chore: import upstream snapshot with attribution
Build and test / Build and test AMD64 Ubuntu 22.04 (push) Failing after 0s
Publish Builder / amazonlinux2023 (push) Failing after 1s
Build and test / UT for Go (push) Has been skipped
Publish KRTE Images / KRTE (push) Failing after 1s
Build and test / Integration Test (push) Has been skipped
Build and test / Upload Code Coverage (push) Has been skipped
Publish Builder / rockylinux9 (push) Failing after 1s
Publish Builder / ubuntu22.04 (push) Failing after 0s
Publish Builder / ubuntu24.04 (push) Failing after 0s
Publish Gpu Builder / publish-gpu-builder (push) Failing after 1s
Publish Test Images / PyTest (push) Failing after 0s
Build and test / UT for Cpp (push) Has been cancelled
Build and test / Build and test AMD64 Ubuntu 22.04 (push) Failing after 0s
Publish Builder / amazonlinux2023 (push) Failing after 1s
Build and test / UT for Go (push) Has been skipped
Publish KRTE Images / KRTE (push) Failing after 1s
Build and test / Integration Test (push) Has been skipped
Build and test / Upload Code Coverage (push) Has been skipped
Publish Builder / rockylinux9 (push) Failing after 1s
Publish Builder / ubuntu22.04 (push) Failing after 0s
Publish Builder / ubuntu24.04 (push) Failing after 0s
Publish Gpu Builder / publish-gpu-builder (push) Failing after 1s
Publish Test Images / PyTest (push) Failing after 0s
Build and test / UT for Cpp (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
CDC sync test cases module.
|
||||
|
||||
This module contains organized test cases for CDC synchronization testing.
|
||||
Each test file focuses on a specific category of operations.
|
||||
"""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,390 @@
|
||||
"""
|
||||
CDC sync tests for alias operations.
|
||||
"""
|
||||
|
||||
import time
|
||||
import pytest
|
||||
from common.common_type import CaseLabel
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
class TestCDCSyncAlias(TestCDCSyncBase):
|
||||
"""Test CDC sync for alias operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
logger.info("Setting up test method")
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
logger.info("Starting teardown method")
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
logger.info(f"Cleaning up {len(self.resources_to_cleanup)} resources")
|
||||
|
||||
# First pass: cleanup aliases (must be done before collections)
|
||||
logger.info("Cleaning up aliases first")
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "alias":
|
||||
logger.info(f"Cleaning up alias: {resource_name}")
|
||||
try:
|
||||
upstream_client.drop_alias(resource_name)
|
||||
logger.info(f"Successfully dropped alias: {resource_name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to drop alias {resource_name}: {e}")
|
||||
|
||||
# Second pass: cleanup collections (after aliases are removed)
|
||||
logger.info("Cleaning up collections after aliases")
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
logger.info(f"Cleaning up collection: {resource_name}")
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
logger.info("Waiting 1 second for cleanup to sync to downstream")
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
else:
|
||||
logger.info("No upstream client found, skipping cleanup")
|
||||
logger.info("Teardown method completed")
|
||||
|
||||
def test_create_alias(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test CREATE_ALIAS operation sync."""
|
||||
logger.info("=== Starting test_create_alias ===")
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_alias_create")
|
||||
alias_name = self.gen_unique_name("test_alias_create")
|
||||
logger.info(f"Generated collection name: {collection_name}")
|
||||
logger.info(f"Generated alias name: {alias_name}")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
self.resources_to_cleanup.append(("alias", alias_name))
|
||||
|
||||
# Initial cleanup
|
||||
logger.info(f"Performing initial cleanup for collection: {collection_name}")
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection
|
||||
logger.info(f"Creating collection: {collection_name}")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
logger.info(f"Collection {collection_name} created in upstream")
|
||||
|
||||
# Wait for creation to sync
|
||||
logger.info(
|
||||
f"Waiting for collection {collection_name} to sync to downstream (timeout: {sync_timeout}s)"
|
||||
)
|
||||
|
||||
def check_create():
|
||||
has_collection = downstream_client.has_collection(collection_name)
|
||||
if has_collection:
|
||||
logger.info(f"Collection {collection_name} found in downstream")
|
||||
return has_collection
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
|
||||
# Create alias
|
||||
logger.info(f"Creating alias {alias_name} for collection {collection_name}")
|
||||
upstream_client.create_alias(collection_name, alias_name)
|
||||
logger.info(f"Alias {alias_name} created in upstream")
|
||||
|
||||
# Verify alias exists in upstream
|
||||
logger.info("Verifying alias exists in upstream")
|
||||
upstream_aliases_result = upstream_client.list_aliases()
|
||||
logger.info(f"Upstream aliases result: {upstream_aliases_result}")
|
||||
upstream_aliases = upstream_aliases_result.get("aliases", [])
|
||||
assert alias_name in upstream_aliases
|
||||
logger.info(f"Confirmed alias {alias_name} exists in upstream")
|
||||
|
||||
# Verify alias points to correct collection using describe_alias
|
||||
logger.info(
|
||||
f"Verifying alias {alias_name} points to collection {collection_name}"
|
||||
)
|
||||
alias_desc = upstream_client.describe_alias(alias_name)
|
||||
logger.info(f"Alias description: {alias_desc}")
|
||||
assert alias_desc.get("collection_name") == collection_name
|
||||
logger.info(
|
||||
f"Confirmed alias {alias_name} correctly points to {collection_name}"
|
||||
)
|
||||
|
||||
# Wait for alias sync to downstream
|
||||
logger.info(
|
||||
f"Waiting for alias {alias_name} to sync to downstream (timeout: {sync_timeout}s)"
|
||||
)
|
||||
|
||||
def check_alias():
|
||||
try:
|
||||
downstream_aliases_result = downstream_client.list_aliases()
|
||||
logger.info(f"Downstream aliases result: {downstream_aliases_result}")
|
||||
downstream_aliases = downstream_aliases_result.get("aliases", [])
|
||||
if alias_name in downstream_aliases:
|
||||
logger.info(f"Alias {alias_name} found in downstream")
|
||||
# Also verify alias points to correct collection in downstream
|
||||
try:
|
||||
downstream_alias_desc = downstream_client.describe_alias(
|
||||
alias_name
|
||||
)
|
||||
logger.info(
|
||||
f"Downstream alias description: {downstream_alias_desc}"
|
||||
)
|
||||
if (
|
||||
downstream_alias_desc.get("collection_name")
|
||||
== collection_name
|
||||
):
|
||||
logger.info(
|
||||
f"Downstream alias {alias_name} correctly points to {collection_name}"
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
f"Downstream alias {alias_name} points to wrong collection: {downstream_alias_desc.get('collection_name')}"
|
||||
)
|
||||
return False
|
||||
except Exception as desc_e:
|
||||
logger.warning(f"Error describing downstream alias: {desc_e}")
|
||||
return False
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking downstream aliases: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_alias, sync_timeout, f"create alias {alias_name}"
|
||||
)
|
||||
logger.info("=== test_create_alias completed successfully ===")
|
||||
|
||||
def test_drop_alias(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DROP_ALIAS operation sync."""
|
||||
logger.info("=== Starting test_drop_alias ===")
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_alias_drop")
|
||||
alias_name = self.gen_unique_name("test_alias_drop")
|
||||
logger.info(f"Generated collection name: {collection_name}")
|
||||
logger.info(f"Generated alias name: {alias_name}")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
self.resources_to_cleanup.append(("alias", alias_name))
|
||||
|
||||
# Initial cleanup
|
||||
logger.info(f"Performing initial cleanup for collection: {collection_name}")
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection and alias
|
||||
logger.info(f"Creating collection: {collection_name}")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
logger.info(f"Collection {collection_name} created in upstream")
|
||||
|
||||
logger.info(f"Creating alias {alias_name} for collection {collection_name}")
|
||||
upstream_client.create_alias(collection_name, alias_name)
|
||||
logger.info(f"Alias {alias_name} created in upstream")
|
||||
|
||||
# Wait for setup to sync
|
||||
logger.info(
|
||||
f"Waiting for collection and alias setup to sync to downstream (timeout: {sync_timeout}s)"
|
||||
)
|
||||
|
||||
def check_setup():
|
||||
try:
|
||||
has_collection = downstream_client.has_collection(collection_name)
|
||||
downstream_aliases_result = downstream_client.list_aliases()
|
||||
downstream_aliases = downstream_aliases_result.get("aliases", [])
|
||||
has_alias = alias_name in downstream_aliases
|
||||
logger.info(
|
||||
f"Downstream - has_collection: {has_collection}, has_alias: {has_alias}"
|
||||
)
|
||||
return has_collection and has_alias
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking downstream setup: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup, sync_timeout, f"setup collection and alias {collection_name}"
|
||||
)
|
||||
|
||||
# Drop alias
|
||||
logger.info(f"Dropping alias {alias_name} from upstream")
|
||||
upstream_client.drop_alias(alias_name)
|
||||
logger.info(f"Alias {alias_name} dropped from upstream")
|
||||
|
||||
# Verify alias is dropped in upstream
|
||||
logger.info("Verifying alias is dropped in upstream")
|
||||
upstream_aliases_result = upstream_client.list_aliases()
|
||||
logger.info(f"Upstream aliases result after drop: {upstream_aliases_result}")
|
||||
upstream_aliases = upstream_aliases_result.get("aliases", [])
|
||||
assert alias_name not in upstream_aliases
|
||||
logger.info(f"Confirmed alias {alias_name} is dropped from upstream")
|
||||
|
||||
# Wait for drop to sync to downstream
|
||||
logger.info(
|
||||
f"Waiting for alias drop to sync to downstream (timeout: {sync_timeout}s)"
|
||||
)
|
||||
|
||||
def check_drop():
|
||||
try:
|
||||
downstream_aliases_result = downstream_client.list_aliases()
|
||||
logger.info(f"Downstream aliases result: {downstream_aliases_result}")
|
||||
downstream_aliases = downstream_aliases_result.get("aliases", [])
|
||||
is_dropped = alias_name not in downstream_aliases
|
||||
if is_dropped:
|
||||
logger.info(
|
||||
f"Alias {alias_name} successfully dropped from downstream"
|
||||
)
|
||||
return is_dropped
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking downstream aliases during drop: {e}")
|
||||
return True # If error, assume alias is dropped
|
||||
|
||||
assert self.wait_for_sync(check_drop, sync_timeout, f"drop alias {alias_name}")
|
||||
logger.info("=== test_drop_alias completed successfully ===")
|
||||
|
||||
def test_alter_alias(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test ALTER_ALIAS operation sync."""
|
||||
logger.info("=== Starting test_alter_alias ===")
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
old_collection = self.gen_unique_name("test_col_alias_old")
|
||||
new_collection = self.gen_unique_name("test_col_alias_new")
|
||||
alias_name = self.gen_unique_name("test_alias_alter")
|
||||
logger.info(f"Generated old collection name: {old_collection}")
|
||||
logger.info(f"Generated new collection name: {new_collection}")
|
||||
logger.info(f"Generated alias name: {alias_name}")
|
||||
self.resources_to_cleanup.append(("collection", old_collection))
|
||||
self.resources_to_cleanup.append(("collection", new_collection))
|
||||
self.resources_to_cleanup.append(("alias", alias_name))
|
||||
|
||||
# Initial cleanup
|
||||
logger.info(
|
||||
f"Performing initial cleanup for collections: {old_collection}, {new_collection}"
|
||||
)
|
||||
self.cleanup_collection(upstream_client, old_collection)
|
||||
self.cleanup_collection(upstream_client, new_collection)
|
||||
|
||||
# Create both collections
|
||||
logger.info(f"Creating old collection: {old_collection}")
|
||||
upstream_client.create_collection(
|
||||
collection_name=old_collection,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
logger.info(f"Old collection {old_collection} created in upstream")
|
||||
|
||||
logger.info(f"Creating new collection: {new_collection}")
|
||||
upstream_client.create_collection(
|
||||
collection_name=new_collection,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
logger.info(f"New collection {new_collection} created in upstream")
|
||||
|
||||
# Create alias pointing to old collection
|
||||
logger.info(
|
||||
f"Creating alias {alias_name} pointing to old collection {old_collection}"
|
||||
)
|
||||
upstream_client.create_alias(old_collection, alias_name)
|
||||
logger.info(
|
||||
f"Alias {alias_name} created in upstream pointing to {old_collection}"
|
||||
)
|
||||
|
||||
# Wait for setup to sync
|
||||
logger.info(
|
||||
f"Waiting for collections and alias setup to sync to downstream (timeout: {sync_timeout}s)"
|
||||
)
|
||||
|
||||
def check_setup():
|
||||
try:
|
||||
has_old = downstream_client.has_collection(old_collection)
|
||||
has_new = downstream_client.has_collection(new_collection)
|
||||
downstream_aliases_result = downstream_client.list_aliases()
|
||||
downstream_aliases = downstream_aliases_result.get("aliases", [])
|
||||
has_alias = alias_name in downstream_aliases
|
||||
logger.info(
|
||||
f"Downstream - has_old: {has_old}, has_new: {has_new}, has_alias: {has_alias}"
|
||||
)
|
||||
return has_old and has_new and has_alias
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking downstream setup: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup, sync_timeout, "setup collections and alias"
|
||||
)
|
||||
|
||||
# Alter alias to point to new collection
|
||||
logger.info(
|
||||
f"Altering alias {alias_name} to point to new collection {new_collection}"
|
||||
)
|
||||
upstream_client.alter_alias(new_collection, alias_name)
|
||||
logger.info(
|
||||
f"Alias {alias_name} altered in upstream to point to {new_collection}"
|
||||
)
|
||||
|
||||
# Verify alias alteration in upstream
|
||||
logger.info(
|
||||
f"Verifying alias {alias_name} now points to {new_collection} in upstream"
|
||||
)
|
||||
upstream_alias_desc = upstream_client.describe_alias(alias_name)
|
||||
logger.info(f"Upstream alias description after alter: {upstream_alias_desc}")
|
||||
assert upstream_alias_desc.get("collection_name") == new_collection
|
||||
logger.info(
|
||||
f"Confirmed upstream alias {alias_name} now points to {new_collection}"
|
||||
)
|
||||
|
||||
# Wait for alter to sync
|
||||
logger.info(
|
||||
f"Waiting for alias alter to sync to downstream (timeout: {sync_timeout}s)"
|
||||
)
|
||||
|
||||
def check_alter():
|
||||
try:
|
||||
# Check if alias still exists and points to correct collection after alter
|
||||
downstream_aliases_result = downstream_client.list_aliases()
|
||||
logger.info(
|
||||
f"Downstream aliases result after alter: {downstream_aliases_result}"
|
||||
)
|
||||
downstream_aliases = downstream_aliases_result.get("aliases", [])
|
||||
if alias_name in downstream_aliases:
|
||||
logger.info(f"Alias {alias_name} found in downstream after alter")
|
||||
# Verify alias points to new collection
|
||||
try:
|
||||
downstream_alias_desc = downstream_client.describe_alias(
|
||||
alias_name
|
||||
)
|
||||
logger.info(
|
||||
f"Downstream alias description after alter: {downstream_alias_desc}"
|
||||
)
|
||||
if (
|
||||
downstream_alias_desc.get("collection_name")
|
||||
== new_collection
|
||||
):
|
||||
logger.info(
|
||||
f"Downstream alias {alias_name} correctly points to {new_collection}"
|
||||
)
|
||||
return True
|
||||
else:
|
||||
logger.warning(
|
||||
f"Downstream alias {alias_name} still points to old collection: {downstream_alias_desc.get('collection_name')}"
|
||||
)
|
||||
return False
|
||||
except Exception as desc_e:
|
||||
logger.warning(
|
||||
f"Error describing downstream alias after alter: {desc_e}"
|
||||
)
|
||||
return False
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking downstream aliases after alter: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_alter, sync_timeout, f"alter alias {alias_name}"
|
||||
)
|
||||
logger.info("=== test_alter_alias completed successfully ===")
|
||||
@@ -0,0 +1,907 @@
|
||||
"""
|
||||
CDC sync tests for collection DDL operations.
|
||||
"""
|
||||
|
||||
import time
|
||||
import pytest
|
||||
from pymilvus import DataType, Collection
|
||||
from common.common_type import CaseLabel
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
class TestCDCSyncCollectionDDL(TestCDCSyncBase):
|
||||
"""Test CDC sync for collection DDL operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def test_create_collection(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test CREATE_COLLECTION operation sync."""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_col_create")
|
||||
|
||||
# Log test start
|
||||
self.log_test_start(
|
||||
"test_create_collection", "CREATE_COLLECTION", collection_name
|
||||
)
|
||||
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Log operation
|
||||
self.log_operation(
|
||||
"CREATE_COLLECTION", "collection", collection_name, "upstream"
|
||||
)
|
||||
|
||||
# Create collection in upstream
|
||||
schema = self.create_default_schema(upstream_client)
|
||||
logger.info(f"[SCHEMA] Collection schema: {schema}")
|
||||
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema
|
||||
)
|
||||
|
||||
# Verify upstream creation
|
||||
upstream_exists = upstream_client.has_collection(collection_name)
|
||||
self.log_resource_state(
|
||||
"collection",
|
||||
collection_name,
|
||||
"exists" if upstream_exists else "missing",
|
||||
"upstream",
|
||||
)
|
||||
assert upstream_exists, (
|
||||
f"Collection {collection_name} not created in upstream"
|
||||
)
|
||||
|
||||
# Log sync verification start
|
||||
self.log_sync_verification(
|
||||
"CREATE_COLLECTION", collection_name, "exists in downstream"
|
||||
)
|
||||
|
||||
# Wait for sync to downstream
|
||||
def check_sync():
|
||||
exists = downstream_client.has_collection(collection_name)
|
||||
if exists:
|
||||
self.log_resource_state(
|
||||
"collection",
|
||||
collection_name,
|
||||
"exists",
|
||||
"downstream",
|
||||
"Sync confirmed",
|
||||
)
|
||||
return exists
|
||||
|
||||
sync_success = self.wait_for_sync(
|
||||
check_sync, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
assert sync_success, (
|
||||
f"Collection {collection_name} failed to sync to downstream"
|
||||
)
|
||||
|
||||
# Log test success
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_create_collection", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] Test failed with error: {e}")
|
||||
self.log_test_end("test_create_collection", False, duration)
|
||||
raise
|
||||
|
||||
def test_drop_collection(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DROP_COLLECTION operation sync."""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_col_drop")
|
||||
|
||||
# Log test start
|
||||
self.log_test_start("test_drop_collection", "DROP_COLLECTION", collection_name)
|
||||
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection first
|
||||
self.log_operation(
|
||||
"CREATE_COLLECTION", "collection", collection_name, "upstream"
|
||||
)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
|
||||
# Drop collection in upstream
|
||||
self.log_operation(
|
||||
"DROP_COLLECTION", "collection", collection_name, "upstream"
|
||||
)
|
||||
upstream_client.drop_collection(collection_name)
|
||||
|
||||
# Verify upstream drop
|
||||
upstream_exists = upstream_client.has_collection(collection_name)
|
||||
self.log_resource_state(
|
||||
"collection",
|
||||
collection_name,
|
||||
"missing" if not upstream_exists else "exists",
|
||||
"upstream",
|
||||
)
|
||||
assert not upstream_exists, (
|
||||
f"Collection {collection_name} still exists in upstream after drop"
|
||||
)
|
||||
|
||||
# Log sync verification start
|
||||
self.log_sync_verification(
|
||||
"DROP_COLLECTION", collection_name, "missing from downstream"
|
||||
)
|
||||
|
||||
# Wait for drop to sync
|
||||
def check_drop():
|
||||
exists = downstream_client.has_collection(collection_name)
|
||||
if not exists:
|
||||
self.log_resource_state(
|
||||
"collection",
|
||||
collection_name,
|
||||
"missing",
|
||||
"downstream",
|
||||
"Drop synced",
|
||||
)
|
||||
return not exists
|
||||
|
||||
sync_success = self.wait_for_sync(
|
||||
check_drop, sync_timeout, f"drop collection {collection_name}"
|
||||
)
|
||||
assert sync_success, (
|
||||
f"Collection {collection_name} drop failed to sync to downstream"
|
||||
)
|
||||
|
||||
# Log test success
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_drop_collection", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] Test failed with error: {e}")
|
||||
self.log_test_end("test_drop_collection", False, duration)
|
||||
raise
|
||||
|
||||
def test_rename_collection(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test RENAME_COLLECTION operation sync."""
|
||||
start_time = time.time()
|
||||
|
||||
old_name = self.gen_unique_name("test_col_rename_old")
|
||||
new_name = self.gen_unique_name("test_col_rename_new")
|
||||
|
||||
# Log test start
|
||||
self.log_test_start(
|
||||
"test_rename_collection", "RENAME_COLLECTION", f"{old_name} -> {new_name}"
|
||||
)
|
||||
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", old_name))
|
||||
self.resources_to_cleanup.append(("collection", new_name))
|
||||
|
||||
try:
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, old_name)
|
||||
self.cleanup_collection(upstream_client, new_name)
|
||||
|
||||
# Create collection first
|
||||
self.log_operation("CREATE_COLLECTION", "collection", old_name, "upstream")
|
||||
upstream_client.create_collection(
|
||||
collection_name=old_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(old_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {old_name}"
|
||||
)
|
||||
|
||||
# Rename collection
|
||||
rename_start_time = time.time()
|
||||
self.log_operation(
|
||||
"RENAME_COLLECTION",
|
||||
"collection",
|
||||
f"{old_name} -> {new_name}",
|
||||
"upstream",
|
||||
)
|
||||
|
||||
try:
|
||||
upstream_client.rename_collection(old_name, new_name)
|
||||
rename_duration = time.time() - rename_start_time
|
||||
logger.info(
|
||||
f"[SUCCESS] Rename operation completed in {rename_duration:.2f}s"
|
||||
)
|
||||
except Exception as e:
|
||||
rename_duration = time.time() - rename_start_time
|
||||
logger.error(
|
||||
f"[FAILED] Rename operation failed after {rename_duration:.2f}s: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
# Verify rename in upstream
|
||||
old_exists = upstream_client.has_collection(old_name)
|
||||
new_exists = upstream_client.has_collection(new_name)
|
||||
self.log_resource_state(
|
||||
"collection",
|
||||
old_name,
|
||||
"missing" if not old_exists else "exists",
|
||||
"upstream",
|
||||
)
|
||||
self.log_resource_state(
|
||||
"collection",
|
||||
new_name,
|
||||
"exists" if new_exists else "missing",
|
||||
"upstream",
|
||||
)
|
||||
|
||||
assert not old_exists, (
|
||||
f"Old collection {old_name} still exists after rename"
|
||||
)
|
||||
assert new_exists, f"New collection {new_name} not found after rename"
|
||||
|
||||
# Log sync verification start
|
||||
self.log_sync_verification(
|
||||
"RENAME_COLLECTION",
|
||||
f"{old_name} -> {new_name}",
|
||||
"completed in downstream",
|
||||
)
|
||||
|
||||
# Wait for rename to sync
|
||||
def check_rename():
|
||||
return not downstream_client.has_collection(
|
||||
old_name
|
||||
) and downstream_client.has_collection(new_name)
|
||||
|
||||
sync_success = self.wait_for_sync(
|
||||
check_rename,
|
||||
sync_timeout,
|
||||
f"rename collection {old_name} to {new_name}",
|
||||
)
|
||||
assert sync_success, (
|
||||
f"Collection rename from {old_name} to {new_name} failed to sync to downstream"
|
||||
)
|
||||
|
||||
# Log test success
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_rename_collection", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] Test failed with error: {e}")
|
||||
self.log_test_end("test_rename_collection", False, duration)
|
||||
raise
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
class TestCDCSyncCollectionManagement(TestCDCSyncBase):
|
||||
"""Test CDC sync for collection management operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def test_add_collection_field(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test ADD_FIELD operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_add_field")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection
|
||||
schema = self.create_default_schema(upstream_client)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema, consistency_level="Strong"
|
||||
)
|
||||
assert self.wait_for_sync(
|
||||
lambda: upstream_client.has_collection(collection_name),
|
||||
sync_timeout,
|
||||
f"create collection {collection_name}",
|
||||
)
|
||||
|
||||
# Add field
|
||||
upstream_client.add_collection_field(
|
||||
collection_name,
|
||||
field_name="new_field",
|
||||
data_type=DataType.INT64,
|
||||
nullable=True,
|
||||
)
|
||||
print(f"DEBUG: add field {collection_name}")
|
||||
res = upstream_client.describe_collection(collection_name)
|
||||
print(f"DEBUG: describe collection {collection_name}: {res}")
|
||||
|
||||
# Wait for addition to sync
|
||||
def check_add():
|
||||
res = downstream_client.describe_collection(collection_name)
|
||||
logger.info(
|
||||
f"DEBUG: describe collection in downstream {collection_name}: {res}"
|
||||
)
|
||||
return "new_field" in [field["name"] for field in res["fields"]]
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_add, sync_timeout, f"add field {collection_name}"
|
||||
)
|
||||
|
||||
def test_load_collection(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test LOAD_COLLECTION operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_load")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection with proper schema
|
||||
schema = self.create_default_schema(upstream_client)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema, consistency_level="Strong"
|
||||
)
|
||||
|
||||
# Create index (required for loading)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector", index_type="AUTOINDEX", metric_type="L2"
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
|
||||
# Load collection
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for load to sync
|
||||
def check_load():
|
||||
try:
|
||||
# Try to perform a search to verify the collection is loaded
|
||||
query_vector = [[0.1] * 128] # dummy vector
|
||||
downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=[],
|
||||
)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_load, sync_timeout, f"load collection {collection_name}"
|
||||
)
|
||||
|
||||
@pytest.mark.skip(reason="skip multi-replica test")
|
||||
def test_load_collection_multi_replicas(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test LOAD_COLLECTION operation with multiple replicas sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_multi_replicas")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection with proper schema
|
||||
schema = self.create_default_schema(upstream_client)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema, consistency_level="Strong"
|
||||
)
|
||||
|
||||
# Create index (required for loading)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector", index_type="AUTOINDEX", metric_type="L2"
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
|
||||
# Load collection with 2 replicas
|
||||
replica_number = 2
|
||||
logger.info(
|
||||
f"Loading collection {collection_name} with {replica_number} replicas"
|
||||
)
|
||||
upstream_client.load_collection(collection_name, replica_number=replica_number)
|
||||
|
||||
# Verify upstream load with replicas and check replica count
|
||||
def verify_upstream_replicas():
|
||||
try:
|
||||
# Create Collection object to get replica information
|
||||
upstream_collection = Collection(
|
||||
name=collection_name, using=upstream_client._using
|
||||
)
|
||||
|
||||
# Get replicas information
|
||||
replicas = upstream_collection.get_replicas()
|
||||
actual_replica_count = len(replicas.groups)
|
||||
logger.info(
|
||||
f"Upstream collection {collection_name} has {actual_replica_count} replicas"
|
||||
)
|
||||
logger.info(f"Replica details: {replicas}")
|
||||
|
||||
# Verify replica count matches expected
|
||||
if actual_replica_count != replica_number:
|
||||
logger.warning(
|
||||
f"Expected {replica_number} replicas, but found {actual_replica_count}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Try to perform a search to verify the collection is loaded
|
||||
query_vector = [[0.1] * 128]
|
||||
upstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=[],
|
||||
)
|
||||
logger.info(
|
||||
f"Upstream collection {collection_name} loaded successfully with {actual_replica_count} replicas"
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Upstream load verification failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
verify_upstream_replicas,
|
||||
sync_timeout,
|
||||
f"load collection {collection_name} with {replica_number} replicas in upstream",
|
||||
)
|
||||
|
||||
# Wait for load with replicas to sync to downstream
|
||||
def check_downstream_replicas():
|
||||
try:
|
||||
# Create Collection object to get replica information
|
||||
downstream_collection = Collection(
|
||||
name=collection_name, using=downstream_client._using
|
||||
)
|
||||
|
||||
# Get replicas information
|
||||
replicas = downstream_collection.get_replicas()
|
||||
actual_replica_count = len(replicas.groups)
|
||||
logger.info(
|
||||
f"Downstream collection {collection_name} has {actual_replica_count} replicas"
|
||||
)
|
||||
logger.info(f"Replica details: {replicas}")
|
||||
|
||||
# Verify replica count matches expected
|
||||
if actual_replica_count != replica_number:
|
||||
logger.warning(
|
||||
f"Expected {replica_number} replicas in downstream, but found {actual_replica_count}"
|
||||
)
|
||||
return False
|
||||
|
||||
# Try to perform a search to verify the collection is loaded in downstream
|
||||
query_vector = [[0.1] * 128]
|
||||
downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=[],
|
||||
)
|
||||
logger.info(
|
||||
f"Downstream collection {collection_name} loaded successfully with {actual_replica_count} replicas"
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Downstream load check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_downstream_replicas,
|
||||
sync_timeout,
|
||||
f"load collection {collection_name} with {replica_number} replicas in downstream",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Successfully verified multi-replica load sync for collection {collection_name}"
|
||||
)
|
||||
logger.info(f"Both upstream and downstream have {replica_number} replicas")
|
||||
|
||||
# Now test release with multiple replicas
|
||||
logger.info(
|
||||
f"Testing release operation for multi-replica collection {collection_name}"
|
||||
)
|
||||
upstream_client.release_collection(collection_name)
|
||||
|
||||
# Verify upstream release
|
||||
def verify_upstream_release():
|
||||
try:
|
||||
# Try to search - should fail if released
|
||||
query_vector = [[0.1] * 128]
|
||||
upstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=[],
|
||||
)
|
||||
logger.warning(
|
||||
f"Upstream collection {collection_name} is still loaded (search succeeded)"
|
||||
)
|
||||
return False # If search succeeds, collection is still loaded
|
||||
except Exception as e:
|
||||
logger.info(
|
||||
f"Upstream collection {collection_name} released successfully (search failed as expected): {e}"
|
||||
)
|
||||
return True # If search fails, collection is released
|
||||
|
||||
assert self.wait_for_sync(
|
||||
verify_upstream_release,
|
||||
sync_timeout,
|
||||
f"release collection {collection_name} in upstream",
|
||||
)
|
||||
|
||||
# Wait for release to sync to downstream
|
||||
def check_downstream_release():
|
||||
try:
|
||||
# Try to search - should fail if released
|
||||
query_vector = [[0.1] * 128]
|
||||
downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=[],
|
||||
)
|
||||
logger.warning(
|
||||
f"Downstream collection {collection_name} is still loaded (search succeeded)"
|
||||
)
|
||||
return False # If search succeeds, collection is still loaded
|
||||
except Exception as e:
|
||||
logger.info(
|
||||
f"Downstream collection {collection_name} released successfully (search failed as expected): {e}"
|
||||
)
|
||||
return True # If search fails, collection is released
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_downstream_release,
|
||||
sync_timeout,
|
||||
f"release collection {collection_name} in downstream",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Successfully verified multi-replica release sync for collection {collection_name}"
|
||||
)
|
||||
logger.info(
|
||||
f"Both upstream and downstream released the collection with {replica_number} replicas"
|
||||
)
|
||||
|
||||
def test_release_collection(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test RELEASE_COLLECTION operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_release")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection with proper schema
|
||||
schema = self.create_default_schema(upstream_client)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema, consistency_level="Strong"
|
||||
)
|
||||
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector", index_type="AUTOINDEX", metric_type="L2"
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for setup to sync
|
||||
def check_setup():
|
||||
try:
|
||||
query_vector = [[0.1] * 128]
|
||||
downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=[],
|
||||
)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup, sync_timeout, f"setup and load collection {collection_name}"
|
||||
)
|
||||
|
||||
# Release collection
|
||||
upstream_client.release_collection(collection_name)
|
||||
|
||||
# Wait for release to sync
|
||||
def check_release():
|
||||
try:
|
||||
# Try to search - should fail if released
|
||||
query_vector = [[0.1] * 128]
|
||||
downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=[],
|
||||
)
|
||||
return False # If search succeeds, collection is still loaded
|
||||
except:
|
||||
return True # If search fails, collection is released
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_release, sync_timeout, f"release collection {collection_name}"
|
||||
)
|
||||
|
||||
def test_flush(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test FLUSH operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_flush")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection with proper schema
|
||||
schema = self.create_default_schema(upstream_client)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema, consistency_level="Strong"
|
||||
)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
|
||||
# Insert data (without immediate flush)
|
||||
test_data = self.generate_test_data(5000)
|
||||
insert_result = upstream_client.insert(collection_name, test_data)
|
||||
|
||||
# Verify data is not visible before flush
|
||||
stats_before = upstream_client.get_collection_stats(collection_name)
|
||||
logger.info(f"Stats before flush: {stats_before}")
|
||||
|
||||
# Flush collection
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Wait for flush data to be visible with timeout
|
||||
expected_count = (
|
||||
insert_result.get("insert_count", 100) if insert_result else 100
|
||||
)
|
||||
|
||||
def check_flush_stats():
|
||||
try:
|
||||
stats = upstream_client.get_collection_stats(collection_name)
|
||||
logger.info(
|
||||
f"DEBUG: get collection stats in upstream {collection_name}: {stats}"
|
||||
)
|
||||
row_count = stats.get("row_count", 0)
|
||||
logger.info(
|
||||
f"Current row count: {row_count}, expected: {expected_count}"
|
||||
)
|
||||
return row_count >= expected_count
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking stats: {e}")
|
||||
return False
|
||||
|
||||
# Use timeout for waiting flush stats to update
|
||||
timeout = 30 # 30 seconds timeout
|
||||
assert self.wait_for_sync(
|
||||
check_flush_stats,
|
||||
timeout,
|
||||
f"flush data visible in stats (expected: {expected_count})",
|
||||
)
|
||||
|
||||
# Get final stats after flush
|
||||
stats_after = upstream_client.get_collection_stats(collection_name)
|
||||
logger.info(f"Stats after flush: {stats_after}")
|
||||
|
||||
# Wait for flush to sync downstream
|
||||
def check_flush():
|
||||
try:
|
||||
downstream_stats = downstream_client.get_collection_stats(
|
||||
collection_name
|
||||
)
|
||||
logger.info(
|
||||
f"DEBUG: get collection stats in downstream {collection_name}: {downstream_stats}"
|
||||
)
|
||||
return downstream_stats.get("row_count", 0) >= 100
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_flush, sync_timeout, f"flush collection {collection_name}"
|
||||
)
|
||||
|
||||
def test_load_collection_with_load_fields(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test LOAD_COLLECTION operation with load_fields parameter sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_load_fields")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection with comprehensive schema (has multiple fields)
|
||||
schema = self.create_comprehensive_schema(upstream_client)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema, consistency_level="Strong"
|
||||
)
|
||||
|
||||
# Create index for float_vector field (required for loading)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="float_vector", index_type="AUTOINDEX", metric_type="L2"
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
|
||||
# Insert some test data
|
||||
test_data = self.generate_comprehensive_test_data(100)
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Load collection with specific fields only (float_vector + id + varchar_field)
|
||||
load_fields = ["float_vector", "id", "varchar_field"]
|
||||
upstream_client.load_collection(collection_name, load_fields=load_fields)
|
||||
|
||||
# Verify upstream load operation succeeded
|
||||
def verify_upstream_load():
|
||||
try:
|
||||
# Try to search to verify collection is loaded
|
||||
query_vector = [[0.1] * 128]
|
||||
upstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=["varchar_field"],
|
||||
anns_field="float_vector",
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Upstream load verification failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
verify_upstream_load,
|
||||
sync_timeout,
|
||||
f"verify upstream load with load_fields in {collection_name}",
|
||||
)
|
||||
|
||||
# Verify downstream sync - collection should be loaded and searchable
|
||||
def check_downstream_load_sync():
|
||||
try:
|
||||
query_vector = [[0.1] * 128]
|
||||
result = downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=["varchar_field"],
|
||||
anns_field="float_vector",
|
||||
)
|
||||
return len(result) > 0 and len(result[0]) >= 0
|
||||
except Exception as e:
|
||||
logger.warning(f"Downstream load sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_downstream_load_sync,
|
||||
sync_timeout,
|
||||
f"verify downstream load sync for {collection_name}",
|
||||
)
|
||||
|
||||
# Additional verification: test that both loaded and unloaded fields can be output
|
||||
# (since load_fields only affects memory usage, not field accessibility)
|
||||
def verify_all_fields_accessible():
|
||||
try:
|
||||
query_vector = [[0.1] * 128]
|
||||
# Test accessing both loaded and unloaded fields
|
||||
result1 = downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=["varchar_field"], # loaded field
|
||||
anns_field="float_vector",
|
||||
)
|
||||
|
||||
result2 = downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
output_fields=[
|
||||
"float_field"
|
||||
], # unloaded field (but should still be accessible)
|
||||
anns_field="float_vector",
|
||||
)
|
||||
|
||||
return len(result1) > 0 and len(result2) > 0
|
||||
except Exception as e:
|
||||
logger.warning(f"Field accessibility verification failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
verify_all_fields_accessible,
|
||||
sync_timeout,
|
||||
f"verify all fields accessible in {collection_name}",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Successfully tested load_collection with load_fields: {load_fields}"
|
||||
)
|
||||
logger.info("Verified CDC sync of load operation with load_fields parameter")
|
||||
@@ -0,0 +1,276 @@
|
||||
"""
|
||||
CDC sync tests for collection property operations.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
class TestCDCSyncCollectionProperties(TestCDCSyncBase):
|
||||
"""Test CDC sync for collection property (alter/drop) operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def _create_basic_collection(self, client, c_name):
|
||||
"""Create a default schema collection, insert 100 rows, and create HNSW index."""
|
||||
schema = self.create_default_schema(client)
|
||||
client.create_collection(
|
||||
collection_name=c_name,
|
||||
schema=schema,
|
||||
consistency_level="Strong",
|
||||
)
|
||||
|
||||
# Insert 100 rows
|
||||
test_data = self.generate_test_data(100)
|
||||
client.insert(c_name, test_data)
|
||||
|
||||
# Create HNSW index on vector field
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
client.create_index(c_name, index_params)
|
||||
|
||||
def test_ttl_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test ALTER_COLLECTION_PROPERTIES (TTL) sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
c_name = self.gen_unique_name("test_col_ttl")
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
# Create collection
|
||||
self._create_basic_collection(upstream_client, c_name)
|
||||
|
||||
# Wait for collection to sync downstream
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
# Alter collection TTL property
|
||||
upstream_client.alter_collection_properties(
|
||||
collection_name=c_name,
|
||||
properties={"collection.ttl.seconds": "3600"},
|
||||
)
|
||||
|
||||
# Wait for property to sync downstream
|
||||
def check_ttl():
|
||||
try:
|
||||
desc = downstream_client.describe_collection(c_name)
|
||||
props = desc.get("properties", {})
|
||||
logger.info(f"Downstream collection properties: {props}")
|
||||
return str(props.get("collection.ttl.seconds", "")) == "3600"
|
||||
except Exception as e:
|
||||
logger.warning(f"Check TTL sync failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_ttl, sync_timeout, f"TTL property sync for {c_name}")
|
||||
|
||||
def test_mmap_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test ALTER_COLLECTION_PROPERTIES (mmap.enabled) sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
c_name = self.gen_unique_name("test_col_mmap")
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
# Create collection
|
||||
self._create_basic_collection(upstream_client, c_name)
|
||||
|
||||
# Wait for collection to sync downstream
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
# Alter mmap.enabled property
|
||||
upstream_client.alter_collection_properties(
|
||||
collection_name=c_name,
|
||||
properties={"mmap.enabled": "true"},
|
||||
)
|
||||
|
||||
# Wait for property to sync downstream
|
||||
def check_mmap():
|
||||
try:
|
||||
desc = downstream_client.describe_collection(c_name)
|
||||
props = desc.get("properties", {})
|
||||
logger.info(f"Downstream collection properties: {props}")
|
||||
return str(props.get("mmap.enabled", "")).lower() == "true"
|
||||
except Exception as e:
|
||||
logger.warning(f"Check mmap sync failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_mmap, sync_timeout, f"mmap property sync for {c_name}")
|
||||
|
||||
def test_autocompaction_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test ALTER_COLLECTION_PROPERTIES (autocompaction.enabled) sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
c_name = self.gen_unique_name("test_col_autocomp")
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
# Create collection
|
||||
self._create_basic_collection(upstream_client, c_name)
|
||||
|
||||
# Wait for collection to sync downstream
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
# Alter autocompaction property
|
||||
upstream_client.alter_collection_properties(
|
||||
collection_name=c_name,
|
||||
properties={"collection.autocompaction.enabled": "true"},
|
||||
)
|
||||
|
||||
# Wait for property to sync downstream
|
||||
def check_autocomp():
|
||||
try:
|
||||
desc = downstream_client.describe_collection(c_name)
|
||||
props = desc.get("properties", {})
|
||||
logger.info(f"Downstream collection properties: {props}")
|
||||
return str(props.get("collection.autocompaction.enabled", "")).lower() == "true"
|
||||
except Exception as e:
|
||||
logger.warning(f"Check autocompaction sync failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_autocomp, sync_timeout, f"autocompaction property sync for {c_name}")
|
||||
|
||||
def test_alter_multiple_properties(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test ALTER_COLLECTION_PROPERTIES with multiple properties at once sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
c_name = self.gen_unique_name("test_col_multi_props")
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
# Create collection
|
||||
self._create_basic_collection(upstream_client, c_name)
|
||||
|
||||
# Wait for collection to sync downstream
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
# Alter all 3 properties at once
|
||||
upstream_client.alter_collection_properties(
|
||||
collection_name=c_name,
|
||||
properties={
|
||||
"collection.ttl.seconds": "3600",
|
||||
"mmap.enabled": "true",
|
||||
"collection.autocompaction.enabled": "true",
|
||||
},
|
||||
)
|
||||
|
||||
# Wait for all 3 properties to sync downstream
|
||||
def check_all_props():
|
||||
try:
|
||||
desc = downstream_client.describe_collection(c_name)
|
||||
props = desc.get("properties", {})
|
||||
logger.info(f"Downstream collection properties: {props}")
|
||||
ttl_ok = str(props.get("collection.ttl.seconds", "")) == "3600"
|
||||
mmap_ok = str(props.get("mmap.enabled", "")).lower() == "true"
|
||||
autocomp_ok = str(props.get("collection.autocompaction.enabled", "")).lower() == "true"
|
||||
return ttl_ok and mmap_ok and autocomp_ok
|
||||
except Exception as e:
|
||||
logger.warning(f"Check all properties sync failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_all_props, sync_timeout, f"all properties sync for {c_name}")
|
||||
|
||||
def test_drop_properties_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DROP_COLLECTION_PROPERTIES — set TTL + mmap, drop TTL, verify TTL gone and mmap remains."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
c_name = self.gen_unique_name("test_col_drop_props")
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
# Create collection
|
||||
self._create_basic_collection(upstream_client, c_name)
|
||||
|
||||
# Wait for collection to sync downstream
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
# Set TTL and mmap properties
|
||||
upstream_client.alter_collection_properties(
|
||||
collection_name=c_name,
|
||||
properties={
|
||||
"collection.ttl.seconds": "3600",
|
||||
"mmap.enabled": "true",
|
||||
},
|
||||
)
|
||||
|
||||
# Wait for both properties to sync downstream
|
||||
def check_props_set():
|
||||
try:
|
||||
desc = downstream_client.describe_collection(c_name)
|
||||
props = desc.get("properties", {})
|
||||
ttl_set = str(props.get("collection.ttl.seconds", "")) == "3600"
|
||||
mmap_set = str(props.get("mmap.enabled", "")).lower() == "true"
|
||||
return ttl_set and mmap_set
|
||||
except Exception as e:
|
||||
logger.warning(f"Check properties set failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_props_set, sync_timeout, f"set properties for {c_name}")
|
||||
|
||||
# Drop TTL property
|
||||
upstream_client.drop_collection_properties(
|
||||
collection_name=c_name,
|
||||
property_keys=["collection.ttl.seconds"],
|
||||
)
|
||||
|
||||
# Wait for TTL to be gone but mmap to remain on downstream
|
||||
def check_drop_ttl():
|
||||
try:
|
||||
desc = downstream_client.describe_collection(c_name)
|
||||
props = desc.get("properties", {})
|
||||
logger.info(f"Downstream collection properties after drop: {props}")
|
||||
ttl_gone = "collection.ttl.seconds" not in props
|
||||
mmap_remains = str(props.get("mmap.enabled", "")).lower() == "true"
|
||||
return ttl_gone and mmap_remains
|
||||
except Exception as e:
|
||||
logger.warning(f"Check drop TTL sync failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_drop_ttl, sync_timeout, f"drop TTL property sync for {c_name}")
|
||||
@@ -0,0 +1,256 @@
|
||||
"""
|
||||
CDC sync tests for database operations.
|
||||
"""
|
||||
|
||||
import time
|
||||
import pytest
|
||||
from common.common_type import CaseLabel
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
class TestCDCSyncDatabase(TestCDCSyncBase):
|
||||
"""Test CDC sync for database operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "database":
|
||||
self.cleanup_database(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def test_create_database(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test CREATE_DATABASE operation sync."""
|
||||
start_time = time.time()
|
||||
db_name = self.gen_unique_name("test_db_create")
|
||||
|
||||
# Log test start
|
||||
self.log_test_start("test_create_database", "CREATE_DATABASE", db_name)
|
||||
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("database", db_name))
|
||||
|
||||
try:
|
||||
# Initial cleanup
|
||||
self.cleanup_database(upstream_client, db_name)
|
||||
|
||||
# Log operation
|
||||
self.log_operation("CREATE_DATABASE", "database", db_name, "upstream")
|
||||
|
||||
# Create database in upstream
|
||||
upstream_client.create_database(db_name)
|
||||
|
||||
# Verify upstream creation
|
||||
upstream_databases = upstream_client.list_databases()
|
||||
upstream_exists = db_name in upstream_databases
|
||||
self.log_resource_state(
|
||||
"database",
|
||||
db_name,
|
||||
"exists" if upstream_exists else "missing",
|
||||
"upstream",
|
||||
)
|
||||
assert upstream_exists, f"Database {db_name} not created in upstream"
|
||||
|
||||
# Log sync verification start
|
||||
self.log_sync_verification(
|
||||
"CREATE_DATABASE", db_name, "exists in downstream"
|
||||
)
|
||||
|
||||
# Wait for sync to downstream
|
||||
def check_sync():
|
||||
downstream_databases = downstream_client.list_databases()
|
||||
exists = db_name in downstream_databases
|
||||
if exists:
|
||||
self.log_resource_state(
|
||||
"database", db_name, "exists", "downstream", "Sync confirmed"
|
||||
)
|
||||
return exists
|
||||
|
||||
sync_success = self.wait_for_sync(
|
||||
check_sync, sync_timeout, f"create database {db_name}"
|
||||
)
|
||||
assert sync_success, f"Database {db_name} failed to sync to downstream"
|
||||
|
||||
# Log test success
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_create_database", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] Test failed with error: {e}")
|
||||
self.log_test_end("test_create_database", False, duration)
|
||||
raise
|
||||
|
||||
def test_drop_database(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DROP_DATABASE operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
db_name = self.gen_unique_name("test_db_drop")
|
||||
self.resources_to_cleanup.append(("database", db_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_database(upstream_client, db_name)
|
||||
|
||||
# Create database in upstream first
|
||||
upstream_client.create_database(db_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return db_name in downstream_client.list_databases()
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create database {db_name}"
|
||||
)
|
||||
|
||||
# Drop database in upstream
|
||||
upstream_client.drop_database(db_name)
|
||||
assert db_name not in upstream_client.list_databases()
|
||||
|
||||
# Wait for drop to sync to downstream
|
||||
def check_drop():
|
||||
return db_name not in downstream_client.list_databases()
|
||||
|
||||
assert self.wait_for_sync(check_drop, sync_timeout, f"drop database {db_name}")
|
||||
|
||||
def test_alter_database_properties(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test ALTER_DATABASE_PROPERTIES operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
db_name = self.gen_unique_name("test_db_alter_props")
|
||||
self.resources_to_cleanup.append(("database", db_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_database(upstream_client, db_name)
|
||||
|
||||
# Create database in upstream first
|
||||
upstream_client.create_database(db_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return db_name in downstream_client.list_databases()
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create database {db_name}"
|
||||
)
|
||||
|
||||
# Set multiple database properties
|
||||
properties_to_set = {
|
||||
"database.max.collections": 100,
|
||||
"database.diskQuota.mb": 1024,
|
||||
}
|
||||
|
||||
upstream_client.alter_database_properties(
|
||||
db_name=db_name, properties=properties_to_set
|
||||
)
|
||||
|
||||
# Wait for alter properties to sync
|
||||
def check_alter_properties():
|
||||
try:
|
||||
# Verify database exists
|
||||
if db_name not in downstream_client.list_databases():
|
||||
return False
|
||||
|
||||
# Verify properties are synced correctly
|
||||
downstream_props = downstream_client.describe_database(db_name)
|
||||
logger.info(f"Downstream database properties: {downstream_props}")
|
||||
for key, expected_value in properties_to_set.items():
|
||||
if key not in downstream_props or str(downstream_props[key]) != str(
|
||||
expected_value
|
||||
):
|
||||
return False
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_alter_properties, sync_timeout, f"alter database properties {db_name}"
|
||||
)
|
||||
logger.info(f"Database properties altered successfully for {db_name}")
|
||||
|
||||
def test_drop_database_properties(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test DROP_DATABASE_PROPERTIES operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
db_name = self.gen_unique_name("test_db_drop_props")
|
||||
self.resources_to_cleanup.append(("database", db_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_database(upstream_client, db_name)
|
||||
|
||||
# Create database in upstream first
|
||||
upstream_client.create_database(db_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return db_name in downstream_client.list_databases()
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create database {db_name}"
|
||||
)
|
||||
|
||||
# First set some properties
|
||||
properties_to_set = {
|
||||
"database.max.collections": 200,
|
||||
"database.diskQuota.mb": 2048,
|
||||
}
|
||||
|
||||
upstream_client.alter_database_properties(
|
||||
db_name=db_name, properties=properties_to_set
|
||||
)
|
||||
|
||||
# Wait for initial properties to sync
|
||||
time.sleep(3) # Allow properties to be set
|
||||
|
||||
# Drop specific database properties (only drop one, keep the other)
|
||||
property_keys_to_drop = ["database.max.collections"]
|
||||
|
||||
upstream_client.drop_database_properties(
|
||||
db_name=db_name, property_keys=property_keys_to_drop
|
||||
)
|
||||
|
||||
# Wait for drop properties to sync
|
||||
def check_drop_properties():
|
||||
try:
|
||||
# Verify database exists
|
||||
if db_name not in downstream_client.list_databases():
|
||||
return False
|
||||
|
||||
# Verify properties are synced correctly after drop
|
||||
downstream_props = downstream_client.describe_database(db_name)
|
||||
|
||||
logger.info(f"Downstream database properties: {downstream_props}")
|
||||
# Verify dropped property is not present
|
||||
if "database.max.collections" in downstream_props:
|
||||
return False
|
||||
|
||||
# Verify non-dropped property is still present with correct value
|
||||
if (
|
||||
"database.diskQuota.mb" not in downstream_props
|
||||
or str(downstream_props["database.diskQuota.mb"]) != "2048"
|
||||
):
|
||||
return False
|
||||
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_drop_properties, sync_timeout, f"drop database properties {db_name}"
|
||||
)
|
||||
logger.info(f"Database properties dropped successfully for {db_name}")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,984 @@
|
||||
"""
|
||||
CDC force-promote failover scenario tests.
|
||||
|
||||
Each test exercises a path through the streamingnode replicate interceptor's
|
||||
SwitchReplicateMode + force-promote ack-callback flow:
|
||||
- basic — kill source, promote secondary, verify writable
|
||||
- target_restart — restart secondary mid-promote
|
||||
- source_restart — restart primary mid-promote
|
||||
- incomplete_ddl — release on primary then immediately promote (Ignore=true path)
|
||||
- network_partition — partition source↔target, then promote
|
||||
- endurance (skipif) — repeated promote/restore loop for ENDURANCE_DURATION_MINUTES
|
||||
|
||||
Every test's `finally` restores the A→B topology via switchover_helper so
|
||||
subsequent tests start from a known state.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from common.common_type import CaseLabel
|
||||
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
FAILOVER_TIMEOUT = int(os.getenv("FAILOVER_TIMEOUT", "180"))
|
||||
FORCE_PROMOTE_RPC_TIMEOUT = int(os.getenv("FORCE_PROMOTE_RPC_TIMEOUT", "60"))
|
||||
PROMOTE_RETRY_INTERVAL = 5
|
||||
DEFAULT_INSERT_COUNT = 200
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
class TestCDCForcePromote(TestCDCSyncBase):
|
||||
"""Force-promote failover scenarios."""
|
||||
|
||||
def setup_method(self):
|
||||
self.resources_to_cleanup = []
|
||||
self._upstream_client = None
|
||||
self._downstream_client = None
|
||||
|
||||
def teardown_method(self):
|
||||
"""Safety-net cleanup: iterates resources_to_cleanup if a test raised
|
||||
before its own finally could run. Normal cleanup happens in each test's
|
||||
finally block."""
|
||||
for resource_type, resource_name in getattr(self, "resources_to_cleanup", []):
|
||||
if resource_type != "collection":
|
||||
continue
|
||||
for client in (
|
||||
getattr(self, "_upstream_client", None),
|
||||
getattr(self, "_downstream_client", None),
|
||||
):
|
||||
if client is None:
|
||||
continue
|
||||
try:
|
||||
self.cleanup_collection(client, resource_name)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def do_force_promote(
|
||||
self,
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
timeout=FORCE_PROMOTE_RPC_TIMEOUT,
|
||||
):
|
||||
"""Call force_promote on downstream with a standalone-primary config."""
|
||||
config = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": target_cluster_id,
|
||||
"connection_param": {"uri": downstream_uri, "token": downstream_token},
|
||||
"pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
}
|
||||
],
|
||||
"cross_cluster_topology": [],
|
||||
}
|
||||
downstream_client.update_replicate_configuration(
|
||||
timeout=timeout,
|
||||
force_promote=True,
|
||||
**config,
|
||||
)
|
||||
logger.info(f"[FORCE_PROMOTE] called on {target_cluster_id} (standalone primary config)")
|
||||
|
||||
def is_already_primary_error(self, err):
|
||||
return "force promote can only be used on secondary clusters" in str(
|
||||
err
|
||||
) and "current cluster is primary" in str(err)
|
||||
|
||||
def promote_call_until_writable(
|
||||
self,
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
timeout=FAILOVER_TIMEOUT,
|
||||
):
|
||||
"""Retry force_promote + probe write until success or timeout.
|
||||
|
||||
A probe write creates+drops a throwaway collection. This proves
|
||||
downstream is in standalone-primary mode and accepts DDL.
|
||||
"""
|
||||
deadline = time.time() + timeout
|
||||
last_err = None
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
try:
|
||||
remaining = max(1, int(deadline - time.time()))
|
||||
self.do_force_promote(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
timeout=min(FORCE_PROMOTE_RPC_TIMEOUT, remaining),
|
||||
)
|
||||
except Exception as e:
|
||||
if not self.is_already_primary_error(e):
|
||||
raise
|
||||
logger.info("[FORCE_PROMOTE] target is already primary; probing writable state")
|
||||
probe = self.gen_unique_name("promote_probe", max_length=40)
|
||||
schema = self.create_default_schema(downstream_client)
|
||||
downstream_client.create_collection(probe, schema=schema)
|
||||
downstream_client.drop_collection(probe)
|
||||
logger.info("[FORCE_PROMOTE] probe write succeeded — cluster writable")
|
||||
return
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
logger.warning(f"[FORCE_PROMOTE_RETRY] failed: {e}")
|
||||
sleep_seconds = min(PROMOTE_RETRY_INTERVAL, max(0, deadline - time.time()))
|
||||
if sleep_seconds:
|
||||
time.sleep(sleep_seconds)
|
||||
raise TimeoutError(f"force_promote did not become writable within {timeout}s; last error: {last_err}")
|
||||
|
||||
def cleanup_resources(self):
|
||||
"""Drop any collections registered in resources_to_cleanup."""
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type != "collection":
|
||||
continue
|
||||
for client_label, client in (
|
||||
("upstream", self._upstream_client),
|
||||
("downstream", self._downstream_client),
|
||||
):
|
||||
if client is None:
|
||||
continue
|
||||
try:
|
||||
self.cleanup_collection(client, resource_name)
|
||||
logger.info(f"[CLEANUP] dropped {resource_name} on {client_label}")
|
||||
except Exception as e:
|
||||
logger.warning(f"[CLEANUP] failed to drop {resource_name} on {client_label}: {e}")
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def cleanup_downstream_only_collection(self, collection_name):
|
||||
"""Drop a collection created while downstream is force-promoted primary."""
|
||||
client = self._downstream_client
|
||||
if client is None:
|
||||
return
|
||||
try:
|
||||
if client.has_collection(collection_name):
|
||||
logger.info(f"[CLEANUP] Cleaning up downstream-only collection: {collection_name}")
|
||||
client.drop_collection(collection_name)
|
||||
logger.info(f"[SUCCESS] Downstream-only collection {collection_name} cleaned up successfully")
|
||||
except Exception as e:
|
||||
logger.warning(f"[FAILED] Failed to cleanup downstream-only collection {collection_name}: {e}")
|
||||
finally:
|
||||
self.resources_to_cleanup = [
|
||||
resource for resource in self.resources_to_cleanup if resource != ("collection", collection_name)
|
||||
]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tests
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_force_promote_basic(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
kubectl_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
):
|
||||
"""Insert N → kill source → force_promote(downstream) → verify writable."""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_fp_basic", max_length=50)
|
||||
c_after = f"{c_name}_after"
|
||||
|
||||
self.log_test_start("test_force_promote_basic", "FORCE_PROMOTE_BASIC", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
self.resources_to_cleanup.append(("collection", c_after))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
lambda: downstream_client.has_collection(c_name),
|
||||
sync_timeout,
|
||||
f"create collection {c_name}",
|
||||
)
|
||||
|
||||
data = self.generate_test_data_with_id(DEFAULT_INSERT_COUNT, start_id=0)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_synced():
|
||||
try:
|
||||
res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/{DEFAULT_INSERT_COUNT}")
|
||||
return cnt >= DEFAULT_INSERT_COUNT
|
||||
except Exception as e:
|
||||
logger.warning(f"sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_synced, sync_timeout, f"initial sync {c_name}")
|
||||
|
||||
# Kill source
|
||||
logger.info(f"[FAILOVER] Killing source pods (instance={source_cluster_id})...")
|
||||
kubectl_helper.delete_pods(source_cluster_id)
|
||||
|
||||
# Promote downstream — retry until writable
|
||||
self.promote_call_until_writable(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
|
||||
# On downstream (now standalone primary), create a new collection + insert
|
||||
schema_after = self.create_manual_id_schema(downstream_client)
|
||||
downstream_client.create_collection(collection_name=c_after, schema=schema_after)
|
||||
idx_after = downstream_client.prepare_index_params()
|
||||
idx_after.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
downstream_client.create_index(c_after, idx_after)
|
||||
downstream_client.load_collection(c_after)
|
||||
data_after = self.generate_test_data_with_id(100, start_id=0)
|
||||
downstream_client.insert(c_after, data_after)
|
||||
downstream_client.flush(c_after)
|
||||
|
||||
res = downstream_client.query(collection_name=c_after, filter="", output_fields=["count(*)"])
|
||||
assert res and res[0]["count(*)"] >= 100, f"downstream not writable after force_promote: count={res}"
|
||||
finally:
|
||||
logger.info("[TEARDOWN] Waiting for source pods before topology restore...")
|
||||
kubectl_helper.wait_for_pods_ready(source_cluster_id, timeout=300)
|
||||
self.cleanup_downstream_only_collection(c_after)
|
||||
logger.info("[TEARDOWN] Restoring A→B topology...")
|
||||
try:
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"switchover restore failed: {e}")
|
||||
self.cleanup_resources()
|
||||
self.log_test_end("test_force_promote_basic", True, time.time() - start_time)
|
||||
|
||||
def test_force_promote_during_target_restart(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
kubectl_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
):
|
||||
"""Promote downstream while downstream pods are bouncing.
|
||||
|
||||
Mirrors snippets test_restart_b_during_force_promote.py: promote runs
|
||||
async, restart happens in main thread mid-promote.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_fp_target_restart", max_length=50)
|
||||
c_after = f"{c_name}_after"
|
||||
|
||||
self.log_test_start(
|
||||
"test_force_promote_during_target_restart",
|
||||
"FORCE_PROMOTE_TARGET_RESTART",
|
||||
c_name,
|
||||
)
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
self.resources_to_cleanup.append(("collection", c_after))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
assert self.wait_for_sync(
|
||||
lambda: downstream_client.has_collection(c_name),
|
||||
sync_timeout,
|
||||
f"create collection {c_name}",
|
||||
)
|
||||
|
||||
data = self.generate_test_data_with_id(DEFAULT_INSERT_COUNT, start_id=0)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_synced():
|
||||
try:
|
||||
res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
return (res[0]["count(*)"] if res else 0) >= DEFAULT_INSERT_COUNT
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_synced, sync_timeout, f"sync {c_name}")
|
||||
|
||||
# Promote async — first call almost certainly fails because target dies
|
||||
promote_err = []
|
||||
|
||||
def promote_thread():
|
||||
try:
|
||||
self.do_force_promote(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
except Exception as e:
|
||||
promote_err.append(e)
|
||||
|
||||
th = threading.Thread(target=promote_thread, daemon=True)
|
||||
th.start()
|
||||
|
||||
# Mid-promote: kill target pods, wait for them to come back
|
||||
time.sleep(2)
|
||||
logger.info("[FAILOVER] Killing target pods mid-promote...")
|
||||
kubectl_helper.delete_pods(target_cluster_id)
|
||||
time.sleep(2)
|
||||
kubectl_helper.wait_for_pods_ready(target_cluster_id, timeout=300)
|
||||
|
||||
th.join(timeout=FAILOVER_TIMEOUT)
|
||||
assert not th.is_alive(), "Background force_promote thread did not finish in time"
|
||||
if promote_err:
|
||||
logger.info(f"[EXPECTED] async promote raised: {promote_err[0]}")
|
||||
|
||||
# Retry until writable
|
||||
self.promote_call_until_writable(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
|
||||
schema_after = self.create_manual_id_schema(downstream_client)
|
||||
downstream_client.create_collection(collection_name=c_after, schema=schema_after)
|
||||
idx_after = downstream_client.prepare_index_params()
|
||||
idx_after.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
downstream_client.create_index(c_after, idx_after)
|
||||
downstream_client.load_collection(c_after)
|
||||
data_after = self.generate_test_data_with_id(100, start_id=0)
|
||||
downstream_client.insert(c_after, data_after)
|
||||
downstream_client.flush(c_after)
|
||||
res = downstream_client.query(collection_name=c_after, filter="", output_fields=["count(*)"])
|
||||
assert res and res[0]["count(*)"] >= 100, f"downstream not writable after target restart: {res}"
|
||||
finally:
|
||||
logger.info("[TEARDOWN] Waiting for target pods before topology restore...")
|
||||
kubectl_helper.wait_for_pods_ready(target_cluster_id, timeout=300)
|
||||
self.cleanup_downstream_only_collection(c_after)
|
||||
logger.info("[TEARDOWN] Restoring A→B topology...")
|
||||
try:
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"switchover restore failed: {e}")
|
||||
self.cleanup_resources()
|
||||
self.log_test_end(
|
||||
"test_force_promote_during_target_restart",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def test_force_promote_during_source_restart(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
kubectl_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
):
|
||||
"""Promote downstream while upstream pods are bouncing.
|
||||
|
||||
Mirrors snippets test_restart_a_during_force_promote.py.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_fp_source_restart", max_length=50)
|
||||
c_after = f"{c_name}_after"
|
||||
|
||||
self.log_test_start(
|
||||
"test_force_promote_during_source_restart",
|
||||
"FORCE_PROMOTE_SOURCE_RESTART",
|
||||
c_name,
|
||||
)
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
self.resources_to_cleanup.append(("collection", c_after))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
assert self.wait_for_sync(
|
||||
lambda: downstream_client.has_collection(c_name),
|
||||
sync_timeout,
|
||||
f"create collection {c_name}",
|
||||
)
|
||||
|
||||
data = self.generate_test_data_with_id(DEFAULT_INSERT_COUNT, start_id=0)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_synced():
|
||||
try:
|
||||
res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
return (res[0]["count(*)"] if res else 0) >= DEFAULT_INSERT_COUNT
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_synced, sync_timeout, f"sync {c_name}")
|
||||
|
||||
promote_err = []
|
||||
|
||||
def promote_thread():
|
||||
try:
|
||||
self.do_force_promote(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
except Exception as e:
|
||||
promote_err.append(e)
|
||||
|
||||
th = threading.Thread(target=promote_thread, daemon=True)
|
||||
th.start()
|
||||
|
||||
time.sleep(2)
|
||||
logger.info("[FAILOVER] Killing source pods mid-promote...")
|
||||
kubectl_helper.delete_pods(source_cluster_id)
|
||||
# Don't wait-ready here — promotion should succeed independently
|
||||
# of whether the old primary is back.
|
||||
|
||||
th.join(timeout=FAILOVER_TIMEOUT)
|
||||
assert not th.is_alive(), "Background force_promote thread did not finish in time"
|
||||
if promote_err:
|
||||
logger.info(f"[INFO] async promote raised (may be expected): {promote_err[0]}")
|
||||
|
||||
self.promote_call_until_writable(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
|
||||
schema_after = self.create_manual_id_schema(downstream_client)
|
||||
downstream_client.create_collection(collection_name=c_after, schema=schema_after)
|
||||
idx_after = downstream_client.prepare_index_params()
|
||||
idx_after.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
downstream_client.create_index(c_after, idx_after)
|
||||
downstream_client.load_collection(c_after)
|
||||
data_after = self.generate_test_data_with_id(100, start_id=0)
|
||||
downstream_client.insert(c_after, data_after)
|
||||
downstream_client.flush(c_after)
|
||||
res = downstream_client.query(collection_name=c_after, filter="", output_fields=["count(*)"])
|
||||
assert res and res[0]["count(*)"] >= 100, f"downstream not writable after source restart: {res}"
|
||||
finally:
|
||||
logger.info("[TEARDOWN] Waiting for source pods before topology restore...")
|
||||
kubectl_helper.wait_for_pods_ready(source_cluster_id, timeout=300)
|
||||
self.cleanup_downstream_only_collection(c_after)
|
||||
logger.info("[TEARDOWN] Restoring A→B topology...")
|
||||
try:
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"switchover restore failed: {e}")
|
||||
self.cleanup_resources()
|
||||
self.log_test_end(
|
||||
"test_force_promote_during_source_restart",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def test_force_promote_with_incomplete_ddl(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
):
|
||||
"""release_collection on upstream then immediately force_promote.
|
||||
|
||||
Exercises the Ignore=true path in
|
||||
ackCallbackScheduler.fixIncompleteBroadcastsForForcePromote — DDL that
|
||||
was mid-broadcast when the operator promoted gets marked Ignore=true
|
||||
and resubmitted with replicate header cleared.
|
||||
|
||||
Mirrors snippets test_incomplete_broadcast_ddl.py.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_fp_inc_ddl", max_length=50)
|
||||
c_after = f"{c_name}_after"
|
||||
|
||||
self.log_test_start(
|
||||
"test_force_promote_with_incomplete_ddl",
|
||||
"FORCE_PROMOTE_INCOMPLETE_DDL",
|
||||
c_name,
|
||||
)
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
self.resources_to_cleanup.append(("collection", c_after))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
assert self.wait_for_sync(
|
||||
lambda: downstream_client.has_collection(c_name),
|
||||
sync_timeout,
|
||||
f"create collection {c_name}",
|
||||
)
|
||||
|
||||
data = self.generate_test_data_with_id(DEFAULT_INSERT_COUNT, start_id=0)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_synced():
|
||||
try:
|
||||
res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
return (res[0]["count(*)"] if res else 0) >= DEFAULT_INSERT_COUNT
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_synced, sync_timeout, f"sync {c_name}")
|
||||
|
||||
# In-flight DDL on upstream — may not finish replicating before promote
|
||||
upstream_client.release_collection(c_name)
|
||||
logger.info(f"[INCOMPLETE_DDL] release_collection sent on {c_name}")
|
||||
|
||||
# Immediately promote downstream (no sleep)
|
||||
self.promote_call_until_writable(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
|
||||
# On downstream (now primary), create new collection + insert
|
||||
schema_after = self.create_manual_id_schema(downstream_client)
|
||||
downstream_client.create_collection(collection_name=c_after, schema=schema_after)
|
||||
idx_after = downstream_client.prepare_index_params()
|
||||
idx_after.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
downstream_client.create_index(c_after, idx_after)
|
||||
downstream_client.load_collection(c_after)
|
||||
data_after = self.generate_test_data_with_id(100, start_id=0)
|
||||
downstream_client.insert(c_after, data_after)
|
||||
downstream_client.flush(c_after)
|
||||
res = downstream_client.query(collection_name=c_after, filter="", output_fields=["count(*)"])
|
||||
assert res and res[0]["count(*)"] >= 100, f"downstream not writable after incomplete DDL: {res}"
|
||||
finally:
|
||||
self.cleanup_downstream_only_collection(c_after)
|
||||
logger.info("[TEARDOWN] Restoring A→B topology...")
|
||||
try:
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"switchover restore failed: {e}")
|
||||
self.cleanup_resources()
|
||||
self.log_test_end(
|
||||
"test_force_promote_with_incomplete_ddl",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def _build_partition_manifest(self, source_cluster_id, target_cluster_id, milvus_ns):
|
||||
"""Return a NetworkChaos dict that bidirectionally partitions source ↔ target."""
|
||||
return {
|
||||
"apiVersion": "chaos-mesh.org/v1alpha1",
|
||||
"kind": "NetworkChaos",
|
||||
"metadata": {
|
||||
"name": f"{source_cluster_id}-failover-partition",
|
||||
"namespace": milvus_ns,
|
||||
},
|
||||
"spec": {
|
||||
"action": "partition",
|
||||
"mode": "all",
|
||||
"selector": {
|
||||
"namespaces": [milvus_ns],
|
||||
"labelSelectors": {"app.kubernetes.io/instance": source_cluster_id},
|
||||
},
|
||||
"direction": "both",
|
||||
"target": {
|
||||
"mode": "all",
|
||||
"selector": {
|
||||
"namespaces": [milvus_ns],
|
||||
"labelSelectors": {"app.kubernetes.io/instance": target_cluster_id},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def test_force_promote_with_network_partition(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
milvus_ns,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
):
|
||||
"""Apply NetworkChaos partition between source/target, then force_promote.
|
||||
|
||||
K8s-native scenario that the snippets tests can't reproduce. Simulates
|
||||
the realistic emergency that motivates force_promote: secondary cannot
|
||||
reach primary, operator force-promotes secondary.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_fp_partition", max_length=50)
|
||||
c_after = f"{c_name}_after"
|
||||
chaos_name = f"{source_cluster_id}-failover-partition"
|
||||
|
||||
self.log_test_start(
|
||||
"test_force_promote_with_network_partition",
|
||||
"FORCE_PROMOTE_PARTITION",
|
||||
c_name,
|
||||
)
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
self.resources_to_cleanup.append(("collection", c_after))
|
||||
|
||||
chaos_path = None
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
assert self.wait_for_sync(
|
||||
lambda: downstream_client.has_collection(c_name),
|
||||
sync_timeout,
|
||||
f"create collection {c_name}",
|
||||
)
|
||||
|
||||
data = self.generate_test_data_with_id(DEFAULT_INSERT_COUNT, start_id=0)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_synced():
|
||||
try:
|
||||
res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
return (res[0]["count(*)"] if res else 0) >= DEFAULT_INSERT_COUNT
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_synced, sync_timeout, f"sync {c_name}")
|
||||
|
||||
# Apply NetworkChaos
|
||||
manifest = self._build_partition_manifest(source_cluster_id, target_cluster_id, milvus_ns)
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
|
||||
yaml.safe_dump(manifest, f)
|
||||
chaos_path = f.name
|
||||
logger.info(f"[CHAOS] applying NetworkChaos partition: {chaos_path}")
|
||||
apply_result = subprocess.run(
|
||||
["kubectl", "apply", "-f", chaos_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert apply_result.returncode == 0, f"kubectl apply failed: {apply_result.stderr}"
|
||||
|
||||
# Let the partition propagate
|
||||
time.sleep(10)
|
||||
|
||||
# Force promote downstream — it can't reach upstream
|
||||
self.promote_call_until_writable(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
|
||||
# Verify writable
|
||||
schema_after = self.create_manual_id_schema(downstream_client)
|
||||
downstream_client.create_collection(collection_name=c_after, schema=schema_after)
|
||||
idx_after = downstream_client.prepare_index_params()
|
||||
idx_after.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
downstream_client.create_index(c_after, idx_after)
|
||||
downstream_client.load_collection(c_after)
|
||||
data_after = self.generate_test_data_with_id(100, start_id=0)
|
||||
downstream_client.insert(c_after, data_after)
|
||||
downstream_client.flush(c_after)
|
||||
res = downstream_client.query(collection_name=c_after, filter="", output_fields=["count(*)"])
|
||||
assert res and res[0]["count(*)"] >= 100, f"downstream not writable after partition+promote: {res}"
|
||||
finally:
|
||||
# Remove the partition first so switchover can fan-out across clusters
|
||||
logger.info(f"[CHAOS] cleaning up NetworkChaos {chaos_name}")
|
||||
subprocess.run(
|
||||
["kubectl", "delete", "networkchaos", chaos_name, "-n", milvus_ns],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if chaos_path and os.path.exists(chaos_path):
|
||||
os.unlink(chaos_path)
|
||||
# Allow partition to clear before switchover
|
||||
time.sleep(10)
|
||||
self.cleanup_downstream_only_collection(c_after)
|
||||
logger.info("[TEARDOWN] Restoring A→B topology...")
|
||||
try:
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"switchover restore failed: {e}")
|
||||
self.cleanup_resources()
|
||||
self.log_test_end(
|
||||
"test_force_promote_with_network_partition",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def _do_one_endurance_iteration(
|
||||
self,
|
||||
iteration,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
):
|
||||
"""One endurance iteration: DDL + insert + force_promote + DDL on new primary."""
|
||||
c_before = f"endurance_before_{iteration}"
|
||||
c_after = f"endurance_after_{iteration}"
|
||||
|
||||
# Reset to A→B topology (in case prior iteration left state behind)
|
||||
try:
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"[iter {iteration}] pre-iter switchover failed") from e
|
||||
|
||||
# DDL before failover on upstream
|
||||
self.cleanup_collection(upstream_client, c_before)
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_before, schema=schema)
|
||||
idx = upstream_client.prepare_index_params()
|
||||
idx.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_before, idx)
|
||||
upstream_client.load_collection(c_before)
|
||||
assert self.wait_for_sync(
|
||||
lambda: downstream_client.has_collection(c_before),
|
||||
sync_timeout,
|
||||
f"[iter {iteration}] create {c_before}",
|
||||
)
|
||||
data = self.generate_test_data_with_id(100, start_id=0)
|
||||
upstream_client.insert(c_before, data)
|
||||
upstream_client.flush(c_before)
|
||||
|
||||
def check_synced():
|
||||
try:
|
||||
res = downstream_client.query(collection_name=c_before, filter="", output_fields=["count(*)"])
|
||||
return (res[0]["count(*)"] if res else 0) >= 100
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_synced, sync_timeout, f"[iter {iteration}] sync {c_before}")
|
||||
|
||||
# In-flight DDL
|
||||
upstream_client.release_collection(c_before)
|
||||
|
||||
# Force promote
|
||||
self.promote_call_until_writable(
|
||||
downstream_client,
|
||||
target_cluster_id,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
|
||||
# DDL after failover on downstream
|
||||
schema_after = self.create_manual_id_schema(downstream_client)
|
||||
downstream_client.create_collection(collection_name=c_after, schema=schema_after)
|
||||
idx_after = downstream_client.prepare_index_params()
|
||||
idx_after.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
downstream_client.create_index(c_after, idx_after)
|
||||
downstream_client.load_collection(c_after)
|
||||
data_after = self.generate_test_data_with_id(100, start_id=0)
|
||||
downstream_client.insert(c_after, data_after)
|
||||
downstream_client.flush(c_after)
|
||||
res = downstream_client.query(collection_name=c_after, filter="", output_fields=["count(*)"])
|
||||
assert res and res[0]["count(*)"] >= 100, f"[iter {iteration}] downstream count check failed: {res}"
|
||||
|
||||
# Cleanup both
|
||||
try:
|
||||
downstream_client.drop_collection(c_after)
|
||||
except Exception as e:
|
||||
logger.warning(f"[iter {iteration}] drop {c_after} on downstream: {e}")
|
||||
try:
|
||||
downstream_client.drop_collection(c_before)
|
||||
except Exception as e:
|
||||
logger.warning(f"[iter {iteration}] drop {c_before} on downstream: {e}")
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not os.getenv("RUN_ENDURANCE"),
|
||||
reason="Endurance test only runs when RUN_ENDURANCE is set",
|
||||
)
|
||||
def test_endurance_force_promote(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
):
|
||||
"""Endurance loop: repeated force_promote + DDL for ENDURANCE_DURATION_MINUTES."""
|
||||
duration_minutes = int(os.getenv("ENDURANCE_DURATION_MINUTES", "30"))
|
||||
deadline = time.time() + duration_minutes * 60
|
||||
iteration = 0
|
||||
start_time = time.time()
|
||||
self.log_test_start("test_endurance_force_promote", "ENDURANCE_FORCE_PROMOTE", "")
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
|
||||
try:
|
||||
while time.time() < deadline:
|
||||
iteration += 1
|
||||
logger.info(f"=== Endurance iteration {iteration} ===")
|
||||
self._do_one_endurance_iteration(
|
||||
iteration,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
pchannel_num,
|
||||
)
|
||||
logger.info(f"PASSED endurance: {iteration} iterations in {duration_minutes}m")
|
||||
finally:
|
||||
logger.info("[TEARDOWN] Restoring A→B topology after endurance...")
|
||||
try:
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"switchover restore failed: {e}")
|
||||
self.log_test_end("test_endurance_force_promote", True, time.time() - start_time)
|
||||
@@ -0,0 +1,39 @@
|
||||
import importlib
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
|
||||
if importlib.util.find_spec("yaml") is None:
|
||||
yaml = types.ModuleType("yaml")
|
||||
yaml.safe_dump = lambda *args, **kwargs: None
|
||||
sys.modules.setdefault("yaml", yaml)
|
||||
|
||||
_TestCDCForcePromote = importlib.import_module("cdc.testcases.test_force_promote").TestCDCForcePromote
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self):
|
||||
self.collections = {"downstream_only"}
|
||||
self.dropped = []
|
||||
|
||||
def has_collection(self, collection_name):
|
||||
return collection_name in self.collections
|
||||
|
||||
def drop_collection(self, collection_name):
|
||||
self.dropped.append(collection_name)
|
||||
self.collections.discard(collection_name)
|
||||
|
||||
|
||||
def test_cleanup_downstream_only_collection_before_topology_restore():
|
||||
test = _TestCDCForcePromote()
|
||||
test.resources_to_cleanup = [
|
||||
("collection", "normal_collection"),
|
||||
("collection", "downstream_only"),
|
||||
]
|
||||
test._downstream_client = FakeClient()
|
||||
|
||||
test.cleanup_downstream_only_collection("downstream_only")
|
||||
|
||||
assert test._downstream_client.dropped == ["downstream_only"]
|
||||
assert ("collection", "downstream_only") not in test.resources_to_cleanup
|
||||
assert ("collection", "normal_collection") in test.resources_to_cleanup
|
||||
@@ -0,0 +1,715 @@
|
||||
"""
|
||||
CDC sync tests for full-text search (BM25), text match, and phrase match operations.
|
||||
"""
|
||||
|
||||
import random
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from pymilvus import AnnSearchRequest, DataType, RRFRanker
|
||||
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
class TestCDCSyncFTSAndText(TestCDCSyncBase):
|
||||
"""Test CDC sync for full-text search, text match, and phrase match operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Test 1: FTS insert and search replication
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("analyzer_type", ["standard", "english"])
|
||||
def test_fts_insert_and_search(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
analyzer_type,
|
||||
):
|
||||
"""Test FTS (BM25) insert and search replication.
|
||||
|
||||
Creates an FTS collection with BM25 function, inserts 200 docs, creates
|
||||
SPARSE_INVERTED_INDEX (BM25) and HNSW indexes, then verifies that FTS search
|
||||
results replicate to downstream with sufficient overlap.
|
||||
"""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("fts_search", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_fts_insert_and_search",
|
||||
f"FTS_INSERT_SEARCH(analyzer={analyzer_type})",
|
||||
collection_name,
|
||||
)
|
||||
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create FTS schema and collection
|
||||
logger.info(f"[CREATE] Creating FTS collection '{collection_name}' with analyzer_type='{analyzer_type}'")
|
||||
schema = self.create_fts_schema(upstream_client, analyzer_type)
|
||||
upstream_client.create_collection(collection_name, schema=schema)
|
||||
|
||||
# Insert 200 documents
|
||||
fts_data = self.generate_fts_data(200)
|
||||
logger.info(f"[INSERT] Inserting {len(fts_data)} FTS documents upstream")
|
||||
result = upstream_client.insert(collection_name, fts_data)
|
||||
inserted_count = result.get("insert_count", len(fts_data))
|
||||
logger.info(f"[INSERT] Inserted {inserted_count} documents")
|
||||
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Create SPARSE_INVERTED_INDEX on sparse_output (BM25)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="sparse_output",
|
||||
index_type="SPARSE_INVERTED_INDEX",
|
||||
metric_type="BM25",
|
||||
params={"bm25_k1": 1.5, "bm25_b": 0.75},
|
||||
)
|
||||
# Create HNSW on dense_vector
|
||||
index_params.add_index(
|
||||
field_name="dense_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Run FTS search on upstream
|
||||
query_text = "vector database similarity search"
|
||||
logger.info(f"[SEARCH] Running FTS search upstream with query: '{query_text}'")
|
||||
upstream_results = upstream_client.search(
|
||||
collection_name,
|
||||
data=[query_text],
|
||||
anns_field="sparse_output",
|
||||
limit=10,
|
||||
search_params={"metric_type": "BM25"},
|
||||
output_fields=["text_field", "category"],
|
||||
)
|
||||
upstream_ids = set()
|
||||
if upstream_results and len(upstream_results) > 0:
|
||||
for hit in upstream_results[0]:
|
||||
upstream_ids.add(hit.get("id") or hit.id)
|
||||
|
||||
logger.info(f"[SEARCH] Upstream FTS returned {len(upstream_ids)} results")
|
||||
|
||||
# Wait for collection to appear downstream
|
||||
def check_collection_exists():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_collection_exists,
|
||||
sync_timeout,
|
||||
f"collection '{collection_name}' creation sync",
|
||||
), f"Collection '{collection_name}' did not sync to downstream"
|
||||
|
||||
# Wait for data + index to sync and downstream search results overlap
|
||||
def check_fts_overlap():
|
||||
try:
|
||||
ds_results = downstream_client.search(
|
||||
collection_name,
|
||||
data=[query_text],
|
||||
anns_field="sparse_output",
|
||||
limit=10,
|
||||
search_params={"metric_type": "BM25"},
|
||||
output_fields=["text_field", "category"],
|
||||
)
|
||||
if not ds_results or len(ds_results) == 0:
|
||||
return False
|
||||
downstream_ids = set()
|
||||
for hit in ds_results[0]:
|
||||
downstream_ids.add(hit.get("id") or hit.id)
|
||||
if not downstream_ids:
|
||||
return False
|
||||
if len(upstream_ids) == 0:
|
||||
return len(downstream_ids) > 0
|
||||
overlap = len(upstream_ids & downstream_ids) / max(len(upstream_ids), 1)
|
||||
logger.info(
|
||||
f"[OVERLAP] FTS search overlap: {overlap:.2f} "
|
||||
f"(upstream={len(upstream_ids)}, downstream={len(downstream_ids)})"
|
||||
)
|
||||
return overlap >= self.SEARCH_OVERLAP_THRESHOLD
|
||||
except Exception as e:
|
||||
logger.warning(f"FTS overlap check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_fts_overlap,
|
||||
sync_timeout,
|
||||
f"FTS search overlap sync (analyzer={analyzer_type})",
|
||||
), f"FTS search results did not reach overlap threshold {self.SEARCH_OVERLAP_THRESHOLD} on downstream"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_fts_insert_and_search", True, duration)
|
||||
|
||||
except Exception as exc:
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_fts_insert_and_search", False, duration)
|
||||
raise exc
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Test 2: TEXT_MATCH sync
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("analyzer_type", ["standard", "english"])
|
||||
def test_text_match_sync(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
analyzer_type,
|
||||
):
|
||||
"""Test TEXT_MATCH query replication.
|
||||
|
||||
Creates a collection with a VARCHAR field that has analyzer + match enabled,
|
||||
inserts 100 rows, and verifies that TEXT_MATCH queries return the same count
|
||||
on both upstream and downstream.
|
||||
"""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("text_match", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_text_match_sync",
|
||||
f"TEXT_MATCH(analyzer={analyzer_type})",
|
||||
collection_name,
|
||||
)
|
||||
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Build schema with analyzer-enabled VARCHAR
|
||||
schema = upstream_client.create_schema(enable_dynamic_field=False)
|
||||
schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True)
|
||||
schema.add_field("dense_vector", DataType.FLOAT_VECTOR, dim=128)
|
||||
schema.add_field(
|
||||
"text_field",
|
||||
DataType.VARCHAR,
|
||||
max_length=2048,
|
||||
enable_analyzer=True,
|
||||
enable_match=True,
|
||||
analyzer_params={"type": analyzer_type},
|
||||
)
|
||||
|
||||
logger.info(f"[CREATE] Creating text-match collection '{collection_name}'")
|
||||
upstream_client.create_collection(collection_name, schema=schema)
|
||||
|
||||
# Insert 100 rows from FTS_SENTENCES
|
||||
data = []
|
||||
for i in range(100):
|
||||
data.append(
|
||||
{
|
||||
"dense_vector": [random.random() for _ in range(128)],
|
||||
"text_field": self.FTS_SENTENCES[i % len(self.FTS_SENTENCES)],
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(f"[INSERT] Inserting {len(data)} rows upstream")
|
||||
upstream_client.insert(collection_name, data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Create HNSW index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="dense_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Run TEXT_MATCH query on upstream
|
||||
filter_expr = "TEXT_MATCH(text_field, 'vector database')"
|
||||
logger.info(f"[QUERY] Upstream TEXT_MATCH query: {filter_expr}")
|
||||
upstream_results = upstream_client.query(
|
||||
collection_name,
|
||||
filter=filter_expr,
|
||||
output_fields=["id", "text_field"],
|
||||
limit=100,
|
||||
)
|
||||
upstream_count = len(upstream_results)
|
||||
logger.info(f"[QUERY] Upstream TEXT_MATCH returned {upstream_count} rows")
|
||||
|
||||
# Wait for collection to appear downstream
|
||||
def check_collection_exists():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_collection_exists,
|
||||
sync_timeout,
|
||||
f"collection '{collection_name}' creation sync",
|
||||
)
|
||||
|
||||
# Wait for downstream query count to match
|
||||
def check_count_match():
|
||||
try:
|
||||
ds_results = downstream_client.query(
|
||||
collection_name,
|
||||
filter=filter_expr,
|
||||
output_fields=["id", "text_field"],
|
||||
limit=100,
|
||||
)
|
||||
ds_count = len(ds_results)
|
||||
logger.info(f"[VERIFY] TEXT_MATCH downstream count={ds_count}, upstream count={upstream_count}")
|
||||
return ds_count == upstream_count
|
||||
except Exception as e:
|
||||
logger.warning(f"TEXT_MATCH count check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_count_match,
|
||||
sync_timeout,
|
||||
f"TEXT_MATCH query count sync (analyzer={analyzer_type})",
|
||||
), f"TEXT_MATCH query count mismatch between upstream ({upstream_count}) and downstream after timeout"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_text_match_sync", True, duration)
|
||||
|
||||
except Exception as exc:
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_text_match_sync", False, duration)
|
||||
raise exc
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Test 3: PHRASE_MATCH sync
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("analyzer_type", ["standard", "english"])
|
||||
def test_phrase_match_sync(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
analyzer_type,
|
||||
):
|
||||
"""Test PHRASE_MATCH query replication.
|
||||
|
||||
Same collection setup as text_match. Queries PHRASE_MATCH with slop=1 and
|
||||
verifies that upstream and downstream return the same count.
|
||||
"""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("phrase_match", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_phrase_match_sync",
|
||||
f"PHRASE_MATCH(analyzer={analyzer_type})",
|
||||
collection_name,
|
||||
)
|
||||
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Build schema with analyzer-enabled VARCHAR
|
||||
schema = upstream_client.create_schema(enable_dynamic_field=False)
|
||||
schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True)
|
||||
schema.add_field("dense_vector", DataType.FLOAT_VECTOR, dim=128)
|
||||
schema.add_field(
|
||||
"text_field",
|
||||
DataType.VARCHAR,
|
||||
max_length=2048,
|
||||
enable_analyzer=True,
|
||||
enable_match=True,
|
||||
analyzer_params={"type": analyzer_type},
|
||||
)
|
||||
|
||||
logger.info(f"[CREATE] Creating phrase-match collection '{collection_name}'")
|
||||
upstream_client.create_collection(collection_name, schema=schema)
|
||||
|
||||
# Insert 100 rows from FTS_SENTENCES
|
||||
data = []
|
||||
for i in range(100):
|
||||
data.append(
|
||||
{
|
||||
"dense_vector": [random.random() for _ in range(128)],
|
||||
"text_field": self.FTS_SENTENCES[i % len(self.FTS_SENTENCES)],
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(f"[INSERT] Inserting {len(data)} rows upstream")
|
||||
upstream_client.insert(collection_name, data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Create HNSW index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="dense_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Run PHRASE_MATCH query on upstream (slop=1)
|
||||
filter_expr = "PHRASE_MATCH(text_field, 'brown fox', 1)"
|
||||
logger.info(f"[QUERY] Upstream PHRASE_MATCH query: {filter_expr}")
|
||||
upstream_results = upstream_client.query(
|
||||
collection_name,
|
||||
filter=filter_expr,
|
||||
output_fields=["id", "text_field"],
|
||||
limit=100,
|
||||
)
|
||||
upstream_count = len(upstream_results)
|
||||
logger.info(f"[QUERY] Upstream PHRASE_MATCH returned {upstream_count} rows")
|
||||
|
||||
# Wait for collection to appear downstream
|
||||
def check_collection_exists():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_collection_exists,
|
||||
sync_timeout,
|
||||
f"collection '{collection_name}' creation sync",
|
||||
)
|
||||
|
||||
# Wait for downstream query count to match
|
||||
def check_count_match():
|
||||
try:
|
||||
ds_results = downstream_client.query(
|
||||
collection_name,
|
||||
filter=filter_expr,
|
||||
output_fields=["id", "text_field"],
|
||||
limit=100,
|
||||
)
|
||||
ds_count = len(ds_results)
|
||||
logger.info(f"[VERIFY] PHRASE_MATCH downstream count={ds_count}, upstream count={upstream_count}")
|
||||
return ds_count == upstream_count
|
||||
except Exception as e:
|
||||
logger.warning(f"PHRASE_MATCH count check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_count_match,
|
||||
sync_timeout,
|
||||
f"PHRASE_MATCH query count sync (analyzer={analyzer_type})",
|
||||
), f"PHRASE_MATCH query count mismatch between upstream ({upstream_count}) and downstream after timeout"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_phrase_match_sync", True, duration)
|
||||
|
||||
except Exception as exc:
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_phrase_match_sync", False, duration)
|
||||
raise exc
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Test 4: Hybrid search (FTS + dense) replication
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def test_hybrid_search_fts_dense(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
):
|
||||
"""Test hybrid search (BM25 sparse + HNSW dense) replication.
|
||||
|
||||
Creates an FTS collection, inserts 300 docs, builds both sparse (BM25) and
|
||||
dense (HNSW) indexes, and verifies that hybrid search results on downstream
|
||||
have sufficient overlap with upstream results.
|
||||
"""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("hybrid_fts", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_hybrid_search_fts_dense",
|
||||
"HYBRID_SEARCH_FTS_DENSE",
|
||||
collection_name,
|
||||
)
|
||||
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create FTS schema (standard analyzer)
|
||||
schema = self.create_fts_schema(upstream_client, "standard")
|
||||
logger.info(f"[CREATE] Creating hybrid-search collection '{collection_name}'")
|
||||
upstream_client.create_collection(collection_name, schema=schema)
|
||||
|
||||
# Insert 300 documents
|
||||
fts_data = self.generate_fts_data(300)
|
||||
logger.info(f"[INSERT] Inserting {len(fts_data)} documents upstream")
|
||||
upstream_client.insert(collection_name, fts_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Create both indexes
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="sparse_output",
|
||||
index_type="SPARSE_INVERTED_INDEX",
|
||||
metric_type="BM25",
|
||||
params={"bm25_k1": 1.5, "bm25_b": 0.75},
|
||||
)
|
||||
index_params.add_index(
|
||||
field_name="dense_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Build hybrid search requests
|
||||
sparse_req = AnnSearchRequest(
|
||||
data=["vector database similarity search"],
|
||||
anns_field="sparse_output",
|
||||
param={"metric_type": "BM25"},
|
||||
limit=10,
|
||||
)
|
||||
dense_query_vec = [random.random() for _ in range(128)]
|
||||
dense_req = AnnSearchRequest(
|
||||
data=[dense_query_vec],
|
||||
anns_field="dense_vector",
|
||||
param={"metric_type": "L2", "params": {"ef": 64}},
|
||||
limit=10,
|
||||
)
|
||||
|
||||
logger.info("[SEARCH] Running hybrid search on upstream")
|
||||
upstream_results = upstream_client.hybrid_search(
|
||||
collection_name,
|
||||
reqs=[sparse_req, dense_req],
|
||||
ranker=RRFRanker(),
|
||||
limit=10,
|
||||
output_fields=["text_field", "category"],
|
||||
)
|
||||
|
||||
upstream_ids = set()
|
||||
if upstream_results and len(upstream_results) > 0:
|
||||
for hit in upstream_results[0]:
|
||||
upstream_ids.add(hit.get("id") or hit.id)
|
||||
|
||||
logger.info(f"[SEARCH] Upstream hybrid search returned {len(upstream_ids)} results")
|
||||
|
||||
# Wait for collection to appear downstream
|
||||
def check_collection_exists():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_collection_exists,
|
||||
sync_timeout,
|
||||
f"collection '{collection_name}' creation sync",
|
||||
)
|
||||
|
||||
# Wait for downstream hybrid search overlap
|
||||
def check_hybrid_overlap():
|
||||
try:
|
||||
ds_sparse_req = AnnSearchRequest(
|
||||
data=["vector database similarity search"],
|
||||
anns_field="sparse_output",
|
||||
param={"metric_type": "BM25"},
|
||||
limit=10,
|
||||
)
|
||||
ds_dense_req = AnnSearchRequest(
|
||||
data=[dense_query_vec],
|
||||
anns_field="dense_vector",
|
||||
param={"metric_type": "L2", "params": {"ef": 64}},
|
||||
limit=10,
|
||||
)
|
||||
ds_results = downstream_client.hybrid_search(
|
||||
collection_name,
|
||||
reqs=[ds_sparse_req, ds_dense_req],
|
||||
ranker=RRFRanker(),
|
||||
limit=10,
|
||||
output_fields=["text_field", "category"],
|
||||
)
|
||||
if not ds_results or len(ds_results) == 0:
|
||||
return False
|
||||
downstream_ids = set()
|
||||
for hit in ds_results[0]:
|
||||
downstream_ids.add(hit.get("id") or hit.id)
|
||||
if not downstream_ids:
|
||||
return False
|
||||
if len(upstream_ids) == 0:
|
||||
return len(downstream_ids) > 0
|
||||
overlap = len(upstream_ids & downstream_ids) / max(len(upstream_ids), 1)
|
||||
logger.info(
|
||||
f"[OVERLAP] Hybrid search overlap: {overlap:.2f} "
|
||||
f"(upstream={len(upstream_ids)}, downstream={len(downstream_ids)})"
|
||||
)
|
||||
return overlap >= self.SEARCH_OVERLAP_THRESHOLD
|
||||
except Exception as e:
|
||||
logger.warning(f"Hybrid overlap check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_hybrid_overlap,
|
||||
sync_timeout,
|
||||
"hybrid search (FTS + dense) overlap sync",
|
||||
), f"Hybrid search results did not reach overlap threshold {self.SEARCH_OVERLAP_THRESHOLD} on downstream"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_hybrid_search_fts_dense", True, duration)
|
||||
|
||||
except Exception as exc:
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_hybrid_search_fts_dense", False, duration)
|
||||
raise exc
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Test 5: FTS after switchover
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def test_fts_after_switchover(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
):
|
||||
"""Test FTS replication continues correctly after CDC topology switchover.
|
||||
|
||||
1. Create FTS collection, insert 100 docs, build index, verify sync.
|
||||
2. Perform switchover so downstream becomes the new source.
|
||||
3. Insert 50 more docs into the new source (original downstream).
|
||||
4. Verify FTS search works on the new downstream (original upstream).
|
||||
5. Switch back to original topology.
|
||||
"""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("fts_switchover", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_fts_after_switchover",
|
||||
"FTS_AFTER_SWITCHOVER",
|
||||
collection_name,
|
||||
)
|
||||
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Phase 1: Create FTS collection and index on original upstream
|
||||
logger.info(f"[PHASE1] Creating FTS collection '{collection_name}' on upstream")
|
||||
schema = self.create_fts_schema(upstream_client, "standard")
|
||||
upstream_client.create_collection(collection_name, schema=schema)
|
||||
|
||||
fts_data = self.generate_fts_data(100)
|
||||
logger.info(f"[PHASE1] Inserting {len(fts_data)} documents upstream")
|
||||
upstream_client.insert(collection_name, fts_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="sparse_output",
|
||||
index_type="SPARSE_INVERTED_INDEX",
|
||||
metric_type="BM25",
|
||||
params={"bm25_k1": 1.5, "bm25_b": 0.75},
|
||||
)
|
||||
index_params.add_index(
|
||||
field_name="dense_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Verify initial sync to downstream
|
||||
def check_initial_sync():
|
||||
if not downstream_client.has_collection(collection_name):
|
||||
return False
|
||||
try:
|
||||
ds_results = downstream_client.search(
|
||||
collection_name,
|
||||
data=["vector database"],
|
||||
anns_field="sparse_output",
|
||||
limit=5,
|
||||
search_params={"metric_type": "BM25"},
|
||||
output_fields=["text_field"],
|
||||
)
|
||||
return ds_results is not None and len(ds_results) > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_initial_sync,
|
||||
sync_timeout,
|
||||
f"initial FTS sync for '{collection_name}'",
|
||||
), f"Initial FTS sync failed for collection '{collection_name}'"
|
||||
|
||||
logger.info("[PHASE1] Initial FTS sync verified")
|
||||
|
||||
# Phase 2: Switchover — downstream becomes new source
|
||||
logger.info(f"[PHASE2] Switching CDC direction: {target_cluster_id} -> {source_cluster_id}")
|
||||
switchover_helper(target_cluster_id, source_cluster_id)
|
||||
|
||||
# Insert 50 more docs to the new source (original downstream)
|
||||
extra_data = self.generate_fts_data(50)
|
||||
logger.info(f"[PHASE2] Inserting {len(extra_data)} additional docs to new source (downstream_client)")
|
||||
downstream_client.insert(collection_name, extra_data)
|
||||
downstream_client.flush(collection_name)
|
||||
|
||||
# Phase 3: Verify FTS search works on new downstream (original upstream)
|
||||
query_text = "distributed database replication"
|
||||
|
||||
def check_fts_on_new_downstream():
|
||||
try:
|
||||
results = upstream_client.search(
|
||||
collection_name,
|
||||
data=[query_text],
|
||||
anns_field="sparse_output",
|
||||
limit=5,
|
||||
search_params={"metric_type": "BM25"},
|
||||
output_fields=["text_field"],
|
||||
)
|
||||
return results is not None and len(results) > 0
|
||||
except Exception as e:
|
||||
logger.warning(f"FTS on new downstream check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_fts_on_new_downstream,
|
||||
sync_timeout,
|
||||
"FTS search on new downstream after switchover",
|
||||
), "FTS search on new downstream (original upstream) failed after switchover"
|
||||
|
||||
logger.info("[PHASE3] FTS search verified on new downstream after switchover")
|
||||
|
||||
# Phase 4: Switch back to original topology
|
||||
logger.info(f"[PHASE4] Switching back to original topology: {source_cluster_id} -> {target_cluster_id}")
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_fts_after_switchover", True, duration)
|
||||
|
||||
except Exception as exc:
|
||||
# Best-effort restore original topology on failure
|
||||
try:
|
||||
logger.warning("[RECOVER] Attempting to restore original CDC topology after failure")
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
except Exception as restore_exc:
|
||||
logger.error(f"[RECOVER] Failed to restore topology: {restore_exc}")
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_fts_after_switchover", False, duration)
|
||||
raise exc
|
||||
@@ -0,0 +1,52 @@
|
||||
import pytest
|
||||
from chaos.checker import Import2PCChecker
|
||||
from common import common_func as cf
|
||||
from common.common_type import CaseLabel
|
||||
from pymilvus import connections
|
||||
|
||||
|
||||
class TestCDCImport2PC:
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
def test_import_2pc_manual_commit_replicates_to_downstream(
|
||||
self,
|
||||
upstream_client,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
import_2pc_minio_endpoint,
|
||||
import_2pc_minio_bucket,
|
||||
import_2pc_downstream_minio_endpoint,
|
||||
import_2pc_downstream_minio_bucket,
|
||||
import_2pc_rows,
|
||||
sync_timeout,
|
||||
):
|
||||
"""
|
||||
target: verify Import 2PC manual commit is a valid CDC workload
|
||||
method: create an upstream manual import job with auto_commit=false, stage the same parquet files in both
|
||||
object stores, wait for primary and secondary Uncommitted, then CommitImport on upstream
|
||||
expected: imported PKs stay invisible on both clusters before commit and become visible on both clusters
|
||||
after the replicated CommitImport is consumed
|
||||
"""
|
||||
collection_name = cf.gen_unique_str("CDCImport2PC_")
|
||||
connections.connect("default", uri=upstream_uri, token=upstream_token)
|
||||
checker = Import2PCChecker(
|
||||
collection_name=collection_name,
|
||||
rows_per_import=import_2pc_rows,
|
||||
minio_endpoint=import_2pc_minio_endpoint,
|
||||
bucket_name=import_2pc_minio_bucket,
|
||||
uri=upstream_uri,
|
||||
token=upstream_token,
|
||||
downstream_uri=downstream_uri,
|
||||
downstream_token=downstream_token,
|
||||
downstream_minio_endpoint=import_2pc_downstream_minio_endpoint,
|
||||
downstream_bucket_name=import_2pc_downstream_minio_bucket,
|
||||
visibility_timeout=max(sync_timeout, 180),
|
||||
)
|
||||
try:
|
||||
res, ok = checker.run_task()
|
||||
assert ok, res
|
||||
finally:
|
||||
checker.terminate()
|
||||
if upstream_client.has_collection(collection_name):
|
||||
upstream_client.drop_collection(collection_name)
|
||||
@@ -0,0 +1,731 @@
|
||||
"""
|
||||
CDC sync tests for index operations.
|
||||
"""
|
||||
|
||||
import time
|
||||
import random
|
||||
import pytest
|
||||
from common.common_type import CaseLabel
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
class TestCDCSyncIndex(TestCDCSyncBase):
|
||||
"""Test CDC sync for index operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def test_create_index(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test CREATE_INDEX operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_create_idx")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
|
||||
# Create index
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="IVF_FLAT",
|
||||
metric_type="L2",
|
||||
params={"nlist": 128},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for index creation to sync
|
||||
def check_index():
|
||||
try:
|
||||
downstream_indexes = downstream_client.list_indexes(collection_name)
|
||||
return len(downstream_indexes) > 0
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_index, sync_timeout, f"create index on {collection_name}"
|
||||
)
|
||||
|
||||
def test_drop_index(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DROP_INDEX operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_drop_idx")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection and index
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="IVF_FLAT",
|
||||
metric_type="L2",
|
||||
params={"nlist": 128},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for setup to sync
|
||||
def check_setup():
|
||||
try:
|
||||
return (
|
||||
downstream_client.has_collection(collection_name)
|
||||
and len(downstream_client.list_indexes(collection_name)) > 0
|
||||
)
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup, sync_timeout, f"setup collection and index {collection_name}"
|
||||
)
|
||||
|
||||
# Drop index
|
||||
upstream_client.drop_index(collection_name, "vector")
|
||||
|
||||
# Wait for index drop to sync
|
||||
def check_drop():
|
||||
try:
|
||||
downstream_indexes = downstream_client.list_indexes(collection_name)
|
||||
return len(downstream_indexes) == 0
|
||||
except:
|
||||
return True # If error, assume index is dropped
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_drop, sync_timeout, f"drop index on {collection_name}"
|
||||
)
|
||||
|
||||
def test_create_vector_indexes_comprehensive(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test CREATE_INDEX operation sync for all vector index types."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
# Test cases for different vector types and their applicable indexes
|
||||
test_cases = [
|
||||
# FLOAT_VECTOR indexes
|
||||
{
|
||||
"field_name": "float_vector",
|
||||
"field_type": "FLOAT_VECTOR",
|
||||
"index_tests": [
|
||||
{"index_type": "FLAT", "metric_type": "L2", "params": {}},
|
||||
{
|
||||
"index_type": "IVF_FLAT",
|
||||
"metric_type": "L2",
|
||||
"params": {"nlist": 128},
|
||||
},
|
||||
{
|
||||
"index_type": "IVF_SQ8",
|
||||
"metric_type": "L2",
|
||||
"params": {"nlist": 128},
|
||||
},
|
||||
{
|
||||
"index_type": "IVF_PQ",
|
||||
"metric_type": "L2",
|
||||
"params": {"nlist": 128, "m": 16, "nbits": 8},
|
||||
},
|
||||
{
|
||||
"index_type": "HNSW",
|
||||
"metric_type": "L2",
|
||||
"params": {"M": 16, "efConstruction": 200},
|
||||
},
|
||||
],
|
||||
},
|
||||
# FLOAT16_VECTOR indexes
|
||||
{
|
||||
"field_name": "float16_vector",
|
||||
"field_type": "FLOAT16_VECTOR",
|
||||
"index_tests": [
|
||||
{"index_type": "FLAT", "metric_type": "L2", "params": {}},
|
||||
{
|
||||
"index_type": "IVF_FLAT",
|
||||
"metric_type": "L2",
|
||||
"params": {"nlist": 128},
|
||||
},
|
||||
{
|
||||
"index_type": "HNSW",
|
||||
"metric_type": "L2",
|
||||
"params": {"M": 16, "efConstruction": 200},
|
||||
},
|
||||
],
|
||||
},
|
||||
# BINARY_VECTOR indexes
|
||||
{
|
||||
"field_name": "binary_vector",
|
||||
"field_type": "BINARY_VECTOR",
|
||||
"index_tests": [
|
||||
{"index_type": "BIN_FLAT", "metric_type": "HAMMING", "params": {}},
|
||||
{
|
||||
"index_type": "BIN_IVF_FLAT",
|
||||
"metric_type": "HAMMING",
|
||||
"params": {"nlist": 128},
|
||||
},
|
||||
],
|
||||
},
|
||||
# SPARSE_FLOAT_VECTOR indexes
|
||||
{
|
||||
"field_name": "sparse_vector",
|
||||
"field_type": "SPARSE_FLOAT_VECTOR",
|
||||
"index_tests": [
|
||||
{
|
||||
"index_type": "SPARSE_INVERTED_INDEX",
|
||||
"metric_type": "IP",
|
||||
"params": {},
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
for test_case in test_cases:
|
||||
for index_test in test_case["index_tests"]:
|
||||
collection_name = self.gen_unique_name(
|
||||
f"test_idx_{test_case['field_type'].lower()}_{index_test['index_type'].lower()}"
|
||||
)
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"[INDEX_TEST] Testing {test_case['field_type']} with {index_test['index_type']} index"
|
||||
)
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection with specific vector field
|
||||
schema = self._create_vector_schema(
|
||||
upstream_client,
|
||||
test_case["field_name"],
|
||||
test_case["field_type"],
|
||||
)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema
|
||||
)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create,
|
||||
sync_timeout,
|
||||
f"create collection {collection_name}",
|
||||
)
|
||||
|
||||
# Insert test data before creating index (better practice)
|
||||
test_data = self._generate_test_data_for_vector_field(
|
||||
test_case["field_name"], test_case["field_type"], 100
|
||||
)
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
logger.info(
|
||||
f"[DATA_INSERTED] Inserted 100 records before creating {test_case['field_type']} index"
|
||||
)
|
||||
|
||||
# Create specific index
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name=test_case["field_name"],
|
||||
index_type=index_test["index_type"],
|
||||
metric_type=index_test["metric_type"],
|
||||
params=index_test["params"],
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for index creation to sync
|
||||
def check_index():
|
||||
try:
|
||||
downstream_indexes = downstream_client.list_indexes(
|
||||
collection_name
|
||||
)
|
||||
return len(downstream_indexes) > 0
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_index,
|
||||
sync_timeout,
|
||||
f"create {index_test['index_type']} index on {collection_name}",
|
||||
)
|
||||
|
||||
# Verify index details
|
||||
try:
|
||||
index_info = downstream_client.describe_index(
|
||||
collection_name, test_case["field_name"]
|
||||
)
|
||||
logger.info(
|
||||
f"[INDEX_VERIFICATION] {index_test['index_type']} index created successfully: {index_info}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to verify {index_test['index_type']} index details: {e}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[INDEX_ERROR] Failed to test {test_case['field_type']} with {index_test['index_type']}: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
def test_create_scalar_indexes_comprehensive(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test CREATE_INDEX operation sync for all scalar field index types."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
# Test cases for different scalar types and their applicable indexes
|
||||
test_cases = [
|
||||
# VARCHAR indexes
|
||||
{
|
||||
"field_name": "varchar_field",
|
||||
"field_type": "VARCHAR",
|
||||
"index_tests": [
|
||||
{"index_type": "INVERTED", "params": {}},
|
||||
{"index_type": "Trie", "params": {}},
|
||||
],
|
||||
},
|
||||
# BOOL indexes
|
||||
{
|
||||
"field_name": "bool_field",
|
||||
"field_type": "BOOL",
|
||||
"index_tests": [
|
||||
{"index_type": "INVERTED", "params": {}},
|
||||
],
|
||||
},
|
||||
# INT32 indexes
|
||||
{
|
||||
"field_name": "int32_field",
|
||||
"field_type": "INT32",
|
||||
"index_tests": [
|
||||
{"index_type": "INVERTED", "params": {}},
|
||||
{"index_type": "STL_SORT", "params": {}},
|
||||
],
|
||||
},
|
||||
# INT64 indexes
|
||||
{
|
||||
"field_name": "int64_field",
|
||||
"field_type": "INT64",
|
||||
"index_tests": [
|
||||
{"index_type": "INVERTED", "params": {}},
|
||||
{"index_type": "STL_SORT", "params": {}},
|
||||
],
|
||||
},
|
||||
# FLOAT indexes
|
||||
{
|
||||
"field_name": "float_field",
|
||||
"field_type": "FLOAT",
|
||||
"index_tests": [
|
||||
{"index_type": "INVERTED", "params": {}},
|
||||
],
|
||||
},
|
||||
# DOUBLE indexes
|
||||
{
|
||||
"field_name": "double_field",
|
||||
"field_type": "DOUBLE",
|
||||
"index_tests": [
|
||||
{"index_type": "INVERTED", "params": {}},
|
||||
],
|
||||
},
|
||||
# ARRAY indexes
|
||||
{
|
||||
"field_name": "int32_array",
|
||||
"field_type": "ARRAY",
|
||||
"element_type": "INT32",
|
||||
"index_tests": [
|
||||
{"index_type": "INVERTED", "params": {}},
|
||||
],
|
||||
},
|
||||
# JSON indexes - use AUTOINDEX (recommended) with proper JSON path syntax
|
||||
{
|
||||
"field_name": "json_field",
|
||||
"field_type": "JSON",
|
||||
"index_tests": [
|
||||
{
|
||||
"index_type": "AUTOINDEX",
|
||||
"params": {
|
||||
"json_path": 'json_field["name"]',
|
||||
"json_cast_type": "VARCHAR",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
for test_case in test_cases:
|
||||
for index_test in test_case["index_tests"]:
|
||||
collection_name = self.gen_unique_name(
|
||||
f"test_idx_{test_case['field_type'].lower()}_{index_test['index_type'].lower()}"
|
||||
)
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"[SCALAR_INDEX_TEST] Testing {test_case['field_type']} with {index_test['index_type']} index"
|
||||
)
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection with specific scalar field
|
||||
schema = self._create_scalar_schema(upstream_client, test_case)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema
|
||||
)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create,
|
||||
sync_timeout,
|
||||
f"create collection {collection_name}",
|
||||
)
|
||||
|
||||
# Insert test data before creating index (better practice)
|
||||
test_data = self._generate_test_data_for_scalar_field(
|
||||
test_case["field_name"], test_case["field_type"], 100
|
||||
)
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
logger.info(
|
||||
f"[DATA_INSERTED] Inserted 100 records before creating {test_case['field_type']} index"
|
||||
)
|
||||
|
||||
# Create specific index
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
if test_case["field_type"] == "JSON":
|
||||
# JSON fields need special handling with index_name
|
||||
index_params.add_index(
|
||||
field_name=test_case["field_name"],
|
||||
index_type=index_test["index_type"],
|
||||
index_name=f"{test_case['field_name']}_name_index",
|
||||
params=index_test["params"],
|
||||
)
|
||||
else:
|
||||
index_params.add_index(
|
||||
field_name=test_case["field_name"],
|
||||
index_type=index_test["index_type"],
|
||||
params=index_test["params"],
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for index creation to sync
|
||||
def check_index():
|
||||
try:
|
||||
downstream_indexes = downstream_client.list_indexes(
|
||||
collection_name
|
||||
)
|
||||
return len(downstream_indexes) > 0
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_index,
|
||||
sync_timeout,
|
||||
f"create {index_test['index_type']} index on {collection_name}",
|
||||
)
|
||||
|
||||
# Verify index details
|
||||
try:
|
||||
index_info = downstream_client.describe_index(
|
||||
collection_name, test_case["field_name"]
|
||||
)
|
||||
logger.info(
|
||||
f"[SCALAR_INDEX_VERIFICATION] {index_test['index_type']} index created successfully: {index_info}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to verify {index_test['index_type']} scalar index details: {e}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[SCALAR_INDEX_ERROR] Failed to test {test_case['field_type']} with {index_test['index_type']}: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
def _create_vector_schema(self, client, field_name, field_type):
|
||||
"""Create schema for vector index testing."""
|
||||
from pymilvus import DataType
|
||||
|
||||
schema = client.create_schema(enable_dynamic_field=True)
|
||||
schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True)
|
||||
|
||||
if field_type == "FLOAT_VECTOR":
|
||||
schema.add_field(field_name, DataType.FLOAT_VECTOR, dim=128)
|
||||
elif field_type == "FLOAT16_VECTOR":
|
||||
schema.add_field(field_name, DataType.FLOAT16_VECTOR, dim=64)
|
||||
elif field_type == "BFLOAT16_VECTOR":
|
||||
schema.add_field(field_name, DataType.BFLOAT16_VECTOR, dim=64)
|
||||
elif field_type == "BINARY_VECTOR":
|
||||
schema.add_field(field_name, DataType.BINARY_VECTOR, dim=128)
|
||||
elif field_type == "SPARSE_FLOAT_VECTOR":
|
||||
schema.add_field(field_name, DataType.SPARSE_FLOAT_VECTOR)
|
||||
elif field_type == "INT8_VECTOR":
|
||||
schema.add_field(field_name, DataType.INT8_VECTOR, dim=128)
|
||||
|
||||
return schema
|
||||
|
||||
def _create_scalar_schema(self, client, test_case):
|
||||
"""Create schema for scalar index testing."""
|
||||
from pymilvus import DataType
|
||||
|
||||
schema = client.create_schema(enable_dynamic_field=True)
|
||||
schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True)
|
||||
schema.add_field(
|
||||
"vector", DataType.FLOAT_VECTOR, dim=128
|
||||
) # Required for collection
|
||||
|
||||
field_name = test_case["field_name"]
|
||||
field_type = test_case["field_type"]
|
||||
|
||||
if field_type == "VARCHAR":
|
||||
schema.add_field(field_name, DataType.VARCHAR, max_length=1000)
|
||||
elif field_type == "BOOL":
|
||||
schema.add_field(field_name, DataType.BOOL)
|
||||
elif field_type == "INT32":
|
||||
schema.add_field(field_name, DataType.INT32)
|
||||
elif field_type == "INT64":
|
||||
schema.add_field(field_name, DataType.INT64)
|
||||
elif field_type == "FLOAT":
|
||||
schema.add_field(field_name, DataType.FLOAT)
|
||||
elif field_type == "DOUBLE":
|
||||
schema.add_field(field_name, DataType.DOUBLE)
|
||||
elif field_type == "ARRAY" and test_case.get("element_type") == "INT32":
|
||||
schema.add_field(
|
||||
field_name,
|
||||
DataType.ARRAY,
|
||||
element_type=DataType.INT32,
|
||||
max_capacity=100,
|
||||
)
|
||||
elif field_type == "JSON":
|
||||
schema.add_field(field_name, DataType.JSON)
|
||||
|
||||
return schema
|
||||
|
||||
def _generate_test_data_for_vector_field(self, field_name, field_type, count=100):
|
||||
"""Generate test data for specific vector field type."""
|
||||
from pymilvus import DataType
|
||||
|
||||
data = []
|
||||
for _ in range(count):
|
||||
record = {}
|
||||
|
||||
if field_type == "FLOAT_VECTOR":
|
||||
vectors = self._gen_vectors(1, 128, DataType.FLOAT_VECTOR)
|
||||
record[field_name] = vectors[0]
|
||||
elif field_type == "FLOAT16_VECTOR":
|
||||
vectors = self._gen_vectors(1, 64, DataType.FLOAT16_VECTOR)
|
||||
record[field_name] = vectors[0]
|
||||
elif field_type == "BFLOAT16_VECTOR":
|
||||
vectors = self._gen_vectors(1, 64, DataType.BFLOAT16_VECTOR)
|
||||
record[field_name] = vectors[0]
|
||||
elif field_type == "BINARY_VECTOR":
|
||||
vectors = self._gen_vectors(1, 128, DataType.BINARY_VECTOR)
|
||||
record[field_name] = vectors[0]
|
||||
elif field_type == "SPARSE_FLOAT_VECTOR":
|
||||
vectors = self._gen_vectors(1, 1000, DataType.SPARSE_FLOAT_VECTOR)
|
||||
record[field_name] = vectors[0]
|
||||
elif field_type == "INT8_VECTOR":
|
||||
vectors = self._gen_vectors(1, 128, DataType.INT8_VECTOR)
|
||||
record[field_name] = vectors[0]
|
||||
|
||||
data.append(record)
|
||||
|
||||
return data
|
||||
|
||||
def _generate_test_data_for_scalar_field(self, field_name, field_type, count=100):
|
||||
"""Generate test data for specific scalar field type."""
|
||||
data = []
|
||||
for i in range(count):
|
||||
record = {
|
||||
"vector": [
|
||||
random.random() for _ in range(128)
|
||||
], # Required base vector field
|
||||
}
|
||||
|
||||
if field_type == "VARCHAR":
|
||||
record[field_name] = f"test_string_{i}_{random.randint(1000, 9999)}"
|
||||
elif field_type == "BOOL":
|
||||
record[field_name] = random.choice([True, False])
|
||||
elif field_type == "INT32":
|
||||
record[field_name] = random.randint(-1000000, 1000000)
|
||||
elif field_type == "INT64":
|
||||
record[field_name] = random.randint(-1000000000, 1000000000)
|
||||
elif field_type == "FLOAT":
|
||||
record[field_name] = random.uniform(-1000.0, 1000.0)
|
||||
elif field_type == "DOUBLE":
|
||||
record[field_name] = random.uniform(-1000.0, 1000.0)
|
||||
elif field_type == "ARRAY":
|
||||
record[field_name] = [
|
||||
random.randint(-100, 100) for _ in range(random.randint(1, 10))
|
||||
]
|
||||
elif field_type == "JSON":
|
||||
record[field_name] = {
|
||||
"name": f"test_item_{i}",
|
||||
"value": random.randint(1, 1000),
|
||||
"category": random.choice(["A", "B", "C"]),
|
||||
"metadata": {
|
||||
"score": random.uniform(0.0, 100.0),
|
||||
"created": f"2024-01-{random.randint(1, 28):02d}",
|
||||
},
|
||||
}
|
||||
|
||||
data.append(record)
|
||||
|
||||
return data
|
||||
|
||||
def test_create_bfloat16_int8_vector_indexes(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test CREATE_INDEX operation sync for BFLOAT16_VECTOR and INT8_VECTOR (combined test due to 4-vector limit)."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
# Test cases for BFLOAT16_VECTOR and INT8_VECTOR
|
||||
test_cases = [
|
||||
# BFLOAT16_VECTOR indexes - use AUTOINDEX for compatibility
|
||||
{
|
||||
"field_name": "bfloat16_vector",
|
||||
"field_type": "BFLOAT16_VECTOR",
|
||||
"index_tests": [
|
||||
{"index_type": "AUTOINDEX", "metric_type": "L2", "params": {}},
|
||||
],
|
||||
},
|
||||
# INT8_VECTOR indexes - use AUTOINDEX for compatibility
|
||||
{
|
||||
"field_name": "int8_vector",
|
||||
"field_type": "INT8_VECTOR",
|
||||
"index_tests": [
|
||||
{"index_type": "AUTOINDEX", "metric_type": "L2", "params": {}},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
for test_case in test_cases:
|
||||
for index_test in test_case["index_tests"]:
|
||||
collection_name = self.gen_unique_name(
|
||||
f"test_idx_{test_case['field_type'].lower()}_{index_test['index_type'].lower()}"
|
||||
)
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"[{test_case['field_type']}_INDEX_TEST] Testing {test_case['field_type']} with {index_test['index_type']} index"
|
||||
)
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection with specific vector field
|
||||
schema = self._create_vector_schema(
|
||||
upstream_client,
|
||||
test_case["field_name"],
|
||||
test_case["field_type"],
|
||||
)
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name, schema=schema
|
||||
)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create,
|
||||
sync_timeout,
|
||||
f"create collection {collection_name}",
|
||||
)
|
||||
|
||||
# Insert test data before creating index (better practice)
|
||||
if test_case["field_type"] == "BFLOAT16_VECTOR":
|
||||
test_data = self.generate_bfloat16_test_data(100)
|
||||
else: # INT8_VECTOR
|
||||
test_data = self.generate_int8_test_data(100)
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
logger.info(
|
||||
f"[DATA_INSERTED] Inserted 100 records before creating {test_case['field_type']} index"
|
||||
)
|
||||
|
||||
# Create specific index
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name=test_case["field_name"],
|
||||
index_type=index_test["index_type"],
|
||||
metric_type=index_test["metric_type"],
|
||||
params=index_test["params"],
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for index creation to sync
|
||||
def check_index():
|
||||
try:
|
||||
downstream_indexes = downstream_client.list_indexes(
|
||||
collection_name
|
||||
)
|
||||
return len(downstream_indexes) > 0
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_index,
|
||||
sync_timeout,
|
||||
f"create {index_test['index_type']} index on {collection_name}",
|
||||
)
|
||||
|
||||
# Verify index details
|
||||
try:
|
||||
index_info = downstream_client.describe_index(
|
||||
collection_name, test_case["field_name"]
|
||||
)
|
||||
logger.info(
|
||||
f"[{test_case['field_type']}_INDEX_VERIFICATION] {index_test['index_type']} index created successfully: {index_info}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to verify {index_test['index_type']} {test_case['field_type']} index details: {e}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[{test_case['field_type']}_INDEX_ERROR] Failed to test {test_case['field_type']} with {index_test['index_type']}: {e}"
|
||||
)
|
||||
raise
|
||||
@@ -0,0 +1,316 @@
|
||||
"""
|
||||
CDC sync tests for multi-database operations.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
from pymilvus import MilvusClient
|
||||
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
class TestCDCSyncMultiDatabase(TestCDCSyncBase):
|
||||
"""Test CDC sync for operations across multiple databases."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_uri = getattr(self, "_upstream_uri", None)
|
||||
upstream_token = getattr(self, "_upstream_token", None)
|
||||
|
||||
if upstream_uri:
|
||||
# Re-create a default-db client for cleanup
|
||||
try:
|
||||
client = MilvusClient(uri=upstream_uri, token=upstream_token)
|
||||
except Exception as e:
|
||||
logger.warning(f"[CLEANUP] Failed to create upstream client: {e}")
|
||||
return
|
||||
|
||||
# Clean up collections in databases first, then databases
|
||||
for resource_type, resource_data in self.resources_to_cleanup:
|
||||
if resource_type == "collection_in_db":
|
||||
db_name, c_name = resource_data
|
||||
try:
|
||||
db_client = MilvusClient(
|
||||
uri=upstream_uri,
|
||||
token=upstream_token,
|
||||
db_name=db_name,
|
||||
)
|
||||
if db_client.has_collection(c_name):
|
||||
logger.info(f"[CLEANUP] Dropping collection {c_name} in db {db_name}")
|
||||
db_client.drop_collection(c_name)
|
||||
db_client.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"[CLEANUP] Failed to drop collection {c_name} in db {db_name}: {e}")
|
||||
|
||||
for resource_type, resource_data in self.resources_to_cleanup:
|
||||
if resource_type == "database":
|
||||
db_name = resource_data
|
||||
self.cleanup_database(client, db_name)
|
||||
|
||||
client.close()
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def test_create_collections_in_multiple_dbs(
|
||||
self,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
):
|
||||
"""Test creating collections in multiple databases syncs to downstream."""
|
||||
self._upstream_uri = upstream_uri
|
||||
self._upstream_token = upstream_token
|
||||
|
||||
db_name_1 = self.gen_unique_name("test_mdb_db1")
|
||||
db_name_2 = self.gen_unique_name("test_mdb_db2")
|
||||
c_name_1 = self.gen_unique_name("test_mdb_col1")
|
||||
c_name_2 = self.gen_unique_name("test_mdb_col2")
|
||||
|
||||
self.resources_to_cleanup.append(("collection_in_db", (db_name_1, c_name_1)))
|
||||
self.resources_to_cleanup.append(("collection_in_db", (db_name_2, c_name_2)))
|
||||
self.resources_to_cleanup.append(("database", db_name_1))
|
||||
self.resources_to_cleanup.append(("database", db_name_2))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_database(upstream_client, db_name_1)
|
||||
self.cleanup_database(upstream_client, db_name_2)
|
||||
|
||||
# Create databases
|
||||
upstream_client.create_database(db_name_1)
|
||||
upstream_client.create_database(db_name_2)
|
||||
|
||||
# Create DB-scoped clients
|
||||
up_db1_client = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name_1)
|
||||
up_db2_client = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name_2)
|
||||
|
||||
# Create collection in db1 with 100 rows
|
||||
schema1 = self.create_default_schema(up_db1_client)
|
||||
up_db1_client.create_collection(collection_name=c_name_1, schema=schema1, consistency_level="Strong")
|
||||
up_db1_client.insert(c_name_1, self.generate_test_data(100))
|
||||
up_db1_client.flush(c_name_1)
|
||||
|
||||
# Create collection in db2 with 200 rows
|
||||
schema2 = self.create_default_schema(up_db2_client)
|
||||
up_db2_client.create_collection(collection_name=c_name_2, schema=schema2, consistency_level="Strong")
|
||||
up_db2_client.insert(c_name_2, self.generate_test_data(200))
|
||||
up_db2_client.flush(c_name_2)
|
||||
|
||||
up_db1_client.close()
|
||||
up_db2_client.close()
|
||||
|
||||
# Wait for both databases and collections to appear on downstream
|
||||
def check_db1_collection():
|
||||
try:
|
||||
if db_name_1 not in downstream_client.list_databases():
|
||||
return False
|
||||
dn_db1 = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name_1)
|
||||
exists = dn_db1.has_collection(c_name_1)
|
||||
if exists:
|
||||
stats = dn_db1.get_collection_stats(c_name_1)
|
||||
logger.info(f"Downstream db1 collection stats: {stats}")
|
||||
dn_db1.close()
|
||||
return exists
|
||||
except Exception as e:
|
||||
logger.warning(f"Check db1 collection failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_db1_collection, sync_timeout, f"collection {c_name_1} in {db_name_1}")
|
||||
|
||||
def check_db2_collection():
|
||||
try:
|
||||
if db_name_2 not in downstream_client.list_databases():
|
||||
return False
|
||||
dn_db2 = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name_2)
|
||||
exists = dn_db2.has_collection(c_name_2)
|
||||
if exists:
|
||||
stats = dn_db2.get_collection_stats(c_name_2)
|
||||
logger.info(f"Downstream db2 collection stats: {stats}")
|
||||
dn_db2.close()
|
||||
return exists
|
||||
except Exception as e:
|
||||
logger.warning(f"Check db2 collection failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_db2_collection, sync_timeout, f"collection {c_name_2} in {db_name_2}")
|
||||
|
||||
# Verify row counts
|
||||
dn_db1 = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name_1)
|
||||
dn_db2 = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name_2)
|
||||
try:
|
||||
stats1 = dn_db1.get_collection_stats(c_name_1)
|
||||
stats2 = dn_db2.get_collection_stats(c_name_2)
|
||||
logger.info(f"DB1 collection row count: {stats1.get('row_count')}")
|
||||
logger.info(f"DB2 collection row count: {stats2.get('row_count')}")
|
||||
assert stats1.get("row_count", 0) >= 100, (
|
||||
f"Expected >= 100 rows in {c_name_1}, got {stats1.get('row_count')}"
|
||||
)
|
||||
assert stats2.get("row_count", 0) >= 200, (
|
||||
f"Expected >= 200 rows in {c_name_2}, got {stats2.get('row_count')}"
|
||||
)
|
||||
finally:
|
||||
dn_db1.close()
|
||||
dn_db2.close()
|
||||
|
||||
def test_drop_db_with_collections(
|
||||
self,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
):
|
||||
"""Test drop DB with collection syncs to downstream (DB gone from downstream)."""
|
||||
self._upstream_uri = upstream_uri
|
||||
self._upstream_token = upstream_token
|
||||
|
||||
db_name = self.gen_unique_name("test_mdb_drop_db")
|
||||
c_name = self.gen_unique_name("test_mdb_drop_col")
|
||||
|
||||
self.resources_to_cleanup.append(("collection_in_db", (db_name, c_name)))
|
||||
self.resources_to_cleanup.append(("database", db_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_database(upstream_client, db_name)
|
||||
|
||||
# Create database and collection
|
||||
upstream_client.create_database(db_name)
|
||||
up_db_client = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name)
|
||||
schema = self.create_default_schema(up_db_client)
|
||||
up_db_client.create_collection(collection_name=c_name, schema=schema, consistency_level="Strong")
|
||||
up_db_client.insert(c_name, self.generate_test_data(50))
|
||||
up_db_client.flush(c_name)
|
||||
up_db_client.close()
|
||||
|
||||
# Wait for DB + collection to sync to downstream
|
||||
def check_created():
|
||||
try:
|
||||
if db_name not in downstream_client.list_databases():
|
||||
return False
|
||||
dn = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name)
|
||||
exists = dn.has_collection(c_name)
|
||||
dn.close()
|
||||
return exists
|
||||
except Exception as e:
|
||||
logger.warning(f"Check DB+collection created: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_created, sync_timeout, f"create db {db_name} with collection")
|
||||
|
||||
# Drop collection then DB in upstream
|
||||
up_db_client2 = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name)
|
||||
up_db_client2.drop_collection(c_name)
|
||||
up_db_client2.close()
|
||||
upstream_client.drop_database(db_name)
|
||||
|
||||
assert db_name not in upstream_client.list_databases(), (
|
||||
f"Database {db_name} still exists in upstream after drop"
|
||||
)
|
||||
|
||||
# Wait for DB to be gone on downstream
|
||||
def check_db_dropped():
|
||||
return db_name not in downstream_client.list_databases()
|
||||
|
||||
assert self.wait_for_sync(check_db_dropped, sync_timeout, f"drop database {db_name}")
|
||||
|
||||
def test_cross_db_operations(
|
||||
self,
|
||||
upstream_uri,
|
||||
upstream_token,
|
||||
downstream_uri,
|
||||
downstream_token,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
):
|
||||
"""Test cross-DB operations: create DB, collection, insert, alter DB properties, create 2nd collection."""
|
||||
self._upstream_uri = upstream_uri
|
||||
self._upstream_token = upstream_token
|
||||
|
||||
db_name = self.gen_unique_name("test_mdb_cross_db")
|
||||
c_name_1 = self.gen_unique_name("test_mdb_cross_col1")
|
||||
c_name_2 = self.gen_unique_name("test_mdb_cross_col2")
|
||||
|
||||
self.resources_to_cleanup.append(("collection_in_db", (db_name, c_name_1)))
|
||||
self.resources_to_cleanup.append(("collection_in_db", (db_name, c_name_2)))
|
||||
self.resources_to_cleanup.append(("database", db_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_database(upstream_client, db_name)
|
||||
|
||||
# Step 1: Create database
|
||||
upstream_client.create_database(db_name)
|
||||
|
||||
# Step 2: Create DB-scoped client and 1st collection with insert
|
||||
up_db_client = MilvusClient(uri=upstream_uri, token=upstream_token, db_name=db_name)
|
||||
schema1 = self.create_default_schema(up_db_client)
|
||||
up_db_client.create_collection(collection_name=c_name_1, schema=schema1, consistency_level="Strong")
|
||||
up_db_client.insert(c_name_1, self.generate_test_data(100))
|
||||
up_db_client.flush(c_name_1)
|
||||
|
||||
# Step 3: Alter DB properties
|
||||
upstream_client.alter_database_properties(
|
||||
db_name=db_name,
|
||||
properties={"database.max.collections": 10},
|
||||
)
|
||||
|
||||
# Step 4: Create 2nd collection in the same DB
|
||||
schema2 = self.create_default_schema(up_db_client)
|
||||
up_db_client.create_collection(collection_name=c_name_2, schema=schema2, consistency_level="Strong")
|
||||
up_db_client.close()
|
||||
|
||||
# Wait for all operations to sync to downstream
|
||||
|
||||
# Check database exists on downstream
|
||||
def check_db_exists():
|
||||
return db_name in downstream_client.list_databases()
|
||||
|
||||
assert self.wait_for_sync(check_db_exists, sync_timeout, f"create database {db_name}")
|
||||
|
||||
# Check 1st collection exists on downstream
|
||||
def check_col1():
|
||||
try:
|
||||
dn = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name)
|
||||
exists = dn.has_collection(c_name_1)
|
||||
dn.close()
|
||||
return exists
|
||||
except Exception as e:
|
||||
logger.warning(f"Check col1 sync failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_col1, sync_timeout, f"collection {c_name_1} in {db_name}")
|
||||
|
||||
# Check 2nd collection exists on downstream
|
||||
def check_col2():
|
||||
try:
|
||||
dn = MilvusClient(uri=downstream_uri, token=downstream_token, db_name=db_name)
|
||||
exists = dn.has_collection(c_name_2)
|
||||
dn.close()
|
||||
return exists
|
||||
except Exception as e:
|
||||
logger.warning(f"Check col2 sync failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_col2, sync_timeout, f"collection {c_name_2} in {db_name}")
|
||||
|
||||
# Check DB properties synced
|
||||
def check_db_props():
|
||||
try:
|
||||
if db_name not in downstream_client.list_databases():
|
||||
return False
|
||||
props = downstream_client.describe_database(db_name)
|
||||
logger.info(f"Downstream database properties: {props}")
|
||||
return str(props.get("database.max.collections", "")) == "10"
|
||||
except Exception as e:
|
||||
logger.warning(f"Check DB properties sync failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_db_props, sync_timeout, f"DB properties sync for {db_name}")
|
||||
@@ -0,0 +1,513 @@
|
||||
"""
|
||||
CDC sync tests for partition operations.
|
||||
"""
|
||||
|
||||
import time
|
||||
import pytest
|
||||
from common.common_type import CaseLabel
|
||||
from .base import TestCDCSyncBase
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
class TestCDCSyncPartition(TestCDCSyncBase):
|
||||
"""Test CDC sync for partition operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def test_create_partition(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test CREATE_PARTITION operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_part_create")
|
||||
partition_name = self.gen_unique_name("test_part_create")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create collection {collection_name}"
|
||||
)
|
||||
|
||||
# Create partition
|
||||
upstream_client.create_partition(collection_name, partition_name)
|
||||
|
||||
# Verify partition exists in upstream
|
||||
upstream_partitions = upstream_client.list_partitions(collection_name)
|
||||
assert partition_name in upstream_partitions
|
||||
|
||||
# Wait for partition sync to downstream
|
||||
def check_partition():
|
||||
try:
|
||||
downstream_partitions = downstream_client.list_partitions(
|
||||
collection_name
|
||||
)
|
||||
return partition_name in downstream_partitions
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_partition, sync_timeout, f"create partition {partition_name}"
|
||||
)
|
||||
|
||||
def test_drop_partition(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DROP_PARTITION operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_part_drop")
|
||||
partition_name = self.gen_unique_name("test_part_drop")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection and partition
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
upstream_client.create_partition(collection_name, partition_name)
|
||||
|
||||
# Wait for setup to sync
|
||||
def check_setup():
|
||||
try:
|
||||
return downstream_client.has_collection(
|
||||
collection_name
|
||||
) and partition_name in downstream_client.list_partitions(
|
||||
collection_name
|
||||
)
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup,
|
||||
sync_timeout,
|
||||
f"setup collection and partition {collection_name}",
|
||||
)
|
||||
|
||||
# Drop partition
|
||||
upstream_client.drop_partition(collection_name, partition_name)
|
||||
|
||||
# Verify partition is dropped in upstream
|
||||
upstream_partitions = upstream_client.list_partitions(collection_name)
|
||||
assert partition_name not in upstream_partitions
|
||||
|
||||
# Wait for drop to sync to downstream
|
||||
def check_drop():
|
||||
try:
|
||||
downstream_partitions = downstream_client.list_partitions(
|
||||
collection_name
|
||||
)
|
||||
return partition_name not in downstream_partitions
|
||||
except:
|
||||
return True # If error, assume partition is dropped
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_drop, sync_timeout, f"drop partition {partition_name}"
|
||||
)
|
||||
|
||||
def test_load_partition(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test LOAD_PARTITION operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_part_load")
|
||||
partition_name = self.gen_unique_name("test_part_load")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection, partition, and index
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
upstream_client.create_partition(collection_name, partition_name)
|
||||
|
||||
# Create index and load collection (required for querying/searching)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector", index_type="AUTOINDEX", metric_type="L2"
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
|
||||
# Wait for setup to sync
|
||||
def check_setup():
|
||||
try:
|
||||
return downstream_client.has_collection(
|
||||
collection_name
|
||||
) and partition_name in downstream_client.list_partitions(
|
||||
collection_name
|
||||
)
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup,
|
||||
sync_timeout,
|
||||
f"setup collection and partition {collection_name}",
|
||||
)
|
||||
|
||||
# Load partition
|
||||
upstream_client.load_partitions(collection_name, [partition_name])
|
||||
# check partition load state in upstream
|
||||
upstream_load_state = upstream_client.get_load_state(
|
||||
collection_name, partition_name
|
||||
)
|
||||
load_state = str(upstream_load_state["state"])
|
||||
print(f"DEBUG: partition load state in upstream: {load_state}")
|
||||
|
||||
# Wait for load to sync
|
||||
def check_load():
|
||||
try:
|
||||
# Check partition load state
|
||||
load_state = downstream_client.get_load_state(
|
||||
collection_name=collection_name, partition_name=partition_name
|
||||
)
|
||||
print(
|
||||
f"DEBUG: partition load state in check_load: {load_state['state']}"
|
||||
)
|
||||
return "Loaded" == str(load_state["state"])
|
||||
except Exception as e:
|
||||
print(f"DEBUG: get_load_state exception in check_load: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_load, sync_timeout, f"load partition {partition_name}"
|
||||
)
|
||||
|
||||
def test_release_partition(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test RELEASE_PARTITION operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_part_release")
|
||||
partition_name = self.gen_unique_name("test_part_release")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection, partition, index, and load
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
upstream_client.create_partition(collection_name, partition_name)
|
||||
|
||||
# Create index and load collection (required for querying/searching)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector", index_type="AUTOINDEX", metric_type="L2"
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
upstream_client.load_partitions(collection_name, [partition_name])
|
||||
for p_name in [partition_name, None]:
|
||||
data = [{"vector": [0.1] * 128, "id": i} for i in range(100)]
|
||||
upstream_client.insert(collection_name, data, partition_name=p_name)
|
||||
|
||||
# Wait for setup to sync
|
||||
def check_setup():
|
||||
try:
|
||||
query_vector = [[0.1] * 128]
|
||||
downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
partition_names=[partition_name],
|
||||
output_fields=[],
|
||||
)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup, sync_timeout, f"setup and load partition {partition_name}"
|
||||
)
|
||||
|
||||
# Release partition
|
||||
upstream_client.release_partitions(collection_name, [partition_name])
|
||||
|
||||
# check partition load state in upstream
|
||||
upstream_load_state = upstream_client.get_load_state(
|
||||
collection_name, partition_name
|
||||
)
|
||||
load_state = str(upstream_load_state["state"])
|
||||
print(f"DEBUG: partition load state in upstream: {load_state}")
|
||||
|
||||
# check partition load state in upstream
|
||||
def check_release():
|
||||
try:
|
||||
query_vector = [[0.1] * 128]
|
||||
res = upstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
partition_names=[partition_name],
|
||||
output_fields=[],
|
||||
)
|
||||
print(
|
||||
f"DEBUG: released partition {partition_name} can still be searched: {res}"
|
||||
)
|
||||
print(
|
||||
f"DEBUG: released partition {partition_name} can still be searched"
|
||||
)
|
||||
return False
|
||||
except:
|
||||
print(f"DEBUG: released partition {partition_name} cannot be searched")
|
||||
return True
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_release, sync_timeout, f"release partition {partition_name}"
|
||||
)
|
||||
|
||||
# check partition load state in downstream
|
||||
def check_release():
|
||||
try:
|
||||
query_vector = [[0.1] * 128]
|
||||
downstream_client.search(
|
||||
collection_name=collection_name,
|
||||
data=query_vector,
|
||||
limit=1,
|
||||
partition_names=[partition_name],
|
||||
output_fields=[],
|
||||
)
|
||||
print(
|
||||
f"DEBUG: released partition {partition_name} can still be searched"
|
||||
)
|
||||
return False
|
||||
except:
|
||||
print(f"DEBUG: released partition {partition_name} cannot be searched")
|
||||
return True
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_release, sync_timeout, f"release partition {partition_name}"
|
||||
)
|
||||
|
||||
def test_partition_insert(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test INSERT operation to partition sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_part_insert")
|
||||
partition_name = self.gen_unique_name("test_part_insert")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection and partition
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
upstream_client.create_partition(collection_name, partition_name)
|
||||
|
||||
# Create index and load collection (required for querying/searching)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector", index_type="AUTOINDEX", metric_type="L2"
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for setup to sync
|
||||
def check_setup():
|
||||
try:
|
||||
return downstream_client.has_collection(
|
||||
collection_name
|
||||
) and partition_name in downstream_client.list_partitions(
|
||||
collection_name
|
||||
)
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup,
|
||||
sync_timeout,
|
||||
f"setup collection and partition {collection_name}",
|
||||
)
|
||||
|
||||
# Insert data to specific partition
|
||||
test_data = self.generate_test_data(100)
|
||||
result = upstream_client.insert(
|
||||
collection_name, test_data, partition_name=partition_name
|
||||
)
|
||||
inserted_count = result.get("insert_count", len(test_data))
|
||||
|
||||
# Flush to ensure data is persisted
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Wait for data sync to downstream partition by querying
|
||||
def check_data():
|
||||
try:
|
||||
# Query data in specific partition
|
||||
result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
partition_names=[partition_name],
|
||||
)
|
||||
count = result[0]["count(*)"] if result else 0
|
||||
return count >= inserted_count
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_data, sync_timeout, f"insert data to partition {partition_name}"
|
||||
)
|
||||
|
||||
def test_partition_delete(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DELETE operation from partition sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
collection_name = self.gen_unique_name("test_col_part_delete")
|
||||
partition_name = self.gen_unique_name("test_part_delete")
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create collection and partition
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
consistency_level="Strong",
|
||||
)
|
||||
upstream_client.create_partition(collection_name, partition_name)
|
||||
|
||||
# Create index and load collection (required for querying/searching)
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector", index_type="AUTOINDEX", metric_type="L2"
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for setup to sync
|
||||
def check_setup():
|
||||
try:
|
||||
return downstream_client.has_collection(
|
||||
collection_name
|
||||
) and partition_name in downstream_client.list_partitions(
|
||||
collection_name
|
||||
)
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_setup,
|
||||
sync_timeout,
|
||||
f"setup collection and partition {collection_name}",
|
||||
)
|
||||
|
||||
# Insert data to partition
|
||||
test_data = self.generate_test_data(100)
|
||||
upstream_client.insert(
|
||||
collection_name, test_data, partition_name=partition_name
|
||||
)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Wait for initial data sync by querying partition
|
||||
def check_data():
|
||||
try:
|
||||
result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
partition_names=[partition_name],
|
||||
)
|
||||
count = result[0]["count(*)"] if result else 0
|
||||
return count >= 100
|
||||
except:
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_data, sync_timeout, f"initial data sync to partition {partition_name}"
|
||||
)
|
||||
|
||||
# Delete some data from partition
|
||||
delete_ids = list(range(20)) # Delete first 20 records
|
||||
upstream_client.delete(
|
||||
collection_name, filter=f"id in {delete_ids}", partition_name=partition_name
|
||||
)
|
||||
upstream_client.flush(collection_name)
|
||||
deleted_result = upstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter=f"id in {delete_ids}",
|
||||
output_fields=["id"],
|
||||
partition_names=[partition_name],
|
||||
)
|
||||
total_count = upstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
partition_names=[partition_name],
|
||||
)
|
||||
total_count = total_count[0]["count(*)"] if total_count else 0
|
||||
print(f"DEBUG: deleted_result in upstream: {deleted_result}")
|
||||
print(f"DEBUG: total_count in upstream: {total_count}")
|
||||
|
||||
# Wait for delete to sync by querying partition
|
||||
def check_delete():
|
||||
try:
|
||||
# Query for the deleted records in partition - should return empty
|
||||
deleted_result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter=f"id in {delete_ids}",
|
||||
output_fields=["id"],
|
||||
partition_names=[partition_name],
|
||||
)
|
||||
# Query total count in partition
|
||||
count_result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
partition_names=[partition_name],
|
||||
)
|
||||
deleted_count = len(deleted_result) if deleted_result else 0
|
||||
total_count = count_result[0]["count(*)"] if count_result else 0
|
||||
|
||||
print(f"DEBUG: deleted_result in check_delete: {deleted_result}")
|
||||
print(f"DEBUG: count_result in check_delete: {count_result}")
|
||||
print(
|
||||
f"DEBUG: deleted_count: {deleted_count}, total_count: {total_count}"
|
||||
)
|
||||
|
||||
# Verify deleted records are gone and total count is correct in partition
|
||||
return deleted_count == 0 and total_count == 80
|
||||
except Exception as e:
|
||||
print(f"DEBUG: query exception in check_delete: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_delete, sync_timeout, f"delete data from partition {partition_name}"
|
||||
)
|
||||
@@ -0,0 +1,873 @@
|
||||
"""
|
||||
CDC sync tests for RBAC operations.
|
||||
"""
|
||||
|
||||
import time
|
||||
import pytest
|
||||
from common.common_type import CaseLabel
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
class TestCDCSyncRBAC(TestCDCSyncBase):
|
||||
"""Test CDC sync for RBAC operations."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "user":
|
||||
self.cleanup_user(upstream_client, resource_name)
|
||||
elif resource_type == "role":
|
||||
self.cleanup_role(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def test_create_role(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test CREATE_ROLE operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
role_name = self.gen_unique_name("test_role_create")
|
||||
self.resources_to_cleanup.append(("role", role_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_role(upstream_client, role_name)
|
||||
|
||||
# Create role in upstream
|
||||
upstream_client.create_role(role_name)
|
||||
assert role_name in upstream_client.list_roles()
|
||||
|
||||
# Wait for sync to downstream
|
||||
def check_sync():
|
||||
return role_name in downstream_client.list_roles()
|
||||
|
||||
assert self.wait_for_sync(check_sync, sync_timeout, f"create role {role_name}")
|
||||
|
||||
def test_drop_role(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DROP_ROLE operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
role_name = self.gen_unique_name("test_role_drop")
|
||||
self.resources_to_cleanup.append(("role", role_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_role(upstream_client, role_name)
|
||||
|
||||
# Create role first
|
||||
upstream_client.create_role(role_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return role_name in downstream_client.list_roles()
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create role {role_name}"
|
||||
)
|
||||
|
||||
# Drop role in upstream
|
||||
upstream_client.drop_role(role_name)
|
||||
assert role_name not in upstream_client.list_roles()
|
||||
|
||||
# Wait for drop to sync
|
||||
def check_drop():
|
||||
return role_name not in downstream_client.list_roles()
|
||||
|
||||
assert self.wait_for_sync(check_drop, sync_timeout, f"drop role {role_name}")
|
||||
|
||||
def test_create_user(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test CREATE_USER operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
username = self.gen_unique_name("test_user_create", max_length=31)
|
||||
password = "TestPass123!"
|
||||
self.resources_to_cleanup.append(("user", username))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_user(upstream_client, username)
|
||||
|
||||
# Create user in upstream
|
||||
upstream_client.create_user(username, password)
|
||||
assert username in upstream_client.list_users()
|
||||
|
||||
# Wait for sync to downstream
|
||||
def check_sync():
|
||||
return username in downstream_client.list_users()
|
||||
|
||||
assert self.wait_for_sync(check_sync, sync_timeout, f"create user {username}")
|
||||
|
||||
def test_drop_user(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test DROP_USER operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
username = self.gen_unique_name("test_user_drop", max_length=31)
|
||||
password = "TestPass123!"
|
||||
self.resources_to_cleanup.append(("user", username))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_user(upstream_client, username)
|
||||
|
||||
# Create user first
|
||||
upstream_client.create_user(username, password)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return username in downstream_client.list_users()
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create user {username}")
|
||||
|
||||
# Drop user in upstream
|
||||
upstream_client.drop_user(username)
|
||||
assert username not in upstream_client.list_users()
|
||||
|
||||
# Wait for drop to sync
|
||||
def check_drop():
|
||||
return username not in downstream_client.list_users()
|
||||
|
||||
assert self.wait_for_sync(check_drop, sync_timeout, f"drop user {username}")
|
||||
|
||||
def test_grant_role(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test GRANT_ROLE operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
username = self.gen_unique_name("test_user_grant", max_length=31)
|
||||
role_name = self.gen_unique_name("test_role_grant")
|
||||
password = "TestPass123!"
|
||||
self.resources_to_cleanup.append(("user", username))
|
||||
self.resources_to_cleanup.append(("role", role_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_user(upstream_client, username)
|
||||
self.cleanup_role(upstream_client, role_name)
|
||||
|
||||
# Create user and role
|
||||
upstream_client.create_user(username, password)
|
||||
upstream_client.create_role(role_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return (
|
||||
username in downstream_client.list_users()
|
||||
and role_name in downstream_client.list_roles()
|
||||
)
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, "create user/role for grant"
|
||||
)
|
||||
|
||||
# Grant role to user
|
||||
upstream_client.grant_role(username, role_name)
|
||||
|
||||
# Wait for grant to sync
|
||||
def check_grant():
|
||||
# Allow operation to propagate
|
||||
time.sleep(2)
|
||||
# Verify role is bound to user using describe_user
|
||||
try:
|
||||
user_info = downstream_client.describe_user(username)
|
||||
user_roles = user_info.get("roles", [])
|
||||
print(f"DEBUG: user_roles in check_grant: {user_roles}")
|
||||
return role_name in user_roles
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking user roles: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_grant, sync_timeout, f"grant role {role_name} to user {username}"
|
||||
)
|
||||
|
||||
def test_revoke_role(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test REVOKE_ROLE operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
username = self.gen_unique_name("test_user_revoke", max_length=31)
|
||||
role_name = self.gen_unique_name("test_role_revoke")
|
||||
password = "TestPass123!"
|
||||
self.resources_to_cleanup.append(("user", username))
|
||||
self.resources_to_cleanup.append(("role", role_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_user(upstream_client, username)
|
||||
self.cleanup_role(upstream_client, role_name)
|
||||
|
||||
# Create user and role, then grant role
|
||||
upstream_client.create_user(username, password)
|
||||
upstream_client.create_role(role_name)
|
||||
upstream_client.grant_role(username, role_name)
|
||||
|
||||
# Wait for setup to sync
|
||||
time.sleep(5)
|
||||
|
||||
# Revoke role from user
|
||||
upstream_client.revoke_role(username, role_name)
|
||||
|
||||
# Wait for revoke to sync
|
||||
def check_revoke():
|
||||
time.sleep(2) # Allow operation to propagate
|
||||
# Verify role is removed from user using describe_user
|
||||
try:
|
||||
user_info = downstream_client.describe_user(username)
|
||||
user_roles = user_info.get("roles", [])
|
||||
print(f"DEBUG: user_roles in check_revoke: {user_roles}")
|
||||
return role_name not in user_roles
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking user roles: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_revoke, sync_timeout, f"revoke role {role_name} from user {username}"
|
||||
)
|
||||
|
||||
def test_grant_privilege(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test GRANT_PRIVILEGE operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
role_name = self.gen_unique_name("test_role_priv_grant")
|
||||
self.resources_to_cleanup.append(("role", role_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_role(upstream_client, role_name)
|
||||
|
||||
# Create role
|
||||
upstream_client.create_role(role_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return role_name in downstream_client.list_roles()
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create role for privilege {role_name}"
|
||||
)
|
||||
|
||||
# Grant privilege to role
|
||||
upstream_client.grant_privilege(
|
||||
role_name=role_name,
|
||||
object_type="Collection",
|
||||
privilege="Search",
|
||||
object_name="*",
|
||||
)
|
||||
|
||||
# check privilege in upstream
|
||||
upstream_privilege = upstream_client.describe_role(role_name)
|
||||
print(f"DEBUG: upstream_privilege in check_grant: {upstream_privilege}")
|
||||
|
||||
# Wait for privilege grant to sync
|
||||
def check_grant():
|
||||
time.sleep(2)
|
||||
# Verify privilege is actually granted using describe_role
|
||||
try:
|
||||
role_info = downstream_client.describe_role(role_name)
|
||||
print(f"DEBUG: role_info in check_grant: {role_info}")
|
||||
for privilege_info in role_info["privileges"]:
|
||||
print(f"DEBUG: privilege_info in check_grant: {privilege_info}")
|
||||
print(
|
||||
f"DEBUG: privilege_info.get('object_type') in check_grant: {privilege_info.get('object_type')}"
|
||||
)
|
||||
print(
|
||||
f"DEBUG: privilege_info.get('privilege') in check_grant: {privilege_info.get('privilege')}"
|
||||
)
|
||||
print(
|
||||
f"DEBUG: privilege_info.get('object_name') in check_grant: {privilege_info.get('object_name')}"
|
||||
)
|
||||
if (
|
||||
privilege_info.get("object_type") == "Collection"
|
||||
and privilege_info.get("privilege") == "Search"
|
||||
and privilege_info.get("object_name") == "*"
|
||||
):
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking role privileges: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_grant, sync_timeout, f"grant privilege to role {role_name}"
|
||||
)
|
||||
|
||||
def test_revoke_privilege(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test REVOKE_PRIVILEGE operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
role_name = self.gen_unique_name("test_role_priv_revoke")
|
||||
self.resources_to_cleanup.append(("role", role_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_role(upstream_client, role_name)
|
||||
|
||||
# Create role and grant privilege
|
||||
upstream_client.create_role(role_name)
|
||||
|
||||
upstream_client.grant_privilege(
|
||||
role_name=role_name,
|
||||
object_type="Collection",
|
||||
privilege="Search",
|
||||
object_name="*",
|
||||
)
|
||||
|
||||
# Wait for setup
|
||||
time.sleep(3)
|
||||
|
||||
# Revoke privilege from role
|
||||
upstream_client.revoke_privilege(
|
||||
role_name=role_name,
|
||||
object_type="Collection",
|
||||
privilege="Search",
|
||||
object_name="*",
|
||||
)
|
||||
|
||||
# Wait for revoke to sync
|
||||
def check_revoke():
|
||||
time.sleep(2)
|
||||
# Verify privilege is actually revoked using describe_role
|
||||
try:
|
||||
role_info = downstream_client.describe_role(role_name)
|
||||
for privilege_info in role_info["privileges"]:
|
||||
if (
|
||||
privilege_info.get("object_type") == "Collection"
|
||||
and privilege_info.get("privilege") == "Search"
|
||||
and privilege_info.get("object_name") == "*"
|
||||
):
|
||||
return False # Should not find the privilege
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking role privileges: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_revoke, sync_timeout, f"revoke privilege from role {role_name}"
|
||||
)
|
||||
|
||||
def test_update_password(
|
||||
self, upstream_client, downstream_client, sync_timeout, downstream_uri
|
||||
):
|
||||
"""Test UPDATE_PASSWORD operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
username = self.gen_unique_name("test_user_pwd", max_length=31)
|
||||
old_password = "OldPass123!"
|
||||
new_password = "NewPass456!"
|
||||
self.resources_to_cleanup.append(("user", username))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_user(upstream_client, username)
|
||||
|
||||
# Create user first
|
||||
upstream_client.create_user(username, old_password)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return username in downstream_client.list_users()
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create user {username}")
|
||||
|
||||
# Update password in upstream
|
||||
upstream_client.update_password(username, old_password, new_password)
|
||||
|
||||
# Wait for password update to sync
|
||||
def check_password_update():
|
||||
time.sleep(2)
|
||||
# Verify password update by trying to create a client with new credentials
|
||||
try:
|
||||
from pymilvus import MilvusClient
|
||||
|
||||
# Try to connect with new password
|
||||
test_client = MilvusClient(
|
||||
uri=downstream_uri, token=f"{username}:{new_password}"
|
||||
)
|
||||
# Simple operation to verify connection works
|
||||
test_client.list_collections()
|
||||
test_client.close()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Error verifying new password: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_password_update, sync_timeout, f"update password for user {username}"
|
||||
)
|
||||
|
||||
def test_create_privilege_group(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test CREATE_PRIVILEGE_GROUP operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
group_name = self.gen_unique_name("test_priv_group")
|
||||
# Note: privilege groups need special cleanup handling
|
||||
|
||||
# Initial cleanup - try to drop if exists
|
||||
try:
|
||||
upstream_client.drop_privilege_group(group_name)
|
||||
except:
|
||||
pass # Ignore if doesn't exist
|
||||
|
||||
# Create privilege group in upstream
|
||||
upstream_client.create_privilege_group(group_name)
|
||||
|
||||
# Verify in upstream
|
||||
def check_create():
|
||||
try:
|
||||
upstream_groups = upstream_client.list_privilege_groups()
|
||||
print(f"DEBUG: upstream_groups in check_create: {upstream_groups}")
|
||||
return group_name in [
|
||||
group["privilege_group"] for group in upstream_groups
|
||||
]
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create privilege group {group_name}"
|
||||
)
|
||||
|
||||
# Wait for sync to downstream
|
||||
def check_sync():
|
||||
try:
|
||||
downstream_groups = downstream_client.list_privilege_groups()
|
||||
return group_name in [
|
||||
group["privilege_group"] for group in downstream_groups
|
||||
]
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_sync, sync_timeout, f"create privilege group {group_name}"
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
try:
|
||||
upstream_client.drop_privilege_group(group_name)
|
||||
except:
|
||||
pass
|
||||
|
||||
def test_drop_privilege_group(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test DROP_PRIVILEGE_GROUP operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
group_name = self.gen_unique_name("test_priv_group_drop")
|
||||
|
||||
# Initial cleanup
|
||||
try:
|
||||
upstream_client.drop_privilege_group(group_name)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Create privilege group first
|
||||
upstream_client.create_privilege_group(group_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
try:
|
||||
return group_name in [
|
||||
group["privilege_group"]
|
||||
for group in downstream_client.list_privilege_groups()
|
||||
]
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create privilege group {group_name}"
|
||||
)
|
||||
|
||||
# Drop privilege group in upstream
|
||||
upstream_client.drop_privilege_group(group_name)
|
||||
|
||||
# Verify dropped in upstream
|
||||
upstream_groups = upstream_client.list_privilege_groups()
|
||||
assert group_name not in [group["privilege_group"] for group in upstream_groups]
|
||||
|
||||
# Wait for drop to sync
|
||||
def check_drop():
|
||||
try:
|
||||
downstream_groups = downstream_client.list_privilege_groups()
|
||||
return group_name not in [
|
||||
group["privilege_group"] for group in downstream_groups
|
||||
]
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_drop, sync_timeout, f"drop privilege group {group_name}"
|
||||
)
|
||||
|
||||
def test_grant_privilege_v2(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test GRANT_PRIVILEGE_V2 operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
role_name = self.gen_unique_name("test_role_priv_v2")
|
||||
self.resources_to_cleanup.append(("role", role_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_role(upstream_client, role_name)
|
||||
|
||||
# Create role
|
||||
upstream_client.create_role(role_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
return role_name in downstream_client.list_roles()
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create role for privilege v2 {role_name}"
|
||||
)
|
||||
|
||||
# Grant privilege using v2 API
|
||||
upstream_client.grant_privilege_v2(
|
||||
role_name=role_name, privilege="Query", collection_name="*"
|
||||
)
|
||||
# check privilege in upstream
|
||||
upstream_privilege = upstream_client.describe_role(role_name)["privileges"]
|
||||
print(f"DEBUG: upstream_privilege in check_grant: {upstream_privilege}")
|
||||
|
||||
# Wait for privilege grant to sync
|
||||
def check_grant():
|
||||
time.sleep(2)
|
||||
# Verify privilege is actually granted using describe_role
|
||||
try:
|
||||
role_info = downstream_client.describe_role(role_name)
|
||||
for privilege_info in role_info["privileges"]:
|
||||
if (
|
||||
privilege_info.get("privilege") == "Query"
|
||||
and privilege_info.get("object_name") == "*"
|
||||
):
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking role privileges v2: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_grant, sync_timeout, f"grant privilege v2 to role {role_name}"
|
||||
)
|
||||
|
||||
def test_revoke_privilege_v2(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test REVOKE_PRIVILEGE_V2 operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
role_name = self.gen_unique_name("test_role_revoke_v2")
|
||||
self.resources_to_cleanup.append(("role", role_name))
|
||||
|
||||
# Initial cleanup
|
||||
self.cleanup_role(upstream_client, role_name)
|
||||
|
||||
# Create role and grant privilege using v2 API
|
||||
upstream_client.create_role(role_name)
|
||||
upstream_client.grant_privilege_v2(
|
||||
role_name=role_name, privilege="Query", collection_name="*"
|
||||
)
|
||||
|
||||
# Wait for setup
|
||||
time.sleep(3)
|
||||
|
||||
# check privilege in upstream
|
||||
upstream_privilege = upstream_client.describe_role(role_name)["privileges"]
|
||||
print(f"DEBUG: upstream_privilege in check_revoke: {upstream_privilege}")
|
||||
|
||||
# check privilege in downstream
|
||||
def check_grant():
|
||||
try:
|
||||
role_info = downstream_client.describe_role(role_name)
|
||||
for privilege_info in role_info["privileges"]:
|
||||
if (
|
||||
privilege_info.get("privilege") == "Query"
|
||||
and privilege_info.get("object_name") == "*"
|
||||
):
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking role privileges v2: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_grant, sync_timeout, f"grant privilege v2 to role {role_name}"
|
||||
)
|
||||
|
||||
# Revoke privilege using v2 API
|
||||
upstream_client.revoke_privilege_v2(
|
||||
role_name=role_name, privilege="Query", collection_name="*"
|
||||
)
|
||||
# check privilege in upstream
|
||||
upstream_privilege = upstream_client.describe_role(role_name)["privileges"]
|
||||
print(f"DEBUG: upstream_privilege in check_revoke: {upstream_privilege}")
|
||||
|
||||
# Wait for revoke to sync
|
||||
def check_revoke():
|
||||
time.sleep(2)
|
||||
# Verify privilege is actually revoked using describe_role
|
||||
try:
|
||||
role_info = downstream_client.describe_role(role_name)
|
||||
for privilege_info in role_info["privileges"]:
|
||||
if (
|
||||
privilege_info.get("privilege") == "Query"
|
||||
and privilege_info.get("object_name") == "*"
|
||||
):
|
||||
return False # Should not find the privilege
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking role privileges v2: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_revoke, sync_timeout, f"revoke privilege v2 from role {role_name}"
|
||||
)
|
||||
|
||||
def test_add_privileges_to_group(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test ADD_PRIVILEGES_TO_GROUP operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
group_name = self.gen_unique_name("test_priv_group_add")
|
||||
|
||||
# Initial cleanup
|
||||
try:
|
||||
upstream_client.drop_privilege_group(group_name)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Create privilege group first
|
||||
upstream_client.create_privilege_group(group_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
try:
|
||||
downstream_groups = downstream_client.list_privilege_groups()
|
||||
print(f"DEBUG: downstream_groups in check_create: {downstream_groups}")
|
||||
return group_name in [
|
||||
group["privilege_group"] for group in downstream_groups
|
||||
]
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create privilege group {group_name}"
|
||||
)
|
||||
|
||||
# Add privileges to group in upstream
|
||||
privileges_to_add = ["Search", "Query"]
|
||||
upstream_client.add_privileges_to_group(group_name, privileges_to_add)
|
||||
|
||||
# Wait for add privileges to sync
|
||||
def check_upstream():
|
||||
client = upstream_client
|
||||
try:
|
||||
groups = client.list_privilege_groups()
|
||||
print(f"DEBUG: groups in check_add: {groups}")
|
||||
assert group_name in [group["privilege_group"] for group in groups]
|
||||
group_info = None
|
||||
for group in groups:
|
||||
if group["privilege_group"] == group_name:
|
||||
group_info = group
|
||||
break
|
||||
assert group_info is not None
|
||||
assert "Search" in group_info["privileges"]
|
||||
assert "Query" in group_info["privileges"]
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_upstream, sync_timeout, f"add privileges to group {group_name}"
|
||||
)
|
||||
|
||||
# Wait for add privileges to sync
|
||||
def check_downstream():
|
||||
client = upstream_client
|
||||
try:
|
||||
groups = client.list_privilege_groups()
|
||||
print(f"DEBUG: groups in check_add: {groups}")
|
||||
assert group_name in [group["privilege_group"] for group in groups]
|
||||
group_info = None
|
||||
for group in groups:
|
||||
if group["privilege_group"] == group_name:
|
||||
group_info = group
|
||||
break
|
||||
assert group_info is not None
|
||||
assert "Search" in group_info["privileges"]
|
||||
assert "Query" in group_info["privileges"]
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_downstream, sync_timeout, f"add privileges to group {group_name}"
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
try:
|
||||
upstream_client.drop_privilege_group(group_name)
|
||||
except:
|
||||
pass
|
||||
|
||||
def test_remove_privileges_from_group(
|
||||
self, upstream_client, downstream_client, sync_timeout
|
||||
):
|
||||
"""Test REMOVE_PRIVILEGES_FROM_GROUP operation sync."""
|
||||
# Store upstream client for teardown
|
||||
self._upstream_client = upstream_client
|
||||
|
||||
group_name = self.gen_unique_name("test_priv_group_add")
|
||||
|
||||
# Initial cleanup
|
||||
try:
|
||||
upstream_client.drop_privilege_group(group_name)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Create privilege group first
|
||||
upstream_client.create_privilege_group(group_name)
|
||||
|
||||
# Wait for creation to sync
|
||||
def check_create():
|
||||
try:
|
||||
downstream_groups = downstream_client.list_privilege_groups()
|
||||
print(f"DEBUG: downstream_groups in check_create: {downstream_groups}")
|
||||
return group_name in [
|
||||
group["privilege_group"] for group in downstream_groups
|
||||
]
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_create, sync_timeout, f"create privilege group {group_name}"
|
||||
)
|
||||
|
||||
# Add privileges to group in upstream
|
||||
privileges_to_add = ["Search", "Query"]
|
||||
upstream_client.add_privileges_to_group(group_name, privileges_to_add)
|
||||
|
||||
# Wait for add privileges to sync
|
||||
def check_upstream():
|
||||
client = upstream_client
|
||||
try:
|
||||
groups = client.list_privilege_groups()
|
||||
print(f"DEBUG: groups in check_add: {groups}")
|
||||
assert group_name in [group["privilege_group"] for group in groups]
|
||||
group_info = None
|
||||
for group in groups:
|
||||
if group["privilege_group"] == group_name:
|
||||
group_info = group
|
||||
break
|
||||
assert group_info is not None
|
||||
assert "Search" in group_info["privileges"]
|
||||
assert "Query" in group_info["privileges"]
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_upstream, sync_timeout, f"add privileges to group {group_name}"
|
||||
)
|
||||
|
||||
# Wait for add privileges to sync
|
||||
def check_downstream():
|
||||
client = upstream_client
|
||||
try:
|
||||
groups = client.list_privilege_groups()
|
||||
print(f"DEBUG: groups in check_add: {groups}")
|
||||
assert group_name in [group["privilege_group"] for group in groups]
|
||||
group_info = None
|
||||
for group in groups:
|
||||
if group["privilege_group"] == group_name:
|
||||
group_info = group
|
||||
break
|
||||
assert group_info is not None
|
||||
assert "Search" in group_info["privileges"]
|
||||
assert "Query" in group_info["privileges"]
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_downstream, sync_timeout, f"add privileges to group {group_name}"
|
||||
)
|
||||
|
||||
# remove privileges from group in upstream
|
||||
privileges_to_remove = ["Search", "Query"]
|
||||
upstream_client.remove_privileges_from_group(group_name, privileges_to_remove)
|
||||
|
||||
# Wait for remove privileges to sync
|
||||
def check_upstream():
|
||||
client = upstream_client
|
||||
try:
|
||||
groups = client.list_privilege_groups()
|
||||
print(f"DEBUG: groups in check_remove: {groups}")
|
||||
assert group_name in [group["privilege_group"] for group in groups]
|
||||
group_info = None
|
||||
for group in groups:
|
||||
if group["privilege_group"] == group_name:
|
||||
group_info = group
|
||||
break
|
||||
assert group_info is not None
|
||||
assert "Search" not in group_info["privileges"]
|
||||
assert "Query" not in group_info["privileges"]
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_upstream, sync_timeout, f"remove privileges from group {group_name}"
|
||||
)
|
||||
|
||||
# Wait for remove privileges to sync
|
||||
def check_downstream():
|
||||
client = downstream_client
|
||||
try:
|
||||
groups = client.list_privilege_groups()
|
||||
print(f"DEBUG: groups in check_remove: {groups}")
|
||||
assert group_name in [group["privilege_group"] for group in groups]
|
||||
group_info = None
|
||||
for group in groups:
|
||||
if group["privilege_group"] == group_name:
|
||||
group_info = group
|
||||
break
|
||||
assert group_info is not None
|
||||
assert "Search" not in group_info["privileges"]
|
||||
assert "Query" not in group_info["privileges"]
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error checking privilege groups: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_downstream, sync_timeout, f"remove privileges from group {group_name}"
|
||||
)
|
||||
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
CDC non-replication tests for resource group operations.
|
||||
|
||||
Resource groups and replica assignment are per-cluster state; by design
|
||||
CDC does NOT propagate RG create/drop/update/transfer-replica to the
|
||||
downstream cluster. These tests guard that invariant.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
class TestCDCSyncResourceGroup(TestCDCSyncBase):
|
||||
"""Verify that resource group operations are NOT replicated by CDC."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup both upstream and downstream (downstream won't auto-sync)."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
downstream_client = getattr(self, "_downstream_client", None)
|
||||
|
||||
for client in (upstream_client, downstream_client):
|
||||
if not client:
|
||||
continue
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "resource_group":
|
||||
self._cleanup_resource_group(client, resource_name)
|
||||
elif resource_type == "collection":
|
||||
self.cleanup_collection(client, resource_name)
|
||||
|
||||
def _cleanup_resource_group(self, client, rg_name):
|
||||
"""Clean up resource group if exists (skipping default RG)."""
|
||||
try:
|
||||
existing = client.list_resource_groups()
|
||||
if rg_name in existing:
|
||||
logger.info(f"[CLEANUP] Cleaning up resource group: {rg_name}")
|
||||
client.drop_resource_group(rg_name)
|
||||
logger.info(f"[SUCCESS] Resource group {rg_name} cleaned up successfully")
|
||||
else:
|
||||
logger.debug(f"Resource group {rg_name} does not exist, skipping cleanup")
|
||||
except Exception as e:
|
||||
logger.warning(f"[FAILED] Failed to cleanup resource group {rg_name}: {e}")
|
||||
|
||||
def test_create_resource_group_not_replicated(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Creating an RG on upstream must NOT create it on downstream."""
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
|
||||
rg_name = self.gen_unique_name("test_rg_create")
|
||||
self.resources_to_cleanup.append(("resource_group", rg_name))
|
||||
|
||||
self._cleanup_resource_group(upstream_client, rg_name)
|
||||
self._cleanup_resource_group(downstream_client, rg_name)
|
||||
|
||||
upstream_client.create_resource_group(rg_name)
|
||||
assert rg_name in upstream_client.list_resource_groups(), f"Resource group {rg_name} not created in upstream"
|
||||
|
||||
# Wait the full sync window; if CDC were going to leak the RG it would
|
||||
# have shown up by now.
|
||||
time.sleep(sync_timeout)
|
||||
downstream_rgs = downstream_client.list_resource_groups()
|
||||
assert rg_name not in downstream_rgs, (
|
||||
f"Resource group {rg_name} unexpectedly appeared on downstream (RG ops must not replicate)"
|
||||
)
|
||||
|
||||
def test_drop_resource_group_not_replicated(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Dropping an RG on upstream must NOT drop it on downstream."""
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
|
||||
rg_name = self.gen_unique_name("test_rg_drop")
|
||||
self.resources_to_cleanup.append(("resource_group", rg_name))
|
||||
|
||||
self._cleanup_resource_group(upstream_client, rg_name)
|
||||
self._cleanup_resource_group(downstream_client, rg_name)
|
||||
|
||||
# Create the RG on BOTH sides (independently, since create isn't replicated either).
|
||||
upstream_client.create_resource_group(rg_name)
|
||||
downstream_client.create_resource_group(rg_name)
|
||||
assert rg_name in upstream_client.list_resource_groups()
|
||||
assert rg_name in downstream_client.list_resource_groups()
|
||||
|
||||
# Drop only on upstream; downstream must retain its own copy.
|
||||
upstream_client.drop_resource_group(rg_name)
|
||||
assert rg_name not in upstream_client.list_resource_groups(), (
|
||||
f"Resource group {rg_name} still exists in upstream after drop"
|
||||
)
|
||||
|
||||
time.sleep(sync_timeout)
|
||||
downstream_rgs = downstream_client.list_resource_groups()
|
||||
assert rg_name in downstream_rgs, (
|
||||
f"Resource group {rg_name} was dropped on downstream after upstream drop "
|
||||
f"(RG ops must not replicate). downstream RGs: {downstream_rgs}"
|
||||
)
|
||||
|
||||
def test_update_resource_group_not_replicated(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Updating RG config on upstream must NOT reconfigure downstream's RG."""
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
|
||||
rg_name = self.gen_unique_name("test_rg_update")
|
||||
self.resources_to_cleanup.append(("resource_group", rg_name))
|
||||
|
||||
self._cleanup_resource_group(upstream_client, rg_name)
|
||||
self._cleanup_resource_group(downstream_client, rg_name)
|
||||
|
||||
upstream_config = {"requests": {"node_num": 0}, "limits": {"node_num": 2}}
|
||||
downstream_config = {"requests": {"node_num": 0}, "limits": {"node_num": 1}}
|
||||
|
||||
# Create the RG on both sides with different configs.
|
||||
upstream_client.create_resource_group(rg_name, config=upstream_config)
|
||||
downstream_client.create_resource_group(rg_name, config=downstream_config)
|
||||
downstream_desc_before = downstream_client.describe_resource_group(rg_name)
|
||||
logger.info(f"Downstream RG before upstream update: {downstream_desc_before}")
|
||||
|
||||
# Upstream has already been created with its config; nothing else to
|
||||
# update since config drift itself would be the leak. Wait and confirm
|
||||
# downstream config is unchanged. ResourceGroupInfo has no __eq__, so
|
||||
# compare the str() form (includes config/limits/requests/nodes).
|
||||
time.sleep(sync_timeout)
|
||||
downstream_desc_after = downstream_client.describe_resource_group(rg_name)
|
||||
logger.info(f"Downstream RG after upstream update: {downstream_desc_after}")
|
||||
assert str(downstream_desc_after) == str(downstream_desc_before), (
|
||||
f"Downstream RG {rg_name} was modified by upstream config (RG ops must not replicate). "
|
||||
f"before={downstream_desc_before}, after={downstream_desc_after}"
|
||||
)
|
||||
|
||||
def test_transfer_replica_not_replicated(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Creating multiple RGs on upstream must NOT create any of them on downstream."""
|
||||
self._upstream_client = upstream_client
|
||||
self._downstream_client = downstream_client
|
||||
|
||||
rg_name_1 = self.gen_unique_name("test_rg_transfer_1")
|
||||
rg_name_2 = self.gen_unique_name("test_rg_transfer_2")
|
||||
self.resources_to_cleanup.append(("resource_group", rg_name_1))
|
||||
self.resources_to_cleanup.append(("resource_group", rg_name_2))
|
||||
|
||||
self._cleanup_resource_group(upstream_client, rg_name_1)
|
||||
self._cleanup_resource_group(upstream_client, rg_name_2)
|
||||
self._cleanup_resource_group(downstream_client, rg_name_1)
|
||||
self._cleanup_resource_group(downstream_client, rg_name_2)
|
||||
|
||||
upstream_client.create_resource_group(rg_name_1)
|
||||
upstream_client.create_resource_group(rg_name_2)
|
||||
upstream_rgs = upstream_client.list_resource_groups()
|
||||
assert rg_name_1 in upstream_rgs
|
||||
assert rg_name_2 in upstream_rgs
|
||||
|
||||
time.sleep(sync_timeout)
|
||||
downstream_rgs = downstream_client.list_resource_groups()
|
||||
assert rg_name_1 not in downstream_rgs and rg_name_2 not in downstream_rgs, (
|
||||
f"Upstream RGs leaked to downstream (RG ops must not replicate). downstream RGs: {downstream_rgs}"
|
||||
)
|
||||
@@ -0,0 +1,690 @@
|
||||
"""
|
||||
CDC sync tests for advanced schema features (dynamic fields, nullable, default values,
|
||||
partition keys, clustering keys, and combinations thereof).
|
||||
"""
|
||||
|
||||
import random
|
||||
import time
|
||||
|
||||
from pymilvus import DataType
|
||||
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
class TestCDCSyncSchemaFeatures(TestCDCSyncBase):
|
||||
"""Test CDC sync for advanced schema features."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
def test_dynamic_schema_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test that dynamic schema fields are correctly replicated via CDC."""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_dynamic_schema", max_length=50)
|
||||
|
||||
self.log_test_start("test_dynamic_schema_sync", "DYNAMIC_SCHEMA", collection_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create dynamic schema collection
|
||||
self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_dynamic_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Create HNSW index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="float_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for collection creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}")
|
||||
|
||||
# Insert 200 rows with extra dynamic fields
|
||||
extra_fields = {"extra_int": int, "extra_str": str, "extra_float": float}
|
||||
test_data = self.generate_dynamic_data(200, extra_fields=extra_fields)
|
||||
self.log_data_operation("INSERT", collection_name, len(test_data), "- dynamic schema data")
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Query extra_int > 0 on upstream
|
||||
self.log_sync_verification("DYNAMIC_SCHEMA", collection_name, "extra_int > 0 count matches downstream")
|
||||
upstream_result = upstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="extra_int > 0",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
upstream_count = upstream_result[0]["count(*)"] if upstream_result else 0
|
||||
logger.info(f"[UPSTREAM] extra_int > 0 count: {upstream_count}")
|
||||
|
||||
# Wait for sync and verify downstream count matches
|
||||
def check_data():
|
||||
try:
|
||||
down_result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="extra_int > 0",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
down_count = down_result[0]["count(*)"] if down_result else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream extra_int > 0 count: {down_count}/{upstream_count}")
|
||||
return down_count == upstream_count
|
||||
except Exception as e:
|
||||
logger.warning(f"Dynamic schema sync check failed: {e}")
|
||||
return False
|
||||
|
||||
sync_success = self.wait_for_sync(check_data, sync_timeout, f"dynamic schema data sync {collection_name}")
|
||||
assert sync_success, f"Dynamic schema data failed to sync to downstream for {collection_name}"
|
||||
|
||||
# Verify data sampling for extra fields
|
||||
match, mismatch, details = self.verify_data_sampling(
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
collection_name,
|
||||
sample_ratio=0.1,
|
||||
output_fields=["id", "varchar_field", "extra_int", "extra_str", "extra_float"],
|
||||
)
|
||||
logger.info(f"[VERIFY] Dynamic schema sampling: match={match}, mismatch={mismatch}")
|
||||
assert mismatch == 0, f"Dynamic field data mismatch detected: {details}"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_dynamic_schema_sync", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] test_dynamic_schema_sync failed: {e}")
|
||||
self.log_test_end("test_dynamic_schema_sync", False, duration)
|
||||
raise
|
||||
|
||||
def test_nullable_fields_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test that nullable field values (including NULLs) are correctly replicated via CDC."""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_nullable_flds", max_length=50)
|
||||
|
||||
self.log_test_start("test_nullable_fields_sync", "NULLABLE_FIELDS", collection_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create nullable schema collection
|
||||
self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_nullable_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Create index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="float_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for collection creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}")
|
||||
|
||||
# Insert 200 rows with null_ratio=0.3
|
||||
test_data = self.generate_nullable_data(200, null_ratio=0.3)
|
||||
self.log_data_operation("INSERT", collection_name, len(test_data), "- nullable data null_ratio=0.3")
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Count nulls and not-nulls for nullable_int64 on upstream
|
||||
null_result = upstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="nullable_int64 is null",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
not_null_result = upstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="nullable_int64 is not null",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
upstream_null_count = null_result[0]["count(*)"] if null_result else 0
|
||||
upstream_not_null_count = not_null_result[0]["count(*)"] if not_null_result else 0
|
||||
logger.info(f"[UPSTREAM] nullable_int64 null={upstream_null_count}, not_null={upstream_not_null_count}")
|
||||
|
||||
self.log_sync_verification("NULLABLE_FIELDS", collection_name, "null counts match downstream")
|
||||
|
||||
# Wait for sync and verify null/not-null counts match on downstream
|
||||
def check_null_counts():
|
||||
try:
|
||||
d_null = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="nullable_int64 is null",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
d_not_null = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="nullable_int64 is not null",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
d_null_count = d_null[0]["count(*)"] if d_null else 0
|
||||
d_not_null_count = d_not_null[0]["count(*)"] if d_not_null else 0
|
||||
logger.info(
|
||||
f"[SYNC_PROGRESS] downstream nullable_int64 null={d_null_count}/{upstream_null_count}, "
|
||||
f"not_null={d_not_null_count}/{upstream_not_null_count}"
|
||||
)
|
||||
return d_null_count == upstream_null_count and d_not_null_count == upstream_not_null_count
|
||||
except Exception as e:
|
||||
logger.warning(f"Nullable sync check failed: {e}")
|
||||
return False
|
||||
|
||||
sync_success = self.wait_for_sync(
|
||||
check_null_counts, sync_timeout, f"nullable fields sync {collection_name}"
|
||||
)
|
||||
assert sync_success, f"Nullable field counts failed to sync to downstream for {collection_name}"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_nullable_fields_sync", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] test_nullable_fields_sync failed: {e}")
|
||||
self.log_test_end("test_nullable_fields_sync", False, duration)
|
||||
raise
|
||||
|
||||
def test_default_values_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test that default field values are applied and replicated correctly via CDC."""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_default_vals", max_length=50)
|
||||
|
||||
self.log_test_start("test_default_values_sync", "DEFAULT_VALUES", collection_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create default values schema collection
|
||||
self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_default_values_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Create index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="float_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for collection creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}")
|
||||
|
||||
# Insert 100 rows providing ONLY float_vector — default fields are omitted
|
||||
test_data = [{"float_vector": [random.random() for _ in range(128)]} for _ in range(100)]
|
||||
self.log_data_operation("INSERT", collection_name, len(test_data), "- only float_vector provided")
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Query default_varchar == "default" on upstream — should be 100
|
||||
upstream_result = upstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter='default_varchar == "default"',
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
upstream_count = upstream_result[0]["count(*)"] if upstream_result else 0
|
||||
logger.info(f'[UPSTREAM] default_varchar == "default" count: {upstream_count}')
|
||||
assert upstream_count == 100, f"Expected 100 rows with default varchar on upstream, got {upstream_count}"
|
||||
|
||||
self.log_sync_verification(
|
||||
"DEFAULT_VALUES", collection_name, 'default_varchar == "default" count=100 on downstream'
|
||||
)
|
||||
|
||||
# Wait for sync and verify same count on downstream
|
||||
def check_defaults():
|
||||
try:
|
||||
down_result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter='default_varchar == "default"',
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
down_count = down_result[0]["count(*)"] if down_result else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream default_varchar count: {down_count}/100")
|
||||
return down_count == 100
|
||||
except Exception as e:
|
||||
logger.warning(f"Default values sync check failed: {e}")
|
||||
return False
|
||||
|
||||
sync_success = self.wait_for_sync(check_defaults, sync_timeout, f"default values sync {collection_name}")
|
||||
assert sync_success, f"Default value data failed to sync to downstream for {collection_name}"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_default_values_sync", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] test_default_values_sync failed: {e}")
|
||||
self.log_test_end("test_default_values_sync", False, duration)
|
||||
raise
|
||||
|
||||
def test_partition_key_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test that partition key schema and data are correctly replicated via CDC."""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_part_key", max_length=50)
|
||||
|
||||
self.log_test_start("test_partition_key_sync", "PARTITION_KEY", collection_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create partition key schema (VarChar key)
|
||||
self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_partition_key_schema(upstream_client, key_type="VarChar"),
|
||||
)
|
||||
|
||||
# Create index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="float_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for collection creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}")
|
||||
|
||||
# Insert 500 rows with random categories as partition key values
|
||||
categories = ["cat_A", "cat_B", "cat_C", "cat_D", "cat_E"]
|
||||
test_data = [
|
||||
{
|
||||
"float_vector": [random.random() for _ in range(128)],
|
||||
"partition_key_field": random.choice(categories),
|
||||
"data_field": f"data_{i}_{random.randint(1000, 9999)}",
|
||||
}
|
||||
for i in range(500)
|
||||
]
|
||||
self.log_data_operation("INSERT", collection_name, len(test_data), "- partition key data")
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
self.log_sync_verification(
|
||||
"PARTITION_KEY", collection_name, "count=500 and partition_key field on downstream"
|
||||
)
|
||||
|
||||
# Wait for sync and verify count=500 on downstream
|
||||
def check_data():
|
||||
try:
|
||||
result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
count = result[0]["count(*)"] if result else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {count}/500")
|
||||
return count >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Partition key sync check failed: {e}")
|
||||
return False
|
||||
|
||||
sync_success = self.wait_for_sync(check_data, sync_timeout, f"partition key data sync {collection_name}")
|
||||
assert sync_success, f"Partition key data failed to sync to downstream for {collection_name}"
|
||||
|
||||
# Verify partition_key_field is present in downstream describe_collection
|
||||
downstream_info = downstream_client.describe_collection(collection_name)
|
||||
downstream_fields = [f["name"] for f in downstream_info.get("fields", [])]
|
||||
logger.info(f"[VERIFY] Downstream fields: {downstream_fields}")
|
||||
assert "partition_key_field" in downstream_fields, (
|
||||
f"partition_key_field not found in downstream collection schema: {downstream_fields}"
|
||||
)
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_partition_key_sync", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] test_partition_key_sync failed: {e}")
|
||||
self.log_test_end("test_partition_key_sync", False, duration)
|
||||
raise
|
||||
|
||||
def test_clustering_key_sync(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test that clustering key schema and data are correctly replicated via CDC."""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_cluster_key", max_length=50)
|
||||
|
||||
self.log_test_start("test_clustering_key_sync", "CLUSTERING_KEY", collection_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Create clustering key schema
|
||||
self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=self.create_clustering_key_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Create index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="float_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for collection creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}")
|
||||
|
||||
# Insert 300 rows
|
||||
test_data = [
|
||||
{
|
||||
"float_vector": [random.random() for _ in range(128)],
|
||||
"clustering_key_field": random.randint(0, 1000),
|
||||
"data_field": f"data_{i}_{random.randint(1000, 9999)}",
|
||||
}
|
||||
for i in range(300)
|
||||
]
|
||||
self.log_data_operation("INSERT", collection_name, len(test_data), "- clustering key data")
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
self.log_sync_verification(
|
||||
"CLUSTERING_KEY", collection_name, "count=300 and clustering_key field on downstream"
|
||||
)
|
||||
|
||||
# Wait for sync and verify count=300 on downstream
|
||||
def check_data():
|
||||
try:
|
||||
result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
count = result[0]["count(*)"] if result else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {count}/300")
|
||||
return count >= 300
|
||||
except Exception as e:
|
||||
logger.warning(f"Clustering key sync check failed: {e}")
|
||||
return False
|
||||
|
||||
sync_success = self.wait_for_sync(check_data, sync_timeout, f"clustering key data sync {collection_name}")
|
||||
assert sync_success, f"Clustering key data failed to sync to downstream for {collection_name}"
|
||||
|
||||
# Verify clustering_key_field is present in downstream describe_collection
|
||||
downstream_info = downstream_client.describe_collection(collection_name)
|
||||
downstream_fields = [f["name"] for f in downstream_info.get("fields", [])]
|
||||
logger.info(f"[VERIFY] Downstream fields: {downstream_fields}")
|
||||
assert "clustering_key_field" in downstream_fields, (
|
||||
f"clustering_key_field not found in downstream collection schema: {downstream_fields}"
|
||||
)
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_clustering_key_sync", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] test_clustering_key_sync failed: {e}")
|
||||
self.log_test_end("test_clustering_key_sync", False, duration)
|
||||
raise
|
||||
|
||||
def test_nullable_with_defaults(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test that nullable fields combined with default values sync correctly via CDC.
|
||||
|
||||
Inserts 100 rows in three patterns:
|
||||
i%3==0: explicit values provided for both fields
|
||||
i%3==1: None values (explicit null) provided for both fields
|
||||
i%3==2: fields omitted entirely (uses default / null)
|
||||
"""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_null_default", max_length=50)
|
||||
|
||||
self.log_test_start("test_nullable_with_defaults", "NULLABLE_WITH_DEFAULTS", collection_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Build custom schema: nullable_with_default (INT64, nullable, default=42)
|
||||
# nullable_no_default (VARCHAR, nullable)
|
||||
schema = upstream_client.create_schema()
|
||||
schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True)
|
||||
schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128)
|
||||
schema.add_field(
|
||||
"nullable_with_default",
|
||||
DataType.INT64,
|
||||
nullable=True,
|
||||
default_value=42,
|
||||
)
|
||||
schema.add_field(
|
||||
"nullable_no_default",
|
||||
DataType.VARCHAR,
|
||||
max_length=256,
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
# Create index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="float_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for collection creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}")
|
||||
|
||||
# Insert 100 rows across 3 patterns
|
||||
test_data = []
|
||||
for i in range(100):
|
||||
record = {"float_vector": [random.random() for _ in range(128)]}
|
||||
if i % 3 == 0:
|
||||
# Explicit values
|
||||
record["nullable_with_default"] = random.randint(100, 999)
|
||||
record["nullable_no_default"] = f"explicit_{i}"
|
||||
elif i % 3 == 1:
|
||||
# Explicit None (null)
|
||||
record["nullable_with_default"] = None
|
||||
record["nullable_no_default"] = None
|
||||
# i%3==2: omit both fields — server applies default/null
|
||||
test_data.append(record)
|
||||
|
||||
self.log_data_operation("INSERT", collection_name, len(test_data), "- nullable+default mixed pattern")
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
self.log_sync_verification("NULLABLE_WITH_DEFAULTS", collection_name, "total count=100 on downstream")
|
||||
|
||||
# Wait for sync and verify total count on downstream
|
||||
def check_data():
|
||||
try:
|
||||
result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
count = result[0]["count(*)"] if result else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {count}/100")
|
||||
return count >= 100
|
||||
except Exception as e:
|
||||
logger.warning(f"Nullable+defaults sync check failed: {e}")
|
||||
return False
|
||||
|
||||
sync_success = self.wait_for_sync(check_data, sync_timeout, f"nullable+defaults sync {collection_name}")
|
||||
assert sync_success, f"Nullable-with-defaults data failed to sync to downstream for {collection_name}"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_nullable_with_defaults", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] test_nullable_with_defaults failed: {e}")
|
||||
self.log_test_end("test_nullable_with_defaults", False, duration)
|
||||
raise
|
||||
|
||||
def test_dynamic_with_partition_key(self, upstream_client, downstream_client, sync_timeout):
|
||||
"""Test that dynamic fields combined with a partition key schema sync correctly via CDC."""
|
||||
start_time = time.time()
|
||||
collection_name = self.gen_unique_name("test_dyn_part_key", max_length=50)
|
||||
|
||||
self.log_test_start("test_dynamic_with_partition_key", "DYNAMIC_WITH_PARTITION_KEY", collection_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", collection_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, collection_name)
|
||||
|
||||
# Build schema: enable_dynamic_field=True + VARCHAR partition key
|
||||
schema = upstream_client.create_schema(enable_dynamic_field=True)
|
||||
schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True)
|
||||
schema.add_field("float_vector", DataType.FLOAT_VECTOR, dim=128)
|
||||
schema.add_field(
|
||||
"pk_field",
|
||||
DataType.VARCHAR,
|
||||
max_length=64,
|
||||
is_partition_key=True,
|
||||
)
|
||||
|
||||
self.log_operation("CREATE_COLLECTION", "collection", collection_name, "upstream")
|
||||
upstream_client.create_collection(
|
||||
collection_name=collection_name,
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
# Create index and load
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="float_vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 8, "efConstruction": 64},
|
||||
)
|
||||
upstream_client.create_index(collection_name, index_params)
|
||||
upstream_client.load_collection(collection_name)
|
||||
|
||||
# Wait for collection creation to sync
|
||||
def check_create():
|
||||
return downstream_client.has_collection(collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {collection_name}")
|
||||
|
||||
# Insert 300 rows with dynamic fields + partition key values
|
||||
partitions = ["region_A", "region_B", "region_C", "region_D"]
|
||||
test_data = [
|
||||
{
|
||||
"float_vector": [random.random() for _ in range(128)],
|
||||
"pk_field": random.choice(partitions),
|
||||
# dynamic fields
|
||||
"dynamic_num": random.randint(1, 10000),
|
||||
"dynamic_tag": f"tag_{i % 10}",
|
||||
"dynamic_score": random.uniform(0.0, 100.0),
|
||||
}
|
||||
for i in range(300)
|
||||
]
|
||||
self.log_data_operation("INSERT", collection_name, len(test_data), "- dynamic+partition_key data")
|
||||
upstream_client.insert(collection_name, test_data)
|
||||
upstream_client.flush(collection_name)
|
||||
|
||||
# Query dynamic_num > 0 on upstream
|
||||
upstream_result = upstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="dynamic_num > 0",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
upstream_count = upstream_result[0]["count(*)"] if upstream_result else 0
|
||||
logger.info(f"[UPSTREAM] dynamic_num > 0 count: {upstream_count}")
|
||||
|
||||
self.log_sync_verification(
|
||||
"DYNAMIC_WITH_PARTITION_KEY",
|
||||
collection_name,
|
||||
f"dynamic_num > 0 count={upstream_count} on downstream",
|
||||
)
|
||||
|
||||
# Wait for sync and verify the count matches on downstream
|
||||
def check_data():
|
||||
try:
|
||||
down_result = downstream_client.query(
|
||||
collection_name=collection_name,
|
||||
filter="dynamic_num > 0",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
down_count = down_result[0]["count(*)"] if down_result else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream dynamic_num > 0 count: {down_count}/{upstream_count}")
|
||||
return down_count == upstream_count
|
||||
except Exception as e:
|
||||
logger.warning(f"Dynamic+partition_key sync check failed: {e}")
|
||||
return False
|
||||
|
||||
sync_success = self.wait_for_sync(check_data, sync_timeout, f"dynamic+partition_key sync {collection_name}")
|
||||
assert sync_success, f"Dynamic+partition_key data failed to sync to downstream for {collection_name}"
|
||||
|
||||
duration = time.time() - start_time
|
||||
self.log_test_end("test_dynamic_with_partition_key", True, duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(f"[ERROR] test_dynamic_with_partition_key failed: {e}")
|
||||
self.log_test_end("test_dynamic_with_partition_key", False, duration)
|
||||
raise
|
||||
@@ -0,0 +1,570 @@
|
||||
"""
|
||||
CDC sync tests for search and query result verification across vector types.
|
||||
"""
|
||||
|
||||
import random
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from pymilvus import AnnSearchRequest, DataType, RRFRanker
|
||||
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
# fmt: off
|
||||
VECTOR_PARAMS = [
|
||||
("FLOAT_VECTOR", "HNSW", "COSINE", 128),
|
||||
("FLOAT_VECTOR", "IVF_FLAT", "L2", 128),
|
||||
("FLOAT16_VECTOR", "HNSW", "L2", 64),
|
||||
("BFLOAT16_VECTOR", "HNSW", "L2", 64),
|
||||
("INT8_VECTOR", "HNSW", "COSINE", 64),
|
||||
("BINARY_VECTOR", "BIN_FLAT", "HAMMING", 128),
|
||||
("SPARSE_FLOAT_VECTOR", "SPARSE_INVERTED_INDEX", "IP", 0),
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
|
||||
class TestCDCSyncSearchVerification(TestCDCSyncBase):
|
||||
"""Test CDC sync for search and query result verification across vector types."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Internal helper
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def _setup_collection(self, client, c_name, vector_type, index_type, metric, dim):
|
||||
"""
|
||||
Create a single-vector-schema collection, insert 500 records,
|
||||
create an index, and load.
|
||||
|
||||
Returns the collection name (same as c_name).
|
||||
"""
|
||||
schema = self.create_single_vector_schema(client, vector_type=vector_type, dim=dim)
|
||||
client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
# Insert 500 records
|
||||
data = self.generate_single_vector_data(500, vector_type=vector_type, dim=dim)
|
||||
client.insert(c_name, data)
|
||||
client.flush(c_name)
|
||||
|
||||
# Build index
|
||||
index_params = client.prepare_index_params()
|
||||
if index_type == "IVF_FLAT":
|
||||
idx_params = {"nlist": 64}
|
||||
elif index_type == "HNSW":
|
||||
idx_params = {"M": 16, "efConstruction": 200}
|
||||
else:
|
||||
idx_params = {}
|
||||
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type=index_type,
|
||||
metric_type=metric,
|
||||
params=idx_params,
|
||||
)
|
||||
client.create_index(c_name, index_params)
|
||||
client.load_collection(c_name)
|
||||
|
||||
return c_name
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Tests
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,index_type,metric,dim",
|
||||
VECTOR_PARAMS,
|
||||
ids=[p[0] + "_" + p[1] for p in VECTOR_PARAMS],
|
||||
)
|
||||
def test_search_result_consistency(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
vector_type,
|
||||
index_type,
|
||||
metric,
|
||||
dim,
|
||||
):
|
||||
"""Verify that ANN search results are consistent between upstream and downstream."""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name(f"test_src_{vector_type[:4].lower()}", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_search_result_consistency",
|
||||
f"SEARCH/{vector_type}/{index_type}",
|
||||
c_name,
|
||||
)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
self._setup_collection(upstream_client, c_name, vector_type, index_type, metric, dim)
|
||||
|
||||
# Wait for at least 500 records to appear on downstream
|
||||
def check_sync():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500")
|
||||
return cnt >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), (
|
||||
f"Downstream did not receive 500 records within {sync_timeout}s"
|
||||
)
|
||||
|
||||
# Build 5 random query vectors
|
||||
dtype = getattr(DataType, vector_type)
|
||||
query_vectors = self._gen_vectors(5, dim if dim > 0 else 1000, dtype)
|
||||
|
||||
avg_overlap, _, _ = self.verify_search_consistency(
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
c_name,
|
||||
query_vectors,
|
||||
anns_field="vector",
|
||||
limit=10,
|
||||
metric_type=metric,
|
||||
)
|
||||
|
||||
assert avg_overlap >= self.SEARCH_OVERLAP_THRESHOLD, (
|
||||
f"Search overlap {avg_overlap:.4f} is below threshold "
|
||||
f"{self.SEARCH_OVERLAP_THRESHOLD} for {vector_type}/{index_type}"
|
||||
)
|
||||
|
||||
finally:
|
||||
self.log_test_end(
|
||||
"test_search_result_consistency",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,index_type,metric,dim",
|
||||
VECTOR_PARAMS,
|
||||
ids=[p[0] + "_" + p[1] for p in VECTOR_PARAMS],
|
||||
)
|
||||
def test_query_data_sampling(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
vector_type,
|
||||
index_type,
|
||||
metric,
|
||||
dim,
|
||||
):
|
||||
"""Verify scalar field values are identical on both sides via random sampling."""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name(f"test_qds_{vector_type[:4].lower()}", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_query_data_sampling",
|
||||
f"QUERY_SAMPLE/{vector_type}/{index_type}",
|
||||
c_name,
|
||||
)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
self._setup_collection(upstream_client, c_name, vector_type, index_type, metric, dim)
|
||||
|
||||
def check_sync():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500")
|
||||
return cnt >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), (
|
||||
f"Downstream did not receive 500 records within {sync_timeout}s"
|
||||
)
|
||||
|
||||
output_fields = ["id", "int_field", "varchar_field", "float_field"]
|
||||
match_count, mismatch_count, mismatch_details = self.verify_data_sampling(
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
c_name,
|
||||
sample_ratio=0.2,
|
||||
output_fields=output_fields,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"[RESULT] Sampling — match={match_count}, mismatch={mismatch_count}, details={mismatch_details[:3]}"
|
||||
)
|
||||
assert mismatch_count == 0, f"Found {mismatch_count} mismatched records: {mismatch_details[:5]}"
|
||||
|
||||
finally:
|
||||
self.log_test_end(
|
||||
"test_query_data_sampling",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def test_hybrid_search_consistency(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
):
|
||||
"""Verify hybrid search (dense + sparse, RRF ranker) results are consistent."""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_hybrid_srch", max_length=50)
|
||||
|
||||
self.log_test_start("test_hybrid_search_consistency", "HYBRID_SEARCH", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
# Build schema: dense FloatVector(128) + sparse
|
||||
schema = upstream_client.create_schema(enable_dynamic_field=True)
|
||||
schema.add_field("id", DataType.INT64, is_primary=True, auto_id=True)
|
||||
schema.add_field("dense", DataType.FLOAT_VECTOR, dim=128)
|
||||
schema.add_field("sparse", DataType.SPARSE_FLOAT_VECTOR)
|
||||
schema.add_field("int_field", DataType.INT64)
|
||||
schema.add_field("varchar_field", DataType.VARCHAR, max_length=256)
|
||||
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
# Insert 300 records
|
||||
dense_vecs = self._gen_vectors(300, 128, DataType.FLOAT_VECTOR)
|
||||
sparse_vecs = self._gen_vectors(300, 1000, DataType.SPARSE_FLOAT_VECTOR)
|
||||
data = [
|
||||
{
|
||||
"dense": dense_vecs[i],
|
||||
"sparse": sparse_vecs[i],
|
||||
"int_field": random.randint(0, 1000),
|
||||
"varchar_field": f"hybrid_{i}_{random.randint(1000, 9999)}",
|
||||
}
|
||||
for i in range(300)
|
||||
]
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
# Create indexes
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="dense",
|
||||
index_type="HNSW",
|
||||
metric_type="COSINE",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
index_params.add_index(
|
||||
field_name="sparse",
|
||||
index_type="SPARSE_INVERTED_INDEX",
|
||||
metric_type="IP",
|
||||
params={},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
|
||||
# Wait for downstream sync
|
||||
def check_sync():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/300")
|
||||
return cnt >= 300
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_sync, sync_timeout, f"hybrid data sync {c_name}"), (
|
||||
f"Downstream did not receive 300 records within {sync_timeout}s"
|
||||
)
|
||||
|
||||
# Build hybrid search requests
|
||||
q_dense = self._gen_vectors(1, 128, DataType.FLOAT_VECTOR)[0]
|
||||
q_sparse = self._gen_vectors(1, 1000, DataType.SPARSE_FLOAT_VECTOR)[0]
|
||||
|
||||
dense_req = AnnSearchRequest(
|
||||
data=[q_dense],
|
||||
anns_field="dense",
|
||||
param={"metric_type": "COSINE", "params": {"ef": 64}},
|
||||
limit=10,
|
||||
)
|
||||
sparse_req = AnnSearchRequest(
|
||||
data=[q_sparse],
|
||||
anns_field="sparse",
|
||||
param={"metric_type": "IP"},
|
||||
limit=10,
|
||||
)
|
||||
|
||||
up_results = upstream_client.hybrid_search(
|
||||
collection_name=c_name,
|
||||
reqs=[dense_req, sparse_req],
|
||||
ranker=RRFRanker(),
|
||||
limit=10,
|
||||
output_fields=["id"],
|
||||
)
|
||||
down_results = downstream_client.hybrid_search(
|
||||
collection_name=c_name,
|
||||
reqs=[dense_req, sparse_req],
|
||||
ranker=RRFRanker(),
|
||||
limit=10,
|
||||
output_fields=["id"],
|
||||
)
|
||||
|
||||
up_pks = set(hit["id"] for hit in up_results[0]) if up_results else set()
|
||||
down_pks = set(hit["id"] for hit in down_results[0]) if down_results else set()
|
||||
union_size = len(up_pks | down_pks)
|
||||
overlap = len(up_pks & down_pks) / union_size if union_size > 0 else 1.0
|
||||
|
||||
logger.info(f"[RESULT] Hybrid search PK overlap={overlap:.4f} (up={len(up_pks)}, down={len(down_pks)})")
|
||||
assert overlap >= self.SEARCH_OVERLAP_THRESHOLD, (
|
||||
f"Hybrid search overlap {overlap:.4f} below threshold {self.SEARCH_OVERLAP_THRESHOLD}"
|
||||
)
|
||||
|
||||
finally:
|
||||
self.log_test_end(
|
||||
"test_hybrid_search_consistency",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def test_search_iterator_consistency(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
):
|
||||
"""Verify search iterator returns the same PK set on upstream and downstream."""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_srch_iter", max_length=50)
|
||||
|
||||
self.log_test_start("test_search_iterator_consistency", "SEARCH_ITERATOR", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
self._setup_collection(upstream_client, c_name, "FLOAT_VECTOR", "HNSW", "COSINE", 128)
|
||||
|
||||
def check_sync():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500")
|
||||
return cnt >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), (
|
||||
f"Downstream did not receive 500 records within {sync_timeout}s"
|
||||
)
|
||||
|
||||
query_vec = self._gen_vectors(1, 128, DataType.FLOAT_VECTOR)[0]
|
||||
search_params = {"metric_type": "COSINE", "params": {"ef": 64}}
|
||||
|
||||
def _collect_iterator_pks(client):
|
||||
pks = set()
|
||||
iterator = client.search_iterator(
|
||||
collection_name=c_name,
|
||||
data=[query_vec],
|
||||
anns_field="vector",
|
||||
batch_size=50,
|
||||
limit=200,
|
||||
param=search_params,
|
||||
output_fields=["id"],
|
||||
)
|
||||
while True:
|
||||
batch = iterator.next()
|
||||
if not batch:
|
||||
iterator.close()
|
||||
break
|
||||
for hit in batch:
|
||||
pks.add(hit["id"])
|
||||
return pks
|
||||
|
||||
up_pks = _collect_iterator_pks(upstream_client)
|
||||
down_pks = _collect_iterator_pks(downstream_client)
|
||||
union_size = len(up_pks | down_pks)
|
||||
overlap = len(up_pks & down_pks) / union_size if union_size > 0 else 1.0
|
||||
|
||||
logger.info(f"[RESULT] Search iterator overlap={overlap:.4f} (up={len(up_pks)}, down={len(down_pks)})")
|
||||
assert overlap >= self.SEARCH_OVERLAP_THRESHOLD, (
|
||||
f"Search iterator PK overlap {overlap:.4f} below threshold {self.SEARCH_OVERLAP_THRESHOLD}"
|
||||
)
|
||||
|
||||
finally:
|
||||
self.log_test_end(
|
||||
"test_search_iterator_consistency",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def test_query_iterator_consistency(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
):
|
||||
"""Verify that a query iterator retrieves identical PK sets from both sides."""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_qry_iter", max_length=50)
|
||||
|
||||
self.log_test_start("test_query_iterator_consistency", "QUERY_ITERATOR", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
self._setup_collection(upstream_client, c_name, "FLOAT_VECTOR", "HNSW", "COSINE", 128)
|
||||
|
||||
def check_sync():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500")
|
||||
return cnt >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), (
|
||||
f"Downstream did not receive 500 records within {sync_timeout}s"
|
||||
)
|
||||
|
||||
up_count, down_count, match = self.verify_iterator_consistency(
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
c_name,
|
||||
batch_size=100,
|
||||
)
|
||||
|
||||
logger.info(f"[RESULT] Query iterator — upstream={up_count}, downstream={down_count}, match={match}")
|
||||
assert match, f"Query iterator PK sets differ: upstream={up_count}, downstream={down_count}"
|
||||
|
||||
finally:
|
||||
self.log_test_end(
|
||||
"test_query_iterator_consistency",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def test_search_with_filter_consistency(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
):
|
||||
"""Verify filtered search produces consistent results honoring the filter predicate."""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_srch_filter", max_length=50)
|
||||
|
||||
self.log_test_start("test_search_with_filter_consistency", "SEARCH_WITH_FILTER", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
self._setup_collection(upstream_client, c_name, "FLOAT_VECTOR", "HNSW", "COSINE", 128)
|
||||
|
||||
def check_sync():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500")
|
||||
return cnt >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_sync, sync_timeout, f"data sync 500 records {c_name}"), (
|
||||
f"Downstream did not receive 500 records within {sync_timeout}s"
|
||||
)
|
||||
|
||||
filter_expr = "int_field > 500"
|
||||
query_vec = self._gen_vectors(1, 128, DataType.FLOAT_VECTOR)[0]
|
||||
search_params = {"metric_type": "COSINE"}
|
||||
|
||||
up_results = upstream_client.search(
|
||||
collection_name=c_name,
|
||||
data=[query_vec],
|
||||
anns_field="vector",
|
||||
search_params=search_params,
|
||||
filter=filter_expr,
|
||||
limit=10,
|
||||
output_fields=["id", "int_field"],
|
||||
)
|
||||
down_results = downstream_client.search(
|
||||
collection_name=c_name,
|
||||
data=[query_vec],
|
||||
anns_field="vector",
|
||||
search_params=search_params,
|
||||
filter=filter_expr,
|
||||
limit=10,
|
||||
output_fields=["id", "int_field"],
|
||||
)
|
||||
|
||||
# Verify filter is honoured on both sides
|
||||
for hit in up_results[0] if up_results else []:
|
||||
assert hit["int_field"] > 500, f"Filter violated on upstream: int_field={hit['int_field']}"
|
||||
for hit in down_results[0] if down_results else []:
|
||||
assert hit["int_field"] > 500, f"Filter violated on downstream: int_field={hit['int_field']}"
|
||||
|
||||
# Verify PK overlap
|
||||
up_pks = set(hit["id"] for hit in up_results[0]) if up_results else set()
|
||||
down_pks = set(hit["id"] for hit in down_results[0]) if down_results else set()
|
||||
union_size = len(up_pks | down_pks)
|
||||
overlap = len(up_pks & down_pks) / union_size if union_size > 0 else 1.0
|
||||
|
||||
logger.info(f"[RESULT] Filtered search overlap={overlap:.4f} (up={len(up_pks)}, down={len(down_pks)})")
|
||||
assert overlap >= self.SEARCH_OVERLAP_THRESHOLD, (
|
||||
f"Filtered search overlap {overlap:.4f} below threshold {self.SEARCH_OVERLAP_THRESHOLD}"
|
||||
)
|
||||
|
||||
finally:
|
||||
self.log_test_end(
|
||||
"test_search_with_filter_consistency",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
@@ -0,0 +1,412 @@
|
||||
"""
|
||||
CDC topology setup and configuration test cases.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from common.common_type import CaseLabel
|
||||
|
||||
from cdc.conftest import CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, apply_replicate_configuration
|
||||
|
||||
from .base import TestCDCSyncBase
|
||||
|
||||
|
||||
@pytest.mark.tags(CaseLabel.CDC)
|
||||
@pytest.mark.skip(reason="Skip topology setup test")
|
||||
class TestCDCTopologySetup(TestCDCSyncBase):
|
||||
"""Test CDC topology setup, switching, and configuration management."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method."""
|
||||
# Cleanup will be handled by individual test methods as needed
|
||||
pass
|
||||
|
||||
def test_normal_topology_setup(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
upstream_uri,
|
||||
downstream_uri,
|
||||
upstream_token,
|
||||
downstream_token,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
pchannel_num,
|
||||
):
|
||||
"""Test case 1: Normal upstream/downstream topology setup."""
|
||||
# Create a basic topology configuration
|
||||
config = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {"uri": upstream_uri, "token": upstream_token},
|
||||
"pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
},
|
||||
{
|
||||
"cluster_id": target_cluster_id,
|
||||
"connection_param": {
|
||||
"uri": downstream_uri,
|
||||
"token": downstream_token,
|
||||
},
|
||||
"pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
},
|
||||
],
|
||||
"cross_cluster_topology": [
|
||||
{
|
||||
"source_cluster_id": source_cluster_id,
|
||||
"target_cluster_id": target_cluster_id,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
# Test normal topology setup
|
||||
apply_replicate_configuration([(upstream_client, config), (downstream_client, config)])
|
||||
|
||||
# Wait for configuration to take effect
|
||||
time.sleep(3)
|
||||
|
||||
# Verify topology is working by creating a test collection
|
||||
test_collection_name = self.gen_unique_name("topology_test")
|
||||
self.resources_to_cleanup.append(("collection", test_collection_name))
|
||||
|
||||
# Create collection on upstream
|
||||
upstream_client.create_collection(
|
||||
collection_name=test_collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Verify it syncs to downstream
|
||||
def check_sync():
|
||||
return downstream_client.has_collection(test_collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_sync, 30, f"collection {test_collection_name} sync")
|
||||
|
||||
# Cleanup
|
||||
self.cleanup_collection(upstream_client, test_collection_name)
|
||||
|
||||
def test_switch_upstream_downstream(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
upstream_uri,
|
||||
downstream_uri,
|
||||
upstream_token,
|
||||
downstream_token,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
pchannel_num,
|
||||
):
|
||||
"""Test case 2: Switch upstream/downstream after initial setup."""
|
||||
# First setup normal topology (source -> target)
|
||||
original_config = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {"uri": upstream_uri, "token": upstream_token},
|
||||
"pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
},
|
||||
{
|
||||
"cluster_id": target_cluster_id,
|
||||
"connection_param": {
|
||||
"uri": downstream_uri,
|
||||
"token": downstream_token,
|
||||
},
|
||||
"pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
},
|
||||
],
|
||||
"cross_cluster_topology": [
|
||||
{
|
||||
"source_cluster_id": source_cluster_id,
|
||||
"target_cluster_id": target_cluster_id,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
apply_replicate_configuration([(upstream_client, original_config), (downstream_client, original_config)])
|
||||
time.sleep(3)
|
||||
|
||||
# Now switch the direction (target -> source)
|
||||
switched_config = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {"uri": upstream_uri, "token": upstream_token},
|
||||
"pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
},
|
||||
{
|
||||
"cluster_id": target_cluster_id,
|
||||
"connection_param": {
|
||||
"uri": downstream_uri,
|
||||
"token": downstream_token,
|
||||
},
|
||||
"pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
},
|
||||
],
|
||||
"cross_cluster_topology": [
|
||||
{
|
||||
"source_cluster_id": target_cluster_id, # Switched
|
||||
"target_cluster_id": source_cluster_id, # Switched
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
# Apply switched configuration
|
||||
apply_replicate_configuration([(upstream_client, switched_config), (downstream_client, switched_config)])
|
||||
time.sleep(3)
|
||||
|
||||
# Test the switched topology by creating collection on downstream (now source)
|
||||
test_collection_name = self.gen_unique_name("switch_test")
|
||||
self.resources_to_cleanup.append(("collection", test_collection_name))
|
||||
|
||||
downstream_client.create_collection(
|
||||
collection_name=test_collection_name,
|
||||
schema=self.create_default_schema(downstream_client),
|
||||
)
|
||||
|
||||
# Verify it syncs to upstream (now target)
|
||||
def check_switched_sync():
|
||||
return upstream_client.has_collection(test_collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_switched_sync, 30, f"switched collection {test_collection_name} sync")
|
||||
|
||||
# Cleanup
|
||||
self.cleanup_collection(downstream_client, test_collection_name)
|
||||
|
||||
def test_invalid_topology_structures(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
upstream_uri,
|
||||
downstream_uri,
|
||||
upstream_token,
|
||||
downstream_token,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
):
|
||||
"""Test case 4: Invalid topology structure cases."""
|
||||
|
||||
# Test case 4.1: Missing cluster in cross_cluster_topology
|
||||
invalid_config_1 = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {"uri": upstream_uri, "token": upstream_token},
|
||||
"pchannels": [f"{source_cluster_id}-rootcoord-dml_0"],
|
||||
}
|
||||
],
|
||||
"cross_cluster_topology": [
|
||||
{
|
||||
"source_cluster_id": source_cluster_id,
|
||||
"target_cluster_id": "nonexistent_cluster", # This cluster is not defined
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with pytest.raises(Exception):
|
||||
upstream_client.update_replicate_configuration(
|
||||
timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_1
|
||||
)
|
||||
|
||||
# Test case 4.2: Circular dependency (A -> B, B -> A)
|
||||
invalid_config_2 = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {"uri": upstream_uri, "token": upstream_token},
|
||||
"pchannels": [f"{source_cluster_id}-rootcoord-dml_0"],
|
||||
},
|
||||
{
|
||||
"cluster_id": target_cluster_id,
|
||||
"connection_param": {
|
||||
"uri": downstream_uri,
|
||||
"token": downstream_token,
|
||||
},
|
||||
"pchannels": [f"{target_cluster_id}-rootcoord-dml_0"],
|
||||
},
|
||||
],
|
||||
"cross_cluster_topology": [
|
||||
{
|
||||
"source_cluster_id": source_cluster_id,
|
||||
"target_cluster_id": target_cluster_id,
|
||||
},
|
||||
{
|
||||
"source_cluster_id": target_cluster_id,
|
||||
"target_cluster_id": source_cluster_id, # Circular dependency
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# This may or may not fail depending on implementation, but test it
|
||||
with pytest.raises(Exception):
|
||||
upstream_client.update_replicate_configuration(
|
||||
timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_2
|
||||
)
|
||||
|
||||
# Test case 4.3: Invalid connection parameters
|
||||
invalid_config_3 = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {
|
||||
"uri": "http://invalid-host:19530", # Invalid URI
|
||||
"token": "invalid:token",
|
||||
},
|
||||
"pchannels": [f"{source_cluster_id}-rootcoord-dml_0"],
|
||||
},
|
||||
{
|
||||
"cluster_id": target_cluster_id,
|
||||
"connection_param": {
|
||||
"uri": downstream_uri,
|
||||
"token": downstream_token,
|
||||
},
|
||||
"pchannels": [f"{target_cluster_id}-rootcoord-dml_0"],
|
||||
},
|
||||
],
|
||||
"cross_cluster_topology": [
|
||||
{
|
||||
"source_cluster_id": source_cluster_id,
|
||||
"target_cluster_id": target_cluster_id,
|
||||
}
|
||||
],
|
||||
}
|
||||
with pytest.raises(Exception):
|
||||
upstream_client.update_replicate_configuration(
|
||||
timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_3
|
||||
)
|
||||
|
||||
# Test case 4.4: Empty cluster list but non-empty topology
|
||||
invalid_config_4 = {
|
||||
"clusters": [], # Empty clusters
|
||||
"cross_cluster_topology": [
|
||||
{
|
||||
"source_cluster_id": source_cluster_id,
|
||||
"target_cluster_id": target_cluster_id,
|
||||
}
|
||||
],
|
||||
}
|
||||
with pytest.raises(Exception):
|
||||
upstream_client.update_replicate_configuration(
|
||||
timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_4
|
||||
)
|
||||
|
||||
# Test case 4.5: Invalid pchannel format
|
||||
invalid_config_5 = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {"uri": upstream_uri, "token": upstream_token},
|
||||
"pchannels": ["invalid-channel-format"], # Invalid format
|
||||
}
|
||||
],
|
||||
"cross_cluster_topology": [],
|
||||
}
|
||||
with pytest.raises(Exception):
|
||||
upstream_client.update_replicate_configuration(
|
||||
timeout=CDC_UPDATE_REPLICATE_TIMEOUT_SECONDS, **invalid_config_5
|
||||
)
|
||||
|
||||
@pytest.mark.order(-1)
|
||||
def test_clear_configuration_disconnect(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
upstream_uri,
|
||||
downstream_uri,
|
||||
upstream_token,
|
||||
downstream_token,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
pchannel_num,
|
||||
):
|
||||
"""Test case 3: Clear configuration and disconnect CDC."""
|
||||
# First setup normal topology
|
||||
config = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {"uri": upstream_uri, "token": upstream_token},
|
||||
"pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
},
|
||||
{
|
||||
"cluster_id": target_cluster_id,
|
||||
"connection_param": {
|
||||
"uri": downstream_uri,
|
||||
"token": downstream_token,
|
||||
},
|
||||
"pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
},
|
||||
],
|
||||
"cross_cluster_topology": [
|
||||
{
|
||||
"source_cluster_id": source_cluster_id,
|
||||
"target_cluster_id": target_cluster_id,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
apply_replicate_configuration([(upstream_client, config), (downstream_client, config)])
|
||||
time.sleep(3)
|
||||
|
||||
# Verify topology is working
|
||||
test_collection_name = self.gen_unique_name("clear_config_test")
|
||||
upstream_client.create_collection(
|
||||
collection_name=test_collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
|
||||
def check_initial_sync():
|
||||
return downstream_client.has_collection(test_collection_name)
|
||||
|
||||
assert self.wait_for_sync(check_initial_sync, 30, f"initial collection {test_collection_name} sync")
|
||||
|
||||
# Now clear the configuration (empty topology)
|
||||
empty_upstream_config = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": source_cluster_id,
|
||||
"connection_param": {"uri": upstream_uri, "token": upstream_token},
|
||||
"pchannels": [f"{source_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
}
|
||||
],
|
||||
"cross_cluster_topology": [],
|
||||
}
|
||||
empty_downstream_config = {
|
||||
"clusters": [
|
||||
{
|
||||
"cluster_id": target_cluster_id,
|
||||
"connection_param": {
|
||||
"uri": downstream_uri,
|
||||
"token": downstream_token,
|
||||
},
|
||||
"pchannels": [f"{target_cluster_id}-rootcoord-dml_{i}" for i in range(pchannel_num)],
|
||||
}
|
||||
],
|
||||
"cross_cluster_topology": [],
|
||||
}
|
||||
# Apply empty configuration to disconnect CDC
|
||||
apply_replicate_configuration(
|
||||
[(upstream_client, empty_upstream_config), (downstream_client, empty_downstream_config)]
|
||||
)
|
||||
time.sleep(3)
|
||||
|
||||
# Test that CDC is disconnected - create new collection and verify it doesn't sync
|
||||
disconnected_collection_name = self.gen_unique_name("disconnected_test")
|
||||
upstream_client.create_collection(
|
||||
collection_name=disconnected_collection_name,
|
||||
schema=self.create_default_schema(upstream_client),
|
||||
)
|
||||
|
||||
# Wait a reasonable time and verify it doesn't sync
|
||||
time.sleep(10)
|
||||
assert not downstream_client.has_collection(disconnected_collection_name), (
|
||||
"Collection should not sync after CDC disconnection"
|
||||
)
|
||||
|
||||
# Cleanup
|
||||
self.cleanup_collection(upstream_client, test_collection_name)
|
||||
self.cleanup_collection(upstream_client, disconnected_collection_name)
|
||||
@@ -0,0 +1,793 @@
|
||||
"""
|
||||
CDC sync tests for topology switchover and failover scenarios.
|
||||
"""
|
||||
|
||||
import random
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .base import TestCDCSyncBase, logger
|
||||
|
||||
|
||||
class TestCDCSyncSwitchover(TestCDCSyncBase):
|
||||
"""Test CDC sync behaviour during and after topology switchover / failover."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.resources_to_cleanup = []
|
||||
|
||||
def teardown_method(self):
|
||||
"""Cleanup after each test method - only cleanup upstream, downstream will sync."""
|
||||
upstream_client = getattr(self, "_upstream_client", None)
|
||||
|
||||
if upstream_client:
|
||||
for resource_type, resource_name in self.resources_to_cleanup:
|
||||
if resource_type == "collection":
|
||||
self.cleanup_collection(upstream_client, resource_name)
|
||||
|
||||
time.sleep(1) # Allow cleanup to sync to downstream
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Tests
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
def test_switchover_basic(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
):
|
||||
"""
|
||||
Basic switchover: insert 200 records, verify sync, switchover, insert 200 more
|
||||
on the new source, verify count == 400 on both sides, sample verify, switch back.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_sw_basic", max_length=50)
|
||||
|
||||
self.log_test_start("test_switchover_basic", "SWITCHOVER_BASIC", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
|
||||
# Wait for collection to appear on downstream
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
# Insert 200 records on upstream (original source)
|
||||
batch1 = self.generate_test_data_with_id(200, start_id=0)
|
||||
upstream_client.insert(c_name, batch1)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_200():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/200")
|
||||
return cnt >= 200
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_200, sync_timeout, f"initial 200-record sync {c_name}")
|
||||
|
||||
# Switchover: target becomes new source
|
||||
logger.info("[SWITCHOVER] Initiating basic switchover...")
|
||||
switchover_helper(target_cluster_id, source_cluster_id)
|
||||
|
||||
# Insert 200 more records on the new source (previously downstream)
|
||||
batch2 = self.generate_test_data_with_id(200, start_id=200)
|
||||
downstream_client.insert(c_name, batch2)
|
||||
downstream_client.flush(c_name)
|
||||
|
||||
def check_400_upstream():
|
||||
try:
|
||||
res = upstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] upstream count after switchover: {cnt}/400")
|
||||
return cnt >= 400
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
def check_400_downstream():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count after switchover: {cnt}/400")
|
||||
return cnt >= 400
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_400_upstream, sync_timeout, f"400-record sync upstream {c_name}")
|
||||
assert self.wait_for_sync(check_400_downstream, sync_timeout, f"400-record sync downstream {c_name}")
|
||||
|
||||
# Sample verify
|
||||
match_count, mismatch_count, details = self.verify_data_sampling(
|
||||
downstream_client, # new source
|
||||
upstream_client, # new target
|
||||
c_name,
|
||||
sample_ratio=0.2,
|
||||
output_fields=["id", "vector"],
|
||||
)
|
||||
assert mismatch_count == 0, f"Data mismatch after basic switchover: {details[:5]}"
|
||||
|
||||
finally:
|
||||
# Restore original topology
|
||||
logger.info("[SWITCHOVER] Restoring original topology after test_switchover_basic...")
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
self.log_test_end("test_switchover_basic", True, time.time() - start_time)
|
||||
|
||||
def test_switchover_during_writes(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
):
|
||||
"""
|
||||
Switchover during concurrent writes: background thread inserts 10 batches of 100
|
||||
with 2 s between each; switchover fires at t=12 s; join thread, flush, verify
|
||||
both sides have the same count >= total_inserted.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_sw_writes", max_length=50)
|
||||
|
||||
self.log_test_start("test_switchover_during_writes", "SWITCHOVER_DURING_WRITES", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
total_inserted = 0
|
||||
lock = threading.Lock()
|
||||
write_error = []
|
||||
|
||||
def background_insert():
|
||||
nonlocal total_inserted
|
||||
for batch_idx in range(10):
|
||||
try:
|
||||
start_id = batch_idx * 100
|
||||
data = self.generate_test_data_with_id(100, start_id=start_id)
|
||||
# Cap pymilvus's retry loop at 10 (default 75). A batch
|
||||
# that lands on the old primary after it flips to
|
||||
# replica returns STREAMING_CODE_REPLICATE_VIOLATION;
|
||||
# the default retry burns ~213 s per failed batch and
|
||||
# makes the thread runtime unbounded when the role flip
|
||||
# takes longer than one batch's cadence. 10 retries
|
||||
# with the decorator's 3 s backoff cap bound each
|
||||
# failure to ~16 s.
|
||||
upstream_client.insert(c_name, data, retry_times=10)
|
||||
with lock:
|
||||
total_inserted += 100
|
||||
logger.info(f"[BACKGROUND] Inserted batch {batch_idx + 1}/10 (total: {total_inserted})")
|
||||
except Exception as e:
|
||||
logger.error(f"[BACKGROUND] Insert failed on batch {batch_idx}: {e}")
|
||||
write_error.append(e)
|
||||
time.sleep(2)
|
||||
|
||||
insert_thread = threading.Thread(target=background_insert, daemon=True)
|
||||
insert_thread.start()
|
||||
|
||||
# Switchover after 12 s (mid-way through background inserts)
|
||||
time.sleep(12)
|
||||
logger.info("[SWITCHOVER] Initiating switchover during writes...")
|
||||
switchover_helper(target_cluster_id, source_cluster_id)
|
||||
|
||||
# With retry_times=10 each failed batch burns ~16 s (vs ~213 s with
|
||||
# the pymilvus default of 75). Worst case: all 10 batches fail →
|
||||
# ~10 * (16 + 2) = 180 s. 180 s gives ~6x margin.
|
||||
insert_thread.join(timeout=180)
|
||||
assert not insert_thread.is_alive(), "Background insert thread did not finish in time"
|
||||
|
||||
# Transient STREAMING_CODE_REPLICATE_VIOLATION failures are EXPECTED
|
||||
# during the role flip: writes that arrive on the old primary after
|
||||
# it becomes a replica are rejected by design. The invariant that
|
||||
# matters is that every batch which SUCCEEDED ends up replicated
|
||||
# consistently (enforced by the up_cnt == down_cnt check below).
|
||||
if write_error:
|
||||
logger.warning(
|
||||
f"Background inserts had {len(write_error)} transient failure(s) during switchover: {write_error}"
|
||||
)
|
||||
|
||||
# After switchover, downstream is the new primary. Flushing the
|
||||
# old primary (upstream) hits the replica-role rate limiter
|
||||
# (rate=0.1/s) and exhausts pymilvus's 75 retries. Flush the
|
||||
# current primary instead.
|
||||
downstream_client.flush(c_name)
|
||||
|
||||
expected = total_inserted
|
||||
logger.info(f"[INFO] Total inserted: {expected}")
|
||||
|
||||
def check_both(client, label):
|
||||
def _check():
|
||||
try:
|
||||
res = client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] {label} count: {cnt}/{expected}")
|
||||
return cnt >= expected
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed ({label}): {e}")
|
||||
return False
|
||||
|
||||
return _check
|
||||
|
||||
assert self.wait_for_sync(
|
||||
check_both(upstream_client, "upstream"),
|
||||
sync_timeout,
|
||||
f"upstream count>={expected} after switchover-during-writes",
|
||||
)
|
||||
assert self.wait_for_sync(
|
||||
check_both(downstream_client, "downstream"),
|
||||
sync_timeout,
|
||||
f"downstream count>={expected} after switchover-during-writes",
|
||||
)
|
||||
|
||||
up_res = upstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
down_res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
up_cnt = up_res[0]["count(*)"] if up_res else 0
|
||||
down_cnt = down_res[0]["count(*)"] if down_res else 0
|
||||
assert up_cnt == down_cnt, (
|
||||
f"Count mismatch after switchover-during-writes: upstream={up_cnt}, downstream={down_cnt}"
|
||||
)
|
||||
|
||||
finally:
|
||||
logger.info("[SWITCHOVER] Restoring original topology after test_switchover_during_writes...")
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
self.log_test_end("test_switchover_during_writes", True, time.time() - start_time)
|
||||
|
||||
def test_switchover_with_all_data_types(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
):
|
||||
"""
|
||||
Switchover with comprehensive data types: insert 100 records, verify sync,
|
||||
switchover, verify scalar field sampling, switch back.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_sw_dtypes", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_switchover_with_all_data_types",
|
||||
"SWITCHOVER_ALL_DTYPES",
|
||||
c_name,
|
||||
)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
schema = self.create_comprehensive_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
# Index every vector field; load_collection fails otherwise.
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(field_name="float_vector", index_type="AUTOINDEX", metric_type="L2")
|
||||
index_params.add_index(field_name="float16_vector", index_type="AUTOINDEX", metric_type="L2")
|
||||
index_params.add_index(field_name="binary_vector", index_type="BIN_FLAT", metric_type="HAMMING")
|
||||
index_params.add_index(field_name="sparse_vector", index_type="SPARSE_INVERTED_INDEX", metric_type="IP")
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
data = self.generate_comprehensive_test_data(100)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_100():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/100")
|
||||
return cnt >= 100
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_100, sync_timeout, f"100-record sync {c_name}")
|
||||
|
||||
# Switchover
|
||||
logger.info("[SWITCHOVER] Initiating switchover with all data types...")
|
||||
switchover_helper(target_cluster_id, source_cluster_id)
|
||||
|
||||
# After switchover, verify scalar field sampling (new source is downstream)
|
||||
scalar_fields = [
|
||||
"bool_field",
|
||||
"int8_field",
|
||||
"int16_field",
|
||||
"int32_field",
|
||||
"int64_field",
|
||||
"float_field",
|
||||
"double_field",
|
||||
"varchar_field",
|
||||
]
|
||||
match_count, mismatch_count, details = self.verify_data_sampling(
|
||||
downstream_client, # new source
|
||||
upstream_client, # new target
|
||||
c_name,
|
||||
sample_ratio=0.3,
|
||||
output_fields=scalar_fields,
|
||||
)
|
||||
logger.info(f"[RESULT] All-dtypes sampling — match={match_count}, mismatch={mismatch_count}")
|
||||
assert mismatch_count == 0, f"Data mismatch after switchover with all types: {details[:5]}"
|
||||
|
||||
finally:
|
||||
logger.info("[SWITCHOVER] Restoring original topology after test_switchover_with_all_data_types...")
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
self.log_test_end(
|
||||
"test_switchover_with_all_data_types",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def test_switchover_with_loaded_collection(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
):
|
||||
"""
|
||||
Switchover with a loaded collection: verify search works on downstream before
|
||||
switchover; after switchover verify search works on both sides.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_sw_loaded", max_length=50)
|
||||
|
||||
self.log_test_start(
|
||||
"test_switchover_with_loaded_collection",
|
||||
"SWITCHOVER_LOADED",
|
||||
c_name,
|
||||
)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
schema = self.create_default_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
data = self.generate_test_data(200)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_200():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/200")
|
||||
return cnt >= 200
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_200, sync_timeout, f"200-record sync {c_name}")
|
||||
|
||||
# Verify search works on downstream before switchover
|
||||
q_vec = [random.random() for _ in range(128)]
|
||||
pre_down_results = downstream_client.search(
|
||||
collection_name=c_name,
|
||||
data=[q_vec],
|
||||
anns_field="vector",
|
||||
search_params={"metric_type": "L2"},
|
||||
limit=5,
|
||||
output_fields=["id"],
|
||||
)
|
||||
assert pre_down_results and len(pre_down_results[0]) > 0, (
|
||||
"Search on downstream returned no results before switchover"
|
||||
)
|
||||
|
||||
# Switchover
|
||||
logger.info("[SWITCHOVER] Initiating switchover with loaded collection...")
|
||||
switchover_helper(target_cluster_id, source_cluster_id)
|
||||
|
||||
# Verify search still works on both sides after switchover
|
||||
for client, label in [(upstream_client, "upstream"), (downstream_client, "downstream")]:
|
||||
results = client.search(
|
||||
collection_name=c_name,
|
||||
data=[q_vec],
|
||||
anns_field="vector",
|
||||
search_params={"metric_type": "L2"},
|
||||
limit=5,
|
||||
output_fields=["id"],
|
||||
)
|
||||
assert results and len(results[0]) > 0, f"Search returned no results on {label} after switchover"
|
||||
logger.info(f"[VERIFY] Search on {label} after switchover returned {len(results[0])} results — OK")
|
||||
|
||||
finally:
|
||||
logger.info("[SWITCHOVER] Restoring original topology after test_switchover_with_loaded_collection...")
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
self.log_test_end(
|
||||
"test_switchover_with_loaded_collection",
|
||||
True,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
def test_switchover_with_index(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
):
|
||||
"""
|
||||
Switchover after index creation: wait for index to sync, switchover, verify
|
||||
list_indexes returns the same index on both sides.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_sw_index", max_length=50)
|
||||
|
||||
self.log_test_start("test_switchover_with_index", "SWITCHOVER_INDEX", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
schema = self.create_default_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
# Insert data first
|
||||
data = self.generate_test_data(200)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
# Create HNSW index
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="COSINE",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
|
||||
# Wait for index to sync to downstream
|
||||
def check_index_sync():
|
||||
try:
|
||||
indexes = downstream_client.list_indexes(c_name)
|
||||
result = len(indexes) > 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream indexes: {indexes} (has_index={result})")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.warning(f"Index sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_index_sync, sync_timeout, f"index sync {c_name}"), (
|
||||
f"Index did not sync to downstream within {sync_timeout}s"
|
||||
)
|
||||
|
||||
# Switchover
|
||||
logger.info("[SWITCHOVER] Initiating switchover with index...")
|
||||
switchover_helper(target_cluster_id, source_cluster_id)
|
||||
|
||||
# Verify list_indexes on both sides
|
||||
up_indexes = upstream_client.list_indexes(c_name)
|
||||
down_indexes = downstream_client.list_indexes(c_name)
|
||||
|
||||
logger.info(f"[VERIFY] After switchover — upstream indexes={up_indexes}, downstream indexes={down_indexes}")
|
||||
assert len(up_indexes) > 0, "No indexes on upstream after switchover"
|
||||
assert len(down_indexes) > 0, "No indexes on downstream after switchover"
|
||||
assert set(up_indexes) == set(down_indexes), (
|
||||
f"Index lists differ after switchover: up={up_indexes}, down={down_indexes}"
|
||||
)
|
||||
|
||||
finally:
|
||||
logger.info("[SWITCHOVER] Restoring original topology after test_switchover_with_index...")
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
self.log_test_end("test_switchover_with_index", True, time.time() - start_time)
|
||||
|
||||
def test_rapid_switchover_stress(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
):
|
||||
"""
|
||||
Rapid-switchover stress: loop 5 times — write 50 records to current source,
|
||||
switchover. After loop, wait 30 s and verify counts match. Restore original topology.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_sw_stress", max_length=50)
|
||||
|
||||
self.log_test_start("test_rapid_switchover_stress", "RAPID_SWITCHOVER_STRESS", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="COSINE",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
# Track topology state
|
||||
current_source_id = source_cluster_id
|
||||
current_target_id = target_cluster_id
|
||||
current_source_client = upstream_client
|
||||
total_inserted = 0
|
||||
|
||||
for iteration in range(5):
|
||||
# Write 50 records to current source
|
||||
start_id = iteration * 50
|
||||
data = self.generate_test_data_with_id(50, start_id=start_id)
|
||||
current_source_client.insert(c_name, data)
|
||||
current_source_client.flush(c_name)
|
||||
total_inserted += 50
|
||||
|
||||
logger.info(
|
||||
f"[STRESS] Iteration {iteration + 1}/5: inserted 50 records "
|
||||
f"(total={total_inserted}), switching over..."
|
||||
)
|
||||
|
||||
# Switchover
|
||||
switchover_helper(current_target_id, current_source_id)
|
||||
|
||||
# Swap topology
|
||||
current_source_id, current_target_id = current_target_id, current_source_id
|
||||
current_source_client = downstream_client if current_source_id == target_cluster_id else upstream_client
|
||||
|
||||
logger.info(
|
||||
f"[STRESS] All 5 switchovers done. Waiting 30s for final sync (total_inserted={total_inserted})..."
|
||||
)
|
||||
time.sleep(30)
|
||||
|
||||
# Verify counts match on both sides
|
||||
up_res = upstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
down_res = downstream_client.query(collection_name=c_name, filter="", output_fields=["count(*)"])
|
||||
up_cnt = up_res[0]["count(*)"] if up_res else 0
|
||||
down_cnt = down_res[0]["count(*)"] if down_res else 0
|
||||
|
||||
logger.info(f"[VERIFY] After stress — upstream={up_cnt}, downstream={down_cnt}, expected>={total_inserted}")
|
||||
assert up_cnt >= total_inserted, f"Upstream count {up_cnt} < expected {total_inserted} after stress"
|
||||
assert down_cnt >= total_inserted, f"Downstream count {down_cnt} < expected {total_inserted} after stress"
|
||||
assert up_cnt == down_cnt, f"Count mismatch after stress: upstream={up_cnt}, downstream={down_cnt}"
|
||||
|
||||
finally:
|
||||
# Restore to original topology
|
||||
logger.info("[SWITCHOVER] Restoring original topology after test_rapid_switchover_stress...")
|
||||
switchover_helper(source_cluster_id, target_cluster_id)
|
||||
self.log_test_end("test_rapid_switchover_stress", True, time.time() - start_time)
|
||||
|
||||
def test_failover_source_down(
|
||||
self,
|
||||
upstream_client,
|
||||
downstream_client,
|
||||
sync_timeout,
|
||||
switchover_helper,
|
||||
source_cluster_id,
|
||||
target_cluster_id,
|
||||
milvus_ns,
|
||||
):
|
||||
"""
|
||||
Failover when source goes down: insert 500 records, verify sync, kill source pods,
|
||||
verify target count >= 500, wait for source to recover and verify count >= 500.
|
||||
"""
|
||||
start_time = time.time()
|
||||
c_name = self.gen_unique_name("test_failover_src", max_length=50)
|
||||
|
||||
self.log_test_start("test_failover_source_down", "FAILOVER_SOURCE_DOWN", c_name)
|
||||
self._upstream_client = upstream_client
|
||||
self.resources_to_cleanup.append(("collection", c_name))
|
||||
|
||||
try:
|
||||
self.cleanup_collection(upstream_client, c_name)
|
||||
|
||||
schema = self.create_manual_id_schema(upstream_client)
|
||||
upstream_client.create_collection(collection_name=c_name, schema=schema)
|
||||
|
||||
index_params = upstream_client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="vector",
|
||||
index_type="HNSW",
|
||||
metric_type="L2",
|
||||
params={"M": 16, "efConstruction": 200},
|
||||
)
|
||||
upstream_client.create_index(c_name, index_params)
|
||||
upstream_client.load_collection(c_name)
|
||||
|
||||
def check_create():
|
||||
return downstream_client.has_collection(c_name)
|
||||
|
||||
assert self.wait_for_sync(check_create, sync_timeout, f"create collection {c_name}")
|
||||
|
||||
# Insert 500 records
|
||||
data = self.generate_test_data_with_id(500, start_id=0)
|
||||
upstream_client.insert(c_name, data)
|
||||
upstream_client.flush(c_name)
|
||||
|
||||
def check_500_downstream():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[SYNC_PROGRESS] downstream count: {cnt}/500")
|
||||
return cnt >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Sync check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_500_downstream, sync_timeout, f"500-record sync {c_name}"), (
|
||||
f"Downstream did not receive 500 records within {sync_timeout}s"
|
||||
)
|
||||
|
||||
# Kill source pods forcefully
|
||||
logger.info(f"[FAILOVER] Killing source pods (instance={source_cluster_id}, ns={milvus_ns})...")
|
||||
kill_cmd = [
|
||||
"kubectl",
|
||||
"delete",
|
||||
"pods",
|
||||
"-l",
|
||||
f"app.kubernetes.io/instance={source_cluster_id}",
|
||||
"-n",
|
||||
milvus_ns,
|
||||
"--grace-period=0",
|
||||
"--force",
|
||||
]
|
||||
result = subprocess.run(kill_cmd, capture_output=True, text=True)
|
||||
logger.info(
|
||||
f"[FAILOVER] kubectl output: stdout={result.stdout!r}, stderr={result.stderr!r}, rc={result.returncode}"
|
||||
)
|
||||
|
||||
# Wait 60 s and verify target still has data
|
||||
logger.info("[FAILOVER] Waiting 60s after pod kill...")
|
||||
time.sleep(60)
|
||||
|
||||
def check_target_intact():
|
||||
try:
|
||||
res = downstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[FAILOVER] Target count after kill: {cnt}")
|
||||
return cnt >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Target check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_target_intact, 30, f"target intact after source kill {c_name}"), (
|
||||
"Target lost data after source pod kill"
|
||||
)
|
||||
|
||||
# Wait 120 s more for source to recover, then verify
|
||||
logger.info("[FAILOVER] Waiting 120s for source recovery...")
|
||||
time.sleep(120)
|
||||
|
||||
def check_source_recovered():
|
||||
try:
|
||||
res = upstream_client.query(
|
||||
collection_name=c_name,
|
||||
filter="",
|
||||
output_fields=["count(*)"],
|
||||
)
|
||||
cnt = res[0]["count(*)"] if res else 0
|
||||
logger.info(f"[FAILOVER] Source count after recovery: {cnt}")
|
||||
return cnt >= 500
|
||||
except Exception as e:
|
||||
logger.warning(f"Source recovery check failed: {e}")
|
||||
return False
|
||||
|
||||
assert self.wait_for_sync(check_source_recovered, 60, f"source recovery {c_name}"), (
|
||||
"Source did not recover with expected data count within timeout"
|
||||
)
|
||||
|
||||
finally:
|
||||
self.log_test_end("test_failover_source_down", True, time.time() - start_time)
|
||||
Reference in New Issue
Block a user