wehub-resource-sync
856 lines
31 KiB
Python
856 lines
31 KiB
Python
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import ray
|
|
from ray import serve
|
|
from ray._common.test_utils import SignalActor, wait_for_condition
|
|
from ray.serve.schema import CeleryAdapterConfig, TaskProcessorConfig
|
|
from ray.serve.task_consumer import (
|
|
instantiate_adapter_from_config,
|
|
task_consumer,
|
|
task_handler,
|
|
)
|
|
from ray.tests.conftest import external_redis # noqa: F401
|
|
|
|
|
|
@ray.remote
|
|
class ProcessedTasksTracker:
|
|
def __init__(self):
|
|
self.processed_tasks = set()
|
|
|
|
def add_task(self, task_data):
|
|
self.processed_tasks.add(task_data)
|
|
|
|
def get_processed_tasks(self):
|
|
return self.processed_tasks
|
|
|
|
def get_count(self):
|
|
return len(self.processed_tasks)
|
|
|
|
|
|
@ray.remote
|
|
def send_request_to_queue(
|
|
processor_config: TaskProcessorConfig, data, task_name="process_request"
|
|
):
|
|
adapter_instance_global = instantiate_adapter_from_config(
|
|
task_processor_config=processor_config
|
|
)
|
|
result = adapter_instance_global.enqueue_task_sync(task_name, args=[data])
|
|
assert result.id is not None
|
|
return result.id
|
|
|
|
|
|
@pytest.fixture(scope="function")
|
|
def temp_queue_directory():
|
|
"""Creates a temporary directory with 'queue', 'results', and 'control' subdirectories for task consumer tests."""
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
tmpdir_path = Path(tmpdir)
|
|
|
|
data_folder_queue = tmpdir_path / "queue"
|
|
data_folder_queue.mkdir()
|
|
|
|
results_path = tmpdir_path / "results"
|
|
results_path.mkdir()
|
|
|
|
control_path = tmpdir_path / "control"
|
|
control_path.mkdir()
|
|
|
|
yield {
|
|
"queue_path": data_folder_queue,
|
|
"results_path": results_path,
|
|
"control_path": control_path,
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="function")
|
|
def transport_options(temp_queue_directory):
|
|
"""Create standard transport options for filesystem broker."""
|
|
|
|
queue_path = temp_queue_directory["queue_path"]
|
|
control_path = temp_queue_directory["control_path"]
|
|
|
|
return {
|
|
# Incoming message queue - where new task messages are written when sent to broker
|
|
"data_folder_in": str(queue_path),
|
|
# Outgoing message storage - where task results and responses are written after completion
|
|
"data_folder_out": str(queue_path),
|
|
# Processed message archive - where messages are moved after successful processing
|
|
"data_folder_processed": str(queue_path),
|
|
# Control message storage - where Celery management and control commands are stored
|
|
"control_folder": str(control_path),
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="function")
|
|
def create_processor_config(temp_queue_directory, transport_options):
|
|
"""Create a TaskProcessorConfig with common defaults."""
|
|
|
|
def _create(
|
|
failed_task_queue_name=None, unprocessable_task_queue_name=None, **kwargs
|
|
):
|
|
results_path = temp_queue_directory["results_path"]
|
|
|
|
config_params = {
|
|
"queue_name": "my_default_app_queue",
|
|
"adapter_config": CeleryAdapterConfig(
|
|
broker_url="filesystem://",
|
|
backend_url=f"file://{results_path}",
|
|
broker_transport_options=transport_options,
|
|
),
|
|
}
|
|
|
|
# Add dead letter queue names if provided
|
|
if failed_task_queue_name is not None:
|
|
config_params["failed_task_queue_name"] = failed_task_queue_name
|
|
if unprocessable_task_queue_name is not None:
|
|
config_params[
|
|
"unprocessable_task_queue_name"
|
|
] = unprocessable_task_queue_name
|
|
|
|
config_params.update(kwargs)
|
|
|
|
return TaskProcessorConfig(**config_params)
|
|
|
|
return _create
|
|
|
|
|
|
def _get_task_counts_by_routing_key(queue_path):
|
|
"""Counts tasks in a queue directory by reading the routing key from each message."""
|
|
counts = defaultdict(int)
|
|
if not queue_path.exists():
|
|
return counts
|
|
|
|
# Celery doesn't provide a way to get the queue size.
|
|
# so we've to levarage the broker's API to get the queue size.
|
|
# Since we are using the filesystem broker in tests, we can read the files in the queue directory to get the queue size.
|
|
for msg_file in queue_path.iterdir():
|
|
if msg_file.is_file():
|
|
try:
|
|
with open(msg_file, "r") as f:
|
|
data = json.load(f)
|
|
routing_key = (
|
|
data.get("properties", {})
|
|
.get("delivery_info", {})
|
|
.get("routing_key")
|
|
)
|
|
if routing_key:
|
|
counts[routing_key] += 1
|
|
except (json.JSONDecodeError, IOError):
|
|
# Ignore files that aren't valid JSON or are otherwise unreadable
|
|
continue
|
|
return counts
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
|
|
class TestTaskConsumerWithRayServe:
|
|
"""Test task consumer integration with Ray Serve."""
|
|
|
|
def test_task_consumer_as_serve_deployment(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that task consumers can be used as Ray Serve deployments."""
|
|
processor_config = create_processor_config()
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
def __init__(self):
|
|
self.data_received = None
|
|
self.task_received = False
|
|
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data):
|
|
self.task_received = True
|
|
self.data_received = data
|
|
|
|
def assert_task_received(self):
|
|
assert self.task_received is True
|
|
assert self.data_received is not None
|
|
assert self.data_received == "test_data_1"
|
|
|
|
# Deploy the consumer as a Serve deployment
|
|
handle = serve.run(ServeTaskConsumer.bind())
|
|
send_request_to_queue.remote(processor_config, "test_data_1")
|
|
|
|
def assert_result():
|
|
try:
|
|
# `assert_task_received` will throw AssertionError if the task was not received or data is not as expected
|
|
handle.assert_task_received.remote().result()
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
wait_for_condition(assert_result)
|
|
|
|
def test_task_consumer_as_serve_deployment_with_failed_task(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that task consumers can be used as Ray Serve deployments."""
|
|
processor_config = create_processor_config(
|
|
failed_task_queue_name="my_failed_task_queue"
|
|
)
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
def __init__(self):
|
|
self.num_calls = 0
|
|
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data):
|
|
self.num_calls += 1
|
|
raise ValueError("Task failed as expected")
|
|
|
|
def get_num_calls(self):
|
|
return self.num_calls
|
|
|
|
handle = serve.run(ServeTaskConsumer.bind())
|
|
task_id_ref = send_request_to_queue.remote(processor_config, "test_data_1")
|
|
task_id = ray.get(task_id_ref)
|
|
|
|
adapter_instance = instantiate_adapter_from_config(
|
|
task_processor_config=processor_config
|
|
)
|
|
|
|
def assert_result():
|
|
result = adapter_instance.get_task_status_sync(task_id)
|
|
|
|
if (
|
|
result.status == "FAILURE"
|
|
and result.result is not None
|
|
and isinstance(result.result, ValueError)
|
|
and str(result.result) == "Task failed as expected"
|
|
and handle.get_num_calls.remote().result()
|
|
== 1 + processor_config.max_retries
|
|
):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
wait_for_condition(assert_result, timeout=20)
|
|
|
|
def test_task_consumer_persistence_across_restarts(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that tasks persist in queue and get executed after deployment restart."""
|
|
# Setup
|
|
config = create_processor_config()
|
|
tracker = ProcessedTasksTracker.remote()
|
|
signal1 = SignalActor.remote()
|
|
|
|
@serve.deployment(
|
|
num_replicas=1, graceful_shutdown_timeout_s=60, max_ongoing_requests=1
|
|
)
|
|
@task_consumer(task_processor_config=config)
|
|
class TaskConsumer:
|
|
def __init__(self, tracker_ref, signal_ref):
|
|
self.tracker, self.signal = tracker_ref, signal_ref
|
|
self.local_processed = []
|
|
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data):
|
|
ray.get(self.signal.wait.remote()) # Block until signal
|
|
self.local_processed.append(data)
|
|
ray.get(self.tracker.add_task.remote(data))
|
|
return f"Processed: {data}"
|
|
|
|
def get_local_processed(self):
|
|
return self.local_processed
|
|
|
|
# Deploy first version and send tasks
|
|
serve.run(TaskConsumer.bind(tracker, signal1), name="app_v1")
|
|
|
|
num_tasks = 20
|
|
for i in range(num_tasks):
|
|
ray.get(send_request_to_queue.remote(config, f"task_{i}"))
|
|
|
|
# Process exactly 1 task, then restart deployment
|
|
wait_for_condition(
|
|
lambda: ray.get(signal1.cur_num_waiters.remote()) == 1, timeout=10
|
|
)
|
|
ray.get(signal1.send.remote(clear=True)) # Allow 1 task to complete
|
|
wait_for_condition(lambda: ray.get(tracker.get_count.remote()) == 1, timeout=10)
|
|
|
|
# Shutdown first deployment
|
|
serve.delete("app_v1", _blocking=False)
|
|
ray.get(signal1.send.remote()) # Release any stuck tasks
|
|
wait_for_condition(
|
|
lambda: "app_v1" not in serve.status().applications, timeout=100
|
|
)
|
|
|
|
tasks_before_restart = ray.get(tracker.get_count.remote())
|
|
assert (
|
|
tasks_before_restart >= 2 and tasks_before_restart < num_tasks
|
|
), f"Expected at least 2 tasks processed and atleast one less than num_tasks, got {tasks_before_restart}"
|
|
|
|
# Deploy second version and process remaining tasks
|
|
signal2 = SignalActor.remote()
|
|
handle = serve.run(TaskConsumer.bind(tracker, signal2), name="app_v2")
|
|
|
|
wait_for_condition(
|
|
lambda: ray.get(signal2.cur_num_waiters.remote()) == 1, timeout=10
|
|
)
|
|
ray.get(signal2.send.remote()) # Process all remaining tasks
|
|
wait_for_condition(
|
|
lambda: ray.get(tracker.get_count.remote()) == num_tasks, timeout=100
|
|
)
|
|
|
|
# Verify all tasks were processed and distributed correctly
|
|
expected_tasks = {f"task_{i}" for i in range(num_tasks)}
|
|
final_tasks = ray.get(tracker.get_processed_tasks.remote())
|
|
second_deployment_tasks = handle.get_local_processed.remote().result()
|
|
|
|
assert (
|
|
final_tasks == expected_tasks
|
|
), f"Missing tasks: {expected_tasks - final_tasks}"
|
|
assert (
|
|
len(second_deployment_tasks) == num_tasks - tasks_before_restart
|
|
), f"Second deployment processed {len(second_deployment_tasks)} tasks, expected {num_tasks - tasks_before_restart}"
|
|
|
|
def test_task_consumer_as_serve_deployment_with_async_task_handler(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that Celery adapter raises NotImplementedError for async task handlers."""
|
|
processor_config = create_processor_config()
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
def __init__(self):
|
|
self.data_received = None
|
|
self.task_received = False
|
|
|
|
# This async task handler should raise NotImplementedError when registered
|
|
@task_handler(name="process_request")
|
|
async def process_request(self, data):
|
|
self.task_received = True
|
|
self.data_received = data
|
|
|
|
# Error is raised during deployment initialization when Celery adapter
|
|
# tries to register the async handler. The deployment fails with a
|
|
# RuntimeError (the underlying NotImplementedError is logged but wrapped).
|
|
with pytest.raises(RuntimeError):
|
|
serve.run(ServeTaskConsumer.bind())
|
|
|
|
def test_task_consumer_metrics(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that task processor metrics are collected and exposed correctly."""
|
|
processor_config = create_processor_config()
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
def __init__(self):
|
|
self.task_received = False
|
|
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data):
|
|
self.task_received = True
|
|
|
|
def get_task_received(self) -> bool:
|
|
return self.task_received
|
|
|
|
handle = serve.run(ServeTaskConsumer.bind())
|
|
send_request_to_queue.remote(processor_config, "test_data_1")
|
|
|
|
def assert_task_received():
|
|
return handle.get_task_received.remote().result()
|
|
|
|
wait_for_condition(assert_task_received, timeout=20)
|
|
|
|
adapter_instance = instantiate_adapter_from_config(
|
|
task_processor_config=processor_config
|
|
)
|
|
metrics = adapter_instance.get_metrics_sync()
|
|
|
|
assert len(metrics) == 1
|
|
worker_name = next(iter(metrics))
|
|
worker_stats = metrics[worker_name]
|
|
|
|
# Check that the total number of processed tasks is correct.
|
|
assert worker_stats["pool"]["threads"] == 1
|
|
assert worker_stats["pool"]["max-concurrency"] == 1
|
|
assert worker_stats["total"]["process_request"] == 1
|
|
assert worker_stats["broker"]["transport"] == "filesystem"
|
|
|
|
def test_task_consumer_health_check(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that the health check for the task processor works correctly."""
|
|
processor_config = create_processor_config()
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
pass
|
|
|
|
serve.run(ServeTaskConsumer.bind())
|
|
|
|
adapter_instance = instantiate_adapter_from_config(
|
|
task_processor_config=processor_config
|
|
)
|
|
|
|
def check_health():
|
|
health_status = adapter_instance.health_check_sync()
|
|
return len(health_status) > 0
|
|
|
|
# Wait for the worker to be ready
|
|
wait_for_condition(check_health, timeout=20)
|
|
|
|
health_status = adapter_instance.health_check_sync()
|
|
assert len(health_status) == 1
|
|
|
|
worker_reply = health_status[0]
|
|
assert len(worker_reply) == 1
|
|
worker_name = next(iter(worker_reply))
|
|
assert worker_reply[worker_name] == {"ok": "pong"}
|
|
|
|
def test_task_processor_with_cancel_tasks_and_app_custom_config(
|
|
self, external_redis, serve_instance # noqa: F811
|
|
):
|
|
"""Test the cancel task functionality with celery broker."""
|
|
redis_address = os.environ.get("RAY_REDIS_ADDRESS")
|
|
|
|
processor_config = TaskProcessorConfig(
|
|
queue_name="my_app_queue",
|
|
adapter_config=CeleryAdapterConfig(
|
|
broker_url=f"redis://{redis_address}/0",
|
|
backend_url=f"redis://{redis_address}/1",
|
|
app_custom_config={"worker_prefetch_multiplier": 1},
|
|
),
|
|
)
|
|
|
|
signal = SignalActor.remote()
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class MyTaskConsumer:
|
|
def __init__(self, signal_actor):
|
|
self._signal = signal_actor
|
|
self.message_received = []
|
|
|
|
@task_handler(name="process")
|
|
def process(self, data):
|
|
ray.get(self._signal.wait.remote())
|
|
self.message_received.append(data)
|
|
|
|
def get_message_received(self):
|
|
return self.message_received
|
|
|
|
handle = serve.run(MyTaskConsumer.bind(signal), name="app_v1")
|
|
|
|
task_ids = []
|
|
for i in range(2):
|
|
task_id_ref = send_request_to_queue.remote(
|
|
processor_config, f"test_data_{i}", task_name="process"
|
|
)
|
|
task_ids.append(ray.get(task_id_ref))
|
|
|
|
wait_for_condition(
|
|
lambda: ray.get(signal.cur_num_waiters.remote()) == 1, timeout=10
|
|
)
|
|
|
|
adapter_instance = instantiate_adapter_from_config(
|
|
task_processor_config=processor_config
|
|
)
|
|
adapter_instance.cancel_task_sync(task_ids[1])
|
|
|
|
ray.get(signal.send.remote())
|
|
|
|
def check_revoked():
|
|
status = adapter_instance.get_task_status_sync(task_ids[1])
|
|
return status.status == "REVOKED"
|
|
|
|
wait_for_condition(check_revoked, timeout=20)
|
|
|
|
assert "test_data_0" in handle.get_message_received.remote().result()
|
|
assert "test_data_1" not in handle.get_message_received.remote().result()
|
|
|
|
serve.delete("app_v1")
|
|
|
|
def test_task_consumer_with_task_custom_config(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that task consumer works with app custom config."""
|
|
processor_config = create_processor_config()
|
|
processor_config.adapter_config.task_custom_config = {
|
|
"retry_backoff_max": 1,
|
|
"retry_kwargs": {"max_retries": 10},
|
|
}
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
def __init__(self):
|
|
self.num_calls = 0
|
|
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data):
|
|
self.num_calls += 1
|
|
raise ValueError("Task failed as expected")
|
|
|
|
def get_num_calls(self):
|
|
return self.num_calls
|
|
|
|
handle = serve.run(ServeTaskConsumer.bind())
|
|
|
|
send_request_to_queue.remote(processor_config, "test_data_0")
|
|
|
|
wait_for_condition(
|
|
lambda: handle.get_num_calls.remote().result() == 11, timeout=20
|
|
)
|
|
|
|
def test_task_consumer_failed_task_queue_consumption(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that failed tasks can be consumed from the failed task queue with the correct arguments."""
|
|
# Create first processor config with failed task queue
|
|
failed_queue_name = "failed_task_queue"
|
|
failing_processor_config = create_processor_config(
|
|
failed_task_queue_name=failed_queue_name
|
|
)
|
|
|
|
# Create second processor config that consumes from the failed queue
|
|
failed_processor_config = create_processor_config()
|
|
failed_processor_config.queue_name = failed_queue_name
|
|
|
|
# First consumer that always fails
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=failing_processor_config)
|
|
class FailingTaskConsumer:
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data):
|
|
raise ValueError("Test error message from first consumer")
|
|
|
|
# Second consumer that processes failed tasks
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=failed_processor_config)
|
|
class FailedTaskConsumer:
|
|
def __init__(self):
|
|
self.received_error = None
|
|
self.received_task_id = None
|
|
self.received_original_args = None
|
|
|
|
@task_handler(name="process_request")
|
|
def process_request(self, task_id, exception_msg, args, kwargs, einfo):
|
|
self.received_task_id = task_id
|
|
self.received_error = exception_msg
|
|
self.received_original_args = args
|
|
|
|
def get_received_error(self):
|
|
return self.received_error
|
|
|
|
def get_received_task_id(self):
|
|
return self.received_task_id
|
|
|
|
def get_received_original_args(self):
|
|
return self.received_original_args
|
|
|
|
# Deploy both consumers
|
|
serve.run(
|
|
FailingTaskConsumer.bind(),
|
|
name="failing_task_consumer",
|
|
route_prefix="/failing_task_consumer",
|
|
)
|
|
handle_2 = serve.run(
|
|
FailedTaskConsumer.bind(),
|
|
name="failed_task_consumer",
|
|
route_prefix="/failed_task_consumer",
|
|
)
|
|
|
|
# Send a task to the first consumer (which will fail)
|
|
task_id = send_request_to_queue.remote(failing_processor_config, "test_data_1")
|
|
|
|
# Verify the received data
|
|
def assert_failed_task_received():
|
|
received_error = handle_2.get_received_error.remote().result()
|
|
received_task_id = handle_2.get_received_task_id.remote().result()
|
|
received_original_args = (
|
|
handle_2.get_received_original_args.remote().result()
|
|
)
|
|
|
|
args_data = "['test_data_1']"
|
|
err_msg = "ValueError: Test error message from first consumer"
|
|
|
|
assert err_msg in received_error
|
|
assert received_task_id == ray.get(task_id)
|
|
assert received_original_args == args_data
|
|
|
|
return True
|
|
|
|
wait_for_condition(assert_failed_task_received, timeout=20)
|
|
|
|
def test_multiple_task_consumers_in_single_app(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that multiple task consumers can coexist in a single Ray Serve application."""
|
|
orchestrator_config = create_processor_config()
|
|
orchestrator_config.queue_name = "orchestrator_queue"
|
|
|
|
worker_config = create_processor_config()
|
|
worker_config.queue_name = "worker_queue"
|
|
|
|
@serve.deployment(name="worker-deployment")
|
|
@task_consumer(task_processor_config=worker_config)
|
|
class WorkerTaskConsumer:
|
|
def __init__(self):
|
|
self.task_count = 0
|
|
|
|
@task_handler(name="process_data")
|
|
def process_data(self, payload):
|
|
self.task_count += 1
|
|
return f"Worker processed: {payload}"
|
|
|
|
def get_worker_task_count(self):
|
|
return self.task_count
|
|
|
|
@serve.deployment(name="orchestrator-deployment")
|
|
@task_consumer(task_processor_config=orchestrator_config)
|
|
class OrchestratorTaskConsumer:
|
|
def __init__(self, worker_deployment):
|
|
self.worker_deployment = worker_deployment
|
|
self.message_received = []
|
|
|
|
@task_handler(name="orchestrate_task")
|
|
def orchestrate_task(self, payload):
|
|
send_request_to_queue.remote(
|
|
worker_config, payload, task_name="process_data"
|
|
)
|
|
self.message_received.append(payload)
|
|
|
|
return f"Orchestrated complex task for payload: {payload}"
|
|
|
|
async def get_worker_task_count(self):
|
|
return await self.worker_deployment.get_worker_task_count.remote()
|
|
|
|
def get_message_received(self):
|
|
return self.message_received
|
|
|
|
worker_deployment = WorkerTaskConsumer.bind()
|
|
orchestrator_deployment = OrchestratorTaskConsumer.bind(worker_deployment)
|
|
|
|
handle = serve.run(orchestrator_deployment, name="multi_consumer_app")
|
|
|
|
num_tasks_to_send = 3
|
|
data_sent_to_orchestrator = []
|
|
for i in range(num_tasks_to_send):
|
|
data_id = f"data_{i}"
|
|
send_request_to_queue.remote(
|
|
orchestrator_config, data_id, task_name="orchestrate_task"
|
|
)
|
|
data_sent_to_orchestrator.append(data_id)
|
|
|
|
# Wait for tasks to be processed properly
|
|
def check_data_processed_properly():
|
|
worker_count = handle.get_worker_task_count.remote().result()
|
|
data_received_by_orchestrator = (
|
|
handle.get_message_received.remote().result()
|
|
)
|
|
|
|
return worker_count == num_tasks_to_send and set(
|
|
data_received_by_orchestrator
|
|
) == set(data_sent_to_orchestrator)
|
|
|
|
wait_for_condition(check_data_processed_properly, timeout=300)
|
|
|
|
def test_task_consumer_with_one_queue_and_multiple_different_tasks(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that task consumers can handle multiple different tasks in the same queue."""
|
|
processor_config = create_processor_config()
|
|
|
|
@serve.deployment
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class MyTaskConsumer:
|
|
def __init__(self):
|
|
self.message_received = []
|
|
|
|
@task_handler(name="process_data")
|
|
def process_data(self, data):
|
|
self.message_received.append(data)
|
|
|
|
@task_handler(name="process_data2")
|
|
def process_data2(self, data):
|
|
self.message_received.append(data)
|
|
|
|
def get_message_received(self):
|
|
return self.message_received
|
|
|
|
handle = serve.run(MyTaskConsumer.bind())
|
|
|
|
send_request_to_queue.remote(
|
|
processor_config, "test_data_1", task_name="process_data"
|
|
)
|
|
send_request_to_queue.remote(
|
|
processor_config, "test_data_2", task_name="process_data2"
|
|
)
|
|
send_request_to_queue.remote(
|
|
processor_config, "test_data_3", task_name="process_data"
|
|
)
|
|
|
|
wait_for_condition(
|
|
lambda: "test_data_1" in handle.get_message_received.remote().result()
|
|
)
|
|
wait_for_condition(
|
|
lambda: "test_data_2" in handle.get_message_received.remote().result()
|
|
)
|
|
wait_for_condition(
|
|
lambda: "test_data_3" in handle.get_message_received.remote().result()
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
|
|
class TestTaskConsumerWithDLQsConfiguration:
|
|
"""Test task consumer with dead letter queues."""
|
|
|
|
def _assert_queue_counts(
|
|
self,
|
|
temp_queue_directory,
|
|
processor_config,
|
|
expected_main=0,
|
|
expected_unprocessable=0,
|
|
expected_failed=0,
|
|
timeout=15,
|
|
):
|
|
"""Helper to assert expected task counts in different queues."""
|
|
|
|
def check_counts():
|
|
queue_path = Path(temp_queue_directory["queue_path"])
|
|
counts = _get_task_counts_by_routing_key(queue_path)
|
|
|
|
main_count = counts.get(processor_config.queue_name, 0)
|
|
unprocessable_count = counts.get(
|
|
getattr(processor_config, "unprocessable_task_queue_name", ""), 0
|
|
)
|
|
failed_count = counts.get(
|
|
getattr(processor_config, "failed_task_queue_name", ""), 0
|
|
)
|
|
|
|
return (
|
|
main_count == expected_main
|
|
and unprocessable_count == expected_unprocessable
|
|
and failed_count == expected_failed
|
|
)
|
|
|
|
wait_for_condition(check_counts, timeout=timeout)
|
|
|
|
def test_task_consumer_as_serve_deployment_with_unknown_task(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that unknown tasks are sent to the unprocessable task queue."""
|
|
processor_config = create_processor_config(
|
|
unprocessable_task_queue_name="unprocessable_task_queue"
|
|
)
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data):
|
|
pass
|
|
|
|
serve.run(ServeTaskConsumer.bind())
|
|
|
|
# Send a task with an unknown name
|
|
send_request_to_queue.remote(
|
|
processor_config, "test_data_1", task_name="unregistered_task"
|
|
)
|
|
|
|
self._assert_queue_counts(
|
|
temp_queue_directory,
|
|
processor_config,
|
|
expected_main=0,
|
|
expected_unprocessable=1,
|
|
timeout=10,
|
|
)
|
|
|
|
def test_task_consumer_as_serve_deployment_with_failed_task_and_dead_letter_queue(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that failed tasks are sent to the failed task queue."""
|
|
processor_config = create_processor_config(
|
|
failed_task_queue_name="failed_task_queue"
|
|
)
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data):
|
|
raise ValueError("Task failed as expected")
|
|
|
|
serve.run(ServeTaskConsumer.bind())
|
|
send_request_to_queue.remote(processor_config, "test_data_1")
|
|
|
|
self._assert_queue_counts(
|
|
temp_queue_directory, processor_config, expected_main=0, expected_failed=1
|
|
)
|
|
|
|
def test_task_consumer_with_mismatched_arguments(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that tasks with mismatched arguments are sent to the unprocessable task queue."""
|
|
processor_config = create_processor_config(
|
|
unprocessable_task_queue_name="unprocessable_task_queue",
|
|
failed_task_queue_name="failed_task_queue",
|
|
)
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
@task_handler(name="process_request")
|
|
def process_request(self, arg1, arg2): # Expects two arguments
|
|
pass
|
|
|
|
serve.run(ServeTaskConsumer.bind())
|
|
|
|
# Send a task with only one argument, which should cause a TypeError
|
|
send_request_to_queue.remote(processor_config, ["test_data_1"])
|
|
|
|
self._assert_queue_counts(
|
|
temp_queue_directory,
|
|
processor_config,
|
|
expected_main=0,
|
|
expected_failed=1,
|
|
)
|
|
|
|
def test_task_consumer_with_argument_type_mismatch(
|
|
self, temp_queue_directory, serve_instance, create_processor_config
|
|
):
|
|
"""Test that tasks with argument type mismatches are sent to the unprocessable task queue."""
|
|
processor_config = create_processor_config(
|
|
unprocessable_task_queue_name="unprocessable_task_queue",
|
|
failed_task_queue_name="failed_task_queue",
|
|
)
|
|
|
|
@serve.deployment(max_ongoing_requests=1)
|
|
@task_consumer(task_processor_config=processor_config)
|
|
class ServeTaskConsumer:
|
|
@task_handler(name="process_request")
|
|
def process_request(self, data: str):
|
|
return len(data) # This will fail if data is not a sequence
|
|
|
|
serve.run(ServeTaskConsumer.bind())
|
|
|
|
# Send an integer, for which len() is undefined, causing a TypeError
|
|
send_request_to_queue.remote(processor_config, 12345)
|
|
|
|
self._assert_queue_counts(
|
|
temp_queue_directory,
|
|
processor_config,
|
|
expected_main=0,
|
|
expected_failed=1,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main(["-v", "-s", __file__]))
|