ray-project--ray/python/ray/serve/tests/test_custom_autoscaling_metrics.py

import asyncio
import sys
from typing import Dict

import pytest

import ray
from ray import serve
from ray._common.test_utils import SignalActor, wait_for_condition
from ray.serve._private.common import DeploymentID
from ray.serve._private.test_utils import check_num_replicas_eq
from ray.serve.config import AutoscalingContext, AutoscalingPolicy


def get_autoscaling_metrics_from_controller(
    client, deployment_id: DeploymentID
) -> Dict[str, float]:
    """Get autoscaling metrics from the controller for testing."""
    ref = client._controller._get_metrics_for_deployment_for_testing.remote(
        deployment_id
    )
    return ray.get(ref)


def custom_autoscaling_policy(ctx: AutoscalingContext):
    aggregated_counter = sum(
        x for x in ctx.aggregated_metrics.get("counter", {}).values()
    )
    max_counter = sum(
        [x[-1].value for x in ctx.raw_metrics.get("counter", {}).values()]
    )
    if max_counter == aggregated_counter == 10:
        return 3, {}
    else:
        return 1, {}


# Example from doc/source/serve/doc_code/autoscaling_policy.py
def max_cpu_usage_autoscaling_policy(ctx: AutoscalingContext):
    cpu_usage_metric = ctx.aggregated_metrics.get("cpu_usage", {})
    max_cpu_usage = list(cpu_usage_metric.values())[-1] if cpu_usage_metric else 0

    if max_cpu_usage > 80:
        return min(ctx.capacity_adjusted_max_replicas, ctx.current_num_replicas + 1), {}
    elif max_cpu_usage < 30:
        return max(ctx.capacity_adjusted_min_replicas, ctx.current_num_replicas - 1), {}
    else:
        return ctx.current_num_replicas, {}


class TestCustomServeMetrics:
    """Check that redeploying a deployment doesn't reset its start time."""

    def test_custom_serve_metrics(self, serve_instance):
        @serve.deployment(
            autoscaling_config={
                "min_replicas": 1,
                "max_replicas": 5,
                "upscale_delay_s": 0.5,
                "downscale_delay_s": 0.5,
                "metrics_interval_s": 0.1,
                "look_back_period_s": 1,
            }
        )
        class DummyMetricIncrementer:
            def __init__(self):
                self.counter = 0

            async def __call__(self) -> str:
                self.counter += 1
                return "Hello, world"

            def record_autoscaling_stats(self) -> Dict[str, int]:
                # Increments each time the deployment has been called
                return {"counter": self.counter}

        app_name = "test_custom_metrics_app"
        handle = serve.run(
            DummyMetricIncrementer.bind(), name=app_name, route_prefix="/"
        )
        dep_id = DeploymentID(name="DummyMetricIncrementer", app_name=app_name)

        # Call deployment 3 times
        [handle.remote() for _ in range(3)]

        def check_counter_value():
            metrics = get_autoscaling_metrics_from_controller(serve_instance, dep_id)
            return "counter" in metrics and metrics["counter"][-1][0].value == 3

        # The final counter value recorded by the controller should be 3
        wait_for_condition(
            check_counter_value,
            timeout=15,
        )

    def test_custom_serve_timeout(self, serve_instance):
        @serve.deployment(
            autoscaling_config={
                "min_replicas": 1,
                "max_replicas": 5,
                "upscale_delay_s": 2,
                "downscale_delay_s": 10,
                "metrics_interval_s": 1,
                "look_back_period_s": 2,
            }
        )
        class DummyMetricTimeout:
            def __init__(self):
                self.counter = 0

            async def __call__(self) -> str:
                self.counter += 1
                return "Hello, world"

            async def record_autoscaling_stats(self) -> Dict[str, int]:
                # Block here until it is forced to cancel due to timeout beyond RAY_SERVE_RECORD_AUTOSCALING_STATS_TIMEOUT_S
                await asyncio.sleep(1000)

        app_name = "test_custom_metrics_app"
        handle = serve.run(DummyMetricTimeout.bind(), name=app_name, route_prefix="/")
        dep_id = DeploymentID(name="DummyMetricTimeout", app_name=app_name)
        # Call deployment 3 times
        [handle.remote() for _ in range(3)]
        # There should be no counter metric because asyncio timeout would have stopped the method execution
        metrics = get_autoscaling_metrics_from_controller(serve_instance, dep_id)
        assert metrics.get("counter", None) is None

    def test_custom_serve_invalid_metric_type(self, serve_instance):
        @serve.deployment(
            autoscaling_config={
                "min_replicas": 1,
                "max_replicas": 5,
                "upscale_delay_s": 2,
                "downscale_delay_s": 10,
                "metrics_interval_s": 1,
                "look_back_period_s": 2,
            }
        )
        class DummyInvalidMetric:
            def __init__(self):
                self.counter = 0

            async def __call__(self) -> str:
                self.counter += 1
                return "Hello, world"

            def record_autoscaling_stats(self) -> Dict[str, str]:
                # Return an invalid metric dict whose valuse are neither int nor float
                return {"counter": "not_an_int"}

        app_name = "test_custom_metrics_app"
        handle = serve.run(DummyInvalidMetric.bind(), name=app_name, route_prefix="/")
        dep_id = DeploymentID(name="DummyInvalidMetric", app_name=app_name)
        # Call deployment 3 times
        [handle.remote() for _ in range(3)]
        # There should be no counter metric because it failed validation, must be int or float
        metrics = get_autoscaling_metrics_from_controller(serve_instance, dep_id)
        assert metrics.get("counter", None) is None

    def test_policy_using_custom_metrics(self, serve_instance):
        signal = SignalActor.remote()

        @serve.deployment(
            autoscaling_config={
                "min_replicas": 1,
                "max_replicas": 5,
                "upscale_delay_s": 2,
                "downscale_delay_s": 1,
                "metrics_interval_s": 0.1,
                "look_back_period_s": 1,
                "target_ongoing_requests": 10,
                "policy": AutoscalingPolicy(policy_function=custom_autoscaling_policy),
            },
            max_ongoing_requests=100,
        )
        class CustomMetricsDeployment:
            def __init__(self):
                self.counter = 0

            async def __call__(self) -> str:
                self.counter += 1
                await signal.wait.remote()
                return "Hello, world"

            def record_autoscaling_stats(self) -> Dict[str, int]:
                return {"counter": self.counter}

        handle = serve.run(CustomMetricsDeployment.bind())
        [handle.remote() for _ in range(10)]
        wait_for_condition(lambda: ray.get(signal.cur_num_waiters.remote()) == 10)
        wait_for_condition(
            check_num_replicas_eq, name="CustomMetricsDeployment", target=3
        )
        signal.send.remote()

    def test_max_cpu_usage_autoscaling_policy(self, serve_instance):
        """Test autoscaling policy based on max CPU usage from documentation example."""
        signal = SignalActor.remote()

        @serve.deployment(
            autoscaling_config={
                "min_replicas": 1,
                "max_replicas": 5,
                "upscale_delay_s": 0.5,
                "downscale_delay_s": 0.5,
                "metrics_interval_s": 0.1,
                "look_back_period_s": 1,
                "target_ongoing_requests": 10,
                "policy": AutoscalingPolicy(
                    policy_function=max_cpu_usage_autoscaling_policy
                ),
            },
            max_ongoing_requests=100,
        )
        class MaxCpuUsageDeployment:
            def __init__(self):
                self.cpu_usage = 0

            async def __call__(self) -> str:
                self.cpu_usage += 1
                await signal.wait.remote()
                return "Hello, world"

            def record_autoscaling_stats(self) -> Dict[str, int]:
                return {"cpu_usage": self.cpu_usage}

        handle = serve.run(MaxCpuUsageDeployment.bind())

        # Test scale up when CPU usage > 80
        # Set CPU usage to 90 to trigger scale up
        dep_id = DeploymentID(name="MaxCpuUsageDeployment")

        # Send requests to increase CPU usage
        [handle.remote() for _ in range(90)]
        wait_for_condition(lambda: ray.get(signal.cur_num_waiters.remote()) == 90)

        # Wait for metrics to be recorded and policy to trigger scale up
        def check_scale_up():
            metrics = get_autoscaling_metrics_from_controller(serve_instance, dep_id)
            return "cpu_usage" in metrics and metrics["cpu_usage"][-1][0].value >= 90

        wait_for_condition(check_scale_up, timeout=10)

        # Should scale up to 2 replicas due to high CPU usage
        wait_for_condition(
            check_num_replicas_eq, name="MaxCpuUsageDeployment", target=2, timeout=15
        )

        # Release signal and test scale down when CPU usage < 30
        signal.send.remote()
        wait_for_condition(lambda: ray.get(signal.cur_num_waiters.remote()) == 0)

        signal = SignalActor.remote()
        # Reset CPU usage to low value by creating new deployment instance
        # This simulates low CPU usage scenario
        @serve.deployment(
            autoscaling_config={
                "min_replicas": 1,
                "max_replicas": 5,
                "upscale_delay_s": 0.5,
                "downscale_delay_s": 0.5,
                "metrics_interval_s": 0.1,
                "look_back_period_s": 1,
                "target_ongoing_requests": 10,
                "policy": AutoscalingPolicy(
                    policy_function=max_cpu_usage_autoscaling_policy
                ),
            },
            max_ongoing_requests=100,
        )
        class LowCpuUsageDeployment:
            def __init__(self):
                self.cpu_usage = 0

            async def __call__(self) -> str:
                self.cpu_usage += 1
                await signal.wait.remote()
                return "Hello, world"

            def record_autoscaling_stats(self) -> Dict[str, int]:
                # Return low CPU usage to trigger scale down
                return {"cpu_usage": 20}

        handle = serve.run(LowCpuUsageDeployment.bind())

        # Send a few requests to establish low CPU usage
        [handle.remote() for _ in range(5)]
        wait_for_condition(lambda: ray.get(signal.cur_num_waiters.remote()) == 5)

        # Wait for metrics to be recorded
        dep_id_low = DeploymentID(name="LowCpuUsageDeployment")

        def check_low_cpu():
            metrics = get_autoscaling_metrics_from_controller(
                serve_instance, dep_id_low
            )
            return "cpu_usage" in metrics and metrics["cpu_usage"][-1][0].value <= 30

        wait_for_condition(check_low_cpu, timeout=10)

        # Should downscale to 1 replica due to low CPU usage
        wait_for_condition(
            check_num_replicas_eq, name="LowCpuUsageDeployment", target=1, timeout=15
        )

        signal.send.remote()

    def test_record_autoscaling_stats_without_autoscaling_config(self, serve_instance):
        """Test that record_autoscaling_stats doesn't crash when using num_replicas instead of autoscaling_config.

        When a deployment defines record_autoscaling_stats but uses fixed num_replicas,
        the replica should start successfully without crashing.
        """

        @serve.deployment(num_replicas=1)
        class DeploymentWithCustomMetricsNoAutoscaling:
            async def record_autoscaling_stats(self) -> Dict[str, float]:
                return {"qps": 1.0}

            async def __call__(self):
                return "ok"

        app_name = "test_custom_metrics_no_autoscaling"
        handle = serve.run(
            DeploymentWithCustomMetricsNoAutoscaling.bind(),
            name=app_name,
            route_prefix="/test_no_autoscaling",
        )

        response = handle.remote().result()
        assert response == "ok"


if __name__ == "__main__":
    sys.exit(pytest.main(["-v", "-s", __file__]))