ray-project--ray/python/ray/serve/tests/test_standalone.py

"""
The test file for all standalone tests that doesn't
requires a shared Serve instance.
"""

import asyncio
import logging
import os
import random
import socket
import sys
import time

import httpx
import pytest

import ray
import ray._private.state as state
from ray import serve
from ray._common.test_utils import run_string_as_driver, wait_for_condition
from ray._raylet import GcsClient
from ray.cluster_utils import Cluster, cluster_not_supported
from ray.serve._private.api import serve_start_async
from ray.serve._private.constants import (
    RAY_SERVE_ENABLE_HA_PROXY,
    SERVE_DEFAULT_APP_NAME,
    SERVE_NAMESPACE,
    SERVE_PROXY_NAME,
)
from ray.serve._private.default_impl import create_cluster_node_info_cache
from ray.serve._private.http_util import set_socket_reuse_port
from ray.serve._private.test_utils import expected_proxy_actors
from ray.serve._private.utils import block_until_http_ready, format_actor_name
from ray.serve.config import (
    ControllerOptions,
    GangSchedulingConfig,
    HTTPOptions,
    ProxyLocation,
)
from ray.serve.context import _get_global_client
from ray.serve.schema import ServeApplicationSchema, ServeDeploySchema
from ray.util.state import list_actors


def _get_random_port() -> int:
    return random.randint(10000, 65535)


@pytest.fixture
def ray_cluster():
    if cluster_not_supported:
        pytest.skip("Cluster not supported")
    cluster = Cluster()
    yield Cluster()
    serve.shutdown()
    ray.shutdown()
    cluster.shutdown()


@pytest.fixture()
def lower_slow_startup_threshold_and_reset():
    original_slow_startup_warning_s = os.environ.get("SERVE_SLOW_STARTUP_WARNING_S")
    original_slow_startup_warning_period_s = os.environ.get(
        "SERVE_SLOW_STARTUP_WARNING_PERIOD_S"
    )
    # Lower slow startup warning threshold to 1 second to reduce test duration
    os.environ["SERVE_SLOW_STARTUP_WARNING_S"] = "1"
    os.environ["SERVE_SLOW_STARTUP_WARNING_PERIOD_S"] = "1"

    ray.init(num_cpus=2)
    serve.start()
    client = _get_global_client()

    yield client

    serve.shutdown()
    ray.shutdown()

    # Reset slow startup warning threshold to prevent state sharing across unit
    # tests
    if original_slow_startup_warning_s is not None:
        os.environ["SERVE_SLOW_STARTUP_WARNING_S"] = original_slow_startup_warning_s
    if original_slow_startup_warning_period_s is not None:
        os.environ[
            "SERVE_SLOW_STARTUP_WARNING_PERIOD_S"
        ] = original_slow_startup_warning_period_s


def test_deployment(ray_cluster):
    # https://github.com/ray-project/ray/issues/11437

    cluster = ray_cluster
    head_node = cluster.add_node(num_cpus=6)

    # Create first job, check we can run a simple serve endpoint
    ray.init(head_node.address, namespace=SERVE_NAMESPACE)
    first_job_id = ray.get_runtime_context().get_job_id()
    serve.start()

    @serve.deployment
    def f(*args):
        return "from_f"

    handle = serve.run(f.bind(), name="f", route_prefix="/say_hi_f")
    assert handle.remote().result() == "from_f"
    assert httpx.get("http://localhost:8000/say_hi_f").text == "from_f"

    serve.context._global_client = None
    ray.shutdown()

    # Create the second job, make sure we can still create new deployments.
    ray.init(head_node.address, namespace="serve")
    assert ray.get_runtime_context().get_job_id() != first_job_id

    @serve.deployment
    def g(*args):
        return "from_g"

    handle = serve.run(g.bind(), name="g", route_prefix="/say_hi_g")
    assert handle.remote().result() == "from_g"
    assert httpx.get("http://localhost:8000/say_hi_g").text == "from_g"
    assert httpx.get("http://localhost:8000/say_hi_f").text == "from_f"


def test_connect(ray_shutdown):
    # Check that you can make API calls from within a deployment.
    ray.init(num_cpus=8, namespace="serve")
    serve.start()

    @serve.deployment
    def connect_in_deployment(*args):
        serve.run(
            connect_in_deployment.options(name="deployment-ception").bind(),
            name="app2",
            route_prefix="/app2",
        )

    handle = serve.run(connect_in_deployment.bind())
    handle.remote().result()
    assert "deployment-ception" in serve.status().applications["app2"].deployments


def test_set_socket_reuse_port():
    sock = socket.socket()
    if hasattr(socket, "SO_REUSEPORT"):
        # If the flag exists, we should be able to to use it
        assert set_socket_reuse_port(sock)
    elif sys.platform == "linux":
        # If the flag doesn't exist, but we are only mordern version
        # of linux, we should be able to force set this flag.
        assert set_socket_reuse_port(sock)
    else:
        # Otherwise, it should graceful fail without exception.
        assert not set_socket_reuse_port(sock)


def _reuse_port_is_available():
    sock = socket.socket()
    return set_socket_reuse_port(sock)


@pytest.mark.skipif(
    not _reuse_port_is_available(),
    reason=(
        "Port sharing only works on newer verion of Linux. "
        "This test can only be ran when port sharing is supported."
    ),
)
def test_multiple_routers(ray_cluster):
    cluster = ray_cluster
    head_node = cluster.add_node(num_cpus=4)
    cluster.add_node(num_cpus=4)

    ray.init(head_node.address)
    assert len(ray.nodes()) == 2
    serve.start(http_options=dict(port=8005, location="EveryNode"))

    @serve.deployment(
        num_replicas=2,
        ray_actor_options={"num_cpus": 3},
    )
    class A:
        def __call__(self, *args):
            return "hi"

    serve.run(A.bind())

    gcs_client = GcsClient(address=ray.get_runtime_context().gcs_address)
    cluster_node_info_cache = create_cluster_node_info_cache(gcs_client)
    cluster_node_info_cache.update()

    def get_proxy_names():
        proxy_names = []
        for node_id in cluster_node_info_cache.get_alive_node_ids():
            proxy_names.append(
                format_actor_name(
                    SERVE_PROXY_NAME,
                    node_id,
                )
            )
        return proxy_names

    wait_for_condition(lambda: len(get_proxy_names()) == 2)
    original_proxy_names = get_proxy_names()

    # Two actors should be started.
    def get_first_two_actors():
        try:
            ray.get_actor(original_proxy_names[0], namespace=SERVE_NAMESPACE)
            ray.get_actor(original_proxy_names[1], namespace=SERVE_NAMESPACE)
            return True
        except ValueError:
            return False

    wait_for_condition(get_first_two_actors)

    # Wait for the actors to come up.
    ray.get(block_until_http_ready.remote("http://127.0.0.1:8005/-/routes"))

    # Kill one of the servers, the HTTP server should still function.
    ray.kill(
        ray.get_actor(get_proxy_names()[0], namespace=SERVE_NAMESPACE), no_restart=True
    )
    ray.get(block_until_http_ready.remote("http://127.0.0.1:8005/-/routes"))

    # Add a new node to the cluster. This should trigger a new router to get
    # started.
    new_node = cluster.add_node(num_cpus=4)
    cluster_node_info_cache.update()

    wait_for_condition(lambda: len(get_proxy_names()) == 3)
    (third_proxy,) = set(get_proxy_names()) - set(original_proxy_names)

    serve.run(A.options(num_replicas=3).bind())

    def get_third_actor():
        try:
            ray.get_actor(third_proxy, namespace=SERVE_NAMESPACE)
            return True
        # IndexErrors covers when cluster resources aren't updated yet.
        except (IndexError, ValueError):
            return False

    wait_for_condition(get_third_actor)

    # Remove the newly-added node from the cluster. The corresponding actor
    # should be removed as well.
    cluster.remove_node(new_node)
    cluster_node_info_cache.update()

    def third_actor_removed():
        try:
            ray.get_actor(third_proxy, namespace=SERVE_NAMESPACE)
            return False
        except ValueError:
            return True

    # Check that the actor is gone and the HTTP server still functions.
    wait_for_condition(third_actor_removed)
    ray.get(block_until_http_ready.remote("http://127.0.0.1:8005/-/routes"))


@pytest.mark.skipif(
    RAY_SERVE_ENABLE_HA_PROXY,
    reason="HAProxy ingress: user HTTP middleware runs on the replica, but the "
    "/-/routes endpoint is served by HAProxy, so middleware-injected headers "
    "are absent there.",
)
def test_middleware(ray_shutdown):
    from starlette.middleware import Middleware
    from starlette.middleware.cors import CORSMiddleware

    port = _get_random_port()
    # `middlewares` in HTTPOptions has been removed; passing it raises an error.
    # Use Serve's FastAPI integration to configure middlewares instead.
    with pytest.raises(ValueError, match="`middlewares` in HTTPOptions"):
        serve.start(
            http_options=dict(
                port=port,
                middlewares=[
                    Middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"])
                ],
            )
        )


@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows")
def test_http_root_path(ray_shutdown):
    @serve.deployment
    def hello():
        return "hello"

    port = _get_random_port()
    root_path = "/serve"
    serve.start(http_options=dict(root_path=root_path, port=port))
    serve.run(hello.bind(), route_prefix="/hello")

    # check routing works as expected
    resp = httpx.get(f"http://127.0.0.1:{port}{root_path}/hello")
    assert resp.status_code == 200
    assert resp.text == "hello"

    # check advertized routes are prefixed correctly
    resp = httpx.get(f"http://127.0.0.1:{port}{root_path}/-/routes")
    assert resp.status_code == 200
    assert resp.json() == {"/hello": "default"}


@pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows")
def test_http_proxy_fail_loudly(ray_shutdown):
    # Test that if the http server fail to start, serve.start should fail.
    with pytest.raises(RuntimeError):
        serve.start(http_options={"host": "bad.ip.address"})


def test_no_http(ray_shutdown):
    # The following should have the same effect.
    # All of these should disable the HTTP proxy.
    options = [
        {"http_options": {"host": None}},
        {"proxy_location": "Disabled"},
        {"http_options": {"location": "NoServer"}},  # deprecated override
    ]

    address = ray.init(num_cpus=8)["address"]
    for i, option in enumerate(options):
        print(f"[{i + 1}/{len(options)}] Running with {option}")
        serve.start(**option)

        # Only controller actor should exist
        live_actors = list_actors(
            address=address,
            filters=[("state", "=", "ALIVE")],
        )
        assert len(live_actors) == 1
        controller = serve.context._global_client._controller
        assert len(ray.get(controller.get_proxies.remote())) == 0

        # Test that the handle still works.
        @serve.deployment
        def hello(*args):
            return "hello"

        handle = serve.run(hello.bind())

        assert handle.remote().result() == "hello"
        serve.shutdown()


def test_http_head_only(ray_cluster):
    cluster = ray_cluster
    head_node = cluster.add_node(num_cpus=4, dashboard_port=_get_random_port())
    cluster.add_node(num_cpus=4)

    ray.init(head_node.address)
    assert len(ray.nodes()) == 2

    serve.start(proxy_location="HeadOnly", http_options={"port": _get_random_port()})

    # Controller and proxy on the head node. Under HAProxy the proxy is the
    # HAProxyManager alongside the fallback ProxyActor, which registers asynchronously.
    expected_classes = {"ServeController", *expected_proxy_actors()}

    def check_head_only_actors():
        actors = list_actors(
            address=head_node.address, filters=[("state", "=", "ALIVE")]
        )
        assert {actor.class_name for actor in actors} == expected_classes
        assert all(actor.node_id == head_node.node_id for actor in actors)
        return True

    wait_for_condition(check_head_only_actors)


def test_instance_in_non_anonymous_namespace(ray_shutdown):
    # Can start instance in non-anonymous namespace.
    ray.init(namespace="foo")
    serve.start()


def test_checkpoint_isolation_namespace(ray_shutdown):
    info = ray.init(namespace="test_namespace1")

    address = info["address"]

    driver_template = """
import ray
from ray import serve

ray.init(address="{address}", namespace="{namespace}")

serve.start(http_options={{"port": {port}}})

@serve.deployment
class A:
    pass

serve.run(A.bind())"""

    run_string_as_driver(
        driver_template.format(address=address, namespace="test_namespace1", port=8000)
    )
    run_string_as_driver(
        driver_template.format(address=address, namespace="test_namespace2", port=8001)
    )


def test_serve_start_different_http_checkpoint_options_warning(
    ray_shutdown, propagate_logs, caplog
):
    logger = logging.getLogger("ray.serve")
    caplog.set_level(logging.WARNING, logger="ray.serve")

    warning_msg = []

    class WarningHandler(logging.Handler):
        def emit(self, record):
            warning_msg.append(self.format(record))

    logger.addHandler(WarningHandler())

    ray.init(namespace="serve-test")
    serve.start()

    # create a different config
    test_http = dict(host="127.1.1.8", port=_get_random_port())

    serve.start(http_options=test_http)

    for test_config, msg in zip([["host", "port"]], warning_msg):
        for test_msg in test_config:
            if "Autoscaling metrics pusher thread" in msg:
                continue
            assert test_msg in msg


def test_recovering_controller_no_redeploy():
    """Ensure controller doesn't redeploy running deployments when recovering."""
    ray_context = ray.init(namespace="x")
    address = ray_context.address_info["address"]
    serve.start()
    client = _get_global_client()

    @serve.deployment
    def f():
        pass

    serve.run(f.bind())

    num_actors = len(list_actors(address, filters=[("state", "=", "ALIVE")]))
    assert num_actors > 0

    pid = ray.get(client._controller.get_pid.remote())

    ray.kill(client._controller, no_restart=False)

    wait_for_condition(lambda: ray.get(client._controller.get_pid.remote()) != pid)

    # Confirm that no new deployment is deployed over the next 5 seconds
    with pytest.raises(RuntimeError):
        wait_for_condition(
            lambda: len(list_actors(address, filters=[("state", "=", "ALIVE")]))
            > num_actors,
            timeout=5,
        )

    serve.shutdown()
    ray.shutdown()


def test_updating_status_message(lower_slow_startup_threshold_and_reset):
    """Check if status message says if a serve deployment has taken a long time"""

    @serve.deployment(
        num_replicas=5,
        ray_actor_options={"num_cpus": 1},
    )
    def f(*args):
        pass

    serve._run(f.bind(), _blocking=False)

    def updating_message():
        deployment_status = (
            serve.status().applications[SERVE_DEFAULT_APP_NAME].deployments["f"]
        )
        message_substring = "more than 1s to be scheduled."
        return (deployment_status.status == "UPDATING") and (
            message_substring in deployment_status.message
        )

    wait_for_condition(updating_message, timeout=20)


def test_gang_updating_status_message(lower_slow_startup_threshold_and_reset):
    @serve.deployment(
        num_replicas=4,
        ray_actor_options={"num_cpus": 1},
        gang_scheduling_config=GangSchedulingConfig(gang_size=2),
    )
    def g(*args):
        pass

    serve._run(g.bind(), _blocking=False)

    def updating_message():
        deployment_status = (
            serve.status().applications[SERVE_DEFAULT_APP_NAME].deployments["g"]
        )
        message_substring = "more than 1s to be scheduled."
        return (deployment_status.status == "UPDATING") and (
            message_substring in deployment_status.message
        )

    wait_for_condition(updating_message, timeout=20)


def test_unhealthy_override_updating_status(lower_slow_startup_threshold_and_reset):
    """
    Check that if status is UNHEALTHY and there is a resource availability
    issue, the status should not change. The issue that caused the deployment to
    be unhealthy should be prioritized over this resource availability issue.
    """

    @serve.deployment
    class f:
        def __init__(self):
            self.num = 1 / 0

        def __call__(self, request):
            pass

    serve._run(f.bind(), _blocking=False)

    wait_for_condition(
        lambda: serve.status()
        .applications[SERVE_DEFAULT_APP_NAME]
        .deployments["f"]
        .status
        == "DEPLOY_FAILED",
        timeout=20,
    )

    with pytest.raises(RuntimeError):
        wait_for_condition(
            lambda: serve.status()
            .applications[SERVE_DEFAULT_APP_NAME]
            .deployments["f"]
            .status
            == "UPDATING",
            timeout=10,
        )


@serve.deployment(ray_actor_options={"num_cpus": 0})
class Waiter:
    def __init__(self):
        time.sleep(5)

    def __call__(self, *args):
        return "May I take your order?"


WaiterNode = Waiter.bind()


def test_build_app_task_uses_zero_cpus(ray_shutdown):
    """Check that the task to build an app uses zero CPUs."""
    ray.init(num_cpus=0)
    serve.start()

    _get_global_client().deploy_apps(
        ServeDeploySchema(
            applications=[
                ServeApplicationSchema(
                    name="default",
                    route_prefix="/",
                    import_path="ray.serve.tests.test_standalone.WaiterNode",
                )
            ],
        )
    )

    # If the task required any resources, this would fail.
    wait_for_condition(
        lambda: httpx.get("http://localhost:8000/").text == "May I take your order?"
    )

    serve.shutdown()
    ray.shutdown()


def _deploy_flaky_app(counter_file, fail_count: int):
    os.environ["FLAKY_BUILD_COUNTER_FILE"] = str(counter_file)
    os.environ["FLAKY_BUILD_FAIL_COUNT"] = str(fail_count)
    counter_file.write_text("0")
    ray.init(num_cpus=1)
    serve.start()
    _get_global_client().deploy_apps(
        ServeDeploySchema(
            applications=[
                ServeApplicationSchema(
                    name="flaky_app",
                    route_prefix="/flaky",
                    import_path="ray.serve.tests.test_config_files.flaky_build.node",
                )
            ]
        )
    )


def test_build_app_retries_until_success(ray_shutdown, tmp_path):
    """A flaky build that succeeds on the 4th attempt deploys cleanly."""
    counter_file = tmp_path / "counter.txt"
    try:
        _deploy_flaky_app(counter_file, fail_count=3)
        wait_for_condition(
            lambda: serve.status().applications["flaky_app"].status == "RUNNING",
            timeout=60,
        )
        assert int(counter_file.read_text()) == 4
    finally:
        os.environ.pop("FLAKY_BUILD_COUNTER_FILE", None)
        os.environ.pop("FLAKY_BUILD_FAIL_COUNT", None)


def test_build_app_fails_after_retries_exhausted(ray_shutdown, tmp_path):
    """If the build keeps failing, the app status surfaces the user error."""
    counter_file = tmp_path / "counter.txt"
    try:
        _deploy_flaky_app(counter_file, fail_count=10)
        wait_for_condition(
            lambda: serve.status().applications["flaky_app"].status == "DEPLOY_FAILED",
            timeout=60,
        )
        assert "flaky build failure" in serve.status().applications["flaky_app"].message
        assert int(counter_file.read_text()) == 4
    finally:
        os.environ.pop("FLAKY_BUILD_COUNTER_FILE", None)
        os.environ.pop("FLAKY_BUILD_FAIL_COUNT", None)


@pytest.mark.parametrize(
    "options",
    [
        # No proxy_location and no location -> default EveryNode.
        {
            "proxy_location": None,
            "http_options": None,
            "expected": ProxyLocation.EveryNode,
        },
        {
            "proxy_location": None,
            "http_options": {"test": "test"},
            "expected": ProxyLocation.EveryNode,
        },
        # Explicit (deprecated) `location` is a back-compat override.
        {
            "proxy_location": None,
            "http_options": {"location": "NoServer"},
            "expected": ProxyLocation.Disabled,
        },
        # location=None now means "unset" -> defers to proxy_location (EveryNode),
        # NOT Disabled (the old None-means-Disabled overload is gone).
        {
            "proxy_location": None,
            "http_options": {"location": None},
            "expected": ProxyLocation.EveryNode,
        },
        # Bare HTTPOptions() defers to proxy_location -> EveryNode (was HeadOnly).
        {
            "proxy_location": None,
            "http_options": HTTPOptions(),
            "expected": ProxyLocation.EveryNode,
        },
        {
            "proxy_location": "Disabled",
            "http_options": None,
            "expected": ProxyLocation.Disabled,
        },
        {
            "proxy_location": "Disabled",
            "http_options": {},
            "expected": ProxyLocation.Disabled,
        },
        {
            "proxy_location": "Disabled",
            "http_options": {"host": "foobar"},
            "expected": ProxyLocation.Disabled,
        },
        # Explicit `location` overrides `proxy_location`.
        {
            "proxy_location": "Disabled",
            "http_options": {"location": "HeadOnly"},
            "expected": ProxyLocation.HeadOnly,
        },
    ],
)
def test_serve_start_proxy_location(ray_shutdown, options):
    expected = options.pop("expected")
    serve.start(**options)
    client = _get_global_client()
    assert client.get_serve_details()["proxy_location"] == expected


@pytest.mark.parametrize(
    "controller_options",
    [
        ControllerOptions(
            runtime_env={
                "env_vars": {
                    "RAY_SERVE_TEST_CONTROLLER_ENV": "from-model",
                    "RAY_SERVE_TEST_CONTROLLER_ENV_2": "second",
                }
            }
        ),
        # Same options passed as a plain dict -- the API coerces it
        # through ``ControllerOptions.model_validate``.
        {
            "runtime_env": {
                "env_vars": {
                    "RAY_SERVE_TEST_CONTROLLER_ENV": "from-model",
                    "RAY_SERVE_TEST_CONTROLLER_ENV_2": "second",
                }
            }
        },
    ],
)
def test_serve_start_controller_options(ray_shutdown, controller_options):
    """``ControllerOptions.runtime_env.env_vars`` lands on the controller actor.

    Uses a custom RAY_SERVE_TEST_CONTROLLER_ENV var (not a real Serve knob)
    so the assertion is decoupled from whatever the Anyscale env hook
    auto-injects. The merge semantics are the env_hook's contract; this
    test only asserts that *our* requested env_vars made it through.
    """
    serve.start(controller_options=controller_options)
    client = _get_global_client()

    # Reach into the controller actor to read its own os.environ; the
    # controller is a singleton named actor on the head node, so a remote
    # task on the same handle runs in the same process.
    def _read_env(self, *, keys):
        import os as _os

        return {k: _os.environ.get(k) for k in keys}

    env_seen = ray.get(
        client._controller.__ray_call__.remote(
            _read_env,
            keys=[
                "RAY_SERVE_TEST_CONTROLLER_ENV",
                "RAY_SERVE_TEST_CONTROLLER_ENV_2",
            ],
        )
    )
    assert env_seen["RAY_SERVE_TEST_CONTROLLER_ENV"] == "from-model"
    assert env_seen["RAY_SERVE_TEST_CONTROLLER_ENV_2"] == "second"


def test_serve_start_controller_options_rejects_disallowed_runtime_env(
    ray_shutdown,
):
    """Bad runtime_env fails at the caller, not from a Ray task."""
    from pydantic import ValidationError

    with pytest.raises(ValidationError) as exc:
        serve.start(controller_options={"runtime_env": {"pip": ["numpy"]}})
    assert "only supports ['env_vars']" in str(exc.value)


def test_serve_start_does_not_leak_idle_worker(ray_shutdown):
    """Regression test for #63596 / PR #63597.

    Before the fix, ``serve_start_async`` ran ``_start_controller`` as a remote
    Ray task and returned the controller ``ActorHandle`` cross-process to the
    caller (the Dashboard Agent). That transfer inserted the handle's
    ObjectRefs into the executor worker's ``stored_in_objects``, pinning the
    worker IDLE forever (it could never drain). Accumulated across calls in a
    long-lived caller, this eventually OOM'd the head node.

    With the inline fix, controller creation runs in the caller process — there
    is no executor worker to pin. This test drives ``serve_start_async`` (the
    exact #63596 path the Dashboard Agent uses) and asserts the symptom.

    ``ray._private.state.workers()`` reports all WORKER-type processes, which
    includes the actor-hosting workers for the controller and HTTP proxies.
    Those are created on ``serve_start_async`` and reaped on ``serve.shutdown()``
    each cycle, so they do not accumulate. The leaked ``_start_controller``
    executor, by contrast, was pinned IDLE and SURVIVED ``serve.shutdown()``
    (its ``object_id_refs_`` never drained) — so it accumulated across cycles
    and grew the count. The non-growth assertion below catches exactly that.
    """

    def _worker_count() -> int:
        return len(state.workers())

    def _settled_worker_count() -> int:
        # ``state.workers()`` reflects the GCS worker table, which updates
        # asynchronously after processes exit and the raylet deregisters them.
        # A fixed sleep is unsafe under CI contention: a not-yet-deregistered
        # worker would false-fail the fixed code. Settle by polling until two
        # consecutive reads agree, so reaping of the controller/proxy workers
        # is complete before sampling. After shutdown only a leaked (pinned)
        # executor survives.
        seen = {"prev": None, "stable": False}

        def _stable() -> bool:
            cur = _worker_count()
            if seen["prev"] is not None and cur == seen["prev"]:
                seen["stable"] = True
            seen["prev"] = cur
            return seen["stable"]

        wait_for_condition(_stable, timeout=30)
        return _worker_count()

    cycles = 3
    counts = []
    for _ in range(cycles):
        asyncio.run(serve_start_async())
        serve.shutdown()
        counts.append(_settled_worker_count())

    # The count must not grow across cycles — that growth was the leak.
    # Allow equal-or-fewer (reaping may reduce it); forbid growth.
    assert counts[-1] <= counts[0], (
        f"Idle WORKER count grew across {cycles} serve_start_async()/shutdown() "
        f"cycles: {counts}. Growth indicates a pinned (leaked) executor worker "
        f"— the #63596 regression."
    )


if __name__ == "__main__":
    sys.exit(pytest.main(["-v", "-s", __file__]))