ray-project--ray/python/ray/serve/_private/constants.py

import logging
import os
from typing import List

from ray._common.network_utils import get_all_interfaces_ip
from ray.serve._private.constants_utils import (
    get_env_bool,
    get_env_float,
    get_env_float_non_negative,
    get_env_float_positive,
    get_env_int,
    get_env_int_non_negative,
    get_env_int_positive,
    get_env_str,
    parse_latency_buckets,
    str_to_list,
)

#: Logger used by serve components
SERVE_LOGGER_NAME = "ray.serve"
logger = logging.getLogger(SERVE_LOGGER_NAME)

#: Actor name used to register controller
SERVE_CONTROLLER_NAME = "SERVE_CONTROLLER_ACTOR"
SERVE_DEPLOYMENT_ACTOR_PREFIX = "SERVE_DEPLOYMENT_ACTOR::"

# Reserved runtime_env keys used to hydrate deployment actor context.
# Unlike replicas which use _set_internal_replica_context() during init,
# deployment actors are user-defined Ray actors. Serve controller can't
# inject constructor params. Env vars via runtime_env are the reasonable
# injection point that doesn't require modifying the user's class.
RAY_SERVE_INTERNAL_DEPLOYMENT_APP_NAME_ENV_VAR = (
    "RAY_SERVE_INTERNAL_DEPLOYMENT_APP_NAME"
)
RAY_SERVE_INTERNAL_DEPLOYMENT_NAME_ENV_VAR = "RAY_SERVE_INTERNAL_DEPLOYMENT_NAME"
RAY_SERVE_INTERNAL_DEPLOYMENT_ACTOR_NAME_ENV_VAR = (
    "RAY_SERVE_INTERNAL_DEPLOYMENT_ACTOR_NAME"
)
RAY_SERVE_INTERNAL_DEPLOYMENT_CODE_VERSION_ENV_VAR = (
    "RAY_SERVE_INTERNAL_DEPLOYMENT_CODE_VERSION"
)

#: Actor name used to register HTTP proxy actor
SERVE_PROXY_NAME = "SERVE_PROXY_ACTOR"

#: Ray namespace used for all Serve actors
SERVE_NAMESPACE = "serve"

DEFAULT_HTTP_HOST = os.environ.get("RAY_SERVE_DEFAULT_HTTP_HOST")

#: HTTP Port
DEFAULT_HTTP_PORT = 8000

#: Fallback proxy HTTP port
RAY_SERVE_FALLBACK_PROXY_HTTP_PORT = get_env_int_positive(
    "RAY_SERVE_FALLBACK_PROXY_HTTP_PORT", 8500
)

#: Uvicorn timeout_keep_alive Config
DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_S = 90

#: gRPC Port
DEFAULT_GRPC_PORT = 9000

#: Fallback proxy gRPC port
RAY_SERVE_FALLBACK_PROXY_GRPC_PORT = get_env_int_positive(
    "RAY_SERVE_FALLBACK_PROXY_GRPC_PORT", 9500
)

#: Default Serve application name
SERVE_DEFAULT_APP_NAME = "default"

#: Max concurrency
ASYNC_CONCURRENCY = int(1e6)

# How long to sleep between control loop cycles on the controller.
CONTROL_LOOP_INTERVAL_S = get_env_float_non_negative(
    "RAY_SERVE_CONTROL_LOOP_INTERVAL_S", 0.1
)

#: Max time to wait for HTTP proxy in `serve.start()`.
HTTP_PROXY_TIMEOUT = 60

# Max retry on deployment constructor is
# min(num_replicas * MAX_PER_REPLICA_RETRY_COUNT, max_constructor_retry_count)
MAX_PER_REPLICA_RETRY_COUNT = get_env_int("RAY_SERVE_MAX_PER_REPLICA_RETRY_COUNT", 3)

#: Max processing latency metric configuration.
#: Rolling window duration for calculating max processing latency (in seconds).
RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_WINDOW_S = float(
    get_env_str("RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_WINDOW_S", "60")
)
#: Interval for reporting max processing latency metric (in seconds).
RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_REPORT_INTERVAL_S = float(
    get_env_str("RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_REPORT_INTERVAL_S", "10")
)
#: Number of buckets for the rolling window (determines granularity).
RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_NUM_BUCKETS = int(
    get_env_str("RAY_SERVE_REPLICA_MAX_PROCESSING_LATENCY_NUM_BUCKETS", "6")
)


# If you are wondering why we are using histogram buckets, please refer to
# https://prometheus.io/docs/practices/histograms/
# short answer is that its cheaper to calculate percentiles on the histogram
# than to calculate them on raw data, both in terms of time and space.

#: Default histogram buckets for latency tracker.
DEFAULT_LATENCY_BUCKET_MS = [
    1,
    2,
    5,
    10,
    20,
    50,
    100,
    200,
    300,
    400,
    500,
    1000,
    2000,
    # 5 seconds
    5000,
    # 10 seconds
    10000,
    # 60 seconds
    60000,
    # 2min
    120000,
    # 5 min
    300000,
    # 10 min
    600000,
]

# Example usage:
# RAY_SERVE_REQUEST_LATENCY_BUCKET_MS="1,2,3,4"
# RAY_SERVE_MODEL_LOAD_LATENCY_BUCKET_MS="1,2,3,4"
#: Histogram buckets for request latency.
REQUEST_LATENCY_BUCKETS_MS = parse_latency_buckets(
    get_env_str(
        "RAY_SERVE_REQUEST_LATENCY_BUCKETS_MS",
        get_env_str("REQUEST_LATENCY_BUCKETS_MS", ""),
    ),
    DEFAULT_LATENCY_BUCKET_MS,
)
#: Histogram buckets for model load/unload latency.
MODEL_LOAD_LATENCY_BUCKETS_MS = parse_latency_buckets(
    get_env_str(
        "RAY_SERVE_MODEL_LOAD_LATENCY_BUCKETS_MS",
        get_env_str("MODEL_LOAD_LATENCY_BUCKETS_MS", ""),
    ),
    DEFAULT_LATENCY_BUCKET_MS,
)

#: Histogram buckets for replica startup and reconfigure latency.
#: These are longer operations (constructor, model loading) so buckets start higher.
DEFAULT_REPLICA_STARTUP_SHUTDOWN_LATENCY_BUCKETS_MS = [
    5,
    20,
    50,
    100,
    250,
    500,
    1000,
    2000,
    5000,
    10000,
    20000,
    30000,
    60000,
    120000,
    240000,
]
REPLICA_STARTUP_SHUTDOWN_LATENCY_BUCKETS_MS = parse_latency_buckets(
    get_env_str("RAY_SERVE_REPLICA_STARTUP_SHUTDOWN_LATENCY_BUCKETS_MS", ""),
    DEFAULT_REPLICA_STARTUP_SHUTDOWN_LATENCY_BUCKETS_MS,
)

#: Histogram buckets for batch execution time in milliseconds.
BATCH_EXECUTION_TIME_BUCKETS_MS = REQUEST_LATENCY_BUCKETS_MS

#: Histogram buckets for batch wait time in milliseconds.
BATCH_WAIT_TIME_BUCKETS_MS = REQUEST_LATENCY_BUCKETS_MS

#: Histogram buckets for batch utilization percentage.
DEFAULT_BATCH_UTILIZATION_BUCKETS_PERCENT = [
    5,
    10,
    20,
    30,
    40,
    50,
    60,
    70,
    80,
    90,
    95,
    99,
    100,
]
BATCH_UTILIZATION_BUCKETS_PERCENT = parse_latency_buckets(
    get_env_str(
        "RAY_SERVE_BATCH_UTILIZATION_BUCKETS_PERCENT",
        "",
    ),
    DEFAULT_BATCH_UTILIZATION_BUCKETS_PERCENT,
)

#: Replica utilization metric configuration.
#: Rolling window duration for calculating replica utilization (in seconds).
RAY_SERVE_REPLICA_UTILIZATION_WINDOW_S = float(
    get_env_str("RAY_SERVE_REPLICA_UTILIZATION_WINDOW_S", "600")
)
#: Interval for reporting replica utilization metric (in seconds).
RAY_SERVE_REPLICA_UTILIZATION_REPORT_INTERVAL_S = float(
    get_env_str("RAY_SERVE_REPLICA_UTILIZATION_REPORT_INTERVAL_S", "10")
)
#: Number of buckets for the rolling window (determines granularity).
RAY_SERVE_REPLICA_UTILIZATION_NUM_BUCKETS = int(
    get_env_str("RAY_SERVE_REPLICA_UTILIZATION_NUM_BUCKETS", "60")
)

#: Histogram buckets for actual batch size.
DEFAULT_BATCH_SIZE_BUCKETS = [
    1,
    2,
    4,
    8,
    16,
    32,
    64,
    128,
    256,
    512,
    1024,
]
BATCH_SIZE_BUCKETS = parse_latency_buckets(
    get_env_str(
        "RAY_SERVE_BATCH_SIZE_BUCKETS",
        "",
    ),
    DEFAULT_BATCH_SIZE_BUCKETS,
)

#: Name of deployment health check method implemented by user.
HEALTH_CHECK_METHOD = "check_health"

#: Name of deployment reconfiguration method implemented by user.
RECONFIGURE_METHOD = "reconfigure"

#: Limit the number of cached handles because each handle has long poll
#: overhead. See https://github.com/ray-project/ray/issues/18980
MAX_CACHED_HANDLES = get_env_int_positive("RAY_SERVE_MAX_CACHED_HANDLES", 100)

#: Because ServeController will accept one long poll request per handle, its
#: concurrency needs to scale as O(num_handles)
CONTROLLER_MAX_CONCURRENCY = get_env_int_positive(
    "RAY_SERVE_CONTROLLER_MAX_CONCURRENCY", 15_000
)

DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_S = 20
DEFAULT_GRACEFUL_SHUTDOWN_WAIT_LOOP_S = 2
DEFAULT_HEALTH_CHECK_PERIOD_S = 10
DEFAULT_HEALTH_CHECK_TIMEOUT_S = 30
DEFAULT_MAX_ONGOING_REQUESTS = 5
DEFAULT_TARGET_ONGOING_REQUESTS = 2
DEFAULT_CONSUMER_CONCURRENCY = DEFAULT_MAX_ONGOING_REQUESTS
DEFAULT_CONSTRUCTOR_RETRY_COUNT = 20
DEFAULT_ROLLING_UPDATE_PERCENTAGE = 0.2

# HTTP Proxy health check configs
PROXY_HEALTH_CHECK_TIMEOUT_S = get_env_float_positive(
    "RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S", 10.0
)

PROXY_HEALTH_CHECK_PERIOD_S = get_env_float_positive(
    "RAY_SERVE_PROXY_HEALTH_CHECK_PERIOD_S", 10.0
)
PROXY_READY_CHECK_TIMEOUT_S = get_env_float_positive(
    "RAY_SERVE_PROXY_READY_CHECK_TIMEOUT_S", 5.0
)

# Number of times in a row that a HTTP proxy must fail the health check before
# being marked unhealthy.
PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD = 3

# The minimum drain period for a HTTP proxy.
PROXY_MIN_DRAINING_PERIOD_S = get_env_float_positive(
    "RAY_SERVE_PROXY_MIN_DRAINING_PERIOD_S", 30.0
)
# The time in seconds that the http proxy state waits before
# rechecking whether the proxy actor is drained or not.
PROXY_DRAIN_CHECK_PERIOD_S = 5

#: Number of times in a row that a replica must fail the health check before
#: being marked unhealthy.
REPLICA_HEALTH_CHECK_UNHEALTHY_THRESHOLD = 3

# Watchdog that detects a wedged user code event loop when user code runs in a
# separate thread (RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD=1) and no user-defined
# check_health is present. The main loop periodically schedules asyncio.sleep(0) on
# the user loop; if the probe times out MAX_FAIL times consecutively, check_health
# raises immediately so the replica is restarted without waiting for the controller's
# RPC timeout. Set MAX_FAIL=0 to disable.
USER_HEALTH_CHECK_PROBE_INTERVAL_S = get_env_float_positive(
    "RAY_SERVE_USER_HEALTH_CHECK_PROBE_INTERVAL_S",
    60.0,
)
USER_HEALTH_CHECK_PROBE_TIMEOUT_S = get_env_float_positive(
    "RAY_SERVE_USER_HEALTH_CHECK_PROBE_TIMEOUT_S",
    300.0,
)
USER_HEALTH_CHECK_PROBE_MAX_FAIL = get_env_int_non_negative(
    "RAY_SERVE_USER_HEALTH_CHECK_PROBE_MAX_FAIL",
    3,
)

# Controller polls deployment-scoped actors with ``__ray_ready__`` (same idea as
# replica health checks). Defaults match deployment replica timing; override via env.
DEPLOYMENT_ACTOR_HEALTH_CHECK_PERIOD_S = get_env_float_positive(
    "RAY_SERVE_DEPLOYMENT_ACTOR_HEALTH_CHECK_PERIOD_S",
    float(DEFAULT_HEALTH_CHECK_PERIOD_S),
)
DEPLOYMENT_ACTOR_HEALTH_CHECK_TIMEOUT_S = get_env_float_positive(
    "RAY_SERVE_DEPLOYMENT_ACTOR_HEALTH_CHECK_TIMEOUT_S",
    float(DEFAULT_HEALTH_CHECK_TIMEOUT_S),
)
DEPLOYMENT_ACTOR_HEALTH_CHECK_UNHEALTHY_THRESHOLD = get_env_int_positive(
    "RAY_SERVE_DEPLOYMENT_ACTOR_HEALTH_CHECK_UNHEALTHY_THRESHOLD",
    REPLICA_HEALTH_CHECK_UNHEALTHY_THRESHOLD,
)

# The time in seconds that the Serve client waits before rechecking deployment state
CLIENT_POLLING_INTERVAL_S = 1.0

# The time in seconds that the Serve client waits before checking if
# deployment has been created
CLIENT_CHECK_CREATION_POLLING_INTERVAL_S = 0.1

# Timeout for GCS internal KV service
RAY_SERVE_KV_TIMEOUT_S = get_env_float_positive("RAY_SERVE_KV_TIMEOUT_S", None)

# Timeout for GCS RPC request
RAY_GCS_RPC_TIMEOUT_S = 3.0

# Maximum duration to wait until broadcasting a long poll update if there are
# still replicas in the RECOVERING state.
RECOVERING_LONG_POLL_BROADCAST_TIMEOUT_S = 10.0

# Minimum duration to wait until broadcasting model IDs.
PUSH_MULTIPLEXED_MODEL_IDS_INTERVAL_S = 0.1

# Deprecation message for V1 migrations.
MIGRATION_MESSAGE = (
    "See https://docs.ray.io/en/latest/serve/index.html for more information."
)

# Environment variable name for to specify the encoding of the log messages
RAY_SERVE_LOG_ENCODING = "TEXT"


# Setting RAY_SERVE_LOG_TO_STDERR=0 will disable logging to the stdout and stderr.
# Also, redirect them to serve's log files.
RAY_SERVE_LOG_TO_STDERR = get_env_bool("RAY_SERVE_LOG_TO_STDERR", "1")

# Logging format attributes
SERVE_LOG_REQUEST_ID = "request_id"
SERVE_LOG_ROUTE = "route"
SERVE_LOG_APPLICATION = "application"
SERVE_LOG_DEPLOYMENT = "deployment"
SERVE_LOG_REPLICA = "replica"
SERVE_LOG_COMPONENT = "component_name"
SERVE_LOG_COMPONENT_ID = "component_id"
SERVE_LOG_MESSAGE = "message"
# This is a reserved for python logging module attribute, it should not be changed.
SERVE_LOG_LEVEL_NAME = "levelname"
SERVE_LOG_TIME = "asctime"

# Logging format with record key to format string dict
SERVE_LOG_RECORD_FORMAT = {
    SERVE_LOG_REQUEST_ID: "%(request_id)s",
    SERVE_LOG_APPLICATION: "%(application)s",
    SERVE_LOG_MESSAGE: "-- %(message)s",
    SERVE_LOG_LEVEL_NAME: "%(levelname)s",
    SERVE_LOG_TIME: "%(asctime)s",
}

# There are some attributes that we only use internally or don't provide values to the
# users. Adding to this set will remove them from structured logs.
SERVE_LOG_UNWANTED_ATTRS = {
    "serve_access_log",
    "task_id",
    "job_id",
    "skip_context_filter",
}

RAY_SERVE_HTTP_KEEP_ALIVE_TIMEOUT_S = get_env_int_non_negative(
    "RAY_SERVE_HTTP_KEEP_ALIVE_TIMEOUT_S", 0
)

RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S = 0.0

SERVE_LOG_EXTRA_FIELDS = "ray_serve_extra_fields"

# Serve HTTP request header key for routing requests.
SERVE_MULTIPLEXED_MODEL_ID = "serve_multiplexed_model_id"

# Serve HTTP request header key for session-stickiness routing.
# Stored as the operator wrote it (no ``-``/``_`` mangling); set via
# ``RAY_SERVE_SESSION_ID_HEADER_KEY`` (default ``x-session-id``). Compare
# against incoming header names with ``_matches_session_id_header`` from
# ``http_util`` -- that helper tolerates intermediate proxies that swap
# ``-`` and ``_`` (nginx, AWS API Gateway, ...).
SERVE_SESSION_ID = get_env_str("RAY_SERVE_SESSION_ID_HEADER_KEY", "x-session-id")

# HTTP request ID
SERVE_HTTP_REQUEST_ID_HEADER = "x-request-id"

# Feature flag to turn on node locality routing for proxies. On by default.
RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTING = get_env_bool(
    "RAY_SERVE_PROXY_PREFER_LOCAL_NODE_ROUTING", "1"
)

# Feature flag to turn on AZ locality routing for proxies. On by default.
RAY_SERVE_PROXY_PREFER_LOCAL_AZ_ROUTING = get_env_bool(
    "RAY_SERVE_PROXY_PREFER_LOCAL_AZ_ROUTING", "1"
)

# Serve HTTP proxy callback import path.
RAY_SERVE_HTTP_PROXY_CALLBACK_IMPORT_PATH = get_env_str(
    "RAY_SERVE_HTTP_PROXY_CALLBACK_IMPORT_PATH", None
)
# Serve controller callback import path.
RAY_SERVE_CONTROLLER_CALLBACK_IMPORT_PATH = get_env_str(
    "RAY_SERVE_CONTROLLER_CALLBACK_IMPORT_PATH", None
)

# Maximum timeout allowed for record_autoscaling_stats to run.
RAY_SERVE_RECORD_AUTOSCALING_STATS_TIMEOUT_S = get_env_float(
    "RAY_SERVE_RECORD_AUTOSCALING_STATS_TIMEOUT_S", 10.0
)

# Factor of look_back_period_s for autoscaling metric record interval.
# Record interval = look_back_period_s * factor. Used by both router and replica.
RAY_SERVE_AUTOSCALING_METRIC_RECORD_INTERVAL_FACTOR = get_env_float(
    "RAY_SERVE_AUTOSCALING_METRIC_RECORD_INTERVAL_FACTOR", 0.2
)

# Replica autoscaling metrics push interval.
RAY_SERVE_REPLICA_AUTOSCALING_METRIC_PUSH_INTERVAL_S = get_env_float(
    "RAY_SERVE_REPLICA_AUTOSCALING_METRIC_PUSH_INTERVAL_S", 10.0
)

# Handle autoscaling metrics push interval. (This interval will affect the cold start time period)
RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S = get_env_float(
    "RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S",
    10.0,
)

# Async inference task queue metrics push interval.
RAY_SERVE_ASYNC_INFERENCE_TASK_QUEUE_METRIC_PUSH_INTERVAL_S = get_env_float(
    "RAY_SERVE_ASYNC_INFERENCE_TASK_QUEUE_METRIC_PUSH_INTERVAL_S", 10.0
)

# Serve multiplexed matching timeout.
# This is the timeout for the matching process of multiplexed requests. To avoid
# thundering herd problem, the timeout value will be randomized between this value
# and this value * 2. The unit is second.
# If the matching process takes longer than the timeout, the request will be
# fallen to the default routing strategy.
RAY_SERVE_MULTIPLEXED_MODEL_ID_MATCHING_TIMEOUT_S = get_env_float_non_negative(
    "RAY_SERVE_MULTIPLEXED_MODEL_ID_MATCHING_TIMEOUT_S", 1.0
)

# Enable memray in all Serve actors.
RAY_SERVE_ENABLE_MEMORY_PROFILING = get_env_bool(
    "RAY_SERVE_ENABLE_MEMORY_PROFILING", "0"
)

# Max value allowed for max_replicas_per_node option.
# TODO(jjyao) the <= 100 limitation is an artificial one
# and is due to the fact that Ray core only supports resource
# precision up to 0.0001.
# This limitation should be lifted in the long term.
MAX_REPLICAS_PER_NODE_MAX_VALUE = 100

# Argument name for passing in the gRPC context into a replica.
GRPC_CONTEXT_ARG_NAME = "grpc_context"

# Whether or not to forcefully kill replicas that fail health checks.
RAY_SERVE_FORCE_STOP_UNHEALTHY_REPLICAS = get_env_bool(
    "RAY_SERVE_FORCE_STOP_UNHEALTHY_REPLICAS", "0"
)

# How often (in seconds) the controller re-records an unchanged status gauge
# value for replicas and applications. Setting this to 0 disables caching
# (every control loop iteration records the gauge, matching pre-optimization
# behavior).
RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S = get_env_float_non_negative(
    "RAY_SERVE_STATUS_GAUGE_REPORT_INTERVAL_S", 10.0
)

# Initial deadline for queue length responses in the router.
RAY_SERVE_QUEUE_LENGTH_RESPONSE_DEADLINE_S = get_env_float(
    "RAY_SERVE_QUEUE_LENGTH_RESPONSE_DEADLINE_S", 0.1
)

# Maximum deadline for queue length responses in the router (in backoff).
RAY_SERVE_MAX_QUEUE_LENGTH_RESPONSE_DEADLINE_S = get_env_float(
    "RAY_SERVE_MAX_QUEUE_LENGTH_RESPONSE_DEADLINE_S", 1.0
)

# Length of time to respect entries in the queue length cache when routing requests.
RAY_SERVE_QUEUE_LENGTH_CACHE_TIMEOUT_S = get_env_float_non_negative(
    "RAY_SERVE_QUEUE_LENGTH_CACHE_TIMEOUT_S", 10.0
)

# Minimum interval between router queue length gauge updates per replica.
# Throttling reduces metrics overhead on the hot path. Set to 0 to disable throttling.
RAY_SERVE_ROUTER_QUEUE_LEN_GAUGE_THROTTLE_S = get_env_float_non_negative(
    "RAY_SERVE_ROUTER_QUEUE_LEN_GAUGE_THROTTLE_S", 0.1
)

# Backoff seconds when choosing router failed, backoff time is calculated as
# initial_backoff_s * backoff_multiplier ** attempt.
# The default backoff time is [0, 0.025, 0.05, 0.1, 0.2, 0.4, 0.5, 0.5 ... ].
RAY_SERVE_ROUTER_RETRY_INITIAL_BACKOFF_S = get_env_float(
    "RAY_SERVE_ROUTER_RETRY_INITIAL_BACKOFF_S", 0.025
)
RAY_SERVE_ROUTER_RETRY_BACKOFF_MULTIPLIER = get_env_int(
    "RAY_SERVE_ROUTER_RETRY_BACKOFF_MULTIPLIER", 2
)
RAY_SERVE_ROUTER_RETRY_MAX_BACKOFF_S = get_env_float(
    "RAY_SERVE_ROUTER_RETRY_MAX_BACKOFF_S", 0.5
)

# The default autoscaling policy to use if none is specified.
DEFAULT_AUTOSCALING_POLICY_NAME = (
    "ray.serve.autoscaling_policy:default_autoscaling_policy"
)

# Feature flag to enable collecting all queued and ongoing request
# metrics at handles instead of replicas. ON by default.
RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE = get_env_bool(
    "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE", "1"
)

RAY_SERVE_MIN_HANDLE_METRICS_TIMEOUT_S = get_env_float_non_negative(
    "RAY_SERVE_MIN_HANDLE_METRICS_TIMEOUT_S", 10.0
)

# Default is 2GiB, the max for a signed int.
RAY_SERVE_GRPC_MAX_MESSAGE_SIZE = get_env_int(
    "RAY_SERVE_GRPC_MAX_MESSAGE_SIZE", (2 * 1024 * 1024 * 1024) - 1
)

RAY_SERVE_REPLICA_GRPC_MAX_MESSAGE_LENGTH = get_env_int(
    # Default max message length in gRPC is 4MB, we keep that default
    "RAY_SERVE_REPLICA_GRPC_MAX_MESSAGE_LENGTH",
    4 * 1024 * 1024,
)

# Default options passed when constructing gRPC servers.
DEFAULT_GRPC_SERVER_OPTIONS = [
    ("grpc.max_send_message_length", RAY_SERVE_GRPC_MAX_MESSAGE_SIZE),
    ("grpc.max_receive_message_length", RAY_SERVE_GRPC_MAX_MESSAGE_SIZE),
]

# Timeout for gracefully shutting down metrics pusher, e.g. in routers or replicas
METRICS_PUSHER_GRACEFUL_SHUTDOWN_TIMEOUT_S = 10

# Feature flag to set `enable_task_events=True` on Serve-managed actors.
RAY_SERVE_ENABLE_TASK_EVENTS = get_env_bool("RAY_SERVE_ENABLE_TASK_EVENTS", "0")

# This is deprecated and will be removed in the future.
RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY = get_env_bool(
    "RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY", "0"
)

# Use pack instead of spread scheduling strategy.
RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY = get_env_bool(
    "RAY_SERVE_USE_PACK_SCHEDULING_STRATEGY",
    os.environ.get("RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY", "0"),
)

# Comma-separated list of custom resources prioritized in scheduling. Sorted from highest to lowest priority.
# Example: "customx,customy"
RAY_SERVE_HIGH_PRIORITY_CUSTOM_RESOURCES: List[str] = str_to_list(
    get_env_str("RAY_SERVE_HIGH_PRIORITY_CUSTOM_RESOURCES", "")
)

# Feature flag to always override local_testing_mode to True in serve.run.
# This is used for internal testing to avoid passing the flag to every invocation.
RAY_SERVE_FORCE_LOCAL_TESTING_MODE = get_env_bool(
    "RAY_SERVE_FORCE_LOCAL_TESTING_MODE", "0"
)

# Run sync methods defined in the replica in a thread pool by default.
RAY_SERVE_RUN_SYNC_IN_THREADPOOL = get_env_bool("RAY_SERVE_RUN_SYNC_IN_THREADPOOL", "0")

RAY_SERVE_RUN_SYNC_IN_THREADPOOL_WARNING = (
    "Calling sync method '{method_name}' directly on the "
    "asyncio loop. In a future version, sync methods will be run in a "
    "threadpool by default. Ensure your sync methods are thread safe "
    "or keep the existing behavior by making them `async def`. Opt "
    "into the new behavior by setting "
    "RAY_SERVE_RUN_SYNC_IN_THREADPOOL=1."
)

# Feature flag to turn off GC optimizations in the proxy (in case there is a
# memory leak or negative performance impact).
RAY_SERVE_ENABLE_PROXY_GC_OPTIMIZATIONS = get_env_bool(
    "RAY_SERVE_ENABLE_PROXY_GC_OPTIMIZATIONS", "1"
)

# Used for gc.set_threshold() when proxy GC optimizations are enabled.
RAY_SERVE_PROXY_GC_THRESHOLD = get_env_int("RAY_SERVE_PROXY_GC_THRESHOLD", 700)

# Interval at which cached metrics will be exported using the Ray metric API.
# Set to `0` to disable caching entirely.
RAY_SERVE_METRICS_EXPORT_INTERVAL_MS = get_env_int(
    "RAY_SERVE_METRICS_EXPORT_INTERVAL_MS", 100
)

# The default request router class to use if none is specified.
DEFAULT_REQUEST_ROUTER_PATH = (
    "ray.serve._private.request_router:PowerOfTwoChoicesRequestRouter"
)

# The default request routing period to use if none is specified.
DEFAULT_REQUEST_ROUTING_STATS_PERIOD_S = 10

# The default request routing timeout to use if none is specified.
DEFAULT_REQUEST_ROUTING_STATS_TIMEOUT_S = 30

# Name of deployment request routing stats method implemented by user.
REQUEST_ROUTING_STATS_METHOD = "record_routing_stats"

# Name of deployment static replica metadata method implemented by user.
RECORD_REPLICA_METADATA_METHOD = "record_replica_metadata"

# By default, we run user code in a separate event loop.
# This flag can be set to 0 to run user code in the same event loop as the
# replica's main event loop.
RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD = get_env_bool(
    "RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD", "1"
)

# By default, we run the router in a separate event loop.
# This flag can be set to 0 to run the router in the same event loop as the
# replica's main event loop.
RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP = get_env_bool(
    "RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP", "1"
)

# For now, this is used only for testing. In the suite of tests that
# use gRPC to send requests, we flip this flag on.
RAY_SERVE_USE_GRPC_BY_DEFAULT = (
    os.environ.get("RAY_SERVE_USE_GRPC_BY_DEFAULT", "0") == "1"
)

RAY_SERVE_PROXY_USE_GRPC = os.environ.get("RAY_SERVE_PROXY_USE_GRPC") == "1" or (
    not os.environ.get("RAY_SERVE_PROXY_USE_GRPC") == "0"
    and RAY_SERVE_USE_GRPC_BY_DEFAULT
)

# The default buffer size for request path logs. Setting to 1 will ensure
# logs are flushed to file handler immediately, otherwise it will be buffered
# and flushed to file handler when the buffer is full or when there is a log
# line with level ERROR.
RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE = get_env_int(
    "RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE", 1
)

# Feature flag to fail the deployment if the rank is not set.
# TODO (abrar): Remove this flag after the feature is stable.
RAY_SERVE_FAIL_ON_RANK_ERROR = get_env_bool("RAY_SERVE_FAIL_ON_RANK_ERROR", "0")

# Stopped replicas to retain per deployment for dashboard log access. 0 disables.
RAY_SERVE_RETAINED_DEAD_REPLICAS = get_env_int_non_negative(
    "RAY_SERVE_RETAINED_DEAD_REPLICAS", 10
)

# The message to return when the replica is healthy.
HEALTHY_MESSAGE = "success"
NO_ROUTES_MESSAGE = "Route table is not populated yet."
NO_REPLICAS_MESSAGE = "No replicas are available yet."
DRAINING_MESSAGE = "This node is being drained."

# Feature flag to enable a limited form of direct ingress where ingress applications
# listen on port 8000 (HTTP) and 9000 (gRPC). No proxies will be started.
RAY_SERVE_ENABLE_DIRECT_INGRESS = (
    os.environ.get("RAY_SERVE_ENABLE_DIRECT_INGRESS", "0") == "1"
)

# Feature flag to use HAProxy.
RAY_SERVE_ENABLE_HA_PROXY = os.environ.get("RAY_SERVE_ENABLE_HA_PROXY", "0") == "1"

# Feature flag to include client IP address in HTTP access logs.
# Off by default for privacy; set to "1" to enable.
RAY_SERVE_LOG_CLIENT_ADDRESS = (
    os.environ.get("RAY_SERVE_LOG_CLIENT_ADDRESS", "0") == "1"
)

# Absolute path to an HAProxy binary. When set, it takes precedence over the
# bundled ray-haproxy package.
RAY_SERVE_HAPROXY_BINARY_PATH = get_env_str("RAY_SERVE_HAPROXY_BINARY_PATH", "")

# HAProxy configuration defaults
# Maximum number of concurrent connections
RAY_SERVE_HAPROXY_MAXCONN = int(os.environ.get("RAY_SERVE_HAPROXY_MAXCONN", "20000"))

# Number of threads for HAProxy
RAY_SERVE_HAPROXY_NBTHREAD = int(os.environ.get("RAY_SERVE_HAPROXY_NBTHREAD", "4"))

# HAProxy configuration file location
RAY_SERVE_HAPROXY_CONFIG_FILE_LOC = os.environ.get(
    "RAY_SERVE_HAPROXY_CONFIG_FILE_LOC", "/tmp/haproxy-serve/haproxy.cfg"
)

# HAProxy admin socket path
RAY_SERVE_HAPROXY_SOCKET_PATH = os.environ.get(
    "RAY_SERVE_HAPROXY_SOCKET_PATH", "/tmp/haproxy-serve/admin.sock"
)

# Enable HAProxy optimized configuration (server state persistence, etc.)
# Disabled by default to prevent test suite interference
RAY_SERVE_ENABLE_HAPROXY_OPTIMIZED_CONFIG = (
    os.environ.get("RAY_SERVE_ENABLE_HAPROXY_OPTIMIZED_CONFIG", "1") == "1"
)

# HAProxy server state path
RAY_SERVE_HAPROXY_SERVER_STATE_BASE = os.environ.get(
    "RAY_SERVE_HAPROXY_SERVER_STATE_BASE", "/tmp/haproxy-serve"
)

# HAProxy server state path
RAY_SERVE_HAPROXY_SERVER_STATE_FILE = os.environ.get(
    "RAY_SERVE_HAPROXY_SERVER_STATE_FILE", "/tmp/haproxy-serve/server-state"
)

# HAProxy hard stop after timeout
RAY_SERVE_HAPROXY_HARD_STOP_AFTER_S = int(
    os.environ.get("RAY_SERVE_HAPROXY_HARD_STOP_AFTER_S", "120")
)

# Timeout for a spawned HAProxy to take over the admin socket (pid-verified).
# Generous: a reload under load transfers listener FDs from a busy predecessor.
RAY_SERVE_HAPROXY_STARTUP_TIMEOUT_S = int(
    os.environ.get("RAY_SERVE_HAPROXY_STARTUP_TIMEOUT_S", "30")
)

# Minimum spacing between HAProxy reloads. Broadcasts arriving inside
# the window are batched into one apply; without it, autoscaling churn
# can fire reloads tens of ms apart.
RAY_SERVE_HAPROXY_BROADCAST_COALESCE_S = get_env_float_non_negative(
    "RAY_SERVE_HAPROXY_BROADCAST_COALESCE_S", 0.1
)

# Histogram boundaries (seconds) for serve_haproxy_update_latency_s: the time
# from the first coalesced controller broadcast to the HAProxy reload finishing.
RAY_SERVE_HAPROXY_UPDATE_LATENCY_BUCKETS_S = [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30]

# Controls whether HAProxy system metrics are reported. On by default.
RAY_SERVE_HAPROXY_METRICS_ENABLED = get_env_bool(
    "RAY_SERVE_HAPROXY_METRICS_ENABLED", "1"
)

# How often (seconds) each HAProxyManager samples and emits node-level HAProxy
# observability gauges (process count and broadcasted-vs-reported target mismatch).
RAY_SERVE_HAPROXY_METRICS_REPORT_INTERVAL_S = get_env_float_non_negative(
    "RAY_SERVE_HAPROXY_METRICS_REPORT_INTERVAL_S", 10.0
)

# HAProxy metrics export port
RAY_SERVE_HAPROXY_METRICS_PORT = int(
    os.environ.get("RAY_SERVE_HAPROXY_METRICS_PORT", "9101")
)

# HAProxy stats UI port
RAY_SERVE_HAPROXY_STATS_PORT = get_env_int("RAY_SERVE_HAPROXY_STATS_PORT", 8404)

# Per-worker-node override for the proxy's HTTP/gRPC bind ports. Head node exempt.
# Prefer http_options.port / grpc_options.port. This override only matters when
# proxies are colocated on one machine and need distinct ports without SO_REUSEPORT.
# Parallels RAY_SERVE_HAPROXY_STATS_PORT / RAY_SERVE_HAPROXY_METRICS_PORT.
RAY_SERVE_WORKER_PROXY_HTTP_PORT = get_env_int("RAY_SERVE_WORKER_PROXY_HTTP_PORT", None)
RAY_SERVE_WORKER_PROXY_GRPC_PORT = get_env_int("RAY_SERVE_WORKER_PROXY_GRPC_PORT", None)

# HAProxy log target (single sink). Accepts any syntax HAProxy's `log` directive
# supports, e.g. "127.0.0.1:514" (UDP syslog) or "/dev/log" (unix datagram socket).
RAY_SERVE_HAPROXY_LOG_TARGET = get_env_str(
    "RAY_SERVE_HAPROXY_LOG_TARGET", "127.0.0.1:514"
)

# HAProxy timeout configurations (in seconds, None = no timeout)
RAY_SERVE_HAPROXY_TIMEOUT_SERVER_S = (
    int(os.environ.get("RAY_SERVE_HAPROXY_TIMEOUT_SERVER_S"))
    if os.environ.get("RAY_SERVE_HAPROXY_TIMEOUT_SERVER_S")
    else None
)

RAY_SERVE_HAPROXY_TIMEOUT_CONNECT_S = (
    int(os.environ.get("RAY_SERVE_HAPROXY_TIMEOUT_CONNECT_S"))
    if os.environ.get("RAY_SERVE_HAPROXY_TIMEOUT_CONNECT_S")
    else None
)

# When enabled, adds 'option http-no-delay' to the HAProxy config defaults,
# setting TCP_NODELAY on both client and server connections.
#
# Default is ON. The streaming serving case (the dominant Ray Serve workload
# today -- streaming LLM completions, SSE, gRPC streaming) is hostile to
# Nagle's algorithm: when the upstream emits a small first chunk (e.g. the
# first SSE event), Nagle holds it in the kernel buffer waiting for either
# more data or the delayed-ACK timer, which lands as added TTFT. Set to "0"
# only if you have a non-streaming HAProxy workload that benefits from
# packet coalescing.
RAY_SERVE_HAPROXY_TCP_NODELAY = get_env_bool("RAY_SERVE_HAPROXY_TCP_NODELAY", "1")

# HAProxy timeout client
RAY_SERVE_HAPROXY_TIMEOUT_CLIENT_S = int(
    os.environ.get("RAY_SERVE_HAPROXY_TIMEOUT_CLIENT_S", "3600")
)

# Number of consecutive failed server health checks that must occur
# before haproxy marks the server as down.
RAY_SERVE_HAPROXY_HEALTH_CHECK_FALL = int(
    os.environ.get("RAY_SERVE_HAPROXY_HEALTH_CHECK_FALL", "2")
)

# Number of consecutive successful server health checks that must occur
# before haproxy marks the server as up.
RAY_SERVE_HAPROXY_HEALTH_CHECK_RISE = int(
    os.environ.get("RAY_SERVE_HAPROXY_HEALTH_CHECK_RISE", "2")
)

# Time interval between each haproxy health check attempt. Also the
# timeout of each health check before being considered as failed.
RAY_SERVE_HAPROXY_HEALTH_CHECK_INTER = os.environ.get(
    "RAY_SERVE_HAPROXY_HEALTH_CHECK_INTER", "5s"
)

# Time interval between each haproxy health check attempt when the server is in any of the transition states: UP - transitionally DOWN or DOWN - transitionally UP
RAY_SERVE_HAPROXY_HEALTH_CHECK_FASTINTER = os.environ.get(
    "RAY_SERVE_HAPROXY_HEALTH_CHECK_FASTINTER", "250ms"
)

# Time interval between each haproxy health check attempt when the server is in the DOWN state
RAY_SERVE_HAPROXY_HEALTH_CHECK_DOWNINTER = os.environ.get(
    "RAY_SERVE_HAPROXY_HEALTH_CHECK_DOWNINTER", "250ms"
)

# The balancing algorithm to use in HAProxy backends. Default is leastconn.
RAY_SERVE_HAPROXY_BALANCE_ALGORITHM = get_env_str(
    "RAY_SERVE_HAPROXY_BALANCE_ALGORITHM", "leastconn"
)

# Timeout shared by the ingress-request-router Lua call and the frontend
# `wait-for-body` directive. Bounds head-of-line blocking on POSTs when a
# router replica is unhealthy.
RAY_SERVE_HAPROXY_INGRESS_REQUEST_ROUTER_TIMEOUT_S = get_env_int(
    "RAY_SERVE_HAPROXY_INGRESS_REQUEST_ROUTER_TIMEOUT_S", 5
)

# Opt-in HAProxy retry knobs on the `-via-ingress-request-router` backend.
# `retry-on` token reference:
# https://docs.haproxy.org/2.8/configuration.html#4-retry-on
# Retry policy for the HAProxy `defaults` block, inherited by every backend.
# Defaults to `conn-failure` only: nothing was sent to the replica, so the
# request is safe to replay for any method (we deliberately avoid empty-response
# / 503, which can double-execute non-idempotent requests or retry deliberate
# backpressure). Set RAY_SERVE_HAPROXY_RETRY_ON to override globally.
RAY_SERVE_HAPROXY_RETRY_ON = get_env_str("RAY_SERVE_HAPROXY_RETRY_ON", "conn-failure")
RAY_SERVE_HAPROXY_RETRIES = get_env_int_non_negative("RAY_SERVE_HAPROXY_RETRIES", None)
# Same retry policy as above; defaults to the global value so the ingress
# request router shares one policy unless explicitly overridden.
RAY_SERVE_HAPROXY_INGRESS_RETRY_ON = get_env_str(
    "RAY_SERVE_HAPROXY_INGRESS_RETRY_ON", RAY_SERVE_HAPROXY_RETRY_ON
)
RAY_SERVE_HAPROXY_INGRESS_RETRIES = get_env_int_non_negative(
    "RAY_SERVE_HAPROXY_INGRESS_RETRIES", RAY_SERVE_HAPROXY_RETRIES
)
RAY_SERVE_HAPROXY_INGRESS_TIMEOUT_SERVER_S = get_env_int_non_negative(
    "RAY_SERVE_HAPROXY_INGRESS_TIMEOUT_SERVER_S", None
)

# Per-buffer byte cap for HAProxy when the ingress-request-router Lua action is
# active. Bodies longer than this are truncated; the Lua forwards what it has
# with an `X-Body-Truncated: <bytes>/<content-length>` header so the router can
# do best-effort prefix matching. Memory cost is ~2 * bufsize * maxconn.
# Only consulted when RAY_SERVE_INGRESS_REQUEST_ROUTER_FORWARD_BODY=1.
RAY_SERVE_HAPROXY_INGRESS_REQUEST_ROUTER_BUFSIZE = get_env_int(
    "RAY_SERVE_HAPROXY_INGRESS_REQUEST_ROUTER_BUFSIZE", 262144
)

# HAProxy tuning flags
RAY_SERVE_HAPROXY_TUNE_BUFSIZE = get_env_int(
    "RAY_SERVE_HAPROXY_TUNE_BUFSIZE", 16384  # 16KB
)
RAY_SERVE_HAPROXY_H2_MAX_FRAME_SIZE = get_env_int(
    "RAY_SERVE_HAPROXY_H2_MAX_FRAME_SIZE", 1024 * 16
)  # 16KB
RAY_SERVE_HAPROXY_H2_BE_INITIAL_WINDOW_SIZE = get_env_int(
    "RAY_SERVE_HAPROXY_H2_BE_INITIAL_WINDOW_SIZE", 1024 * 64
)  # 64KB
RAY_SERVE_HAPROXY_H2_BE_MAX_CONCURRENT_STREAMS = get_env_int(
    "RAY_SERVE_HAPROXY_H2_BE_MAX_CONCURRENT_STREAMS", 100
)
RAY_SERVE_HAPROXY_H2_FE_INITIAL_WINDOW_SIZE = get_env_int(
    "RAY_SERVE_HAPROXY_H2_FE_INITIAL_WINDOW_SIZE", 1024 * 64
)  # 64KB
RAY_SERVE_HAPROXY_H2_FE_MAX_CONCURRENT_STREAMS = get_env_int(
    "RAY_SERVE_HAPROXY_H2_FE_MAX_CONCURRENT_STREAMS", 100
)

# Escape hatch: when true, HAProxy forwards the (possibly truncated) request
# body to /internal/route and the router reads it. Off by default because for
# large payloads the body buffering / re-emit cost adds noticeable time-to-
# first-response. Skipping the forward is fine for any policy whose decision
# does not depend on the request body: round-robin and power-of-two ignore
# the body entirely, and session-aware policies key on the ``x-session-id``
# header (forwarded with the request line) rather than the body.
#
# Flip this to true if the configured request router needs the body for its
# decision, e.g. prefix-aware / prefix-cache routing.
RAY_SERVE_INGRESS_REQUEST_ROUTER_FORWARD_BODY = get_env_bool(
    "RAY_SERVE_INGRESS_REQUEST_ROUTER_FORWARD_BODY", False
)

# Emit per-request metrics from the ingress-request-router data path:
# - truncated body counter
# - router consultation latency histogram
# - replica-id mismatch counter (router pinned X, HAProxy used Y after fallthrough)
#
# When enabled, HAProxy logs an RFC 5424 line with metric fields in the
# structured-data section to RAY_SERVE_HAPROXY_METRICS_SOCKET_PATH, and the
# HAProxy proxy actor parses each datagram into ray.serve.metrics Counter /
# Histogram objects. When disabled, neither the log target nor the Lua timing
# calls are rendered into the generated config -- there is no runtime cost.
RAY_SERVE_INGRESS_REQUEST_ROUTER_METRICS_ENABLED = get_env_bool(
    "RAY_SERVE_INGRESS_REQUEST_ROUTER_METRICS_ENABLED", "0"
)

# Unix dgram socket that HAProxy writes the structured metric log lines to.
# Bound by the proxy actor before HAProxy is started. Only consulted when
# RAY_SERVE_INGRESS_REQUEST_ROUTER_METRICS_ENABLED is true.
RAY_SERVE_HAPROXY_METRICS_SOCKET_PATH = os.environ.get(
    "RAY_SERVE_HAPROXY_METRICS_SOCKET_PATH", "/tmp/haproxy-serve/metrics.sock"
)

RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT = int(
    os.environ.get("RAY_SERVE_DIRECT_INGRESS_MIN_HTTP_PORT", "30000")
)
RAY_SERVE_DIRECT_INGRESS_MIN_GRPC_PORT = int(
    os.environ.get("RAY_SERVE_DIRECT_INGRESS_MIN_GRPC_PORT", "40000")
)
RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT = int(
    os.environ.get("RAY_SERVE_DIRECT_INGRESS_MAX_HTTP_PORT", "31000")
)
RAY_SERVE_DIRECT_INGRESS_MAX_GRPC_PORT = int(
    os.environ.get("RAY_SERVE_DIRECT_INGRESS_MAX_GRPC_PORT", "41000")
)
RAY_SERVE_DIRECT_INGRESS_PORT_RETRY_COUNT = int(
    os.environ.get("RAY_SERVE_DIRECT_INGRESS_PORT_RETRY_COUNT", "100")
)

# Hold released replica ports out of the pool this long so proxies can
# drop their stale slot before a new replica grabs the same port. 0 disables.
# Defaults to hard-stop-after plus a margin: soft-stopped (reloaded-out)
# HAProxy workers run no health checks and keep routing to their frozen
# server list until hard-stop-after fires, so a freed port must stay out
# of the pool at least that long or another app's replica can inherit the
# old app's traffic. The margin covers the broadcast/reload lag before an
# old worker's hard-stop clock starts.
RAY_SERVE_PORT_QUARANTINE_S = get_env_float_non_negative(
    "RAY_SERVE_PORT_QUARANTINE_S",
    float(RAY_SERVE_HAPROXY_HARD_STOP_AFTER_S + 30),
)

# The minimum drain period for a HTTP proxy.
# If RAY_SERVE_FORCE_STOP_UNHEALTHY_REPLICAS is set to 1,
# then the minimum draining period is 0.
RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S = float(
    os.environ.get("RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S", "30")
)

# Grace added on top of the min draining period when flooring an ingress
# deployment's graceful_shutdown_timeout_s.
RAY_SERVE_DIRECT_INGRESS_SHUTDOWN_BUFFER_S = 5

# HTTP request timeout
SERVE_HTTP_REQUEST_TIMEOUT_S_HEADER = "x-request-timeout-seconds"

# HTTP request disconnect disabled
SERVE_HTTP_REQUEST_DISCONNECT_DISABLED_HEADER = "x-request-disconnect-disabled"

# Path to tracing exporter function
# If empty string (default), then tracing is disabled
RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH = os.environ.get(
    "RAY_SERVE_TRACING_EXPORTER_IMPORT_PATH", ""
)
DEFAULT_TRACING_EXPORTER_IMPORT_PATH = (
    "ray.serve._private.tracing_utils:default_tracing_exporter"
)
RAY_SERVE_TRACING_SAMPLING_RATIO = float(
    os.environ.get("RAY_SERVE_TRACING_SAMPLING_RATIO", 0.01)
)

# If throughput optimized Ray Serve is enabled, set the following constants.
# This should be at the end.
RAY_SERVE_THROUGHPUT_OPTIMIZED = get_env_bool("RAY_SERVE_THROUGHPUT_OPTIMIZED", "0")
if RAY_SERVE_THROUGHPUT_OPTIMIZED:
    RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD = get_env_bool(
        "RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD", "0"
    )
    RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE = get_env_int(
        "RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE", 1000
    )
    RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP = get_env_bool(
        "RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP", "0"
    )
    RAY_SERVE_LOG_TO_STDERR = get_env_bool("RAY_SERVE_LOG_TO_STDERR", "0")
    RAY_SERVE_USE_GRPC_BY_DEFAULT = get_env_bool("RAY_SERVE_USE_GRPC_BY_DEFAULT", "1")
    RAY_SERVE_ENABLE_DIRECT_INGRESS = get_env_bool(
        "RAY_SERVE_ENABLE_DIRECT_INGRESS", "1"
    )

if RAY_SERVE_ENABLE_HA_PROXY:
    # Direct ingress must be enabled if HAProxy is enabled.
    RAY_SERVE_ENABLE_DIRECT_INGRESS = True

    # Replica HTTP ports must be reachable from HAProxy on remote nodes, so
    # the effective default binds to all interfaces regardless of
    # RAY_SERVE_DEFAULT_HTTP_HOST.
    if DEFAULT_HTTP_HOST not in (None, get_all_interfaces_ip()):
        logger.warning(
            f"RAY_SERVE_DEFAULT_HTTP_HOST={DEFAULT_HTTP_HOST!r} is ignored "
            "because RAY_SERVE_ENABLE_HA_PROXY=1 forces host to all interfaces "
            "so HAProxy on other nodes can reach Serve HTTP ports."
        )
    DEFAULT_HTTP_HOST = get_all_interfaces_ip()

if RAY_SERVE_INGRESS_REQUEST_ROUTER_METRICS_ENABLED:
    RAY_SERVE_HAPROXY_METRICS_ENABLED = True

# Feature flag to aggregate metrics at the controller instead of the replicas or handles.
RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER = get_env_bool(
    "RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER", "0"
)

# Feature flag to include high-cardinality source tags on Serve controller metrics.
# Disable this to keep deployment/application tags while dropping source identifiers
# like replica IDs from controller-emitted metrics.
RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS = get_env_bool(
    "RAY_SERVE_CONTROLLER_METRICS_INCLUDE_HIGH_CARDINALITY_TAGS", "1"
)

# Feature flag to use compact (low-cardinality) namespace tags on long poll metrics.
# When enabled, metric tags use only the LongPollNamespace enum name
# (e.g., "DEPLOYMENT_CONFIG") instead of the full key string which includes
# per-deployment identifiers. This bounds metric cardinality to ~6 namespace types
# instead of scaling with the number of deployments.
# Recommended for workloads with a large number (>1000) of deployments.
RAY_SERVE_COMPACT_LONG_POLL_METRIC_TAGS = get_env_bool(
    "RAY_SERVE_COMPACT_LONG_POLL_METRIC_TAGS", "0"
)
# Key for the decision counters in default autoscaling policy state
SERVE_AUTOSCALING_DECISION_COUNTERS_KEY = "__decision_counters"
# Key for the wall-clock timestamp when a scaling decision was first observed
SERVE_AUTOSCALING_DECISION_TIMESTAMP_KEY = "__decision_timestamp"

# Event loop monitoring interval in seconds.
# This is how often the event loop lag is measured.
RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S = get_env_float_positive(
    "RAY_SERVE_EVENT_LOOP_MONITORING_INTERVAL_S", 5.0
)

# Histogram buckets for event loop scheduling latency in milliseconds.
# These are tuned for detecting event loop blocking:
# - < 10ms: healthy
# - 10-50ms: acceptable under load
# - 50-100ms: concerning, investigate
# - 100-500ms: problematic, likely blocking code
# - > 500ms: severe, definitely blocking
# - > 5s: catastrophic
SERVE_EVENT_LOOP_LATENCY_HISTOGRAM_BOUNDARIES_MS = [
    1,  # 1ms
    5,  # 5ms
    10,  # 10ms
    25,  # 25ms
    50,  # 50ms
    100,  # 100ms
    250,  # 250ms
    500,  # 500ms
    1000,  # 1s
    2500,  # 2.5s
    5000,  # 5s
    10000,  # 10s
]