HAPROXY_HEALTHZ_RULES_TEMPLATE = """    # Health check endpoint
    acl healthcheck path -i {{ config.health_check_endpoint }}
    # Keep health checks out of the access log but still record their metric:
    # tag them `debug` so the access-log target (level info) drops them while the
    # metrics socket (level debug) keeps them. Mirrors the proxy, which records
    # the healthz metric with should_record_access_log=False.
    http-request set-log-level debug if healthcheck
    {%- if config.metrics_enabled %}
    http-request set-var-fmt(txn.serve_route) {{ config.health_check_endpoint | haproxy_fmt }} if healthcheck
    {%- endif %}
{%- if not health_info.healthy %}
    # Override: force health checks to fail (used by drain/disable)
    http-request return status {{ health_info.status }} content-type text/plain string "{{ health_info.health_message }}" if healthcheck
{%- elif backends %}
    # 200 if any backend has at least one server UP
{%-   for backend in backends %}
    acl backend_{{ backend.name or 'unknown' }}_server_up nbsrv({{ backend.name or 'unknown' }}) ge 1
{%-   endfor %}
    # Any backend with a server UP passes the health check (OR logic)
{%-   for backend in backends %}
    http-request return status {{ health_info.status }} content-type text/plain string "{{ health_info.health_message }}" if healthcheck backend_{{ backend.name or 'unknown' }}_server_up
{%-   endfor %}
    http-request return status 503 content-type text/plain string "Service Unavailable" if healthcheck
{%- elif config.is_head %}
    # Head is the always-on ingress endpoint, so it stays ready with no backends.
    # Mirrors is_head in ProxyRouter.ready_for_traffic.
    http-request return status {{ health_info.status }} content-type text/plain string "{{ health_info.health_message }}" if healthcheck
{%- endif %}
"""

# Same shape as HAPROXY_HEALTHZ_RULES_TEMPLATE, but emits gRPC trailers-only
# responses. The OK / UNAVAILABLE distinction maps to grpc-status 0 / 14
# (gRPC clients map HTTP 503 to UNAVAILABLE, which is why every shape here
# uses HTTP 200 and signals state through grpc-status).
HAPROXY_GRPC_HEALTHZ_RULES_TEMPLATE = """    # Health check endpoint (gRPC `Healthz`)
    acl is_healthz path /ray.serve.RayServeAPIService/Healthz
    # Suppress logging for health checks
    http-request set-log-level silent if is_healthz
{%- if not health_info.healthy %}
    # Override: force health checks to fail (used by drain/disable)
    http-request return status 200 content-type application/grpc hdr grpc-status 14 hdr grpc-message "{{ health_info.health_message }}" if is_healthz
{%- elif backends %}
    # OK if any backend has at least one server UP
{%-   for backend in backends %}
    acl backend_{{ backend.name or 'unknown' }}_server_up nbsrv({{ backend.name or 'unknown' }}) ge 1
{%-   endfor %}
    # Any backend with a server UP passes the health check (OR logic)
{%-   for backend in backends %}
    http-request return status 200 content-type application/grpc hdr grpc-status 0 if is_healthz backend_{{ backend.name or 'unknown' }}_server_up
{%-   endfor %}
    http-request return status 200 content-type application/grpc hdr grpc-status 14 hdr grpc-message "Service Unavailable" if is_healthz
{%- elif config.is_head %}
    # Head is the always-on ingress endpoint, so it stays ready with no backends.
    # Mirrors is_head in ProxyRouter.ready_for_traffic.
    http-request return status 200 content-type application/grpc hdr grpc-status 0 if is_healthz
{%- endif %}
"""

HAPROXY_CONFIG_TEMPLATE = """global
    # Access/event log at `info`. The system endpoints (/-/healthz, /-/routes)
    # tag their logs `debug` so they are dropped here but still reach the
    # rfc5424 metrics socket (level `debug`) when metrics are enabled -- mirroring
    # the proxy, which records their metric but not their access log.
    log {{ config.log_target }} local0 info
    stats socket {{ config.socket_path }} mode 666 level admin expose-fd listeners
    stats timeout 30s
    maxconn {{ config.maxconn }}
    nbthread {{ config.nbthread }}
    {%- if has_ingress_request_router %}
    lua-load-per-thread {{ ingress_request_router_lua_path }}
    {%- endif %}
    {%- if has_ingress_request_router and ingress_request_router_forward_body %}
    tune.bufsize {{ ingress_request_router_bufsize }}
    {%- else %}
    tune.bufsize {{ config.bufsize }}
    {%- endif %}
    {%- if config.enable_hap_optimization %}
    server-state-base {{ config.server_state_base }}
    server-state-file {{ config.server_state_file }}
    {%- endif %}
    {%- if config.hard_stop_after_s is not none %}
    hard-stop-after {{ config.hard_stop_after_s }}s
    {%- endif %}
    {%- if config.grpc_enabled %}
    tune.h2.max-frame-size {{ config.h2_max_frame_size }}
    tune.h2.be.initial-window-size {{config.h2_be_initial_window_size}}
    tune.h2.be.max-concurrent-streams {{config.h2_be_max_concurrent_streams}}
    tune.h2.fe.initial-window-size {{config.h2_fe_initial_window_size}}
    tune.h2.fe.max-concurrent-streams {{config.h2_fe_max_concurrent_streams}}
    {%- endif %}
defaults
    mode http
    option log-health-checks
    {% if config.timeout_connect_s is not none %}timeout connect {{ config.timeout_connect_s }}s{% endif %}
    {% if config.timeout_client_s is not none %}timeout client {{ config.timeout_client_s }}s{% endif %}
    {% if config.timeout_server_s is not none %}timeout server {{ config.timeout_server_s }}s{% endif %}
    {% if config.timeout_http_request_s is not none %}timeout http-request {{ config.timeout_http_request_s }}s{% endif %}
    {% if config.timeout_http_keep_alive_s is not none %}timeout http-keep-alive {{ config.timeout_http_keep_alive_s }}s{% endif %}
    {% if config.timeout_queue_s is not none %}timeout queue {{ config.timeout_queue_s }}s{% endif %}
    log global
    option httplog
    option abortonclose
    option splice-request
    option splice-response
    # On a retry, use a different slot (`1`). retry-on defaults to connect
    # failures only (nothing was sent → safe to replay); override globally via
    # RAY_SERVE_HAPROXY_RETRY_ON. Inherited by every backend.
    option redispatch 1
    retry-on {{ config.retry_on }}
    {%- if config.retries is not none %}
    retries {{ config.retries }}
    {%- endif %}
    {%- if config.tcp_nodelay %}
    # Set TCP_NODELAY on all connections
    option http-no-delay
    {%- endif %}
    {%- if config.enable_hap_optimization %}
    option idle-close-on-response
    {%- endif %}
    # Normalize 502/503/504 to 500 per Serve's default behavior. 503
    # covers HAProxy's own "all retries exhausted / no server" response.
    {%- if config.error_file_path %}
    errorfile 502 {{ config.error_file_path }}
    errorfile 503 {{ config.error_file_path }}
    errorfile 504 {{ config.error_file_path }}
    {%- endif %}
    {%- if config.enable_hap_optimization %}
    load-server-state-from-file global
    {%- endif %}
    balance {{ config.balance_algorithm }}
frontend prometheus
    bind :{{ config.metrics_port }}
    mode http
    http-request use-service prometheus-exporter if { path {{ config.metrics_uri }} }
    no log
frontend http_frontend
    bind {{ config.frontend_host }}:{{ config.frontend_port }}
    {%- if config.metrics_enabled %}
    log global
    # Per-request HTTP ingress metrics. One RFC 5424 line per request matched to
    # a Serve app backend, scraped into the serve_num_http_* /
    # serve_http_request_latency_ms families (the metrics the Python proxy emits
    # in non-HAProxy mode). Goes only to the rfc5424 target below; the inherited
    # rfc3164 targets do not include the SD section, so their byte stream is
    # unchanged. The general fields come from txn.serve_* vars set per backend
    # below; %ST/%Ta/%ts render unquoted (HAProxy does not quote those aliases).
    # term_state (%ts) is HAProxy's 2-char session termination state; a leading "C"
    # means the client aborted, which the collector maps to status 499 to match the
    # Python proxy's client-disconnect convention. When ingress-request-router
    # metrics are also enabled, the router-specific fields are appended to the same
    # line.
    log {{ metrics_socket_path }} len 8192 format rfc5424 local1 debug
    log-format-sd "%{+Q,+E}o [serve@1 app=%[var(txn.serve_app)] route=%[var(txn.serve_route)] method=%HM status=%ST latency_ms=%Ta deployment=%[var(txn.serve_deployment)] term_state=%ts{% if ingress_request_router_metrics_enabled and has_ingress_request_router %} intended=%[var(txn.ingress_request_router_target)] actual=%s router_latency_us=%[var(txn.ingress_request_router_latency_us)] body_truncated_full_length=%[var(txn.ingress_request_router_truncated_full_length)] via_router=%[var(txn.via_ingress_request_router)] failed=%[var(txn.ingress_request_router_failed)]{% endif %}]"
    {%- endif %}
    {%- if config.root_path %}
    # Strip the configured global root_path so the health/routes endpoints, the
    # per-backend path ACLs, and the path forwarded to replicas are all
    # root_path-agnostic. Mirrors the native Serve proxy, which mounts the app
    # under root_path. An exact root_path match becomes "/", and paths outside
    # root_path are left unchanged.
    http-request set-path / if { path {{ config.root_path }} }
    http-request set-path %[path,regsub(^{{ config.root_path }}/,/)] if { path_beg {{ config.root_path }}/ }
    {%- endif %}
{{ healthz_rules|safe }}
    # Routes endpoint
    acl routes path -i /-/routes
    # Like health checks: kept out of the access log (tagged `debug`); its metric
    # is recorded (route=/-/routes, app unset) when metrics are enabled.
    http-request set-log-level debug if routes
    {%- if config.metrics_enabled %}
    http-request set-var-fmt(txn.serve_route) {{ '/-/routes' | haproxy_fmt }} if routes
    {%- endif %}
    http-request return status {{ route_info.status }} content-type {{ route_info.routes_content_type }} string "{{ route_info.routes_message }}" if routes

    {%- if config.inject_process_id_header and config.reload_id %}
    # Inject unique reload ID as header to track which HAProxy instance handled the request (testing only)
    http-request set-header x-haproxy-reload-id {{ config.reload_id }}
    {%- endif %}
    # Per-backend path ACLs (used for both ingress-request-router dispatch
    # and static use_backend selection below).
{%- for backend in backends %}
    acl is_{{ backend.name or 'unknown' }} path_beg {{ '/' if not backend.path_prefix or backend.path_prefix == '/' else backend.path_prefix ~ '/' }}
    acl is_{{ backend.name or 'unknown' }} path {{ backend.path_prefix or '/' }}
{%- endfor %}
    {%- if config.metrics_enabled %}
    # Per-request HTTP metric vars (app / route / ingress deployment), set on the
    # first matching backend. Backends are sorted longest-prefix-first and the
    # !found guard makes the longest match win, mirroring the use_backend rules
    # below. Requests that match no app backend (e.g. /-/routes, 404s) leave
    # these unset, so the collector can skip them.
    {%- for backend in backends %}
    http-request set-var-fmt(txn.serve_app) {{ (backend.app_name or 'unknown') | haproxy_fmt }} if is_{{ backend.name or 'unknown' }} !{ var(txn.serve_app) -m found }
    http-request set-var-fmt(txn.serve_route) {{ (backend.path_prefix or '/') | haproxy_fmt }} if is_{{ backend.name or 'unknown' }} !{ var(txn.serve_route) -m found }
    {%- if backend.ingress_deployment_name %}
    http-request set-var-fmt(txn.serve_deployment) {{ backend.ingress_deployment_name | haproxy_fmt }} if is_{{ backend.name or 'unknown' }} !{ var(txn.serve_deployment) -m found }
    {%- endif %}
    {%- endfor %}
    {%- endif %}
    {%- if has_ingress_request_router %}
    # Set txn.ingress_request_router_app to the first matching router-bearing
    # backend. Backends are sorted longest-prefix-first, and the !found guard
    # ensures only the longest match wins.
    {%- for backend in backends %}
    {%- if backend.ingress_request_router_servers %}
    http-request set-var(txn.ingress_request_router_app) str({{ backend.name or 'unknown' }}) if is_{{ backend.name or 'unknown' }} !{ var(txn.ingress_request_router_app) -m found }
    {%- endif %}
    {%- endfor %}
    acl has_ingress_request_router_app var(txn.ingress_request_router_app) -m found
    {%- if ingress_request_router_forward_body %}
    http-request wait-for-body time {{ ingress_request_router_timeout_s }}s if METH_POST has_ingress_request_router_app
    {%- endif %}
    http-request lua.route_via_ingress_request_router if METH_POST has_ingress_request_router_app
    # A pin-miss is recoverable only if its app has a fallback proxy. Mark it
    # per app so the 503 below fails loud for apps with none.
    {%- for backend in backends %}
    {%- if backend.ingress_request_router_servers and backend.fallback_server %}
    http-request set-var(txn.ingress_request_router_recoverable) str(1) if { var(txn.ingress_request_router_app) -m str "{{ backend.name or 'unknown' }}" } { var(txn.ingress_request_router_failed) -m str "unknown_replica_id" }
    {%- endif %}
    {%- endfor %}
    # 503 on any router failure except a recoverable pin-miss. Must precede the
    # use_backend rules so failures never fall through to the primary backend.
    http-request return status 503 content-type text/plain lf-string "Ingress request router failed: %[var(txn.ingress_request_router_failed)]" hdr X-Serve-Reason %[var(txn.ingress_request_router_failed)] if { var(txn.ingress_request_router_failed) -m found } !{ var(txn.ingress_request_router_recoverable) -m found }
    {%- endif %}
    # Static routing based on path prefixes in decreasing length then alphabetical order
{%- for backend in backends %}
    {%- if has_ingress_request_router and backend.ingress_request_router_servers %}
    use_backend {{ backend.name or 'unknown' }}-via-ingress-request-router if is_{{ backend.name or 'unknown' }} { var(txn.via_ingress_request_router) -m found }
    {%- if backend.fallback_server %}
    # Pin-miss recovery: route into the router backend, which picks the fallback.
    use_backend {{ backend.name or 'unknown' }}-via-ingress-request-router if is_{{ backend.name or 'unknown' }} { var(txn.ingress_request_router_failed) -m str "unknown_replica_id" }
    {%- endif %}
    {%- endif %}
    use_backend {{ backend.name or 'unknown' }} if is_{{ backend.name or 'unknown' }}
{%- endfor %}
    default_backend default_backend
backend default_backend
    http-request return status 404 content-type text/plain lf-string "Path \'%[path]\' not found. Ping http://.../-/routes for available routes."
{%- for item in backends_with_health_config %}
{%- set backend = item.backend %}
{%- set hc = item.health_config %}
backend {{ backend.name or 'unknown' }}
    log global
    # Enable HTTP connection reuse for better performance
    http-reuse always
    # Set backend-specific timeouts, overriding defaults if specified
    {%- if backend.timeout_connect_s is not none %}
    timeout connect {{ backend.timeout_connect_s }}s
    {%- endif %}
    {%- if backend.timeout_server_s is not none %}
    timeout server {{ backend.timeout_server_s }}s
    {%- endif %}
    {%- if backend.timeout_client_s is not none %}
    timeout client {{ backend.timeout_client_s }}s
    {%- endif %}
    {%- if backend.timeout_http_request_s is not none %}
    timeout http-request {{ backend.timeout_http_request_s }}s
    {%- endif %}
    {%- if backend.timeout_queue_s is not none %}
    timeout queue {{ backend.timeout_queue_s }}s
    {%- endif %}
    # Set timeouts to support keep-alive connections
    {%- if backend.timeout_http_keep_alive_s is not none %}
    timeout http-keep-alive {{ backend.timeout_http_keep_alive_s }}s
    {%- endif %}
    {%- if backend.timeout_tunnel_s is not none %}
    timeout tunnel {{ backend.timeout_tunnel_s }}s
    {%- endif %}
    # Health check configuration - use backend-specific or global defaults
    {%- if hc.health_path %}
    # HTTP health check with custom path
    option httpchk GET {{ hc.health_path }}
    http-check expect status 200
    {%- endif %}
    {{ hc.default_server_directive }}
    # Servers in this backend
    {%- for server in backend.servers %}
    server {{ server.name }} {{ server.host }}:{{ server.port }} check
    {%- endfor %}
    {%- if backend.fallback_server %}
    # Fallback to head node's Serve proxy when no ingress replicas are available
    server {{ backend.fallback_server.name }} {{ backend.fallback_server.host }}:{{ backend.fallback_server.port }} check backup
    {%- endif %}
{%- if has_ingress_request_router and backend.ingress_request_router_servers %}
backend {{ backend.name or 'unknown' }}-via-ingress-request-router
    log global
    # Keep the pinned data-plane path on the same connection policy as the
    # primary backend. For streamed responses, forcing server-close can leave
    # HAProxy holding unread server-side FINs under a burst while worker
    # threads are still routing other requests.
    http-reuse always
    # Inherits the defaults block's `option redispatch 1` + retry-on, so a
    # DOWN/slow pinned server falls through to a different replica instead of
    # head-of-line-blocking on the original pick. One retry policy everywhere.
    {%- if backend.timeout_connect_s is not none %}
    timeout connect {{ backend.timeout_connect_s }}s
    {%- endif %}
    {%- if config.ingress_timeout_server_s is not none %}
    timeout server {{ config.ingress_timeout_server_s }}s
    {%- elif backend.timeout_server_s is not none %}
    timeout server {{ backend.timeout_server_s }}s
    {%- endif %}
    {%- if backend.timeout_http_keep_alive_s is not none %}
    timeout http-keep-alive {{ backend.timeout_http_keep_alive_s }}s
    {%- endif %}
    {%- for server in backend.servers %}
    use-server {{ server.name }} if { var(txn.ingress_request_router_target) -m str "{{ server.name }}" }
    {%- endfor %}
    {%- if backend.fallback_server %}
    # Pin-miss: route to the fallback Serve proxy, which re-pins via its own
    # router. If the fallback is DOWN this use-server is skipped and the request
    # load-balances onto a primary replica in this backend, so affinity lapses
    # until the fallback's health check passes. That is plain selection-time
    # fallthrough, not `option redispatch` (which only re-picks after a
    # connection failure to an already-selected server).
    use-server {{ backend.fallback_server.name }} if { var(txn.ingress_request_router_failed) -m str "unknown_replica_id" }
    {%- endif %}
    # `track` allows us to mirror primary-backend health and avoid double-checking.
    {%- for server in backend.servers %}
    server {{ server.name }} {{ server.host }}:{{ server.port }} track {{ backend.name or 'unknown' }}/{{ server.name }}
    {%- endfor %}
    {%- if backend.fallback_server %}
    server {{ backend.fallback_server.name }} {{ backend.fallback_server.host }}:{{ backend.fallback_server.port }} track {{ backend.name or 'unknown' }}/{{ backend.fallback_server.name }} backup
    {%- endif %}
{%- endif %}
{%- endfor %}
{%- if config.grpc_enabled %}
frontend grpc_frontend
    # gRPC requires HTTP/2. HAProxy decodes H2 frames into HTTP request
    # semantics in `mode http` when `proto h2` is on the bind line.
    bind {{ config.grpc_frontend_host }}:{{ config.grpc_frontend_port }} proto h2
    mode http
    log global
    # No per-request ingress-metrics log line here (unlike http_frontend): a
    # gRPC call's real status is `grpc-status`, sent in HTTP/2 trailers on
    # success, and HAProxy cannot read trailers in log-format (haproxy/haproxy#112).
    # gRPC ingress metrics are therefore emitted by the replica instead.

{{ grpc_healthz_rules|safe }}

    # ListApplications must aggregate across all apps, so it goes to the
    # head-node fallback Serve proxy rather than an individual replica.
    acl is_list_applications path /ray.serve.RayServeAPIService/ListApplications
{%- if grpc_fallback_backend_with_health_config %}
    use_backend grpc_fallback_backend if is_list_applications
{%- else %}
    http-request return status 200 content-type application/grpc hdr grpc-status 14 hdr grpc-message "ListApplications is unavailable" if is_list_applications
{%- endif %}

    # Route per-app on the `application` metadata that Ray Serve clients attach.
{%- for backend in grpc_backends %}
    acl is_{{ backend.name or 'unknown' }} req.hdr(application) -m str {{ backend.app_name }}
    use_backend {{ backend.name or 'unknown' }} if is_{{ backend.name or 'unknown' }}
{%- endfor %}
{%- if grpc_backends|length == 1 %}
    # With exactly one app deployed, route there regardless of metadata so
    # clients can call it without setting the `application` header.
    default_backend {{ grpc_backends[0].name or 'unknown' }}
{%- else %}
    # Zero apps, or multiple apps without a matching `application` header.
    default_backend default_grpc_backend
{%- endif %}

{%- if grpc_fallback_backend_with_health_config %}
{%- set backend = grpc_fallback_backend_with_health_config.backend %}
{%- set hc = grpc_fallback_backend_with_health_config.health_config %}
{%- if backend.servers %}
{%- set server = backend.servers[0] %}
backend grpc_fallback_backend
    mode http
    log global
    # gRPC health check: replay a complete unary `Healthz` request via
    # `tcp-check send-binary` and match the healthy message in the response.
    # `http-check` can't be used because its body is truncated at the first NUL
    # byte and a gRPC frame always starts with the NUL compression flag, so the
    # server would get a message-less unary and stall until timeout.
    option tcp-check
    tcp-check connect
    tcp-check send-binary {{ hc.grpc_healthcheck_request_hex }}
    tcp-check expect binary {{ hc.grpc_healthcheck_expect_hex }}
    {{ hc.default_server_directive }}
    # `proto h2` makes HAProxy speak HTTP/2 cleartext to the fallback gRPC server.
    server {{ server.name }} {{ server.host }}:{{ server.port }} proto h2 check
{%- endif %}
{%- endif %}
{%- if grpc_backends|length != 1 %}
backend default_grpc_backend
    mode http
    log global
    # Trailers-only NOT_FOUND. gRPC clients surface this as
    # grpc.StatusCode.NOT_FOUND; an HTTP 503 would map to UNAVAILABLE instead.
    acl has_application_header req.hdr(application) -m found
    http-request return status 200 content-type application/grpc hdr grpc-status 5 hdr grpc-message "Application '%[req.hdr(application)]' not found. Ping /ray.serve.RayServeAPIService/ListApplications for available applications." if has_application_header
    http-request return status 200 content-type application/grpc hdr grpc-status 5 hdr grpc-message "Application metadata not set. Ping /ray.serve.RayServeAPIService/ListApplications for available applications."
{%- endif %}
{%- for item in grpc_backends_with_health_config %}
{%- set backend = item.backend %}
{%- set hc = item.health_config %}
backend {{ backend.name or 'unknown' }}
    mode http
    log global
    http-reuse always
    {%- if backend.timeout_connect_s is not none %}
    timeout connect {{ backend.timeout_connect_s }}s
    {%- endif %}
    {%- if backend.timeout_server_s is not none %}
    timeout server {{ backend.timeout_server_s }}s
    {%- endif %}
    {%- if backend.timeout_client_s is not none %}
    timeout client {{ backend.timeout_client_s }}s
    {%- endif %}
    # gRPC health check: replay a complete unary `Healthz` request via
    # `tcp-check send-binary` and match the healthy message in the response.
    # `http-check` can't be used because its body is truncated at the first NUL
    # byte and a gRPC frame always starts with the NUL compression flag, so the
    # server would get a message-less unary and stall until timeout.
    option tcp-check
    tcp-check connect
    tcp-check send-binary {{ hc.grpc_healthcheck_request_hex }}
    tcp-check expect binary {{ hc.grpc_healthcheck_expect_hex }}
    {{ hc.default_server_directive }}
    # `proto h2` makes HAProxy speak HTTP/2 cleartext to backend gRPC servers.
    {%- for server in backend.servers %}
    server {{ server.name }} {{ server.host }}:{{ server.port }} proto h2 check
    {%- endfor %}
    {%- if backend.fallback_server %}
    server {{ backend.fallback_server.name }} {{ backend.fallback_server.host }}:{{ backend.fallback_server.port }} proto h2 check backup
    {%- endif %}
{%- endfor %}
{%- endif %}
listen stats
  bind *:{{ config.stats_port }}
  stats enable
  stats uri {{ config.stats_uri }}
  stats refresh 1s
"""