chore: import upstream snapshot with attribution

java/serve/pom_template.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+{auto_gen_header}
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xmlns="http://maven.apache.org/POM/4.0.0"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <parent>
+    <groupId>io.ray</groupId>
+    <artifactId>ray-superpom</artifactId>
+    <version>2.0.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>ray-serve</artifactId>
+  <name>ray serve</name>
+  <description>java for ray serve</description>
+
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.ray</groupId>
+      <artifactId>ray-api</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>io.ray</groupId>
+      <artifactId>ray-runtime</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+{generated_bzl_deps}
+  </dependencies>
+</project>

java/serve/src/main/java/io/ray/serve/api/Serve.java

@@ -0,0 +1,420 @@
+package io.ray.serve.api;
+
+import com.google.common.base.Preconditions;
+import io.ray.api.BaseActorHandle;
+import io.ray.api.PyActorHandle;
+import io.ray.api.Ray;
+import io.ray.api.exception.RayActorException;
+import io.ray.api.exception.RayTimeoutException;
+import io.ray.api.function.PyActorClass;
+import io.ray.api.function.PyActorMethod;
+import io.ray.api.options.ActorLifetime;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.dag.Graph;
+import io.ray.serve.deployment.Application;
+import io.ray.serve.deployment.Deployment;
+import io.ray.serve.deployment.DeploymentCreator;
+import io.ray.serve.deployment.DeploymentRoute;
+import io.ray.serve.exception.RayServeException;
+import io.ray.serve.generated.ActorNameList;
+import io.ray.serve.handle.DeploymentHandle;
+import io.ray.serve.poll.LongPollClientFactory;
+import io.ray.serve.replica.ReplicaContext;
+import io.ray.serve.util.CollectionUtil;
+import io.ray.serve.util.MessageFormatter;
+import io.ray.serve.util.ServeProtoUtil;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Ray Serve global API. TODO: will be riched in the Java SDK/API PR. */
+public class Serve {
+  private static final Logger LOGGER = LoggerFactory.getLogger(Serve.class);
+
+  private static ReplicaContext INTERNAL_REPLICA_CONTEXT;
+
+  private static ServeControllerClient GLOBAL_CLIENT;
+
+  /**
+   * Initialize a serve instance.
+   *
+   * @param config Configuration options for Serve.
+   * @return
+   */
+  public static synchronized ServeControllerClient start(Map<String, String> config) {
+    return serveStart(config);
+  }
+
+  private static synchronized ServeControllerClient serveStart(Map<String, String> config) {
+
+    try {
+      ServeControllerClient client = getGlobalClient(true);
+      LOGGER.info("Connecting to existing Serve app in namespace {}", Constants.SERVE_NAMESPACE);
+      return client;
+    } catch (RayServeException | IllegalStateException e) {
+      LOGGER.info(
+          "There is no Serve instance running on this Ray cluster. A new one will be started.");
+    }
+
+    // Initialize ray if needed.
+    if (!Ray.isInitialized()) {
+      init();
+    }
+
+    int httpPort =
+        Optional.ofNullable(config)
+            .map(m -> m.get(RayServeConfig.PROXY_HTTP_PORT))
+            .map(Integer::parseInt)
+            .orElse(Integer.valueOf(System.getProperty(RayServeConfig.PROXY_HTTP_PORT, "8000")));
+    PyActorHandle controllerAvatar =
+        Ray.actor(
+                PyActorClass.of("ray.serve._private.controller_avatar", "ServeControllerAvatar"),
+                httpPort)
+            .setName(Constants.SERVE_CONTROLLER_NAME + "_AVATAR")
+            .setLifetime(ActorLifetime.DETACHED)
+            .setMaxRestarts(-1)
+            .setMaxConcurrency(1)
+            .remote();
+
+    controllerAvatar.task(PyActorMethod.of("check_alive")).remote().get();
+
+    PyActorHandle controller =
+        (PyActorHandle)
+            Ray.getActor(Constants.SERVE_CONTROLLER_NAME, Constants.SERVE_NAMESPACE).get();
+
+    ActorNameList actorNameList =
+        ServeProtoUtil.bytesToProto(
+            (byte[]) controller.task(PyActorMethod.of("get_proxy_names")).remote().get(),
+            ActorNameList::parseFrom);
+    if (actorNameList != null && !CollectionUtil.isEmpty(actorNameList.getNamesList())) {
+      try {
+        for (String name : actorNameList.getNamesList()) {
+          PyActorHandle proxyActorHandle =
+              (PyActorHandle) Ray.getActor(name, Constants.SERVE_NAMESPACE).get();
+          proxyActorHandle
+              .task(PyActorMethod.of("ready"))
+              .remote()
+              .get(Constants.PROXY_TIMEOUT_S * 1000);
+        }
+      } catch (RayTimeoutException e) {
+        String errMsg =
+            MessageFormatter.format(
+                "HTTP proxies not available after {}s.", Constants.PROXY_TIMEOUT_S);
+        LOGGER.error(errMsg, e);
+        throw new RayServeException(errMsg, e);
+      }
+    }
+
+    ServeControllerClient client = new ServeControllerClient(controller);
+    setGlobalClient(client);
+    LOGGER.info("Started Serve in namespace {}", Constants.SERVE_NAMESPACE);
+    return client;
+  }
+
+  /**
+   * Completely shut down the connected Serve instance.
+   *
+   * <p>Shuts down all processes and deletes all state associated with the instance.
+   */
+  public static void shutdown() {
+    ServeControllerClient client = null;
+    try {
+      client = getGlobalClient();
+    } catch (RayServeException | IllegalStateException e) {
+      LOGGER.info(
+          "Nothing to shut down. There's no Serve application running on this Ray cluster.");
+      return;
+    }
+
+    LongPollClientFactory.stop();
+    client.shutdown(null);
+    clearContext();
+  }
+
+  public static void clearContext() {
+    setGlobalClient(null);
+    setInternalReplicaContext(null);
+  }
+
+  /**
+   * Define a Serve deployment.
+   *
+   * @return DeploymentCreator
+   */
+  public static DeploymentCreator deployment() {
+    return new DeploymentCreator();
+  }
+
+  /**
+   * Set replica information to global context.
+   *
+   * @param deploymentName deployment name
+   * @param replicaTag replica tag
+   * @param servableObject the servable object of the specified replica.
+   * @param config
+   */
+  public static void setInternalReplicaContext(
+      String deploymentName,
+      String replicaTag,
+      Object servableObject,
+      Map<String, String> config,
+      String appName) {
+    INTERNAL_REPLICA_CONTEXT =
+        new ReplicaContext(deploymentName, replicaTag, servableObject, config, appName);
+  }
+
+  /**
+   * Set replica information to global context.
+   *
+   * @param replicaContext
+   */
+  public static void setInternalReplicaContext(ReplicaContext replicaContext) {
+    INTERNAL_REPLICA_CONTEXT = replicaContext;
+  }
+
+  /**
+   * If called from a deployment, returns the deployment and replica tag.
+   *
+   * <p>A replica tag uniquely identifies a single replica for a Ray Serve deployment at runtime.
+   * Replica tags are of the form `<deployment_name>#<random letters>`.
+   *
+   * @return the replica context if it exists, or throw RayServeException.
+   */
+  public static ReplicaContext getReplicaContext() {
+    if (INTERNAL_REPLICA_CONTEXT == null) {
+      throw new RayServeException(
+          "`Serve.getReplicaContext()` may only be called from within a Ray Serve deployment.");
+    }
+    return INTERNAL_REPLICA_CONTEXT;
+  }
+
+  /**
+   * Gets the global client, which stores the controller's handle.
+   *
+   * @param healthCheckController If True, run a health check on the cached controller if it exists.
+   *     If the check fails, try reconnecting to the controller.
+   * @return ServeControllerClient to the running Serve controller. If there is no running
+   *     controller and raise_if_no_controller_running is set to False, returns None.
+   */
+  public static ServeControllerClient getGlobalClient(boolean healthCheckController) {
+    try {
+      if (GLOBAL_CLIENT != null) {
+        if (healthCheckController) {
+          ((PyActorHandle) GLOBAL_CLIENT.getController())
+              .task(PyActorMethod.of("check_alive"))
+              .remote();
+        }
+        return GLOBAL_CLIENT;
+      }
+    } catch (RayActorException e) {
+      LOGGER.info("The cached controller has died. Reconnecting.");
+      setGlobalClient(null);
+    }
+    return connect();
+  }
+
+  /**
+   * Gets the global client, which stores the controller's handle.
+   *
+   * @return ServeControllerClient to the running Serve controller. If there is no running
+   *     controller and raise_if_no_controller_running is set to False, returns None.
+   */
+  public static ServeControllerClient getGlobalClient() {
+    return getGlobalClient(false);
+  }
+
+  private static void setGlobalClient(ServeControllerClient client) {
+    GLOBAL_CLIENT = client;
+  }
+
+  /**
+   * Connect to an existing Serve instance on this Ray cluster.
+   *
+   * <p>If calling from the driver program, the Serve instance on this Ray cluster must first have
+   * been initialized using `Serve.start`.
+   *
+   * <p>If called from within a replica, this will connect to the same Serve instance that the
+   * replica is running in.
+   *
+   * @return
+   */
+  private static synchronized ServeControllerClient connect() {
+
+    if (GLOBAL_CLIENT != null) {
+      return GLOBAL_CLIENT;
+    }
+
+    // Initialize ray if needed.
+    if (!Ray.isInitialized()) {
+      init();
+    }
+
+    Optional<BaseActorHandle> optional =
+        Ray.getActor(Constants.SERVE_CONTROLLER_NAME, Constants.SERVE_NAMESPACE);
+    Preconditions.checkState(
+        optional.isPresent(),
+        MessageFormatter.format(
+            "There is no instance running on this Ray cluster. "
+                + "Please call `serve.start() to start one."));
+    LOGGER.info(
+        "Got controller handle with name `{}` in namespace `{}`.",
+        Constants.SERVE_CONTROLLER_NAME,
+        Constants.SERVE_NAMESPACE);
+
+    ServeControllerClient client = new ServeControllerClient(optional.get());
+
+    setGlobalClient(client);
+    return client;
+  }
+
+  /**
+   * Dynamically fetch a handle to a Deployment object.
+   *
+   * <p>This can be used to update and redeploy a deployment without access to the original
+   * definition.
+   *
+   * @param name name of the deployment. This must have already been deployed.
+   * @return Deployment
+   * @deprecated {@value Constants#MIGRATION_MESSAGE}
+   */
+  @Deprecated
+  public static Deployment getDeployment(String name) {
+    LOGGER.warn(Constants.MIGRATION_MESSAGE);
+    DeploymentRoute deploymentRoute = getGlobalClient().getDeploymentInfo(name);
+    if (deploymentRoute == null) {
+      throw new RayServeException(
+          MessageFormatter.format(
+              "Deployment {} was not found. Did you call Deployment.deploy?", name));
+    }
+
+    // TODO use DeploymentCreator
+    return new Deployment(
+        name,
+        deploymentRoute.getDeploymentInfo().getDeploymentConfig(),
+        deploymentRoute.getDeploymentInfo().getReplicaConfig(),
+        deploymentRoute.getDeploymentInfo().getVersion());
+  }
+
+  /**
+   * Run an application and return a handle to its ingress deployment.
+   *
+   * @param target A Serve application returned by `Deployment.bind()`.
+   * @return A handle that can be used to call the application.
+   */
+  public static DeploymentHandle run(Application target) {
+    return run(target, true, Constants.SERVE_DEFAULT_APP_NAME, null, null, false);
+  }
+
+  /**
+   * Run an application and return a handle to its ingress deployment.
+   *
+   * @param target A Serve application returned by `Deployment.bind()`.
+   * @param blocking
+   * @param name Application name. If not provided, this will be the only application running on the
+   *     cluster (it will delete all others).
+   * @param routePrefix Route prefix for HTTP requests. Defaults to '/'.
+   * @param config
+   * @param externalScalerEnabled If true, indicates that an external autoscaler will manage replica
+   *     scaling for this application. Defaults to false.
+   * @return A handle that can be used to call the application.
+   */
+  public static DeploymentHandle run(
+      Application target,
+      boolean blocking,
+      String name,
+      String routePrefix,
+      Map<String, String> config,
+      boolean externalScalerEnabled) {
+
+    if (StringUtils.isBlank(name)) {
+      throw new RayServeException("Application name must a non-empty string.");
+    }
+
+    if (StringUtils.isNotBlank(routePrefix)) {
+      Preconditions.checkArgument(
+          routePrefix.startsWith("/"), "The route_prefix must start with a forward slash ('/')");
+    } else {
+      routePrefix = "/";
+    }
+
+    ServeControllerClient client = serveStart(config);
+
+    List<Deployment> deployments = Graph.build(target.getInternalDagNode(), name);
+    Deployment ingressDeployment = deployments.get(deployments.size() - 1);
+
+    for (Deployment deployment : deployments) {
+      // Overwrite route prefix
+      deployment
+          .getDeploymentConfig()
+          .setVersion(
+              StringUtils.isNotBlank(deployment.getVersion())
+                  ? deployment.getVersion()
+                  : RandomStringUtils.randomAlphabetic(6));
+    }
+
+    client.deployApplication(
+        name,
+        routePrefix,
+        deployments,
+        ingressDeployment.getName(),
+        blocking,
+        externalScalerEnabled);
+    return client.getDeploymentHandle(ingressDeployment.getName(), name, true);
+  }
+
+  private static void init() {
+    System.setProperty("ray.job.namespace", Constants.SERVE_NAMESPACE);
+    Ray.init();
+  }
+
+  /**
+   * Get a handle to the application's ingress deployment by name.
+   *
+   * @param name application name
+   * @return
+   */
+  public static DeploymentHandle getAppHandle(String name) {
+    ServeControllerClient client = getGlobalClient();
+    String ingress =
+        (String)
+            ((PyActorHandle) client.getController())
+                .task(PyActorMethod.of("get_ingress_deployment_name"), name)
+                .remote()
+                .get();
+
+    if (StringUtils.isBlank(ingress)) {
+      throw new RayServeException(
+          MessageFormatter.format("Application '{}' does not exist.", ingress));
+    }
+    return client.getDeploymentHandle(ingress, name, false);
+  }
+
+  /**
+   * Delete an application by its name.
+   *
+   * <p>Deletes the app with all corresponding deployments.
+   *
+   * @param name application name
+   */
+  public static void delete(String name) {
+    delete(name, true);
+  }
+
+  /**
+   * Delete an application by its name.
+   *
+   * <p>Deletes the app with all corresponding deployments.
+   *
+   * @param name application name
+   * @param blocking Wait for the application to be deleted or not.
+   */
+  public static void delete(String name, boolean blocking) {
+    getGlobalClient().deleteApps(Arrays.asList(name), blocking);
+  }
+}

java/serve/src/main/java/io/ray/serve/api/ServeControllerClient.java

@@ -0,0 +1,330 @@
+package io.ray.serve.api;
+
+import com.google.protobuf.ByteString;
+import io.ray.api.ActorHandle;
+import io.ray.api.BaseActorHandle;
+import io.ray.api.PyActorHandle;
+import io.ray.api.Ray;
+import io.ray.api.exception.RayActorException;
+import io.ray.api.exception.RayTimeoutException;
+import io.ray.api.function.PyActorMethod;
+import io.ray.serve.common.Constants;
+import io.ray.serve.controller.ServeController;
+import io.ray.serve.deployment.Deployment;
+import io.ray.serve.deployment.DeploymentRoute;
+import io.ray.serve.exception.RayServeException;
+import io.ray.serve.generated.ApplicationArgs;
+import io.ray.serve.generated.ApplicationStatus;
+import io.ray.serve.generated.DeploymentArgs;
+import io.ray.serve.generated.EndpointInfo;
+import io.ray.serve.generated.StatusOverview;
+import io.ray.serve.handle.DeploymentHandle;
+import io.ray.serve.util.CollectionUtil;
+import io.ray.serve.util.MessageFormatter;
+import io.ray.serve.util.ServeProtoUtil;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ServeControllerClient {
+  private static final Logger LOGGER = LoggerFactory.getLogger(ServeControllerClient.class);
+
+  private static long CLIENT_POLLING_INTERVAL_S = 1;
+
+  private BaseActorHandle controller; // TODO change to PyActorHandle
+
+  private boolean shutdown;
+
+  private Map<String, DeploymentHandle> handleCache = new ConcurrentHashMap<>();
+
+  private String rootUrl;
+
+  @SuppressWarnings("unchecked")
+  public ServeControllerClient(BaseActorHandle controller) {
+    this.controller = controller;
+    this.rootUrl =
+        controller instanceof PyActorHandle
+            ? (String)
+                ((PyActorHandle) controller).task(PyActorMethod.of("get_root_url")).remote().get()
+            : ((ActorHandle<ServeController>) controller)
+                .task(ServeController::getRootUrl)
+                .remote()
+                .get();
+  }
+
+  /**
+   * Retrieve DeploymentHandle for service deployment to invoke it from Java.
+   *
+   * @param deploymentName A registered service deployment.
+   * @param appName application name
+   * @param missingOk If true, then Serve won't check the deployment is registered.
+   * @return
+   */
+  @SuppressWarnings("unchecked")
+  public DeploymentHandle getDeploymentHandle(
+      String deploymentName, String appName, boolean missingOk) {
+    String cacheKey =
+        StringUtils.join(
+            new Object[] {deploymentName, appName, missingOk}, Constants.SEPARATOR_HASH);
+    if (handleCache.containsKey(cacheKey)) {
+      return handleCache.get(cacheKey);
+    }
+
+    Map<String, EndpointInfo> endpoints = null;
+    if (controller instanceof PyActorHandle) {
+      endpoints =
+          ServeProtoUtil.parseEndpointSet(
+              (byte[])
+                  ((PyActorHandle) controller)
+                      .task(PyActorMethod.of(Constants.CONTROLLER_GET_ALL_ENDPOINTS_METHOD))
+                      .remote()
+                      .get());
+    } else {
+      LOGGER.warn("Client currently only supports the Python controller.");
+      endpoints =
+          ServeProtoUtil.parseEndpointSet(
+              ((ActorHandle<? extends ServeController>) controller)
+                  .task(ServeController::getAllEndpoints)
+                  .remote()
+                  .get());
+    }
+
+    if (!missingOk && (endpoints == null || !endpoints.containsKey(deploymentName))) {
+      throw new RayServeException(
+          MessageFormatter.format("Deployment {} does not exist.", deploymentName));
+    }
+
+    DeploymentHandle handle = new DeploymentHandle(deploymentName, appName);
+    handleCache.put(cacheKey, handle);
+    return handle;
+  }
+
+  /**
+   * Completely shut down the connected Serve instance.
+   *
+   * <p>Shuts down all processes and deletes all state associated with the instance.
+   *
+   * @param timeoutS The unit is second.
+   */
+  public synchronized void shutdown(Long timeoutS) {
+    if (Ray.isInitialized() && !shutdown) {
+
+      if (timeoutS == null) {
+        timeoutS = 30L;
+      }
+
+      try {
+        ((PyActorHandle) controller)
+            .task(PyActorMethod.of("graceful_shutdown"))
+            .remote()
+            .get(timeoutS * 1000);
+      } catch (RayActorException e) {
+        // Controller has been shut down.
+        return;
+      } catch (RayTimeoutException e) {
+        LOGGER.warn(
+            "Controller failed to shut down within {}s. Check controller logs for more details.",
+            timeoutS);
+      }
+      shutdown = true;
+    }
+  }
+
+  public String getRootUrl() {
+    return rootUrl;
+  }
+
+  /**
+   * @deprecated {@value Constants#MIGRATION_MESSAGE}
+   * @param name
+   * @return
+   */
+  @Deprecated
+  public DeploymentRoute getDeploymentInfo(String name) {
+    return DeploymentRoute.fromProtoBytes(
+        (byte[])
+            ((PyActorHandle) controller)
+                .task(PyActorMethod.of("get_deployment_info"), name)
+                .remote()
+                .get());
+  }
+
+  public BaseActorHandle getController() {
+    return controller;
+  }
+
+  /**
+   * Deployment an application with deployment list.
+   *
+   * @param name application name.
+   * @param routePrefix route prefix for the application.
+   * @param deployments deployment list.
+   * @param ingressDeploymentName name of the ingress deployment (the one that is exposed over
+   *     HTTP).
+   * @param blocking Wait for the applications to be deployed or not.
+   * @param externalScalerEnabled If true, indicates that an external autoscaler will manage replica
+   *     scaling for this application.
+   */
+  public void deployApplication(
+      String name,
+      String routePrefix,
+      List<Deployment> deployments,
+      String ingressDeploymentName,
+      boolean blocking,
+      boolean externalScalerEnabled) {
+
+    Object[] deploymentArgsArray = new Object[deployments.size()];
+
+    for (int i = 0; i < deployments.size(); i++) {
+      Deployment deployment = deployments.get(i);
+      DeploymentArgs.Builder deploymentArgs =
+          DeploymentArgs.newBuilder()
+              .setDeploymentName(deployment.getName())
+              .setReplicaConfig(ByteString.copyFrom(deployment.getReplicaConfig().toProtoBytes()))
+              .setDeploymentConfig(
+                  ByteString.copyFrom(deployment.getDeploymentConfig().toProtoBytes()))
+              .setIngress(deployment.isIngress())
+              .setDeployerJobId(Ray.getRuntimeContext().getCurrentJobId().toString());
+      if (deployment.getName() == ingressDeploymentName) {
+        deploymentArgs.setRoutePrefix(routePrefix);
+      }
+      deploymentArgsArray[i] = deploymentArgs.build().toByteArray();
+    }
+
+    ApplicationArgs.Builder applicationArgs =
+        ApplicationArgs.newBuilder().setExternalScalerEnabled(externalScalerEnabled);
+    byte[] applicationArgsBytes = applicationArgs.build().toByteArray();
+
+    ((PyActorHandle) controller)
+        .task(
+            PyActorMethod.of("deploy_application"), name, deploymentArgsArray, applicationArgsBytes)
+        .remote()
+        .get();
+
+    if (blocking) {
+      waitForApplicationRunning(name, null);
+      for (Deployment deployment : deployments) {
+        logDeploymentReady(
+            deployment.getName(),
+            deployment.getVersion(),
+            "component=serve deployment=" + deployment.getName());
+      }
+    }
+  }
+
+  /**
+   * Waits for the named application to enter "RUNNING" status.
+   *
+   * @param name application name
+   * @param timeoutS unit: second
+   */
+  private void waitForApplicationRunning(String name, Long timeoutS) {
+    long start = System.currentTimeMillis();
+    while (timeoutS == null || System.currentTimeMillis() - start < timeoutS * 1000) {
+
+      StatusOverview status = getServeStatus(name);
+      if (status.getAppStatus().getStatus() == ApplicationStatus.APPLICATION_STATUS_RUNNING) {
+        return;
+      } else if (status.getAppStatus().getStatus()
+          == ApplicationStatus.APPLICATION_STATUS_DEPLOY_FAILED) {
+        throw new RayServeException(
+            MessageFormatter.format(
+                "Deploying application {} is failed: {}",
+                name,
+                status.getAppStatus().getMessage()));
+      }
+
+      LOGGER.debug(
+          "Waiting for {} to be RUNNING, current status: {}.",
+          name,
+          status.getAppStatus().getStatus());
+      try {
+        Thread.sleep(CLIENT_POLLING_INTERVAL_S * 1000);
+      } catch (InterruptedException e) {
+      }
+    }
+
+    throw new RayServeException(
+        MessageFormatter.format(
+            "Application {} did not become RUNNING after {}s.", name, timeoutS));
+  }
+
+  private void logDeploymentReady(String name, String version, String tag) {
+    LOGGER.info(
+        "Deployment '{}{}' is ready. {}",
+        name,
+        StringUtils.isNotBlank(version) ? "':'" + version : "",
+        tag);
+  }
+
+  /**
+   * Delete the specified applications.
+   *
+   * @param names application names
+   * @param blocking Wait for the applications to be deleted or not.
+   */
+  public void deleteApps(List<String> names, boolean blocking) {
+    if (CollectionUtil.isEmpty(names)) {
+      return;
+    }
+
+    LOGGER.info("Deleting app {}", names);
+
+    ((PyActorHandle) controller)
+        .task(PyActorMethod.of("delete_apps"), names.toArray())
+        .remote()
+        .get();
+
+    if (blocking) {
+      long start = System.currentTimeMillis();
+      List<String> undeleted = new ArrayList<>(names);
+
+      while (System.currentTimeMillis() - start < 60 * 1000) {
+
+        Iterator<String> iterator = undeleted.iterator();
+        while (iterator.hasNext()) {
+
+          String name = iterator.next();
+          StatusOverview status = getServeStatus(name);
+          if (status.getAppStatus().getStatus()
+              == ApplicationStatus.APPLICATION_STATUS_NOT_STARTED) {
+            iterator.remove();
+          }
+        }
+
+        if (undeleted.isEmpty()) {
+          return;
+        }
+
+        try {
+          Thread.sleep(CLIENT_POLLING_INTERVAL_S * 1000);
+        } catch (InterruptedException e) {
+        }
+      }
+      throw new RayServeException(
+          MessageFormatter.format(
+              "Some of these applications weren't deleted after 60s: {}", names));
+    }
+  }
+
+  /**
+   * Return the status of the specified application.
+   *
+   * @param name application name
+   * @return
+   */
+  public StatusOverview getServeStatus(String name) {
+    byte[] statusBytes =
+        (byte[])
+            ((PyActorHandle) controller)
+                .task(PyActorMethod.of("get_serve_status"), name)
+                .remote()
+                .get();
+    return ServeProtoUtil.bytesToProto(statusBytes, StatusOverview::parseFrom);
+  }
+}

java/serve/src/main/java/io/ray/serve/common/Constants.java

@@ -0,0 +1,65 @@
+package io.ray.serve.common;
+
+import com.google.common.collect.Lists;
+import java.util.List;
+
+/** Ray Serve common constants. */
+public class Constants {
+
+  /** Name of deployment reconfiguration method implemented by user. */
+  public static final String RECONFIGURE_METHOD = "reconfigure";
+
+  /** Default histogram buckets for latency tracker. */
+  public static final List<Double> DEFAULT_LATENCY_BUCKET_MS =
+      Lists.newArrayList(
+          1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0);
+
+  /** Name of controller listen_for_change method. */
+  public static final String CONTROLLER_LISTEN_FOR_CHANGE_METHOD = "listen_for_change_java";
+
+  /** Name of controller get_all_endpoints method. */
+  public static final String CONTROLLER_GET_ALL_ENDPOINTS_METHOD = "get_all_endpoints_java";
+
+  public static final String SERVE_CONTROLLER_NAME = "SERVE_CONTROLLER_ACTOR";
+
+  public static final String SERVE_NAMESPACE = "serve";
+
+  public static final String CALL_METHOD = "call";
+
+  public static final String UTF8 = "UTF-8";
+
+  public static final String CHECK_HEALTH_METHOD = "checkHealth";
+
+  /** Controller checkpoint path */
+  public static final String DEFAULT_CHECKPOINT_PATH = "ray://";
+
+  /**
+   * Because ServeController will accept one long poll request per handle, its concurrency needs to
+   * scale as O(num_handles)
+   */
+  public static final int CONTROLLER_MAX_CONCURRENCY = 15000;
+
+  /** Max time to wait for proxy in `Serve.start`. Unit: second */
+  public static final int PROXY_TIMEOUT_S = 60;
+
+  public static final Double DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_S = 20.0;
+
+  public static final Double DEFAULT_GRACEFUL_SHUTDOWN_WAIT_LOOP_S = 2.0;
+
+  public static final Double DEFAULT_HEALTH_CHECK_PERIOD_S = 10.0;
+
+  public static final Double DEFAULT_HEALTH_CHECK_TIMEOUT_S = 30.0;
+
+  public static final Double DEFAULT_REQUEST_ROUTING_STATS_PERIOD_S = 10.0;
+
+  public static final Double DEFAULT_REQUEST_ROUTING_STATS_TIMEOUT_S = 30.0;
+
+  /** Default Serve application name */
+  public static final String SERVE_DEFAULT_APP_NAME = "default";
+
+  public static final String MIGRATION_MESSAGE =
+      "This API is deprecated and may be removed in future Ray releases. "
+          + "Please see https://docs.ray.io/en/latest/serve/index.html for more information.";
+
+  public static final String SEPARATOR_HASH = "#";
+}

java/serve/src/main/java/io/ray/serve/config/AutoscalingConfig.java

@@ -0,0 +1,97 @@
+package io.ray.serve.config;
+
+import java.io.Serializable;
+
+public class AutoscalingConfig implements Serializable {
+  private static final long serialVersionUID = 9135422781025005216L;
+  private int minReplicas = 1;
+  private int maxReplicas = 1;
+  private int targetOngoingRequests = 1;
+  /** How often to scrape for metrics */
+  private double metricsIntervalS = 10.0;
+  /** Time window to average over for metrics. */
+  private double lookBackPeriodS = 30.0;
+  /** Multiplicative "gain" factor to limit scaling decisions */
+  private double smoothingFactor = 1.0;
+  /** How long to wait before scaling down replicas */
+  private double downscaleDelayS = 600.0;
+  /** How long to wait before scaling up replicas */
+  private double upscaleDelayS = 30.0;
+
+  public int getMinReplicas() {
+    return minReplicas;
+  }
+
+  public void setMinReplicas(int minReplicas) {
+    this.minReplicas = minReplicas;
+  }
+
+  public int getMaxReplicas() {
+    return maxReplicas;
+  }
+
+  public void setMaxReplicas(int maxReplicas) {
+    this.maxReplicas = maxReplicas;
+  }
+
+  public int getTargetOngoingRequests() {
+    return targetOngoingRequests;
+  }
+
+  public void setTargetOngoingRequests(int targetOngoingRequests) {
+    this.targetOngoingRequests = targetOngoingRequests;
+  }
+
+  public double getMetricsIntervalS() {
+    return metricsIntervalS;
+  }
+
+  public void setMetricsIntervalS(double metricsIntervalS) {
+    this.metricsIntervalS = metricsIntervalS;
+  }
+
+  public double getLookBackPeriodS() {
+    return lookBackPeriodS;
+  }
+
+  public void setLookBackPeriodS(double lookBackPeriodS) {
+    this.lookBackPeriodS = lookBackPeriodS;
+  }
+
+  public double getSmoothingFactor() {
+    return smoothingFactor;
+  }
+
+  public void setSmoothingFactor(double smoothingFactor) {
+    this.smoothingFactor = smoothingFactor;
+  }
+
+  public double getDownscaleDelayS() {
+    return downscaleDelayS;
+  }
+
+  public void setDownscaleDelayS(double downscaleDelayS) {
+    this.downscaleDelayS = downscaleDelayS;
+  }
+
+  public double getUpscaleDelayS() {
+    return upscaleDelayS;
+  }
+
+  public void setUpscaleDelayS(double upscaleDelayS) {
+    this.upscaleDelayS = upscaleDelayS;
+  }
+
+  public io.ray.serve.generated.AutoscalingConfig toProto() {
+    return io.ray.serve.generated.AutoscalingConfig.newBuilder()
+        .setMinReplicas(minReplicas)
+        .setMaxReplicas(maxReplicas)
+        .setTargetOngoingRequests(targetOngoingRequests)
+        .setMetricsIntervalS(metricsIntervalS)
+        .setLookBackPeriodS(lookBackPeriodS)
+        .setSmoothingFactor(smoothingFactor)
+        .setDownscaleDelayS(downscaleDelayS)
+        .setUpscaleDelayS(upscaleDelayS)
+        .build();
+  }
+}

java/serve/src/main/java/io/ray/serve/config/DeploymentConfig.java

@@ -0,0 +1,336 @@
+package io.ray.serve.config;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import io.ray.runtime.serializer.MessagePackSerializer;
+import io.ray.serve.common.Constants;
+import io.ray.serve.exception.RayServeException;
+import io.ray.serve.generated.DeploymentLanguage;
+import io.ray.serve.util.MessageFormatter;
+import java.io.Serializable;
+
+/** Configuration options for a deployment, to be set by the user. */
+public class DeploymentConfig implements Serializable {
+
+  private static final long serialVersionUID = 5965977837248820843L;
+
+  /**
+   * The number of processes to start up that will handle requests to this deployment. Defaults to
+   * 1.
+   */
+  private Integer numReplicas = 1;
+
+  /**
+   * The maximum number of requests that can be sent to a replica of this deployment without
+   * receiving a response. Defaults to 100.
+   */
+  private Integer maxOngoingRequests = 100;
+
+  /**
+   * Arguments to pass to the reconfigure method of the deployment. The reconfigure method is called
+   * if user_config is not None.
+   */
+  private Object userConfig;
+
+  /**
+   * Duration that deployment replicas will wait until there is no more work to be done before
+   * shutting down.
+   */
+  private Double gracefulShutdownWaitLoopS = Constants.DEFAULT_GRACEFUL_SHUTDOWN_WAIT_LOOP_S;
+
+  /** Controller waits for this duration to forcefully kill the replica for shutdown. */
+  private Double gracefulShutdownTimeoutS = Constants.DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_S;
+
+  /** Frequency at which the controller will health check replicas. */
+  private Double healthCheckPeriodS = Constants.DEFAULT_HEALTH_CHECK_PERIOD_S;
+
+  /**
+   * Timeout that the controller will wait for a response from the replica's health check before
+   * marking it unhealthy.
+   */
+  private Double healthCheckTimeoutS = Constants.DEFAULT_HEALTH_CHECK_TIMEOUT_S;
+
+  private AutoscalingConfig autoscalingConfig;
+
+  private RequestRouterConfig routerConfig;
+
+  /** This flag is used to let replica know they are deplyed from a different language. */
+  private Boolean isCrossLanguage = false;
+
+  /** This property tells the controller the deployment's language. */
+  private DeploymentLanguage deploymentLanguage = DeploymentLanguage.JAVA;
+
+  private String version;
+
+  private String prevVersion;
+
+  private Integer maxConstructorRetryCount = 20;
+
+  public Integer getNumReplicas() {
+    return numReplicas;
+  }
+
+  public DeploymentConfig setNumReplicas(Integer numReplicas) {
+    if (numReplicas != null) {
+      this.numReplicas = numReplicas;
+    }
+    return this;
+  }
+
+  public Integer getMaxOngoingRequests() {
+    return maxOngoingRequests;
+  }
+
+  public DeploymentConfig setMaxOngoingRequests(Integer maxOngoingRequests) {
+    if (maxOngoingRequests != null) {
+      Preconditions.checkArgument(maxOngoingRequests > 0, "max_ongoing_requests must be > 0");
+      this.maxOngoingRequests = maxOngoingRequests;
+    }
+    return this;
+  }
+
+  public Integer getMaxConstructorRetryCount() {
+    return maxConstructorRetryCount;
+  }
+
+  public DeploymentConfig setMaxConstructorRetryCount(Integer maxConstructorRetryCount) {
+    if (maxConstructorRetryCount != null) {
+      Preconditions.checkArgument(
+          maxConstructorRetryCount > 0, "max constructor retry count must be > 0");
+      this.maxConstructorRetryCount = maxConstructorRetryCount;
+    }
+    return this;
+  }
+
+  public Object getUserConfig() {
+    return userConfig;
+  }
+
+  public DeploymentConfig setUserConfig(Object userConfig) {
+    this.userConfig = userConfig;
+    return this;
+  }
+
+  public Double getGracefulShutdownWaitLoopS() {
+    return gracefulShutdownWaitLoopS;
+  }
+
+  public DeploymentConfig setGracefulShutdownWaitLoopS(Double gracefulShutdownWaitLoopS) {
+    if (gracefulShutdownWaitLoopS != null) {
+      this.gracefulShutdownWaitLoopS = gracefulShutdownWaitLoopS;
+    }
+    return this;
+  }
+
+  public Double getGracefulShutdownTimeoutS() {
+    return gracefulShutdownTimeoutS;
+  }
+
+  public DeploymentConfig setGracefulShutdownTimeoutS(Double gracefulShutdownTimeoutS) {
+    if (gracefulShutdownTimeoutS != null) {
+      this.gracefulShutdownTimeoutS = gracefulShutdownTimeoutS;
+    }
+    return this;
+  }
+
+  public Double getHealthCheckPeriodS() {
+    return healthCheckPeriodS;
+  }
+
+  public DeploymentConfig setHealthCheckPeriodS(Double healthCheckPeriodS) {
+    if (healthCheckPeriodS != null) {
+      this.healthCheckPeriodS = healthCheckPeriodS;
+    }
+    return this;
+  }
+
+  public Double getHealthCheckTimeoutS() {
+    return healthCheckTimeoutS;
+  }
+
+  public DeploymentConfig setHealthCheckTimeoutS(Double healthCheckTimeoutS) {
+    if (healthCheckTimeoutS != null) {
+      this.healthCheckTimeoutS = healthCheckTimeoutS;
+    }
+    return this;
+  }
+
+  public Double getRequestRoutingStatsPeriodS() {
+    return routerConfig.getRequestRoutingStatsPeriodS();
+  }
+
+  public DeploymentConfig setRequestRoutingStatsPeriodS(Double requestRoutingStatsPeriodS) {
+    if (requestRoutingStatsPeriodS != null) {
+      routerConfig.setRequestRoutingStatsPeriodS(requestRoutingStatsPeriodS);
+    }
+    return this;
+  }
+
+  public Double getRequestRoutingStatsTimeoutS() {
+    return routerConfig.getRequestRoutingStatsTimeoutS();
+  }
+
+  public DeploymentConfig setRequestRoutingStatsTimeoutS(Double requestRoutingStatsTimeoutS) {
+    if (requestRoutingStatsTimeoutS != null) {
+      routerConfig.setRequestRoutingStatsTimeoutS(requestRoutingStatsTimeoutS);
+    }
+    return this;
+  }
+
+  public AutoscalingConfig getAutoscalingConfig() {
+    return autoscalingConfig;
+  }
+
+  public DeploymentConfig setAutoscalingConfig(AutoscalingConfig autoscalingConfig) {
+    this.autoscalingConfig = autoscalingConfig;
+    return this;
+  }
+
+  public RequestRouterConfig getRequestRouterConfig() {
+    return routerConfig;
+  }
+
+  public DeploymentConfig setRequestRouterConfig(RequestRouterConfig routerConfig) {
+    this.routerConfig = routerConfig;
+    return this;
+  }
+
+  public boolean isCrossLanguage() {
+    return isCrossLanguage;
+  }
+
+  public DeploymentConfig setCrossLanguage(Boolean isCrossLanguage) {
+    if (isCrossLanguage != null) {
+      this.isCrossLanguage = isCrossLanguage;
+    }
+    return this;
+  }
+
+  public DeploymentLanguage getDeploymentLanguage() {
+    return deploymentLanguage;
+  }
+
+  public DeploymentConfig setDeploymentLanguage(DeploymentLanguage deploymentLanguage) {
+    if (deploymentLanguage != null) {
+      this.deploymentLanguage = deploymentLanguage;
+      this.isCrossLanguage = deploymentLanguage != DeploymentLanguage.JAVA;
+    }
+    return this;
+  }
+
+  public String getVersion() {
+    return version;
+  }
+
+  public DeploymentConfig setVersion(String version) {
+    this.version = version;
+    return this;
+  }
+
+  public String getPrevVersion() {
+    return prevVersion;
+  }
+
+  public void setPrevVersion(String prevVersion) {
+    this.prevVersion = prevVersion;
+  }
+
+  public byte[] toProtoBytes() {
+    io.ray.serve.generated.DeploymentConfig.Builder builder =
+        io.ray.serve.generated.DeploymentConfig.newBuilder()
+            .setNumReplicas(numReplicas)
+            .setMaxOngoingRequests(maxOngoingRequests)
+            .setMaxQueuedRequests(-1)
+            .setGracefulShutdownWaitLoopS(gracefulShutdownWaitLoopS)
+            .setGracefulShutdownTimeoutS(gracefulShutdownTimeoutS)
+            .setHealthCheckPeriodS(healthCheckPeriodS)
+            .setHealthCheckTimeoutS(healthCheckTimeoutS)
+            .setIsCrossLanguage(isCrossLanguage)
+            .setDeploymentLanguage(deploymentLanguage)
+            .setVersion(version)
+            .setMaxConstructorRetryCount(maxConstructorRetryCount);
+    if (null != userConfig) {
+      builder.setUserConfig(ByteString.copyFrom(MessagePackSerializer.encode(userConfig).getKey()));
+    }
+    if (null != autoscalingConfig) {
+      builder.setAutoscalingConfig(autoscalingConfig.toProto());
+    }
+    if (null != routerConfig) {
+      builder.setRequestRouterConfig(routerConfig.toProto());
+    }
+    return builder.build().toByteArray();
+  }
+
+  public io.ray.serve.generated.DeploymentConfig toProto() {
+    io.ray.serve.generated.DeploymentConfig.Builder builder =
+        io.ray.serve.generated.DeploymentConfig.newBuilder()
+            .setNumReplicas(numReplicas)
+            .setMaxOngoingRequests(maxOngoingRequests)
+            .setGracefulShutdownWaitLoopS(gracefulShutdownWaitLoopS)
+            .setGracefulShutdownTimeoutS(gracefulShutdownTimeoutS)
+            .setHealthCheckPeriodS(healthCheckPeriodS)
+            .setHealthCheckTimeoutS(healthCheckTimeoutS)
+            .setIsCrossLanguage(isCrossLanguage)
+            .setDeploymentLanguage(deploymentLanguage)
+            .setMaxConstructorRetryCount(maxConstructorRetryCount);
+    if (null != userConfig) {
+      builder.setUserConfig(ByteString.copyFrom(MessagePackSerializer.encode(userConfig).getKey()));
+    }
+    if (null != autoscalingConfig) {
+      builder.setAutoscalingConfig(autoscalingConfig.toProto());
+    }
+    if (null != routerConfig) {
+      builder.setRequestRouterConfig(routerConfig.toProto());
+    }
+    return builder.build();
+  }
+
+  public static DeploymentConfig fromProto(io.ray.serve.generated.DeploymentConfig proto) {
+
+    DeploymentConfig deploymentConfig = new DeploymentConfig();
+    if (proto == null) {
+      return deploymentConfig;
+    }
+    deploymentConfig.setNumReplicas(proto.getNumReplicas());
+    deploymentConfig.setMaxOngoingRequests(proto.getMaxOngoingRequests());
+    deploymentConfig.setGracefulShutdownWaitLoopS(proto.getGracefulShutdownWaitLoopS());
+    deploymentConfig.setGracefulShutdownTimeoutS(proto.getGracefulShutdownTimeoutS());
+    deploymentConfig.setCrossLanguage(proto.getIsCrossLanguage());
+    if (proto.getDeploymentLanguage() == DeploymentLanguage.UNRECOGNIZED) {
+      throw new RayServeException(
+          MessageFormatter.format(
+              "Unrecognized deployment language {}. Deployment language must be in {}.",
+              proto.getDeploymentLanguage(),
+              Lists.newArrayList(DeploymentLanguage.values())));
+    }
+    deploymentConfig.setDeploymentLanguage(proto.getDeploymentLanguage());
+    if (proto.getUserConfig() != null && proto.getUserConfig().size() != 0) {
+      deploymentConfig.setUserConfig(
+          MessagePackSerializer.decode(
+              proto.getUserConfig().toByteArray(), Object.class)); // TODO-xlang
+    }
+    if (proto.getMaxConstructorRetryCount() > 0) {
+      deploymentConfig.setMaxConstructorRetryCount(proto.getMaxConstructorRetryCount());
+    }
+    return deploymentConfig;
+  }
+
+  public static DeploymentConfig fromProtoBytes(byte[] bytes) {
+
+    DeploymentConfig deploymentConfig = new DeploymentConfig();
+    if (bytes == null) {
+      return deploymentConfig;
+    }
+
+    io.ray.serve.generated.DeploymentConfig proto = null;
+    try {
+      proto = io.ray.serve.generated.DeploymentConfig.parseFrom(bytes);
+    } catch (InvalidProtocolBufferException e) {
+      throw new RayServeException("Failed to parse DeploymentConfig from protobuf bytes.", e);
+    }
+
+    return fromProto(proto);
+  }
+}

java/serve/src/main/java/io/ray/serve/config/RayServeConfig.java

@@ -0,0 +1,18 @@
+package io.ray.serve.config;
+
+public class RayServeConfig {
+
+  public static final String PROXY_CLASS = "ray.serve.proxy.class";
+
+  public static final String PROXY_HTTP_PORT = "ray.serve.proxy.http.port";
+
+  public static final String METRICS_ENABLED = "ray.serve.metrics.enabled";
+
+  public static final String LONG_POOL_CLIENT_ENABLED = "ray.serve.long.poll.client.enabled";
+
+  /** The polling interval of long poll thread. Unit: second. */
+  public static final String LONG_POOL_CLIENT_INTERVAL = "ray.serve.long.poll.client.interval_s";
+
+  /** The polling timeout of each long poll invoke. Unit: second. */
+  public static final String LONG_POOL_CLIENT_TIMEOUT_S = "ray.serve.long.poll.client.timeout_s";
+}

java/serve/src/main/java/io/ray/serve/config/ReplicaConfig.java

@@ -0,0 +1,156 @@
+package io.ray.serve.config;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import com.google.gson.Gson;
+import com.google.protobuf.ByteString;
+import io.ray.runtime.serializer.MessagePackSerializer;
+import io.ray.serve.util.MessageFormatter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+
+/** Configuration options for a replica. */
+public class ReplicaConfig {
+
+  private static Gson gson = new Gson();
+
+  private String deploymentDef;
+
+  private Object[] initArgs;
+
+  private Map<String, Object> rayActorOptions;
+
+  private Map<String, Object> resources;
+
+  private static final Set<String> allowedRayActorOptions =
+      Sets.newHashSet(
+          // resource options
+          "accelerator_type",
+          "memory",
+          "num_cpus",
+          "num_gpus",
+          "object_store_memory",
+          "resources",
+          // other options
+          "runtime_env");
+
+  public ReplicaConfig(
+      String deploymentDef, Object[] initArgs, Map<String, Object> rayActorOptions) {
+    this.deploymentDef = deploymentDef;
+    this.initArgs = initArgs != null ? initArgs : new Object[0];
+    this.rayActorOptions = rayActorOptions != null ? rayActorOptions : new HashMap<>();
+    for (String option : this.rayActorOptions.keySet()) {
+      Preconditions.checkArgument(
+          allowedRayActorOptions.contains(option),
+          MessageFormatter.format(
+              "Specifying '{}' in ray_actor_options is not allowed. Allowed options: {}",
+              option,
+              allowedRayActorOptions));
+    }
+    // TODO validate_actor_options
+    if (!this.rayActorOptions.containsKey("num_cpus")) {
+      this.rayActorOptions.put("num_cpus", 1.0);
+    }
+    this.resources = resourcesFromRayOptions(this.rayActorOptions);
+  }
+
+  /**
+   * Determine a task's resource requirements.
+   *
+   * @param rayActorOptions The map that contains resources requirements.
+   * @return A map of the resource requirements for the task.
+   */
+  private Map<String, Object> resourcesFromRayOptions(Map<String, Object> rayActorOptions) {
+
+    Object numCpus = rayActorOptions.get("num_cpus");
+    Object numGpus = rayActorOptions.get("num_gpus");
+    Object memory = rayActorOptions.get("memory");
+    Object objectStoreMemory = rayActorOptions.get("object_store_memory");
+    Object acceleratorType = rayActorOptions.get("accelerator_type");
+
+    Map<String, Object> resources = new HashMap<>();
+    if (numCpus != null) {
+      resources.put("CPU", numCpus);
+    }
+    if (numGpus != null) {
+      resources.put("GPU", numGpus);
+    }
+    if (memory != null) {
+      resources.put("memory", memory);
+    }
+    if (objectStoreMemory != null) {
+      resources.put("object_store_memory", objectStoreMemory);
+    }
+    if (acceleratorType != null) {
+      resources.put("accelerator_type:" + acceleratorType, 0.001);
+    }
+    return resources;
+  }
+
+  public String getDeploymentDef() {
+    return deploymentDef;
+  }
+
+  public void setDeploymentDef(String deploymentDef) {
+    this.deploymentDef = deploymentDef;
+  }
+
+  public Object[] getInitArgs() {
+    return initArgs;
+  }
+
+  public void setInitArgs(Object[] initArgs) {
+    this.initArgs = initArgs;
+  }
+
+  public Map<String, Object> getRayActorOptions() {
+    return rayActorOptions;
+  }
+
+  public void setRayActorOptions(Map<String, Object> rayActorOptions) {
+    this.rayActorOptions = rayActorOptions;
+  }
+
+  public Map<String, Object> getResources() {
+    return resources;
+  }
+
+  public void setResources(Map<String, Object> resources) {
+    this.resources = resources;
+  }
+
+  public byte[] toProtoBytes() {
+    io.ray.serve.generated.ReplicaConfig.Builder builder =
+        io.ray.serve.generated.ReplicaConfig.newBuilder();
+    if (StringUtils.isNotBlank(deploymentDef)) {
+      builder.setDeploymentDefName(deploymentDef);
+      builder.setDeploymentDef(ByteString.copyFromUtf8(deploymentDef)); // TODO-xlang
+    }
+    if (initArgs != null && initArgs.length > 0) {
+      builder.setInitArgs(
+          ByteString.copyFrom(MessagePackSerializer.encode(initArgs).getKey())); // TODO-xlang
+    }
+    if (rayActorOptions != null && !rayActorOptions.isEmpty()) {
+      builder.setRayActorOptions(gson.toJson(rayActorOptions));
+    }
+    return builder.build().toByteArray();
+  }
+
+  public static ReplicaConfig fromProto(io.ray.serve.generated.ReplicaConfig proto) {
+    if (proto == null) {
+      return null;
+    }
+    Object[] initArgs = null;
+    if (0 != proto.getInitArgs().toByteArray().length) {
+      initArgs = MessagePackSerializer.decode(proto.getInitArgs().toByteArray(), null);
+    }
+    ReplicaConfig replicaConfig =
+        new ReplicaConfig(
+            proto.getDeploymentDefName(),
+            initArgs,
+            gson.fromJson(proto.getRayActorOptions(), Map.class));
+    return replicaConfig;
+  }
+}

java/serve/src/main/java/io/ray/serve/config/RequestRouterConfig.java

@@ -0,0 +1,38 @@
+package io.ray.serve.config;
+
+import io.ray.serve.common.Constants;
+import java.io.Serializable;
+
+public class RequestRouterConfig implements Serializable {
+  /** Frequency at which the controller will record request routing stats. */
+  private Double requestRoutingStatsPeriodS = Constants.DEFAULT_REQUEST_ROUTING_STATS_PERIOD_S;
+
+  /**
+   * Timeout that the controller waits for a response from the replica's request routing stats
+   * before retrying.
+   */
+  private Double requestRoutingStatsTimeoutS = Constants.DEFAULT_REQUEST_ROUTING_STATS_TIMEOUT_S;
+
+  public Double getRequestRoutingStatsPeriodS() {
+    return requestRoutingStatsPeriodS;
+  }
+
+  public Double getRequestRoutingStatsTimeoutS() {
+    return requestRoutingStatsTimeoutS;
+  }
+
+  public void setRequestRoutingStatsPeriodS(Double requestRoutingStatsPeriodS) {
+    this.requestRoutingStatsPeriodS = requestRoutingStatsPeriodS;
+  }
+
+  public void setRequestRoutingStatsTimeoutS(Double requestRoutingStatsTimeoutS) {
+    this.requestRoutingStatsTimeoutS = requestRoutingStatsTimeoutS;
+  }
+
+  public io.ray.serve.generated.RequestRouterConfig toProto() {
+    return io.ray.serve.generated.RequestRouterConfig.newBuilder()
+        .setRequestRoutingStatsPeriodS(requestRoutingStatsPeriodS)
+        .setRequestRoutingStatsTimeoutS(requestRoutingStatsTimeoutS)
+        .build();
+  }
+}

java/serve/src/main/java/io/ray/serve/context/RequestContext.java

@@ -0,0 +1,59 @@
+package io.ray.serve.context;
+
+import java.util.Optional;
+
+public class RequestContext {
+
+  private static RequestContext DEFAULT_CONTEXT = new RequestContext("", "", "", "");
+
+  private static ThreadLocal<RequestContext> SERVE_REQUEST_CONTEXT =
+      ThreadLocal.withInitial(() -> DEFAULT_CONTEXT);
+
+  private String route;
+  private String requestId;
+  private String appName;
+  private String multiplexedModelId;
+
+  private RequestContext(
+      String route, String requestId, String appName, String multiplexedModelId) {
+    this.route = route;
+    this.requestId = requestId;
+    this.appName = appName;
+    this.multiplexedModelId = multiplexedModelId;
+  }
+
+  public String getRoute() {
+    return route;
+  }
+
+  public String getRequestId() {
+    return requestId;
+  }
+
+  public String getAppName() {
+    return appName;
+  }
+
+  public String getMultiplexedModelId() {
+    return multiplexedModelId;
+  }
+
+  public static void set(
+      String route, String requestId, String appName, String multiplexedModelId) {
+    SERVE_REQUEST_CONTEXT.set(
+        new RequestContext(
+            Optional.ofNullable(route).orElse(DEFAULT_CONTEXT.getRoute()),
+            Optional.ofNullable(requestId).orElse(DEFAULT_CONTEXT.getRequestId()),
+            Optional.ofNullable(appName).orElse(DEFAULT_CONTEXT.getAppName()),
+            Optional.ofNullable(multiplexedModelId)
+                .orElse(DEFAULT_CONTEXT.getMultiplexedModelId())));
+  }
+
+  public static RequestContext get() {
+    return SERVE_REQUEST_CONTEXT.get();
+  }
+
+  public static void clean() {
+    SERVE_REQUEST_CONTEXT.remove();
+  }
+}

java/serve/src/main/java/io/ray/serve/controller/ServeController.java

@@ -0,0 +1,11 @@
+package io.ray.serve.controller;
+
+import io.ray.serve.poll.LongPollRequest;
+
+public interface ServeController {
+  byte[] getAllEndpoints();
+
+  byte[] listenForChange(LongPollRequest longPollRequest);
+
+  String getRootUrl();
+}

java/serve/src/main/java/io/ray/serve/dag/ClassNode.java

@@ -0,0 +1,27 @@
+package io.ray.serve.dag;
+
+import java.util.Map;
+
+public class ClassNode extends DAGNode {
+
+  private String className;
+
+  public ClassNode(
+      String clsName,
+      Object[] clsArgs,
+      Map<String, Object> clsOptions,
+      Map<String, Object> otherArgsToResolve) {
+    super(clsArgs, clsOptions, otherArgsToResolve);
+    this.className = clsName;
+  }
+
+  @Override
+  public DAGNode copyImpl(
+      Object[] newArgs, Map<String, Object> newOptions, Map<String, Object> newOtherArgsToResolve) {
+    return new ClassNode(className, newArgs, newOptions, newOtherArgsToResolve);
+  }
+
+  public String getClassName() {
+    return className;
+  }
+}

java/serve/src/main/java/io/ray/serve/dag/DAGNode.java

@@ -0,0 +1,87 @@
+package io.ray.serve.dag;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+
+public abstract class DAGNode implements DAGNodeBase {
+
+  private final Object[] boundArgs;
+
+  private final Map<String, Object> boundOptions;
+
+  private final Map<String, Object> boundOtherArgsToResolve;
+
+  private String stableUuid = UUID.randomUUID().toString().replace("-", "");
+
+  public DAGNode(
+      Object[] args, Map<String, Object> options, Map<String, Object> otherArgsToResolve) {
+    this.boundArgs = args != null ? args : new Object[0];
+    this.boundOptions = options != null ? options : new HashMap<>();
+    this.boundOtherArgsToResolve =
+        otherArgsToResolve != null ? otherArgsToResolve : new HashMap<>();
+  }
+
+  @Override
+  public <T> T applyRecursive(Function<DAGNodeBase, T> fn) {
+    if (!(fn instanceof CachingFn)) {
+      Function<DAGNodeBase, T> newFun = new CachingFn<>(fn);
+      return newFun.apply(applyAndReplaceAllChildNodes(node -> node.applyRecursive(newFun)));
+    } else {
+      return fn.apply(applyAndReplaceAllChildNodes(node -> node.applyRecursive(fn)));
+    }
+  }
+
+  @Override
+  public <T> DAGNodeBase applyAndReplaceAllChildNodes(Function<DAGNodeBase, T> fn) {
+    Object[] newArgs = new Object[boundArgs.length];
+    for (int i = 0; i < boundArgs.length; i++) {
+      newArgs[i] =
+          boundArgs[i] instanceof DAGNodeBase ? fn.apply((DAGNodeBase) boundArgs[i]) : boundArgs[i];
+    }
+
+    return copy(newArgs, boundOptions, boundOtherArgsToResolve);
+  }
+
+  @Override
+  public DAGNodeBase copy(
+      Object[] newArgs, Map<String, Object> newOptions, Map<String, Object> newOtherArgsToResolve) {
+    DAGNode instance = (DAGNode) copyImpl(newArgs, newOptions, newOtherArgsToResolve);
+    instance.stableUuid = stableUuid;
+    return instance;
+  }
+
+  public Object[] getBoundArgs() {
+    return boundArgs;
+  }
+
+  public Map<String, Object> getBoundOptions() {
+    return boundOptions;
+  }
+
+  public Map<String, Object> getBoundOtherArgsToResolve() {
+    return boundOtherArgsToResolve;
+  }
+
+  @Override
+  public String getStableUuid() {
+    return stableUuid;
+  }
+
+  private static class CachingFn<T> implements Function<DAGNodeBase, T> {
+
+    private final Map<String, T> cache = new HashMap<>();
+
+    private final Function<DAGNodeBase, T> function;
+
+    public CachingFn(Function<DAGNodeBase, T> function) {
+      this.function = function;
+    }
+
+    @Override
+    public T apply(DAGNodeBase t) {
+      return cache.computeIfAbsent(t.getStableUuid(), key -> function.apply(t));
+    }
+  }
+}

java/serve/src/main/java/io/ray/serve/dag/DAGNodeBase.java

@@ -0,0 +1,21 @@
+package io.ray.serve.dag;
+
+import java.util.Map;
+import java.util.function.Function;
+
+public interface DAGNodeBase {
+
+  <T> T applyRecursive(Function<DAGNodeBase, T> fn);
+
+  <T> DAGNodeBase applyAndReplaceAllChildNodes(Function<DAGNodeBase, T> fn);
+
+  DAGNodeBase copy(
+      Object[] newArgs, Map<String, Object> newOptions, Map<String, Object> newOtherArgsToResolve);
+
+  default DAGNodeBase copyImpl(
+      Object[] newArgs, Map<String, Object> newOptions, Map<String, Object> newOtherArgsToResolve) {
+    throw new UnsupportedOperationException();
+  }
+
+  String getStableUuid();
+}

java/serve/src/main/java/io/ray/serve/dag/DeploymentNode.java

@@ -0,0 +1,44 @@
+package io.ray.serve.dag;
+
+import io.ray.serve.deployment.Deployment;
+import io.ray.serve.handle.DeploymentHandle;
+import java.util.Map;
+
+public class DeploymentNode extends DAGNode {
+
+  private String appName;
+
+  private Deployment deployment;
+
+  private DeploymentHandle deploymentHandle;
+
+  public DeploymentNode(
+      Deployment deployment,
+      String appName,
+      Object[] deploymentInitArgs,
+      Map<String, Object> rayActorOptions,
+      Map<String, Object> otherArgsToResolve) {
+    super(deploymentInitArgs, rayActorOptions, otherArgsToResolve);
+    this.appName = appName;
+    this.deployment = deployment;
+    this.deploymentHandle = new DeploymentHandle(deployment.getName(), appName);
+  }
+
+  @Override
+  public DAGNode copyImpl(
+      Object[] newArgs, Map<String, Object> newOptions, Map<String, Object> newOtherArgsToResolve) {
+    return new DeploymentNode(deployment, appName, newArgs, newOptions, newOtherArgsToResolve);
+  }
+
+  public String getAppName() {
+    return appName;
+  }
+
+  public Deployment getDeployment() {
+    return deployment;
+  }
+
+  public DeploymentHandle getDeploymentHandle() {
+    return deploymentHandle;
+  }
+}

java/serve/src/main/java/io/ray/serve/dag/Graph.java

@@ -0,0 +1,81 @@
+package io.ray.serve.dag;
+
+import io.ray.serve.deployment.Deployment;
+import io.ray.serve.handle.DeploymentHandle;
+import io.ray.serve.util.CommonUtil;
+import io.ray.serve.util.DAGUtil;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+
+public class Graph {
+
+  public static List<Deployment> build(DAGNode rayDagRootNode, String name) {
+    DAGNodeBase serveRootDag =
+        rayDagRootNode.applyRecursive(node -> transformRayDagToServeDag(node, name));
+    return extractDeployments(serveRootDag);
+  }
+
+  public static DAGNodeBase transformRayDagToServeDag(DAGNodeBase dagNode, String appName) {
+    if (dagNode instanceof ClassNode) {
+      ClassNode clsNode = (ClassNode) dagNode;
+      Deployment deploymentShell =
+          (Deployment) clsNode.getBoundOtherArgsToResolve().get("deployment_schema");
+
+      String deploymentName = DAGUtil.getNodeName(clsNode);
+      if (!StringUtils.equals(
+          deploymentShell.getName(), CommonUtil.getDeploymentName(clsNode.getClassName()))) {
+        deploymentName = deploymentShell.getName();
+      }
+
+      Object[] replacedDeploymentInitArgs = new Object[clsNode.getBoundArgs().length];
+      for (int i = 0; i < clsNode.getBoundArgs().length; i++) {
+        replacedDeploymentInitArgs[i] =
+            clsNode.getBoundArgs()[i] instanceof DeploymentNode
+                ? replaceWithHandle((DeploymentNode) clsNode.getBoundArgs()[i])
+                : clsNode.getBoundArgs()[i];
+      }
+
+      Deployment deployment =
+          deploymentShell
+              .options()
+              .setDeploymentDef(clsNode.getClassName())
+              .setName(deploymentName)
+              .setInitArgs(replacedDeploymentInitArgs)
+              .create(false);
+
+      return new DeploymentNode(
+          deployment,
+          appName,
+          clsNode.getBoundArgs(),
+          clsNode.getBoundOptions(),
+          clsNode.getBoundOtherArgsToResolve());
+    }
+
+    return dagNode;
+  }
+
+  public static List<Deployment> extractDeployments(DAGNodeBase rootNode) {
+    Map<String, Deployment> deployments = new LinkedHashMap<>();
+    rootNode.applyRecursive(
+        node -> {
+          if (node instanceof DeploymentNode) {
+            Deployment deployment = ((DeploymentNode) node).getDeployment();
+            deployments.put(deployment.getName(), deployment);
+          }
+          return node;
+        });
+    return deployments.values().stream().collect(Collectors.toList());
+  }
+
+  public static DeploymentHandle replaceWithHandle(DAGNode node) {
+    if (node instanceof DeploymentNode) {
+      DeploymentNode deploymentNode = (DeploymentNode) node;
+      return new DeploymentHandle(
+          deploymentNode.getDeployment().getName(), deploymentNode.getAppName());
+    }
+    return null;
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/Application.java

@@ -0,0 +1,43 @@
+package io.ray.serve.deployment;
+
+import io.ray.serve.dag.DAGNode;
+import io.ray.serve.dag.DAGNodeBase;
+import java.util.Map;
+import java.util.function.Function;
+
+public class Application implements DAGNodeBase {
+  private DAGNode internalDagNode;
+
+  private Application(DAGNode internalDagNode) {
+    this.internalDagNode = internalDagNode;
+  }
+
+  public DAGNode getInternalDagNode() {
+    return internalDagNode;
+  }
+
+  public static Application fromInternalDagNode(DAGNode dagNode) {
+    return new Application(dagNode);
+  }
+
+  @Override
+  public <T> T applyRecursive(Function<DAGNodeBase, T> fn) {
+    return internalDagNode.applyRecursive(fn);
+  }
+
+  @Override
+  public <T> DAGNodeBase applyAndReplaceAllChildNodes(Function<DAGNodeBase, T> fn) {
+    return internalDagNode.applyAndReplaceAllChildNodes(fn);
+  }
+
+  @Override
+  public DAGNodeBase copy(
+      Object[] newArgs, Map<String, Object> newOptions, Map<String, Object> newOtherArgsToResolve) {
+    return internalDagNode.copy(newArgs, newOptions, newOtherArgsToResolve);
+  }
+
+  @Override
+  public String getStableUuid() {
+    return internalDagNode.getStableUuid();
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/Deployment.java

@@ -0,0 +1,135 @@
+package io.ray.serve.deployment;
+
+import com.google.common.base.Preconditions;
+import io.ray.serve.api.Serve;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.config.ReplicaConfig;
+import io.ray.serve.dag.ClassNode;
+import io.ray.serve.dag.DAGNode;
+import io.ray.serve.handle.DeploymentHandle;
+import java.util.HashMap;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Construct a Deployment. CONSTRUCTOR SHOULDN'T BE USED DIRECTLY.
+ *
+ * <p>Deployments should be created, retrieved, and updated using `Serve.deployment.create`,
+ * `Serve.getDeployment`, and `Deployment.options.create`, respectively.
+ */
+public class Deployment {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(Deployment.class);
+
+  private final String name;
+
+  private final DeploymentConfig deploymentConfig;
+
+  private final ReplicaConfig replicaConfig;
+
+  private final String version;
+
+  private boolean ingress;
+
+  // TODO support placement group.
+
+  public Deployment(
+      String name, DeploymentConfig deploymentConfig, ReplicaConfig replicaConfig, String version) {
+
+    Preconditions.checkArgument(
+        version != null || deploymentConfig.getAutoscalingConfig() == null,
+        "Currently autoscaling is only supported for versioned deployments. Try Serve.deployment.setVersion.");
+
+    this.name = name;
+    this.version = version;
+    this.deploymentConfig = deploymentConfig;
+    this.replicaConfig = replicaConfig;
+  }
+
+  /**
+   * Get a ServeHandle to this deployment to invoke it from Java.
+   *
+   * @return ServeHandle
+   * @deprecated {@value Constants#MIGRATION_MESSAGE}
+   */
+  @Deprecated
+  public DeploymentHandle getHandle() {
+    LOGGER.warn(Constants.MIGRATION_MESSAGE);
+    return Serve.getGlobalClient().getDeploymentHandle(name, "", true);
+  }
+
+  /**
+   * Return a copy of this deployment with updated options.
+   *
+   * <p>Only those options passed in will be updated, all others will remain unchanged from the
+   * existing deployment.
+   *
+   * @return
+   */
+  public DeploymentCreator options() {
+
+    return new DeploymentCreator()
+        .setDeploymentDef(this.replicaConfig.getDeploymentDef())
+        .setName(this.name)
+        .setVersion(this.version)
+        .setNumReplicas(this.deploymentConfig.getNumReplicas())
+        .setInitArgs(this.replicaConfig.getInitArgs())
+        .setRayActorOptions(this.replicaConfig.getRayActorOptions())
+        .setUserConfig(this.deploymentConfig.getUserConfig())
+        .setMaxOngoingRequests(this.deploymentConfig.getMaxOngoingRequests())
+        .setAutoscalingConfig(this.deploymentConfig.getAutoscalingConfig())
+        .setGracefulShutdownWaitLoopS(this.deploymentConfig.getGracefulShutdownWaitLoopS())
+        .setGracefulShutdownTimeoutS(this.deploymentConfig.getGracefulShutdownTimeoutS())
+        .setHealthCheckPeriodS(this.deploymentConfig.getHealthCheckPeriodS())
+        .setHealthCheckTimeoutS(this.deploymentConfig.getHealthCheckTimeoutS())
+        .setLanguage(this.deploymentConfig.getDeploymentLanguage());
+  }
+
+  /**
+   * Bind the arguments to the deployment and return an Application.
+   *
+   * <p>The returned Application can be deployed using `serve.run` (or via config file) or bound to
+   * another deployment for composition.
+   *
+   * @param args
+   * @return
+   */
+  public Application bind(Object... args) {
+    Map<String, Object> otherArgsToResolve = new HashMap<>();
+    otherArgsToResolve.put("deployment_schema", this);
+    otherArgsToResolve.put("is_from_serve_deployment", true);
+    DAGNode dagNode =
+        new ClassNode(
+            replicaConfig.getDeploymentDef(),
+            args,
+            replicaConfig.getRayActorOptions(),
+            otherArgsToResolve);
+    return Application.fromInternalDagNode(dagNode);
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public DeploymentConfig getDeploymentConfig() {
+    return deploymentConfig;
+  }
+
+  public ReplicaConfig getReplicaConfig() {
+    return replicaConfig;
+  }
+
+  public String getVersion() {
+    return version;
+  }
+
+  public boolean isIngress() {
+    return ingress;
+  }
+
+  public void setIngress(boolean ingress) {
+    this.ingress = ingress;
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/DeploymentCreator.java

@@ -0,0 +1,245 @@
+package io.ray.serve.deployment;
+
+import com.google.common.base.Preconditions;
+import io.ray.serve.api.Serve;
+import io.ray.serve.config.AutoscalingConfig;
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.config.ReplicaConfig;
+import io.ray.serve.generated.DeploymentLanguage;
+import io.ray.serve.util.CommonUtil;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DeploymentCreator {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(Serve.class);
+
+  private String deploymentDef;
+
+  /**
+   * Globally-unique name identifying this deployment. If not provided, the name of the class or
+   * function will be used.
+   */
+  private String name;
+
+  /**
+   * Version of the deployment. This is used to indicate a code change for the deployment; when it
+   * is re-deployed with a version change, a rolling update of the replicas will be performed. If
+   * not provided, every deployment will be treated as a new version.
+   */
+  @Deprecated private String version;
+
+  /**
+   * The number of processes to start up that will handle requests to this deployment. Defaults to
+   * 1.
+   */
+  private Integer numReplicas;
+
+  /**
+   * Positional args to be passed to the class constructor when starting up deployment replicas.
+   * These can also be passed when you call `.deploy()` on the returned Deployment.
+   */
+  private Object[] initArgs;
+
+  /** Options to be passed to the Ray actor constructor such as resource requirements. */
+  private Map<String, Object> rayActorOptions;
+
+  /**
+   * [experimental] Config to pass to the reconfigure method of the deployment. This can be updated
+   * dynamically without changing the version of the deployment and restarting its replicas.
+   */
+  private Object userConfig;
+
+  /**
+   * The maximum number of queries that will be sent to a replica of this deployment without
+   * receiving a response. Defaults to 100.
+   */
+  private Integer maxOngoingRequests;
+
+  private AutoscalingConfig autoscalingConfig;
+
+  private Double gracefulShutdownWaitLoopS;
+
+  private Double gracefulShutdownTimeoutS;
+
+  private Double healthCheckPeriodS;
+
+  private Double healthCheckTimeoutS;
+
+  private DeploymentLanguage language;
+
+  // TODO is_driver_deployment\placement_group_bundles\placement_group_strategy
+
+  public Deployment create(boolean check) {
+
+    if (check) {
+      Preconditions.checkArgument(
+          numReplicas == null || numReplicas != 0, "num_replicas is expected to larger than 0");
+
+      Preconditions.checkArgument(
+          numReplicas == null || autoscalingConfig == null,
+          "Manually setting num_replicas is not allowed when autoscalingConfig is provided.");
+    }
+
+    if (version != null) {
+      LOGGER.warn(
+          "DeprecationWarning: `version` in `@serve.deployment` has been deprecated. Explicitly specifying version will raise an error in the future!");
+    }
+
+    DeploymentConfig deploymentConfig =
+        new DeploymentConfig()
+            .setNumReplicas(numReplicas != null ? numReplicas : 1)
+            .setMaxOngoingRequests(maxOngoingRequests)
+            .setUserConfig(userConfig)
+            .setAutoscalingConfig(autoscalingConfig)
+            .setGracefulShutdownWaitLoopS(gracefulShutdownWaitLoopS)
+            .setGracefulShutdownTimeoutS(gracefulShutdownTimeoutS)
+            .setHealthCheckPeriodS(healthCheckPeriodS)
+            .setHealthCheckTimeoutS(healthCheckTimeoutS)
+            .setDeploymentLanguage(language);
+
+    ReplicaConfig replicaConfig = new ReplicaConfig(deploymentDef, initArgs, rayActorOptions);
+
+    return new Deployment(
+        StringUtils.isNotBlank(name) ? name : CommonUtil.getDeploymentName(deploymentDef),
+        deploymentConfig,
+        replicaConfig,
+        version);
+  }
+
+  public Deployment create() {
+    return create(true);
+  }
+
+  public Application bind(Object... args) {
+    return create().bind(args);
+  }
+
+  public String getDeploymentDef() {
+    return deploymentDef;
+  }
+
+  public DeploymentCreator setDeploymentDef(String deploymentDef) {
+    this.deploymentDef = deploymentDef;
+    return this;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public DeploymentCreator setName(String name) {
+    this.name = name;
+    return this;
+  }
+
+  public String getVersion() {
+    return version;
+  }
+
+  public DeploymentCreator setVersion(String version) {
+    this.version = version;
+    return this;
+  }
+
+  public Integer getNumReplicas() {
+    return numReplicas;
+  }
+
+  public DeploymentCreator setNumReplicas(Integer numReplicas) {
+    this.numReplicas = numReplicas;
+    return this;
+  }
+
+  public Object[] getInitArgs() {
+    return initArgs;
+  }
+
+  public DeploymentCreator setInitArgs(Object[] initArgs) {
+    this.initArgs = initArgs;
+    return this;
+  }
+
+  public Map<String, Object> getRayActorOptions() {
+    return rayActorOptions;
+  }
+
+  public DeploymentCreator setRayActorOptions(Map<String, Object> rayActorOptions) {
+    this.rayActorOptions = rayActorOptions;
+    return this;
+  }
+
+  public Object getUserConfig() {
+    return userConfig;
+  }
+
+  public DeploymentCreator setUserConfig(Object userConfig) {
+    this.userConfig = userConfig;
+    return this;
+  }
+
+  public Integer getMaxOngoingRequests() {
+    return maxOngoingRequests;
+  }
+
+  public DeploymentCreator setMaxOngoingRequests(Integer maxOngoingRequests) {
+    this.maxOngoingRequests = maxOngoingRequests;
+    return this;
+  }
+
+  public AutoscalingConfig getAutoscalingConfig() {
+    return autoscalingConfig;
+  }
+
+  public DeploymentCreator setAutoscalingConfig(AutoscalingConfig autoscalingConfig) {
+    this.autoscalingConfig = autoscalingConfig;
+    return this;
+  }
+
+  public Double getGracefulShutdownWaitLoopS() {
+    return gracefulShutdownWaitLoopS;
+  }
+
+  public DeploymentCreator setGracefulShutdownWaitLoopS(Double gracefulShutdownWaitLoopS) {
+    this.gracefulShutdownWaitLoopS = gracefulShutdownWaitLoopS;
+    return this;
+  }
+
+  public Double getGracefulShutdownTimeoutS() {
+    return gracefulShutdownTimeoutS;
+  }
+
+  public DeploymentCreator setGracefulShutdownTimeoutS(Double gracefulShutdownTimeoutS) {
+    this.gracefulShutdownTimeoutS = gracefulShutdownTimeoutS;
+    return this;
+  }
+
+  public Double getHealthCheckPeriodS() {
+    return healthCheckPeriodS;
+  }
+
+  public DeploymentCreator setHealthCheckPeriodS(Double healthCheckPeriodS) {
+    this.healthCheckPeriodS = healthCheckPeriodS;
+    return this;
+  }
+
+  public Double getHealthCheckTimeoutS() {
+    return healthCheckTimeoutS;
+  }
+
+  public DeploymentCreator setHealthCheckTimeoutS(Double healthCheckTimeoutS) {
+    this.healthCheckTimeoutS = healthCheckTimeoutS;
+    return this;
+  }
+
+  public DeploymentLanguage getLanguage() {
+    return language;
+  }
+
+  public DeploymentCreator setLanguage(DeploymentLanguage language) {
+    this.language = language;
+    return this;
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/DeploymentId.java

@@ -0,0 +1,40 @@
+package io.ray.serve.deployment;
+
+import java.io.Serializable;
+import org.apache.commons.lang3.StringUtils;
+
+public class DeploymentId implements Serializable {
+  private static final long serialVersionUID = 3423413558240304854L;
+  private final String app;
+  private final String name;
+
+  public DeploymentId(String name, String app) {
+    this.name = name;
+    this.app = app;
+  }
+
+  public String toReplicaActorClassName() {
+    if (StringUtils.isBlank(app)) {
+      return "ServeReplica:" + name;
+    } else {
+      return "ServeReplica:" + app + ":" + name;
+    }
+  }
+
+  public String getApp() {
+    return app;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public String toString() {
+    if (StringUtils.isBlank(app)) {
+      return name;
+    } else {
+      return app + "_" + name;
+    }
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/DeploymentInfo.java

@@ -0,0 +1,97 @@
+package io.ray.serve.deployment;
+
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.config.ReplicaConfig;
+
+public class DeploymentInfo {
+
+  private String name;
+
+  private DeploymentConfig deploymentConfig;
+
+  private ReplicaConfig replicaConfig;
+
+  private Long startTimeMs;
+
+  private String actorName;
+
+  private String version;
+
+  private Long endTimeMs;
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public DeploymentConfig getDeploymentConfig() {
+    return deploymentConfig;
+  }
+
+  public void setDeploymentConfig(DeploymentConfig deploymentConfig) {
+    this.deploymentConfig = deploymentConfig;
+  }
+
+  public ReplicaConfig getReplicaConfig() {
+    return replicaConfig;
+  }
+
+  public void setReplicaConfig(ReplicaConfig replicaConfig) {
+    this.replicaConfig = replicaConfig;
+  }
+
+  public Long getStartTimeMs() {
+    return startTimeMs;
+  }
+
+  public void setStartTimeMs(Long startTimeMs) {
+    this.startTimeMs = startTimeMs;
+  }
+
+  public String getActorName() {
+    return actorName;
+  }
+
+  public void setActorName(String actorName) {
+    this.actorName = actorName;
+  }
+
+  public String getVersion() {
+    return version;
+  }
+
+  public void setVersion(String version) {
+    this.version = version;
+  }
+
+  public Long getEndTimeMs() {
+    return endTimeMs;
+  }
+
+  public void setEndTimeMs(Long endTimeMs) {
+    this.endTimeMs = endTimeMs;
+  }
+
+  public static DeploymentInfo fromProto(io.ray.serve.generated.DeploymentInfo proto) {
+
+    if (proto == null) {
+      return null;
+    }
+    DeploymentInfo deploymentInfo = new DeploymentInfo();
+    deploymentInfo.setName(proto.getName());
+    deploymentInfo.setDeploymentConfig(DeploymentConfig.fromProto(proto.getDeploymentConfig()));
+    deploymentInfo.setReplicaConfig(ReplicaConfig.fromProto(proto.getReplicaConfig()));
+    if (proto.getStartTimeMs() != 0) {
+      deploymentInfo.setStartTimeMs(proto.getStartTimeMs());
+    }
+    deploymentInfo.setActorName(proto.getActorName());
+    deploymentInfo.setVersion(proto.getVersion());
+    if (proto.getEndTimeMs() != 0) {
+      deploymentInfo.setEndTimeMs(proto.getEndTimeMs());
+    }
+    return deploymentInfo;
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/DeploymentRoute.java

@@ -0,0 +1,37 @@
+package io.ray.serve.deployment;
+
+import io.ray.serve.util.ServeProtoUtil;
+
+public class DeploymentRoute {
+
+  private final DeploymentInfo deploymentInfo;
+
+  private final String route;
+
+  public DeploymentRoute(DeploymentInfo deploymentInfo, String route) {
+    this.deploymentInfo = deploymentInfo;
+    this.route = route;
+  }
+
+  public DeploymentInfo getDeploymentInfo() {
+    return deploymentInfo;
+  }
+
+  public String getRoute() {
+    return route;
+  }
+
+  public static DeploymentRoute fromProto(io.ray.serve.generated.DeploymentRoute proto) {
+    if (proto == null) {
+      return null;
+    }
+    return new DeploymentRoute(
+        DeploymentInfo.fromProto(proto.getDeploymentInfo()), proto.getRoute());
+  }
+
+  public static DeploymentRoute fromProtoBytes(byte[] bytes) {
+    io.ray.serve.generated.DeploymentRoute proto =
+        ServeProtoUtil.bytesToProto(bytes, io.ray.serve.generated.DeploymentRoute::parseFrom);
+    return fromProto(proto);
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/DeploymentStatusInfo.java

@@ -0,0 +1,46 @@
+package io.ray.serve.deployment;
+
+import io.ray.serve.generated.DeploymentStatus;
+
+public class DeploymentStatusInfo {
+
+  private String name;
+
+  private DeploymentStatus deploymentStatus;
+
+  private String message = "";
+
+  public DeploymentStatusInfo(String name, DeploymentStatus deploymentStatus, String message) {
+    this.name = name;
+    this.deploymentStatus = deploymentStatus;
+    this.message = message;
+  }
+
+  public static DeploymentStatusInfo fromProto(io.ray.serve.generated.DeploymentStatusInfo proto) {
+    return new DeploymentStatusInfo(proto.getName(), proto.getStatus(), proto.getMessage());
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public DeploymentStatus getDeploymentStatus() {
+    return deploymentStatus;
+  }
+
+  public void setDeploymentStatus(DeploymentStatus deploymentStatus) {
+    this.deploymentStatus = deploymentStatus;
+  }
+
+  public String getMessage() {
+    return message;
+  }
+
+  public void setMessage(String message) {
+    this.message = message;
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/DeploymentVersion.java

@@ -0,0 +1,105 @@
+package io.ray.serve.deployment;
+
+import com.google.gson.Gson;
+import com.google.protobuf.InvalidProtocolBufferException;
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.exception.RayServeException;
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+
+public class DeploymentVersion implements Serializable {
+
+  private static Gson gson = new Gson();
+
+  private static final long serialVersionUID = 3400261981775851058L;
+
+  private String codeVersion;
+
+  private Object userConfig;
+
+  private DeploymentConfig deploymentConfig;
+
+  private Map<String, Object> rayActorOptions;
+
+  private boolean unversioned;
+
+  public DeploymentVersion() {
+    this(null, new DeploymentConfig(), null);
+  }
+
+  public DeploymentVersion(String codeVersion) {
+    this(codeVersion, new DeploymentConfig(), null);
+  }
+
+  public DeploymentVersion(
+      String codeVersion, DeploymentConfig deploymentConfig, Map<String, Object> rayActorOptions) {
+    if (StringUtils.isBlank(codeVersion)) {
+      this.unversioned = true;
+      this.codeVersion = RandomStringUtils.randomAlphabetic(6);
+    } else {
+      this.codeVersion = codeVersion;
+    }
+    if (deploymentConfig == null) {
+      deploymentConfig = new DeploymentConfig();
+    }
+    this.deploymentConfig = deploymentConfig;
+    this.rayActorOptions = rayActorOptions;
+    this.userConfig = deploymentConfig.getUserConfig();
+  }
+
+  public String getCodeVersion() {
+    return codeVersion;
+  }
+
+  public Object getUserConfig() {
+    return userConfig;
+  }
+
+  public DeploymentConfig getDeploymentConfig() {
+    return deploymentConfig;
+  }
+
+  public Map<String, Object> getRayActorOptions() {
+    return rayActorOptions;
+  }
+
+  public boolean isUnversioned() {
+    return unversioned;
+  }
+
+  public static DeploymentVersion fromProtoBytes(byte[] bytes) {
+    if (bytes == null) {
+      return new DeploymentVersion();
+    }
+
+    io.ray.serve.generated.DeploymentVersion proto = null;
+    try {
+      proto = io.ray.serve.generated.DeploymentVersion.parseFrom(bytes);
+    } catch (InvalidProtocolBufferException e) {
+      throw new RayServeException("Failed to parse DeploymentVersion from protobuf bytes.", e);
+    }
+    if (proto == null) {
+      return new DeploymentVersion();
+    }
+    return new DeploymentVersion(
+        proto.getCodeVersion(),
+        DeploymentConfig.fromProto(proto.getDeploymentConfig()),
+        gson.fromJson(proto.getRayActorOptions(), Map.class));
+  }
+
+  public byte[] toProtoBytes() {
+    io.ray.serve.generated.DeploymentVersion.Builder proto =
+        io.ray.serve.generated.DeploymentVersion.newBuilder();
+
+    if (StringUtils.isNotBlank(codeVersion)) {
+      proto.setCodeVersion(codeVersion);
+    }
+    proto.setDeploymentConfig(deploymentConfig.toProto());
+    if (rayActorOptions != null && !rayActorOptions.isEmpty()) {
+      proto.setRayActorOptions(gson.toJson(rayActorOptions));
+    }
+    return proto.build().toByteArray();
+  }
+}

java/serve/src/main/java/io/ray/serve/deployment/DeploymentWrapper.java

@@ -0,0 +1,87 @@
+package io.ray.serve.deployment;
+
+import io.ray.serve.config.DeploymentConfig;
+import java.io.Serializable;
+import java.util.Map;
+
+public class DeploymentWrapper implements Serializable {
+
+  private static final long serialVersionUID = -5366203408463039652L;
+
+  private String name;
+
+  private String deploymentDef;
+
+  private DeploymentConfig deploymentConfig;
+
+  private Object[] initArgs;
+
+  private DeploymentVersion deploymentVersion;
+
+  private Map<String, String> config;
+
+  private String appName;
+
+  public String getAppName() {
+    return appName;
+  }
+
+  public DeploymentWrapper setAppName(String appName) {
+    this.appName = appName;
+    return this;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public DeploymentWrapper setName(String name) {
+    this.name = name;
+    return this;
+  }
+
+  public String getDeploymentDef() {
+    return deploymentDef;
+  }
+
+  public DeploymentWrapper setDeploymentDef(String deploymentDef) {
+    this.deploymentDef = deploymentDef;
+    return this;
+  }
+
+  public DeploymentConfig getDeploymentConfig() {
+    return deploymentConfig;
+  }
+
+  public DeploymentWrapper setDeploymentConfig(DeploymentConfig deploymentConfig) {
+    this.deploymentConfig = deploymentConfig;
+    return this;
+  }
+
+  public Object[] getInitArgs() {
+    return initArgs;
+  }
+
+  public DeploymentWrapper setInitArgs(Object[] initArgs) {
+    this.initArgs = initArgs;
+    return this;
+  }
+
+  public DeploymentVersion getDeploymentVersion() {
+    return deploymentVersion;
+  }
+
+  public DeploymentWrapper setDeploymentVersion(DeploymentVersion deploymentVersion) {
+    this.deploymentVersion = deploymentVersion;
+    return this;
+  }
+
+  public Map<String, String> getConfig() {
+    return config;
+  }
+
+  public DeploymentWrapper setConfig(Map<String, String> config) {
+    this.config = config;
+    return this;
+  }
+}

java/serve/src/main/java/io/ray/serve/exception/RayServeException.java

@@ -0,0 +1,16 @@
+package io.ray.serve.exception;
+
+import io.ray.api.exception.RayException;
+
+public class RayServeException extends RayException {
+
+  private static final long serialVersionUID = 4673951342965950469L;
+
+  public RayServeException(String message) {
+    super(message);
+  }
+
+  public RayServeException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

java/serve/src/main/java/io/ray/serve/handle/DeploymentHandle.java

@@ -0,0 +1,139 @@
+package io.ray.serve.handle;
+
+import io.ray.runtime.metric.Count;
+import io.ray.runtime.metric.Metrics;
+import io.ray.serve.api.Serve;
+import io.ray.serve.common.Constants;
+import io.ray.serve.context.RequestContext;
+import io.ray.serve.deployment.DeploymentId;
+import io.ray.serve.generated.RequestMetadata;
+import io.ray.serve.metrics.RayServeMetrics;
+import io.ray.serve.router.Router;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+
+/** A handle to a service deployment. */
+public class DeploymentHandle implements Serializable {
+
+  private static final long serialVersionUID = 550701184372753496L;
+
+  private DeploymentId deploymentId;
+
+  private HandleOptions handleOptions;
+
+  private String handleTag;
+
+  private transient Count requestCounter;
+
+  private transient volatile Router router;
+
+  public DeploymentHandle(String deploymentName, String appName) {
+    this(deploymentName, appName, null, null);
+  }
+
+  public DeploymentHandle(
+      String deploymentName, String appName, HandleOptions handleOptions, Router router) {
+    this.deploymentId = new DeploymentId(deploymentName, appName);
+    this.handleOptions = handleOptions != null ? handleOptions : new HandleOptions();
+    this.handleTag =
+        StringUtils.isBlank(appName)
+            ? deploymentName + Constants.SEPARATOR_HASH + RandomStringUtils.randomAlphabetic(6)
+            : appName
+                + Constants.SEPARATOR_HASH
+                + deploymentName
+                + Constants.SEPARATOR_HASH
+                + RandomStringUtils.randomAlphabetic(6);
+    this.router = router;
+    initMetrics();
+  }
+
+  private void initMetrics() {
+    Map<String, String> metricsTags = new HashMap<>();
+    metricsTags.put(RayServeMetrics.TAG_HANDLE, handleTag);
+    metricsTags.put(RayServeMetrics.TAG_ENDPOINT, deploymentId.getName());
+    if (StringUtils.isNotBlank(deploymentId.getApp())) {
+      metricsTags.put(RayServeMetrics.TAG_APPLICATION, deploymentId.getApp());
+    }
+    RayServeMetrics.execute(
+        () ->
+            this.requestCounter =
+                Metrics.count()
+                    .name(RayServeMetrics.SERVE_HANDLE_REQUEST_COUNTER.name())
+                    .description(RayServeMetrics.SERVE_HANDLE_REQUEST_COUNTER.getDescription())
+                    .unit("")
+                    .tags(metricsTags)
+                    .register());
+  }
+
+  /**
+   * Returns a Ray ObjectRef whose results can be waited for or retrieved using ray.wait or Ray.get
+   * (or ``await object_ref``), respectively.
+   *
+   * @param parameters The input parameters of the specified method to be invoked in the deployment.
+   * @return ObjectRef
+   */
+  public DeploymentResponse remote(Object... parameters) {
+    RequestContext requestContext = RequestContext.get();
+    RayServeMetrics.execute(() -> requestCounter.inc(1.0));
+    RequestMetadata.Builder requestMetadata = RequestMetadata.newBuilder();
+    requestMetadata.setRequestId(requestContext.getRequestId());
+    requestMetadata.setCallMethod(
+        handleOptions != null ? handleOptions.getMethodName() : Constants.CALL_METHOD);
+    requestMetadata.setRoute(requestContext.getRoute());
+    requestMetadata.setMultiplexedModelId(requestContext.getMultiplexedModelId());
+    return new DeploymentResponse(
+        getOrCreateRouter().assignRequest(requestMetadata.build(), parameters));
+  }
+
+  private Router getOrCreateRouter() {
+    if (router != null) {
+      return router;
+    }
+
+    synchronized (DeploymentHandle.class) {
+      if (router != null) {
+        return router;
+      }
+      router = new Router(Serve.getGlobalClient().getController(), deploymentId);
+    }
+    return router;
+  }
+
+  /**
+   * The constructor of DeploymentHandle is not invoked during deserialization, so it is necessary
+   * to customize some initialization behavior during the deserialization process.
+   *
+   * @param in
+   * @throws IOException
+   * @throws ClassNotFoundException
+   */
+  private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+    in.defaultReadObject();
+    initMetrics();
+  }
+
+  public DeploymentHandle method(String methodName) {
+    handleOptions.setMethodName(methodName);
+    return this;
+  }
+
+  // TODO method(String methodName, String signature)
+
+  public Router getRouter() {
+    return getOrCreateRouter();
+  }
+
+  /**
+   * Whether this handle is actively polling for replica updates.
+   *
+   * @return
+   */
+  public boolean isPolling() {
+    return router.getLongPollClient().isRunning();
+  }
+}

java/serve/src/main/java/io/ray/serve/handle/DeploymentResponse.java

@@ -0,0 +1,31 @@
+package io.ray.serve.handle;
+
+import io.ray.api.ObjectRef;
+
+/**
+ * A future-like object wrapping the result of a unary deployment handle call.
+ *
+ * <p>From outside a deployment, `.result()` can be used to retrieve the output in a blocking
+ * manner.
+ */
+public class DeploymentResponse {
+  private ObjectRef<Object> objectRef;
+
+  public DeploymentResponse(ObjectRef<Object> objectRef) {
+    this.objectRef = objectRef;
+  }
+
+  public Object result() {
+    return objectRef.get();
+  }
+
+  /**
+   * Fetch the result of the handle call synchronously.
+   *
+   * @param timeoutMs The unit is millisecond.
+   * @return call result
+   */
+  public Object result(long timeoutMs) {
+    return objectRef.get(timeoutMs);
+  }
+}

java/serve/src/main/java/io/ray/serve/handle/HandleOptions.java

@@ -0,0 +1,18 @@
+package io.ray.serve.handle;
+
+import java.io.Serializable;
+
+/** Options for each ServeHandle instances. These fields are immutable. */
+public class HandleOptions implements Serializable {
+
+  private static final long serialVersionUID = -279077795726949172L;
+  private String methodName = "call";
+
+  public String getMethodName() {
+    return methodName;
+  }
+
+  public void setMethodName(String methodName) {
+    this.methodName = methodName;
+  }
+}

java/serve/src/main/java/io/ray/serve/metrics/RayServeMetrics.java

@@ -0,0 +1,84 @@
+package io.ray.serve.metrics;
+
+import io.ray.api.Ray;
+
+public enum RayServeMetrics {
+  SERVE_HANDLE_REQUEST_COUNTER(
+      "serve_handle_request_counter",
+      "The number of handle.remote() calls that have been made on this handle."),
+
+  SERVE_NUM_ROUTER_REQUESTS(
+      "serve_num_router_requests", "The number of requests processed by the router."),
+
+  SERVE_DEPLOYMENT_QUEUED_QUERIES(
+      "serve_deployment_queued_queries",
+      "The current number of queries to this deployment waiting to be assigned to a replica."),
+
+  SERVE_DEPLOYMENT_REQUEST_COUNTER(
+      "serve_deployment_request_counter",
+      "The number of queries that have been processed in this replica."),
+
+  SERVE_DEPLOYMENT_ERROR_COUNTER(
+      "serve_deployment_error_counter",
+      "The number of exceptions that have occurred in this replica."),
+
+  SERVE_DEPLOYMENT_REPLICA_STARTS(
+      "serve_deployment_replica_starts",
+      "The number of times this replica has been restarted due to failure."),
+
+  SERVE_DEPLOYMENT_PROCESSING_LATENCY_MS(
+      "serve_deployment_processing_latency_ms", "The latency for queries to be processed."),
+
+  SERVE_REPLICA_PROCESSING_QUERIES(
+      "serve_replica_processing_queries", "The current number of queries being processed."),
+  ;
+
+  public static final String TAG_HANDLE = "handle";
+
+  public static final String TAG_ENDPOINT = "endpoint";
+
+  public static final String TAG_DEPLOYMENT = "deployment";
+
+  public static final String TAG_ROUTE = "route";
+
+  public static final String TAG_REPLICA = "replica";
+
+  public static final String TAG_APPLICATION = "application";
+
+  private static final boolean canBeUsed =
+      Ray.isInitialized() && !Ray.getRuntimeContext().isLocalMode();
+
+  private static volatile boolean enabled = true;
+
+  private String name;
+
+  private String description;
+
+  private RayServeMetrics(String name, String description) {
+    this.name = name;
+    this.description = description;
+  }
+
+  public static void execute(Runnable runnable) {
+    if (!enabled || !canBeUsed) {
+      return;
+    }
+    runnable.run();
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public static void enable() {
+    enabled = true;
+  }
+
+  public static void disable() {
+    enabled = false;
+  }
+}

java/serve/src/main/java/io/ray/serve/poll/KeyListener.java

@@ -0,0 +1,8 @@
+package io.ray.serve.poll;
+
+/** Listener of long poll. It notifies changed object to the specified key. */
+@FunctionalInterface
+public interface KeyListener {
+
+  void notifyChanged(Object updatedObject);
+}

java/serve/src/main/java/io/ray/serve/poll/KeyType.java

@@ -0,0 +1,92 @@
+package io.ray.serve.poll;
+
+import io.ray.serve.exception.RayServeException;
+import java.io.Serializable;
+import java.util.Objects;
+import org.apache.commons.lang3.StringUtils;
+
+/** Key type of long poll. */
+public class KeyType implements Serializable {
+
+  private static final long serialVersionUID = -8838552786234630401L;
+
+  private final LongPollNamespace longPollNamespace;
+
+  private final String key;
+
+  private int hashCode;
+
+  private String name;
+
+  public KeyType(LongPollNamespace longPollNamespace, String key) {
+    this.longPollNamespace = longPollNamespace;
+    this.key = key;
+    this.hashCode = Objects.hash(this.longPollNamespace, this.key);
+    this.name = parseName();
+  }
+
+  public LongPollNamespace getLongPollNamespace() {
+    return longPollNamespace;
+  }
+
+  public String getKey() {
+    return key;
+  }
+
+  @Override
+  public int hashCode() {
+    return hashCode;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    KeyType keyType = (KeyType) obj;
+    return Objects.equals(longPollNamespace, keyType.getLongPollNamespace())
+        && Objects.equals(key, keyType.getKey());
+  }
+
+  private String parseName() {
+    if (longPollNamespace == null && StringUtils.isBlank(key)) {
+      return "";
+    }
+    if (longPollNamespace == null) {
+      return key;
+    }
+    if (StringUtils.isBlank(key)) {
+      return longPollNamespace.name();
+    }
+    return "(" + longPollNamespace.name() + "," + key + ")";
+  }
+
+  @Override
+  public String toString() {
+    return name;
+  }
+
+  public static KeyType parseFrom(String key) {
+    if (key == null) {
+      return null;
+    }
+    if (StringUtils.isBlank(key)) {
+      return new KeyType(null, null);
+    }
+    if (key.startsWith("(")) {
+      String[] fields = StringUtils.split(StringUtils.substring(key, 1, key.length() - 1), ",", 2);
+      if (fields.length != 2) {
+        throw new RayServeException("Illegal KeyType string: " + key);
+      }
+      return new KeyType(LongPollNamespace.parseFrom(fields[0]), fields[1].trim());
+    }
+    LongPollNamespace longPollNamespace = LongPollNamespace.parseFrom(key);
+    if (null != longPollNamespace) {
+      return new KeyType(longPollNamespace, null);
+    }
+    return new KeyType(null, key);
+  }
+}

java/serve/src/main/java/io/ray/serve/poll/LongPollClient.java

@@ -0,0 +1,28 @@
+package io.ray.serve.poll;
+
+import com.google.common.base.Preconditions;
+import io.ray.api.BaseActorHandle;
+import java.util.Map;
+
+/** The asynchronous long polling client. */
+public class LongPollClient {
+
+  private Map<KeyType, KeyListener> keyListeners;
+
+  private boolean running;
+
+  public LongPollClient(BaseActorHandle hostActor, Map<KeyType, KeyListener> keyListeners) {
+    Preconditions.checkArgument(keyListeners != null && keyListeners.size() != 0);
+    LongPollClientFactory.register(hostActor, keyListeners);
+    this.keyListeners = keyListeners;
+    this.running = true;
+  }
+
+  public Map<KeyType, KeyListener> getKeyListeners() {
+    return keyListeners;
+  }
+
+  public boolean isRunning() {
+    return running;
+  }
+}

java/serve/src/main/java/io/ray/serve/poll/LongPollClientFactory.java

@@ -0,0 +1,249 @@
+package io.ray.serve.poll;
+
+import com.google.common.base.Preconditions;
+import io.ray.api.ActorHandle;
+import io.ray.api.BaseActorHandle;
+import io.ray.api.ObjectRef;
+import io.ray.api.PyActorHandle;
+import io.ray.api.Ray;
+import io.ray.api.exception.RayActorException;
+import io.ray.api.exception.RayTaskException;
+import io.ray.api.exception.RayTimeoutException;
+import io.ray.api.function.PyActorMethod;
+import io.ray.serve.api.Serve;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.controller.ServeController;
+import io.ray.serve.generated.DeploymentTargetInfo;
+import io.ray.serve.replica.ReplicaContext;
+import io.ray.serve.util.CollectionUtil;
+import io.ray.serve.util.ServeProtoUtil;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** The long poll client factory that holds a asynchronous singleton thread. */
+public class LongPollClientFactory {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(LongPollClientFactory.class);
+
+  /** Handle to actor embedding LongPollHost. */
+  private static BaseActorHandle hostActor;
+
+  /** A set mapping keys to callbacks to be called on state update for the corresponding keys. */
+  private static final Map<KeyType, KeyListener> KEY_LISTENERS = new ConcurrentHashMap<>();
+
+  public static final Map<KeyType, Integer> SNAPSHOT_IDS = new ConcurrentHashMap<>();
+
+  public static final Map<KeyType, Object> OBJECT_SNAPSHOTS = new ConcurrentHashMap<>();
+
+  private static ScheduledExecutorService scheduledExecutorService;
+
+  private static boolean inited = false;
+
+  private static long longPollTimoutS = 1L;
+
+  public static final Map<LongPollNamespace, Function<byte[], Object>> DESERIALIZERS =
+      new HashMap<>();
+
+  static {
+    DESERIALIZERS.put(LongPollNamespace.ROUTE_TABLE, ServeProtoUtil::parseEndpointSet);
+    DESERIALIZERS.put(
+        LongPollNamespace.DEPLOYMENT_TARGETS,
+        bytes -> ServeProtoUtil.bytesToProto(bytes, DeploymentTargetInfo::parseFrom));
+  }
+
+  public static void register(BaseActorHandle hostActor, Map<KeyType, KeyListener> keyListeners) {
+    init(hostActor);
+    if (!inited) {
+      return;
+    }
+    KEY_LISTENERS.putAll(keyListeners);
+    for (KeyType keyType : keyListeners.keySet()) {
+      SNAPSHOT_IDS.put(keyType, -1);
+    }
+    LOGGER.info("LongPollClient registered keys: {}.", keyListeners.keySet());
+    try {
+      pollNext();
+    } catch (RayTimeoutException e) {
+      LOGGER.info("Register poll timeout. keys:{}", keyListeners.keySet());
+    }
+  }
+
+  public static synchronized void init(BaseActorHandle hostActor) {
+    if (inited) {
+      return;
+    }
+    long intervalS = 10L;
+    try {
+      ReplicaContext replicaContext = Serve.getReplicaContext();
+      boolean enabled =
+          Optional.ofNullable(replicaContext.getConfig())
+              .map(config -> config.get(RayServeConfig.LONG_POOL_CLIENT_ENABLED))
+              .map(Boolean::valueOf)
+              .orElse(true);
+      if (!enabled) {
+        LOGGER.info("LongPollClient is disabled.");
+        return;
+      }
+      if (null == hostActor) {
+        hostActor = Ray.getActor(Constants.SERVE_CONTROLLER_NAME, Constants.SERVE_NAMESPACE).get();
+      }
+      intervalS =
+          Optional.ofNullable(replicaContext.getConfig())
+              .map(config -> config.get(RayServeConfig.LONG_POOL_CLIENT_INTERVAL))
+              .map(Long::valueOf)
+              .orElse(10L);
+      longPollTimoutS =
+          Optional.ofNullable(replicaContext.getConfig())
+              .map(config -> config.get(RayServeConfig.LONG_POOL_CLIENT_TIMEOUT_S))
+              .map(Long::valueOf)
+              .orElse(10L);
+    } catch (Exception e) {
+      LOGGER.info(
+          "Serve.getReplicaContext()` may only be called from within a Ray Serve deployment.");
+    }
+
+    Preconditions.checkNotNull(hostActor);
+    LongPollClientFactory.hostActor = hostActor;
+
+    scheduledExecutorService =
+        Executors.newSingleThreadScheduledExecutor(
+            new ThreadFactory() {
+              @Override
+              public Thread newThread(Runnable r) {
+                Thread thread = new Thread(r, "ray-serve-long-poll-client-thread");
+                thread.setDaemon(true);
+                return thread;
+              }
+            });
+    long finalIntervalS = intervalS;
+    scheduledExecutorService.scheduleWithFixedDelay(
+        () -> {
+          try {
+            pollNext();
+          } catch (RayTimeoutException e) {
+            LOGGER.info(
+                "long poll timeout in {} seconds, execute next poll after {} seconds.",
+                longPollTimoutS,
+                finalIntervalS);
+          } catch (RayActorException e) {
+            LOGGER.error("LongPollClient failed to connect to host. Shutting down.");
+            stop();
+          } catch (RayTaskException e) {
+            LOGGER.error("LongPollHost errored", e);
+          } catch (Throwable e) {
+            LOGGER.error("LongPollClient failed to update object of key {}", SNAPSHOT_IDS, e);
+          }
+        },
+        0L,
+        intervalS,
+        TimeUnit.SECONDS);
+    inited = true;
+    LOGGER.info("LongPollClient was initialized");
+  }
+
+  /** Poll the updates. */
+  @SuppressWarnings("unchecked")
+  public static synchronized void pollNext() {
+    LOGGER.info("LongPollClient polls next snapshotIds {}", SNAPSHOT_IDS);
+    LongPollRequest longPollRequest = new LongPollRequest(SNAPSHOT_IDS);
+    LongPollResult longPollResult = null;
+    if (hostActor instanceof PyActorHandle) {
+      // Poll from python controller.
+      ObjectRef<Object> currentRef =
+          ((PyActorHandle) hostActor)
+              .task(
+                  PyActorMethod.of(Constants.CONTROLLER_LISTEN_FOR_CHANGE_METHOD),
+                  longPollRequest.toProtobuf().toByteArray())
+              .remote();
+      Object data = Ray.get(currentRef, longPollTimoutS * 1000);
+      longPollResult = LongPollResult.parseFrom((byte[]) data);
+    } else {
+      // Poll from java controller.
+      ObjectRef<byte[]> currentRef =
+          ((ActorHandle<ServeController>) hostActor)
+              .task(ServeController::listenForChange, longPollRequest)
+              .remote();
+      longPollResult = LongPollResult.parseFrom(currentRef.get(longPollTimoutS * 1000));
+    }
+    processUpdate(longPollResult == null ? null : longPollResult.getUpdatedObjects());
+  }
+
+  public static void processUpdate(Map<KeyType, UpdatedObject> updates) {
+    if (updates == null || updates.isEmpty()) {
+      LOGGER.info("LongPollClient received nothing.");
+      return;
+    }
+    LOGGER.info("LongPollClient received updates for keys: {}", updates.keySet());
+    for (Map.Entry<KeyType, UpdatedObject> entry : updates.entrySet()) {
+      KeyType keyType = entry.getKey();
+      UpdatedObject updatedObject = entry.getValue();
+
+      Object objectSnapshot = updatedObject.getObjectSnapshot();
+
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug(
+            "The updated object for key {} is {}",
+            keyType,
+            ReflectionToStringBuilder.toString(objectSnapshot));
+      }
+
+      KeyListener keyListener = KEY_LISTENERS.get(entry.getKey());
+      if (keyListener == null) {
+        LOGGER.warn(
+            "LongPollClient has no listener for key: {}, maybe this key was garbage collected.",
+            entry.getKey());
+        continue;
+      }
+      KEY_LISTENERS.get(entry.getKey()).notifyChanged(objectSnapshot);
+      OBJECT_SNAPSHOTS.put(entry.getKey(), objectSnapshot);
+      SNAPSHOT_IDS.put(entry.getKey(), entry.getValue().getSnapshotId());
+    }
+  }
+
+  public static void unregister(Set<KeyType> keys) {
+    if (CollectionUtil.isEmpty(keys)) {
+      return;
+    }
+    for (KeyType keyType : keys) {
+      SNAPSHOT_IDS.remove(keyType);
+      KEY_LISTENERS.remove(keyType);
+      OBJECT_SNAPSHOTS.remove(keyType);
+    }
+    LOGGER.info("LongPollClient unregistered keys: {}.", keys);
+  }
+
+  public static synchronized void stop() {
+    if (!inited) {
+      return;
+    }
+    if (scheduledExecutorService != null) {
+      scheduledExecutorService.shutdown();
+      try {
+        scheduledExecutorService.awaitTermination(longPollTimoutS, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        LOGGER.error("awaitTermination error, the exception is ", e);
+      }
+    }
+    KEY_LISTENERS.clear();
+    OBJECT_SNAPSHOTS.clear();
+    SNAPSHOT_IDS.clear();
+    inited = false;
+    LOGGER.info("LongPollClient was stopped.");
+  }
+
+  public static boolean isInitialized() {
+    return inited;
+  }
+}

java/serve/src/main/java/io/ray/serve/poll/LongPollNamespace.java

@@ -0,0 +1,24 @@
+package io.ray.serve.poll;
+
+import org.apache.commons.lang3.StringUtils;
+
+/** The long poll namespace enum. */
+public enum LongPollNamespace {
+  DEPLOYMENT_TARGETS,
+
+  ROUTE_TABLE;
+
+  public static LongPollNamespace parseFrom(String key) {
+    for (LongPollNamespace namespace : LongPollNamespace.values()) {
+      if (StringUtils.equals(key, namespace.name())) {
+        return namespace;
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public String toString() {
+    return this.getClass().getSimpleName() + "." + this.name();
+  }
+}

java/serve/src/main/java/io/ray/serve/poll/LongPollRequest.java

@@ -0,0 +1,30 @@
+package io.ray.serve.poll;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class LongPollRequest implements Serializable {
+
+  private static final long serialVersionUID = 6700308335001848617L;
+
+  private Map<KeyType, Integer> keysToSnapshotIds;
+
+  public LongPollRequest(Map<KeyType, Integer> keysToSnapshotIds) {
+    this.keysToSnapshotIds = keysToSnapshotIds;
+  }
+
+  public Map<KeyType, Integer> getKeysToSnapshotIds() {
+    return keysToSnapshotIds;
+  }
+
+  public io.ray.serve.generated.LongPollRequest toProtobuf() {
+    io.ray.serve.generated.LongPollRequest.Builder builder =
+        io.ray.serve.generated.LongPollRequest.newBuilder();
+    if (keysToSnapshotIds != null) {
+      keysToSnapshotIds.entrySet().stream()
+          .forEach(
+              entry -> builder.putKeysToSnapshotIds(entry.getKey().toString(), entry.getValue()));
+    }
+    return builder.build();
+  }
+}

java/serve/src/main/java/io/ray/serve/poll/LongPollResult.java

@@ -0,0 +1,49 @@
+package io.ray.serve.poll;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import io.ray.serve.exception.RayServeException;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+public class LongPollResult implements Serializable {
+
+  private static final long serialVersionUID = 6603042077360718438L;
+
+  Map<KeyType, UpdatedObject> updatedObjects;
+
+  public Map<KeyType, UpdatedObject> getUpdatedObjects() {
+    return updatedObjects;
+  }
+
+  public void setUpdatedObjects(Map<KeyType, UpdatedObject> updatedObjects) {
+    this.updatedObjects = updatedObjects;
+  }
+
+  public static LongPollResult parseFrom(byte[] longPollResultBytes) {
+    if (longPollResultBytes == null) {
+      return null;
+    }
+    io.ray.serve.generated.LongPollResult pbLongPollResult = null;
+    try {
+      pbLongPollResult = io.ray.serve.generated.LongPollResult.parseFrom(longPollResultBytes);
+    } catch (InvalidProtocolBufferException e) {
+      throw new RayServeException("Failed to parse LongPollResult from protobuf bytes.", e);
+    }
+    if (pbLongPollResult == null) {
+      return null;
+    }
+    LongPollResult longPollResult = new LongPollResult();
+    if (pbLongPollResult.getUpdatedObjectsMap() != null) {
+      Map<KeyType, UpdatedObject> updatedObjects =
+          new HashMap<>(pbLongPollResult.getUpdatedObjectsMap().size());
+      for (Map.Entry<String, io.ray.serve.generated.UpdatedObject> entry :
+          pbLongPollResult.getUpdatedObjectsMap().entrySet()) {
+        KeyType keyType = KeyType.parseFrom(entry.getKey());
+        updatedObjects.put(keyType, UpdatedObject.parseFrom(keyType, entry.getValue()));
+      }
+      longPollResult.setUpdatedObjects(updatedObjects);
+    }
+    return longPollResult;
+  }
+}

java/serve/src/main/java/io/ray/serve/poll/UpdatedObject.java

@@ -0,0 +1,49 @@
+package io.ray.serve.poll;
+
+import io.ray.serve.exception.RayServeException;
+import java.io.Serializable;
+import java.util.function.Function;
+
+public class UpdatedObject implements Serializable {
+
+  private static final long serialVersionUID = 4079205733405537177L;
+
+  private int snapshotId;
+
+  private Object objectSnapshot;
+
+  public int getSnapshotId() {
+    return snapshotId;
+  }
+
+  public void setSnapshotId(int snapshotId) {
+    this.snapshotId = snapshotId;
+  }
+
+  public Object getObjectSnapshot() {
+    return objectSnapshot;
+  }
+
+  public void setObjectSnapshot(Object objectSnapshot) {
+    this.objectSnapshot = objectSnapshot;
+  }
+
+  public static UpdatedObject parseFrom(
+      KeyType keyType, io.ray.serve.generated.UpdatedObject pbUpdatedObject) {
+    if (pbUpdatedObject == null) {
+      return null;
+    }
+    UpdatedObject updatedObject = new UpdatedObject();
+    updatedObject.setSnapshotId(pbUpdatedObject.getSnapshotId());
+
+    Function<byte[], Object> deserializer =
+        LongPollClientFactory.DESERIALIZERS.get(keyType.getLongPollNamespace());
+    if (deserializer == null) {
+      throw new RayServeException(
+          "No deserializer for LongPollNamespace: " + keyType.getLongPollNamespace());
+    }
+    updatedObject.setObjectSnapshot(
+        deserializer.apply(pbUpdatedObject.getObjectSnapshot().toByteArray()));
+    return updatedObject;
+  }
+}

java/serve/src/main/java/io/ray/serve/replica/RayServeReplica.java

@@ -0,0 +1,22 @@
+package io.ray.serve.replica;
+
+public interface RayServeReplica {
+
+  Object handleRequest(Object requestMetadata, Object requestArgs);
+
+  default Object reconfigure(byte[] deploymentConfigBytes) {
+    return null;
+  }
+
+  default boolean checkHealth() {
+    return true;
+  }
+
+  default boolean prepareForShutdown() {
+    return true;
+  }
+
+  default int getNumOngoingRequests() {
+    return 0;
+  }
+}

java/serve/src/main/java/io/ray/serve/replica/RayServeReplicaImpl.java

@@ -0,0 +1,397 @@
+package io.ray.serve.replica;
+
+import com.google.common.collect.ImmutableMap;
+import io.ray.api.BaseActorHandle;
+import io.ray.runtime.metric.Count;
+import io.ray.runtime.metric.Gauge;
+import io.ray.runtime.metric.Histogram;
+import io.ray.runtime.metric.Metrics;
+import io.ray.serve.api.Serve;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.context.RequestContext;
+import io.ray.serve.deployment.DeploymentId;
+import io.ray.serve.deployment.DeploymentVersion;
+import io.ray.serve.exception.RayServeException;
+import io.ray.serve.generated.RequestMetadata;
+import io.ray.serve.metrics.RayServeMetrics;
+import io.ray.serve.router.Query;
+import io.ray.serve.util.MessageFormatter;
+import io.ray.serve.util.ReflectUtil;
+import java.lang.reflect.Method;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles requests with the provided callable. */
+public class RayServeReplicaImpl implements RayServeReplica {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(RayServeReplicaImpl.class);
+
+  private String deploymentName;
+
+  private DeploymentId deploymentId;
+
+  private String replicaTag;
+
+  private DeploymentConfig config;
+
+  private AtomicInteger numOngoingRequests = new AtomicInteger();
+
+  private Object callable;
+
+  private Count requestCounter;
+
+  private Count errorCounter;
+
+  private Count restartCounter;
+
+  private Histogram processingLatencyTracker;
+
+  private Gauge numProcessingItems;
+
+  private DeploymentVersion version;
+
+  private boolean isDeleted = false;
+
+  private final Method checkHealthMethod;
+
+  private final Method callMethod;
+
+  public RayServeReplicaImpl(
+      Object callable,
+      DeploymentConfig deploymentConfig,
+      DeploymentVersion version,
+      BaseActorHandle actorHandle,
+      String appName) {
+    this.deploymentName = Serve.getReplicaContext().getDeploymentName();
+    this.deploymentId = new DeploymentId(this.deploymentName, appName);
+    this.replicaTag = Serve.getReplicaContext().getReplicaTag();
+    this.callable = callable;
+    this.config = deploymentConfig;
+    this.version = version;
+    this.checkHealthMethod = getRunnerMethod(Constants.CHECK_HEALTH_METHOD, null, true);
+    this.callMethod = getRunnerMethod(Constants.CALL_METHOD, new Object[] {new Object()}, true);
+    registerMetrics();
+  }
+
+  private void registerMetrics() {
+    RayServeMetrics.execute(
+        () ->
+            requestCounter =
+                Metrics.count()
+                    .name(RayServeMetrics.SERVE_DEPLOYMENT_REQUEST_COUNTER.getName())
+                    .description(RayServeMetrics.SERVE_DEPLOYMENT_REQUEST_COUNTER.getDescription())
+                    .unit("")
+                    .tags(
+                        ImmutableMap.of(
+                            RayServeMetrics.TAG_DEPLOYMENT,
+                            deploymentName,
+                            RayServeMetrics.TAG_REPLICA,
+                            replicaTag))
+                    .register());
+
+    RayServeMetrics.execute(
+        () ->
+            errorCounter =
+                Metrics.count()
+                    .name(RayServeMetrics.SERVE_DEPLOYMENT_ERROR_COUNTER.getName())
+                    .description(RayServeMetrics.SERVE_DEPLOYMENT_ERROR_COUNTER.getDescription())
+                    .unit("")
+                    .tags(
+                        ImmutableMap.of(
+                            RayServeMetrics.TAG_DEPLOYMENT,
+                            deploymentName,
+                            RayServeMetrics.TAG_REPLICA,
+                            replicaTag))
+                    .register());
+
+    RayServeMetrics.execute(
+        () ->
+            restartCounter =
+                Metrics.count()
+                    .name(RayServeMetrics.SERVE_DEPLOYMENT_REPLICA_STARTS.getName())
+                    .description(RayServeMetrics.SERVE_DEPLOYMENT_REPLICA_STARTS.getDescription())
+                    .unit("")
+                    .tags(
+                        ImmutableMap.of(
+                            RayServeMetrics.TAG_DEPLOYMENT,
+                            deploymentName,
+                            RayServeMetrics.TAG_REPLICA,
+                            replicaTag))
+                    .register());
+
+    RayServeMetrics.execute(
+        () ->
+            processingLatencyTracker =
+                Metrics.histogram()
+                    .name(RayServeMetrics.SERVE_DEPLOYMENT_PROCESSING_LATENCY_MS.getName())
+                    .description(
+                        RayServeMetrics.SERVE_DEPLOYMENT_PROCESSING_LATENCY_MS.getDescription())
+                    .unit("")
+                    .boundaries(Constants.DEFAULT_LATENCY_BUCKET_MS)
+                    .tags(
+                        ImmutableMap.of(
+                            RayServeMetrics.TAG_DEPLOYMENT,
+                            deploymentName,
+                            RayServeMetrics.TAG_REPLICA,
+                            replicaTag))
+                    .register());
+
+    RayServeMetrics.execute(
+        () ->
+            numProcessingItems =
+                Metrics.gauge()
+                    .name(RayServeMetrics.SERVE_REPLICA_PROCESSING_QUERIES.getName())
+                    .description(RayServeMetrics.SERVE_REPLICA_PROCESSING_QUERIES.getDescription())
+                    .unit("")
+                    .tags(
+                        ImmutableMap.of(
+                            RayServeMetrics.TAG_DEPLOYMENT,
+                            deploymentName,
+                            RayServeMetrics.TAG_REPLICA,
+                            replicaTag))
+                    .register());
+
+    RayServeMetrics.execute(() -> restartCounter.inc(1.0));
+  }
+
+  @Override
+  public Object handleRequest(Object requestMetadata, Object requestArgs) {
+    // TODO 1101 wrap_user_method_call
+    long startTime = System.currentTimeMillis();
+    Query request = new Query((RequestMetadata) requestMetadata, requestArgs);
+    LOGGER.debug(
+        "Replica {} received request {}", replicaTag, request.getMetadata().getRequestId());
+
+    numOngoingRequests.incrementAndGet();
+    RayServeMetrics.execute(() -> numProcessingItems.update(numOngoingRequests.get()));
+    Object result = invokeSingle(request);
+    numOngoingRequests.decrementAndGet();
+
+    long requestTimeMs = System.currentTimeMillis() - startTime;
+    LOGGER.debug(
+        "Replica {} finished request {} in {}ms",
+        replicaTag,
+        request.getMetadata().getRequestId(),
+        requestTimeMs);
+
+    return result;
+  }
+
+  private Object invokeSingle(Query requestItem) {
+
+    long start = System.currentTimeMillis();
+    Method methodToCall = null;
+    RequestMetadata requestMetadata = requestItem.getMetadata();
+    try {
+      RequestContext.set(
+          requestMetadata.getRoute(),
+          requestMetadata.getRequestId(),
+          deploymentId.getApp(),
+          requestMetadata.getMultiplexedModelId());
+      LOGGER.info("Started executing request {}", requestMetadata.getRequestId());
+
+      Object[] args = parseRequestItem(requestItem);
+      methodToCall =
+          args.length == 1 && callMethod != null
+              ? callMethod
+              : getRunnerMethod(requestItem.getMetadata().getCallMethod(), args, false);
+      Object result = methodToCall.invoke(callable, args);
+      RayServeMetrics.execute(() -> requestCounter.inc(1.0));
+      return result;
+    } catch (Throwable e) {
+      RayServeMetrics.execute(() -> errorCounter.inc(1.0));
+      throw new RayServeException(
+          MessageFormatter.format(
+              "Replica {} failed to invoke method {}",
+              replicaTag,
+              methodToCall == null ? "unknown" : methodToCall.getName()),
+          e);
+    } finally {
+      RayServeMetrics.execute(
+          () -> processingLatencyTracker.update(System.currentTimeMillis() - start));
+      RequestContext.clean();
+    }
+  }
+
+  private Object[] parseRequestItem(Query requestItem) {
+    if (requestItem.getArgs() == null) {
+      return new Object[0];
+    }
+
+    if (requestItem.getArgs() instanceof Object[]) {
+      return (Object[]) requestItem.getArgs();
+    }
+
+    return new Object[] {requestItem.getArgs()};
+  }
+
+  private Method getRunnerMethod(String methodName, Object[] args, boolean isNullable) {
+    try {
+      return ReflectUtil.getMethod(callable.getClass(), methodName, args);
+    } catch (NoSuchMethodException e) {
+      String errMsg =
+          MessageFormatter.format(
+              "Tried to call a method {} that does not exist. Available methods: {}",
+              methodName,
+              ReflectUtil.getMethodStrings(callable.getClass()));
+      if (isNullable) {
+        LOGGER.warn(errMsg);
+        return null;
+      } else {
+        LOGGER.error(errMsg, e);
+        throw new RayServeException(errMsg, e);
+      }
+    }
+  }
+
+  /**
+   * Perform graceful shutdown. Trigger a graceful shutdown protocol that will wait for all the
+   * queued tasks to be completed and return to the controller.
+   *
+   * @return true if it is ready for shutdown.
+   */
+  @Override
+  public synchronized boolean prepareForShutdown() {
+    while (true) {
+      // Sleep first because we want to make sure all the routers receive the notification to remove
+      // this replica first.
+      try {
+        Thread.sleep((long) (config.getGracefulShutdownWaitLoopS() * 1000));
+      } catch (InterruptedException e) {
+        LOGGER.error(
+            "Replica {} was interrupted in sheep when draining pending queries", replicaTag);
+      }
+      int numOngoingRequest = getNumOngoingRequests();
+      if (numOngoingRequest > 0) {
+        LOGGER.info(
+            "Waiting for an additional {}s to shut down because there are {} ongoing requests.",
+            config.getGracefulShutdownWaitLoopS(),
+            numOngoingRequest);
+      } else {
+        LOGGER.info("Graceful shutdown complete; replica exiting.");
+        break;
+      }
+    }
+
+    // Explicitly call the del method to trigger clean up. We set isDeleted = true after
+    // succssifully calling it so the destructor is called only once.
+    try {
+      if (!isDeleted) {
+        ReflectUtil.getMethod(callable.getClass(), "del").invoke(callable);
+      }
+    } catch (NoSuchMethodException e) {
+      LOGGER.warn("Deployment {} has no del method.", deploymentName);
+    } catch (Throwable e) {
+      LOGGER.error("Exception during graceful shutdown of replica.");
+    } finally {
+      isDeleted = true;
+    }
+    return true;
+  }
+
+  @Override
+  public int getNumOngoingRequests() {
+    return numOngoingRequests.get();
+  }
+
+  @Override
+  public DeploymentVersion reconfigure(byte[] deploymentConfigBytes) {
+    config = DeploymentConfig.fromProtoBytes(deploymentConfigBytes);
+    Object userConfig = config.getUserConfig();
+    DeploymentVersion deploymentVersion =
+        new DeploymentVersion(version.getCodeVersion(), config, version.getRayActorOptions());
+    version = deploymentVersion;
+    if (userConfig != null) {
+      updateUserConfig(userConfig);
+    }
+    return version;
+  }
+
+  public void updateUserConfig(Object userConfig) {
+    LOGGER.info(
+        "Replica {} of deployment {} reconfigure userConfig: {}",
+        replicaTag,
+        deploymentName,
+        userConfig);
+    try {
+      ReflectUtil.getMethod(callable.getClass(), Constants.RECONFIGURE_METHOD, userConfig)
+          .invoke(callable, userConfig);
+    } catch (NoSuchMethodException e) {
+      String errMsg =
+          MessageFormatter.format(
+              "userConfig specified but deployment {} missing {} method",
+              deploymentId,
+              Constants.RECONFIGURE_METHOD);
+      LOGGER.error(errMsg);
+      throw new RayServeException(errMsg, e);
+    } catch (Throwable e) {
+      String errMsg =
+          MessageFormatter.format(
+              "Replica {} of deployment {} failed to reconfigure userConfig {}",
+              replicaTag,
+              deploymentId,
+              userConfig);
+      LOGGER.error(errMsg);
+      throw new RayServeException(errMsg, e);
+    } finally {
+      LOGGER.info(
+          "Replica {} of deployment {} finished reconfiguring userConfig: {}",
+          replicaTag,
+          deploymentId,
+          userConfig);
+    }
+  }
+
+  public DeploymentVersion getVersion() {
+    return version;
+  }
+
+  @Override
+  public boolean checkHealth() {
+    if (checkHealthMethod == null) {
+      return true;
+    }
+    boolean result = true;
+    try {
+      LOGGER.info(
+          "Replica {} of deployment {} check health of {}",
+          replicaTag,
+          deploymentName,
+          callable.getClass().getName());
+      Object isHealthy = checkHealthMethod.invoke(callable);
+      if (!(isHealthy instanceof Boolean)) {
+        LOGGER.error(
+            "The health check result {} of {} in replica {} of deployment {} is illegal.",
+            isHealthy == null ? "null" : isHealthy.getClass().getName() + ":" + isHealthy,
+            callable.getClass().getName(),
+            replicaTag,
+            deploymentName);
+        result = false;
+      } else {
+        result = (boolean) isHealthy;
+      }
+    } catch (Throwable e) {
+      LOGGER.error(
+          "Replica {} of deployment {} failed to check health of {}",
+          replicaTag,
+          deploymentName,
+          callable.getClass().getName(),
+          e);
+      result = false;
+    } finally {
+      LOGGER.info(
+          "The health check result of {} in replica {} of deployment {} is {}.",
+          callable.getClass().getName(),
+          replicaTag,
+          deploymentName,
+          result);
+    }
+    return result;
+  }
+
+  public Object getCallable() {
+    return callable;
+  }
+}

java/serve/src/main/java/io/ray/serve/replica/RayServeWrappedReplica.java

@@ -0,0 +1,234 @@
+package io.ray.serve.replica;
+
+import com.google.common.base.Preconditions;
+import io.ray.api.BaseActorHandle;
+import io.ray.api.Ray;
+import io.ray.runtime.serializer.MessagePackSerializer;
+import io.ray.serve.api.Serve;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.deployment.DeploymentVersion;
+import io.ray.serve.deployment.DeploymentWrapper;
+import io.ray.serve.exception.RayServeException;
+import io.ray.serve.metrics.RayServeMetrics;
+import io.ray.serve.util.MessageFormatter;
+import io.ray.serve.util.ReflectUtil;
+import io.ray.serve.util.ServeProtoUtil;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Replica class wrapping the provided class. Note that Java function is not supported now. */
+public class RayServeWrappedReplica implements RayServeReplica {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(RayServeReplicaImpl.class);
+
+  private DeploymentWrapper deploymentInfo;
+
+  private RayServeReplicaImpl replica;
+
+  public RayServeWrappedReplica(
+      String deploymentName,
+      String replicaTag,
+      String deploymentDef,
+      byte[] initArgsbytes,
+      byte[] deploymentConfigBytes,
+      byte[] deploymentVersionBytes,
+      String appName) {
+
+    // Parse DeploymentConfig.
+    DeploymentConfig deploymentConfig = DeploymentConfig.fromProtoBytes(deploymentConfigBytes);
+
+    // Parse init args.
+    Object[] initArgs = null;
+    try {
+      initArgs = parseInitArgs(initArgsbytes);
+    } catch (IOException e) {
+      String errMsg =
+          MessageFormatter.format(
+              "Failed to initialize replica {} of deployment {}",
+              replicaTag,
+              deploymentInfo.getName());
+      LOGGER.error(errMsg, e);
+      throw new RayServeException(errMsg, e);
+    }
+
+    DeploymentVersion version = DeploymentVersion.fromProtoBytes(deploymentVersionBytes);
+
+    DeploymentWrapper deploymentWrapper = new DeploymentWrapper();
+    deploymentWrapper.setName(deploymentName);
+    deploymentWrapper.setDeploymentDef(deploymentDef);
+    deploymentWrapper.setDeploymentConfig(deploymentConfig);
+    deploymentWrapper.setInitArgs(initArgs);
+    deploymentWrapper.setDeploymentVersion(version);
+    deploymentWrapper.setAppName(appName);
+
+    // Init replica.
+    init(deploymentWrapper, replicaTag);
+  }
+
+  public RayServeWrappedReplica(DeploymentWrapper deploymentWrapper, String replicaTag) {
+    init(deploymentWrapper, replicaTag);
+  }
+
+  @SuppressWarnings("rawtypes")
+  private void init(DeploymentWrapper deploymentWrapper, String replicaTag) {
+    try {
+      // Set the controller name so that Serve.connect() in the user's code will connect to the
+      // instance that this deployment is running in.
+      Serve.setInternalReplicaContext(
+          deploymentWrapper.getName(),
+          replicaTag,
+          null,
+          deploymentWrapper.getConfig(),
+          deploymentWrapper.getAppName());
+
+      // Instantiate the object defined by deploymentDef.
+      Class deploymentClass =
+          Class.forName(
+              deploymentWrapper.getDeploymentDef(),
+              true,
+              Optional.ofNullable(Thread.currentThread().getContextClassLoader())
+                  .orElse(getClass().getClassLoader()));
+      Object callable =
+          ReflectUtil.getConstructor(deploymentClass, deploymentWrapper.getInitArgs())
+              .newInstance(deploymentWrapper.getInitArgs());
+      Serve.getReplicaContext().setServableObject(callable);
+
+      // Get the controller by name.
+      Optional<BaseActorHandle> optional =
+          Ray.getActor(Constants.SERVE_CONTROLLER_NAME, Constants.SERVE_NAMESPACE);
+      Preconditions.checkState(optional.isPresent(), "Controller does not exist");
+
+      // Enable metrics.
+      enableMetrics(deploymentWrapper.getConfig());
+
+      // Construct worker replica.
+      this.replica =
+          new RayServeReplicaImpl(
+              callable,
+              deploymentWrapper.getDeploymentConfig(),
+              deploymentWrapper.getDeploymentVersion(),
+              optional.get(),
+              deploymentWrapper.getAppName());
+      this.deploymentInfo = deploymentWrapper;
+    } catch (Throwable e) {
+      String errMsg =
+          MessageFormatter.format(
+              "Failed to initialize replica {} of deployment {}",
+              replicaTag,
+              deploymentWrapper.getName());
+      LOGGER.error(errMsg, e);
+      throw new RayServeException(errMsg, e);
+    }
+  }
+
+  private void enableMetrics(Map<String, String> config) {
+    Optional.ofNullable(config)
+        .map(conf -> conf.get(RayServeConfig.METRICS_ENABLED))
+        .ifPresent(
+            enabled -> {
+              if (Boolean.valueOf(enabled)) {
+                RayServeMetrics.enable();
+              } else {
+                RayServeMetrics.disable();
+              }
+            });
+  }
+
+  private Object[] parseInitArgs(byte[] initArgsbytes) throws IOException {
+
+    if (initArgsbytes == null || initArgsbytes.length == 0) {
+      return new Object[0];
+    }
+
+    return MessagePackSerializer.decode(initArgsbytes, Object[].class);
+  }
+
+  /**
+   * The entry method to process the request.
+   *
+   * @param requestMetadata the real type is byte[] if this invocation is cross-language. Otherwise,
+   *     the real type is {@link io.ray.serve.generated.RequestMetadata}.
+   * @param requestArgs The input parameters of the specified method of the object defined by
+   *     deploymentDef. The real type is serialized {@link io.ray.serve.generated.RequestWrapper} if
+   *     this invocation is cross-language. Otherwise, the real type is Object[].
+   * @return the result of request being processed
+   */
+  @Override
+  public Object handleRequest(Object requestMetadata, Object requestArgs) {
+    return replica.handleRequest(
+        ServeProtoUtil.parseRequestMetadata((byte[]) requestMetadata), requestArgs);
+  }
+
+  /**
+   * Check if the actor is healthy.
+   *
+   * @return true if the actor is health, or return false.
+   */
+  @Override
+  public boolean checkHealth() {
+    return replica.checkHealth();
+  }
+
+  /**
+   * Tell the caller this replica is successfully launched.
+   *
+   * @return
+   */
+  public boolean isAllocated() {
+    return true;
+  }
+
+  /**
+   * Tell the caller this replica is successfully initialized.
+   *
+   * @return
+   */
+  public Object isInitialized(byte[] deploymentConfigBytes) {
+    Object deploymentVersion = reconfigure(deploymentConfigBytes);
+    checkHealth();
+    return deploymentVersion;
+  }
+
+  /**
+   * Wait until there is no request in processing. It is used for stopping replica gracefully.
+   *
+   * @return true if it is ready for shutdown.
+   */
+  @Override
+  public boolean prepareForShutdown() {
+    return replica.prepareForShutdown();
+  }
+
+  /**
+   * Reconfigure user's configuration in the callable object through its reconfigure method.
+   *
+   * @param userConfig new user's configuration
+   * @return DeploymentVersion. If the current invocation is crossing language, the
+   *     DeploymentVersion is serialized to protobuf byte[].
+   */
+  @Override
+  public Object reconfigure(byte[] deploymentConfigBytes) {
+    DeploymentVersion deploymentVersion = replica.reconfigure(deploymentConfigBytes);
+    return deploymentVersion.toProtoBytes();
+  }
+
+  /**
+   * Get the deployment version of the current replica.
+   *
+   * @return DeploymentVersion. If the current invocation is crossing language, the
+   *     DeploymentVersion is serialized to protobuf byte[].
+   */
+  public Object getVersion() {
+    DeploymentVersion deploymentVersion = replica.getVersion();
+    return deploymentVersion.toProtoBytes();
+  }
+
+  public Object getCallable() {
+    return replica.getCallable();
+  }
+}

java/serve/src/main/java/io/ray/serve/replica/ReplicaContext.java

@@ -0,0 +1,70 @@
+package io.ray.serve.replica;
+
+import java.util.Map;
+
+/** Stores data for Serve API calls from within deployments. */
+public class ReplicaContext {
+
+  private String deploymentName;
+
+  private String replicaTag;
+
+  private Object servableObject;
+
+  private Map<String, String> config;
+
+  private String appName;
+
+  public ReplicaContext(
+      String deploymentName,
+      String replicaTag,
+      Object servableObject,
+      Map<String, String> config,
+      String appName) {
+    this.deploymentName = deploymentName;
+    this.replicaTag = replicaTag;
+    this.servableObject = servableObject;
+    this.config = config;
+    this.appName = appName;
+  }
+
+  public String getDeploymentName() {
+    return deploymentName;
+  }
+
+  public void setDeploymentName(String deploymentName) {
+    this.deploymentName = deploymentName;
+  }
+
+  public String getReplicaTag() {
+    return replicaTag;
+  }
+
+  public void setReplicaTag(String replicaTag) {
+    this.replicaTag = replicaTag;
+  }
+
+  public Object getServableObject() {
+    return servableObject;
+  }
+
+  public void setServableObject(Object servableObject) {
+    this.servableObject = servableObject;
+  }
+
+  public Map<String, String> getConfig() {
+    return config;
+  }
+
+  public void setConfig(Map<String, String> config) {
+    this.config = config;
+  }
+
+  public String getAppName() {
+    return appName;
+  }
+
+  public void setAppName(String appName) {
+    this.appName = appName;
+  }
+}

java/serve/src/main/java/io/ray/serve/replica/ReplicaName.java

@@ -0,0 +1,30 @@
+package io.ray.serve.replica;
+
+public class ReplicaName {
+
+  private String deploymentTag;
+
+  private String replicaSuffix;
+
+  private String replicaTag = "";
+
+  private String delimiter = "#";
+
+  public ReplicaName(String deploymentTag, String replicaSuffix) {
+    this.deploymentTag = deploymentTag;
+    this.replicaSuffix = replicaSuffix;
+    this.replicaTag = deploymentTag + this.delimiter + replicaSuffix;
+  }
+
+  public String getDeploymentTag() {
+    return deploymentTag;
+  }
+
+  public String getReplicaSuffix() {
+    return replicaSuffix;
+  }
+
+  public String getReplicaTag() {
+    return replicaTag;
+  }
+}

java/serve/src/main/java/io/ray/serve/router/Query.java

@@ -0,0 +1,28 @@
+package io.ray.serve.router;
+
+import io.ray.serve.generated.RequestMetadata;
+
+/** Wrap request arguments and meta data. */
+public class Query {
+
+  private RequestMetadata metadata;
+
+  /**
+   * If this query is cross-language, the args is serialized {@link
+   * io.ray.serve.generated.RequestWrapper}. Otherwise, it is Object[].
+   */
+  private Object args;
+
+  public Query(RequestMetadata requestMetadata, Object args) {
+    this.metadata = requestMetadata;
+    this.args = args;
+  }
+
+  public RequestMetadata getMetadata() {
+    return metadata;
+  }
+
+  public Object getArgs() {
+    return args;
+  }
+}

java/serve/src/main/java/io/ray/serve/router/ReplicaSet.java

@@ -0,0 +1,131 @@
+package io.ray.serve.router;
+
+import com.google.common.collect.Sets;
+import io.ray.api.ActorHandle;
+import io.ray.api.BaseActorHandle;
+import io.ray.api.ObjectRef;
+import io.ray.api.PyActorHandle;
+import io.ray.api.Ray;
+import io.ray.api.call.PyActorTaskCaller;
+import io.ray.api.function.PyActorMethod;
+import io.ray.serve.common.Constants;
+import io.ray.serve.exception.RayServeException;
+import io.ray.serve.generated.DeploymentTargetInfo;
+import io.ray.serve.replica.RayServeWrappedReplica;
+import io.ray.serve.util.CollectionUtil;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.RandomUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Data structure representing a set of replica actor handles. */
+public class ReplicaSet { // TODO ReplicaScheduler
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(ReplicaSet.class);
+
+  // The key is the name of the actor, and the value is a set of all flight queries objectrefs of
+  // the actor.
+  private final Map<String, Set<ObjectRef<Object>>> inFlightQueries;
+
+  // Map the actor name to the handle of the actor.
+  private final Map<String, BaseActorHandle> allActorHandles;
+
+  private boolean hasPullReplica = false;
+
+  public ReplicaSet(String deploymentName) {
+    this.inFlightQueries = new ConcurrentHashMap<>();
+    this.allActorHandles = new ConcurrentHashMap<>();
+  }
+
+  public synchronized void updateWorkerReplicas(Object deploymentTargetInfo) {
+    if (null != deploymentTargetInfo) {
+      Set<String> actorNameSet =
+          new HashSet<>(((DeploymentTargetInfo) deploymentTargetInfo).getReplicaNamesList());
+      Set<String> added = new HashSet<>(Sets.difference(actorNameSet, inFlightQueries.keySet()));
+      Set<String> removed = new HashSet<>(Sets.difference(inFlightQueries.keySet(), actorNameSet));
+      added.forEach(
+          name -> {
+            Optional<BaseActorHandle> handleOptional =
+                Ray.getActor(name, Constants.SERVE_NAMESPACE);
+            if (handleOptional.isPresent()) {
+              allActorHandles.put(name, handleOptional.get());
+              inFlightQueries.put(name, Sets.newConcurrentHashSet());
+            } else {
+              LOGGER.warn("Can not get actor handle. actor name is {}", name);
+            }
+          });
+      removed.forEach(inFlightQueries::remove);
+      removed.forEach(allActorHandles::remove);
+      if (added.size() > 0 || removed.size() > 0) {
+        LOGGER.info("ReplicaSet: +{}, -{} replicas.", added.size(), removed.size());
+      }
+    }
+    hasPullReplica = true;
+  }
+
+  /**
+   * Given a query, submit it to a replica and return the object ref. This method will keep track of
+   * the in flight queries for each replicas and only send a query to available replicas (determined
+   * by the max_concurrent_quries value.)
+   *
+   * @param query the incoming query.
+   * @return ray.ObjectRef
+   */
+  public ObjectRef<Object> assignReplica(Query query) {
+    ObjectRef<Object> assignedRef = tryAssignReplica(query);
+    return assignedRef;
+  }
+
+  /**
+   * Try to assign query to a replica, return the object ref if succeeded or return None if it can't
+   * assign this query to any replicas.
+   *
+   * @param query query the incoming query.
+   * @return ray.ObjectRef
+   */
+  @SuppressWarnings("unchecked")
+  private ObjectRef<Object> tryAssignReplica(Query query) {
+    int loopCount = 0;
+    while (!hasPullReplica && loopCount < 50) {
+      try {
+        TimeUnit.MICROSECONDS.sleep(20);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+      loopCount++;
+    }
+    List<BaseActorHandle> handles = new ArrayList<>(allActorHandles.values());
+    if (CollectionUtil.isEmpty(handles)) {
+      throw new RayServeException("ReplicaSet found no replica.");
+    }
+    int randomIndex = RandomUtils.nextInt(0, handles.size());
+    BaseActorHandle replica =
+        handles.get(randomIndex); // TODO controll concurrency using maxOngoingRequests
+    LOGGER.debug("Assigned query {} to replica {}.", query.getMetadata().getRequestId(), replica);
+    if (replica instanceof PyActorHandle) {
+      Object[] args =
+          Stream.concat(
+                  Stream.of(query.getMetadata().toByteArray()),
+                  Arrays.stream((Object[]) query.getArgs()))
+              .toArray();
+      PyActorTaskCaller<Object> pyCaller =
+          new PyActorTaskCaller<>(
+              (PyActorHandle) replica, PyActorMethod.of("handle_request_from_java"), args);
+      return pyCaller.remote();
+    } else {
+      return ((ActorHandle<RayServeWrappedReplica>) replica)
+          .task(
+              RayServeWrappedReplica::handleRequest,
+              query.getMetadata().toByteArray(),
+              query.getArgs())
+          .remote();
+    }
+  }
+
+  public Map<String, Set<ObjectRef<Object>>> getInFlightQueries() {
+    return inFlightQueries;
+  }
+}

java/serve/src/main/java/io/ray/serve/router/Router.java

@@ -0,0 +1,98 @@
+package io.ray.serve.router;
+
+import com.google.common.collect.ImmutableMap;
+import io.ray.api.BaseActorHandle;
+import io.ray.api.ObjectRef;
+import io.ray.runtime.metric.Count;
+import io.ray.runtime.metric.Gauge;
+import io.ray.runtime.metric.Metrics;
+import io.ray.serve.deployment.DeploymentId;
+import io.ray.serve.generated.RequestMetadata;
+import io.ray.serve.metrics.RayServeMetrics;
+import io.ray.serve.poll.KeyListener;
+import io.ray.serve.poll.KeyType;
+import io.ray.serve.poll.LongPollClient;
+import io.ray.serve.poll.LongPollNamespace;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** Router process incoming queries: assign a replica. */
+public class Router {
+
+  private ReplicaSet replicaSet;
+  private LongPollClient longPollClient;
+
+  private Count numRouterRequests;
+  private AtomicInteger numQueuedQueries = new AtomicInteger();
+  private Gauge numQueuedQueriesGauge;
+
+  public Router(BaseActorHandle controllerHandle, DeploymentId deploymentId) {
+    this.replicaSet = new ReplicaSet(deploymentId.getName());
+
+    RayServeMetrics.execute(
+        () ->
+            this.numRouterRequests =
+                Metrics.count()
+                    .name(RayServeMetrics.SERVE_NUM_ROUTER_REQUESTS.getName())
+                    .description(RayServeMetrics.SERVE_NUM_ROUTER_REQUESTS.getDescription())
+                    .unit("")
+                    .tags(
+                        ImmutableMap.of(
+                            RayServeMetrics.TAG_DEPLOYMENT,
+                            deploymentId.getName(),
+                            RayServeMetrics.TAG_APPLICATION,
+                            deploymentId.getApp()))
+                    .register());
+
+    RayServeMetrics.execute(
+        () ->
+            this.numQueuedQueriesGauge =
+                Metrics.gauge()
+                    .name(RayServeMetrics.SERVE_DEPLOYMENT_QUEUED_QUERIES.getName())
+                    .description(RayServeMetrics.SERVE_DEPLOYMENT_QUEUED_QUERIES.getDescription())
+                    .unit("")
+                    .tags(
+                        ImmutableMap.of(
+                            RayServeMetrics.TAG_DEPLOYMENT,
+                            deploymentId.getName(),
+                            RayServeMetrics.TAG_APPLICATION,
+                            deploymentId.getApp()))
+                    .register());
+
+    Map<KeyType, KeyListener> keyListeners = new HashMap<>();
+    keyListeners.put(
+        new KeyType(LongPollNamespace.DEPLOYMENT_TARGETS, deploymentId.getName()),
+        deploymentTargetInfo ->
+            replicaSet.updateWorkerReplicas(deploymentTargetInfo)); // cross language
+    this.longPollClient = new LongPollClient(controllerHandle, keyListeners);
+  }
+
+  /**
+   * Assign a query and returns an object ref represent the result.
+   *
+   * @param requestMetadata the metadata of incoming queries.
+   * @param requestArgs the request body of incoming queries.
+   * @return ray.ObjectRef
+   */
+  public ObjectRef<Object> assignRequest(RequestMetadata requestMetadata, Object[] requestArgs) {
+    RayServeMetrics.execute(() -> numRouterRequests.inc(1));
+    numQueuedQueries.incrementAndGet();
+    RayServeMetrics.execute(() -> numQueuedQueriesGauge.update(numQueuedQueries.get()));
+
+    try {
+      return replicaSet.assignReplica(new Query(requestMetadata, requestArgs));
+    } finally {
+      numQueuedQueries.decrementAndGet();
+      RayServeMetrics.execute(() -> numQueuedQueriesGauge.update(numQueuedQueries.get()));
+    }
+  }
+
+  public ReplicaSet getReplicaSet() {
+    return replicaSet;
+  }
+
+  public LongPollClient getLongPollClient() {
+    return longPollClient;
+  }
+}

java/serve/src/main/java/io/ray/serve/util/CollectionUtil.java

@@ -0,0 +1,10 @@
+package io.ray.serve.util;
+
+import java.util.Collection;
+
+public class CollectionUtil {
+
+  public static <E> boolean isEmpty(Collection<E> collection) {
+    return collection == null || collection.isEmpty();
+  }
+}

java/serve/src/main/java/io/ray/serve/util/CommonUtil.java

@@ -0,0 +1,10 @@
+package io.ray.serve.util;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class CommonUtil {
+
+  public static String getDeploymentName(String deploymentDef) {
+    return StringUtils.substringAfterLast(StringUtils.substringAfterLast(deploymentDef, "."), "$");
+  }
+}

java/serve/src/main/java/io/ray/serve/util/DAGUtil.java

@@ -0,0 +1,19 @@
+package io.ray.serve.util;
+
+import io.ray.serve.dag.ClassNode;
+import io.ray.serve.dag.DAGNode;
+import java.util.Optional;
+
+public class DAGUtil {
+
+  public static String getNodeName(DAGNode node) {
+    String nodeName = null;
+    if (node instanceof ClassNode) {
+      nodeName =
+          Optional.ofNullable(node.getBoundOptions())
+              .map(options -> (String) options.get("name"))
+              .orElse(CommonUtil.getDeploymentName(((ClassNode) node).getClassName()));
+    }
+    return nodeName;
+  }
+}

java/serve/src/main/java/io/ray/serve/util/MessageFormatter.java

@@ -0,0 +1,9 @@
+package io.ray.serve.util;
+
+/** Ray Serve common log tool. */
+public class MessageFormatter {
+
+  public static String format(String messagePattern, Object... args) {
+    return org.slf4j.helpers.MessageFormatter.arrayFormat(messagePattern, args).getMessage();
+  }
+}

java/serve/src/main/java/io/ray/serve/util/ProtobufBytesParser.java

@@ -0,0 +1,9 @@
+package io.ray.serve.util;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+@FunctionalInterface
+public interface ProtobufBytesParser<T> {
+
+  T parse(byte[] bytes) throws InvalidProtocolBufferException;
+}

java/serve/src/main/java/io/ray/serve/util/ReflectUtil.java

@@ -0,0 +1,195 @@
+package io.ray.serve.util;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Executable;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+
+/** Tool class for reflection. */
+public class ReflectUtil {
+
+  /**
+   * Get types of the parameters in input array, and make the types into a new array.
+   *
+   * @param parameters The input parameter array
+   * @return Type array corresponding to the input parameter array
+   */
+  @SuppressWarnings("rawtypes")
+  private static Class[] getParameterTypes(Object[] parameters) {
+    Class[] parameterTypes = null;
+    if (ArrayUtils.isEmpty(parameters)) {
+      return null;
+    }
+    parameterTypes = new Class[parameters.length];
+    for (int i = 0; i < parameters.length; i++) {
+      parameterTypes[i] = parameters[i].getClass();
+    }
+    return parameterTypes;
+  }
+
+  /**
+   * Get a constructor whose each parameter's type is the most closed to the corresponding input
+   * parameter in terms of Java inheritance system, while {@link Class#getConstructor(Class...)}
+   * returns the one based on class's equal.
+   *
+   * @param targetClass the constructor's class
+   * @param parameters the input parameters
+   * @return a matching constructor of the target class
+   * @throws NoSuchMethodException if a matching method is not found
+   */
+  @SuppressWarnings("rawtypes")
+  public static Constructor getConstructor(Class targetClass, Object... parameters)
+      throws NoSuchMethodException {
+    return reflect(
+        targetClass.getConstructors(), (candidate) -> true, ".<init>", targetClass, parameters);
+  }
+
+  /**
+   * Get a method whose each parameter's type is the most closed to the corresponding input
+   * parameter in terms of Java inheritance system, while {@link Class#getMethod(String, Class...)}
+   * returns the one based on class's equal.
+   *
+   * @param targetClass the constructor's class
+   * @param name the specified method's name
+   * @param parameters the input parameters
+   * @return a matching method of the target class
+   * @throws NoSuchMethodException if a matching method is not found
+   */
+  @SuppressWarnings("rawtypes")
+  public static Method getMethod(Class targetClass, String name, Object... parameters)
+      throws NoSuchMethodException {
+    return reflect(
+        targetClass.getMethods(),
+        (candidate) -> StringUtils.equals(name, candidate.getName()),
+        "." + name,
+        targetClass,
+        parameters);
+  }
+
+  /**
+   * Get an object of {@link Executable} of the specified class according to initialization
+   * parameters. The result's every parameter's type is the same as, or is a subclass or
+   * subinterface of, the type of the corresponding input parameter. This method returns the result
+   * whose each parameter's type is the most closed to the corresponding input parameter in terms of
+   * Java inheritance system.
+   *
+   * @param <T> the type of result which extends {@link Executable}
+   * @param candidates a set of candidates
+   * @param filter the filter deciding whether to select the input candidate
+   * @param message a message representing the target executable object
+   * @param targetClass the constructor's class
+   * @param parameters the input parameters
+   * @return a matching executable of the target class
+   * @throws NoSuchMethodException if a matching method is not found
+   */
+  @SuppressWarnings("rawtypes")
+  private static <T extends Executable> T reflect(
+      T[] candidates,
+      Function<T, Boolean> filter,
+      String message,
+      Class targetClass,
+      Object... parameters)
+      throws NoSuchMethodException {
+    Class[] parameterTypes = getParameterTypes(parameters);
+    T result = null;
+    for (int i = 0; i < candidates.length; i++) {
+      T candidate = candidates[i];
+      if (filter.apply(candidate)
+          && assignable(parameterTypes, candidate.getParameterTypes())
+          && (result == null
+              || assignable(candidate.getParameterTypes(), result.getParameterTypes()))) {
+        result = candidate;
+      }
+    }
+    if (result == null) {
+      throw new NoSuchMethodException(
+          targetClass.getName() + message + argumentTypesToString(parameterTypes));
+    }
+    return result;
+  }
+
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  private static boolean assignable(Class[] from, Class[] to) {
+    if (from == null) {
+      return to == null || to.length == 0;
+    }
+
+    if (to == null) {
+      return from.length == 0;
+    }
+
+    if (from.length != to.length) {
+      return false;
+    }
+
+    for (int i = 0; i < from.length; i++) {
+      if (!to[i].isAssignableFrom(from[i])) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  /**
+   * It is copied from {@link Class#argumentTypesToString(Class[])}.
+   *
+   * @param argTypes array of Class object
+   * @return Formatted string of the input Class array.
+   */
+  private static String argumentTypesToString(Class<?>[] argTypes) {
+    StringBuilder buf = new StringBuilder();
+    buf.append("(");
+    if (argTypes != null) {
+      for (int i = 0; i < argTypes.length; i++) {
+        if (i > 0) {
+          buf.append(", ");
+        }
+        Class<?> c = argTypes[i];
+        buf.append((c == null) ? "null" : c.getName());
+      }
+    }
+    buf.append(")");
+    return buf.toString();
+  }
+
+  /**
+   * Get a string representing the specified class's all methods.
+   *
+   * @param targetClass the input class
+   * @return the formatted string of the specified class's all methods.
+   */
+  @SuppressWarnings("rawtypes")
+  public static List<String> getMethodStrings(Class targetClass) {
+    if (targetClass == null) {
+      return null;
+    }
+    Method[] methods = targetClass.getMethods();
+    if (methods == null || methods.length == 0) {
+      return null;
+    }
+    List<String> methodStrings = new ArrayList<>();
+    for (int i = 0; i < methods.length; i++) {
+      methodStrings.add(methods[i].toString());
+    }
+    return methodStrings;
+  }
+
+  @SuppressWarnings("unchecked")
+  public static <T> List<T> getInstancesByClassNames(String classNames, Class<T> cls)
+      throws ClassNotFoundException, InstantiationException, IllegalAccessException,
+          IllegalArgumentException, InvocationTargetException, NoSuchMethodException,
+          SecurityException {
+    String[] classNameArray = StringUtils.split(classNames, ";");
+    List<T> isntances = new ArrayList<>();
+    for (String className : classNameArray) {
+      isntances.add((T) Class.forName(className).getConstructor().newInstance());
+    }
+    return isntances;
+  }
+}

java/serve/src/main/java/io/ray/serve/util/ServeProtoUtil.java

@@ -0,0 +1,96 @@
+package io.ray.serve.util;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import io.ray.serve.common.Constants;
+import io.ray.serve.exception.RayServeException;
+import io.ray.serve.generated.EndpointInfo;
+import io.ray.serve.generated.EndpointSet;
+import io.ray.serve.generated.RequestMetadata;
+import io.ray.serve.generated.RequestWrapper;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+
+public class ServeProtoUtil {
+
+  public static final String SERVE_PROTO_PARSE_ERROR_MSG =
+      "Failed to parse {} from protobuf bytes.";
+
+  public static RequestMetadata parseRequestMetadata(byte[] requestMetadataBytes) {
+
+    // Get a builder from RequestMetadata(bytes) or create a new one.
+    RequestMetadata.Builder builder = null;
+    if (requestMetadataBytes == null) {
+      builder = RequestMetadata.newBuilder();
+    } else {
+      RequestMetadata requestMetadata = null;
+      try {
+        requestMetadata = RequestMetadata.parseFrom(requestMetadataBytes);
+      } catch (InvalidProtocolBufferException e) {
+        throw new RayServeException("Failed to parse RequestMetadata from protobuf bytes.", e);
+      }
+      if (requestMetadata == null) {
+        builder = RequestMetadata.newBuilder();
+      } else {
+        builder = RequestMetadata.newBuilder(requestMetadata);
+      }
+    }
+
+    // Set default values.
+    if (StringUtils.isBlank(builder.getCallMethod())) {
+      builder.setCallMethod(Constants.CALL_METHOD);
+    }
+
+    return builder.build();
+  }
+
+  public static RequestWrapper parseRequestWrapper(byte[] httpRequestWrapperBytes) {
+
+    // Get a builder from HTTPRequestWrapper(bytes) or create a new one.
+    RequestWrapper.Builder builder = null;
+    if (httpRequestWrapperBytes == null) {
+      builder = RequestWrapper.newBuilder();
+    } else {
+      RequestWrapper requestWrapper = null;
+      try {
+        requestWrapper = RequestWrapper.parseFrom(httpRequestWrapperBytes);
+      } catch (InvalidProtocolBufferException e) {
+        throw new RayServeException("Failed to parse RequestWrapper from protobuf bytes.", e);
+      }
+      if (requestWrapper == null) {
+        builder = RequestWrapper.newBuilder();
+      } else {
+        builder = RequestWrapper.newBuilder(requestWrapper);
+      }
+    }
+
+    return builder.build();
+  }
+
+  public static Map<String, EndpointInfo> parseEndpointSet(byte[] endpointSetBytes) {
+    if (endpointSetBytes == null) {
+      return null;
+    }
+    EndpointSet endpointSet = null;
+    try {
+      endpointSet = EndpointSet.parseFrom(endpointSetBytes);
+    } catch (InvalidProtocolBufferException e) {
+      throw new RayServeException("Failed to parse EndpointSet from protobuf bytes.", e);
+    }
+    if (endpointSet == null) {
+      return null;
+    }
+    return endpointSet.getEndpointsMap();
+  }
+
+  public static <T> T bytesToProto(byte[] bytes, ProtobufBytesParser<T> protobufBytesParser) {
+    if (bytes == null) {
+      return null;
+    }
+    try {
+      T proto = protobufBytesParser.parse(bytes);
+      return proto;
+    } catch (InvalidProtocolBufferException e) {
+      throw new RayServeException("Failed to parse protobuf bytes.", e);
+    }
+  }
+}

java/serve/src/main/java/io/ray/serve/util/SocketUtil.java

@@ -0,0 +1,49 @@
+package io.ray.serve.util;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+
+public class SocketUtil {
+
+  public static final int PORT_RANGE_MAX = 65535;
+
+  public static int findAvailableTcpPort(int minPort) {
+    int portRange = PORT_RANGE_MAX - minPort;
+    int candidatePort = minPort;
+    int searchCounter = 0;
+    while (!isPortAvailable(candidatePort)) {
+      candidatePort++;
+      if (++searchCounter > portRange) {
+        throw new IllegalStateException(
+            String.format(
+                "Could not find an available tcp port in the range [%d, %d] after %d attempts.",
+                minPort, PORT_RANGE_MAX, searchCounter));
+      }
+    }
+    return candidatePort;
+  }
+
+  public static boolean isPortAvailable(int port) {
+    ServerSocket socket;
+    try {
+      socket = new ServerSocket();
+    } catch (IOException e) {
+      throw new IllegalStateException("Unable to create ServerSocket.", e);
+    }
+
+    try {
+      InetSocketAddress sa = new InetSocketAddress(port);
+      socket.bind(sa);
+      return true;
+    } catch (IOException ex) {
+      return false;
+    } finally {
+      try {
+        socket.close();
+      } catch (IOException ex) {
+        // ignore this exception for now
+      }
+    }
+  }
+}

java/serve/src/main/resources/test_python_deployment.py

@@ -0,0 +1,20 @@
+# This file is used by CrossLanguageDeploymentTest.java to test cross-language
+# invocation.
+from ray import serve
+
+
+def echo_server(v):
+    return v
+
+
+@serve.deployment
+class Counter(object):
+    def __init__(self, value):
+        self.value = int(value)
+
+    def increase(self, delta):
+        self.value += int(delta)
+        return str(self.value)
+
+    def reconfigure(self, value_str):
+        self.value = int(value_str)

java/serve/src/test/java/io/ray/serve/BaseServeTest.java

@@ -0,0 +1,82 @@
+package io.ray.serve;
+
+import io.ray.api.Ray;
+import io.ray.serve.api.Serve;
+import io.ray.serve.api.ServeControllerClient;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.poll.LongPollClientFactory;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.collections.Maps;
+
+public abstract class BaseServeTest {
+  private static final Logger LOGGER = LoggerFactory.getLogger(BaseServeTest.class);
+  protected static ServeControllerClient client = null;
+
+  @BeforeMethod(alwaysRun = true)
+  public static void startServe() {
+    Map<String, String> config = Maps.newHashMap();
+    // The default port 8000 is occupied by other processes on the ci platform.
+    config.put(RayServeConfig.PROXY_HTTP_PORT, "8341"); // TODO(liuyang-my) Get an available port.
+    client = Serve.start(config);
+  }
+
+  @AfterMethod(alwaysRun = true)
+  public static void shutdownServe() {
+    try {
+      Serve.shutdown();
+    } catch (Exception e) {
+      LOGGER.error("serve shutdown error", e);
+    }
+    try {
+      Ray.shutdown();
+      LOGGER.info("Base serve test shutdown ray. Is initialized:{}", Ray.isInitialized());
+    } catch (Exception e) {
+      LOGGER.error("ray shutdown error", e);
+    }
+  }
+
+  private static boolean previousInited;
+
+  private static String previousNamespace;
+
+  public static void initRay() {
+    previousInited = Ray.isInitialized();
+    LOGGER.info("Base serve test. Previous inited:{}", previousInited);
+    previousNamespace = System.getProperty("ray.job.namespace");
+    LOGGER.info("Base serve test. Previous namespace:{}", previousNamespace);
+
+    System.setProperty("ray.job.namespace", Constants.SERVE_NAMESPACE);
+    Ray.init();
+  }
+
+  public static void shutdownRay() {
+    if (!previousInited) {
+      Ray.shutdown();
+      LOGGER.info("Base serve test shutdown ray. Is initialized:{}", Ray.isInitialized());
+    }
+    if (previousNamespace == null) {
+      System.clearProperty("ray.job.namespace");
+    } else {
+      System.setProperty("ray.job.namespace", previousNamespace);
+    }
+  }
+
+  public static void clearContext() {
+    Serve.clearContext();
+  }
+
+  public static void stopLongPoll() {
+    LongPollClientFactory.stop();
+  }
+
+  public static void clearAndShutdownRay() {
+    stopLongPoll();
+    shutdownRay();
+    clearContext();
+  }
+}

java/serve/src/test/java/io/ray/serve/BaseServeTest2.java

@@ -0,0 +1,43 @@
+package io.ray.serve;
+
+import io.ray.api.Ray;
+import io.ray.serve.api.Serve;
+import io.ray.serve.config.RayServeConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+
+public abstract class BaseServeTest2 {
+  private static final Logger LOGGER = LoggerFactory.getLogger(BaseServeTest2.class);
+
+  private String previousHttpPort;
+
+  @BeforeMethod(alwaysRun = true)
+  public void initConfig() {
+    // The default port 8000 is occupied by other processes on the ci platform.
+    previousHttpPort =
+        System.setProperty(
+            RayServeConfig.PROXY_HTTP_PORT, "8341"); // TODO(liuyang-my) Get an available port.
+  }
+
+  @AfterMethod(alwaysRun = true)
+  public void shutdownServe() {
+    try {
+      Serve.shutdown();
+    } catch (Exception e) {
+      LOGGER.error("serve shutdown error", e);
+    }
+    try {
+      Ray.shutdown();
+      LOGGER.info("Base serve test shutdown ray. Is initialized:{}", Ray.isInitialized());
+    } catch (Exception e) {
+      LOGGER.error("ray shutdown error", e);
+    }
+    if (previousHttpPort == null) {
+      System.clearProperty(RayServeConfig.PROXY_HTTP_PORT);
+    } else {
+      System.setProperty(RayServeConfig.PROXY_HTTP_PORT, previousHttpPort);
+    }
+  }
+}

java/serve/src/test/java/io/ray/serve/DummyServeController.java

@@ -0,0 +1,51 @@
+package io.ray.serve;
+
+import io.ray.serve.controller.ServeController;
+import io.ray.serve.generated.EndpointInfo;
+import io.ray.serve.generated.EndpointSet;
+import io.ray.serve.poll.LongPollRequest;
+import io.ray.serve.util.ServeProtoUtil;
+import java.util.Map;
+
+public class DummyServeController implements ServeController {
+  private Map<String, EndpointInfo> endpoints;
+
+  private byte[] longPollResult;
+
+  private String rootUrl;
+
+  public DummyServeController(String rootUrl) {
+    this.rootUrl = rootUrl;
+  }
+
+  @Override
+  public byte[] getAllEndpoints() {
+    EndpointSet.Builder builder = EndpointSet.newBuilder();
+    builder.putAllEndpoints(endpoints);
+    return builder.build().toByteArray();
+  }
+
+  public boolean setEndpoints(byte[] endpoints) {
+    this.endpoints = ServeProtoUtil.parseEndpointSet(endpoints);
+    return true;
+  }
+
+  public boolean setLongPollResult(byte[] longPollResult) {
+    this.longPollResult = longPollResult;
+    return true;
+  }
+
+  @Override
+  public byte[] listenForChange(LongPollRequest longPollRequest) {
+    return longPollResult;
+  }
+
+  @Override
+  public String getRootUrl() {
+    return rootUrl;
+  }
+
+  public void setRootUrl(String rootUrl) {
+    this.rootUrl = rootUrl;
+  }
+}

java/serve/src/test/java/io/ray/serve/api/ServeTest.java

@@ -0,0 +1,96 @@
+package io.ray.serve.api;
+
+import io.ray.api.PyActorHandle;
+import io.ray.api.Ray;
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.DummyServeController;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.exception.RayServeException;
+import io.ray.serve.replica.ReplicaContext;
+import io.ray.serve.replica.ReplicaName;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+import org.testng.collections.Maps;
+
+public class ServeTest {
+
+  @Test
+  public void getReplicaContextNormalTest() {
+    try {
+      String dummyName = "getReplicaContextNormalTest";
+      ReplicaName replicaName = new ReplicaName(dummyName, RandomStringUtils.randomAlphabetic(6));
+      Object servableObject = new Object();
+      Serve.setInternalReplicaContext(
+          replicaName.getDeploymentTag(), replicaName.getReplicaTag(), servableObject, null, null);
+
+      ReplicaContext replicaContext = Serve.getReplicaContext();
+      Assert.assertNotNull(replicaContext, "no replica context");
+      Assert.assertEquals(replicaContext.getDeploymentName(), replicaName.getDeploymentTag());
+      Assert.assertEquals(replicaContext.getReplicaTag(), replicaName.getReplicaTag());
+    } finally {
+      BaseServeTest.clearContext();
+    }
+  }
+
+  @Test
+  public void getReplicaContextNotExistTest() {
+    Assert.assertThrows(RayServeException.class, () -> Serve.getReplicaContext());
+  }
+
+  @Test(groups = {"cluster"})
+  public void startTest() {
+    try {
+      // The default port 8000 is occupied by other processes on the ci platform.
+      Map<String, String> config = Maps.newHashMap();
+      config.put(RayServeConfig.PROXY_HTTP_PORT, "8341");
+      Serve.start(config);
+
+      Optional<PyActorHandle> controller = Ray.getActor(Constants.SERVE_CONTROLLER_NAME);
+      Assert.assertTrue(controller.isPresent());
+
+      Serve.shutdown();
+      controller = Ray.getActor(Constants.SERVE_CONTROLLER_NAME);
+      Assert.assertFalse(controller.isPresent());
+    } finally {
+      BaseServeTest.shutdownServe();
+    }
+  }
+
+  @Test(groups = {"cluster"})
+  public void getGlobalClientNormalTest() {
+    try {
+      BaseServeTest.startServe();
+
+      ServeControllerClient client = Serve.getGlobalClient(true);
+      Assert.assertNotNull(client);
+    } finally {
+      BaseServeTest.shutdownServe();
+    }
+  }
+
+  @Test
+  public void getGlobalClientExceptionTest() {
+    Assert.assertThrows(IllegalStateException.class, () -> Serve.getGlobalClient());
+  }
+
+  @Test
+  public void getGlobalClientInReplicaTest() {
+    try {
+      BaseServeTest.initRay();
+
+      // Mock controller.
+      Ray.actor(DummyServeController::new, "").setName(Constants.SERVE_CONTROLLER_NAME).remote();
+
+      // Get client.
+      ServeControllerClient client = Serve.getGlobalClient();
+      Assert.assertNotNull(client);
+    } finally {
+      BaseServeTest.shutdownRay();
+      BaseServeTest.clearContext();
+    }
+  }
+}

java/serve/src/test/java/io/ray/serve/context/RequestContextTest.java

@@ -0,0 +1,36 @@
+package io.ray.serve.context;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class RequestContextTest {
+
+  @Test
+  public void test() {
+    RequestContext context = RequestContext.get();
+    Assert.assertEquals(context.getRoute(), "");
+    Assert.assertEquals(context.getRequestId(), "");
+    Assert.assertEquals(context.getAppName(), "");
+    Assert.assertEquals(context.getMultiplexedModelId(), "");
+
+    String route = "route";
+    String requestId = "requestId";
+    String appName = "appName";
+    String multiplexedModelId = "multiplexedModelId";
+
+    RequestContext.set(route, requestId, appName, multiplexedModelId);
+    context = RequestContext.get();
+    Assert.assertEquals(context.getRoute(), route);
+    Assert.assertEquals(context.getRequestId(), requestId);
+    Assert.assertEquals(context.getAppName(), appName);
+    Assert.assertEquals(context.getMultiplexedModelId(), multiplexedModelId);
+
+    RequestContext.clean();
+    context = RequestContext.get();
+    Assert.assertEquals(context.getRoute(), "");
+    Assert.assertEquals(context.getRequestId(), "");
+    Assert.assertEquals(context.getAppName(), "");
+    Assert.assertEquals(context.getMultiplexedModelId(), "");
+    RequestContext.clean();
+  }
+}

java/serve/src/test/java/io/ray/serve/deployment/CrossLanguageDeploymentTest.java

@@ -0,0 +1,122 @@
+package io.ray.serve.deployment;
+
+import io.ray.api.ObjectRef;
+import io.ray.api.Ray;
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.api.Serve;
+import io.ray.serve.generated.DeploymentLanguage;
+import io.ray.serve.handle.DeploymentHandle;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.commons.io.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Test(groups = {"cluster"})
+public class CrossLanguageDeploymentTest extends BaseServeTest {
+  private static final String PYTHON_MODULE = "test_python_deployment";
+
+  @BeforeClass
+  public void beforeClass() {
+    // Delete and re-create the temp dir.
+    File tempDir =
+        new File(
+            System.getProperty("java.io.tmpdir")
+                + File.separator
+                + "ray_serve_cross_language_test");
+    FileUtils.deleteQuietly(tempDir);
+    tempDir.mkdirs();
+    tempDir.deleteOnExit();
+
+    // Write the test Python file to the temp dir.
+    InputStream in =
+        CrossLanguageDeploymentTest.class.getResourceAsStream(
+            File.separator + PYTHON_MODULE + ".py");
+    File pythonFile = new File(tempDir.getAbsolutePath() + File.separator + PYTHON_MODULE + ".py");
+    try {
+      FileUtils.copyInputStreamToFile(in, pythonFile);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+
+    System.setProperty(
+        "ray.job.code-search-path",
+        System.getProperty("java.class.path") + File.pathSeparator + tempDir.getAbsolutePath());
+  }
+
+  @Test
+  public void createPyClassTest() {
+    Application deployment =
+        Serve.deployment()
+            .setLanguage(DeploymentLanguage.PYTHON)
+            .setName("createPyClassTest")
+            .setDeploymentDef(PYTHON_MODULE + ".Counter")
+            .setNumReplicas(1)
+            .bind("28");
+
+    DeploymentHandle handle = Serve.run(deployment);
+    Assert.assertEquals(handle.method("increase").remote("6").result(), "34");
+  }
+
+  @Test
+  public void createPyClassWithObjectRefTest() {
+    Application deployment =
+        Serve.deployment()
+            .setLanguage(DeploymentLanguage.PYTHON)
+            .setName("createPyClassWithObjectRefTest")
+            .setDeploymentDef(PYTHON_MODULE + ".Counter")
+            .setNumReplicas(1)
+            .bind("28");
+
+    DeploymentHandle handle = Serve.run(deployment);
+    ObjectRef<Integer> numRef = Ray.put(10);
+    Assert.assertEquals(handle.method("increase").remote(numRef).result(), "38");
+  }
+
+  @Test
+  public void createPyMethodTest() {
+    Application deployment =
+        Serve.deployment()
+            .setLanguage(DeploymentLanguage.PYTHON)
+            .setName("createPyMethodTest")
+            .setDeploymentDef(PYTHON_MODULE + ".echo_server")
+            .setNumReplicas(1)
+            .bind();
+    DeploymentHandle handle = Serve.run(deployment);
+    Assert.assertEquals(handle.method("__call__").remote("6").result(), "6");
+  }
+
+  @Test
+  public void createPyMethodWithObjectRefTest() {
+    Application deployment =
+        Serve.deployment()
+            .setLanguage(DeploymentLanguage.PYTHON)
+            .setName("createPyMethodWithObjectRefTest")
+            .setDeploymentDef(PYTHON_MODULE + ".echo_server")
+            .setNumReplicas(1)
+            .bind();
+    DeploymentHandle handle = Serve.run(deployment);
+    ObjectRef<String> numRef = Ray.put("10");
+    Assert.assertEquals(handle.method("__call__").remote(numRef).result(), "10");
+  }
+
+  @Test
+  public void userConfigTest() throws InterruptedException {
+    Application deployment =
+        Serve.deployment()
+            .setLanguage(DeploymentLanguage.PYTHON)
+            .setName("userConfigTest")
+            .setDeploymentDef(PYTHON_MODULE + ".Counter")
+            .setNumReplicas(1)
+            .setUserConfig("1")
+            .bind("28");
+    DeploymentHandle handle = Serve.run(deployment);
+    Assert.assertEquals(handle.method("increase").remote("6").result(), "7");
+    //    deployment.options().setUserConfig("3").create().deploy(true);
+    //    TimeUnit.SECONDS.sleep(20L);
+    //    Assert.assertEquals(handle.method("increase").remote("6").result(), "9");
+    // TOOD update user config
+  }
+}

java/serve/src/test/java/io/ray/serve/deployment/DeploymentGraphTest.java

@@ -0,0 +1,131 @@
+package io.ray.serve.deployment;
+
+import io.ray.api.Ray;
+import io.ray.serve.BaseServeTest2;
+import io.ray.serve.api.Serve;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.generated.ApplicationStatus;
+import io.ray.serve.generated.StatusOverview;
+import io.ray.serve.handle.DeploymentHandle;
+import io.ray.serve.handle.DeploymentResponse;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class DeploymentGraphTest extends BaseServeTest2 {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(DeploymentGraphTest.class);
+
+  private String previousHttpPort;
+
+  @BeforeMethod(alwaysRun = true)
+  public void before() {
+    // The default port 8000 is occupied by other processes on the ci platform.
+    previousHttpPort = System.getProperty(RayServeConfig.PROXY_HTTP_PORT);
+    System.setProperty(
+        RayServeConfig.PROXY_HTTP_PORT, "8341"); // TODO(liuyang-my) Get an available port.
+  }
+
+  @AfterMethod(alwaysRun = true)
+  public void after() {
+    try {
+      Serve.shutdown();
+    } catch (Exception e) {
+      LOGGER.error("serve shutdown error", e);
+    }
+    try {
+      Ray.shutdown();
+      LOGGER.info("Base serve test shutdown ray. Is initialized:{}", Ray.isInitialized());
+    } catch (Exception e) {
+      LOGGER.error("ray shutdown error", e);
+    }
+    if (previousHttpPort == null) {
+      System.clearProperty(RayServeConfig.PROXY_HTTP_PORT);
+    } else {
+      System.setProperty(RayServeConfig.PROXY_HTTP_PORT, previousHttpPort);
+    }
+  }
+
+  public static class Counter {
+    private AtomicInteger count;
+
+    public Counter(String count) {
+      this.count = new AtomicInteger(Integer.parseInt(count));
+    }
+
+    public String call(String input) {
+      return String.valueOf(count.addAndGet(Integer.parseInt(input)));
+    }
+  }
+
+  @Test(groups = {"cluster"})
+  public void bindTest() {
+    Application deployment =
+        Serve.deployment().setDeploymentDef(Counter.class.getName()).setNumReplicas(1).bind("2");
+
+    DeploymentHandle handle = Serve.run(deployment);
+    Assert.assertEquals(handle.remote("2").result(), "4");
+  }
+
+  public static class ModelA {
+    public String call(String msg) {
+      return "A:" + msg;
+    }
+  }
+
+  public static class ModelB {
+    public String call(String msg) {
+      return "B:" + msg;
+    }
+  }
+
+  public static class Combiner {
+    private DeploymentHandle modelAHandle;
+    private DeploymentHandle modelBHandle;
+
+    public Combiner(DeploymentHandle modelAHandle, DeploymentHandle modelBHandle) {
+      LOGGER.info("Combiner.");
+      this.modelAHandle = modelAHandle;
+      this.modelBHandle = modelBHandle;
+    }
+
+    public String call(String request) {
+      DeploymentResponse responseA = modelAHandle.remote(request);
+      DeploymentResponse responseB = modelBHandle.remote(request);
+      return responseA.result() + "," + responseB.result();
+    }
+  }
+
+  @Test(groups = {"cluster"})
+  public void testPassHandle() {
+    Application modelA = Serve.deployment().setDeploymentDef(ModelA.class.getName()).bind();
+    Application modelB = Serve.deployment().setDeploymentDef(ModelB.class.getName()).bind();
+
+    Application driver =
+        Serve.deployment().setDeploymentDef(Combiner.class.getName()).bind(modelA, modelB);
+    DeploymentHandle handle = Serve.run(driver);
+    Assert.assertEquals(handle.remote("test").result(), "A:test,B:test");
+  }
+
+  @Test(groups = {"cluster"})
+  public void statusTest() {
+    Application deployment =
+        Serve.deployment().setDeploymentDef(Counter.class.getName()).setNumReplicas(1).bind("2");
+    Serve.run(deployment);
+
+    StatusOverview status =
+        Serve.getGlobalClient().getServeStatus(Constants.SERVE_DEFAULT_APP_NAME);
+    Assert.assertEquals(
+        status.getAppStatus().getStatus(), ApplicationStatus.APPLICATION_STATUS_RUNNING);
+
+    Serve.delete(Constants.SERVE_DEFAULT_APP_NAME);
+    status = Serve.getGlobalClient().getServeStatus(Constants.SERVE_DEFAULT_APP_NAME);
+    Assert.assertEquals(
+        status.getAppStatus().getStatus(), ApplicationStatus.APPLICATION_STATUS_NOT_STARTED);
+  }
+}

java/serve/src/test/java/io/ray/serve/deployment/DeploymentTest.java

@@ -0,0 +1,275 @@
+package io.ray.serve.deployment;
+
+import io.ray.api.Ray;
+import io.ray.serve.BaseServeTest2;
+import io.ray.serve.api.Serve;
+import io.ray.serve.config.AutoscalingConfig;
+import io.ray.serve.handle.DeploymentHandle;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.apache.hc.client5.http.classic.HttpClient;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
+import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.apache.hc.core5.http.io.entity.StringEntity;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+@Test(groups = {"cluster"})
+public class DeploymentTest extends BaseServeTest2 {
+
+  @Test
+  public void deployTest() {
+    // Deploy deployment.
+    String deploymentName = "exampleEcho";
+
+    Application deployment =
+        Serve.deployment()
+            .setName(deploymentName)
+            .setDeploymentDef(ExampleEchoDeployment.class.getName())
+            .setNumReplicas(1)
+            .setUserConfig("_test")
+            .bind("echo_");
+    DeploymentHandle handle = Serve.run(deployment);
+    Assert.assertEquals(handle.method("call").remote("6").result(), "echo_6_test");
+    Assert.assertTrue((boolean) handle.method("checkHealth").remote().result());
+  }
+
+  @Test
+  public void httpExposeDeploymentTest() throws IOException {
+    // Deploy deployment.
+    String deploymentName = "exampleEcho";
+
+    Application deployment =
+        Serve.deployment()
+            .setName(deploymentName)
+            .setDeploymentDef(ExampleEchoDeployment.class.getName())
+            .setNumReplicas(1)
+            .setUserConfig("_test")
+            .bind("echo_");
+    Serve.run(deployment);
+
+    HttpClient httpClient = HttpClientBuilder.create().build();
+    HttpGet httpGet = new HttpGet("http://127.0.0.1:8341/" + deploymentName + "?input=testhttpget");
+    try (CloseableHttpResponse httpResponse = (CloseableHttpResponse) httpClient.execute(httpGet)) {
+      byte[] body = EntityUtils.toByteArray(httpResponse.getEntity());
+      String response = new String(body, StandardCharsets.UTF_8);
+      Assert.assertEquals(response, "echo_testhttpget_test");
+    }
+    HttpPost httpPost = new HttpPost("http://127.0.0.1:8341/" + deploymentName);
+    httpPost.setEntity(new StringEntity("testhttppost"));
+    try (CloseableHttpResponse httpResponse =
+        (CloseableHttpResponse) httpClient.execute(httpPost)) {
+      byte[] body = EntityUtils.toByteArray(httpResponse.getEntity());
+      String response = new String(body, StandardCharsets.UTF_8);
+      Assert.assertEquals(response, "echo_testhttppost_test");
+    }
+  }
+
+  @Test(enabled = false)
+  public void updateDeploymentTest() {
+    String deploymentName = "exampleEcho";
+
+    Application deployment =
+        Serve.deployment()
+            .setName(deploymentName)
+            .setDeploymentDef(ExampleEchoDeployment.class.getName())
+            .setNumReplicas(1)
+            .setUserConfig("_test")
+            .bind("echo_");
+    Serve.run(deployment);
+
+    Deployment deployed = Serve.getDeployment(deploymentName);
+    Serve.run(deployed.options().setNumReplicas(2).bind("echo_"));
+  }
+
+  @Test
+  public void autoScaleTest() {
+    String deploymentName = "exampleEcho";
+    AutoscalingConfig autoscalingConfig = new AutoscalingConfig();
+    autoscalingConfig.setMinReplicas(1);
+    autoscalingConfig.setMaxReplicas(3);
+    autoscalingConfig.setTargetOngoingRequests(10);
+    Application deployment =
+        Serve.deployment()
+            .setName(deploymentName)
+            .setDeploymentDef(ExampleEchoDeployment.class.getName())
+            .setAutoscalingConfig(autoscalingConfig)
+            .setUserConfig("_test")
+            .setVersion("v1")
+            .bind("echo_");
+
+    DeploymentHandle handle = Serve.run(deployment);
+    Assert.assertEquals(handle.method("call").remote("6").result(), "echo_6_test");
+  }
+
+  @Test(enabled = false)
+  public void userConfigTest() {
+    String deploymentName = "exampleEcho";
+    Application deployment =
+        Serve.deployment()
+            .setName(deploymentName)
+            .setDeploymentDef(ExampleEchoDeployment.class.getName())
+            .setNumReplicas(1)
+            .setUserConfig("_test")
+            .bind("echo_");
+    Serve.run(deployment);
+
+    Serve.run(Serve.getDeployment(deploymentName).options().setUserConfig("_new").bind());
+    Assert.assertEquals(
+        Serve.getAppHandle(deploymentName).method("call").remote("6").result(), "echo_6_new");
+    // TOOD update user config
+  }
+
+  @Test
+  public void externalScalerEnabledTest() {
+    /*
+     * This test verifies that the external_scaler_enabled flag is properly passed through
+     * the Java Serve API to the Python controller.
+     *
+     * WHY WE DON'T TEST VIA HTTP DASHBOARD API:
+     * The external scaler HTTP REST API endpoint (/api/v1/applications/{app}/deployments/{dep}/scale)
+     * is hosted by the Ray dashboard on port 8265. However, in the Java test framework:
+     *
+     * 1. Each test creates an ephemeral Ray session with a fresh actor registry
+     * 2. The Serve controller is created programmatically via Java API (Serve.run)
+     * 3. The dashboard runs in a separate process and discovers controllers by querying
+     *    Ray's actor registry for "SERVE_CONTROLLER_ACTOR" in the "serve" namespace
+     * 4. Due to timing and process isolation issues in test environments, the dashboard's
+     *    get_serve_controller() method often fails to find the controller, returning 503
+     *    with error "Serve controller is not available"
+     *
+     * For testing purposes, we verify the flag is correctly passed to the controller by:
+     * 1. Deploying an application with external_scaler_enabled=true
+     * 2. Verifying the deployment succeeds and is functional
+     * 3. Checking that the controller actor exists with the correct configuration
+     *
+     * The actual HTTP scaling functionality is tested in Python integration tests where
+     * the dashboard and controller have proper lifecycle management.
+     */
+    String appName = "externalScalerApp";
+    String deploymentName = "exampleEcho";
+    Application deployment =
+        Serve.deployment()
+            .setName(deploymentName)
+            .setDeploymentDef(ExampleEchoDeployment.class.getName())
+            .setNumReplicas(1)
+            .setUserConfig("_test")
+            .bind("echo_");
+
+    // Deploy with external_scaler_enabled=true - this passes the flag through
+    DeploymentHandle handle = Serve.run(deployment, true, appName, "/", null, true);
+
+    // Verify the deployment is functional
+    Assert.assertEquals(handle.method("call").remote("5").result(), "echo_5_test");
+    Assert.assertTrue((boolean) handle.method("checkHealth").remote().result());
+
+    // Verify the controller actor exists in the correct namespace
+    java.util.Optional<io.ray.api.BaseActorHandle> controllerOpt =
+        Ray.getActor("SERVE_CONTROLLER_ACTOR", "serve");
+    Assert.assertTrue(
+        controllerOpt.isPresent(), "Serve controller actor should exist in 'serve' namespace");
+
+    // Verify that the external_scaler_enabled flag is actually set to TRUE in the controller
+    // by calling the controller's get_external_scaler_enabled method
+    io.ray.api.PyActorHandle controller = (io.ray.api.PyActorHandle) controllerOpt.get();
+    try {
+      // Call the Python controller's get_external_scaler_enabled method
+      // This is a helper method added specifically for Java tests that returns a simple boolean
+      Object result =
+          controller
+              .task(io.ray.api.function.PyActorMethod.of("get_external_scaler_enabled"), appName)
+              .remote()
+              .get();
+
+      // Verify the flag is set to True
+      Assert.assertTrue(
+          Boolean.TRUE.equals(result),
+          "external_scaler_enabled should be True for app '" + appName + "', but was: " + result);
+
+      // Also verify application is running
+      io.ray.serve.api.ServeControllerClient client = io.ray.serve.api.Serve.getGlobalClient();
+      io.ray.serve.generated.StatusOverview status = client.getServeStatus(appName);
+      Assert.assertEquals(
+          status.getAppStatus().getStatus(),
+          io.ray.serve.generated.ApplicationStatus.APPLICATION_STATUS_RUNNING,
+          "Application should be in RUNNING status");
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to verify external_scaler_enabled flag", e);
+    }
+  }
+
+  @Test
+  public void externalScalerDisabledTest() {
+    /*
+     * This test verifies that the external_scaler_enabled flag defaults to false and
+     * applications can be deployed with external scaling explicitly disabled.
+     *
+     * This is the complement to externalScalerEnabledTest - verifying that the flag
+     * can be set to false (which is also the default behavior).
+     *
+     * In production, when external_scaler_enabled=false, attempts to scale via the
+     * HTTP dashboard API would return 412 (Precondition Failed) with an
+     * ExternalScalerDisabledError. However, as explained in externalScalerEnabledTest,
+     * we cannot reliably test the HTTP API in this test environment due to dashboard
+     * and controller lifecycle management issues.
+     *
+     * For testing purposes, we verify:
+     * 1. Applications can be deployed with external_scaler_enabled=false
+     * 2. The deployment succeeds and is functional
+     * 3. Checking that the controller actor exists with the correct configuration
+     */
+    String appName = "normalApp";
+    String deploymentName = "exampleEcho";
+    Application deployment =
+        Serve.deployment()
+            .setName(deploymentName)
+            .setDeploymentDef(ExampleEchoDeployment.class.getName())
+            .setNumReplicas(1)
+            .setUserConfig("_test")
+            .bind("echo_");
+
+    // Deploy with external_scaler_enabled=false (explicit)
+    DeploymentHandle handle = Serve.run(deployment, true, appName, "/", null, false);
+
+    // Verify the deployment is functional
+    Assert.assertEquals(handle.method("call").remote("7").result(), "echo_7_test");
+    Assert.assertTrue((boolean) handle.method("checkHealth").remote().result());
+
+    // Verify the controller actor exists in the correct namespace
+    java.util.Optional<io.ray.api.BaseActorHandle> controllerOpt =
+        Ray.getActor("SERVE_CONTROLLER_ACTOR", "serve");
+    Assert.assertTrue(
+        controllerOpt.isPresent(), "Serve controller actor should exist in 'serve' namespace");
+
+    // Verify that the external_scaler_enabled flag is actually set to FALSE in the controller
+    // by calling the controller's get_external_scaler_enabled method
+    io.ray.api.PyActorHandle controller = (io.ray.api.PyActorHandle) controllerOpt.get();
+    try {
+      // Call the Python controller's get_external_scaler_enabled method
+      // This is a helper method added specifically for Java tests that returns a simple boolean
+      Object result =
+          controller
+              .task(io.ray.api.function.PyActorMethod.of("get_external_scaler_enabled"), appName)
+              .remote()
+              .get();
+
+      // Verify the flag is set to False
+      Assert.assertFalse(
+          Boolean.TRUE.equals(result),
+          "external_scaler_enabled should be False for app '" + appName + "', but was: " + result);
+
+      // Also verify application is running
+      io.ray.serve.api.ServeControllerClient client = io.ray.serve.api.Serve.getGlobalClient();
+      io.ray.serve.generated.StatusOverview status = client.getServeStatus(appName);
+      Assert.assertEquals(
+          status.getAppStatus().getStatus(),
+          io.ray.serve.generated.ApplicationStatus.APPLICATION_STATUS_RUNNING,
+          "Application should be in RUNNING status");
+    } catch (Exception e) {
+      throw new RuntimeException("Failed to verify external_scaler_enabled flag", e);
+    }
+  }
+}

java/serve/src/test/java/io/ray/serve/deployment/ExampleEchoDeployment.java

@@ -0,0 +1,32 @@
+package io.ray.serve.deployment;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ExampleEchoDeployment {
+  String prefix;
+  String suffix = "";
+  private static final Logger LOGGER = LoggerFactory.getLogger(ExampleEchoDeployment.class);
+
+  public ExampleEchoDeployment(Object prefix) {
+    LOGGER.info("receive init args: {}", prefix);
+    this.prefix = (String) prefix;
+  }
+
+  public String call(Object input) {
+    LOGGER.info("receive call request: {}", input);
+    return this.prefix + input + this.suffix;
+  }
+
+  public boolean checkHealth() {
+    return true;
+  }
+
+  public Object reconfigure(Object userConfig) {
+    LOGGER.info("receive userconfig: {}", userConfig);
+    if (null != userConfig) {
+      this.suffix = userConfig.toString();
+    }
+    return null;
+  }
+}

java/serve/src/test/java/io/ray/serve/docdemo/HttpStrategyCalcOnRayServe.java

@@ -0,0 +1,120 @@
+package io.ray.serve.docdemo;
+
+import com.google.gson.Gson;
+import io.ray.serve.api.Serve;
+import io.ray.serve.deployment.Application;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import org.apache.hc.client5.http.fluent.Request;
+
+public class HttpStrategyCalcOnRayServe {
+
+  public void deploy() {
+    Serve.start(null);
+
+    Application deployment =
+        Serve.deployment()
+            .setName("http-strategy")
+            .setDeploymentDef(HttpStrategyOnRayServe.class.getName())
+            .setNumReplicas(4)
+            .bind();
+    Serve.run(deployment);
+  }
+
+  // docs-http-start
+  private Gson gson = new Gson();
+
+  public String httpCalc(Long time, String bank, String indicator) {
+    Map<String, Object> data = new HashMap<>();
+    data.put("time", time);
+    data.put("bank", bank);
+    data.put("indicator", indicator);
+
+    String result;
+    try {
+      result =
+          Request.post("http://127.0.0.1:8000/http-strategy")
+              .bodyString(gson.toJson(data), null)
+              .execute()
+              .returnContent()
+              .asString();
+    } catch (IOException e) {
+      result = "error";
+    }
+
+    return result;
+  }
+  // docs-http-end
+
+  // docs-calc-start
+  public List<String> calc(Long time, Map<String, List<List<String>>> banksAndIndicators) {
+
+    List<String> results = new ArrayList<>();
+    for (Entry<String, List<List<String>>> e : banksAndIndicators.entrySet()) {
+      String bank = e.getKey();
+      for (List<String> indicators : e.getValue()) {
+        for (String indicator : indicators) {
+          results.add(httpCalc(time, bank, indicator));
+        }
+      }
+    }
+    return results;
+  }
+  // docs-calc-end
+
+  // docs-parallel-calc-start
+  private ExecutorService executorService = Executors.newFixedThreadPool(4);
+
+  public List<String> parallelCalc(Long time, Map<String, List<List<String>>> banksAndIndicators) {
+
+    List<String> results = new ArrayList<>();
+    List<Future<String>> futures = new ArrayList<>();
+    for (Entry<String, List<List<String>>> e : banksAndIndicators.entrySet()) {
+      String bank = e.getKey();
+      for (List<String> indicators : e.getValue()) {
+        for (String indicator : indicators) {
+          futures.add(executorService.submit(() -> httpCalc(time, bank, indicator)));
+        }
+      }
+    }
+    for (Future<String> future : futures) {
+      try {
+        results.add(future.get());
+      } catch (InterruptedException | ExecutionException e1) {
+        results.add("error");
+      }
+    }
+    return results;
+  }
+  // docs-parallel-calc-end
+
+  // docs-main-start
+  public static void main(String[] args) {
+
+    long time = System.currentTimeMillis();
+    String bank1 = "demo_bank_1";
+    String bank2 = "demo_bank_2";
+    String indicator1 = "demo_indicator_1";
+    String indicator2 = "demo_indicator_2";
+    Map<String, List<List<String>>> banksAndIndicators = new HashMap<>();
+    banksAndIndicators.put(bank1, Arrays.asList(Arrays.asList(indicator1, indicator2)));
+    banksAndIndicators.put(
+        bank2, Arrays.asList(Arrays.asList(indicator1), Arrays.asList(indicator2)));
+
+    HttpStrategyCalcOnRayServe strategy = new HttpStrategyCalcOnRayServe();
+    strategy.deploy();
+    List<String> results = strategy.parallelCalc(time, banksAndIndicators);
+
+    System.out.println(results);
+  }
+  // docs-main-end
+}

java/serve/src/test/java/io/ray/serve/docdemo/HttpStrategyOnRayServe.java

@@ -0,0 +1,22 @@
+package io.ray.serve.docdemo;
+
+// docs-strategy-start
+import com.google.gson.Gson;
+
+public class HttpStrategyOnRayServe {
+
+  static class BankIndicator {
+    long time;
+    String bank;
+    String indicator;
+  }
+
+  private Gson gson = new Gson();
+
+  public String call(String dataJson) {
+    BankIndicator data = gson.fromJson(dataJson, BankIndicator.class);
+    // do bank data calculation
+    return data.bank + "-" + data.indicator + "-" + data.time; // Demo;
+  }
+}
+// docs-strategy-end

java/serve/src/test/java/io/ray/serve/docdemo/ManageDeployment.java

@@ -0,0 +1,81 @@
+package io.ray.serve.docdemo;
+
+import io.ray.serve.api.Serve;
+import io.ray.serve.deployment.Application;
+import io.ray.serve.deployment.Deployment;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class ManageDeployment {
+
+  // docs-create-start
+  public static class Counter {
+
+    private AtomicInteger value;
+
+    public Counter(String value) {
+      this.value = new AtomicInteger(Integer.valueOf(value));
+    }
+
+    public String call(String delta) {
+      return String.valueOf(value.addAndGet(Integer.valueOf(delta)));
+    }
+  }
+
+  public void create() {
+    Application app =
+        Serve.deployment()
+            .setName("counter")
+            .setDeploymentDef(Counter.class.getName())
+            .setNumReplicas(1)
+            .bind("1");
+    Serve.run(app);
+  }
+  // docs-create-end
+
+  // docs-query-start
+  public Deployment query() {
+    Deployment deployment = Serve.getDeployment("counter");
+    return deployment;
+  }
+  // docs-query-end
+
+  // docs-update-start
+  public void update() {
+    Application app =
+        Serve.deployment()
+            .setName("counter")
+            .setDeploymentDef(Counter.class.getName())
+            .setNumReplicas(1)
+            .bind("2");
+    Serve.run(app);
+  }
+  // docs-update-end
+
+  // docs-scale-start
+  public void scaleOut() {
+    Deployment deployment = Serve.getDeployment("counter");
+
+    // Scale up to 2 replicas.
+    Serve.run(deployment.options().setNumReplicas(2).bind());
+
+    // Scale down to 1 replica.
+    Serve.run(deployment.options().setNumReplicas(1).bind());
+  }
+  // docs-scale-end
+
+  // docs-resource-start
+  public void manageResource() {
+    Map<String, Object> rayActorOptions = new HashMap<>();
+    rayActorOptions.put("num_gpus", 1);
+    Application app =
+        Serve.deployment()
+            .setName("counter")
+            .setDeploymentDef(Counter.class.getName())
+            .setRayActorOptions(rayActorOptions)
+            .bind();
+    Serve.run(app);
+  }
+  // docs-resource-end
+}

java/serve/src/test/java/io/ray/serve/docdemo/ManagePythonDeployment.java

@@ -0,0 +1,30 @@
+package io.ray.serve.docdemo;
+
+import io.ray.serve.api.Serve;
+import io.ray.serve.deployment.Application;
+import io.ray.serve.generated.DeploymentLanguage;
+import io.ray.serve.handle.DeploymentHandle;
+import java.io.File;
+
+public class ManagePythonDeployment {
+
+  public static void main(String[] args) {
+
+    System.setProperty(
+        "ray.job.code-search-path",
+        System.getProperty("java.class.path") + File.pathSeparator + "/path/to/code/");
+
+    Serve.start(null);
+
+    Application deployment =
+        Serve.deployment()
+            .setLanguage(DeploymentLanguage.PYTHON)
+            .setName("counter")
+            .setDeploymentDef("counter.Counter")
+            .setNumReplicas(1)
+            .bind("1");
+    DeploymentHandle handle = Serve.run(deployment);
+
+    System.out.println(handle.method("increase").remote("2").result());
+  }
+}

java/serve/src/test/java/io/ray/serve/docdemo/Strategy.java

@@ -0,0 +1,35 @@
+package io.ray.serve.docdemo;
+
+// docs-strategy-start
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+public class Strategy {
+
+  public List<String> calc(Long time, Map<String, List<List<String>>> banksAndIndicators) {
+    List<String> results = new ArrayList<>();
+    for (Entry<String, List<List<String>>> e : banksAndIndicators.entrySet()) {
+      String bank = e.getKey();
+      for (List<String> indicators : e.getValue()) {
+        results.addAll(calcBankIndicators(time, bank, indicators));
+      }
+    }
+    return results;
+  }
+
+  public List<String> calcBankIndicators(Long time, String bank, List<String> indicators) {
+    List<String> results = new ArrayList<>();
+    for (String indicator : indicators) {
+      results.add(calcIndicator(time, bank, indicator));
+    }
+    return results;
+  }
+
+  public String calcIndicator(Long time, String bank, String indicator) {
+    // do bank data calculation
+    return bank + "-" + indicator + "-" + time; // Demo;
+  }
+}
+// docs-strategy-end

java/serve/src/test/java/io/ray/serve/docdemo/StrategyCalc.java

@@ -0,0 +1,28 @@
+package io.ray.serve.docdemo;
+
+// docs-strategy-calc-start
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class StrategyCalc {
+
+  public static void main(String[] args) {
+    long time = System.currentTimeMillis();
+    String bank1 = "demo_bank_1";
+    String bank2 = "demo_bank_2";
+    String indicator1 = "demo_indicator_1";
+    String indicator2 = "demo_indicator_2";
+    Map<String, List<List<String>>> banksAndIndicators = new HashMap<>();
+    banksAndIndicators.put(bank1, Arrays.asList(Arrays.asList(indicator1, indicator2)));
+    banksAndIndicators.put(
+        bank2, Arrays.asList(Arrays.asList(indicator1), Arrays.asList(indicator2)));
+
+    Strategy strategy = new Strategy();
+    List<String> results = strategy.calc(time, banksAndIndicators);
+
+    System.out.println(results);
+  }
+}
+// docs-strategy-calc-end

java/serve/src/test/java/io/ray/serve/docdemo/StrategyCalcOnRayServe.java

@@ -0,0 +1,95 @@
+package io.ray.serve.docdemo;
+
+import io.ray.serve.api.Serve;
+import io.ray.serve.deployment.Application;
+import io.ray.serve.deployment.Deployment;
+import io.ray.serve.handle.DeploymentResponse;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+public class StrategyCalcOnRayServe {
+
+  // docs-deploy-start
+  public void deploy() {
+    Serve.start(null);
+
+    Application deployment =
+        Serve.deployment()
+            .setName("strategy")
+            .setDeploymentDef(StrategyOnRayServe.class.getName())
+            .setNumReplicas(4)
+            .bind();
+    Serve.run(deployment);
+  }
+  // docs-deploy-end
+
+  // docs-calc-start
+  public List<String> calc(Long time, Map<String, List<List<String>>> banksAndIndicators) {
+    Deployment deployment = Serve.getDeployment("strategy");
+
+    List<String> results = new ArrayList<>();
+    for (Entry<String, List<List<String>>> e : banksAndIndicators.entrySet()) {
+      String bank = e.getKey();
+      for (List<String> indicators : e.getValue()) {
+        for (String indicator : indicators) {
+          results.add(
+              (String)
+                  deployment
+                      .getHandle()
+                      .method("calcIndicator")
+                      .remote(time, bank, indicator)
+                      .result());
+        }
+      }
+    }
+    return results;
+  }
+  // docs-calc-end
+
+  // docs-parallel-calc-start
+  public List<String> parallelCalc(Long time, Map<String, List<List<String>>> banksAndIndicators) {
+    Deployment deployment = Serve.getDeployment("strategy");
+
+    List<String> results = new ArrayList<>();
+    List<DeploymentResponse> responses = new ArrayList<>();
+    for (Entry<String, List<List<String>>> e : banksAndIndicators.entrySet()) {
+      String bank = e.getKey();
+      for (List<String> indicators : e.getValue()) {
+        for (String indicator : indicators) {
+          responses.add(
+              deployment.getHandle().method("calcIndicator").remote(time, bank, indicator));
+        }
+      }
+    }
+    for (DeploymentResponse response : responses) {
+      results.add((String) response.result());
+    }
+    return results;
+  }
+  // docs-parallel-calc-end
+
+  // docs-main-start
+  public static void main(String[] args) {
+
+    long time = System.currentTimeMillis();
+    String bank1 = "demo_bank_1";
+    String bank2 = "demo_bank_2";
+    String indicator1 = "demo_indicator_1";
+    String indicator2 = "demo_indicator_2";
+    Map<String, List<List<String>>> banksAndIndicators = new HashMap<>();
+    banksAndIndicators.put(bank1, Arrays.asList(Arrays.asList(indicator1, indicator2)));
+    banksAndIndicators.put(
+        bank2, Arrays.asList(Arrays.asList(indicator1), Arrays.asList(indicator2)));
+
+    StrategyCalcOnRayServe strategy = new StrategyCalcOnRayServe();
+    strategy.deploy();
+    List<String> results = strategy.parallelCalc(time, banksAndIndicators);
+
+    System.out.println(results);
+  }
+  // docs-main-end
+}

java/serve/src/test/java/io/ray/serve/docdemo/StrategyOnRayServe.java

@@ -0,0 +1,11 @@
+package io.ray.serve.docdemo;
+
+// docs-strategy-start
+public class StrategyOnRayServe {
+
+  public String calcIndicator(Long time, String bank, String indicator) {
+    // do bank data calculation
+    return bank + "-" + indicator + "-" + time; // Demo;
+  }
+}
+// docs-strategy-end

java/serve/src/test/java/io/ray/serve/docdemo/test/HttpStrategyCalcOnRayServeTest.java

@@ -0,0 +1,71 @@
+package io.ray.serve.docdemo.test;
+
+import com.google.gson.Gson;
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.docdemo.HttpStrategyCalcOnRayServe;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hc.client5.http.fluent.Request;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class HttpStrategyCalcOnRayServeTest extends BaseServeTest {
+
+  public static class HttpStrategyCalcOnRayServeForTest extends HttpStrategyCalcOnRayServe {
+
+    private Gson gson = new Gson();
+
+    @Override
+    public String httpCalc(Long time, String bank, String indicator) {
+      Map<String, Object> data = new HashMap<>();
+      data.put("time", time);
+      data.put("bank", bank);
+      data.put("indicator", indicator);
+      String result;
+      try {
+        result =
+            Request.post("http://127.0.0.1:8341/http-strategy")
+                .bodyString(gson.toJson(data), null)
+                .execute()
+                .returnContent()
+                .asString();
+      } catch (IOException e) {
+        result = "error";
+      }
+      return result;
+    }
+  }
+
+  @Test(groups = {"cluster"})
+  public void test() {
+    String prefix = "HttpStrategyCalcOnRayServeTest";
+    String bank1 = prefix + "_bank_1";
+    String bank2 = prefix + "_bank_2";
+    String indicator1 = prefix + "_indicator_1";
+    String indicator2 = prefix + "_indicator_2";
+    long time = System.currentTimeMillis();
+
+    Map<String, List<List<String>>> banksAndIndicators = new HashMap<>();
+    banksAndIndicators.put(bank1, Arrays.asList(Arrays.asList(indicator1, indicator2)));
+    banksAndIndicators.put(
+        bank2, Arrays.asList(Arrays.asList(indicator1), Arrays.asList(indicator2)));
+
+    HttpStrategyCalcOnRayServe strategy = new HttpStrategyCalcOnRayServeForTest();
+    strategy.deploy();
+
+    List<String> results = strategy.calc(time, banksAndIndicators);
+    Assert.assertTrue(results.contains(bank1 + "-" + indicator1 + "-" + time));
+    Assert.assertTrue(results.contains(bank1 + "-" + indicator2 + "-" + time));
+    Assert.assertTrue(results.contains(bank2 + "-" + indicator1 + "-" + time));
+    Assert.assertTrue(results.contains(bank2 + "-" + indicator1 + "-" + time));
+
+    results = strategy.parallelCalc(time, banksAndIndicators);
+    Assert.assertTrue(results.contains(bank1 + "-" + indicator1 + "-" + time));
+    Assert.assertTrue(results.contains(bank1 + "-" + indicator2 + "-" + time));
+    Assert.assertTrue(results.contains(bank2 + "-" + indicator1 + "-" + time));
+    Assert.assertTrue(results.contains(bank2 + "-" + indicator1 + "-" + time));
+  }
+}

java/serve/src/test/java/io/ray/serve/docdemo/test/ManageDeploymentTest.java

@@ -0,0 +1,31 @@
+package io.ray.serve.docdemo.test;
+
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.deployment.Deployment;
+import io.ray.serve.docdemo.ManageDeployment;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class ManageDeploymentTest extends BaseServeTest {
+
+  @Test(
+      enabled = false,
+      groups = {"cluster"})
+  public void test() {
+    ManageDeployment manageDeployment = new ManageDeployment();
+    manageDeployment.create();
+
+    Deployment deployment = manageDeployment.query();
+    Assert.assertEquals(deployment.getName(), "counter");
+    Assert.assertEquals(deployment.getDeploymentConfig().getNumReplicas().intValue(), 1);
+    Assert.assertEquals(deployment.getReplicaConfig().getInitArgs()[0], "1");
+
+    manageDeployment.update();
+    deployment = manageDeployment.query();
+    Assert.assertEquals(deployment.getReplicaConfig().getInitArgs()[0], "2");
+
+    manageDeployment.scaleOut();
+    deployment = manageDeployment.query();
+    Assert.assertEquals(deployment.getDeploymentConfig().getNumReplicas().intValue(), 1);
+  }
+}

java/serve/src/test/java/io/ray/serve/docdemo/test/StrategyCalcOnRayServeTest.java

@@ -0,0 +1,45 @@
+package io.ray.serve.docdemo.test;
+
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.docdemo.StrategyCalcOnRayServe;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class StrategyCalcOnRayServeTest extends BaseServeTest {
+
+  @Test(
+      enabled = false,
+      groups = {"cluster"})
+  public void test() {
+    String prefix = "StrategyCalcOnRayServeTest";
+    String bank1 = prefix + "_bank_1";
+    String bank2 = prefix + "_bank_2";
+    String indicator1 = prefix + "_indicator_1";
+    String indicator2 = prefix + "_indicator_2";
+    long time = System.currentTimeMillis();
+
+    Map<String, List<List<String>>> banksAndIndicators = new HashMap<>();
+    banksAndIndicators.put(bank1, Arrays.asList(Arrays.asList(indicator1, indicator2)));
+    banksAndIndicators.put(
+        bank2, Arrays.asList(Arrays.asList(indicator1), Arrays.asList(indicator2)));
+
+    StrategyCalcOnRayServe strategy = new StrategyCalcOnRayServe();
+    strategy.deploy();
+
+    List<String> results = strategy.calc(time, banksAndIndicators);
+    Assert.assertTrue(results.contains(bank1 + "-" + indicator1 + "-" + time));
+    Assert.assertTrue(results.contains(bank1 + "-" + indicator2 + "-" + time));
+    Assert.assertTrue(results.contains(bank2 + "-" + indicator1 + "-" + time));
+    Assert.assertTrue(results.contains(bank2 + "-" + indicator1 + "-" + time));
+
+    results = strategy.parallelCalc(time, banksAndIndicators);
+    Assert.assertTrue(results.contains(bank1 + "-" + indicator1 + "-" + time));
+    Assert.assertTrue(results.contains(bank1 + "-" + indicator2 + "-" + time));
+    Assert.assertTrue(results.contains(bank2 + "-" + indicator1 + "-" + time));
+    Assert.assertTrue(results.contains(bank2 + "-" + indicator1 + "-" + time));
+  }
+}

java/serve/src/test/java/io/ray/serve/docdemo/test/StrategyCalcTest.java

@@ -0,0 +1,12 @@
+package io.ray.serve.docdemo.test;
+
+import io.ray.serve.docdemo.StrategyCalc;
+import org.testng.annotations.Test;
+
+public class StrategyCalcTest {
+
+  @Test
+  public void test() {
+    StrategyCalc.main(null);
+  }
+}

java/serve/src/test/java/io/ray/serve/poll/KeyListenerTest.java

@@ -0,0 +1,15 @@
+package io.ray.serve.poll;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class KeyListenerTest {
+
+  @Test
+  public void test() throws Throwable {
+    int[] a = new int[] {0};
+    KeyListener keyListener = (x) -> ((int[]) x)[0] = 1;
+    keyListener.notifyChanged(a);
+    Assert.assertEquals(a[0], 1);
+  }
+}

java/serve/src/test/java/io/ray/serve/poll/KeyTypeTest.java

@@ -0,0 +1,70 @@
+package io.ray.serve.poll;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class KeyTypeTest {
+
+  @Test
+  public void equalsTest() {
+    KeyType k1 = new KeyType(LongPollNamespace.ROUTE_TABLE, "k1");
+    KeyType k2 = new KeyType(LongPollNamespace.ROUTE_TABLE, "k1");
+    KeyType k3 = new KeyType(LongPollNamespace.ROUTE_TABLE, null);
+    KeyType k4 = new KeyType(LongPollNamespace.ROUTE_TABLE, "k4");
+
+    Assert.assertTrue(k1.equals(k1));
+    Assert.assertTrue(k1.equals(k2));
+    Assert.assertTrue(k2.equals(k1));
+    Assert.assertFalse(k1.equals(k3));
+    Assert.assertFalse(k3.equals(k1));
+    Assert.assertFalse(k1.equals(k4));
+    Assert.assertFalse(k4.equals(k1));
+  }
+
+  @Test
+  public void hashCodeTest() {
+    KeyType k1 = new KeyType(LongPollNamespace.ROUTE_TABLE, "k1");
+    KeyType k2 = new KeyType(LongPollNamespace.ROUTE_TABLE, "k1");
+    KeyType k3 = new KeyType(LongPollNamespace.ROUTE_TABLE, null);
+    KeyType k4 = new KeyType(LongPollNamespace.ROUTE_TABLE, "k4");
+
+    Assert.assertEquals(k1.hashCode(), k2.hashCode());
+    Assert.assertNotEquals(k1.hashCode(), k3.hashCode());
+    Assert.assertNotEquals(k1.hashCode(), k4.hashCode());
+  }
+
+  @Test
+  public void toStringTest() {
+    KeyType k1 = new KeyType(LongPollNamespace.ROUTE_TABLE, "k1");
+    Assert.assertEquals(k1.toString(), "(ROUTE_TABLE,k1)");
+
+    KeyType k2 = new KeyType(LongPollNamespace.ROUTE_TABLE, null);
+    Assert.assertEquals(k2.toString(), "ROUTE_TABLE");
+
+    KeyType k3 = new KeyType(null, "k3");
+    Assert.assertEquals(k3.toString(), "k3");
+
+    KeyType k4 = new KeyType(null, null);
+    Assert.assertEquals(k4.toString(), "");
+  }
+
+  @Test
+  public void parseFromTest() {
+    KeyType k1 = KeyType.parseFrom("(ROUTE_TABLE,k1)");
+    Assert.assertEquals(k1.getLongPollNamespace(), LongPollNamespace.ROUTE_TABLE);
+    Assert.assertEquals(k1.getKey(), "k1");
+
+    KeyType k2 = KeyType.parseFrom("ROUTE_TABLE");
+    Assert.assertEquals(k2.getLongPollNamespace(), LongPollNamespace.ROUTE_TABLE);
+    Assert.assertNull(k2.getKey());
+
+    String key3 = "k3";
+    KeyType k3 = KeyType.parseFrom(key3);
+    Assert.assertEquals(k3.getKey(), key3);
+    Assert.assertNull(k3.getLongPollNamespace());
+
+    KeyType k4 = KeyType.parseFrom("");
+    Assert.assertNull(k4.getKey());
+    Assert.assertNull(k4.getLongPollNamespace());
+  }
+}

java/serve/src/test/java/io/ray/serve/poll/LongPollClientTest.java

@@ -0,0 +1,110 @@
+package io.ray.serve.poll;
+
+import io.ray.api.ActorHandle;
+import io.ray.api.ObjectRef;
+import io.ray.api.Ray;
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.DummyServeController;
+import io.ray.serve.api.Serve;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.generated.EndpointInfo;
+import io.ray.serve.generated.EndpointSet;
+import io.ray.serve.generated.LongPollResult;
+import io.ray.serve.generated.UpdatedObject;
+import io.ray.serve.replica.ReplicaContext;
+import java.util.HashMap;
+import java.util.Map;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class LongPollClientTest {
+  @Test
+  public void disableTest() throws Throwable {
+    Map<String, String> config = new HashMap<>();
+    config.put(RayServeConfig.LONG_POOL_CLIENT_ENABLED, "false");
+
+    ReplicaContext replicaContext = new ReplicaContext(null, null, null, config, null);
+    Serve.setInternalReplicaContext(replicaContext);
+    try {
+      LongPollClientFactory.init(null);
+      Assert.assertFalse(LongPollClientFactory.isInitialized());
+    } finally {
+      Serve.setInternalReplicaContext(null);
+    }
+  }
+
+  @SuppressWarnings({"unchecked", "unused"})
+  @Test
+  public void normalTest() throws Throwable {
+    BaseServeTest.initRay();
+    try {
+      String prefix = "LongPollClientTest_normalTest";
+
+      // Init controller.
+      ActorHandle<DummyServeController> controllerHandle =
+          Ray.actor(DummyServeController::new, "")
+              .setName(Constants.SERVE_CONTROLLER_NAME)
+              .remote();
+
+      Serve.setInternalReplicaContext(null, null, null, null, null);
+
+      // Init route table.
+      String endpointName1 = prefix + "_endpoint1";
+      String endpointName2 = prefix + "_endpoint2";
+      EndpointSet endpointSet =
+          EndpointSet.newBuilder()
+              .putEndpoints(
+                  endpointName1, EndpointInfo.newBuilder().setEndpointName(endpointName1).build())
+              .putEndpoints(
+                  endpointName2, EndpointInfo.newBuilder().setEndpointName(endpointName2).build())
+              .build();
+
+      // Construct a listener map.
+      KeyType keyType = new KeyType(LongPollNamespace.ROUTE_TABLE, null);
+      String[] testData = new String[] {"test"};
+      Map<KeyType, KeyListener> keyListeners = new HashMap<>();
+      keyListeners.put(
+          keyType,
+          (object) ->
+              testData[0] =
+                  ((Map<String, EndpointInfo>) object).get(endpointName1).getEndpointName());
+
+      // Register.
+      LongPollClient longPollClient = new LongPollClient(controllerHandle, keyListeners);
+      Assert.assertTrue(LongPollClientFactory.isInitialized());
+
+      // Construct updated object.
+      int snapshotId = 10;
+      UpdatedObject updatedObject =
+          UpdatedObject.newBuilder()
+              .setSnapshotId(snapshotId)
+              .setObjectSnapshot(endpointSet.toByteString())
+              .build();
+
+      // Mock LongPollResult.
+      LongPollResult longPollResult =
+          LongPollResult.newBuilder().putUpdatedObjects(keyType.toString(), updatedObject).build();
+      ObjectRef<Boolean> mockLongPollResult =
+          controllerHandle
+              .task(DummyServeController::setLongPollResult, longPollResult.toByteArray())
+              .remote();
+      Assert.assertEquals(mockLongPollResult.get().booleanValue(), true);
+
+      // Poll.
+      LongPollClientFactory.pollNext();
+
+      // Validation.
+      Assert.assertEquals(LongPollClientFactory.SNAPSHOT_IDS.get(keyType).intValue(), snapshotId);
+      Assert.assertEquals(
+          ((Map<String, EndpointInfo>) LongPollClientFactory.OBJECT_SNAPSHOTS.get(keyType)).size(),
+          2);
+      Assert.assertEquals(testData[0], endpointName1);
+
+      LongPollClientFactory.stop();
+      Assert.assertFalse(LongPollClientFactory.isInitialized());
+    } finally {
+      BaseServeTest.clearAndShutdownRay();
+    }
+  }
+}

java/serve/src/test/java/io/ray/serve/poll/LongPollNamespaceTest.java

@@ -0,0 +1,24 @@
+package io.ray.serve.poll;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class LongPollNamespaceTest {
+
+  @Test
+  public void toStringTest() {
+    String key = LongPollNamespace.ROUTE_TABLE.toString();
+    Assert.assertEquals(key, "LongPollNamespace.ROUTE_TABLE");
+  }
+
+  @Test
+  public void parseFromTest() {
+    String key = "ROUTE_TABLE";
+    LongPollNamespace longPollNamespace = LongPollNamespace.parseFrom(key);
+    Assert.assertEquals(longPollNamespace, LongPollNamespace.ROUTE_TABLE);
+
+    String unknown = "unknown";
+    longPollNamespace = LongPollNamespace.parseFrom(unknown);
+    Assert.assertNull(longPollNamespace);
+  }
+}

java/serve/src/test/java/io/ray/serve/repdemo/DeploymentDemo.java

@@ -0,0 +1,24 @@
+package io.ray.serve.repdemo;
+
+import io.ray.serve.api.Serve;
+import io.ray.serve.deployment.Application;
+import io.ray.serve.handle.DeploymentHandle;
+
+public class DeploymentDemo {
+  private String msg;
+
+  public DeploymentDemo(String msg) {
+    this.msg = msg;
+  }
+
+  public String call() {
+    return msg;
+  }
+
+  public static void main(String[] args) {
+    Application deployment =
+        Serve.deployment().setDeploymentDef(DeploymentDemo.class.getName()).bind();
+    DeploymentHandle handle = Serve.run(deployment);
+    System.out.println(handle.remote().result());
+  }
+}

java/serve/src/test/java/io/ray/serve/repdemo/Driver.java

@@ -0,0 +1,43 @@
+package io.ray.serve.repdemo;
+
+import io.ray.serve.api.Serve;
+import io.ray.serve.deployment.Application;
+import io.ray.serve.handle.DeploymentHandle;
+import io.ray.serve.handle.DeploymentResponse;
+
+public class Driver {
+  private DeploymentHandle modelAHandle;
+  private DeploymentHandle modelBHandle;
+
+  public Driver(DeploymentHandle modelAHandle, DeploymentHandle modelBHandle) {
+    this.modelAHandle = modelAHandle;
+    this.modelBHandle = modelBHandle;
+  }
+
+  public String call(String request) {
+    DeploymentResponse responseA = modelAHandle.remote(request);
+    DeploymentResponse responseB = modelBHandle.remote(request);
+    return (String) responseA.result() + responseB.result();
+  }
+
+  public static class ModelA {
+    public String call(String msg) {
+      return msg;
+    }
+  }
+
+  public static class ModelB {
+    public String call(String msg) {
+      return msg;
+    }
+  }
+
+  public static void main(String[] args) {
+    Application modelA = Serve.deployment().setDeploymentDef(ModelA.class.getName()).bind();
+    Application modelB = Serve.deployment().setDeploymentDef(ModelB.class.getName()).bind();
+
+    Application driver =
+        Serve.deployment().setDeploymentDef(Driver.class.getName()).bind(modelA, modelB);
+    Serve.run(driver);
+  }
+}

java/serve/src/test/java/io/ray/serve/repdemo/Hello.java

@@ -0,0 +1,42 @@
+package io.ray.serve.repdemo;
+
+import io.ray.serve.api.Serve;
+import io.ray.serve.deployment.Application;
+import java.util.Map;
+
+public class Hello {
+
+  public static class HelloWorldArgs {
+    private String message;
+
+    public String getMessage() {
+      return message;
+    }
+
+    public void setMessage(String message) {
+      this.message = message;
+    }
+  }
+
+  public static class HelloWorld {
+    private String message;
+
+    public HelloWorld(String message) {
+      this.message = message;
+    }
+
+    public String call() {
+      return message;
+    }
+  }
+
+  public static Application appBuilder(Map<String, String> args) {
+    return Serve.deployment()
+        .setDeploymentDef(HelloWorld.class.getName())
+        .bind(args.get("message"));
+  }
+
+  public static Application typedAppBuilder(HelloWorldArgs args) {
+    return Serve.deployment().setDeploymentDef(HelloWorld.class.getName()).bind(args.getMessage());
+  }
+}

java/serve/src/test/java/io/ray/serve/repdemo/Text.java

@@ -0,0 +1,43 @@
+package io.ray.serve.repdemo;
+
+import io.ray.serve.api.Serve;
+import io.ray.serve.deployment.Application;
+import io.ray.serve.handle.DeploymentHandle;
+
+public class Text {
+
+  public static class Hello {
+    public String call() {
+      return "Hello";
+    }
+  }
+
+  public static class World {
+    public String call() {
+      return " world!";
+    }
+  }
+
+  public static class Ingress {
+    private DeploymentHandle helloHandle;
+    private DeploymentHandle worldHandle;
+
+    public Ingress(DeploymentHandle helloHandle, DeploymentHandle worldHandle) {
+      this.helloHandle = helloHandle;
+      this.worldHandle = worldHandle;
+    }
+
+    public String call() {
+      return (String) helloHandle.remote().result() + worldHandle.remote().result();
+    }
+  }
+
+  public static Application app() {
+    Application hello = Serve.deployment().setDeploymentDef(Hello.class.getName()).bind();
+    Application world = Serve.deployment().setDeploymentDef(World.class.getName()).bind();
+
+    Application app =
+        Serve.deployment().setDeploymentDef(Ingress.class.getName()).bind(hello, world);
+    return app;
+  }
+}

java/serve/src/test/java/io/ray/serve/replica/DummyReplica.java

@@ -0,0 +1,21 @@
+package io.ray.serve.replica;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class DummyReplica {
+
+  private AtomicInteger counter = new AtomicInteger();
+
+  public String call() {
+    return String.valueOf(counter.incrementAndGet());
+  }
+
+  public void reconfigure(Object userConfig) {
+    counter.set(0);
+  }
+
+  public void reconfigure(Map<String, String> userConfig) {
+    counter.set(Integer.valueOf(userConfig.get("value")));
+  }
+}

java/serve/src/test/java/io/ray/serve/replica/RayServeReplicaTest.java

@@ -0,0 +1,128 @@
+package io.ray.serve.replica;
+
+import com.google.common.collect.ImmutableMap;
+import io.ray.api.ActorHandle;
+import io.ray.api.ObjectRef;
+import io.ray.api.Ray;
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.DummyServeController;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.deployment.DeploymentVersion;
+import io.ray.serve.deployment.DeploymentWrapper;
+import io.ray.serve.generated.DeploymentLanguage;
+import io.ray.serve.generated.RequestMetadata;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class RayServeReplicaTest {
+  @Test
+  public void test() throws IOException {
+
+    try {
+      BaseServeTest.initRay();
+
+      String prefix = "RayServeReplicaTest";
+      String deploymentName = prefix + "_deployment";
+      String replicaTag = prefix + "_replica";
+      String version = "v1";
+      Map<String, String> config = new HashMap<>();
+      config.put(RayServeConfig.LONG_POOL_CLIENT_ENABLED, "false");
+
+      ActorHandle<DummyServeController> controllerHandle =
+          Ray.actor(DummyServeController::new, "")
+              .setName(Constants.SERVE_CONTROLLER_NAME)
+              .remote();
+      controllerHandle
+          .task(DummyServeController::getRootUrl)
+          .remote()
+          .get(); // Wait for the dummy controller to be ready.
+
+      DeploymentConfig deploymentConfig =
+          new DeploymentConfig().setDeploymentLanguage(DeploymentLanguage.JAVA);
+      DeploymentWrapper deploymentWrapper =
+          new DeploymentWrapper()
+              .setName(deploymentName)
+              .setDeploymentConfig(deploymentConfig)
+              .setDeploymentVersion(new DeploymentVersion(version))
+              .setDeploymentDef(DummyReplica.class.getName())
+              .setAppName("app_test");
+
+      ActorHandle<RayServeWrappedReplica> replicHandle =
+          Ray.actor(RayServeWrappedReplica::new, deploymentWrapper, replicaTag).remote();
+
+      // ready
+      Assert.assertTrue(replicHandle.task(RayServeWrappedReplica::checkHealth).remote().get());
+
+      // handle request
+      RequestMetadata.Builder requestMetadata = RequestMetadata.newBuilder();
+      requestMetadata.setRequestId(RandomStringUtils.randomAlphabetic(10));
+      requestMetadata.setCallMethod(Constants.CALL_METHOD);
+
+      ObjectRef<Object> resultRef =
+          replicHandle
+              .task(
+                  RayServeWrappedReplica::handleRequest,
+                  requestMetadata.build().toByteArray(),
+                  new Object[0])
+              .remote();
+      Assert.assertEquals((String) resultRef.get(), "1");
+
+      // reconfigure
+      ObjectRef<Object> versionRef =
+          replicHandle
+              .task(
+                  RayServeWrappedReplica::reconfigure,
+                  (new DeploymentConfig().setVersion("")).toProtoBytes())
+              .remote();
+      Assert.assertEquals(
+          DeploymentVersion.fromProtoBytes((byte[]) (versionRef.get())).getCodeVersion(), version);
+
+      deploymentConfig.setUserConfig(new Object()).setVersion("");
+      replicHandle
+          .task(RayServeWrappedReplica::reconfigure, deploymentConfig.toProtoBytes())
+          .remote()
+          .get();
+      resultRef =
+          replicHandle
+              .task(
+                  RayServeWrappedReplica::handleRequest,
+                  requestMetadata.build().toByteArray(),
+                  new Object[0])
+              .remote();
+      Assert.assertEquals((String) resultRef.get(), "1");
+
+      deploymentConfig = deploymentConfig.setUserConfig(ImmutableMap.of("value", "100"));
+      replicHandle
+          .task(RayServeWrappedReplica::reconfigure, deploymentConfig.toProtoBytes())
+          .remote()
+          .get();
+      resultRef =
+          replicHandle
+              .task(
+                  RayServeWrappedReplica::handleRequest,
+                  requestMetadata.build().toByteArray(),
+                  new Object[0])
+              .remote();
+      Assert.assertEquals((String) resultRef.get(), "101");
+
+      // get version
+      versionRef = replicHandle.task(RayServeWrappedReplica::getVersion).remote();
+      Assert.assertEquals(
+          DeploymentVersion.fromProtoBytes((byte[]) (versionRef.get())).getCodeVersion(), version);
+
+      // prepare for shutdown
+      ObjectRef<Boolean> shutdownRef =
+          replicHandle.task(RayServeWrappedReplica::prepareForShutdown).remote();
+      Assert.assertTrue(shutdownRef.get());
+
+    } finally {
+      BaseServeTest.clearAndShutdownRay();
+    }
+  }
+}

java/serve/src/test/java/io/ray/serve/replica/ReplicaNameTest.java

@@ -0,0 +1,19 @@
+package io.ray.serve.replica;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class ReplicaNameTest {
+
+  @Test
+  public void test() {
+    String deploymentTag = "ReplicaNameTest";
+    String replicaSuffix = RandomStringUtils.randomAlphabetic(6);
+    ReplicaName replicaName = new ReplicaName(deploymentTag, replicaSuffix);
+
+    Assert.assertEquals(replicaName.getDeploymentTag(), deploymentTag);
+    Assert.assertEquals(replicaName.getReplicaSuffix(), replicaSuffix);
+    Assert.assertEquals(replicaName.getReplicaTag(), deploymentTag + "#" + replicaSuffix);
+  }
+}

java/serve/src/test/java/io/ray/serve/router/ReplicaSetTest.java

@@ -0,0 +1,98 @@
+package io.ray.serve.router;
+
+import io.ray.api.ActorHandle;
+import io.ray.api.ObjectRef;
+import io.ray.api.Ray;
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.DummyServeController;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.deployment.DeploymentVersion;
+import io.ray.serve.deployment.DeploymentWrapper;
+import io.ray.serve.generated.DeploymentLanguage;
+import io.ray.serve.generated.DeploymentTargetInfo;
+import io.ray.serve.generated.RequestMetadata;
+import io.ray.serve.replica.RayServeWrappedReplica;
+import io.ray.serve.replica.ReplicaContext;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class ReplicaSetTest {
+  private String deploymentName = "ReplicaSetTest";
+
+  @Test
+  public void updateWorkerReplicasTest() {
+    try {
+      BaseServeTest.initRay();
+      ReplicaSet replicaSet = new ReplicaSet(deploymentName);
+      DeploymentTargetInfo.Builder builder = DeploymentTargetInfo.newBuilder();
+
+      replicaSet.updateWorkerReplicas(builder.build());
+      Map<String, Set<ObjectRef<Object>>> inFlightQueries = replicaSet.getInFlightQueries();
+      Assert.assertTrue(inFlightQueries.isEmpty());
+    } finally {
+      BaseServeTest.shutdownRay();
+    }
+  }
+
+  @SuppressWarnings("unused")
+  @Test
+  public void assignReplicaTest() {
+    try {
+      BaseServeTest.initRay();
+
+      String replicaTag = deploymentName + "_replica";
+      String actorName = replicaTag;
+      String version = "v1";
+      String appName = "app1";
+      // Controller
+      ActorHandle<DummyServeController> controllerHandle =
+          Ray.actor(DummyServeController::new, "")
+              .setName(Constants.SERVE_CONTROLLER_NAME)
+              .remote();
+
+      // Replica
+      DeploymentConfig deploymentConfig =
+          new DeploymentConfig().setDeploymentLanguage(DeploymentLanguage.JAVA);
+
+      Object[] initArgs =
+          new Object[] {deploymentName, replicaTag, new Object(), new HashMap<>(), appName};
+
+      DeploymentWrapper deploymentWrapper =
+          new DeploymentWrapper()
+              .setName(deploymentName)
+              .setDeploymentConfig(deploymentConfig)
+              .setDeploymentVersion(new DeploymentVersion(version))
+              .setDeploymentDef(ReplicaContext.class.getName())
+              .setInitArgs(initArgs);
+
+      ActorHandle<RayServeWrappedReplica> replicaHandle =
+          Ray.actor(RayServeWrappedReplica::new, deploymentWrapper, replicaTag)
+              .setName(actorName)
+              .remote();
+      Assert.assertTrue(replicaHandle.task(RayServeWrappedReplica::checkHealth).remote().get());
+
+      // ReplicaSet
+      ReplicaSet replicaSet = new ReplicaSet(deploymentName);
+      DeploymentTargetInfo.Builder builder = DeploymentTargetInfo.newBuilder();
+      builder.addReplicaNames(actorName).setIsAvailable(true);
+      replicaSet.updateWorkerReplicas(builder.build());
+
+      // assign
+      RequestMetadata.Builder requestMetadata = RequestMetadata.newBuilder();
+      requestMetadata.setRequestId(RandomStringUtils.randomAlphabetic(10));
+      requestMetadata.setCallMethod("getDeploymentName");
+
+      Query query = new Query(requestMetadata.build(), null);
+      ObjectRef<Object> resultRef = replicaSet.assignReplica(query);
+
+      Assert.assertEquals((String) resultRef.get(), deploymentName);
+    } finally {
+      BaseServeTest.clearAndShutdownRay();
+    }
+  }
+}

java/serve/src/test/java/io/ray/serve/router/RouterTest.java

@@ -0,0 +1,88 @@
+package io.ray.serve.router;
+
+import io.ray.api.ActorHandle;
+import io.ray.api.ObjectRef;
+import io.ray.api.Ray;
+import io.ray.serve.BaseServeTest;
+import io.ray.serve.DummyServeController;
+import io.ray.serve.api.Serve;
+import io.ray.serve.common.Constants;
+import io.ray.serve.config.DeploymentConfig;
+import io.ray.serve.config.RayServeConfig;
+import io.ray.serve.deployment.DeploymentId;
+import io.ray.serve.deployment.DeploymentVersion;
+import io.ray.serve.deployment.DeploymentWrapper;
+import io.ray.serve.generated.DeploymentLanguage;
+import io.ray.serve.generated.DeploymentTargetInfo;
+import io.ray.serve.generated.RequestMetadata;
+import io.ray.serve.replica.RayServeWrappedReplica;
+import io.ray.serve.replica.ReplicaContext;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class RouterTest {
+  @Test
+  public void test() {
+
+    try {
+      BaseServeTest.initRay();
+
+      String deploymentName = "RouterTest";
+      String replicaTag = deploymentName + "_replica";
+      String actorName = replicaTag;
+      String version = "v1";
+      String appName = "app1";
+      Map<String, String> config = new HashMap<>();
+      config.put(RayServeConfig.LONG_POOL_CLIENT_ENABLED, "false");
+
+      // Controller
+      ActorHandle<DummyServeController> controllerHandle =
+          Ray.actor(DummyServeController::new, "")
+              .setName(Constants.SERVE_CONTROLLER_NAME)
+              .remote();
+
+      // Replica
+      DeploymentConfig deploymentConfig =
+          new DeploymentConfig().setDeploymentLanguage(DeploymentLanguage.JAVA);
+
+      Object[] initArgs =
+          new Object[] {deploymentName, replicaTag, new Object(), new HashMap<>(), appName};
+
+      DeploymentWrapper deploymentWrapper =
+          new DeploymentWrapper()
+              .setName(deploymentName)
+              .setDeploymentConfig(deploymentConfig)
+              .setDeploymentVersion(new DeploymentVersion(version))
+              .setDeploymentDef(ReplicaContext.class.getName())
+              .setInitArgs(initArgs);
+
+      ActorHandle<RayServeWrappedReplica> replicaHandle =
+          Ray.actor(RayServeWrappedReplica::new, deploymentWrapper, replicaTag)
+              .setName(actorName)
+              .remote();
+      Assert.assertTrue(replicaHandle.task(RayServeWrappedReplica::checkHealth).remote().get());
+
+      // Set ReplicaContext
+      Serve.setInternalReplicaContext(null, null, null, config, null);
+
+      // Router
+      Router router = new Router(controllerHandle, new DeploymentId(deploymentName, appName));
+      DeploymentTargetInfo.Builder builder = DeploymentTargetInfo.newBuilder();
+      builder.addReplicaNames(actorName).setIsAvailable(true);
+      router.getReplicaSet().updateWorkerReplicas(builder.build());
+
+      // assign
+      RequestMetadata.Builder requestMetadata = RequestMetadata.newBuilder();
+      requestMetadata.setRequestId(RandomStringUtils.randomAlphabetic(10));
+      requestMetadata.setCallMethod("getDeploymentName");
+
+      ObjectRef<Object> resultRef = router.assignRequest(requestMetadata.build(), null);
+      Assert.assertEquals((String) resultRef.get(), deploymentName);
+    } finally {
+      BaseServeTest.clearAndShutdownRay();
+    }
+  }
+}

java/serve/src/test/java/io/ray/serve/util/CommonUtilTest.java

@@ -0,0 +1,15 @@
+package io.ray.serve.util;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class CommonUtilTest {
+
+  public static class InnerTest {}
+
+  @Test
+  public void getSimpleClassNameTest() {
+    String name = CommonUtil.getDeploymentName(InnerTest.class.getName());
+    Assert.assertEquals(name, "InnerTest");
+  }
+}

java/serve/src/test/java/io/ray/serve/util/LogUtilTest.java

@@ -0,0 +1,13 @@
+package io.ray.serve.util;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class LogUtilTest {
+
+  @Test
+  public void formatTest() {
+    String result = MessageFormatter.format("{},{},{}", "1", "2", "3");
+    Assert.assertEquals(result, "1,2,3");
+  }
+}

java/serve/src/test/java/io/ray/serve/util/ReflectUtilTest.java

@@ -0,0 +1,66 @@
+package io.ray.serve.util;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class ReflectUtilTest {
+
+  static class ReflectExample {
+    public ReflectExample() {}
+
+    public ReflectExample(Integer a) {}
+
+    public ReflectExample(String a) {}
+
+    public void test(String a) {}
+  }
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void getConstructorTest() throws NoSuchMethodException {
+
+    Constructor<ReflectExample> constructor = ReflectUtil.getConstructor(ReflectExample.class);
+    Assert.assertNotNull(constructor);
+
+    constructor = ReflectUtil.getConstructor(ReflectExample.class);
+    Assert.assertNotNull(constructor);
+
+    constructor = ReflectUtil.getConstructor(ReflectExample.class, 2);
+    Assert.assertNotNull(constructor);
+
+    constructor = ReflectUtil.getConstructor(ReflectExample.class, "");
+    Assert.assertNotNull(constructor);
+
+    Assert.assertThrows(
+        NoSuchMethodException.class,
+        () -> ReflectUtil.getConstructor(ReflectExample.class, new HashMap<>()));
+  }
+
+  @Test
+  public void getMethodTest() throws NoSuchMethodException {
+
+    Method method = ReflectUtil.getMethod(ReflectExample.class, "test", "");
+    Assert.assertNotNull(method);
+
+    Assert.assertThrows(
+        NoSuchMethodException.class,
+        () -> ReflectUtil.getMethod(ReflectExample.class, "test", new HashMap<>()));
+  }
+
+  @Test
+  public void getMethodStringsTest() {
+    List<String> methodList = ReflectUtil.getMethodStrings(ReflectExample.class);
+    String result = null;
+    for (String method : methodList) {
+      if (StringUtils.contains(method, "test")) {
+        result = method;
+      }
+    }
+    Assert.assertNotNull(result, "there should be test method");
+  }
+}

java/serve/src/test/java/io/ray/serve/util/ServeProtoUtilTest.java

@@ -0,0 +1,45 @@
+package io.ray.serve.util;
+
+import com.google.protobuf.ByteString;
+import io.ray.serve.generated.RequestMetadata;
+import io.ray.serve.generated.RequestWrapper;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class ServeProtoUtilTest {
+
+  @Test
+  public void parseRequestMetadataTest() {
+    String prefix = "parseRequestMetadataTest";
+    String requestId = RandomStringUtils.randomAlphabetic(10);
+    String callMethod = prefix + "_method";
+    String endpoint = prefix + "_endpoint";
+    String context = prefix + "_context";
+    RequestMetadata requestMetadata =
+        RequestMetadata.newBuilder()
+            .setRequestId(requestId)
+            .setCallMethod(callMethod)
+            .putContext("context", context)
+            .build();
+
+    RequestMetadata result = ServeProtoUtil.parseRequestMetadata(requestMetadata.toByteArray());
+    Assert.assertNotNull(result);
+    Assert.assertEquals(result.getCallMethod(), callMethod);
+    Assert.assertEquals(result.getRequestId(), requestId);
+    Assert.assertEquals(result.getContextMap().get("context"), context);
+  }
+
+  @Test
+  public void parseRequestWrapperTest() {
+    byte[] body = new byte[] {1, 2};
+    RequestWrapper requestWrapper =
+        RequestWrapper.newBuilder().setBody(ByteString.copyFrom(body)).build();
+
+    RequestWrapper result = ServeProtoUtil.parseRequestWrapper(requestWrapper.toByteArray());
+    Assert.assertNotNull(result);
+    byte[] rstBody = result.getBody().toByteArray();
+    Assert.assertEquals(rstBody[0], 1);
+    Assert.assertEquals(rstBody[1], 2);
+  }
+}