chore: import upstream snapshot with attribution

cpp/include/ray/api.h

@@ -0,0 +1,402 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/actor_creator.h>
+#include <ray/api/actor_handle.h>
+#include <ray/api/actor_task_caller.h>
+#include <ray/api/function_manager.h>
+#include <ray/api/logging.h>
+#include <ray/api/object_ref.h>
+#include <ray/api/ray_config.h>
+#include <ray/api/ray_remote.h>
+#include <ray/api/ray_runtime.h>
+#include <ray/api/ray_runtime_holder.h>
+#include <ray/api/runtime_env.h>
+#include <ray/api/task_caller.h>
+#include <ray/api/wait_result.h>
+
+#include <boost/callable_traits.hpp>
+#include <memory>
+#include <msgpack.hpp>
+#include <mutex>
+
+namespace ray {
+
+/// Initialize Ray runtime with config.
+void Init(ray::RayConfig &config);
+
+/// Initialize Ray runtime with config and command-line arguments.
+/// If a parameter is explicitly set in command-line arguments, the parameter value will
+/// be overwritten.
+void Init(ray::RayConfig &config, int argc, char **argv);
+
+/// Initialize Ray runtime with default config.
+void Init();
+
+/// Check if ray::Init has been called yet.
+bool IsInitialized();
+
+/// Shutdown Ray runtime.
+void Shutdown();
+
+/// Store an object in the object store.
+///
+/// \param[in] obj The object which should be stored.
+/// \return ObjectRef A reference to the object in the object store.
+template <typename T>
+ray::ObjectRef<T> Put(const T &obj);
+
+/// Get a single object from the object store.
+/// This method will be blocked until the object is ready.
+///
+/// \param[in] object The object reference which should be returned.
+/// \return shared pointer of the result.
+template <typename T>
+std::shared_ptr<T> Get(const ray::ObjectRef<T> &object);
+
+/// Get a list of objects from the object store.
+/// This method will be blocked until all the objects are ready.
+///
+/// \param[in] objects The object array which should be got.
+/// \return shared pointer array of the result.
+template <typename T>
+std::vector<std::shared_ptr<T>> Get(const std::vector<ray::ObjectRef<T>> &objects);
+
+/// Get a single object from the object store.
+/// This method will be blocked until the object is ready.
+///
+/// \param[in] object The object reference which should be returned.
+/// \param[in] timeout_ms The maximum amount of time in milliseconds to wait before
+/// returning.
+/// \return shared pointer of the result.
+template <typename T>
+std::shared_ptr<T> Get(const ray::ObjectRef<T> &object, const int &timeout_ms);
+
+/// Get a list of objects from the object store.
+/// This method will be blocked until all the objects are ready.
+///
+/// \param[in] objects The object array which should be got.
+/// \param[in] timeout_ms The maximum amount of time in milliseconds to wait before
+/// returning.
+/// \return shared pointer array of the result.
+template <typename T>
+std::vector<std::shared_ptr<T>> Get(const std::vector<ray::ObjectRef<T>> &objects,
+                                    const int &timeout_ms);
+
+/// Wait for a list of objects to be locally available,
+/// until specified number of objects are ready, or specified timeout has passed.
+///
+/// \param[in] objects The object array which should be waited.
+/// \param[in] num_objects The minimum number of objects to wait.
+/// \param[in] timeout_ms The maximum wait time in milliseconds.
+/// \return Two arrays, one containing locally available objects, one containing the
+/// rest.
+template <typename T>
+WaitResult<T> Wait(const std::vector<ray::ObjectRef<T>> &objects,
+                   int num_objects,
+                   int timeout_ms);
+
+/// Create a `TaskCaller` for calling remote function.
+/// It is used for normal task, such as ray::Task(Plus1).Remote(1),
+/// ray::Task(Plus).Remote(1, 2).
+/// \param[in] func The function to be remote executed.
+/// \return TaskCaller.
+template <typename F>
+ray::internal::TaskCaller<F> Task(F func);
+
+template <typename R>
+ray::internal::TaskCaller<PyFunction<R>> Task(PyFunction<R> func);
+
+template <typename R>
+ray::internal::TaskCaller<JavaFunction<R>> Task(JavaFunction<R> func);
+
+/// Generic version of creating an actor
+/// It is used for creating an actor, such as: ActorCreator<Counter> creator =
+/// ray::Actor(Counter::FactoryCreate<int>).Remote(1);
+template <typename F>
+ray::internal::ActorCreator<F> Actor(F create_func);
+
+ray::internal::ActorCreator<PyActorClass> Actor(PyActorClass func);
+
+ray::internal::ActorCreator<JavaActorClass> Actor(JavaActorClass func);
+
+/// Get a handle to a named actor in current namespace.
+/// The actor must have been created with name specified.
+///
+/// \param[in] actor_name The name of the named actor.
+/// \return An ActorHandle to the actor if the actor of specified name exists or an
+/// empty optional object.
+template <typename T>
+boost::optional<ActorHandle<T>> GetActor(const std::string &actor_name);
+
+/// Get a handle to a named actor in the given namespace.
+/// The actor must have been created with name specified.
+///
+/// \param[in] actor_name The name of the named actor.
+/// \param[in] namespace The namespace of the actor.
+/// \return An ActorHandle to the actor if the actor of specified name exists in
+/// specifiled namespace or an empty optional object.
+template <typename T>
+boost::optional<ActorHandle<T>> GetActor(const std::string &actor_name,
+                                         const std::string &ray_namespace);
+
+/// Intentionally exit the current actor.
+/// It is used to disconnect an actor and exit the worker.
+/// \Throws RayException if the current process is a driver or the current worker is not
+/// an actor.
+void ExitActor();
+
+template <typename T>
+std::vector<std::shared_ptr<T>> Get(const std::vector<std::string> &ids);
+
+template <typename T>
+std::vector<std::shared_ptr<T>> Get(const std::vector<std::string> &ids,
+                                    const int &timeout_ms);
+
+/// Create a placement group on remote nodes.
+///
+/// \param[in] create_options Creation options of the placement group.
+/// \return A PlacementGroup to the created placement group.
+PlacementGroup CreatePlacementGroup(
+    const ray::PlacementGroupCreationOptions &create_options);
+
+/// Remove a placement group by id.
+///
+/// \param[in] placement_group_id Id of the placement group.
+void RemovePlacementGroup(const std::string &placement_group_id);
+
+std::vector<PlacementGroup> GetAllPlacementGroups();
+
+/// Get a placement group by id.
+PlacementGroup GetPlacementGroupById(const std::string &id);
+
+/// Get a placement group by name.
+PlacementGroup GetPlacementGroup(const std::string &name);
+
+/// Returns true if the current actor was restarted, otherwise false.
+bool WasCurrentActorRestarted();
+
+/// Get the namespace of this job.
+std::string GetNamespace();
+
+// --------- inline implementation ------------
+
+template <typename T>
+inline std::vector<std::string> ObjectRefsToObjectIDs(
+    const std::vector<ray::ObjectRef<T>> &object_refs) {
+  std::vector<std::string> object_ids;
+  for (auto it = object_refs.begin(); it != object_refs.end(); it++) {
+    object_ids.push_back(it->ID());
+  }
+  return object_ids;
+}
+
+template <typename T>
+inline ray::ObjectRef<T> Put(const T &obj) {
+  auto buffer =
+      std::make_shared<msgpack::sbuffer>(ray::internal::Serializer::Serialize(obj));
+  auto id = ray::internal::GetRayRuntime()->Put(buffer);
+  auto ref = ObjectRef<T>(id);
+  // The core worker will add an initial ref to the put ID to
+  // keep it in scope. Now that we've created the frontend
+  // ObjectRef, remove this initial ref.
+  ray::internal::GetRayRuntime()->RemoveLocalReference(id);
+  return ref;
+}
+
+template <typename T>
+inline std::shared_ptr<T> Get(const ray::ObjectRef<T> &object, const int &timeout_ms) {
+  return GetFromRuntime(object, timeout_ms);
+}
+
+template <typename T>
+inline std::vector<std::shared_ptr<T>> Get(const std::vector<std::string> &ids,
+                                           const int &timeout_ms) {
+  auto result = ray::internal::GetRayRuntime()->Get(ids, timeout_ms);
+  std::vector<std::shared_ptr<T>> return_objects;
+  return_objects.reserve(result.size());
+  for (auto it = result.begin(); it != result.end(); it++) {
+    auto obj = ray::internal::Serializer::Deserialize<std::shared_ptr<T>>((*it)->data(),
+                                                                          (*it)->size());
+    return_objects.push_back(std::move(obj));
+  }
+  return return_objects;
+}
+
+template <typename T>
+inline std::vector<std::shared_ptr<T>> Get(const std::vector<ray::ObjectRef<T>> &ids,
+                                           const int &timeout_ms) {
+  auto object_ids = ObjectRefsToObjectIDs<T>(ids);
+  return Get<T>(object_ids, timeout_ms);
+}
+
+template <typename T>
+inline std::shared_ptr<T> Get(const ray::ObjectRef<T> &object) {
+  return Get<T>(object, -1);
+}
+
+template <typename T>
+inline std::vector<std::shared_ptr<T>> Get(const std::vector<std::string> &ids) {
+  return Get<T>(ids, -1);
+}
+
+template <typename T>
+inline std::vector<std::shared_ptr<T>> Get(const std::vector<ray::ObjectRef<T>> &ids) {
+  return Get<T>(ids, -1);
+}
+
+template <typename T>
+inline WaitResult<T> Wait(const std::vector<ray::ObjectRef<T>> &objects,
+                          int num_objects,
+                          int timeout_ms) {
+  auto object_ids = ObjectRefsToObjectIDs<T>(objects);
+  auto results =
+      ray::internal::GetRayRuntime()->Wait(object_ids, num_objects, timeout_ms);
+  std::list<ray::ObjectRef<T>> readys;
+  std::list<ray::ObjectRef<T>> unreadys;
+  for (size_t i = 0; i < results.size(); i++) {
+    if (results[i] == true) {
+      readys.emplace_back(objects[i]);
+    } else {
+      unreadys.emplace_back(objects[i]);
+    }
+  }
+  return WaitResult<T>(std::move(readys), std::move(unreadys));
+}
+
+inline ray::internal::ActorCreator<PyActorClass> Actor(PyActorClass func) {
+  ray::internal::RemoteFunctionHolder remote_func_holder(func.module_name,
+                                                         func.function_name,
+                                                         func.class_name,
+                                                         ray::internal::LangType::PYTHON);
+  return {ray::internal::GetRayRuntime().get(), std::move(remote_func_holder)};
+}
+
+template <typename R>
+inline ray::internal::TaskCaller<PyFunction<R>> Task(PyFunction<R> func) {
+  ray::internal::RemoteFunctionHolder remote_func_holder(
+      func.module_name, func.function_name, "", ray::internal::LangType::PYTHON);
+  return {ray::internal::GetRayRuntime().get(), std::move(remote_func_holder)};
+}
+
+template <typename R>
+inline ray::internal::TaskCaller<JavaFunction<R>> Task(JavaFunction<R> func) {
+  ray::internal::RemoteFunctionHolder remote_func_holder(
+      "", func.function_name, func.class_name, ray::internal::LangType::JAVA);
+  return {ray::internal::GetRayRuntime().get(), std::move(remote_func_holder)};
+}
+
+inline ray::internal::ActorCreator<JavaActorClass> Actor(JavaActorClass func) {
+  ray::internal::RemoteFunctionHolder remote_func_holder(func.module_name,
+                                                         func.function_name,
+                                                         func.class_name,
+                                                         ray::internal::LangType::JAVA);
+  return {ray::internal::GetRayRuntime().get(), std::move(remote_func_holder)};
+}
+
+/// Normal task.
+template <typename F>
+inline ray::internal::TaskCaller<F> Task(F func) {
+  static_assert(!ray::internal::is_python_v<F>, "Must be a cpp function.");
+  static_assert(!std::is_member_function_pointer_v<F>,
+                "Incompatible type: member function cannot be called with ray::Task.");
+  auto func_name = internal::FunctionManager::Instance().GetFunctionName(func);
+  ray::internal::RemoteFunctionHolder remote_func_holder(std::move(func_name));
+  return ray::internal::TaskCaller<F>(ray::internal::GetRayRuntime().get(),
+                                      std::move(remote_func_holder));
+}
+
+/// Creating an actor.
+template <typename F>
+inline ray::internal::ActorCreator<F> Actor(F create_func) {
+  auto func_name = internal::FunctionManager::Instance().GetFunctionName(create_func);
+  // Cpp actor don't need class_name, But java/python calls cpp actor need class name
+  // param.
+  auto class_name = internal::FunctionManager::GetClassNameByFuncName(func_name);
+  ray::internal::RemoteFunctionHolder remote_func_holder(
+      "", std::move(func_name), std::move(class_name), internal::LangType::CPP);
+  return ray::internal::ActorCreator<F>(ray::internal::GetRayRuntime().get(),
+                                        std::move(remote_func_holder));
+}
+
+// Get the cpp actor handle by name.
+template <typename T>
+boost::optional<ActorHandle<T>> GetActor(const std::string &actor_name) {
+  return GetActor<T>(actor_name, "");
+}
+
+template <typename T>
+boost::optional<ActorHandle<T>> GetActor(const std::string &actor_name,
+                                         const std::string &ray_namespace) {
+  if (actor_name.empty()) {
+    return {};
+  }
+
+  auto actor_id = ray::internal::GetRayRuntime()->GetActorId(actor_name, ray_namespace);
+  if (actor_id.empty()) {
+    return {};
+  }
+
+  return ActorHandle<T>(actor_id);
+}
+
+// Get the cross-language actor handle by name.
+inline boost::optional<ActorHandleXlang> GetActor(const std::string &actor_name,
+                                                  const std::string &ray_namespce = "") {
+  if (actor_name.empty()) {
+    return {};
+  }
+
+  auto actor_id = ray::internal::GetRayRuntime()->GetActorId(actor_name, ray_namespce);
+  if (actor_id.empty()) {
+    return {};
+  }
+
+  return ActorHandleXlang(actor_id);
+}
+
+inline void ExitActor() { ray::internal::GetRayRuntime()->ExitActor(); }
+
+inline PlacementGroup CreatePlacementGroup(
+    const ray::PlacementGroupCreationOptions &create_options) {
+  return ray::internal::GetRayRuntime()->CreatePlacementGroup(create_options);
+}
+
+inline void RemovePlacementGroup(const std::string &placement_group_id) {
+  return ray::internal::GetRayRuntime()->RemovePlacementGroup(placement_group_id);
+}
+
+inline std::vector<PlacementGroup> GetAllPlacementGroups() {
+  return ray::internal::GetRayRuntime()->GetAllPlacementGroups();
+}
+
+inline PlacementGroup GetPlacementGroupById(const std::string &id) {
+  return ray::internal::GetRayRuntime()->GetPlacementGroupById(id);
+}
+
+inline PlacementGroup GetPlacementGroup(const std::string &name) {
+  return ray::internal::GetRayRuntime()->GetPlacementGroup(name);
+}
+
+inline bool WasCurrentActorRestarted() {
+  return ray::internal::GetRayRuntime()->WasCurrentActorRestarted();
+}
+
+inline std::string GetNamespace() {
+  return ray::internal::GetRayRuntime()->GetNamespace();
+}
+
+}  // namespace ray

cpp/include/ray/api/actor_creator.h

@@ -0,0 +1,113 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/actor_handle.h>
+#include <ray/api/runtime_env.h>
+#include <ray/api/task_options.h>
+
+namespace ray {
+namespace internal {
+
+template <typename F>
+using GetActorType = std::remove_pointer_t<boost::callable_traits::return_type_t<F>>;
+
+template <typename F>
+class ActorCreator {
+ public:
+  ActorCreator() {}
+
+  ActorCreator(RayRuntime *runtime, RemoteFunctionHolder remote_function_holder)
+      : runtime_(runtime), remote_function_holder_(std::move(remote_function_holder)) {}
+
+  template <typename... Args>
+  ray::ActorHandle<GetActorType<F>, is_x_lang_v<F>> Remote(Args &&...args);
+
+  ActorCreator &SetName(std::string name) {
+    create_options_.name = std::move(name);
+    return *this;
+  }
+
+  ActorCreator &SetName(std::string name, std::string ray_namespace) {
+    create_options_.name = std::move(name);
+    create_options_.ray_namespace = std::move(ray_namespace);
+    return *this;
+  }
+
+  ActorCreator &SetResources(std::unordered_map<std::string, double> resources) {
+    create_options_.resources = std::move(resources);
+    return *this;
+  }
+
+  ActorCreator &SetResource(std::string name, double value) {
+    create_options_.resources.emplace(std::move(name), value);
+    return *this;
+  }
+
+  ActorCreator &SetMaxRestarts(int max_restarts) {
+    create_options_.max_restarts = max_restarts;
+    return *this;
+  }
+
+  ActorCreator &SetMaxConcurrency(int max_concurrency) {
+    create_options_.max_concurrency = max_concurrency;
+    return *this;
+  }
+
+  ActorCreator &SetPlacementGroup(PlacementGroup group, int bundle_index) {
+    create_options_.group = group;
+    create_options_.bundle_index = bundle_index;
+    return *this;
+  }
+
+  ActorCreator &SetRuntimeEnv(const ray::RuntimeEnv &runtime_env) {
+    create_options_.serialized_runtime_env_info = runtime_env.SerializeToRuntimeEnvInfo();
+    return *this;
+  }
+
+ private:
+  RayRuntime *runtime_;
+  RemoteFunctionHolder remote_function_holder_;
+  std::vector<TaskArg> args_;
+  ActorCreationOptions create_options_{};
+};
+
+// ---------- implementation ----------
+template <typename F>
+template <typename... Args>
+ActorHandle<GetActorType<F>, is_x_lang_v<F>> ActorCreator<F>::Remote(Args &&...args) {
+  CheckTaskOptions(create_options_.resources);
+
+  if constexpr (is_x_lang_v<F>) {
+    using ArgsTuple = std::tuple<Args...>;
+    Arguments::WrapArgs<ArgsTuple>(remote_function_holder_.lang_type_,
+                                   &args_,
+                                   std::make_index_sequence<sizeof...(Args)>{},
+                                   std::forward<Args>(args)...);
+  } else {
+    StaticCheck<F, Args...>();
+    using ArgsTuple = RemoveReference_t<boost::callable_traits::args_t<F>>;
+    Arguments::WrapArgs<ArgsTuple>(remote_function_holder_.lang_type_,
+                                   &args_,
+                                   std::make_index_sequence<sizeof...(Args)>{},
+                                   std::forward<Args>(args)...);
+  }
+
+  auto returned_actor_id =
+      runtime_->CreateActor(remote_function_holder_, args_, create_options_);
+  return ActorHandle<GetActorType<F>, is_x_lang_v<F>>(returned_actor_id);
+}
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/actor_handle.h

@@ -0,0 +1,92 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/actor_task_caller.h>
+#include <ray/api/function_manager.h>
+#include <ray/api/ray_runtime_holder.h>
+
+namespace ray {
+
+/// A handle to an actor which can be used to invoke a remote actor method, with the
+/// `Call` method.
+/// \param ActorType The type of the concrete actor class.
+/// Note, the `Call` method is defined in actor_call.generated.h.
+template <typename ActorType, bool IsXlang = false>
+class ActorHandle {
+ public:
+  ActorHandle() = default;
+
+  ActorHandle(const std::string &id) { id_ = id; }
+
+  // Used to identify its type.
+  static bool IsActorHandle() { return true; }
+
+  /// Get a untyped ID of the actor
+  const std::string &ID() const { return id_; }
+
+  /// Include the `Call` methods for calling remote functions.
+  template <typename F>
+  ray::internal::ActorTaskCaller<F> Task(F actor_func) {
+    static_assert(!IsXlang && !ray::internal::is_python_v<F>,
+                  "Actor method is not a member function of actor class.");
+    static_assert(std::is_member_function_pointer_v<F>,
+                  "Actor method is not a member function of actor class.");
+    using Self = boost::callable_traits::class_of_t<F>;
+    static_assert(
+        std::is_same<ActorType, Self>::value || std::is_base_of<Self, ActorType>::value,
+        "Class types must be same.");
+    auto func_name = internal::FunctionManager::Instance().GetFunctionName(actor_func);
+    ray::internal::RemoteFunctionHolder remote_func_holder(func_name);
+    return ray::internal::ActorTaskCaller<F>(
+        internal::GetRayRuntime().get(), id_, std::move(remote_func_holder));
+  }
+
+  template <typename R>
+  ray::internal::ActorTaskCaller<PyActorMethod<R>> Task(PyActorMethod<R> func) {
+    static_assert(IsXlang, "Actor function type does not match actor class");
+    ray::internal::RemoteFunctionHolder remote_func_holder(
+        "", func.function_name, "", ray::internal::LangType::PYTHON);
+    return {ray::internal::GetRayRuntime().get(), id_, std::move(remote_func_holder)};
+  }
+
+  template <typename R>
+  ray::internal::ActorTaskCaller<JavaActorMethod<R>> Task(JavaActorMethod<R> func) {
+    static_assert(IsXlang, "Actor function type does not match actor class");
+    ray::internal::RemoteFunctionHolder remote_func_holder(
+        "", func.function_name, "", ray::internal::LangType::JAVA);
+    return {ray::internal::GetRayRuntime().get(), id_, std::move(remote_func_holder)};
+  }
+
+  void Kill() { Kill(true); }
+  void Kill(bool no_restart) {
+    ray::internal::GetRayRuntime()->KillActor(id_, no_restart);
+  }
+
+  static ActorHandle FromBytes(const std::string &serialized_actor_handle) {
+    std::string id = ray::internal::GetRayRuntime()->DeserializeAndRegisterActorHandle(
+        serialized_actor_handle);
+    return ActorHandle(id);
+  }
+
+  /// Make ActorHandle serializable
+  MSGPACK_DEFINE(id_);
+
+ private:
+  std::string id_;
+};
+
+typedef ActorHandle<void, true> ActorHandleXlang;
+}  // namespace ray

cpp/include/ray/api/actor_task_caller.h

@@ -0,0 +1,97 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/arguments.h>
+#include <ray/api/object_ref.h>
+#include <ray/api/static_check.h>
+#include <ray/api/task_options.h>
+namespace ray {
+namespace internal {
+
+template <typename F>
+class ActorTaskCaller {
+ public:
+  ActorTaskCaller() = default;
+
+  ActorTaskCaller(RayRuntime *runtime,
+                  const std::string &id,
+                  RemoteFunctionHolder remote_function_holder)
+      : runtime_(runtime),
+        id_(id),
+        remote_function_holder_(std::move(remote_function_holder)) {}
+
+  template <typename... Args>
+  ObjectRef<boost::callable_traits::return_type_t<F>> Remote(Args &&...args);
+
+  ActorTaskCaller &SetName(std::string name) {
+    task_options_.name = std::move(name);
+    return *this;
+  }
+
+  ActorTaskCaller &SetResources(std::unordered_map<std::string, double> resources) {
+    task_options_.resources = std::move(resources);
+    return *this;
+  }
+
+  ActorTaskCaller &SetResource(std::string name, double value) {
+    task_options_.resources.emplace(std::move(name), value);
+    return *this;
+  }
+
+ private:
+  RayRuntime *runtime_;
+  std::string id_;
+  RemoteFunctionHolder remote_function_holder_;
+  std::vector<TaskArg> args_;
+  CallOptions task_options_;
+};
+
+// ---------- implementation ----------
+
+template <typename F>
+template <typename... Args>
+ObjectRef<boost::callable_traits::return_type_t<F>> ActorTaskCaller<F>::Remote(
+    Args &&...args) {
+  CheckTaskOptions(task_options_.resources);
+
+  if constexpr (is_x_lang_v<F>) {
+    using ArgsTuple = std::tuple<Args...>;
+    Arguments::WrapArgs<ArgsTuple>(remote_function_holder_.lang_type_,
+                                   &args_,
+                                   std::make_index_sequence<sizeof...(Args)>{},
+                                   std::forward<Args>(args)...);
+  } else {
+    StaticCheck<F, Args...>();
+    using ArgsTuple = RemoveReference_t<RemoveFirst_t<boost::callable_traits::args_t<F>>>;
+    Arguments::WrapArgs<ArgsTuple>(remote_function_holder_.lang_type_,
+                                   &args_,
+                                   std::make_index_sequence<sizeof...(Args)>{},
+                                   std::forward<Args>(args)...);
+  }
+
+  using ReturnType = boost::callable_traits::return_type_t<F>;
+  auto returned_object_id =
+      runtime_->CallActor(remote_function_holder_, id_, args_, task_options_);
+  auto return_ref = ObjectRef<ReturnType>(returned_object_id);
+  // The core worker will add an initial ref to each return ID to keep it in
+  // scope. Now that we've created the frontend ObjectRef, remove this initial
+  // ref.
+  runtime_->RemoveLocalReference(returned_object_id);
+  return return_ref;
+}
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/arguments.h

@@ -0,0 +1,123 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/object_ref.h>
+#include <ray/api/serializer.h>
+#include <ray/api/type_traits.h>
+#include <ray/api/xlang_function.h>
+
+#include <msgpack.hpp>
+#include <type_traits>
+
+namespace ray {
+namespace internal {
+
+class Arguments {
+ public:
+  template <typename OriginArgType, typename InputArgTypes>
+  static void WrapArgsImpl(LangType lang_type,
+                           std::vector<TaskArg> *task_args,
+                           InputArgTypes &&arg) {
+    if constexpr (is_object_ref_v<OriginArgType>) {
+      if (RayRuntimeHolder::Instance().Runtime()->IsLocalMode()) {
+        PushReferenceArg(task_args, std::forward<InputArgTypes>(arg));
+      } else {
+        // After the Object Ref parameter is supported, this exception will be deleted.
+        throw std::invalid_argument(
+            "At present, the Ray C++ API does not support the passing of "
+            "`ray::ObjectRef` parameters. Will support later.");
+      }
+    } else if constexpr (is_object_ref_v<InputArgTypes>) {
+      // core_worker submitting task callback will get the value of an ObjectRef arg, but
+      // local mode we don't call core_worker submit task, so we need get the value of an
+      // ObjectRef arg only for local mode.
+      if (RayRuntimeHolder::Instance().Runtime()->IsLocalMode()) {
+        auto buffer = RayRuntimeHolder::Instance().Runtime()->Get(arg.ID());
+        PushValueArg(task_args, std::move(*buffer));
+      } else {
+        PushReferenceArg(task_args, std::forward<InputArgTypes>(arg));
+      }
+    } else {
+      if (lang_type == LangType::CPP) {
+        if constexpr (is_actor_handle_v<InputArgTypes>) {
+          auto serialized_actor_handle =
+              RayRuntimeHolder::Instance().Runtime()->SerializeActorHandle(arg.ID());
+          msgpack::sbuffer buffer = Serializer::Serialize(serialized_actor_handle);
+          PushValueArg(task_args, std::move(buffer));
+        } else {
+          msgpack::sbuffer buffer =
+              Serializer::Serialize(std::forward<InputArgTypes>(arg));
+          PushValueArg(task_args, std::move(buffer));
+        }
+      } else {
+        // Fill dummy field for handling kwargs.
+        if (lang_type == LangType::PYTHON) {
+          msgpack::sbuffer dummy_buf(METADATA_STR_DUMMY.size());
+          dummy_buf.write(METADATA_STR_DUMMY.data(), METADATA_STR_DUMMY.size());
+          PushValueArg(task_args, std::move(dummy_buf), METADATA_STR_RAW);
+        }
+        // Below applies to both PYTHON and JAVA.
+        auto data_buf = Serializer::Serialize(std::forward<InputArgTypes>(arg));
+        auto len_buf = Serializer::Serialize(data_buf.size());
+
+        msgpack::sbuffer buffer(XLANG_HEADER_LEN + data_buf.size());
+        buffer.write(len_buf.data(), len_buf.size());
+        for (size_t i = 0; i < XLANG_HEADER_LEN - len_buf.size(); ++i) {
+          buffer.write("", 1);
+        }
+        buffer.write(data_buf.data(), data_buf.size());
+
+        PushValueArg(task_args, std::move(buffer), METADATA_STR_XLANG);
+      }
+    }
+  }
+
+  template <typename OriginArgsTuple, size_t... I, typename... InputArgTypes>
+  static void WrapArgs(LangType lang_type,
+                       std::vector<TaskArg> *task_args,
+                       std::index_sequence<I...>,
+                       InputArgTypes &&...args) {
+    (void)std::initializer_list<int>{
+        (WrapArgsImpl<std::tuple_element_t<I, OriginArgsTuple>>(
+             lang_type, task_args, std::forward<InputArgTypes>(args)),
+         0)...};
+    /// Silence gcc warning error.
+    (void)task_args;
+    (void)lang_type;
+  }
+
+ private:
+  static void PushValueArg(std::vector<TaskArg> *task_args,
+                           msgpack::sbuffer &&buffer,
+                           std::string_view meta_str = "") {
+    /// Pass by value.
+    TaskArg task_arg;
+    task_arg.buf = std::move(buffer);
+    if (!meta_str.empty()) task_arg.meta_str = std::move(meta_str);
+    task_args->emplace_back(std::move(task_arg));
+  }
+
+  template <typename TaskArg, typename T>
+  static void PushReferenceArg(std::vector<TaskArg> *task_args, T &&arg) {
+    /// Pass by reference.
+    TaskArg task_arg{};
+    task_arg.id = arg.ID();
+    task_args->emplace_back(std::move(task_arg));
+  }
+};
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/common_types.h

@@ -0,0 +1,56 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <memory>
+#include <msgpack.hpp>
+#include <string_view>
+
+#include "boost/optional.hpp"
+
+namespace ray {
+namespace internal {
+
+struct TaskArg {
+  TaskArg() = default;
+  TaskArg(TaskArg &&rhs) {
+    buf = std::move(rhs.buf);
+    id = rhs.id;
+    meta_str = std::move(rhs.meta_str);
+  }
+
+  TaskArg(const TaskArg &) = delete;
+  TaskArg &operator=(TaskArg const &) = delete;
+  TaskArg &operator=(TaskArg &&) = delete;
+
+  /// If the buf is initialized shows it is a value argument.
+  boost::optional<msgpack::sbuffer> buf;
+  /// If the id is initialized shows it is a reference argument.
+  boost::optional<std::string> id;
+
+  std::string_view meta_str;
+};
+
+using ArgsBuffer = msgpack::sbuffer;
+using ArgsBufferList = std::vector<ArgsBuffer>;
+
+using RemoteFunction = std::function<msgpack::sbuffer(const ArgsBufferList &)>;
+using RemoteFunctionMap_t = std::unordered_map<std::string, RemoteFunction>;
+
+using RemoteMemberFunction =
+    std::function<msgpack::sbuffer(msgpack::sbuffer *, const ArgsBufferList &)>;
+using RemoteMemberFunctionMap_t = std::unordered_map<std::string, RemoteMemberFunction>;
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/function_manager.h

@@ -0,0 +1,354 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/common_types.h>
+#include <ray/api/ray_runtime_holder.h>
+#include <ray/api/serializer.h>
+#include <ray/api/type_traits.h>
+
+#include <boost/callable_traits.hpp>
+#include <functional>
+#include <map>
+#include <memory>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <unordered_map>
+
+namespace ray {
+namespace internal {
+
+template <typename T>
+inline static std::enable_if_t<!std::is_pointer<T>::value, msgpack::sbuffer>
+PackReturnValue(T result) {
+  if constexpr (is_actor_handle_v<T>) {
+    auto serialized_actor_handle =
+        RayRuntimeHolder::Instance().Runtime()->SerializeActorHandle(result.ID());
+    return Serializer::Serialize(serialized_actor_handle);
+  }
+  return Serializer::Serialize(std::move(result));
+}
+
+template <typename T>
+inline static std::enable_if_t<std::is_pointer<T>::value, msgpack::sbuffer>
+PackReturnValue(T result) {
+  return Serializer::Serialize((uint64_t)result);
+}
+
+inline static msgpack::sbuffer PackVoid() {
+  return Serializer::Serialize(msgpack::type::nil_t());
+}
+
+msgpack::sbuffer PackError(std::string error_msg);
+
+/// It's help to invoke functions and member functions, the class Invoker<Function> help
+/// do type erase.
+template <typename Function>
+struct Invoker {
+  /// Invoke functions by networking stream, at first deserialize the binary data to a
+  /// tuple, then call function with tuple.
+  static inline msgpack::sbuffer Apply(const Function &func,
+                                       const ArgsBufferList &args_buffer) {
+    using RetrunType = boost::callable_traits::return_type_t<Function>;
+    using ArgsTuple = RemoveReference_t<boost::callable_traits::args_t<Function>>;
+    if (std::tuple_size<ArgsTuple>::value != args_buffer.size()) {
+      throw std::invalid_argument("Arguments number not match");
+    }
+
+    msgpack::sbuffer result;
+    ArgsTuple tp{};
+    bool is_ok = GetArgsTuple(
+        tp, args_buffer, std::make_index_sequence<std::tuple_size<ArgsTuple>::value>{});
+    if (!is_ok) {
+      throw std::invalid_argument("Arguments error");
+    }
+
+    result = Invoker<Function>::Call<RetrunType>(func, std::move(tp));
+    return result;
+  }
+
+  static inline msgpack::sbuffer ApplyMember(const Function &func,
+                                             msgpack::sbuffer *ptr,
+                                             const ArgsBufferList &args_buffer) {
+    using RetrunType = boost::callable_traits::return_type_t<Function>;
+    using ArgsTuple =
+        RemoveReference_t<RemoveFirst_t<boost::callable_traits::args_t<Function>>>;
+    if (std::tuple_size<ArgsTuple>::value != args_buffer.size()) {
+      throw std::invalid_argument("Arguments number not match");
+    }
+
+    msgpack::sbuffer result;
+    ArgsTuple tp{};
+    bool is_ok = GetArgsTuple(
+        tp, args_buffer, std::make_index_sequence<std::tuple_size<ArgsTuple>::value>{});
+    if (!is_ok) {
+      throw std::invalid_argument("Arguments error");
+    }
+
+    uint64_t actor_ptr = Serializer::Deserialize<uint64_t>(ptr->data(), ptr->size());
+    using Self = boost::callable_traits::class_of_t<Function>;
+    Self *self = (Self *)actor_ptr;
+    result = Invoker<Function>::CallMember<RetrunType>(func, self, std::move(tp));
+
+    return result;
+  }
+
+ private:
+  template <typename T>
+  static inline T ParseArg(const ArgsBuffer &args_buffer, bool &is_ok) {
+    is_ok = true;
+    if constexpr (is_object_ref_v<T>) {
+      // Construct an ObjectRef<T> by id.
+      return T(std::string(args_buffer.data(), args_buffer.size()));
+    } else if constexpr (is_actor_handle_v<T>) {
+      auto actor_handle =
+          Serializer::Deserialize<std::string>(args_buffer.data(), args_buffer.size());
+      return T::FromBytes(actor_handle);
+    } else {
+      auto [success, value] =
+          Serializer::DeserializeWhenNil<T>(args_buffer.data(), args_buffer.size());
+      is_ok = success;
+      return value;
+    }
+  }
+
+  static inline bool GetArgsTuple(std::tuple<> &tup,
+                                  const ArgsBufferList &args_buffer,
+                                  std::index_sequence<>) {
+    return true;
+  }
+
+  template <size_t... I, typename... Args>
+  static inline bool GetArgsTuple(std::tuple<Args...> &tp,
+                                  const ArgsBufferList &args_buffer,
+                                  std::index_sequence<I...>) {
+    bool is_ok = true;
+    (void)std::initializer_list<int>{
+        (std::get<I>(tp) = ParseArg<Args>(args_buffer.at(I), is_ok), 0)...};
+    return is_ok;
+  }
+
+  template <typename R, typename F, typename... Args>
+  static std::enable_if_t<std::is_void<R>::value, msgpack::sbuffer> Call(
+      const F &f, std::tuple<Args...> args) {
+    CallInternal<R>(f, std::make_index_sequence<sizeof...(Args)>{}, std::move(args));
+    return PackVoid();
+  }
+
+  template <typename R, typename F, typename... Args>
+  static std::enable_if_t<!std::is_void<R>::value, msgpack::sbuffer> Call(
+      const F &f, std::tuple<Args...> args) {
+    auto r =
+        CallInternal<R>(f, std::make_index_sequence<sizeof...(Args)>{}, std::move(args));
+    return PackReturnValue(r);
+  }
+
+  template <typename R, typename F, size_t... I, typename... Args>
+  static R CallInternal(const F &f,
+                        const std::index_sequence<I...> &,
+                        std::tuple<Args...> args) {
+    (void)args;
+    using ArgsTuple = boost::callable_traits::args_t<F>;
+    return f(((typename std::tuple_element<I, ArgsTuple>::type)std::get<I>(args))...);
+  }
+
+  template <typename R, typename F, typename Self, typename... Args>
+  static std::enable_if_t<std::is_void<R>::value, msgpack::sbuffer> CallMember(
+      const F &f, Self *self, std::tuple<Args...> args) {
+    CallMemberInternal<R>(
+        f, self, std::make_index_sequence<sizeof...(Args)>{}, std::move(args));
+    return PackVoid();
+  }
+
+  template <typename R, typename F, typename Self, typename... Args>
+  static std::enable_if_t<!std::is_void<R>::value, msgpack::sbuffer> CallMember(
+      const F &f, Self *self, std::tuple<Args...> args) {
+    auto r = CallMemberInternal<R>(
+        f, self, std::make_index_sequence<sizeof...(Args)>{}, std::move(args));
+    return PackReturnValue(r);
+  }
+
+  template <typename R, typename F, typename Self, size_t... I, typename... Args>
+  static R CallMemberInternal(const F &f,
+                              Self *self,
+                              const std::index_sequence<I...> &,
+                              std::tuple<Args...> args) {
+    (void)args;
+    using ArgsTuple = boost::callable_traits::args_t<F>;
+    return (self->*f)(
+        ((typename std::tuple_element<I + 1, ArgsTuple>::type) std::get<I>(args))...);
+  }
+};
+
+/// Manage all ray remote functions, add remote functions by RAY_REMOTE, get functions by
+/// TaskExecutionHandler.
+class FunctionManager {
+ public:
+  static FunctionManager &Instance() {
+    static FunctionManager instance;
+    return instance;
+  }
+
+  std::pair<const RemoteFunctionMap_t &, const RemoteMemberFunctionMap_t &>
+  GetRemoteFunctions() {
+    return std::pair<const RemoteFunctionMap_t &, const RemoteMemberFunctionMap_t &>(
+        map_invokers_, map_mem_func_invokers_);
+  }
+
+  RemoteFunction *GetFunction(const std::string &func_name) {
+    auto it = map_invokers_.find(func_name);
+    if (it == map_invokers_.end()) {
+      return nullptr;
+    }
+
+    return &it->second;
+  }
+
+  template <typename Function>
+  std::enable_if_t<!std::is_member_function_pointer<Function>::value, bool>
+  RegisterRemoteFunction(std::string const &name, const Function &f) {
+    auto pair = func_ptr_to_key_map_.emplace(GetAddress(f), name);
+    if (!pair.second) {
+      throw RayException("Duplicate RAY_REMOTE function: " + name);
+    }
+
+    bool ok = RegisterNonMemberFunc(name, f);
+    if (!ok) {
+      throw RayException("Duplicate RAY_REMOTE function: " + name);
+    }
+
+    return true;
+  }
+
+  template <typename Function>
+  std::enable_if_t<std::is_member_function_pointer<Function>::value, bool>
+  RegisterRemoteFunction(std::string const &name, const Function &f) {
+    using Self = boost::callable_traits::class_of_t<Function>;
+    auto key = std::make_pair(typeid(Self).name(), GetAddress(f));
+    auto pair = mem_func_to_key_map_.emplace(std::move(key), name);
+    if (!pair.second) {
+      throw RayException("Duplicate RAY_REMOTE function: " + name);
+    }
+
+    bool ok = RegisterMemberFunc(name, f);
+    if (!ok) {
+      throw RayException("Duplicate RAY_REMOTE function: " + name);
+    }
+
+    return true;
+  }
+
+  template <typename Function>
+  std::enable_if_t<!std::is_member_function_pointer<Function>::value, std::string>
+  GetFunctionName(const Function &f) {
+    auto it = func_ptr_to_key_map_.find(GetAddress(f));
+    if (it == func_ptr_to_key_map_.end()) {
+      return "";
+    }
+
+    return it->second;
+  }
+
+  template <typename Function>
+  std::enable_if_t<std::is_member_function_pointer<Function>::value, std::string>
+  GetFunctionName(const Function &f) {
+    using Self = boost::callable_traits::class_of_t<Function>;
+    auto key = std::make_pair(typeid(Self).name(), GetAddress(f));
+    auto it = mem_func_to_key_map_.find(key);
+    if (it == mem_func_to_key_map_.end()) {
+      return "";
+    }
+
+    return it->second;
+  }
+
+  RemoteMemberFunction *GetMemberFunction(const std::string &func_name) {
+    auto it = map_mem_func_invokers_.find(func_name);
+    if (it == map_mem_func_invokers_.end()) {
+      return nullptr;
+    }
+
+    return &it->second;
+  }
+
+  static std::string GetClassNameByFuncName(const std::string &func_name) {
+    if (func_name.empty()) {
+      return "";
+    }
+
+    const std::string &prefix = "RAY_FUNC(";
+    size_t start_pos = 0;
+    auto pos = func_name.find(prefix);
+    if (pos != func_name.npos) {
+      start_pos = pos + prefix.size();
+    }
+    auto end_pod = func_name.find_last_of("::");
+    if (end_pod == func_name.npos || end_pod <= start_pos) {
+      return "";
+    }
+
+    return func_name.substr(start_pos, (end_pod - start_pos - 1));
+  }
+
+ private:
+  FunctionManager() = default;
+  ~FunctionManager() = default;
+  FunctionManager(const FunctionManager &) = delete;
+  FunctionManager(FunctionManager &&) = delete;
+
+  template <typename Function>
+  bool RegisterNonMemberFunc(std::string const &name, Function f) {
+    return map_invokers_
+        .emplace(
+            name,
+            std::bind(&Invoker<Function>::Apply, std::move(f), std::placeholders::_1))
+        .second;
+  }
+
+  template <typename Function>
+  bool RegisterMemberFunc(std::string const &name, Function f) {
+    return map_mem_func_invokers_
+        .emplace(name,
+                 std::bind(&Invoker<Function>::ApplyMember,
+                           std::move(f),
+                           std::placeholders::_1,
+                           std::placeholders::_2))
+        .second;
+  }
+
+  template <class Dest, class Source>
+  Dest BitCast(const Source &source) {
+    static_assert(sizeof(Dest) == sizeof(Source),
+                  "BitCast requires source and destination to be the same size");
+
+    Dest dest;
+    memcpy(&dest, &source, sizeof(dest));
+    return dest;
+  }
+
+  template <typename F>
+  std::string GetAddress(F f) {
+    auto arr = BitCast<std::array<char, sizeof(F)>>(f);
+    return std::string(arr.data(), arr.size());
+  }
+
+  RemoteFunctionMap_t map_invokers_;
+  RemoteMemberFunctionMap_t map_mem_func_invokers_;
+  std::unordered_map<std::string, std::string> func_ptr_to_key_map_;
+  std::map<std::pair<std::string, std::string>, std::string> mem_func_to_key_map_;
+};
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/internal_api.h

@@ -0,0 +1,21 @@
+// Copyright 2022 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+namespace ray {
+
+void RunTaskExecutionLoop();
+
+}  // namespace ray

cpp/include/ray/api/logging.h

@@ -0,0 +1,83 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <memory>
+
+#if defined(_WIN32)
+#ifndef _WINDOWS_
+// This is a public header that may be compiled outside of Ray's build system,
+// so we cannot rely on the -DWIN32_LEAN_AND_MEAN compiler flag from ray.bzl.
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN  // Prevent Windows.h from including WinSock.h
+#endif
+#include <Windows.h>  // Force inclusion of WinGDI here to resolve name conflict
+#endif
+#ifdef ERROR  // Should be true unless someone else undef'd it already
+#undef ERROR  // Windows GDI defines this macro; make it a global enum so it doesn't
+              // conflict with our code
+enum { ERROR = 0 };
+#endif
+#endif
+
+#if defined(DEBUG) && DEBUG == 1
+// Bazel defines the DEBUG macro for historical reasons:
+// https://github.com/bazelbuild/bazel/issues/3513#issuecomment-323829248
+// Undefine the DEBUG macro to prevent conflicts with our usage below
+#undef DEBUG
+// Redefine DEBUG as itself to allow any '#ifdef DEBUG' to keep working regardless
+#define DEBUG DEBUG
+#endif
+
+namespace ray {
+
+enum class RayLoggerLevel { DEBUG = -1, INFO = 0, WARNING = 1, ERROR = 2, FATAL = 3 };
+
+#define RAYLOG_INTERNAL(level) *CreateRayLogger(__FILE__, __LINE__, level)
+#define RAYLOG(level)                             \
+  if (IsLevelEnabled(ray::RayLoggerLevel::level)) \
+  RAYLOG_INTERNAL(ray::RayLoggerLevel::level)
+
+// To make the logging lib pluggable with other logging libs and make
+// the implementation unaware by the user, RayLog is only a declaration
+// which hides the implementation into logging.cc file.
+// In logging.cc, we can choose different log libs using different macros.
+
+// This is a log interface which does not output anything.
+class RayLogger {
+ public:
+  virtual ~RayLogger(){};
+
+  virtual bool IsEnabled() const = 0;
+
+  template <typename T>
+  RayLogger &operator<<(const T &t) {
+    if (IsEnabled()) {
+      Stream() << t;
+    }
+    return *this;
+  }
+
+ protected:
+  virtual std::ostream &Stream() = 0;
+};
+
+std::unique_ptr<RayLogger> CreateRayLogger(const char *file_name,
+                                           int line_number,
+                                           RayLoggerLevel severity);
+bool IsLevelEnabled(RayLoggerLevel log_level);
+
+}  // namespace ray

cpp/include/ray/api/metric.h

@@ -0,0 +1,135 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace ray {
+class Metric {
+ public:
+  virtual ~Metric() = 0;
+
+  /// Get the name of this metric.
+  std::string GetName() const;
+
+  /// Record the value for this metric.
+  ///
+  /// \param value The value that we record.
+  /// \param tags The map tag values that we want to record for this metric record.
+  void Record(double value, const std::unordered_map<std::string, std::string> &tags);
+
+ protected:
+  void *metric_ = nullptr;
+};  // class Metric
+
+class Gauge : public Metric {
+ public:
+  /// Gauges keep the last recorded value and drop everything before.
+  /// Unlike counters, gauges can go up or down over time.
+  ///
+  /// This corresponds to Prometheus' gauge metric:
+  /// https://prometheus.io/docs/concepts/metric_types/#gauge
+  ///
+  /// \param[in] name The Name of the metric.
+  /// \param[in] description The Description of the metric.
+  /// \param[in] unit The unit of the metric
+  /// \param[in] tag_keys Tag keys of the metric.
+  Gauge(const std::string &name,
+        const std::string &description,
+        const std::string &unit,
+        const std::vector<std::string> &tag_keys = {});
+
+  ///  Set the gauge to the given `value`.
+  ///
+  /// Tags passed in will take precedence over the metric's default tags.
+  ///
+  /// \param[in] value Value to set the gauge to.
+  /// \param[in] tags Tags to set or override for this gauge.
+  void Set(double value, const std::unordered_map<std::string, std::string> &tags);
+};  // class Gauge
+
+class Histogram : public Metric {
+ public:
+  /// Tracks the size and number of events in buckets.
+  ///
+  /// Histograms allow you to calculate aggregate quantiles
+  /// such as 25, 50, 95, 99 percentile latency for an RPC.
+  ///
+  /// This corresponds to Prometheus' histogram metric:
+  /// https://prometheus.io/docs/concepts/metric_types/#histogram
+  ///
+  /// \param[in] name The Name of the metric.
+  /// \param[in] description The Description of the metric.
+  /// \param[in] unit The unit of the metric
+  /// \param[in] boundaries The Boundaries of histogram buckets.
+  /// \param[in] tag_keys Tag keys of the metric.
+  Histogram(const std::string &name,
+            const std::string &description,
+            const std::string &unit,
+            const std::vector<double> boundaries,
+            const std::vector<std::string> &tag_keys = {});
+
+  /// Observe a given `value` and add it to the appropriate bucket.
+  ///
+  /// Tags passed in will take precedence over the metric's default tags.
+  ///
+  /// \param[in] value The value that we record.
+  /// \param[in] tags The map tag values that we want to record
+  void Observe(double value, const std::unordered_map<std::string, std::string> &tags);
+};  // class Histogram
+
+class Counter : public Metric {
+ public:
+  /// A cumulative metric that is monotonically increasing.
+  ///
+  /// This corresponds to Prometheus' counter metric:
+  /// https://prometheus.io/docs/concepts/metric_types/#counter
+  ///
+  /// \param[in] name The Name of the metric.
+  /// \param[in] description The Description of the metric.
+  /// \param[in] unit The unit of the metric
+  /// \param[in] tag_keys Tag keys of the metric.
+  Counter(const std::string &name,
+          const std::string &description,
+          const std::string &unit,
+          const std::vector<std::string> &tag_keys = {});
+
+  /// Increment the counter by `value` (defaults to 1).
+  ///
+  /// Tags passed in will take precedence over the metric's default tags.
+  ///
+  /// \param[in] value Value to increment the counter by (default=1).
+  /// \param[in] tags The map tag values that we want to record
+  void Inc(double value, const std::unordered_map<std::string, std::string> &tags);
+};  // class Counter
+
+class Sum : public Metric {
+ public:
+  /// A sum up of the metric points.
+  ///
+  /// \param[in] name The Name of the metric.
+  /// \param[in] description The Description of the metric.
+  /// \param[in] unit The unit of the metric
+  /// \param[in] tag_keys Tag keys of the metric.
+  Sum(const std::string &name,
+      const std::string &description,
+      const std::string &unit,
+      const std::vector<std::string> &tag_keys = {});
+};  // class Sum
+
+}  // namespace ray

cpp/include/ray/api/msgpack_adaptor.h

@@ -0,0 +1,88 @@
+// Copyright 2023 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <any>
+#include <msgpack.hpp>
+
+// TODO(Larry Lian) Adapt on windows
+#ifndef _WIN32
+
+namespace msgpack {
+namespace adaptor {
+
+template <>
+struct pack<std::any> {
+  template <typename Stream>
+  msgpack::packer<Stream> &operator()(msgpack::packer<Stream> &o,
+                                      const std::any &v) const {
+    const auto &any_type = v.type();
+    if (any_type == typeid(msgpack::type::nil_t)) {
+      o.pack(std::any_cast<msgpack::type::nil_t>(v));
+    } else if (any_type == typeid(bool)) {
+      o.pack(std::any_cast<bool>(v));
+    } else if (any_type == typeid(uint64_t)) {
+      o.pack(std::any_cast<uint64_t>(v));
+    } else if (any_type == typeid(int64_t)) {
+      o.pack(std::any_cast<int64_t>(v));
+    } else if (any_type == typeid(double)) {
+      o.pack(std::any_cast<double>(v));
+    } else if (any_type == typeid(std::string)) {
+      o.pack(std::any_cast<std::string>(v));
+    } else if (any_type == typeid(std::vector<char>)) {
+      o.pack(std::any_cast<std::vector<char>>(v));
+    } else {
+      throw msgpack::type_error();
+    }
+    return o;
+  }
+};
+
+template <>
+struct convert<std::any> {
+  msgpack::object const &operator()(msgpack::object const &o, std::any &v) const {
+    switch (o.type) {
+    case type::NIL:
+      v = o.as<msgpack::type::nil_t>();
+      break;
+    case type::BOOLEAN:
+      v = o.as<bool>();
+      break;
+    case type::POSITIVE_INTEGER:
+      v = o.as<uint64_t>();
+      break;
+    case type::NEGATIVE_INTEGER:
+      v = o.as<int64_t>();
+      break;
+    case type::FLOAT32:
+    case type::FLOAT64:
+      v = o.as<double>();
+      break;
+    case type::STR:
+      v = o.as<std::string>();
+      break;
+    case type::BIN:
+      v = o.as<std::vector<char>>();
+      break;
+    default:
+      throw msgpack::type_error();
+    }
+    return o;
+  }
+};
+
+}  // namespace adaptor
+}  // namespace msgpack
+#endif

cpp/include/ray/api/object_ref.h

@@ -0,0 +1,225 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/ray_runtime_holder.h>
+#include <ray/api/serializer.h>
+#include <ray/api/type_traits.h>
+
+#include <memory>
+#include <msgpack.hpp>
+#include <utility>
+
+namespace ray {
+
+template <typename T>
+class ObjectRef;
+
+/// Common helper functions used by ObjectRef<T> and ObjectRef<void>;
+inline void CheckResult(const std::shared_ptr<msgpack::sbuffer> &packed_object) {
+  bool has_error =
+      ray::internal::Serializer::HasError(packed_object->data(), packed_object->size());
+  if (has_error) {
+    auto tp = ray::internal::Serializer::Deserialize<std::tuple<int, std::string>>(
+        packed_object->data(), packed_object->size(), 1);
+    std::string err_msg = std::get<1>(tp);
+    throw ray::internal::RayTaskException(err_msg);
+  }
+}
+
+inline void CopyAndAddReference(std::string &dest_id, const std::string &id) {
+  dest_id = id;
+  ray::internal::GetRayRuntime()->AddLocalReference(id);
+}
+
+inline void SubReference(const std::string &id) {
+  ray::internal::GetRayRuntime()->RemoveLocalReference(id);
+}
+
+/// Represents an object in the object store..
+/// \param T The type of object.
+template <typename T>
+class ObjectRef {
+ public:
+  ObjectRef();
+  ~ObjectRef();
+  // Used to identify its type.
+  static bool IsObjectRef() { return true; }
+
+  ObjectRef(ObjectRef &&rhs) {
+    SubReference(rhs.id_);
+    CopyAndAddReference(id_, rhs.id_);
+    rhs.id_ = {};
+  }
+
+  ObjectRef &operator=(ObjectRef &&rhs) {
+    if (rhs == *this) {
+      return *this;
+    }
+
+    SubReference(id_);
+    CopyAndAddReference(id_, rhs.id_);
+    // Rvalues need to be sub after add. Otherwise, the reference_count will become 0 and
+    // be deleted. Cause information such as owner to be deleted.
+    SubReference(rhs.id_);
+    rhs.id_ = {};
+    return *this;
+  }
+
+  ObjectRef(const ObjectRef &rhs) { CopyAndAddReference(id_, rhs.id_); }
+
+  ObjectRef &operator=(const ObjectRef &rhs) {
+    if (rhs == *this) {
+      return *this;
+    }
+
+    SubReference(id_);
+    CopyAndAddReference(id_, rhs.id_);
+    return *this;
+  }
+
+  ObjectRef(const std::string &id);
+
+  bool operator==(const ObjectRef<T> &object) const;
+
+  /// Get a untyped ID of the object
+  const std::string &ID() const;
+
+  /// Get the object from the object store.
+  /// This method will be blocked until the object is ready.
+  ///
+  /// \return shared pointer of the result.
+  std::shared_ptr<T> Get() const;
+
+  /// Get the object from the object store.
+  /// This method will be blocked until the object is ready.
+  ///
+  /// \param timeout_ms The maximum amount of time in milliseconds to wait before
+  /// returning.
+  /// \return shared pointer of the result.
+  std::shared_ptr<T> Get(const int &timeout_ms) const;
+
+  /// Make ObjectRef serializable
+  MSGPACK_DEFINE(id_);
+
+ private:
+  std::string id_;
+};
+
+// ---------- implementation ----------
+template <typename T>
+inline static std::shared_ptr<T> GetFromRuntime(const ObjectRef<T> &object,
+                                                const int &timeout_ms) {
+  auto packed_object = internal::GetRayRuntime()->Get(object.ID(), timeout_ms);
+  CheckResult(packed_object);
+
+  if (ray::internal::Serializer::IsXLang(packed_object->data(), packed_object->size())) {
+    return ray::internal::Serializer::Deserialize<std::shared_ptr<T>>(
+        packed_object->data(), packed_object->size(), internal::XLANG_HEADER_LEN);
+  }
+
+  if constexpr (ray::internal::is_actor_handle_v<T>) {
+    auto actor_handle = ray::internal::Serializer::Deserialize<std::string>(
+        packed_object->data(), packed_object->size());
+    return std::make_shared<T>(T::FromBytes(actor_handle));
+  }
+
+  return ray::internal::Serializer::Deserialize<std::shared_ptr<T>>(
+      packed_object->data(), packed_object->size());
+}
+
+template <typename T>
+ObjectRef<T>::ObjectRef() {}
+
+template <typename T>
+ObjectRef<T>::ObjectRef(const std::string &id) {
+  CopyAndAddReference(id_, id);
+}
+
+template <typename T>
+ObjectRef<T>::~ObjectRef() {
+  SubReference(id_);
+}
+
+template <typename T>
+inline bool ObjectRef<T>::operator==(const ObjectRef<T> &object) const {
+  return id_ == object.id_;
+}
+
+template <typename T>
+const std::string &ObjectRef<T>::ID() const {
+  return id_;
+}
+
+template <typename T>
+inline std::shared_ptr<T> ObjectRef<T>::Get() const {
+  return GetFromRuntime(*this, -1);
+}
+
+template <typename T>
+inline std::shared_ptr<T> ObjectRef<T>::Get(const int &timeout_ms) const {
+  return GetFromRuntime(*this, timeout_ms);
+}
+
+template <>
+class ObjectRef<void> {
+ public:
+  ObjectRef() = default;
+  ~ObjectRef() { SubReference(id_); }
+  // Used to identify its type.
+  static bool IsObjectRef() { return true; }
+
+  ObjectRef(const ObjectRef &rhs) { CopyAndAddReference(id_, rhs.id_); }
+
+  ObjectRef &operator=(const ObjectRef &rhs) {
+    CopyAndAddReference(id_, rhs.id_);
+    return *this;
+  }
+
+  ObjectRef(const std::string &id) { CopyAndAddReference(id_, id); }
+
+  bool operator==(const ObjectRef<void> &object) const { return id_ == object.id_; }
+
+  /// Get a untyped ID of the object
+  const std::string &ID() const { return id_; }
+
+  /// Get the object from the object store.
+  /// This method will be blocked until the object is ready.
+  ///
+  /// \return shared pointer of the result.
+  void Get() const {
+    auto packed_object = internal::GetRayRuntime()->Get(id_);
+    CheckResult(packed_object);
+  }
+
+  /// Get the object from the object store.
+  /// This method will be blocked until the object is ready.
+  ///
+  /// \param timeout_ms The maximum amount of time in milliseconds to wait before
+  /// returning.
+  ///  \return shared pointer of the result.
+  void Get(const int &timeout_ms) const {
+    auto packed_object = internal::GetRayRuntime()->Get(id_, timeout_ms);
+    CheckResult(packed_object);
+  }
+
+  /// Make ObjectRef serializable
+  MSGPACK_DEFINE(id_);
+
+ private:
+  std::string id_;
+};
+
+}  // namespace ray

cpp/include/ray/api/overload.h

@@ -0,0 +1,200 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+namespace ray {
+namespace internal {
+
+struct cv_none {};
+struct cv_const {};
+struct cv_volatile {};
+struct cv_cv {};
+struct ref_none {};
+struct ref_rval {};
+struct ref_lval {};
+
+namespace detail {
+
+//
+// underload for free functions
+//
+
+template <typename... Ts>
+struct underload_free {
+  template <typename R>
+  constexpr auto operator()(R (*f)(Ts...)) const {
+    return f;
+  }
+};
+
+//
+// underload for member functions (also free functions for convenience)
+// unspecialized template accepts any cv qualifier, any reference qualifier
+// and therefore cannot resolve an overload that differs only in qualification
+//
+
+template <typename... Ts>
+struct underload : underload_free<Ts...>,
+                   underload<cv_none, Ts...>,
+                   underload<cv_const, Ts...>,
+                   underload<cv_volatile, Ts...>,
+                   underload<cv_cv, Ts...> {
+  using underload_free<Ts...>::operator();
+  using underload<cv_none, Ts...>::operator();
+  using underload<cv_const, Ts...>::operator();
+  using underload<cv_volatile, Ts...>::operator();
+  using underload<cv_cv, Ts...>::operator();
+  static constexpr bool is_overload_v = true;
+};
+
+//
+// specializations with cv tag
+// accept any reference qualifier
+// cannot resolve overload that differs only in reference qualification
+//
+
+template <typename... Ts>
+struct underload<cv_none, Ts...> : underload<cv_none, ref_none, Ts...>,
+                                   underload<cv_none, ref_rval, Ts...>,
+                                   underload<cv_none, ref_lval, Ts...> {
+  using underload<cv_none, ref_none, Ts...>::operator();
+  using underload<cv_none, ref_rval, Ts...>::operator();
+  using underload<cv_none, ref_lval, Ts...>::operator();
+};
+
+template <typename... Ts>
+struct underload<cv_const, Ts...> : underload<cv_const, ref_none, Ts...>,
+                                    underload<cv_const, ref_rval, Ts...>,
+                                    underload<cv_const, ref_lval, Ts...> {
+  using underload<cv_const, ref_none, Ts...>::operator();
+  using underload<cv_const, ref_rval, Ts...>::operator();
+  using underload<cv_const, ref_lval, Ts...>::operator();
+};
+
+template <typename... Ts>
+struct underload<cv_volatile, Ts...> : underload<cv_volatile, ref_none, Ts...>,
+                                       underload<cv_volatile, ref_rval, Ts...>,
+                                       underload<cv_volatile, ref_lval, Ts...> {
+  using underload<cv_volatile, ref_none, Ts...>::operator();
+  using underload<cv_volatile, ref_rval, Ts...>::operator();
+  using underload<cv_volatile, ref_lval, Ts...>::operator();
+};
+
+template <typename... Ts>
+struct underload<cv_cv, Ts...> : underload<cv_cv, ref_none, Ts...>,
+                                 underload<cv_cv, ref_rval, Ts...>,
+                                 underload<cv_cv, ref_lval, Ts...> {
+  using underload<cv_cv, ref_none, Ts...>::operator();
+  using underload<cv_cv, ref_rval, Ts...>::operator();
+  using underload<cv_cv, ref_lval, Ts...>::operator();
+};
+
+//
+// specializations with reference tag
+// accept any cv qualifier
+// cannot resolve overload that differs only in cv qualification
+//
+
+template <typename... Ts>
+struct underload<ref_none, Ts...> : underload<cv_none, ref_none, Ts...>,
+                                    underload<cv_const, ref_none, Ts...>,
+                                    underload<cv_volatile, ref_none, Ts...>,
+                                    underload<cv_cv, ref_none, Ts...> {
+  using underload<cv_none, ref_none, Ts...>::operator();
+  using underload<cv_const, ref_none, Ts...>::operator();
+  using underload<cv_volatile, ref_none, Ts...>::operator();
+  using underload<cv_cv, ref_none, Ts...>::operator();
+};
+
+template <typename... Ts>
+struct underload<ref_rval, Ts...> : underload<cv_none, ref_rval, Ts...>,
+                                    underload<cv_const, ref_rval, Ts...>,
+                                    underload<cv_volatile, ref_rval, Ts...>,
+                                    underload<cv_cv, ref_rval, Ts...> {
+  using underload<cv_none, ref_rval, Ts...>::operator();
+  using underload<cv_const, ref_rval, Ts...>::operator();
+  using underload<cv_volatile, ref_rval, Ts...>::operator();
+  using underload<cv_cv, ref_rval, Ts...>::operator();
+};
+
+template <typename... Ts>
+struct underload<ref_lval, Ts...> : underload<cv_none, ref_lval, Ts...>,
+                                    underload<cv_const, ref_lval, Ts...>,
+                                    underload<cv_volatile, ref_lval, Ts...>,
+                                    underload<cv_cv, ref_lval, Ts...> {
+  using underload<cv_none, ref_lval, Ts...>::operator();
+  using underload<cv_const, ref_lval, Ts...>::operator();
+  using underload<cv_volatile, ref_lval, Ts...>::operator();
+  using underload<cv_cv, ref_lval, Ts...>::operator();
+};
+
+//
+// specializations with cv tag followed by reference tag
+//
+
+#define UNDERLOAD(CV_TAG, REF_TAG, QUALIFIER)                     \
+  template <typename... Ts>                                       \
+  struct underload<CV_TAG, REF_TAG, Ts...> {                      \
+    template <typename R, typename T>                             \
+    constexpr auto operator()(R (T::*f)(Ts...) QUALIFIER) const { \
+      return f;                                                   \
+    }                                                             \
+    static constexpr bool is_cv_v = true;                         \
+  };
+UNDERLOAD(cv_none, ref_none, )
+UNDERLOAD(cv_const, ref_none, const)
+UNDERLOAD(cv_volatile, ref_none, volatile)
+UNDERLOAD(cv_cv, ref_none, const volatile)
+UNDERLOAD(cv_none, ref_rval, &&)
+UNDERLOAD(cv_const, ref_rval, const &&)
+UNDERLOAD(cv_volatile, ref_rval, volatile &&)
+UNDERLOAD(cv_cv, ref_rval, const volatile &&)
+UNDERLOAD(cv_none, ref_lval, &)
+UNDERLOAD(cv_const, ref_lval, const &)
+UNDERLOAD(cv_volatile, ref_lval, volatile &)
+UNDERLOAD(cv_cv, ref_lval, const volatile &)
+#undef UNDERLOAD
+
+//
+// specializations with reference tag followed by cv tag
+//
+
+#define UNDERLOAD(CV_TAG, REF_TAG) \
+  template <typename... Ts>        \
+  struct underload<REF_TAG, CV_TAG, Ts...> : underload<CV_TAG, REF_TAG, Ts...> {};
+UNDERLOAD(cv_none, ref_none)
+UNDERLOAD(cv_const, ref_none)
+UNDERLOAD(cv_volatile, ref_none)
+UNDERLOAD(cv_cv, ref_none)
+UNDERLOAD(cv_none, ref_rval)
+UNDERLOAD(cv_const, ref_rval)
+UNDERLOAD(cv_volatile, ref_rval)
+UNDERLOAD(cv_cv, ref_rval)
+UNDERLOAD(cv_none, ref_lval)
+UNDERLOAD(cv_const, ref_lval)
+UNDERLOAD(cv_volatile, ref_lval)
+UNDERLOAD(cv_cv, ref_lval)
+#undef UNDERLOAD
+
+}  // namespace detail
+
+template <typename... Ts>
+constexpr detail::underload_free<Ts...> underload_free{};
+
+template <typename... Ts>
+constexpr detail::underload<Ts...> underload{};
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/ray_config.h

@@ -0,0 +1,74 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <ray/api/ray_exception.h>
+#include <ray/api/runtime_env.h>
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "boost/optional.hpp"
+
+namespace ray {
+
+enum class ActorLifetime {
+  NON_DETACHED,
+  DETACHED,
+};
+
+class RayConfig {
+ public:
+  // The address of the Ray cluster to connect to.
+  // If not provided, it will be initialized from environment variable "RAY_ADDRESS" by
+  // default.
+  std::string address = "";
+
+  // Whether or not to run this application in a local mode. This is used for debugging.
+  bool local_mode = false;
+
+  // An array of directories or dynamic library files that specify the search path for
+  // user code. This parameter is not used when the application runs in local mode.
+  // Only searching the top level under a directory.
+  std::vector<std::string> code_search_path;
+
+  // The command line args to be appended as parameters of the `ray start` command. It
+  // takes effect only if Ray head is started by a driver. Run `ray start --help` for
+  // details.
+  std::vector<std::string> head_args = {};
+
+  // The default actor lifetime type, `DETACHED` or `NON_DETACHED`.
+  ActorLifetime default_actor_lifetime = ActorLifetime::NON_DETACHED;
+
+  // The job level runtime environments.
+  boost::optional<RuntimeEnv> runtime_env;
+
+  /* The following are unstable parameters and their use is discouraged. */
+
+  // Prevents external clients without the username from connecting to Redis if provided.
+  boost::optional<std::string> redis_username_;
+
+  // Prevents external clients without the password from connecting to Redis if provided.
+  boost::optional<std::string> redis_password_;
+
+  // A specific flag for internal `default_worker`. Please don't use it in user code.
+  bool is_worker_ = false;
+
+  // A namespace is a logical grouping of jobs and named actors.
+  std::string ray_namespace = "";
+};
+
+}  // namespace ray

cpp/include/ray/api/ray_exception.h

@@ -0,0 +1,68 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <exception>
+#include <string>
+
+namespace ray {
+namespace internal {
+
+class RayException : public std::exception {
+ public:
+  RayException(const std::string &msg) : msg_(msg){};
+
+  const char *what() const noexcept override { return msg_.c_str(); };
+
+  std::string msg_;
+};
+
+class RayActorException : public RayException {
+ public:
+  RayActorException(const std::string &msg) : RayException(msg){};
+};
+
+class RayTaskException : public RayException {
+ public:
+  RayTaskException(const std::string &msg) : RayException(msg){};
+};
+
+class RayWorkerException : public RayException {
+ public:
+  RayWorkerException(const std::string &msg) : RayException(msg){};
+};
+
+class UnreconstructableException : public RayException {
+ public:
+  UnreconstructableException(const std::string &msg) : RayException(msg){};
+};
+
+class RayFunctionNotFound : public RayException {
+ public:
+  RayFunctionNotFound(const std::string &msg) : RayException(msg){};
+};
+
+class RayRuntimeEnvException : public RayException {
+ public:
+  RayRuntimeEnvException(const std::string &msg) : RayException(msg){};
+};
+
+class RayTimeoutException : public RayException {
+ public:
+  RayTimeoutException(const std::string &msg) : RayException(msg){};
+};
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/ray_remote.h

@@ -0,0 +1,83 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/function_manager.h>
+#include <ray/api/overload.h>
+
+#include "boost/utility/string_view.hpp"
+
+namespace ray {
+namespace internal {
+
+inline static std::vector<boost::string_view> GetFunctionNames(boost::string_view str) {
+  std::vector<boost::string_view> output;
+  size_t first = 0;
+
+  while (first < str.size()) {
+    auto second = str.find_first_of(",", first);
+
+    if (first != second) {
+      auto sub_str = str.substr(first, second - first);
+      if (sub_str.find_first_of('(') != boost::string_view::npos) {
+        second = str.find_first_of(")", first) + 1;
+      }
+      if (str[first] == ' ') {
+        first++;
+      }
+
+      auto name = str.substr(first, second - first);
+      if (name.back() == ' ') {
+        name.remove_suffix(1);
+      }
+      output.emplace_back(name);
+    }
+
+    if (second == boost::string_view::npos) break;
+
+    first = second + 1;
+  }
+
+  return output;
+}
+
+template <typename T, typename... U>
+inline static int RegisterRemoteFunctions(const T &t, U... u) {
+  int index = 0;
+  const auto func_names = GetFunctionNames(t);
+  (void)std::initializer_list<int>{
+      (FunctionManager::Instance().RegisterRemoteFunction(
+           std::string(func_names[index].data(), func_names[index].length()), u),
+       index++,
+       0)...};
+  return 0;
+}
+
+#define CONCATENATE_DIRECT(s1, s2) s1##s2
+#define CONCATENATE(s1, s2) CONCATENATE_DIRECT(s1, s2)
+#ifdef _MSC_VER
+#define ANONYMOUS_VARIABLE(str) CONCATENATE(str, __COUNTER__)
+#else
+#define ANONYMOUS_VARIABLE(str) CONCATENATE(str, __LINE__)
+#endif
+}  // namespace internal
+
+#define RAY_REMOTE(...)                 \
+  inline auto ANONYMOUS_VARIABLE(var) = \
+      ray::internal::RegisterRemoteFunctions(#__VA_ARGS__, __VA_ARGS__);
+
+#define RAY_FUNC(f, ...) ray::internal::underload<__VA_ARGS__>(f)
+
+}  // namespace ray

cpp/include/ray/api/ray_runtime.h

@@ -0,0 +1,105 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/common_types.h>
+#include <ray/api/task_options.h>
+#include <ray/api/xlang_function.h>
+
+#include <cstdint>
+#include <memory>
+#include <msgpack.hpp>
+#include <typeinfo>
+#include <vector>
+
+namespace ray {
+namespace internal {
+
+struct RemoteFunctionHolder {
+  RemoteFunctionHolder() = default;
+  RemoteFunctionHolder(const std::string &module_name,
+                       const std::string &function_name,
+                       const std::string &class_name = "",
+                       LangType lang_type = LangType::CPP)
+      : module_name_(module_name),
+        function_name_(function_name),
+        class_name_(class_name),
+        lang_type_(lang_type) {}
+
+  RemoteFunctionHolder(std::string func_name) {
+    if (func_name.empty()) {
+      throw RayException(
+          "Function not found. Please use RAY_REMOTE to register this function.");
+    }
+    function_name_ = std::move(func_name);
+  }
+
+  std::string module_name_;
+  std::string function_name_;
+  std::string class_name_;
+  LangType lang_type_ = LangType::CPP;
+};
+
+class RayRuntime {
+ public:
+  virtual std::string Put(std::shared_ptr<msgpack::sbuffer> data) = 0;
+  virtual std::shared_ptr<msgpack::sbuffer> Get(const std::string &id) = 0;
+
+  virtual std::vector<std::shared_ptr<msgpack::sbuffer>> Get(
+      const std::vector<std::string> &ids) = 0;
+
+  virtual std::shared_ptr<msgpack::sbuffer> Get(const std::string &object_id,
+                                                const int &timeout_ms) = 0;
+
+  virtual std::vector<std::shared_ptr<msgpack::sbuffer>> Get(
+      const std::vector<std::string> &ids, const int &timeout_ms) = 0;
+
+  virtual std::vector<bool> Wait(const std::vector<std::string> &ids,
+                                 int num_objects,
+                                 int timeout_ms) = 0;
+
+  virtual std::string Call(const RemoteFunctionHolder &remote_function_holder,
+                           std::vector<TaskArg> &args,
+                           const CallOptions &task_options) = 0;
+  virtual std::string CreateActor(const RemoteFunctionHolder &remote_function_holder,
+                                  std::vector<TaskArg> &args,
+                                  const ActorCreationOptions &create_options) = 0;
+  virtual std::string CallActor(const RemoteFunctionHolder &remote_function_holder,
+                                const std::string &actor,
+                                std::vector<TaskArg> &args,
+                                const CallOptions &call_options) = 0;
+  virtual void AddLocalReference(const std::string &id) = 0;
+  virtual void RemoveLocalReference(const std::string &id) = 0;
+  virtual std::string GetActorId(const std::string &actor_name,
+                                 const std::string &ray_namespace) = 0;
+  virtual void KillActor(const std::string &str_actor_id, bool no_restart) = 0;
+  virtual void ExitActor() = 0;
+  virtual ray::PlacementGroup CreatePlacementGroup(
+      const ray::PlacementGroupCreationOptions &create_options) = 0;
+  virtual void RemovePlacementGroup(const std::string &group_id) = 0;
+  virtual bool WaitPlacementGroupReady(const std::string &group_id,
+                                       int64_t timeout_seconds) = 0;
+  virtual bool WasCurrentActorRestarted() = 0;
+  virtual std::vector<PlacementGroup> GetAllPlacementGroups() = 0;
+  virtual PlacementGroup GetPlacementGroupById(const std::string &id) = 0;
+  virtual PlacementGroup GetPlacementGroup(const std::string &name) = 0;
+  virtual bool IsLocalMode() { return false; }
+  virtual std::string GetNamespace() = 0;
+  virtual std::string SerializeActorHandle(const std::string &actor_id) = 0;
+  virtual std::string DeserializeAndRegisterActorHandle(
+      const std::string &serialized_actor_handle) = 0;
+};
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/ray_runtime_holder.h

@@ -0,0 +1,48 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/ray_runtime.h>
+
+namespace ray {
+namespace internal {
+
+struct RayRuntimeHolder {
+  static RayRuntimeHolder &Instance() {
+    static RayRuntimeHolder instance;
+    return instance;
+  }
+
+  void Init(std::shared_ptr<RayRuntime> runtime) { runtime_ = runtime; }
+
+  std::shared_ptr<RayRuntime> Runtime() { return runtime_; }
+
+ private:
+  RayRuntimeHolder() = default;
+  ~RayRuntimeHolder() = default;
+  RayRuntimeHolder(RayRuntimeHolder const &) = delete;
+  RayRuntimeHolder(RayRuntimeHolder &&) = delete;
+  RayRuntimeHolder &operator=(RayRuntimeHolder const &) = delete;
+  RayRuntimeHolder &operator=(RayRuntimeHolder &&) = delete;
+
+  std::shared_ptr<RayRuntime> runtime_;
+};
+
+inline static std::shared_ptr<RayRuntime> GetRayRuntime() {
+  return RayRuntimeHolder::Instance().Runtime();
+}
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/runtime_env.h

@@ -0,0 +1,100 @@
+// Copyright 2022 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <ray/api/ray_exception.h>
+
+#include <string>
+
+#include "nlohmann/json.hpp"
+
+namespace ray {
+
+/// This class provides interfaces of setting runtime environments for job/actor/task.
+class RuntimeEnv {
+ public:
+  /// Set a runtime env field by name and Object.
+  /// \param[in] name The runtime env plugin name.
+  /// \param[in] value An object with primitive data type or jsonable type of
+  /// nlohmann/json.
+  template <typename T>
+  void Set(const std::string &name, const T &value);
+
+  /// Get the object of a runtime env field.
+  /// \param[in] name The runtime env plugin name.
+  template <typename T>
+  T Get(const std::string &name) const;
+
+  /// Set a runtime env field by name and json string.
+  /// \param[in] name The runtime env plugin name.
+  /// \param[in] json_str A json string represents the runtime env field.
+  void SetJsonStr(const std::string &name, const std::string &json_str);
+
+  /// Get the json string of a runtime env field.
+  /// \param[in] name The runtime env plugin name.
+  std::string GetJsonStr(const std::string &name) const;
+
+  /// Whether a field is contained.
+  /// \param[in] name The runtime env plugin name.
+  bool Contains(const std::string &name) const;
+
+  /// Remove a field by name.
+  /// \param[in] name The runtime env plugin name.
+  /// \return true if remove an existing field, otherwise false.
+  bool Remove(const std::string &name);
+
+  /// Whether the runtime env is empty.
+  bool Empty() const;
+
+  /// Serialize the runtime env to string.
+  std::string Serialize() const;
+
+  /// Serialize the runtime env to RuntimeEnvInfo.
+  std::string SerializeToRuntimeEnvInfo() const;
+
+  /// Deserialize the runtime env from string.
+  /// \return The deserialized RuntimeEnv instance.
+  static RuntimeEnv Deserialize(const std::string &serialized_runtime_env);
+
+ private:
+  nlohmann::json fields_;
+};
+
+// --------- inline implementation ------------
+
+template <typename T>
+inline void RuntimeEnv::Set(const std::string &name, const T &value) {
+  try {
+    nlohmann::json value_j = value;
+    fields_[name] = value_j;
+  } catch (std::exception &e) {
+    throw ray::internal::RayRuntimeEnvException("Failed to set the field " + name + ": " +
+                                                e.what());
+  }
+}
+
+template <typename T>
+inline T RuntimeEnv::Get(const std::string &name) const {
+  if (!Contains(name)) {
+    throw ray::internal::RayRuntimeEnvException("The field " + name + " not found.");
+  }
+  try {
+    return fields_[name].get<T>();
+  } catch (std::exception &e) {
+    throw ray::internal::RayRuntimeEnvException("Failed to get the field " + name + ": " +
+                                                e.what());
+  }
+}
+
+}  // namespace ray

cpp/include/ray/api/serializer.h

@@ -0,0 +1,86 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/msgpack_adaptor.h>
+#include <ray/api/ray_exception.h>
+#include <ray/api/xlang_function.h>
+
+#include <msgpack.hpp>
+
+namespace ray {
+namespace internal {
+
+class Serializer {
+ public:
+  template <typename T>
+  static msgpack::sbuffer Serialize(const T &t) {
+    msgpack::sbuffer buffer;
+    msgpack::pack(buffer, t);
+    return buffer;
+  }
+
+  static msgpack::sbuffer Serialize(const char *data, size_t size) {
+    msgpack::sbuffer buffer;
+    msgpack::packer<msgpack::sbuffer> packer(&buffer);
+    packer.pack_bin(size);
+    packer.pack_bin_body(data, size);
+    return buffer;
+  }
+
+  template <typename T>
+  static T Deserialize(const char *data, size_t size) {
+    msgpack::unpacked unpacked;
+    msgpack::unpack(unpacked, data, size);
+    return unpacked.get().as<T>();
+  }
+
+  template <typename T>
+  static T Deserialize(const char *data, size_t size, size_t offset) {
+    return Deserialize<T>(data + offset, size - offset);
+  }
+
+  template <typename T>
+  static T Deserialize(const char *data, size_t size, size_t *off) {
+    msgpack::unpacked unpacked = msgpack::unpack(data, size, *off);
+    return unpacked.get().as<T>();
+  }
+
+  template <typename T>
+  static std::pair<bool, T> DeserializeWhenNil(const char *data, size_t size) {
+    T val;
+    size_t off = 0;
+    msgpack::unpacked unpacked = msgpack::unpack(data, size, off);
+    if (!unpacked.get().convert_if_not_nil(val)) {
+      return {false, {}};
+    }
+
+    return {true, val};
+  }
+
+  static bool HasError(char *data, size_t size) {
+    msgpack::unpacked unpacked = msgpack::unpack(data, size);
+    return unpacked.get().is_nil() && size > 1;
+  }
+
+  static bool IsXLang(char *data, size_t size) {
+    msgpack::unpacked unpacked = msgpack::unpack(data, size);
+    return unpacked.get().type == msgpack::type::POSITIVE_INTEGER &&
+           size >= XLANG_HEADER_LEN;
+  }
+};
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/static_check.h

@@ -0,0 +1,73 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/object_ref.h>
+
+#include <boost/callable_traits.hpp>
+#include <type_traits>
+
+namespace ray {
+namespace internal {
+
+template <typename T>
+struct FilterArgType {
+  using type = T;
+};
+
+template <typename T>
+struct FilterArgType<ObjectRef<T>> {
+  using type = T;
+};
+
+template <typename T>
+struct FilterArgType<ObjectRef<T> &> {
+  using type = T;
+};
+
+template <typename T>
+struct FilterArgType<ObjectRef<T> &&> {
+  using type = T;
+};
+
+template <typename T>
+struct FilterArgType<const ObjectRef<T> &> {
+  using type = T;
+};
+
+template <typename F, typename... Args>
+struct is_invocable
+    : std::is_constructible<
+          std::function<void(Args...)>,
+          std::reference_wrapper<typename std::remove_reference<F>::type>> {};
+
+template <typename Function, typename... Args>
+inline std::enable_if_t<!std::is_member_function_pointer<Function>::value> StaticCheck() {
+  static_assert(is_invocable<Function, typename FilterArgType<Args>::type...>::value ||
+                    is_invocable<Function, Args...>::value,
+                "arguments not match");
+}
+
+template <typename Function, typename... Args>
+inline std::enable_if_t<std::is_member_function_pointer<Function>::value> StaticCheck() {
+  using ActorType = boost::callable_traits::class_of_t<Function>;
+  static_assert(
+      is_invocable<Function, ActorType &, typename FilterArgType<Args>::type...>::value ||
+          is_invocable<Function, ActorType &, Args...>::value,
+      "arguments not match");
+}
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/task_caller.h

@@ -0,0 +1,109 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/runtime_env.h>
+#include <ray/api/static_check.h>
+#include <ray/api/task_options.h>
+
+namespace ray {
+namespace internal {
+
+template <typename F>
+class TaskCaller {
+ public:
+  TaskCaller();
+
+  TaskCaller(RayRuntime *runtime, RemoteFunctionHolder remote_function_holder);
+
+  template <typename... Args>
+  ObjectRef<boost::callable_traits::return_type_t<F>> Remote(Args &&...args);
+
+  TaskCaller &SetName(std::string name) {
+    task_options_.name = std::move(name);
+    return *this;
+  }
+
+  TaskCaller &SetResources(std::unordered_map<std::string, double> resources) {
+    task_options_.resources = std::move(resources);
+    return *this;
+  }
+
+  TaskCaller &SetResource(std::string name, double value) {
+    task_options_.resources.emplace(std::move(name), value);
+    return *this;
+  }
+
+  TaskCaller &SetPlacementGroup(PlacementGroup group, int bundle_index) {
+    task_options_.group = group;
+    task_options_.bundle_index = bundle_index;
+    return *this;
+  }
+
+  TaskCaller &SetRuntimeEnv(const ray::RuntimeEnv &runtime_env) {
+    task_options_.serialized_runtime_env_info = runtime_env.SerializeToRuntimeEnvInfo();
+    return *this;
+  }
+
+ private:
+  RayRuntime *runtime_;
+  RemoteFunctionHolder remote_function_holder_{};
+  std::string function_name_;
+  std::vector<TaskArg> args_;
+  CallOptions task_options_;
+};
+
+// ---------- implementation ----------
+
+template <typename F>
+TaskCaller<F>::TaskCaller() {}
+
+template <typename F>
+TaskCaller<F>::TaskCaller(RayRuntime *runtime,
+                          RemoteFunctionHolder remote_function_holder)
+    : runtime_(runtime), remote_function_holder_(std::move(remote_function_holder)) {}
+
+template <typename F>
+template <typename... Args>
+ObjectRef<boost::callable_traits::return_type_t<F>> TaskCaller<F>::Remote(
+    Args &&...args) {
+  CheckTaskOptions(task_options_.resources);
+
+  if constexpr (is_x_lang_v<F>) {
+    using ArgsTuple = std::tuple<Args...>;
+    Arguments::WrapArgs<ArgsTuple>(remote_function_holder_.lang_type_,
+                                   &args_,
+                                   std::make_index_sequence<sizeof...(Args)>{},
+                                   std::forward<Args>(args)...);
+  } else {
+    StaticCheck<F, Args...>();
+    using ArgsTuple = RemoveReference_t<boost::callable_traits::args_t<F>>;
+    Arguments::WrapArgs<ArgsTuple>(remote_function_holder_.lang_type_,
+                                   &args_,
+                                   std::make_index_sequence<sizeof...(Args)>{},
+                                   std::forward<Args>(args)...);
+  }
+
+  auto returned_object_id = runtime_->Call(remote_function_holder_, args_, task_options_);
+  using ReturnType = boost::callable_traits::return_type_t<F>;
+  auto return_ref = ObjectRef<ReturnType>(returned_object_id);
+  // The core worker will add an initial ref to each return ID to keep it in
+  // scope. Now that we've created the frontend ObjectRef, remove this initial
+  // ref.
+  runtime_->RemoveLocalReference(returned_object_id);
+  return return_ref;
+}
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/task_options.h

@@ -0,0 +1,116 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/ray_exception.h>
+
+#include <cmath>
+
+namespace ray {
+namespace internal {
+
+inline void CheckTaskOptions(const std::unordered_map<std::string, double> &resources) {
+  for (auto &pair : resources) {
+    if (pair.first.empty() || pair.second == 0) {
+      throw RayException("Resource values should be positive. Specified resource: " +
+                         pair.first + " = " + std::to_string(pair.second) + ".");
+    }
+    // Note: A resource value should be an integer if it is greater than 1.0.
+    // e.g. 3.0 is a valid resource value, but 3.5 is not.
+    double intpart;
+    if (pair.second > 1 && std::modf(pair.second, &intpart) != 0.0) {
+      throw RayException(
+          "A resource value should be an integer if it is greater than 1.0. Specified "
+          "resource: " +
+          pair.first + " = " + std::to_string(pair.second) + ".");
+    }
+  }
+}
+
+}  // namespace internal
+
+enum class PlacementStrategy {
+  PACK = 0,
+  SPREAD = 1,
+  STRICT_PACK = 2,
+  STRICT_SPREAD = 3,
+  UNRECOGNIZED = -1
+};
+
+enum PlacementGroupState {
+  PENDING = 0,
+  PREPARED = 1,
+  CREATED = 2,
+  REMOVED = 3,
+  RESCHEDULING = 4,
+  UNRECOGNIZED = -1,
+};
+
+struct PlacementGroupCreationOptions {
+  std::string name;
+  std::vector<std::unordered_map<std::string, double>> bundles;
+  PlacementStrategy strategy;
+};
+
+class PlacementGroup {
+ public:
+  PlacementGroup() = default;
+  PlacementGroup(std::string id,
+                 PlacementGroupCreationOptions options,
+                 PlacementGroupState state = PlacementGroupState::UNRECOGNIZED)
+      : id_(std::move(id)), options_(std::move(options)), state_(state) {}
+  std::string GetID() const { return id_; }
+  std::string GetName() { return options_.name; }
+  std::vector<std::unordered_map<std::string, double>> GetBundles() {
+    return options_.bundles;
+  }
+  ray::PlacementGroupState GetState() { return state_; }
+  PlacementStrategy GetStrategy() { return options_.strategy; }
+  bool Wait(int timeout_seconds) { return callback_(id_, timeout_seconds); }
+  void SetWaitCallbak(std::function<bool(const std::string &, int)> callback) {
+    callback_ = std::move(callback);
+  }
+  bool Empty() const { return id_.empty(); }
+
+ private:
+  std::string id_;
+  PlacementGroupCreationOptions options_;
+  PlacementGroupState state_;
+  std::function<bool(const std::string &, int)> callback_;
+};
+
+namespace internal {
+
+struct CallOptions {
+  std::string name;
+  std::unordered_map<std::string, double> resources;
+  PlacementGroup group;
+  int bundle_index;
+  std::string serialized_runtime_env_info;
+};
+
+struct ActorCreationOptions {
+  std::string name;
+  std::string ray_namespace;
+  std::unordered_map<std::string, double> resources;
+  int max_restarts = 0;
+  int max_concurrency = 1;
+  PlacementGroup group;
+  int bundle_index;
+  std::string serialized_runtime_env_info;
+};
+}  // namespace internal
+
+}  // namespace ray

cpp/include/ray/api/type_traits.h

@@ -0,0 +1,88 @@
+// Copyright 2017 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <type_traits>
+
+namespace ray {
+namespace internal {
+
+template <typename>
+struct RemoveFirst;
+
+template <class First, class... Second>
+struct RemoveFirst<std::tuple<First, Second...>> {
+  using type = std::tuple<Second...>;
+};
+
+template <class Tuple>
+using RemoveFirst_t = typename RemoveFirst<Tuple>::type;
+
+template <typename>
+struct RemoveReference;
+
+template <class... T>
+struct RemoveReference<std::tuple<T...>> {
+  using type = std::tuple<std::remove_const_t<std::remove_reference_t<T>>...>;
+};
+
+template <class Tuple>
+using RemoveReference_t = typename RemoveReference<Tuple>::type;
+
+template <class, class = void>
+struct is_object_ref_t : std::false_type {};
+
+template <class T>
+struct is_object_ref_t<T, std::void_t<decltype(std::declval<T>().IsObjectRef())>>
+    : std::true_type {};
+
+template <typename T>
+auto constexpr is_object_ref_v = is_object_ref_t<T>::value;
+
+template <class, class = void>
+struct is_actor_handle_t : std::false_type {};
+
+template <class T>
+struct is_actor_handle_t<T, std::void_t<decltype(std::declval<T>().IsActorHandle())>>
+    : std::true_type {};
+
+template <typename T>
+auto constexpr is_actor_handle_v = is_actor_handle_t<T>::value;
+
+template <class, class = void>
+struct is_python_t : std::false_type {};
+
+template <class T>
+struct is_python_t<T, std::void_t<decltype(std::declval<T>().IsPython())>>
+    : std::true_type {};
+
+template <typename T>
+auto constexpr is_python_v = is_python_t<T>::value;
+
+template <class, class = void>
+struct is_java_t : std::false_type {};
+
+template <class T>
+struct is_java_t<T, std::void_t<decltype(std::declval<T>().IsJava())>> : std::true_type {
+};
+
+template <typename T>
+auto constexpr is_java_v = is_java_t<T>::value;
+
+template <typename T>
+auto constexpr is_x_lang_v = is_java_v<T> || is_python_v<T>;
+
+}  // namespace internal
+}  // namespace ray

cpp/include/ray/api/wait_result.h

@@ -0,0 +1,37 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ray/api/object_ref.h>
+
+#include <list>
+
+namespace ray {
+
+/// \param T The type of object.
+template <typename T>
+class WaitResult {
+ public:
+  /// The object id list of ready objects
+  std::list<ObjectRef<T>> ready;
+  /// The object id list of unready objects
+  std::list<ObjectRef<T>> unready;
+  WaitResult(){};
+  WaitResult(std::list<ObjectRef<T>> &&ready_objects,
+             std::list<ObjectRef<T>> &&unready_objects)
+      : ready(ready_objects), unready(unready_objects){};
+};
+
+}  // namespace ray

cpp/include/ray/api/xlang_function.h

@@ -0,0 +1,85 @@
+// Copyright 2020-2021 The Ray Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <string_view>
+
+namespace ray {
+
+template <typename R>
+struct PyFunction {
+  bool IsPython() { return true; }
+  R operator()() { return {}; }
+
+  std::string module_name;
+  std::string function_name;
+};
+
+struct PyActorClass {
+  bool IsPython() { return true; }
+  void operator()() {}
+
+  std::string module_name;
+  std::string class_name;
+  std::string function_name = "__init__";
+};
+
+template <typename R>
+struct PyActorMethod {
+  bool IsPython() { return true; }
+  R operator()() { return {}; }
+
+  std::string function_name;
+};
+
+struct JavaActorClass {
+  bool IsJava() { return true; }
+  void operator()() {}
+  std::string class_name;
+  std::string module_name = "";
+  std::string function_name = "<init>";
+};
+template <typename R>
+struct JavaActorMethod {
+  bool IsJava() { return true; }
+  R operator()() { return {}; }
+  std::string function_name;
+};
+
+template <typename R>
+struct JavaFunction {
+  bool IsJava() { return true; }
+  R operator()() { return {}; }
+  std::string class_name;
+  std::string function_name;
+};
+
+namespace internal {
+
+enum class LangType {
+  CPP,
+  PYTHON,
+  JAVA,
+};
+
+inline constexpr size_t XLANG_HEADER_LEN = 9;
+inline constexpr std::string_view METADATA_STR_DUMMY = "__RAY_DUMMY__";
+inline constexpr std::string_view METADATA_STR_RAW = "RAW";
+inline constexpr std::string_view METADATA_STR_XLANG = "XLANG";
+
+}  // namespace internal
+
+}  // namespace ray