chore: import upstream snapshot with attribution

apps/cpp_rpc/CMakeLists.txt

@@ -0,0 +1,82 @@
+cmake_policy(SET CMP0069 NEW) # suppress CMake warning about IPO
+
+set(TVM_RPC_SOURCES
+  main.cc
+  rpc_env.cc
+  rpc_server.cc
+)
+
+set(TVM_RPC_LINKER_LIBS "")
+
+if(WIN32)
+  list(APPEND TVM_RPC_SOURCES win32_process.cc)
+endif()
+
+# Set output to same directory as the other TVM libs
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+add_executable(tvm_rpc ${TVM_RPC_SOURCES})
+
+include(CheckIPOSupported)
+check_ipo_supported(RESULT result OUTPUT output)
+if(result)
+  set_property(TARGET tvm_rpc PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE)
+endif()
+
+if(WIN32)
+  target_compile_definitions(tvm_rpc PUBLIC -DNOMINMAX)
+endif()
+
+if (OS)
+   if (OS STREQUAL "Linux")
+      set_property(TARGET tvm_rpc PROPERTY LINK_FLAGS -lpthread)
+   endif()
+endif()
+
+if(USE_OPENCL)
+   if (ANDROID_ABI)
+     if(DEFINED ENV{ANDROID_NDK_MAJOR})
+       if($ENV{ANDROID_NDK_MAJOR} VERSION_LESS "23")
+         set_property(TARGET tvm_rpc PROPERTY LINK_FLAGS -fuse-ld=gold)
+       endif()
+     endif()
+   endif()
+endif()
+
+target_include_directories(
+  tvm_rpc
+  PUBLIC "../../include"
+)
+
+target_link_libraries(tvm_rpc PUBLIC tvm_ffi_header)
+
+if (BUILD_FOR_ANDROID AND USE_HEXAGON)
+  get_hexagon_sdk_property("${USE_HEXAGON_SDK}" "${USE_HEXAGON_ARCH}"
+    DSPRPC_LIB DSPRPC_LIB_DIRS
+  )
+  if(DSPRPC_LIB_DIRS)
+    link_directories(${DSPRPC_LIB_DIRS})
+  else()
+    message(WARNING "Could not locate some Hexagon SDK components")
+  endif()
+  list(APPEND TVM_RPC_LINKER_LIBS cdsprpc log)
+endif()
+
+if(BUILD_STATIC_RUNTIME)
+  foreach(lib ${TVM_RUNTIME_BACKEND_LIBS})
+    if(MSVC)
+      list(APPEND TVM_RPC_LINKER_LIBS "/WHOLEARCHIVE:$<TARGET_FILE:${lib}>")
+    elseif(APPLE)
+      list(APPEND TVM_RPC_LINKER_LIBS "-Wl,-force_load,$<TARGET_FILE:${lib}>")
+    else()
+      list(APPEND TVM_RPC_LINKER_LIBS "-Wl,--whole-archive" "${lib}" "-Wl,--no-whole-archive")
+    endif()
+  endforeach()
+else()
+  if(NOT MSVC AND NOT APPLE)
+    list(APPEND TVM_RPC_LINKER_LIBS tvm_runtime "-Wl,--no-as-needed" ${TVM_RUNTIME_BACKEND_LIBS} "-Wl,--as-needed")
+  else()
+    list(APPEND TVM_RPC_LINKER_LIBS tvm_runtime ${TVM_RUNTIME_BACKEND_LIBS})
+  endif()
+endif()
+
+target_link_libraries(tvm_rpc PRIVATE ${TVM_RPC_LINKER_LIBS})

apps/cpp_rpc/README.md

@@ -0,0 +1,83 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# TVM RPC Server
+This folder contains a simple recipe to make RPC server in c++.
+
+## Usage (Non-Windows)
+- Configure the tvm CMake build with `config.cmake` ensuring that `USE_CPP_RPC` is set to `ON` in the config.
+- If cross compiling for Android, add the following options to the CMake config or specify them when invoking CMake:
+```
+  # Whether to build the C++ RPC server binary
+  set(USE_CPP_RPC ON)
+  # Path to the Android NDK CMake toolchain
+  set(CMAKE_TOOLCHAIN_FILE $ENV{ANDROID_NDK}/build/cmake/android.toolchain.cmake)
+  # The Android ABI and platform to target
+  set(ANDROID_ABI "arm64-v8a")
+  set(ANDROID_PLATFORM android-28)
+  ```
+- Similarly, if cross compiling for embedded Linux add the following options to CMake config:
+```
+  # Needed to ensure pthread is linked
+  set(OS Linux)
+  # Path to the desired C++ cross compiler
+  set(CMAKE_CXX_COMPILER /path/to/cross/compiler/executable)
+```
+- If you need to build cpp_rpc with OpenCL support, specify variable `USE_OPENCL` in the config:
+  ```
+  set(USE_OPENCL ON)
+  ```
+  In this case [OpenCL-wrapper](../../src/runtime/opencl/opencl_wrapper) or OpenCL installed to your system will be used.
+  When OpenCL-wrapper is used, it will dynamically load OpenCL library on the device.
+  If the device doesn't have OpenCL library on it, then you'll see in the runtime that OpenCL library cannot be opened.
+
+  If linking against a custom device OpenCL library is needed, in the config specify the path to the OpenCL SDK containing the include/CL headers and lib/ or lib64/libOpenCL.so:
+```
+  set(USE_OPENCL /path/to/opencl-sdk)
+```
+
+- From within the configured tvm build directory, compile `tvm_runtime` and the `tvm_rpc` server:
+```
+  cmake --build $TVM_ROOT/build --target tvm_runtime tvm_rpc -j$(nproc)
+```
+- Use `./tvm_rpc server` to start the RPC server
+
+## Usage (Windows)
+- Configure the tvm CMake build with `config.cmake` ensuring that `USE_CPP_RPC` is set to `ON` in the config.
+- Install [LLVM pre-build binaries](https://releases.llvm.org/download.html), making sure to select the option to add it to the PATH.
+- Verify Python 3.6 or newer is installed and in the PATH.
+- Use `<tvm_output_dir>\tvm_rpc.exe` to start the RPC server
+
+## How it works
+- The tvm runtime dll is linked along with this executable and when the RPC server starts it will load the tvm runtime library.
+
+```
+Command line usage
+ server       - Start the server
+--host        - The listen address of the server, Default=0.0.0.0 (any)
+--port        - The port of the RPC server, Default=9090
+--port-end    - The end search port of the RPC server, Default=9099
+--tracker     - The RPC tracker address in host:port format e.g. 10.1.1.2:9190 Default=""
+--key         - The key used to identify the device type in tracker. Default=""
+--custom-addr - Custom IP Address to Report to RPC Tracker. Default=""
+--silent      - Whether to run in silent mode. Default=False
+  Example
+  ./tvm_rpc server --host=0.0.0.0 --port=9090 --port-end=9099 --tracker=127.0.0.1:9190 --key=rasp
+```
+
+## Note
+Currently support is only there for Linux / Android / Windows environment and proxy mode isn't supported currently.

apps/cpp_rpc/main.cc

@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_server.cc
+ * \brief RPC Server for TVM.
+ */
+#include <csignal>
+#include <cstdio>
+#include <cstdlib>
+#if defined(__linux__) || defined(__ANDROID__)
+#include <unistd.h>
+#endif
+#include <tvm/runtime/logging.h>
+
+#include <cstring>
+#include <iostream>
+#include <sstream>
+#include <vector>
+
+#include "../../src/support/socket.h"
+#include "../../src/support/utils.h"
+#include "rpc_server.h"
+
+#if defined(_WIN32)
+#include "win32_process.h"
+#endif
+
+using namespace std;
+using namespace tvm::runtime;
+using namespace tvm::support;
+
+static const string kUsage =
+    "Command line usage\n"
+    " server       - Start the server\n"
+    "--host        - The listen address of the server, Default=0.0.0.0 (any)\n"
+    "--port        - The port of the RPC server, Default=9090\n"
+    "--port-end    - The end search port of the RPC server, Default=9099\n"
+    "--tracker     - The RPC tracker address in host:port format e.g. 10.1.1.2:9190 Default=\"\"\n"
+    "--key         - The key used to identify the device type in tracker. Default=\"\"\n"
+    "--custom-addr - Custom IP Address to Report to RPC Tracker. Default=\"\"\n"
+    "--work-dir    - Custom work directory. Default=\"\"\n"
+    "--silent      - Whether to run in silent mode. Default=False\n"
+    "\n"
+    "  Example\n"
+    "  ./tvm_rpc server --host=0.0.0.0 --port=9090 --port-end=9099 "
+    " --tracker=127.0.0.1:9190 --key=rasp"
+    "\n";
+
+/*!
+ * \brief RpcServerArgs.
+ * \arg host The listen address of the server, Default=0.0.0.0 (any)
+ * \arg port The port of the RPC server, Default=9090
+ * \arg port_end The end search port of the RPC server, Default=9099
+ * \arg tracker The address of RPC tracker in host:port format e.g. 10.77.1.234:9190 Default=""
+ * \arg key The key used to identify the device type in tracker. Default=""
+ * \arg custom_addr Custom IP Address to Report to RPC Tracker. Default=""
+ * \arg work_dir Custom work directory. Default=""
+ * \arg silent Whether run in silent mode. Default=False
+ */
+struct RpcServerArgs {
+  string host = "0.0.0.0";
+  int port = 9090;
+  int port_end = 9099;
+  string tracker;
+  string key;
+  string custom_addr;
+  string work_dir;
+  bool silent = false;
+#if defined(WIN32)
+  std::string mmap_path;
+#endif
+};
+
+/*!
+ * \brief PrintArgs print the contents of RpcServerArgs
+ * \param args RpcServerArgs structure
+ */
+void PrintArgs(const RpcServerArgs& args) {
+  LOG(INFO) << "host        = " << args.host;
+  LOG(INFO) << "port        = " << args.port;
+  LOG(INFO) << "port_end    = " << args.port_end;
+  LOG(INFO) << "tracker     = " << args.tracker;
+  LOG(INFO) << "key         = " << args.key;
+  LOG(INFO) << "custom_addr = " << args.custom_addr;
+  LOG(INFO) << "work_dir    = " << args.work_dir;
+  LOG(INFO) << "silent      = " << ((args.silent) ? ("True") : ("False"));
+}
+
+#if defined(__linux__) || defined(__ANDROID__)
+/*!
+ * \brief CtrlCHandler, exits if Ctrl+C is pressed
+ * \param s signal
+ */
+void CtrlCHandler(int s) {
+  LOG(INFO) << "\nUser pressed Ctrl+C, Exiting";
+  exit(1);
+}
+
+/*!
+ * \brief HandleCtrlC Register for handling Ctrl+C event.
+ */
+void HandleCtrlC() {
+  // Ctrl+C handler
+  struct sigaction sigIntHandler;
+  sigIntHandler.sa_handler = CtrlCHandler;
+  sigemptyset(&sigIntHandler.sa_mask);
+  sigIntHandler.sa_flags = 0;
+  sigaction(SIGINT, &sigIntHandler, nullptr);
+}
+#endif
+/*!
+ * \brief GetCmdOption Parse and find the command option.
+ * \param argc arg counter
+ * \param argv arg values
+ * \param option command line option to search for.
+ * \param key whether the option itself is key
+ * \return value corresponding to option.
+ */
+string GetCmdOption(int argc, char* argv[], string option, bool key = false) {
+  string cmd;
+  for (int i = 1; i < argc; ++i) {
+    string arg = argv[i];
+    if (arg.find(option) == 0) {
+      if (key) {
+        cmd = argv[i];
+        return cmd;
+      }
+      // We assume "=" is the end of option.
+      TVM_FFI_ICHECK_EQ(*option.rbegin(), '=');
+      cmd = arg.substr(arg.find('=') + 1);
+      return cmd;
+    }
+  }
+  return cmd;
+}
+
+/*!
+ * \brief ValidateTracker Check the tracker address format is correct and changes the format.
+ * \param tracker The tracker input.
+ * \return result of operation.
+ */
+bool ValidateTracker(string& tracker) {
+  vector<string> list = Split(tracker, ':');
+  if ((list.size() != 2) || (!ValidateIP(list[0])) || (!IsNumber(list[1]))) {
+    return false;
+  }
+  ostringstream ss;
+  ss << "('" << list[0] << "', " << list[1] << ")";
+  tracker = ss.str();
+  return true;
+}
+
+/*!
+ * \brief ParseCmdArgs parses the command line arguments.
+ * \param argc arg counter
+ * \param argv arg values
+ * \param args the output structure which holds the parsed values
+ */
+void ParseCmdArgs(int argc, char* argv[], struct RpcServerArgs& args) {
+  const string silent = GetCmdOption(argc, argv, "--silent", true);
+  if (!silent.empty()) {
+    args.silent = true;
+  }
+
+  const string host = GetCmdOption(argc, argv, "--host=");
+  if (!host.empty()) {
+    if (!ValidateIP(host)) {
+      LOG(WARNING) << "Wrong host address format.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.host = host;
+  }
+
+  const string port = GetCmdOption(argc, argv, "--port=");
+  if (!port.empty()) {
+    if (!IsNumber(port) || stoi(port) > 65535) {
+      LOG(WARNING) << "Wrong port number.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.port = stoi(port);
+  }
+
+  const string port_end = GetCmdOption(argc, argv, "--port-end=");
+  if (!port_end.empty()) {
+    if (!IsNumber(port_end) || stoi(port_end) > 65535) {
+      LOG(WARNING) << "Wrong port-end number.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.port_end = stoi(port_end);
+  }
+
+  string tracker = GetCmdOption(argc, argv, "--tracker=");
+  if (!tracker.empty()) {
+    if (!ValidateTracker(tracker)) {
+      LOG(WARNING) << "Wrong tracker address format.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.tracker = tracker;
+  }
+
+  const string key = GetCmdOption(argc, argv, "--key=");
+  if (!key.empty()) {
+    args.key = key;
+  }
+
+  const string custom_addr = GetCmdOption(argc, argv, "--custom-addr=");
+  if (!custom_addr.empty()) {
+    if (!ValidateIP(custom_addr)) {
+      LOG(WARNING) << "Wrong custom address format.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.custom_addr = custom_addr;
+  }
+#if defined(WIN32)
+  const string mmap_path = GetCmdOption(argc, argv, "--child_proc=");
+  if (!mmap_path.empty()) {
+    args.mmap_path = mmap_path;
+  }
+#endif
+  const string work_dir = GetCmdOption(argc, argv, "--work-dir=");
+  if (!work_dir.empty()) {
+    args.work_dir = work_dir;
+  }
+}
+
+/*!
+ * \brief RpcServer Starts the RPC server.
+ * \param argc arg counter
+ * \param argv arg values
+ * \return result of operation.
+ */
+int RpcServer(int argc, char* argv[]) {
+  RpcServerArgs args;
+
+  /* parse the command line args */
+  ParseCmdArgs(argc, argv, args);
+  PrintArgs(args);
+
+  LOG(INFO) << "Starting CPP Server, Press Ctrl+C to stop.";
+#if defined(__linux__) || defined(__ANDROID__)
+  // Ctrl+C handler
+  HandleCtrlC();
+#endif
+
+#if defined(WIN32)
+  if (!args.mmap_path.empty()) {
+    int ret = 0;
+
+    try {
+      ChildProcSocketHandler(args.mmap_path);
+    } catch (const std::exception&) {
+      ret = -1;
+    }
+
+    return ret;
+  }
+#endif
+
+  RPCServerCreate(args.host, args.port, args.port_end, args.tracker, args.key, args.custom_addr,
+                  args.work_dir, args.silent);
+  return 0;
+}
+
+/*!
+ * \brief main The main function.
+ * \param argc arg counter
+ * \param argv arg values
+ * \return result of operation.
+ */
+int main(int argc, char* argv[]) {
+  if (argc <= 1) {
+    LOG(INFO) << kUsage;
+    return 0;
+  }
+
+  // Runs WSAStartup on Win32, no-op on POSIX
+  Socket::Startup();
+#if defined(_WIN32)
+  SetEnvironmentVariableA("CUDA_CACHE_DISABLE", "1");
+#endif
+
+  if (0 == strcmp(argv[1], "server")) {
+    return RpcServer(argc, argv);
+  }
+
+  LOG(INFO) << kUsage;
+
+  return 0;
+}

apps/cpp_rpc/rpc_env.cc

@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * \file rpc_env.cc
+ * \brief Server environment of the RPC.
+ */
+#include "rpc_env.h"
+
+#include <tvm/ffi/extra/module.h>
+#include <tvm/ffi/function.h>
+#include <tvm/runtime/logging.h>
+
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <system_error>
+#include <vector>
+
+#include "../../src/support/utils.h"
+
+namespace {
+std::string GenerateUntarCommand(const std::string& tar_file, const std::string& output_dir) {
+  std::string untar_cmd;
+  untar_cmd.reserve(512);
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+  untar_cmd += "tar -C ";
+  untar_cmd += output_dir;
+  untar_cmd += " -zxf ";
+  untar_cmd += tar_file;
+#elif defined(_WIN32)
+  untar_cmd += "python -m tarfile -e ";
+  untar_cmd += tar_file;
+  untar_cmd += " ";
+  untar_cmd += output_dir;
+#endif
+  return untar_cmd;
+}
+
+}  // Anonymous namespace
+
+namespace tvm {
+namespace runtime {
+
+RPCEnv::RPCEnv(const std::string& wd) {
+  std::error_code ec;
+  if (!wd.empty()) {
+    base_ = wd + "/.cache";
+    std::filesystem::create_directories(base_, ec);
+    if (ec) {
+      LOG(WARNING) << "Failed to create directory " << base_ << " : " << ec.message();
+    }
+  } else {
+#if defined(ANDROID) || defined(__ANDROID__)
+    std::string pkg_name;
+    if (std::ifstream cmdline("/proc/self/cmdline"); cmdline) {
+      std::getline(cmdline, pkg_name, '\0');
+    }
+    std::string android_base_ = "/data/data/" + pkg_name + "/cache";
+    // Check if application data directory exist. If not exist, usually means we run tvm_rpc from
+    // adb shell terminal.
+    if (!std::filesystem::is_directory(android_base_)) {
+      // Tmp directory is always writable for 'shell' user.
+      android_base_ = "/data/local/tmp";
+    }
+    base_ = android_base_ + "/rpc";
+#elif !defined(_WIN32)
+    base_ = std::filesystem::current_path(ec).string() + "/rpc";
+    if (ec) {
+      base_ = "./rpc";
+    }
+#else
+    base_ = "./rpc";
+#endif
+    std::filesystem::create_directories(base_, ec);
+    if (ec) {
+      LOG(WARNING) << "Failed to create directory " << base_ << " : " << ec.message();
+    }
+  }
+  std::filesystem::permissions(base_, std::filesystem::perms::all,
+                               std::filesystem::perm_options::replace, ec);
+  if (ec) {
+    LOG(WARNING) << "Failed to grant permissions to " << base_ << " : " << ec.message();
+  }
+
+  ffi::Function::SetGlobal(
+      "tvm.rpc.server.workpath",
+      ffi::Function::FromTyped([this](const std::string& path) { return this->GetPath(path); }));
+
+  ffi::Function::SetGlobal("tvm.rpc.server.listdir",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string dir = this->GetPath(path);
+                             std::ostringstream os;
+                             for (auto d : ListDir(dir)) {
+                               os << d << ",";
+                             }
+                             return os.str();
+                           }));
+
+  ffi::Function::SetGlobal("tvm.rpc.server.load_module",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string file_name = this->GetPath(path);
+                             file_name = BuildSharedLibrary(file_name);
+                             LOG(INFO) << "Load module from " << file_name << " ...";
+                             return ffi::Module::LoadFromFile(file_name);
+                           }));
+
+  ffi::Function::SetGlobal("tvm.rpc.server.download_linked_module",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string file_name = this->GetPath(path);
+                             file_name = BuildSharedLibrary(file_name);
+                             std::string bin;
+
+                             std::ifstream fs(file_name, std::ios::in | std::ios::binary);
+                             TVM_FFI_ICHECK(!fs.fail()) << "Cannot open " << file_name;
+                             fs.seekg(0, std::ios::end);
+                             size_t size = static_cast<size_t>(fs.tellg());
+                             fs.seekg(0, std::ios::beg);
+                             bin.resize(size);
+                             fs.read(bin.data(), size);
+                             LOG(INFO) << "Send linked module " << file_name << " to client";
+                             return ffi::Bytes(bin);
+                           }));
+}
+
+/*!
+ * \brief GetPath To get the work path from packed function
+ * \param file_name The file name
+ * \return The full path of file.
+ */
+std::string RPCEnv::GetPath(const std::string& file_name) const {
+  // we assume file_name starts with "/" means file_name is the exact path
+  // and does not create /.rpc/
+  return !file_name.empty() && file_name[0] == '/' ? file_name : base_ + "/" + file_name;
+}
+
+/*!
+ * \brief Remove The RPC Environment cleanup function
+ */
+void RPCEnv::CleanUp() const {
+  std::error_code ec;
+  std::filesystem::remove_all(base_, ec);
+  if (ec) {
+    LOG(WARNING) << "Cleanup " << base_ << " failed: " << ec.message();
+  }
+}
+
+/*!
+ * \brief ListDir get the list of files in a directory
+ * \param dirname The root directory name
+ * \return vector Files in directory.
+ */
+std::vector<std::string> ListDir(const std::string& dirname) {
+  std::error_code ec;
+  std::vector<std::string> vec;
+  auto iter = std::filesystem::directory_iterator(dirname, ec);
+  if (ec) {
+    if (ec == std::errc::no_such_file_or_directory) return vec;
+    TVM_FFI_THROW(InternalError) << "ListDir " << dirname << " error: " << ec.message();
+  }
+  for (const auto& entry : iter) {
+    vec.push_back(entry.path().generic_string());
+  }
+  return vec;
+}
+
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+/*!
+ * \brief LinuxShared Creates a linux shared library
+ * \param output The output file name
+ * \param files The files for building
+ * \param options The compiler options
+ * \param cc The compiler
+ */
+void LinuxShared(const std::string output, const std::vector<std::string>& files,
+                 std::string options = "", std::string cc = "g++") {
+  std::string cmd = cc;
+  cmd += " -shared -fPIC ";
+  cmd += " -o " + output;
+  for (auto f = files.begin(); f != files.end(); ++f) {
+    cmd += " " + *f;
+  }
+  cmd += " " + options;
+  std::string err_msg;
+  auto executed_status = support::Execute(cmd, &err_msg);
+  if (executed_status) {
+    TVM_FFI_THROW(InternalError) << err_msg;
+  }
+}
+#endif
+
+#ifdef _WIN32
+/*!
+ * \brief WindowsShared Creates a Windows shared library
+ * \param output The output file name
+ * \param files The files for building
+ * \param options The compiler options
+ * \param cc The compiler
+ */
+void WindowsShared(const std::string& output, const std::vector<std::string>& files,
+                   const std::string& options = "", const std::string& cc = "clang") {
+  std::string cmd = cc;
+  cmd += " -O2 -flto=full -fuse-ld=lld-link -shared ";
+  cmd += " -o " + output;
+  for (const auto& file : files) {
+    cmd += " " + file;
+  }
+  cmd += " " + options;
+  std::string err_msg;
+  const auto executed_status = support::Execute(cmd, &err_msg);
+  if (executed_status) {
+    TVM_FFI_THROW(InternalError) << err_msg;
+  }
+}
+#endif
+
+/*!
+ * \brief CreateShared Creates a shared library
+ * \param output The output file name
+ * \param files The files for building
+ */
+void CreateShared(const std::string& output, const std::vector<std::string>& files) {
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+  LinuxShared(output, files);
+#elif defined(_WIN32)
+  WindowsShared(output, files);
+#else
+  TVM_FFI_THROW(InternalError) << "Operating system not supported";
+#endif
+}
+
+std::string BuildSharedLibrary(std::string file) {
+  if (support::EndsWith(file, ".so") || support::EndsWith(file, ".dll") ||
+      support::EndsWith(file, ".dylib")) {
+    return file;
+  }
+
+  std::string file_name = file + ".so";
+  if (support::EndsWith(file, ".o")) {
+    CreateShared(file_name, {file});
+  } else if (support::EndsWith(file, ".tar")) {
+    const std::string tmp_dir = "./rpc/tmp/";
+    std::error_code ec;
+    std::filesystem::create_directories(tmp_dir, ec);
+    if (ec) {
+      LOG(WARNING) << "Failed to create directory " << tmp_dir << " : " << ec.message();
+    }
+    std::filesystem::permissions(tmp_dir, std::filesystem::perms::all,
+                                 std::filesystem::perm_options::replace, ec);
+    if (ec) {
+      LOG(WARNING) << "Failed to grant permissions to " << tmp_dir << " : " << ec.message();
+    }
+
+    const std::string cmd = GenerateUntarCommand(file, tmp_dir);
+
+    std::string err_msg;
+    const int executed_status = support::Execute(cmd, &err_msg);
+    if (executed_status) {
+      TVM_FFI_THROW(InternalError) << err_msg;
+    }
+    CreateShared(file_name, ListDir(tmp_dir));
+    std::filesystem::remove_all(tmp_dir, ec);
+    if (ec) {
+      LOG(WARNING) << "Remove " << tmp_dir << " failed: " << ec.message();
+    }
+  } else {
+    file_name = file;
+  }
+  return file_name;
+}
+
+}  // namespace runtime
+}  // namespace tvm

apps/cpp_rpc/rpc_env.h

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_env.h
+ * \brief Server environment of the RPC.
+ */
+#ifndef TVM_APPS_CPP_RPC_ENV_H_
+#define TVM_APPS_CPP_RPC_ENV_H_
+
+#include <tvm/ffi/function.h>
+
+#include <string>
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief ListDir Get the list of files in a directory
+ * \param dirname The root directory name
+ * \return vector Files in directory.
+ */
+std::vector<std::string> ListDir(const std::string& dirname);
+
+/*!
+ * \brief build a shared library if necessary
+ *
+ *        This function will automatically call
+ *        cc.create_shared if the path is in format .o or .tar
+ *        High level handling for .o and .tar file.
+ *        We support this to be consistent with RPC module load.
+ * \param file_in The input file path.
+ *
+ * \return The name of the shared library.
+ */
+std::string BuildSharedLibrary(std::string file_in);
+
+/*!
+ * \brief RPCEnv The RPC Environment parameters for c++ rpc server
+ */
+struct RPCEnv {
+ public:
+  /*!
+   * \brief Constructor Init The RPC Environment initialize function
+   */
+  RPCEnv(const std::string& word_dir = "");
+  /*!
+   * \brief GetPath To get the workpath from packed function
+   * \param name The file name
+   * \return The full path of file.
+   */
+  std::string GetPath(const std::string& file_name) const;
+  /*!
+   * \brief The RPC Environment cleanup function
+   */
+  void CleanUp() const;
+
+ private:
+  /*!
+   * \brief Holds the environment path.
+   */
+  std::string base_;
+};  // RPCEnv
+
+}  // namespace runtime
+}  // namespace tvm
+#endif  // TVM_APPS_CPP_RPC_ENV_H_

apps/cpp_rpc/rpc_server.cc

@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_server.cc
+ * \brief RPC Server implementation.
+ */
+#include <tvm/ffi/function.h>
+#include <tvm/ffi/reflection/registry.h>
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+#include <signal.h>
+#include <sys/select.h>
+#include <sys/wait.h>
+#endif
+#include <chrono>
+#include <future>
+#include <iostream>
+#include <set>
+#include <string>
+#include <thread>
+#include <utility>
+
+#include "../../src/runtime/rpc/rpc_endpoint.h"
+#include "../../src/runtime/rpc/rpc_socket_impl.h"
+#include "../../src/support/socket.h"
+#include "rpc_env.h"
+#include "rpc_server.h"
+#include "rpc_tracker_client.h"
+#if defined(_WIN32)
+#include "win32_process.h"
+#endif
+
+using namespace std::chrono;
+
+namespace tvm {
+namespace runtime {
+
+#ifdef __ANDROID__
+static std::string getNextString(std::stringstream* iss) {
+  std::string str = iss->str();
+  size_t start = iss->tellg();
+  size_t len = str.size();
+  // Skip leading spaces.
+  while (start < len && isspace(str[start])) start++;
+
+  size_t end = start;
+  while (end < len && !isspace(str[end])) end++;
+
+  iss->seekg(end);
+  return str.substr(start, end - start);
+}
+#endif
+
+/*!
+ * \brief RPCServer RPC Server class.
+ *
+ * \param host The listen address of the server, Default=0.0.0.0 (any)
+ *
+ * \param port_search_start The low end of the search range for an
+ *     available port for the RPC, Default=9090
+ *
+ * \param port_search_end The high search the search range for an
+ *     available port for the RPC, Default=9099
+ *
+ * \param tracker The address of RPC tracker in host:port format
+ *     (e.g. "10.77.1.234:9190")
+ *
+ * \param key The key used to identify the device type in tracker.
+ *
+ * \param custom_addr Custom IP Address to Report to RPC Tracker.
+ */
+class RPCServer {
+ public:
+  /*!
+   * \brief Constructor.
+   */
+  RPCServer(std::string host, int port_search_start, int port_search_end, std::string tracker_addr,
+            std::string key, std::string custom_addr, std::string work_dir)
+      : host_(std::move(host)),
+        port_search_start_(port_search_start),
+        my_port_(0),
+        port_search_end_(port_search_end),
+        tracker_addr_(std::move(tracker_addr)),
+        key_(std::move(key)),
+        custom_addr_(std::move(custom_addr)),
+        work_dir_(std::move(work_dir)) {}
+
+  /*!
+   * \brief Destructor.
+   */
+  ~RPCServer() {
+    try {
+      // Free the resources
+      tracker_sock_.Close();
+      listen_sock_.Close();
+    } catch (...) {
+    }
+  }
+
+  /*!
+   * \brief Start Creates the RPC listen process and execution.
+   */
+  void Start() {
+    listen_sock_.Create();
+    my_port_ = listen_sock_.TryBindHost(host_, port_search_start_, port_search_end_);
+    LOG(INFO) << "Bind to " << host_ << ":" << my_port_;
+    listen_sock_.Listen(1);
+    std::future<void> proc(std::async(std::launch::async, &RPCServer::ListenLoopProc, this));
+    proc.get();
+    // Close the listen socket
+    listen_sock_.Close();
+  }
+
+ private:
+  /*!
+   * \brief ListenLoopProc The listen process.
+   */
+  void ListenLoopProc() {
+    TrackerClient tracker(tracker_addr_, key_, custom_addr_, my_port_);
+    while (true) {
+      support::TCPSocket conn;
+      support::SockAddr addr("0.0.0.0", 0);
+      std::string opts;
+      try {
+        // step 1: setup tracker and report to tracker
+        tracker.TryConnect();
+        // step 2: wait for in-coming connections
+        AcceptConnection(&tracker, &conn, &addr, &opts);
+      } catch (const char* msg) {
+        LOG(WARNING) << "Socket exception: " << msg;
+        // close tracker resource
+        tracker.Close();
+        continue;
+      } catch (const std::exception& e) {
+        // close tracker resource
+        tracker.Close();
+        LOG(WARNING) << "Exception standard: " << e.what();
+        continue;
+      }
+
+      int timeout = GetTimeOutFromOpts(opts);
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+      // step 3: serving
+      if (timeout != 0) {
+        const pid_t worker_pid = fork();
+        if (worker_pid == 0) {
+          // Worker process
+          ServerLoopProc(conn, addr, work_dir_);
+          _exit(0);
+        }
+
+        int status = 0;
+        bool timed_out = false;
+        auto start_timer = std::chrono::steady_clock::now();
+        while (true) {
+          // Check worker pid (non-blocking)
+          int ret = waitpid(worker_pid, &status, WNOHANG);
+          if (ret == worker_pid) {
+            break;
+          } else if (ret == -1) {
+            if (errno == EINTR) continue;
+            break;
+          }
+          // Check worker timeout
+          if (std::chrono::steady_clock::now() - start_timer >= std::chrono::seconds(timeout)) {
+            timed_out = true;
+            kill(worker_pid, SIGTERM);
+            waitpid(worker_pid, &status, 0);
+            break;
+          }
+          std::this_thread::sleep_for(std::chrono::milliseconds(50));
+        }
+
+        // Logging.
+        if (timed_out) {
+          LOG(INFO) << "Child pid=" << worker_pid << " killed"
+                    << " (timeout = " << timeout << " sec)"
+                    << ", status = " << status;
+        } else {
+          LOG(INFO) << "Child pid=" << worker_pid << " finished"
+                    << ", status = " << status;
+        }
+      } else {
+        auto pid = fork();
+        if (pid == 0) {
+          ServerLoopProc(conn, addr, work_dir_);
+          _exit(0);
+        }
+        // Wait for the result
+        int status = 0;
+        wait(&status);
+        LOG(INFO) << "Child pid=" << pid << " exited, status =" << status;
+      }
+#elif defined(WIN32)
+      auto start_time = high_resolution_clock::now();
+      try {
+        SpawnRPCChild(conn.sockfd, seconds(timeout));
+      } catch (const std::exception&) {
+      }
+      auto dur = high_resolution_clock::now() - start_time;
+
+      LOG(INFO) << "Serve Time " << duration_cast<milliseconds>(dur).count() << "ms";
+#else
+      LOG(WARNING) << "Unknown platform. It is not known how to bring up the subprocess."
+                   << " RPC will be launched in the main thread.";
+      ServerLoopProc(conn, addr, work_dir_);
+#endif
+      // close from our side.
+      LOG(INFO) << "End session with " << addr.AsString();
+      conn.Close();
+    }
+  }
+
+  /*!
+   * \brief AcceptConnection Accepts the RPC Server connection.
+   * \param tracker Tracker details.
+   * \param conn_sock New connection information.
+   * \param addr New connection address information.
+   * \param opts Parsed options for socket
+   * \param ping_period Timeout for select call waiting
+   */
+  void AcceptConnection(TrackerClient* tracker, support::TCPSocket* conn_sock,
+                        support::SockAddr* addr, std::string* opts, int ping_period = 2) {
+    std::set<std::string> old_keyset;
+    std::string matchkey;
+
+    // Report resource to tracker and get key
+    tracker->ReportResourceAndGetKey(my_port_, &matchkey);
+
+    while (true) {
+      tracker->WaitConnectionAndUpdateKey(listen_sock_, my_port_, ping_period, &matchkey);
+      support::TCPSocket conn = listen_sock_.Accept(addr);
+
+      int code = kRPCMagic;
+      TVM_FFI_ICHECK_EQ(conn.RecvAll(&code, sizeof(code)), sizeof(code));
+      if (code != kRPCMagic) {
+        conn.Close();
+        TVM_FFI_THROW(InternalError) << "Client connected is not TVM RPC server";
+        continue;
+      }
+
+      int keylen = 0;
+      TVM_FFI_ICHECK_EQ(conn.RecvAll(&keylen, sizeof(keylen)), sizeof(keylen));
+
+      const char* CLIENT_HEADER = "client:";
+      const char* SERVER_HEADER = "server:";
+      std::string expect_header = CLIENT_HEADER + matchkey;
+      std::string server_key = SERVER_HEADER + key_;
+      if (size_t(keylen) < expect_header.length()) {
+        conn.Close();
+        LOG(INFO) << "Wrong client header length";
+        continue;
+      }
+
+      TVM_FFI_ICHECK_NE(keylen, 0);
+      std::string remote_key;
+      remote_key.resize(keylen);
+      TVM_FFI_ICHECK_EQ(conn.RecvAll(&remote_key[0], keylen), keylen);
+
+      std::stringstream ssin(remote_key);
+      std::string arg0;
+#ifndef __ANDROID__
+      ssin >> arg0;
+#else
+      arg0 = getNextString(&ssin);
+#endif
+
+      if (arg0 != expect_header) {
+        code = kRPCMismatch;
+        TVM_FFI_ICHECK_EQ(conn.SendAll(&code, sizeof(code)), sizeof(code));
+        conn.Close();
+        LOG(WARNING) << "Mismatch key from" << addr->AsString();
+        continue;
+      } else {
+        code = kRPCSuccess;
+        TVM_FFI_ICHECK_EQ(conn.SendAll(&code, sizeof(code)), sizeof(code));
+        keylen = int(server_key.length());
+        TVM_FFI_ICHECK_EQ(conn.SendAll(&keylen, sizeof(keylen)), sizeof(keylen));
+        TVM_FFI_ICHECK_EQ(conn.SendAll(server_key.c_str(), keylen), keylen);
+        LOG(INFO) << "New session from " << addr->AsString();
+#ifndef __ANDROID__
+        ssin >> *opts;
+#else
+        *opts = getNextString(&ssin);
+#endif
+        *conn_sock = conn;
+        return;
+      }
+    }
+  }
+
+  /*!
+   * \brief ServerLoopProc The Server loop process.
+   * \param sock The socket information
+   * \param addr The socket address information
+   */
+  static void ServerLoopProc(support::TCPSocket sock, support::SockAddr addr,
+                             std::string work_dir) {
+    // Server loop
+    const auto s_time = std::chrono::high_resolution_clock::now();
+    const auto env = RPCEnv(work_dir);
+    RPCServerLoop(int(sock.sockfd));
+    const auto e_time = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<double> elapsed = e_time - s_time;
+    LOG(INFO) << "Finished serving " << addr.AsString() << " after " << elapsed.count() << " sec";
+    env.CleanUp();
+  }
+
+  /*!
+   * \brief GetTimeOutFromOpts Parse and get the timeout option.
+   * \param opts The option string
+   */
+  int GetTimeOutFromOpts(const std::string& opts) const {
+    const std::string option = "-timeout=";
+
+    size_t pos = opts.rfind(option);
+    if (pos != std::string::npos) {
+      const std::string cmd = opts.substr(pos + option.size());
+      TVM_FFI_ICHECK(support::IsNumber(cmd)) << "Timeout is not valid";
+      return std::stoi(cmd);
+    }
+    return 0;
+  }
+
+  std::string host_;
+  int port_search_start_;
+  int my_port_;
+  int port_search_end_;
+  std::string tracker_addr_;
+  std::string key_;
+  std::string custom_addr_;
+  std::string work_dir_;
+  support::TCPSocket listen_sock_;
+  support::TCPSocket tracker_sock_;
+};
+
+#if defined(WIN32)
+/*!
+ * \brief ServerLoopFromChild The Server loop process.
+ * \param socket The socket information
+ */
+void ServerLoopFromChild(SOCKET socket) {
+  // Server loop
+  tvm::support::TCPSocket sock(socket);
+  const auto env = RPCEnv();
+  RPCServerLoop(int(sock.sockfd));
+
+  sock.Close();
+  env.CleanUp();
+}
+#endif
+
+/*!
+ * \brief RPCServerCreate Creates the RPC Server.
+ * \param host The listen address of the server, Default=0.0.0.0 (any)
+ * \param port The port of the RPC server, Default=9090
+ * \param port_end The end search port of the RPC server, Default=9099
+ * \param tracker_addr The address of RPC tracker in host:port format e.g. 10.77.1.234:9190
+ * Default="" \param key The key used to identify the device type in tracker. Default="" \param
+ * custom_addr Custom IP Address to Report to RPC Tracker. Default="" \param silent Whether run in
+ * silent mode. Default=True
+ */
+void RPCServerCreate(std::string host, int port, int port_end, std::string tracker_addr,
+                     std::string key, std::string custom_addr, std::string work_dir, bool silent) {
+  if (silent) {
+  }
+  // Start the rpc server
+  RPCServer rpc(std::move(host), port, port_end, std::move(tracker_addr), std::move(key),
+                std::move(custom_addr), std::move(work_dir));
+  rpc.Start();
+}
+
+TVM_FFI_STATIC_INIT_BLOCK() {
+  namespace refl = tvm::ffi::reflection;
+  refl::GlobalDef().def("rpc.ServerCreate", RPCServerCreate);
+}
+}  // namespace runtime
+}  // namespace tvm

apps/cpp_rpc/rpc_server.h

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_server.h
+ * \brief RPC Server implementation.
+ */
+#ifndef TVM_APPS_CPP_RPC_SERVER_H_
+#define TVM_APPS_CPP_RPC_SERVER_H_
+
+#include <string>
+
+#include "tvm/runtime/base.h"
+
+namespace tvm {
+namespace runtime {
+
+#if defined(WIN32)
+/*!
+ * \brief ServerLoopFromChild The Server loop process.
+ * \param sock The socket information
+ * \param addr The socket address information
+ */
+void ServerLoopFromChild(SOCKET socket);
+#endif
+
+/*!
+ * \brief RPCServerCreate Creates the RPC Server.
+ * \param host The listen address of the server, Default=0.0.0.0 (any)
+ * \param port The port of the RPC server, Default=9090
+ * \param port_end The end search port of the RPC server, Default=9099
+ * \param tracker The address of RPC tracker in host:port format e.g. 10.77.1.234:9190 Default=""
+ * \param key The key used to identify the device type in tracker. Default=""
+ * \param custom_addr Custom IP Address to Report to RPC Tracker. Default=""
+ * \param work_dir Custom work directory. Default=""
+ * \param silent Whether run in silent mode. Default=True
+ */
+void RPCServerCreate(std::string host = "", int port = 9090, int port_end = 9099,
+                     std::string tracker_addr = "", std::string key = "",
+                     std::string custom_addr = "", std::string work_dir = "", bool silent = true);
+}  // namespace runtime
+}  // namespace tvm
+#endif  // TVM_APPS_CPP_RPC_SERVER_H_

apps/cpp_rpc/rpc_tracker_client.h

@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_tracker_client.h
+ * \brief RPC Tracker client to report resources.
+ */
+#ifndef TVM_APPS_CPP_RPC_TRACKER_CLIENT_H_
+#define TVM_APPS_CPP_RPC_TRACKER_CLIENT_H_
+
+#include <chrono>
+#include <iostream>
+#include <random>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "../../src/runtime/rpc/rpc_endpoint.h"
+#include "../../src/support/socket.h"
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief TrackerClient Tracker client class.
+ * \param tracker The address of RPC tracker in host:port format e.g. 10.77.1.234:9190 Default=""
+ * \param key The key used to identify the device type in tracker. Default=""
+ * \param custom_addr Custom IP Address to Report to RPC Tracker. Default=""
+ */
+class TrackerClient {
+ public:
+  /*!
+   * \brief Constructor.
+   */
+  TrackerClient(const std::string& tracker_addr, const std::string& key,
+                const std::string& custom_addr, int port)
+      : tracker_addr_(tracker_addr),
+        key_(key),
+        custom_addr_(custom_addr),
+        port_(port),
+        gen_(std::random_device{}()),
+        dis_(0.0, 1.0) {
+    if (custom_addr_.empty()) {
+      custom_addr_ = "null";
+    } else {
+      // Since custom_addr_ can be either the json value null which is not surrounded by quotes
+      // or a string containing the custom value which json does required to be quoted then we
+      // need to set custom_addr_ to be a string containing the quotes here.
+      custom_addr_ = "\"" + custom_addr_ + "\"";
+    }
+  }
+  /*!
+   * \brief Destructor.
+   */
+  ~TrackerClient() {
+    // Free the resources
+    Close();
+  }
+  /*!
+   * \brief IsValid Check tracker is valid.
+   */
+  bool IsValid() { return (!tracker_addr_.empty() && !tracker_sock_.IsClosed()); }
+  /*!
+   * \brief TryConnect Connect to tracker if the tracker address is valid.
+   */
+  void TryConnect() {
+    if (!tracker_addr_.empty() && (tracker_sock_.IsClosed())) {
+      tracker_sock_ = ConnectWithRetry();
+
+      int code = kRPCTrackerMagic;
+      TVM_FFI_ICHECK_EQ(tracker_sock_.SendAll(&code, sizeof(code)), sizeof(code));
+      TVM_FFI_ICHECK_EQ(tracker_sock_.RecvAll(&code, sizeof(code)), sizeof(code));
+      TVM_FFI_ICHECK_EQ(code, kRPCTrackerMagic) << tracker_addr_.c_str() << " is not RPC Tracker";
+
+      std::ostringstream ss;
+      ss << "[" << static_cast<int>(TrackerCode::kUpdateInfo) << ", {\"key\": \"server:" << key_
+         << "\", \"addr\": [" << custom_addr_ << ", \"" << port_ << "\"]}]";
+      tracker_sock_.SendBytes(ss.str());
+
+      // Receive status and validate
+      std::string remote_status = tracker_sock_.RecvBytes();
+      TVM_FFI_ICHECK_EQ(std::stoi(remote_status), static_cast<int>(TrackerCode::kSuccess));
+    }
+  }
+  /*!
+   * \brief Close Clean up tracker resources.
+   */
+  void Close() {
+    // close tracker resource
+    if (!tracker_sock_.IsClosed()) {
+      tracker_sock_.Close();
+    }
+  }
+  /*!
+   * \brief ReportResourceAndGetKey Report resource to tracker.
+   * \param port listening port.
+   * \param matchkey Random match key output.
+   */
+  void ReportResourceAndGetKey(int port, std::string* matchkey) {
+    if (!tracker_sock_.IsClosed()) {
+      *matchkey = RandomKey(key_ + ":", old_keyset_);
+
+      std::ostringstream ss;
+      ss << "[" << static_cast<int>(TrackerCode::kPut) << ", \"" << key_ << "\", [" << port
+         << ", \"" << *matchkey << "\"], " << custom_addr_ << "]";
+
+      tracker_sock_.SendBytes(ss.str());
+
+      // Receive status and validate
+      std::string remote_status = tracker_sock_.RecvBytes();
+      TVM_FFI_ICHECK_EQ(std::stoi(remote_status), static_cast<int>(TrackerCode::kSuccess));
+    } else {
+      *matchkey = key_;
+    }
+  }
+
+  /*!
+   * \brief ReportResourceAndGetKey Report resource to tracker.
+   * \param listen_sock Listen socket details for select.
+   * \param port listening port.
+   * \param ping_period Select wait time.
+   * \param matchkey Random match key output.
+   */
+  void WaitConnectionAndUpdateKey(support::TCPSocket listen_sock, int port, int ping_period,
+                                  std::string* matchkey) {
+    int unmatch_period_count = 0;
+    int unmatch_timeout = 4;
+    while (true) {
+      if (!tracker_sock_.IsClosed()) {
+        support::PollHelper poller;
+        poller.WatchRead(listen_sock.sockfd);
+        poller.Poll(ping_period * 1000);
+        if (!poller.CheckRead(listen_sock.sockfd)) {
+          std::ostringstream ss;
+          ss << "[" << int(TrackerCode::kGetPendingMatchKeys) << "]";
+          tracker_sock_.SendBytes(ss.str());
+
+          // Receive status and validate
+          std::string pending_keys = tracker_sock_.RecvBytes();
+          old_keyset_.insert(*matchkey);
+
+          // if match key not in pending key set
+          // it means the key is acquired by a client but not used.
+          if (pending_keys.find(*matchkey) == std::string::npos) {
+            unmatch_period_count += 1;
+          } else {
+            unmatch_period_count = 0;
+          }
+          // regenerate match key if key is acquired but not used for a while
+          if (unmatch_period_count * ping_period > unmatch_timeout + ping_period) {
+            LOG(INFO) << "no incoming connections, regenerate key ...";
+
+            *matchkey = RandomKey(key_ + ":", old_keyset_);
+
+            std::ostringstream ss;
+            ss << "[" << static_cast<int>(TrackerCode::kPut) << ", \"" << key_ << "\", [" << port
+               << ", \"" << *matchkey << "\"], " << custom_addr_ << "]";
+            tracker_sock_.SendBytes(ss.str());
+
+            std::string remote_status = tracker_sock_.RecvBytes();
+            TVM_FFI_ICHECK_EQ(std::stoi(remote_status), static_cast<int>(TrackerCode::kSuccess));
+            unmatch_period_count = 0;
+          }
+          continue;
+        }
+      }
+      break;
+    }
+  }
+
+ private:
+  /*!
+   * \brief Connect to a RPC address with retry.
+            This function is only reliable to short period of server restart.
+   * \param timeout Timeout during retry
+   * \param retry_period Number of seconds before we retry again.
+   * \return TCPSocket The socket information if connect is success.
+   */
+  support::TCPSocket ConnectWithRetry(int timeout = 60, int retry_period = 5) {
+    auto tbegin = std::chrono::system_clock::now();
+    while (true) {
+      support::SockAddr addr(tracker_addr_);
+      support::TCPSocket sock;
+      sock.Create();
+      if (sock.Connect(addr)) {
+        LOG(INFO) << "Connected to tracker " << addr.AsString();
+        return sock;
+      }
+
+      auto period = (std::chrono::duration_cast<std::chrono::seconds>(
+                         std::chrono::system_clock::now() - tbegin))
+                        .count();
+      TVM_FFI_ICHECK(period < timeout) << "Failed to connect to tracker " << addr.AsString();
+      LOG(WARNING) << "Cannot connect to tracker " << addr.AsString() << " retry in "
+                   << retry_period << " seconds.";
+      std::this_thread::sleep_for(std::chrono::seconds(retry_period));
+    }
+  }
+  /*!
+   * \brief Random Generate a random number between 0 and 1.
+   * \return random float value.
+   */
+  float Random() { return dis_(gen_); }
+  /*!
+   * \brief Generate a random key.
+   * \param prefix The string prefix.
+   * \return cmap The conflict map set.
+   */
+  std::string RandomKey(const std::string& prefix, const std::set<std::string>& cmap) {
+    if (!cmap.empty()) {
+      while (true) {
+        std::string key = prefix + std::to_string(Random());
+        if (cmap.find(key) == cmap.end()) {
+          return key;
+        }
+      }
+    }
+    return prefix + std::to_string(Random());
+  }
+
+  std::string tracker_addr_;
+  std::string key_;
+  std::string custom_addr_;
+  int port_;
+  support::TCPSocket tracker_sock_;
+  std::set<std::string> old_keyset_;
+  std::mt19937 gen_;
+  std::uniform_real_distribution<float> dis_;
+};
+}  // namespace runtime
+}  // namespace tvm
+#endif  // TVM_APPS_CPP_RPC_TRACKER_CLIENT_H_

apps/cpp_rpc/win32_process.cc

@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include "win32_process.h"
+
+#include <conio.h>
+#include <tvm/runtime/logging.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#include <cstdio>
+#include <memory>
+#include <stdexcept>
+#include <string>
+
+#include "rpc_server.h"
+
+using namespace std::chrono;
+using namespace tvm::runtime;
+
+namespace {
+// The prefix path for the memory mapped file used to store IPC information
+const std::string kMemoryMapPrefix = "/MAPPED_FILE/TVM_RPC";
+// Used to construct unique names for named resources in the parent process
+const std::string kParent = "parent";
+// Used to construct unique names for named resources in the child process
+const std::string kChild = "child";
+// The timeout of the WIN32 events, in the parent and the child
+const milliseconds kEventTimeout(2000);
+
+// Used to create unique WIN32 mmap paths and event names
+int child_counter_ = 0;
+
+/*!
+ * \brief HandleDeleter Deleter for UniqueHandle smart pointer
+ * \param handle The WIN32 HANDLE to manage
+ */
+struct HandleDeleter {
+  void operator()(HANDLE handle) const {
+    if (handle != INVALID_HANDLE_VALUE && handle != nullptr) {
+      CloseHandle(handle);
+    }
+  }
+};
+
+/*!
+ * \brief UniqueHandle Smart pointer to manage a WIN32 HANDLE
+ */
+using UniqueHandle = std::unique_ptr<void, HandleDeleter>;
+
+/*!
+ * \brief MakeUniqueHandle Helper method to construct a UniqueHandle
+ * \param handle The WIN32 HANDLE to manage
+ */
+UniqueHandle MakeUniqueHandle(HANDLE handle) {
+  if (handle == INVALID_HANDLE_VALUE || handle == nullptr) {
+    return nullptr;
+  }
+
+  return UniqueHandle(handle);
+}
+
+/*!
+ * \brief GetSocket Gets the socket info from the parent process and duplicates the socket
+ * \param mmap_path The path to the memory mapped info set by the parent
+ */
+SOCKET GetSocket(const std::string& mmap_path) {
+  WSAPROTOCOL_INFO protocol_info;
+
+  const std::string parent_event_name = mmap_path + kParent;
+  const std::string child_event_name = mmap_path + kChild;
+
+  // Open the events
+  UniqueHandle parent_file_mapping_event;
+  if ((parent_file_mapping_event = MakeUniqueHandle(
+           OpenEventA(SYNCHRONIZE, false, parent_event_name.c_str()))) == nullptr) {
+    TVM_FFI_THROW(InternalError) << "OpenEvent() failed: " << GetLastError();
+  }
+
+  UniqueHandle child_file_mapping_event;
+  if ((child_file_mapping_event = MakeUniqueHandle(
+           OpenEventA(EVENT_MODIFY_STATE, false, child_event_name.c_str()))) == nullptr) {
+    TVM_FFI_THROW(InternalError) << "OpenEvent() failed: " << GetLastError();
+  }
+
+  // Wait for the parent to set the event, notifying WSAPROTOCOL_INFO is ready to be read
+  if (WaitForSingleObject(parent_file_mapping_event.get(), uint32_t(kEventTimeout.count())) !=
+      WAIT_OBJECT_0) {
+    TVM_FFI_THROW(InternalError) << "WaitForSingleObject() failed: " << GetLastError();
+  }
+
+  const UniqueHandle file_map =
+      MakeUniqueHandle(OpenFileMappingA(FILE_MAP_READ | FILE_MAP_WRITE, false, mmap_path.c_str()));
+  if (!file_map) {
+    LOG(INFO) << "CreateFileMapping() failed: " << GetLastError();
+  }
+
+  void* map_view = MapViewOfFile(file_map.get(), FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
+
+  SOCKET sock_duplicated = INVALID_SOCKET;
+
+  if (map_view != nullptr) {
+    memcpy(&protocol_info, map_view, sizeof(WSAPROTOCOL_INFO));
+    UnmapViewOfFile(map_view);
+
+    // Creates the duplicate socket, that was created in the parent
+    sock_duplicated =
+        WSASocket(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, &protocol_info, 0, 0);
+
+    // Let the parent know we are finished duplicating the socket
+    SetEvent(child_file_mapping_event.get());
+  } else {
+    TVM_FFI_THROW(InternalError) << "MapViewOfFile() failed: " << GetLastError();
+  }
+
+  return sock_duplicated;
+}
+}  // Anonymous namespace
+
+namespace tvm {
+namespace runtime {
+/*!
+ * \brief SpawnRPCChild Spawns a child process with a given timeout to run
+ * \param fd The client socket to duplicate in the child
+ * \param timeout The time in seconds to wait for the child to complete before termination
+ */
+void SpawnRPCChild(SOCKET fd, seconds timeout) {
+  STARTUPINFOA startup_info;
+
+  memset(&startup_info, 0, sizeof(startup_info));
+  startup_info.cb = sizeof(startup_info);
+
+  std::string file_map_path = kMemoryMapPrefix + std::to_string(child_counter_++);
+
+  const std::string parent_event_name = file_map_path + kParent;
+  const std::string child_event_name = file_map_path + kChild;
+
+  // Create an event to let the child know the socket info was set to the mmap file
+  UniqueHandle parent_file_mapping_event;
+  if ((parent_file_mapping_event = MakeUniqueHandle(
+           CreateEventA(nullptr, true, false, parent_event_name.c_str()))) == nullptr) {
+    TVM_FFI_THROW(InternalError) << "CreateEvent for parent file mapping failed";
+  }
+
+  UniqueHandle child_file_mapping_event;
+  // An event to let the parent know the socket info was read from the mmap file
+  if ((child_file_mapping_event = MakeUniqueHandle(
+           CreateEventA(nullptr, true, false, child_event_name.c_str()))) == nullptr) {
+    TVM_FFI_THROW(InternalError) << "CreateEvent for child file mapping failed";
+  }
+
+  char current_executable[MAX_PATH];
+
+  // Get the full path of the current executable
+  GetModuleFileNameA(nullptr, current_executable, MAX_PATH);
+
+  std::string child_command_line = current_executable;
+  child_command_line += " server --child_proc=";
+  child_command_line += file_map_path;
+
+  // CreateProcessA requires a non const char*, so we copy our std::string
+  auto command_line_ptr = std::make_unique<char[]>(child_command_line.size() + 1);
+  strcpy(command_line_ptr.get(), child_command_line.c_str());
+
+  PROCESS_INFORMATION child_process_info;
+  if (CreateProcessA(nullptr, command_line_ptr.get(), nullptr, nullptr, false, CREATE_NO_WINDOW,
+                     nullptr, nullptr, &startup_info, &child_process_info)) {
+    // Child process and thread handles must be closed, so wrapped in RAII
+    auto child_process_handle = MakeUniqueHandle(child_process_info.hProcess);
+    auto child_process_thread_handle = MakeUniqueHandle(child_process_info.hThread);
+
+    WSAPROTOCOL_INFO protocol_info;
+    // Get info needed to duplicate the socket
+    if (WSADuplicateSocket(fd, child_process_info.dwProcessId, &protocol_info) == SOCKET_ERROR) {
+      TVM_FFI_THROW(InternalError) << "WSADuplicateSocket(): failed. Error =" << WSAGetLastError();
+    }
+
+    // Create a mmap file to store the info needed for duplicating the SOCKET in the child proc
+    UniqueHandle file_map =
+        MakeUniqueHandle(CreateFileMappingA(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0,
+                                            sizeof(WSAPROTOCOL_INFO), file_map_path.c_str()));
+    if (!file_map) {
+      LOG(INFO) << "CreateFileMapping() failed: " << GetLastError();
+    }
+
+    if (GetLastError() == ERROR_ALREADY_EXISTS) {
+      TVM_FFI_THROW(InternalError) << "CreateFileMapping(): mapping file already exists";
+    } else {
+      void* map_view = MapViewOfFile(file_map.get(), FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
+
+      if (map_view != nullptr) {
+        memcpy(map_view, &protocol_info, sizeof(WSAPROTOCOL_INFO));
+        UnmapViewOfFile(map_view);
+
+        // Let child proc know the mmap file is ready to be read
+        SetEvent(parent_file_mapping_event.get());
+
+        // Wait for the child to finish reading mmap file
+        if (WaitForSingleObject(child_file_mapping_event.get(), uint32_t(kEventTimeout.count())) !=
+            WAIT_OBJECT_0) {
+          TerminateProcess(child_process_handle.get(), 0);
+          TVM_FFI_THROW(InternalError)
+              << "WaitForSingleObject for child file mapping timed out.  Terminating child "
+                 "process.";
+        }
+      } else {
+        TerminateProcess(child_process_handle.get(), 0);
+        TVM_FFI_THROW(InternalError) << "MapViewOfFile() failed: " << GetLastError();
+      }
+    }
+
+    const DWORD process_timeout =
+        timeout.count() ? uint32_t(duration_cast<milliseconds>(timeout).count()) : INFINITE;
+
+    // Wait for child process to exit, or hit configured timeout
+    if (WaitForSingleObject(child_process_handle.get(), process_timeout) != WAIT_OBJECT_0) {
+      LOG(INFO) << "Child process timeout.  Terminating.";
+      TerminateProcess(child_process_handle.get(), 0);
+    }
+  } else {
+    LOG(INFO) << "Create child process failed: " << GetLastError();
+  }
+}
+/*!
+ * \brief ChildProcSocketHandler Ran from the child process and runs server to handle the client
+ * socket \param mmap_path The memory mapped file path that will contain the information to
+ * duplicate the client socket from the parent
+ */
+void ChildProcSocketHandler(const std::string& mmap_path) {
+  SOCKET socket;
+
+  // Set high thread priority to avoid the thread scheduler from
+  // interfering with any measurements in the RPC server.
+  SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL);
+
+  if ((socket = GetSocket(mmap_path)) != INVALID_SOCKET) {
+    tvm::runtime::ServerLoopFromChild(socket);
+  } else {
+    TVM_FFI_THROW(InternalError) << "GetSocket() failed";
+  }
+}
+}  // namespace runtime
+}  // namespace tvm

apps/cpp_rpc/win32_process.h

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file win32_process.h
+ * \brief Win32 process code to mimic a POSIX fork()
+ */
+#ifndef TVM_APPS_CPP_RPC_WIN32_PROCESS_H_
+#define TVM_APPS_CPP_RPC_WIN32_PROCESS_H_
+
+#include <chrono>
+#include <string>
+
+#include "../../src/support/socket.h"
+
+namespace tvm {
+namespace runtime {
+/*!
+ * \brief SpawnRPCChild Spawns a child process with a given timeout to run
+ * \param fd The client socket to duplicate in the child
+ * \param timeout The time in seconds to wait for the child to complete before termination
+ */
+void SpawnRPCChild(SOCKET fd, std::chrono::seconds timeout);
+/*!
+ * \brief ChildProcSocketHandler Ran from the child process and runs server to handle the client
+ * socket \param mmap_path The memory mapped file path that will contain the information to
+ * duplicate the client socket from the parent
+ */
+void ChildProcSocketHandler(const std::string& mmap_path);
+}  // namespace runtime
+}  // namespace tvm
+#endif  // TVM_APPS_CPP_RPC_WIN32_PROCESS_H_