chore: import upstream snapshot with attribution

apps/android_rpc/.gitignore

@@ -0,0 +1,15 @@
+*.iml
+.gradle
+/local.properties
+/.idea/workspace.xml
+/.idea/libraries
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+
+/app/src/main/jni/jni_helper_func.h
+/app/src/main/jni/org_apache_tvm_native_c_api.cc
+/app/src/main/jni/org_apache_tvm_native_c_api.h
+/app/src/main/obj/
+dev_tools/tvmrpc.keystore

apps/android_rpc/README.md

@@ -0,0 +1,170 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+
+# Android TVM RPC
+
+This folder contains Android RPC app that allows us to launch an RPC server on a Android device and connect to it through python script and do testing on the python side as normal TVM RPC.
+
+You will need JDK, [Android NDK](https://developer.android.com/ndk) and an Android device to use this.
+
+## Build and Installation
+
+### <a name="buildapk">Build APK</a>
+
+We use [Gradle](https://gradle.org) to build. Please follow [the installation instruction](https://gradle.org/install) for your operating system.
+
+Before you build the Android application, please refer to [TVM4J Installation Guide](https://github.com/apache/tvm/blob/main/jvm/README.md) and install tvm4j-core to your local maven repository. You can find tvm4j dependency declare in `app/build.gradle`. Modify it if it is necessary.
+
+```
+dependencies {
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    androidTestImplementation('androidx.test.espresso:espresso-core:3.1.0', {
+        exclude group: 'com.android.support', module: 'support-annotations'
+    })
+    implementation 'androidx.appcompat:appcompat:1.6.1'
+    implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
+    implementation 'com.google.android.material:material:1.8.0'
+    implementation files('../../../jvm/core/target/tvm4j-core-0.0.1-SNAPSHOT.jar')
+    testImplementation 'junit:junit:4.13.2'
+}
+```
+
+Now use Gradle to compile JNI, resolve Java dependencies and build the Android application together with tvm4j. Run following script to generate the apk file.
+
+```bash
+export ANDROID_HOME=[Path to your Android SDK, e.g., ~/Android/sdk]
+cd apps/android_rpc
+gradle clean build
+```
+
+In `app/build/outputs/apk` you'll find `app-release-unsigned.apk`, use `dev_tools/gen_keystore.sh` to generate a signature and use `dev_tools/sign_apk.sh` to get the signed apk file `app/build/outputs/apk/release/tvmrpc-release.apk`.
+
+Upload `tvmrpc-release.apk` to your Android device and install it:
+
+```bash
+$ANDROID_HOME/platform-tools/adb install app/build/outputs/apk/release/tvmrpc-release.apk
+```
+
+If you see error:
+
+    adb: failed to install app/build/outputs/apk/release/tvmrpc-release.apk:
+      Failure [INSTALL_FAILED_UPDATE_INCOMPATIBLE:
+      Package org.apache.tvm.tvmrpc signatures do not match the previously installed version; ignoring!]
+
+Run uninstall first:
+
+```bash
+$ANDROID_HOME/platform-tools/adb uninstall org.apache.tvm.tvmrpc
+```
+
+### Build with OpenCL
+
+Application is building with OpenCL support by default.
+[OpenCL-wrapper](../../src/runtime/opencl/opencl_wrapper) is used and will dynamically load OpenCL library on the device.
+If the device doesn't have OpenCL library on it, then you'll see in the runtime that OpenCL library cannot be opened.
+If you want to build this application without OpenCL then set `USE_OPENCL = 0`
+in [config.mk](./app/src/main/jni/make/config.mk)
+
+## Cross Compile and Run on Android Devices
+
+### Architecture and Android Standalone Toolchain
+
+In order to cross compile a shared library (.so) for your android device, you have to know the target triple for the device. (Refer to [Cross-compilation using Clang](https://clang.llvm.org/docs/CrossCompilation.html) for more information). Run `adb shell cat /proc/cpuinfo` to list the device's CPU information.
+
+Now use NDK to generate standalone toolchain for your device. For my test device, I use following command.
+
+```bash
+cd /opt/android-ndk/build/tools/
+./make-standalone-toolchain.sh --platform=android-24 --use-llvm --arch=arm64 --install-dir=/opt/android-toolchain-arm64
+```
+
+If everything goes well, you will find compile tools in `/opt/android-toolchain-arm64/bin`. For example, `bin/aarch64-linux-android-g++` can be used to compile C++ source codes and create shared libraries for arm64 Android devices.
+
+### Cross Compile and Upload to the Android Device
+
+First start an RPC tracker using
+
+```python -m tvm.exec.rpc_tracker --port [PORT]```
+
+and connect your Android device to this RPC tracker via the TVM RPC application. Open the app,
+set the `Address` and `Port` fields to the address and port of the RPC tracker respectively.
+The key should be set to "android" if you wish to avoid modifying the default test script.
+
+After pushing "START RPC" button on the app, you can check the connect by run
+
+```python -m tvm.exec.query_rpc_tracker --port [PORT]```
+
+on your host machine.
+You are supposed to find a free "android" in the queue status.
+
+```
+...
+
+Queue Status
+-------------------------------
+key       total  free  pending
+-------------------------------
+android   1      1     0
+-------------------------------
+```
+
+
+Then checkout [android\_rpc/tests/android\_rpc\_test.py](https://github.com/apache/tvm/blob/main/apps/android_rpc/tests/android_rpc_test.py) and run,
+
+```bash
+# Specify the RPC tracker
+export TVM_TRACKER_HOST=0.0.0.0
+export TVM_TRACKER_PORT=[PORT]
+# Specify the standalone Android C++ compiler
+export TVM_NDK_CC=/opt/android-toolchain-arm64/bin/aarch64-linux-android-g++
+python android_rpc_test.py
+```
+
+This will compile TVM IR to shared libraries (CPU, OpenCL and Vulkan) and run vector addition on your Android device. To verify compiled TVM IR shared libraries on OpenCL target set `'test_opencl = True'` and on Vulkan target set `'test_vulkan = True'` in  [tests/android_rpc_test.py](https://github.com/apache/tvm/blob/main/apps/android_rpc/tests/android_rpc_test.py), by default on CPU target will execute.
+On my test device, it gives following results.
+
+```bash
+Run CPU test ...
+0.000962932 secs/op
+
+Run GPU(OpenCL Flavor) test ...
+0.000155807 secs/op
+
+[23:29:34] /home/tvm/src/runtime/vulkan/vulkan_device_api.cc:674: Cannot initialize vulkan: [23:29:34] /home/tvm/src/runtime/vulkan/vulkan_device_api.cc:512: Check failed: __e == VK_SUCCESS Vulan Error, code=-9: VK_ERROR_INCOMPATIBLE_DRIVER
+
+Stack trace returned 10 entries:
+[bt] (0) /home/user/.local/lib/python3.6/site-packages/tvm-0.4.0-py3.6-linux-x86_64.egg/tvm/libtvm.so(dmlc::StackTrace[abi:cxx11]()+0x53) [0x7f477f5399f3]
+.........
+
+You can still compile vulkan module but cannot run locally
+Run GPU(Vulkan Flavor) test ...
+0.000225198 secs/op
+```
+
+You can define your own TVM operators and test via this RPC app on your Android device to find the most optimized TVM schedule.
+
+### Troubleshooting
+
+If you build the application in Android Studio and see error similar to this one:
+```
+A problem occurred evaluating project ':app'.
+> Failed to apply plugin 'com.android.internal.version-check'.
+   > Minimum supported Gradle version is 7.5. Current version is 7.4. If using the gradle wrapper, try editing the distributionUrl in /Users/echuraev/Workspace/OctoML/tvm_android_test/apps/android_deploy/gradle/wrapper/gradle-wrapper.properties to gradle-7.5-all.zip
+```
+Run project syncing `File -> Sync Project with Gradle Files`. It should sync the
+project and create gradle-wrapper files.

apps/android_rpc/app/.gitignore

@@ -0,0 +1 @@
+/build

apps/android_rpc/app/build.gradle

@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+apply plugin: 'com.android.application'
+
+task generateJniHeaders(type: Exec, description: 'Generate JNI Headers') {
+    def headerPath = "${project.projectDir}/src/main/jni"
+    def classPath = "${project.projectDir}/../../../jvm/core/target/*"
+    def filePath = "${project.projectDir}/../../../jvm/core/src/main/java/org/apache/tvm/LibInfo.java"
+    commandLine "javac", "-h", headerPath, "-classpath", classPath, filePath
+    doLast {
+        file("${headerPath}/org_apache_tvm_LibInfo.h").renameTo(file("${headerPath}/org_apache_tvm_native_c_api.h"))
+    }
+}
+
+task copyFiles(type: Copy, description: 'Copy Sources for ndk-build') {
+    dependsOn "generateJniHeaders"
+    def ndkFilesPath = "${project.projectDir}/../../../jvm/native/src/main/native"
+    def srcPath = "${project.projectDir}/src/main/jni/"
+
+    from "${ndkFilesPath}/org_apache_tvm_native_c_api.cc", "${ndkFilesPath}/jni_helper_func.h"
+    into srcPath
+}
+
+task deleteLibs(type: Delete, description: "Delete Compiled Libraries") {
+    dependsOn "copyFiles"
+    def libsPath = "${project.projectDir}/src/main/libs"
+    delete libsPath
+}
+
+task buildJni(type: Exec, description: 'Build JNI libs') {
+    dependsOn "deleteLibs"
+    def buildPath = "${project.projectDir}/src/main/jni"
+    commandLine "ndk-build", "--directory", buildPath
+}
+
+tasks.withType(JavaCompile) {
+    compileTask -> compileTask.dependsOn buildJni
+}
+
+// gradle.projectsEvaluated {
+//     tasks.withType(JavaCompile) {
+//         options.compilerArgs << "-Xlint:deprecation"
+//     }
+// }
+
+android {
+    compileSdkVersion 33
+    defaultConfig {
+        applicationId "org.apache.tvm.tvmrpc"
+        minSdkVersion 24
+        targetSdkVersion 33
+        versionCode 1
+        versionName "1.0"
+        testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner'
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+    sourceSets {
+        main {
+            jni.srcDirs = []
+            jniLibs.srcDirs = ['src/main/libs']
+        }
+    }
+
+    lint {
+        disable 'Instantiatable' // MainActivity and RPCActivity must extend android.app.Activity
+        disable 'MissingClass' // .RPCWatchdogService was not found in the project or the libraries
+        disable 'IconDipSize' // The image ic_launcher.png varies significantly in its density-independent size
+    }
+    namespace 'org.apache.tvm.tvmrpc'
+
+}
+
+dependencies {
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    androidTestImplementation('androidx.test.espresso:espresso-core:3.1.0', {
+        exclude group: 'com.android.support', module: 'support-annotations'
+    })
+    implementation 'androidx.appcompat:appcompat:1.6.1'
+    implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
+    implementation 'com.google.android.material:material:1.8.0'
+    implementation files('../../../jvm/core/target/tvm4j-core-0.0.1-SNAPSHOT.jar')
+    testImplementation 'junit:junit:4.13.2'
+}

apps/android_rpc/app/src/main/AndroidManifest.xml

@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <uses-permission android:name="android.permission.INTERNET" />
+
+    <application
+        android:allowBackup="true"
+        android:label="@string/app_name"
+        android:supportsRtl="true"
+        android:theme="@style/AppTheme"
+        android:icon="@mipmap/ic_launcher">
+        <uses-native-library
+            android:name="libOpenCL.so"
+            android:required="false"/>
+        <activity
+            android:name=".MainActivity"
+            android:theme="@style/AppTheme.NoActionBar"
+            android:screenOrientation="unspecified"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+        <service android:name=".RPCWatchdogService"
+         android:process=":RPCWatchdogServiceProcess"
+         android:permission="android.permission.BIND_JOB_SERVICE" />
+        <activity
+            android:name=".RPCActivity"
+            android:process=":RPCProcess"
+            android:label="@string/rpc_name"
+            android:theme="@style/AppTheme.NoActionBar"
+            android:exported="false"
+            android:screenOrientation="unspecified">
+        </activity>
+    </application>
+
+</manifest>

apps/android_rpc/app/src/main/java/org/apache/tvm/tvmrpc/MainActivity.java

@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tvm.tvmrpc;
+
+import android.app.AlertDialog;
+import android.content.Context;
+import android.content.DialogInterface;
+import android.content.Intent;
+import android.content.SharedPreferences;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.Looper;
+import android.widget.CompoundButton;
+import android.widget.EditText;
+import androidx.appcompat.app.AppCompatActivity;
+import androidx.appcompat.widget.SwitchCompat;
+import androidx.appcompat.widget.Toolbar;
+
+public class MainActivity extends AppCompatActivity {
+  // wait time before automatic restart of RPC Activity
+  public static final int HANDLER_RESTART_DELAY = 5000;
+
+  private void showDialog(String title, String msg) {
+    AlertDialog.Builder builder = new AlertDialog.Builder(this);
+    builder.setTitle(title);
+    builder.setMessage(msg);
+    builder.setCancelable(true);
+    builder.setNeutralButton(android.R.string.ok, new DialogInterface.OnClickListener() {
+      public void onClick(DialogInterface dialog, int id) {
+        dialog.cancel();
+      }
+    });
+    builder.create().show();
+  }
+
+  public Intent updateRPCPrefs() {
+    System.err.println("updating preferences...");
+    EditText edProxyAddress = findViewById(R.id.input_address);
+    EditText edProxyPort = findViewById(R.id.input_port);
+    EditText edAppKey = findViewById(R.id.input_key);
+    SwitchCompat inputSwitch = findViewById(R.id.switch_persistent);
+
+    final String proxyHost = edProxyAddress.getText().toString();
+    final int proxyPort = Integer.parseInt(edProxyPort.getText().toString());
+    final String key = edAppKey.getText().toString();
+    final boolean isChecked = inputSwitch.isChecked();
+
+    SharedPreferences pref =
+        getApplicationContext().getSharedPreferences("RPCProxyPreference", Context.MODE_PRIVATE);
+    SharedPreferences.Editor editor = pref.edit();
+    editor.putString("input_address", proxyHost);
+    editor.putString("input_port", edProxyPort.getText().toString());
+    editor.putString("input_key", key);
+    editor.putBoolean("input_switch", isChecked);
+    editor.commit();
+
+    Intent intent = new Intent(this, RPCActivity.class);
+    intent.putExtra("host", proxyHost);
+    intent.putExtra("port", proxyPort);
+    intent.putExtra("key", key);
+    return intent;
+  }
+
+  private void setupRelaunch() {
+    final Context context = this;
+    final SwitchCompat switchPersistent = findViewById(R.id.switch_persistent);
+    final Runnable rPCStarter = new Runnable() {
+      public void run() {
+        if (switchPersistent.isChecked()) {
+          System.err.println("relaunching RPC activity...");
+          Intent intent = ((MainActivity) context).updateRPCPrefs();
+          startActivity(intent);
+        }
+      }
+    };
+
+    Handler handler = new Handler(Looper.getMainLooper());
+    handler.postDelayed(rPCStarter, HANDLER_RESTART_DELAY);
+  }
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_main);
+    Toolbar toolbar = findViewById(R.id.toolbar);
+    setSupportActionBar(toolbar);
+    final Context context = this;
+
+    SwitchCompat switchPersistent = findViewById(R.id.switch_persistent);
+    switchPersistent.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
+      @Override
+      public void onCheckedChanged(CompoundButton buttonView, boolean isChecked) {
+        if (isChecked) {
+          System.err.println("automatic RPC restart enabled...");
+          updateRPCPrefs();
+          setupRelaunch();
+        } else {
+          System.err.println("automatic RPC restart disabled...");
+          updateRPCPrefs();
+        }
+      }
+    });
+
+    enableInputView(true);
+  }
+
+  @Override
+  protected void onResume() {
+    System.err.println("MainActivity onResume...");
+    enableInputView(true);
+    setupRelaunch();
+    super.onResume();
+  }
+
+  @Override
+  protected void onDestroy() {
+    super.onDestroy();
+  }
+
+  private void enableInputView(boolean enable) {
+    EditText edProxyAddress = findViewById(R.id.input_address);
+    EditText edProxyPort = findViewById(R.id.input_port);
+    EditText edAppKey = findViewById(R.id.input_key);
+    SwitchCompat input_switch = findViewById(R.id.switch_persistent);
+    edProxyAddress.setEnabled(enable);
+    edProxyPort.setEnabled(enable);
+    edAppKey.setEnabled(enable);
+
+    if (enable) {
+      SharedPreferences pref =
+          getApplicationContext().getSharedPreferences("RPCProxyPreference", Context.MODE_PRIVATE);
+      String inputAddress = pref.getString("input_address", null);
+      if (null != inputAddress)
+        edProxyAddress.setText(inputAddress);
+      String inputPort = pref.getString("input_port", null);
+      if (null != inputPort)
+        edProxyPort.setText(inputPort);
+      String inputKey = pref.getString("input_key", null);
+      if (null != inputKey)
+        edAppKey.setText(inputKey);
+      boolean isChecked = pref.getBoolean("input_switch", false);
+      input_switch.setChecked(isChecked);
+    }
+  }
+}

apps/android_rpc/app/src/main/java/org/apache/tvm/tvmrpc/RPCActivity.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tvm.tvmrpc;
+
+import android.content.Intent;
+import android.os.Bundle;
+import android.view.View;
+import android.widget.Button;
+import androidx.appcompat.app.AppCompatActivity;
+
+public class RPCActivity extends AppCompatActivity {
+  private RPCProcessor tvmServerWorker;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_rpc);
+
+    Button stopRPC = findViewById(R.id.button_stop_rpc);
+    stopRPC.setOnClickListener(new View.OnClickListener() {
+      public void onClick(View v) {
+        System.err.println(tvmServerWorker == null);
+        if (tvmServerWorker != null) {
+          // currently will raise a socket closed exception
+          tvmServerWorker.disconnect();
+        }
+        finish();
+        // prevent Android from recycling the process
+        System.exit(0);
+      }
+    });
+
+    System.err.println("rpc activity onCreate...");
+    Intent intent = getIntent();
+    String host = intent.getStringExtra("host");
+    int port = intent.getIntExtra("port", 9090);
+    String key = intent.getStringExtra("key");
+
+    tvmServerWorker = new RPCProcessor(this);
+    tvmServerWorker.setDaemon(true);
+    tvmServerWorker.start();
+    tvmServerWorker.connect(host, port, key);
+  }
+
+  @Override
+  protected void onDestroy() {
+    System.err.println("rpc activity onDestroy");
+    tvmServerWorker.disconnect();
+    super.onDestroy();
+    android.os.Process.killProcess(android.os.Process.myPid());
+  }
+}

apps/android_rpc/app/src/main/java/org/apache/tvm/tvmrpc/RPCAndroidWatchdog.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tvm.tvmrpc;
+
+import android.app.Activity;
+import org.apache.tvm.rpc.RPCWatchdog;
+
+/**
+ * Watchdog for Android RPC.
+ */
+public class RPCAndroidWatchdog extends RPCWatchdog {
+  public Activity rpc_activity = null;
+  public RPCAndroidWatchdog(Activity activity) {
+    super();
+    rpc_activity = activity;
+  }
+
+  /**
+   * Method to non-destructively terminate the running thread on Android
+   */
+  @Override
+  protected void terminate() {
+    rpc_activity.finish();
+  }
+}

apps/android_rpc/app/src/main/java/org/apache/tvm/tvmrpc/RPCProcessor.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tvm.tvmrpc;
+
+import android.app.Activity;
+import android.os.ParcelFileDescriptor;
+import java.net.Socket;
+import org.apache.tvm.rpc.ConnectTrackerServerProcessor;
+
+/**
+ * Connect to RPC proxy and deal with requests.
+ */
+class RPCProcessor extends Thread {
+  private String host;
+  private int port;
+  private String key;
+  private boolean running = false;
+  private long startTime;
+  private ConnectTrackerServerProcessor currProcessor;
+  private boolean first = true;
+  private Activity rpc_activity = null;
+
+  public RPCProcessor(Activity activity) {
+    super();
+    rpc_activity = activity;
+  }
+
+  @Override
+  public void run() {
+    RPCAndroidWatchdog watchdog = new RPCAndroidWatchdog(rpc_activity);
+    watchdog.start();
+    while (true) {
+      synchronized (this) {
+        currProcessor = null;
+        while (!running) {
+          try {
+            this.wait();
+          } catch (InterruptedException e) {
+          }
+        }
+        try {
+          currProcessor = new ConnectTrackerServerProcessor(host, port, key, watchdog);
+        } catch (Throwable e) {
+          e.printStackTrace();
+          // kill if creating a new processor failed
+          System.exit(0);
+        }
+      }
+      if (currProcessor != null)
+        currProcessor.run();
+      watchdog.finishTimeout();
+    }
+  }
+
+  /**
+   * Disconnect from the proxy server.
+   */
+  synchronized void disconnect() {
+    if (running) {
+      running = false;
+      if (currProcessor != null) {
+        currProcessor.terminate();
+      }
+    }
+  }
+
+  /**
+   * Start rpc processor and connect to the proxy server.
+   * @param host proxy server host.
+   * @param port proxy server port.
+   * @param key proxy server key.
+   */
+  synchronized void connect(String host, int port, String key) {
+    this.host = host;
+    this.port = port;
+    this.key = key;
+    running = true;
+    this.notify();
+  }
+}

apps/android_rpc/app/src/main/jni/Android.mk

@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+LOCAL_PATH := $(call my-dir)
+MY_PATH := $(LOCAL_PATH)
+
+include $(CLEAR_VARS)
+
+LOCAL_PATH := $(MY_PATH)
+ROOT_PATH := $(MY_PATH)/../../../../../..
+
+ifndef config
+	ifneq ("$(wildcard ./config.mk)","")
+	  config ?= config.mk
+	else
+	  config ?= make/config.mk
+	endif
+endif
+
+include $(config)
+
+LOCAL_SRC_FILES := org_apache_tvm_native_c_api.cc
+LOCAL_LDFLAGS := -L$(SYSROOT)/usr/lib/ -llog
+
+LOCAL_C_INCLUDES := $(ROOT_PATH)/include \
+                    $(ROOT_PATH)/3rdparty/tvm-ffi/include \
+                    $(ROOT_PATH)/3rdparty/tvm-ffi/3rdparty/dlpack/include \
+                    $(ROOT_PATH)/3rdparty/OpenCL-Headers
+
+LOCAL_MODULE = tvm4j_runtime_packed
+
+LOCAL_CPP_FEATURES += exceptions
+LOCAL_LDLIBS += -latomic
+LOCAL_ARM_MODE := arm
+
+ifdef ADD_C_INCLUDES
+	LOCAL_C_INCLUDES += $(ADD_C_INCLUDES)
+endif
+
+ifdef ADD_LDLIBS
+	LOCAL_LDLIBS += $(ADD_LDLIBS)
+endif
+
+include $(BUILD_SHARED_LIBRARY)

apps/android_rpc/app/src/main/jni/Application.mk

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ifndef config
+    ifneq ("$(wildcard ./config.mk)","")
+        config ?= config.mk
+    else
+        config ?= make/config.mk
+    endif
+endif
+
+include $(config)
+
+# We target every architecture except armeabi here, for two reasons:
+# 1) armeabi is deprecated in NDK r16 and removed in r17
+# 2) vulkan is not supported in armeabi
+APP_ABI ?= armeabi-v7a arm64-v8a x86 x86_64 mips
+APP_STL := c++_shared
+
+APP_CPPFLAGS += -DTVM4J_ANDROID=1 -std=c++17 -Oz -frtti
+ifeq ($(USE_OPENCL), 1)
+    APP_CPPFLAGS += -DTVM_OPENCL_RUNTIME=1
+endif
+
+ifeq ($(USE_VULKAN), 1)
+    APP_CPPFLAGS += -DTVM_VULKAN_RUNTIME=1
+    APP_LDFLAGS += -lvulkan
+endif
+
+ifeq ($(USE_SORT), 1)
+    APP_CPPFLAGS += -DUSE_SORT=1
+endif
+
+ifeq ($(USE_RANDOM), 1)
+    APP_CPPFLAGS += -DUSE_RANDOM=1
+endif

apps/android_rpc/app/src/main/jni/tvm_runtime.h

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file tvm_runtime.h
+ * \brief Pack all tvm runtime source files
+ */
+#include <sys/stat.h>
+
+#include <fstream>
+
+/* Enable custom logging - this will cause TVM to use a custom implementation
+ * of tvm::runtime::detail::LogMessage. We use this to pass TVM log messages to
+ * Android logcat.
+ */
+#define TVM_LOG_CUSTOMIZE 1
+#define TVM_FFI_USE_LIBBACKTRACE 0
+
+#include "../3rdparty/tvm-ffi/src/ffi/backtrace.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/container.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/dtype.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/error.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/extra/library_module.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/extra/library_module_dynamic_lib.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/extra/library_module_system_lib.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/extra/module.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/function.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/object.cc"
+#include "../3rdparty/tvm-ffi/src/ffi/tensor.cc"
+#include "../src/runtime/cpu_device_api.cc"
+#include "../src/runtime/device_api.cc"
+#include "../src/runtime/file_utils.cc"
+#include "../src/runtime/logging.cc"
+#include "../src/runtime/memory/memory_manager.cc"
+#include "../src/runtime/registry.cc"
+#include "../src/runtime/rpc/rpc_channel.cc"
+#include "../src/runtime/rpc/rpc_endpoint.cc"
+#include "../src/runtime/rpc/rpc_event_impl.cc"
+#include "../src/runtime/rpc/rpc_local_session.cc"
+#include "../src/runtime/rpc/rpc_module.cc"
+#include "../src/runtime/rpc/rpc_server_env.cc"
+#include "../src/runtime/rpc/rpc_session.cc"
+#include "../src/runtime/rpc/rpc_socket_impl.cc"
+#include "../src/runtime/tensor.cc"
+#include "../src/runtime/thread_pool.cc"
+#include "../src/runtime/threading_backend.cc"
+#include "../src/runtime/timer.cc"
+#include "../src/runtime/workspace_pool.cc"
+
+#ifdef TVM_OPENCL_RUNTIME
+#include "../src/runtime/opencl/opencl_device_api.cc"
+#include "../src/runtime/opencl/opencl_module.cc"
+#include "../src/runtime/opencl/opencl_wrapper/opencl_wrapper.cc"
+#include "../src/runtime/source_utils.cc"
+#endif
+
+#ifdef TVM_VULKAN_RUNTIME
+#include "../src/runtime/vulkan/vulkan_amdrgp.cc"
+#include "../src/runtime/vulkan/vulkan_buffer.cc"
+#include "../src/runtime/vulkan/vulkan_common.cc"
+#include "../src/runtime/vulkan/vulkan_device.cc"
+#include "../src/runtime/vulkan/vulkan_device_api.cc"
+#include "../src/runtime/vulkan/vulkan_instance.cc"
+#include "../src/runtime/vulkan/vulkan_module.cc"
+#include "../src/runtime/vulkan/vulkan_stream.cc"
+#include "../src/runtime/vulkan/vulkan_wrapped_func.cc"
+#endif
+
+#ifdef USE_SORT
+#include "../src/runtime/extra/contrib/sort/sort.cc"
+#endif
+
+#ifdef USE_RANDOM
+#include "../src/runtime/extra/contrib/random/random.cc"
+#endif
+
+#include <android/log.h>
+
+namespace tvm {
+namespace runtime {
+namespace detail {
+// Override logging mechanism
+[[noreturn]] void LogFatalImpl(const std::string& file, int lineno, const std::string& message) {
+  std::string m = file + ":" + std::to_string(lineno) + ": " + message;
+  __android_log_write(ANDROID_LOG_FATAL, "TVM_RUNTIME", m.c_str());
+  throw tvm::ffi::Error("InternalError", message, TVMFFIBacktrace(file.c_str(), lineno, "", 0));
+}
+void LogMessageImpl(const std::string& file, int lineno, int level, const std::string& message) {
+  std::string m = file + ":" + std::to_string(lineno) + ": " + message;
+  __android_log_write(ANDROID_LOG_DEBUG + level, "TVM_RUNTIME", m.c_str());
+}
+
+}  // namespace detail
+}  // namespace runtime
+}  // namespace tvm

apps/android_rpc/app/src/main/res/layout/activity_main.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<androidx.coordinatorlayout.widget.CoordinatorLayout
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    tools:context="org.apache.tvm.tvmrpc.MainActivity">
+
+    <com.google.android.material.appbar.AppBarLayout
+        android:layout_height="wrap_content"
+        android:layout_width="match_parent"
+        android:theme="@style/AppTheme.AppBarOverlay">
+
+        <androidx.appcompat.widget.Toolbar
+            android:id="@+id/toolbar"
+            android:layout_width="match_parent"
+            android:layout_height="?attr/actionBarSize"
+            android:background="?attr/colorPrimary"
+            app:popupTheme="@style/AppTheme.PopupOverlay" />
+
+    </com.google.android.material.appbar.AppBarLayout>
+
+    <include layout="@layout/content_main"/>
+
+</androidx.coordinatorlayout.widget.CoordinatorLayout>

apps/android_rpc/app/src/main/res/layout/activity_rpc.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<androidx.coordinatorlayout.widget.CoordinatorLayout
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    tools:context="org.apache.tvm.tvmrpc.RPCActivity">
+
+    <com.google.android.material.appbar.AppBarLayout
+        android:layout_height="wrap_content"
+        android:layout_width="match_parent"
+        android:theme="@style/AppTheme.AppBarOverlay">
+
+        <androidx.appcompat.widget.Toolbar
+            android:id="@+id/toolbar"
+            android:layout_width="match_parent"
+            android:layout_height="?attr/actionBarSize"
+            android:background="?attr/colorPrimary"
+            app:popupTheme="@style/AppTheme.PopupOverlay" />
+
+    </com.google.android.material.appbar.AppBarLayout>
+
+    <include layout="@layout/content_rpc"/>
+
+</androidx.coordinatorlayout.widget.CoordinatorLayout>

apps/android_rpc/app/src/main/res/layout/content_main.xml

@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+              xmlns:tools="http://schemas.android.com/tools"
+              xmlns:app="http://schemas.android.com/apk/res-auto"
+              android:orientation="vertical"
+              android:layout_width="fill_parent"
+              android:layout_height="wrap_content"
+              app:layout_behavior="@string/appbar_scrolling_view_behavior"
+              tools:showIn="@layout/activity_main">
+    <LinearLayout
+        android:orientation="horizontal"
+        android:layout_width="fill_parent"
+        android:layout_height="wrap_content">
+        <TextView
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:text="@string/label_address"/>
+        <EditText
+            android:id="@+id/input_address"
+            android:hint="@string/input_address"
+            android:autofillHints="@string/label_address_hint"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:inputType="phone"
+            android:background="@android:drawable/editbox_background"/>
+    </LinearLayout>
+
+    <LinearLayout
+        android:orientation="horizontal"
+        android:layout_width="fill_parent"
+        android:layout_height="wrap_content">
+        <TextView
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:text="@string/label_port"/>
+        <EditText
+            android:id="@+id/input_port"
+            android:hint="@string/input_port"
+            android:autofillHints="@string/label_port_hint"
+            android:minWidth="100dip"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:inputType="phone"
+            android:background="@android:drawable/editbox_background"/>
+    </LinearLayout>
+
+    <LinearLayout
+        android:orientation="horizontal"
+        android:layout_width="fill_parent"
+        android:layout_height="wrap_content">
+        <TextView
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:text="@string/label_key"/>
+        <EditText
+            android:id="@+id/input_key"
+            android:hint="@string/input_key"
+            android:autofillHints="@string/label_key_hint"
+            android:inputType="text"
+            android:minWidth="100dip"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:background="@android:drawable/editbox_background"/>
+    </LinearLayout>
+
+    <LinearLayout
+        android:orientation="horizontal"
+        android:layout_width="fill_parent"
+        android:layout_height="wrap_content">
+        <TextView
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:text="@string/label_persistent"/>
+        <androidx.appcompat.widget.SwitchCompat
+                android:id="@+id/switch_persistent"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:switchMinWidth="55dp"
+                android:paddingLeft="10dip"
+                android:paddingRight="10dip"
+                android:checked="false"
+                android:textOff="@string/switch_off"
+                android:textOn="@string/switch_on" />
+    </LinearLayout>
+
+</LinearLayout>

apps/android_rpc/app/src/main/res/layout/content_rpc.xml

@@ -0,0 +1,34 @@
+<?xml version='1.0'?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+              xmlns:tools="http://schemas.android.com/tools"
+              xmlns:app="http://schemas.android.com/apk/res-auto"
+              android:orientation="vertical"
+              android:layout_width="fill_parent"
+              android:layout_height="wrap_content"
+              app:layout_behavior="@string/appbar_scrolling_view_behavior"
+              tools:showIn="@layout/activity_rpc">
+        <Button
+            android:id="@+id/button_stop_rpc"
+            android:layout_height="wrap_content"
+            android:layout_width="wrap_content"
+            android:text="@string/stop_rpc" />
+</LinearLayout>

apps/android_rpc/app/src/main/res/values/colors.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<resources>
+    <color name="colorPrimary">#3F51B5</color>
+    <color name="colorPrimaryDark">#303F9F</color>
+    <color name="colorAccent">#06d467</color>
+</resources>

apps/android_rpc/app/src/main/res/values/strings.xml

@@ -0,0 +1,43 @@
+<?xml version='1.0'?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<resources>
+    <string name="app_name">TVM RPC</string>
+    <string name="rpc_name">RPC</string>
+
+    <string name="input_address">Enter the tracker server address</string>
+    <string name="input_port">Enter the tracker server port</string>
+    <string name="input_key">Enter the app connection key</string>
+
+    <string name="label_address">Address</string>
+    <string name="label_port">Port</string>
+    <string name="label_key">Key</string>
+    <string name="label_persistent">Enable RPC</string>
+
+    <string name="label_address_hint">192.168.1.1</string>
+    <string name="label_port_hint">9190</string>
+    <string name="label_key_hint">android</string>
+
+
+    <string name="switch_on">Enabled</string>
+    <string name="switch_off">Disabled</string>
+
+    <string name="stop_rpc">Stop RPC</string>
+</resources>

apps/android_rpc/app/src/main/res/values/styles.xml

@@ -0,0 +1,37 @@
+<?xml version='1.0'?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+        <item name="colorPrimary">@color/colorPrimary</item>
+        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
+        <item name="colorAccent">@color/colorAccent</item>
+    </style>
+    <style name="AppTheme.NoActionBar">
+        <item name="windowActionBar">false</item>
+        <item name="windowNoTitle">true</item>
+    </style>
+    <style name="AppTheme.AppBarOverlay" parent="ThemeOverlay.AppCompat.Dark.ActionBar" />
+    <style name="AppTheme.PopupOverlay" parent="ThemeOverlay.AppCompat.Light" />
+
+</resources>

apps/android_rpc/build.gradle

@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
+buildscript {
+    repositories {
+        gradlePluginPortal()
+        maven {
+            url 'https://maven.google.com'
+        }
+        google()
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:7.4.1'
+
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+    }
+}
+
+allprojects {
+    repositories {
+        gradlePluginPortal()
+        maven {
+          url 'https://maven.google.com'
+        }
+        mavenLocal()
+        mavenCentral()
+        google()
+    }
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+}

apps/android_rpc/dev_tools/gen_keystore.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+CURR_DIR=$(cd `dirname $0`; pwd)
+keytool -genkey -keystore $CURR_DIR/tvmrpc.keystore -alias tvmrpc -keyalg RSA -validity 10000

apps/android_rpc/dev_tools/sign_apk.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+CURR_DIR=$(cd `dirname $0`; pwd)
+APK_DIR=$CURR_DIR/../app/build/outputs/apk/release
+UNSIGNED_APK=$APK_DIR/app-release-unsigned.apk
+SIGNED_APK=$APK_DIR/tvmrpc-release.apk
+jarsigner -verbose -keystore $CURR_DIR/tvmrpc.keystore -signedjar $SIGNED_APK $UNSIGNED_APK 'tvmrpc'
+echo $SIGNED_APK

apps/android_rpc/gradle.properties

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+android.useAndroidX=true
+android.enableJetifier=true

apps/android_rpc/settings.gradle

@@ -0,0 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+include ':app'

apps/android_rpc/tests/android_rpc_test.py

@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: F821
+"""Testcode for Android RPC.
+
+To use it, start an RPC tracker with "python -m tvm.exec.rpc_tracker".
+Use the tracker's address and port when configuring the RPC app.
+Use "android" as the key if you wish to avoid modifying this script.
+"""
+
+import os
+
+import numpy as np
+
+import tvm
+from tvm import rpc, te
+from tvm.support import ndk, utils
+
+# Set to be address of tvm proxy.
+tracker_host = os.environ["TVM_TRACKER_HOST"]
+tracker_port = int(os.environ["TVM_TRACKER_PORT"])
+key = "android"
+
+# Change target configuration.
+# Run `adb shell cat /proc/cpuinfo` to find the arch.
+arch = "arm64"
+target = {"kind": "llvm", "mtriple": f"{arch}-linux-android"}
+
+# whether enable to execute test on OpenCL target
+test_opencl = False
+# whether enable to execute test on Vulkan target
+test_vulkan = False
+
+
+def test_rpc_module():
+    # graph
+    n = tvm.runtime.convert(1024)
+    A = te.placeholder((n,), name="A")
+    B = te.compute(A.shape, lambda *i: A(*i) + 1.0, name="B")
+    a_np = np.random.uniform(size=1024).astype(A.dtype)
+    temp = utils.tempdir()
+
+    # Establish remote connection with target hardware
+    tracker = rpc.connect_tracker(tracker_host, tracker_port)
+    remote = tracker.request(key, priority=0, session_timeout=60)
+
+    mod = tvm.IRModule.from_expr(te.create_prim_func([A, B]).with_attr("global_symbol", "myadd"))
+    sch = tvm.s_tir.Schedule(mod)
+    (x,) = sch.get_loops(block=sch.get_sblock("B"))
+    xo, xi = sch.split(i, [None, 32])
+    sch.bind(xo, "blockIdx.x")
+    sch.bind(xi, "threadIdx.x")
+
+    if test_opencl:
+        f = tvm.compile(sch.mod, target=tvm.target.Target("opencl", host=target))
+        path_dso_cl = temp.relpath("dev_lib_cl.so")
+        f.export_library(path_dso_cl, fcompile=ndk.create_shared)
+
+        print("Run GPU(OpenCL Flavor) test ...")
+        dev = remote.cl(0)
+        remote.upload(path_dso_cl)
+        f1 = remote.load_module("dev_lib_cl.so")
+        a = tvm.runtime.tensor(a_np, dev)
+        b = tvm.runtime.tensor(np.zeros(1024, dtype=A.dtype), dev)
+        time_f = f1.time_evaluator(f1.entry_name, dev, number=10)
+        cost = time_f(a, b).mean
+        print(f"{cost:g} secs/op\n")
+        np.testing.assert_equal(b.numpy(), a.numpy() + 1)
+
+
+if __name__ == "__main__":
+    test_rpc_module()

apps/cpp_rpc/CMakeLists.txt

@@ -0,0 +1,82 @@
+cmake_policy(SET CMP0069 NEW) # suppress CMake warning about IPO
+
+set(TVM_RPC_SOURCES
+  main.cc
+  rpc_env.cc
+  rpc_server.cc
+)
+
+set(TVM_RPC_LINKER_LIBS "")
+
+if(WIN32)
+  list(APPEND TVM_RPC_SOURCES win32_process.cc)
+endif()
+
+# Set output to same directory as the other TVM libs
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+add_executable(tvm_rpc ${TVM_RPC_SOURCES})
+
+include(CheckIPOSupported)
+check_ipo_supported(RESULT result OUTPUT output)
+if(result)
+  set_property(TARGET tvm_rpc PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE)
+endif()
+
+if(WIN32)
+  target_compile_definitions(tvm_rpc PUBLIC -DNOMINMAX)
+endif()
+
+if (OS)
+   if (OS STREQUAL "Linux")
+      set_property(TARGET tvm_rpc PROPERTY LINK_FLAGS -lpthread)
+   endif()
+endif()
+
+if(USE_OPENCL)
+   if (ANDROID_ABI)
+     if(DEFINED ENV{ANDROID_NDK_MAJOR})
+       if($ENV{ANDROID_NDK_MAJOR} VERSION_LESS "23")
+         set_property(TARGET tvm_rpc PROPERTY LINK_FLAGS -fuse-ld=gold)
+       endif()
+     endif()
+   endif()
+endif()
+
+target_include_directories(
+  tvm_rpc
+  PUBLIC "../../include"
+)
+
+target_link_libraries(tvm_rpc PUBLIC tvm_ffi_header)
+
+if (BUILD_FOR_ANDROID AND USE_HEXAGON)
+  get_hexagon_sdk_property("${USE_HEXAGON_SDK}" "${USE_HEXAGON_ARCH}"
+    DSPRPC_LIB DSPRPC_LIB_DIRS
+  )
+  if(DSPRPC_LIB_DIRS)
+    link_directories(${DSPRPC_LIB_DIRS})
+  else()
+    message(WARNING "Could not locate some Hexagon SDK components")
+  endif()
+  list(APPEND TVM_RPC_LINKER_LIBS cdsprpc log)
+endif()
+
+if(BUILD_STATIC_RUNTIME)
+  foreach(lib ${TVM_RUNTIME_BACKEND_LIBS})
+    if(MSVC)
+      list(APPEND TVM_RPC_LINKER_LIBS "/WHOLEARCHIVE:$<TARGET_FILE:${lib}>")
+    elseif(APPLE)
+      list(APPEND TVM_RPC_LINKER_LIBS "-Wl,-force_load,$<TARGET_FILE:${lib}>")
+    else()
+      list(APPEND TVM_RPC_LINKER_LIBS "-Wl,--whole-archive" "${lib}" "-Wl,--no-whole-archive")
+    endif()
+  endforeach()
+else()
+  if(NOT MSVC AND NOT APPLE)
+    list(APPEND TVM_RPC_LINKER_LIBS tvm_runtime "-Wl,--no-as-needed" ${TVM_RUNTIME_BACKEND_LIBS} "-Wl,--as-needed")
+  else()
+    list(APPEND TVM_RPC_LINKER_LIBS tvm_runtime ${TVM_RUNTIME_BACKEND_LIBS})
+  endif()
+endif()
+
+target_link_libraries(tvm_rpc PRIVATE ${TVM_RPC_LINKER_LIBS})

apps/cpp_rpc/README.md

@@ -0,0 +1,83 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# TVM RPC Server
+This folder contains a simple recipe to make RPC server in c++.
+
+## Usage (Non-Windows)
+- Configure the tvm CMake build with `config.cmake` ensuring that `USE_CPP_RPC` is set to `ON` in the config.
+- If cross compiling for Android, add the following options to the CMake config or specify them when invoking CMake:
+```
+  # Whether to build the C++ RPC server binary
+  set(USE_CPP_RPC ON)
+  # Path to the Android NDK CMake toolchain
+  set(CMAKE_TOOLCHAIN_FILE $ENV{ANDROID_NDK}/build/cmake/android.toolchain.cmake)
+  # The Android ABI and platform to target
+  set(ANDROID_ABI "arm64-v8a")
+  set(ANDROID_PLATFORM android-28)
+  ```
+- Similarly, if cross compiling for embedded Linux add the following options to CMake config:
+```
+  # Needed to ensure pthread is linked
+  set(OS Linux)
+  # Path to the desired C++ cross compiler
+  set(CMAKE_CXX_COMPILER /path/to/cross/compiler/executable)
+```
+- If you need to build cpp_rpc with OpenCL support, specify variable `USE_OPENCL` in the config:
+  ```
+  set(USE_OPENCL ON)
+  ```
+  In this case [OpenCL-wrapper](../../src/runtime/opencl/opencl_wrapper) or OpenCL installed to your system will be used.
+  When OpenCL-wrapper is used, it will dynamically load OpenCL library on the device.
+  If the device doesn't have OpenCL library on it, then you'll see in the runtime that OpenCL library cannot be opened.
+
+  If linking against a custom device OpenCL library is needed, in the config specify the path to the OpenCL SDK containing the include/CL headers and lib/ or lib64/libOpenCL.so:
+```
+  set(USE_OPENCL /path/to/opencl-sdk)
+```
+
+- From within the configured tvm build directory, compile `tvm_runtime` and the `tvm_rpc` server:
+```
+  cmake --build $TVM_ROOT/build --target tvm_runtime tvm_rpc -j$(nproc)
+```
+- Use `./tvm_rpc server` to start the RPC server
+
+## Usage (Windows)
+- Configure the tvm CMake build with `config.cmake` ensuring that `USE_CPP_RPC` is set to `ON` in the config.
+- Install [LLVM pre-build binaries](https://releases.llvm.org/download.html), making sure to select the option to add it to the PATH.
+- Verify Python 3.6 or newer is installed and in the PATH.
+- Use `<tvm_output_dir>\tvm_rpc.exe` to start the RPC server
+
+## How it works
+- The tvm runtime dll is linked along with this executable and when the RPC server starts it will load the tvm runtime library.
+
+```
+Command line usage
+ server       - Start the server
+--host        - The listen address of the server, Default=0.0.0.0 (any)
+--port        - The port of the RPC server, Default=9090
+--port-end    - The end search port of the RPC server, Default=9099
+--tracker     - The RPC tracker address in host:port format e.g. 10.1.1.2:9190 Default=""
+--key         - The key used to identify the device type in tracker. Default=""
+--custom-addr - Custom IP Address to Report to RPC Tracker. Default=""
+--silent      - Whether to run in silent mode. Default=False
+  Example
+  ./tvm_rpc server --host=0.0.0.0 --port=9090 --port-end=9099 --tracker=127.0.0.1:9190 --key=rasp
+```
+
+## Note
+Currently support is only there for Linux / Android / Windows environment and proxy mode isn't supported currently.

apps/cpp_rpc/main.cc

@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_server.cc
+ * \brief RPC Server for TVM.
+ */
+#include <csignal>
+#include <cstdio>
+#include <cstdlib>
+#if defined(__linux__) || defined(__ANDROID__)
+#include <unistd.h>
+#endif
+#include <tvm/runtime/logging.h>
+
+#include <cstring>
+#include <iostream>
+#include <sstream>
+#include <vector>
+
+#include "../../src/support/socket.h"
+#include "../../src/support/utils.h"
+#include "rpc_server.h"
+
+#if defined(_WIN32)
+#include "win32_process.h"
+#endif
+
+using namespace std;
+using namespace tvm::runtime;
+using namespace tvm::support;
+
+static const string kUsage =
+    "Command line usage\n"
+    " server       - Start the server\n"
+    "--host        - The listen address of the server, Default=0.0.0.0 (any)\n"
+    "--port        - The port of the RPC server, Default=9090\n"
+    "--port-end    - The end search port of the RPC server, Default=9099\n"
+    "--tracker     - The RPC tracker address in host:port format e.g. 10.1.1.2:9190 Default=\"\"\n"
+    "--key         - The key used to identify the device type in tracker. Default=\"\"\n"
+    "--custom-addr - Custom IP Address to Report to RPC Tracker. Default=\"\"\n"
+    "--work-dir    - Custom work directory. Default=\"\"\n"
+    "--silent      - Whether to run in silent mode. Default=False\n"
+    "\n"
+    "  Example\n"
+    "  ./tvm_rpc server --host=0.0.0.0 --port=9090 --port-end=9099 "
+    " --tracker=127.0.0.1:9190 --key=rasp"
+    "\n";
+
+/*!
+ * \brief RpcServerArgs.
+ * \arg host The listen address of the server, Default=0.0.0.0 (any)
+ * \arg port The port of the RPC server, Default=9090
+ * \arg port_end The end search port of the RPC server, Default=9099
+ * \arg tracker The address of RPC tracker in host:port format e.g. 10.77.1.234:9190 Default=""
+ * \arg key The key used to identify the device type in tracker. Default=""
+ * \arg custom_addr Custom IP Address to Report to RPC Tracker. Default=""
+ * \arg work_dir Custom work directory. Default=""
+ * \arg silent Whether run in silent mode. Default=False
+ */
+struct RpcServerArgs {
+  string host = "0.0.0.0";
+  int port = 9090;
+  int port_end = 9099;
+  string tracker;
+  string key;
+  string custom_addr;
+  string work_dir;
+  bool silent = false;
+#if defined(WIN32)
+  std::string mmap_path;
+#endif
+};
+
+/*!
+ * \brief PrintArgs print the contents of RpcServerArgs
+ * \param args RpcServerArgs structure
+ */
+void PrintArgs(const RpcServerArgs& args) {
+  LOG(INFO) << "host        = " << args.host;
+  LOG(INFO) << "port        = " << args.port;
+  LOG(INFO) << "port_end    = " << args.port_end;
+  LOG(INFO) << "tracker     = " << args.tracker;
+  LOG(INFO) << "key         = " << args.key;
+  LOG(INFO) << "custom_addr = " << args.custom_addr;
+  LOG(INFO) << "work_dir    = " << args.work_dir;
+  LOG(INFO) << "silent      = " << ((args.silent) ? ("True") : ("False"));
+}
+
+#if defined(__linux__) || defined(__ANDROID__)
+/*!
+ * \brief CtrlCHandler, exits if Ctrl+C is pressed
+ * \param s signal
+ */
+void CtrlCHandler(int s) {
+  LOG(INFO) << "\nUser pressed Ctrl+C, Exiting";
+  exit(1);
+}
+
+/*!
+ * \brief HandleCtrlC Register for handling Ctrl+C event.
+ */
+void HandleCtrlC() {
+  // Ctrl+C handler
+  struct sigaction sigIntHandler;
+  sigIntHandler.sa_handler = CtrlCHandler;
+  sigemptyset(&sigIntHandler.sa_mask);
+  sigIntHandler.sa_flags = 0;
+  sigaction(SIGINT, &sigIntHandler, nullptr);
+}
+#endif
+/*!
+ * \brief GetCmdOption Parse and find the command option.
+ * \param argc arg counter
+ * \param argv arg values
+ * \param option command line option to search for.
+ * \param key whether the option itself is key
+ * \return value corresponding to option.
+ */
+string GetCmdOption(int argc, char* argv[], string option, bool key = false) {
+  string cmd;
+  for (int i = 1; i < argc; ++i) {
+    string arg = argv[i];
+    if (arg.find(option) == 0) {
+      if (key) {
+        cmd = argv[i];
+        return cmd;
+      }
+      // We assume "=" is the end of option.
+      TVM_FFI_ICHECK_EQ(*option.rbegin(), '=');
+      cmd = arg.substr(arg.find('=') + 1);
+      return cmd;
+    }
+  }
+  return cmd;
+}
+
+/*!
+ * \brief ValidateTracker Check the tracker address format is correct and changes the format.
+ * \param tracker The tracker input.
+ * \return result of operation.
+ */
+bool ValidateTracker(string& tracker) {
+  vector<string> list = Split(tracker, ':');
+  if ((list.size() != 2) || (!ValidateIP(list[0])) || (!IsNumber(list[1]))) {
+    return false;
+  }
+  ostringstream ss;
+  ss << "('" << list[0] << "', " << list[1] << ")";
+  tracker = ss.str();
+  return true;
+}
+
+/*!
+ * \brief ParseCmdArgs parses the command line arguments.
+ * \param argc arg counter
+ * \param argv arg values
+ * \param args the output structure which holds the parsed values
+ */
+void ParseCmdArgs(int argc, char* argv[], struct RpcServerArgs& args) {
+  const string silent = GetCmdOption(argc, argv, "--silent", true);
+  if (!silent.empty()) {
+    args.silent = true;
+  }
+
+  const string host = GetCmdOption(argc, argv, "--host=");
+  if (!host.empty()) {
+    if (!ValidateIP(host)) {
+      LOG(WARNING) << "Wrong host address format.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.host = host;
+  }
+
+  const string port = GetCmdOption(argc, argv, "--port=");
+  if (!port.empty()) {
+    if (!IsNumber(port) || stoi(port) > 65535) {
+      LOG(WARNING) << "Wrong port number.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.port = stoi(port);
+  }
+
+  const string port_end = GetCmdOption(argc, argv, "--port-end=");
+  if (!port_end.empty()) {
+    if (!IsNumber(port_end) || stoi(port_end) > 65535) {
+      LOG(WARNING) << "Wrong port-end number.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.port_end = stoi(port_end);
+  }
+
+  string tracker = GetCmdOption(argc, argv, "--tracker=");
+  if (!tracker.empty()) {
+    if (!ValidateTracker(tracker)) {
+      LOG(WARNING) << "Wrong tracker address format.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.tracker = tracker;
+  }
+
+  const string key = GetCmdOption(argc, argv, "--key=");
+  if (!key.empty()) {
+    args.key = key;
+  }
+
+  const string custom_addr = GetCmdOption(argc, argv, "--custom-addr=");
+  if (!custom_addr.empty()) {
+    if (!ValidateIP(custom_addr)) {
+      LOG(WARNING) << "Wrong custom address format.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.custom_addr = custom_addr;
+  }
+#if defined(WIN32)
+  const string mmap_path = GetCmdOption(argc, argv, "--child_proc=");
+  if (!mmap_path.empty()) {
+    args.mmap_path = mmap_path;
+  }
+#endif
+  const string work_dir = GetCmdOption(argc, argv, "--work-dir=");
+  if (!work_dir.empty()) {
+    args.work_dir = work_dir;
+  }
+}
+
+/*!
+ * \brief RpcServer Starts the RPC server.
+ * \param argc arg counter
+ * \param argv arg values
+ * \return result of operation.
+ */
+int RpcServer(int argc, char* argv[]) {
+  RpcServerArgs args;
+
+  /* parse the command line args */
+  ParseCmdArgs(argc, argv, args);
+  PrintArgs(args);
+
+  LOG(INFO) << "Starting CPP Server, Press Ctrl+C to stop.";
+#if defined(__linux__) || defined(__ANDROID__)
+  // Ctrl+C handler
+  HandleCtrlC();
+#endif
+
+#if defined(WIN32)
+  if (!args.mmap_path.empty()) {
+    int ret = 0;
+
+    try {
+      ChildProcSocketHandler(args.mmap_path);
+    } catch (const std::exception&) {
+      ret = -1;
+    }
+
+    return ret;
+  }
+#endif
+
+  RPCServerCreate(args.host, args.port, args.port_end, args.tracker, args.key, args.custom_addr,
+                  args.work_dir, args.silent);
+  return 0;
+}
+
+/*!
+ * \brief main The main function.
+ * \param argc arg counter
+ * \param argv arg values
+ * \return result of operation.
+ */
+int main(int argc, char* argv[]) {
+  if (argc <= 1) {
+    LOG(INFO) << kUsage;
+    return 0;
+  }
+
+  // Runs WSAStartup on Win32, no-op on POSIX
+  Socket::Startup();
+#if defined(_WIN32)
+  SetEnvironmentVariableA("CUDA_CACHE_DISABLE", "1");
+#endif
+
+  if (0 == strcmp(argv[1], "server")) {
+    return RpcServer(argc, argv);
+  }
+
+  LOG(INFO) << kUsage;
+
+  return 0;
+}

apps/cpp_rpc/rpc_env.cc

@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*!
+ * \file rpc_env.cc
+ * \brief Server environment of the RPC.
+ */
+#include "rpc_env.h"
+
+#include <tvm/ffi/extra/module.h>
+#include <tvm/ffi/function.h>
+#include <tvm/runtime/logging.h>
+
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <system_error>
+#include <vector>
+
+#include "../../src/support/utils.h"
+
+namespace {
+std::string GenerateUntarCommand(const std::string& tar_file, const std::string& output_dir) {
+  std::string untar_cmd;
+  untar_cmd.reserve(512);
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+  untar_cmd += "tar -C ";
+  untar_cmd += output_dir;
+  untar_cmd += " -zxf ";
+  untar_cmd += tar_file;
+#elif defined(_WIN32)
+  untar_cmd += "python -m tarfile -e ";
+  untar_cmd += tar_file;
+  untar_cmd += " ";
+  untar_cmd += output_dir;
+#endif
+  return untar_cmd;
+}
+
+}  // Anonymous namespace
+
+namespace tvm {
+namespace runtime {
+
+RPCEnv::RPCEnv(const std::string& wd) {
+  std::error_code ec;
+  if (!wd.empty()) {
+    base_ = wd + "/.cache";
+    std::filesystem::create_directories(base_, ec);
+    if (ec) {
+      LOG(WARNING) << "Failed to create directory " << base_ << " : " << ec.message();
+    }
+  } else {
+#if defined(ANDROID) || defined(__ANDROID__)
+    std::string pkg_name;
+    if (std::ifstream cmdline("/proc/self/cmdline"); cmdline) {
+      std::getline(cmdline, pkg_name, '\0');
+    }
+    std::string android_base_ = "/data/data/" + pkg_name + "/cache";
+    // Check if application data directory exist. If not exist, usually means we run tvm_rpc from
+    // adb shell terminal.
+    if (!std::filesystem::is_directory(android_base_)) {
+      // Tmp directory is always writable for 'shell' user.
+      android_base_ = "/data/local/tmp";
+    }
+    base_ = android_base_ + "/rpc";
+#elif !defined(_WIN32)
+    base_ = std::filesystem::current_path(ec).string() + "/rpc";
+    if (ec) {
+      base_ = "./rpc";
+    }
+#else
+    base_ = "./rpc";
+#endif
+    std::filesystem::create_directories(base_, ec);
+    if (ec) {
+      LOG(WARNING) << "Failed to create directory " << base_ << " : " << ec.message();
+    }
+  }
+  std::filesystem::permissions(base_, std::filesystem::perms::all,
+                               std::filesystem::perm_options::replace, ec);
+  if (ec) {
+    LOG(WARNING) << "Failed to grant permissions to " << base_ << " : " << ec.message();
+  }
+
+  ffi::Function::SetGlobal(
+      "tvm.rpc.server.workpath",
+      ffi::Function::FromTyped([this](const std::string& path) { return this->GetPath(path); }));
+
+  ffi::Function::SetGlobal("tvm.rpc.server.listdir",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string dir = this->GetPath(path);
+                             std::ostringstream os;
+                             for (auto d : ListDir(dir)) {
+                               os << d << ",";
+                             }
+                             return os.str();
+                           }));
+
+  ffi::Function::SetGlobal("tvm.rpc.server.load_module",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string file_name = this->GetPath(path);
+                             file_name = BuildSharedLibrary(file_name);
+                             LOG(INFO) << "Load module from " << file_name << " ...";
+                             return ffi::Module::LoadFromFile(file_name);
+                           }));
+
+  ffi::Function::SetGlobal("tvm.rpc.server.download_linked_module",
+                           ffi::Function::FromTyped([this](const std::string& path) {
+                             std::string file_name = this->GetPath(path);
+                             file_name = BuildSharedLibrary(file_name);
+                             std::string bin;
+
+                             std::ifstream fs(file_name, std::ios::in | std::ios::binary);
+                             TVM_FFI_ICHECK(!fs.fail()) << "Cannot open " << file_name;
+                             fs.seekg(0, std::ios::end);
+                             size_t size = static_cast<size_t>(fs.tellg());
+                             fs.seekg(0, std::ios::beg);
+                             bin.resize(size);
+                             fs.read(bin.data(), size);
+                             LOG(INFO) << "Send linked module " << file_name << " to client";
+                             return ffi::Bytes(bin);
+                           }));
+}
+
+/*!
+ * \brief GetPath To get the work path from packed function
+ * \param file_name The file name
+ * \return The full path of file.
+ */
+std::string RPCEnv::GetPath(const std::string& file_name) const {
+  // we assume file_name starts with "/" means file_name is the exact path
+  // and does not create /.rpc/
+  return !file_name.empty() && file_name[0] == '/' ? file_name : base_ + "/" + file_name;
+}
+
+/*!
+ * \brief Remove The RPC Environment cleanup function
+ */
+void RPCEnv::CleanUp() const {
+  std::error_code ec;
+  std::filesystem::remove_all(base_, ec);
+  if (ec) {
+    LOG(WARNING) << "Cleanup " << base_ << " failed: " << ec.message();
+  }
+}
+
+/*!
+ * \brief ListDir get the list of files in a directory
+ * \param dirname The root directory name
+ * \return vector Files in directory.
+ */
+std::vector<std::string> ListDir(const std::string& dirname) {
+  std::error_code ec;
+  std::vector<std::string> vec;
+  auto iter = std::filesystem::directory_iterator(dirname, ec);
+  if (ec) {
+    if (ec == std::errc::no_such_file_or_directory) return vec;
+    TVM_FFI_THROW(InternalError) << "ListDir " << dirname << " error: " << ec.message();
+  }
+  for (const auto& entry : iter) {
+    vec.push_back(entry.path().generic_string());
+  }
+  return vec;
+}
+
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+/*!
+ * \brief LinuxShared Creates a linux shared library
+ * \param output The output file name
+ * \param files The files for building
+ * \param options The compiler options
+ * \param cc The compiler
+ */
+void LinuxShared(const std::string output, const std::vector<std::string>& files,
+                 std::string options = "", std::string cc = "g++") {
+  std::string cmd = cc;
+  cmd += " -shared -fPIC ";
+  cmd += " -o " + output;
+  for (auto f = files.begin(); f != files.end(); ++f) {
+    cmd += " " + *f;
+  }
+  cmd += " " + options;
+  std::string err_msg;
+  auto executed_status = support::Execute(cmd, &err_msg);
+  if (executed_status) {
+    TVM_FFI_THROW(InternalError) << err_msg;
+  }
+}
+#endif
+
+#ifdef _WIN32
+/*!
+ * \brief WindowsShared Creates a Windows shared library
+ * \param output The output file name
+ * \param files The files for building
+ * \param options The compiler options
+ * \param cc The compiler
+ */
+void WindowsShared(const std::string& output, const std::vector<std::string>& files,
+                   const std::string& options = "", const std::string& cc = "clang") {
+  std::string cmd = cc;
+  cmd += " -O2 -flto=full -fuse-ld=lld-link -shared ";
+  cmd += " -o " + output;
+  for (const auto& file : files) {
+    cmd += " " + file;
+  }
+  cmd += " " + options;
+  std::string err_msg;
+  const auto executed_status = support::Execute(cmd, &err_msg);
+  if (executed_status) {
+    TVM_FFI_THROW(InternalError) << err_msg;
+  }
+}
+#endif
+
+/*!
+ * \brief CreateShared Creates a shared library
+ * \param output The output file name
+ * \param files The files for building
+ */
+void CreateShared(const std::string& output, const std::vector<std::string>& files) {
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+  LinuxShared(output, files);
+#elif defined(_WIN32)
+  WindowsShared(output, files);
+#else
+  TVM_FFI_THROW(InternalError) << "Operating system not supported";
+#endif
+}
+
+std::string BuildSharedLibrary(std::string file) {
+  if (support::EndsWith(file, ".so") || support::EndsWith(file, ".dll") ||
+      support::EndsWith(file, ".dylib")) {
+    return file;
+  }
+
+  std::string file_name = file + ".so";
+  if (support::EndsWith(file, ".o")) {
+    CreateShared(file_name, {file});
+  } else if (support::EndsWith(file, ".tar")) {
+    const std::string tmp_dir = "./rpc/tmp/";
+    std::error_code ec;
+    std::filesystem::create_directories(tmp_dir, ec);
+    if (ec) {
+      LOG(WARNING) << "Failed to create directory " << tmp_dir << " : " << ec.message();
+    }
+    std::filesystem::permissions(tmp_dir, std::filesystem::perms::all,
+                                 std::filesystem::perm_options::replace, ec);
+    if (ec) {
+      LOG(WARNING) << "Failed to grant permissions to " << tmp_dir << " : " << ec.message();
+    }
+
+    const std::string cmd = GenerateUntarCommand(file, tmp_dir);
+
+    std::string err_msg;
+    const int executed_status = support::Execute(cmd, &err_msg);
+    if (executed_status) {
+      TVM_FFI_THROW(InternalError) << err_msg;
+    }
+    CreateShared(file_name, ListDir(tmp_dir));
+    std::filesystem::remove_all(tmp_dir, ec);
+    if (ec) {
+      LOG(WARNING) << "Remove " << tmp_dir << " failed: " << ec.message();
+    }
+  } else {
+    file_name = file;
+  }
+  return file_name;
+}
+
+}  // namespace runtime
+}  // namespace tvm

apps/cpp_rpc/rpc_env.h

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_env.h
+ * \brief Server environment of the RPC.
+ */
+#ifndef TVM_APPS_CPP_RPC_ENV_H_
+#define TVM_APPS_CPP_RPC_ENV_H_
+
+#include <tvm/ffi/function.h>
+
+#include <string>
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief ListDir Get the list of files in a directory
+ * \param dirname The root directory name
+ * \return vector Files in directory.
+ */
+std::vector<std::string> ListDir(const std::string& dirname);
+
+/*!
+ * \brief build a shared library if necessary
+ *
+ *        This function will automatically call
+ *        cc.create_shared if the path is in format .o or .tar
+ *        High level handling for .o and .tar file.
+ *        We support this to be consistent with RPC module load.
+ * \param file_in The input file path.
+ *
+ * \return The name of the shared library.
+ */
+std::string BuildSharedLibrary(std::string file_in);
+
+/*!
+ * \brief RPCEnv The RPC Environment parameters for c++ rpc server
+ */
+struct RPCEnv {
+ public:
+  /*!
+   * \brief Constructor Init The RPC Environment initialize function
+   */
+  RPCEnv(const std::string& word_dir = "");
+  /*!
+   * \brief GetPath To get the workpath from packed function
+   * \param name The file name
+   * \return The full path of file.
+   */
+  std::string GetPath(const std::string& file_name) const;
+  /*!
+   * \brief The RPC Environment cleanup function
+   */
+  void CleanUp() const;
+
+ private:
+  /*!
+   * \brief Holds the environment path.
+   */
+  std::string base_;
+};  // RPCEnv
+
+}  // namespace runtime
+}  // namespace tvm
+#endif  // TVM_APPS_CPP_RPC_ENV_H_

apps/cpp_rpc/rpc_server.cc

@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_server.cc
+ * \brief RPC Server implementation.
+ */
+#include <tvm/ffi/function.h>
+#include <tvm/ffi/reflection/registry.h>
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+#include <signal.h>
+#include <sys/select.h>
+#include <sys/wait.h>
+#endif
+#include <chrono>
+#include <future>
+#include <iostream>
+#include <set>
+#include <string>
+#include <thread>
+#include <utility>
+
+#include "../../src/runtime/rpc/rpc_endpoint.h"
+#include "../../src/runtime/rpc/rpc_socket_impl.h"
+#include "../../src/support/socket.h"
+#include "rpc_env.h"
+#include "rpc_server.h"
+#include "rpc_tracker_client.h"
+#if defined(_WIN32)
+#include "win32_process.h"
+#endif
+
+using namespace std::chrono;
+
+namespace tvm {
+namespace runtime {
+
+#ifdef __ANDROID__
+static std::string getNextString(std::stringstream* iss) {
+  std::string str = iss->str();
+  size_t start = iss->tellg();
+  size_t len = str.size();
+  // Skip leading spaces.
+  while (start < len && isspace(str[start])) start++;
+
+  size_t end = start;
+  while (end < len && !isspace(str[end])) end++;
+
+  iss->seekg(end);
+  return str.substr(start, end - start);
+}
+#endif
+
+/*!
+ * \brief RPCServer RPC Server class.
+ *
+ * \param host The listen address of the server, Default=0.0.0.0 (any)
+ *
+ * \param port_search_start The low end of the search range for an
+ *     available port for the RPC, Default=9090
+ *
+ * \param port_search_end The high search the search range for an
+ *     available port for the RPC, Default=9099
+ *
+ * \param tracker The address of RPC tracker in host:port format
+ *     (e.g. "10.77.1.234:9190")
+ *
+ * \param key The key used to identify the device type in tracker.
+ *
+ * \param custom_addr Custom IP Address to Report to RPC Tracker.
+ */
+class RPCServer {
+ public:
+  /*!
+   * \brief Constructor.
+   */
+  RPCServer(std::string host, int port_search_start, int port_search_end, std::string tracker_addr,
+            std::string key, std::string custom_addr, std::string work_dir)
+      : host_(std::move(host)),
+        port_search_start_(port_search_start),
+        my_port_(0),
+        port_search_end_(port_search_end),
+        tracker_addr_(std::move(tracker_addr)),
+        key_(std::move(key)),
+        custom_addr_(std::move(custom_addr)),
+        work_dir_(std::move(work_dir)) {}
+
+  /*!
+   * \brief Destructor.
+   */
+  ~RPCServer() {
+    try {
+      // Free the resources
+      tracker_sock_.Close();
+      listen_sock_.Close();
+    } catch (...) {
+    }
+  }
+
+  /*!
+   * \brief Start Creates the RPC listen process and execution.
+   */
+  void Start() {
+    listen_sock_.Create();
+    my_port_ = listen_sock_.TryBindHost(host_, port_search_start_, port_search_end_);
+    LOG(INFO) << "Bind to " << host_ << ":" << my_port_;
+    listen_sock_.Listen(1);
+    std::future<void> proc(std::async(std::launch::async, &RPCServer::ListenLoopProc, this));
+    proc.get();
+    // Close the listen socket
+    listen_sock_.Close();
+  }
+
+ private:
+  /*!
+   * \brief ListenLoopProc The listen process.
+   */
+  void ListenLoopProc() {
+    TrackerClient tracker(tracker_addr_, key_, custom_addr_, my_port_);
+    while (true) {
+      support::TCPSocket conn;
+      support::SockAddr addr("0.0.0.0", 0);
+      std::string opts;
+      try {
+        // step 1: setup tracker and report to tracker
+        tracker.TryConnect();
+        // step 2: wait for in-coming connections
+        AcceptConnection(&tracker, &conn, &addr, &opts);
+      } catch (const char* msg) {
+        LOG(WARNING) << "Socket exception: " << msg;
+        // close tracker resource
+        tracker.Close();
+        continue;
+      } catch (const std::exception& e) {
+        // close tracker resource
+        tracker.Close();
+        LOG(WARNING) << "Exception standard: " << e.what();
+        continue;
+      }
+
+      int timeout = GetTimeOutFromOpts(opts);
+#if defined(__linux__) || defined(__ANDROID__) || defined(__APPLE__)
+      // step 3: serving
+      if (timeout != 0) {
+        const pid_t worker_pid = fork();
+        if (worker_pid == 0) {
+          // Worker process
+          ServerLoopProc(conn, addr, work_dir_);
+          _exit(0);
+        }
+
+        int status = 0;
+        bool timed_out = false;
+        auto start_timer = std::chrono::steady_clock::now();
+        while (true) {
+          // Check worker pid (non-blocking)
+          int ret = waitpid(worker_pid, &status, WNOHANG);
+          if (ret == worker_pid) {
+            break;
+          } else if (ret == -1) {
+            if (errno == EINTR) continue;
+            break;
+          }
+          // Check worker timeout
+          if (std::chrono::steady_clock::now() - start_timer >= std::chrono::seconds(timeout)) {
+            timed_out = true;
+            kill(worker_pid, SIGTERM);
+            waitpid(worker_pid, &status, 0);
+            break;
+          }
+          std::this_thread::sleep_for(std::chrono::milliseconds(50));
+        }
+
+        // Logging.
+        if (timed_out) {
+          LOG(INFO) << "Child pid=" << worker_pid << " killed"
+                    << " (timeout = " << timeout << " sec)"
+                    << ", status = " << status;
+        } else {
+          LOG(INFO) << "Child pid=" << worker_pid << " finished"
+                    << ", status = " << status;
+        }
+      } else {
+        auto pid = fork();
+        if (pid == 0) {
+          ServerLoopProc(conn, addr, work_dir_);
+          _exit(0);
+        }
+        // Wait for the result
+        int status = 0;
+        wait(&status);
+        LOG(INFO) << "Child pid=" << pid << " exited, status =" << status;
+      }
+#elif defined(WIN32)
+      auto start_time = high_resolution_clock::now();
+      try {
+        SpawnRPCChild(conn.sockfd, seconds(timeout));
+      } catch (const std::exception&) {
+      }
+      auto dur = high_resolution_clock::now() - start_time;
+
+      LOG(INFO) << "Serve Time " << duration_cast<milliseconds>(dur).count() << "ms";
+#else
+      LOG(WARNING) << "Unknown platform. It is not known how to bring up the subprocess."
+                   << " RPC will be launched in the main thread.";
+      ServerLoopProc(conn, addr, work_dir_);
+#endif
+      // close from our side.
+      LOG(INFO) << "End session with " << addr.AsString();
+      conn.Close();
+    }
+  }
+
+  /*!
+   * \brief AcceptConnection Accepts the RPC Server connection.
+   * \param tracker Tracker details.
+   * \param conn_sock New connection information.
+   * \param addr New connection address information.
+   * \param opts Parsed options for socket
+   * \param ping_period Timeout for select call waiting
+   */
+  void AcceptConnection(TrackerClient* tracker, support::TCPSocket* conn_sock,
+                        support::SockAddr* addr, std::string* opts, int ping_period = 2) {
+    std::set<std::string> old_keyset;
+    std::string matchkey;
+
+    // Report resource to tracker and get key
+    tracker->ReportResourceAndGetKey(my_port_, &matchkey);
+
+    while (true) {
+      tracker->WaitConnectionAndUpdateKey(listen_sock_, my_port_, ping_period, &matchkey);
+      support::TCPSocket conn = listen_sock_.Accept(addr);
+
+      int code = kRPCMagic;
+      TVM_FFI_ICHECK_EQ(conn.RecvAll(&code, sizeof(code)), sizeof(code));
+      if (code != kRPCMagic) {
+        conn.Close();
+        TVM_FFI_THROW(InternalError) << "Client connected is not TVM RPC server";
+        continue;
+      }
+
+      int keylen = 0;
+      TVM_FFI_ICHECK_EQ(conn.RecvAll(&keylen, sizeof(keylen)), sizeof(keylen));
+
+      const char* CLIENT_HEADER = "client:";
+      const char* SERVER_HEADER = "server:";
+      std::string expect_header = CLIENT_HEADER + matchkey;
+      std::string server_key = SERVER_HEADER + key_;
+      if (size_t(keylen) < expect_header.length()) {
+        conn.Close();
+        LOG(INFO) << "Wrong client header length";
+        continue;
+      }
+
+      TVM_FFI_ICHECK_NE(keylen, 0);
+      std::string remote_key;
+      remote_key.resize(keylen);
+      TVM_FFI_ICHECK_EQ(conn.RecvAll(&remote_key[0], keylen), keylen);
+
+      std::stringstream ssin(remote_key);
+      std::string arg0;
+#ifndef __ANDROID__
+      ssin >> arg0;
+#else
+      arg0 = getNextString(&ssin);
+#endif
+
+      if (arg0 != expect_header) {
+        code = kRPCMismatch;
+        TVM_FFI_ICHECK_EQ(conn.SendAll(&code, sizeof(code)), sizeof(code));
+        conn.Close();
+        LOG(WARNING) << "Mismatch key from" << addr->AsString();
+        continue;
+      } else {
+        code = kRPCSuccess;
+        TVM_FFI_ICHECK_EQ(conn.SendAll(&code, sizeof(code)), sizeof(code));
+        keylen = int(server_key.length());
+        TVM_FFI_ICHECK_EQ(conn.SendAll(&keylen, sizeof(keylen)), sizeof(keylen));
+        TVM_FFI_ICHECK_EQ(conn.SendAll(server_key.c_str(), keylen), keylen);
+        LOG(INFO) << "New session from " << addr->AsString();
+#ifndef __ANDROID__
+        ssin >> *opts;
+#else
+        *opts = getNextString(&ssin);
+#endif
+        *conn_sock = conn;
+        return;
+      }
+    }
+  }
+
+  /*!
+   * \brief ServerLoopProc The Server loop process.
+   * \param sock The socket information
+   * \param addr The socket address information
+   */
+  static void ServerLoopProc(support::TCPSocket sock, support::SockAddr addr,
+                             std::string work_dir) {
+    // Server loop
+    const auto s_time = std::chrono::high_resolution_clock::now();
+    const auto env = RPCEnv(work_dir);
+    RPCServerLoop(int(sock.sockfd));
+    const auto e_time = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<double> elapsed = e_time - s_time;
+    LOG(INFO) << "Finished serving " << addr.AsString() << " after " << elapsed.count() << " sec";
+    env.CleanUp();
+  }
+
+  /*!
+   * \brief GetTimeOutFromOpts Parse and get the timeout option.
+   * \param opts The option string
+   */
+  int GetTimeOutFromOpts(const std::string& opts) const {
+    const std::string option = "-timeout=";
+
+    size_t pos = opts.rfind(option);
+    if (pos != std::string::npos) {
+      const std::string cmd = opts.substr(pos + option.size());
+      TVM_FFI_ICHECK(support::IsNumber(cmd)) << "Timeout is not valid";
+      return std::stoi(cmd);
+    }
+    return 0;
+  }
+
+  std::string host_;
+  int port_search_start_;
+  int my_port_;
+  int port_search_end_;
+  std::string tracker_addr_;
+  std::string key_;
+  std::string custom_addr_;
+  std::string work_dir_;
+  support::TCPSocket listen_sock_;
+  support::TCPSocket tracker_sock_;
+};
+
+#if defined(WIN32)
+/*!
+ * \brief ServerLoopFromChild The Server loop process.
+ * \param socket The socket information
+ */
+void ServerLoopFromChild(SOCKET socket) {
+  // Server loop
+  tvm::support::TCPSocket sock(socket);
+  const auto env = RPCEnv();
+  RPCServerLoop(int(sock.sockfd));
+
+  sock.Close();
+  env.CleanUp();
+}
+#endif
+
+/*!
+ * \brief RPCServerCreate Creates the RPC Server.
+ * \param host The listen address of the server, Default=0.0.0.0 (any)
+ * \param port The port of the RPC server, Default=9090
+ * \param port_end The end search port of the RPC server, Default=9099
+ * \param tracker_addr The address of RPC tracker in host:port format e.g. 10.77.1.234:9190
+ * Default="" \param key The key used to identify the device type in tracker. Default="" \param
+ * custom_addr Custom IP Address to Report to RPC Tracker. Default="" \param silent Whether run in
+ * silent mode. Default=True
+ */
+void RPCServerCreate(std::string host, int port, int port_end, std::string tracker_addr,
+                     std::string key, std::string custom_addr, std::string work_dir, bool silent) {
+  if (silent) {
+  }
+  // Start the rpc server
+  RPCServer rpc(std::move(host), port, port_end, std::move(tracker_addr), std::move(key),
+                std::move(custom_addr), std::move(work_dir));
+  rpc.Start();
+}
+
+TVM_FFI_STATIC_INIT_BLOCK() {
+  namespace refl = tvm::ffi::reflection;
+  refl::GlobalDef().def("rpc.ServerCreate", RPCServerCreate);
+}
+}  // namespace runtime
+}  // namespace tvm

apps/cpp_rpc/rpc_server.h

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_server.h
+ * \brief RPC Server implementation.
+ */
+#ifndef TVM_APPS_CPP_RPC_SERVER_H_
+#define TVM_APPS_CPP_RPC_SERVER_H_
+
+#include <string>
+
+#include "tvm/runtime/base.h"
+
+namespace tvm {
+namespace runtime {
+
+#if defined(WIN32)
+/*!
+ * \brief ServerLoopFromChild The Server loop process.
+ * \param sock The socket information
+ * \param addr The socket address information
+ */
+void ServerLoopFromChild(SOCKET socket);
+#endif
+
+/*!
+ * \brief RPCServerCreate Creates the RPC Server.
+ * \param host The listen address of the server, Default=0.0.0.0 (any)
+ * \param port The port of the RPC server, Default=9090
+ * \param port_end The end search port of the RPC server, Default=9099
+ * \param tracker The address of RPC tracker in host:port format e.g. 10.77.1.234:9190 Default=""
+ * \param key The key used to identify the device type in tracker. Default=""
+ * \param custom_addr Custom IP Address to Report to RPC Tracker. Default=""
+ * \param work_dir Custom work directory. Default=""
+ * \param silent Whether run in silent mode. Default=True
+ */
+void RPCServerCreate(std::string host = "", int port = 9090, int port_end = 9099,
+                     std::string tracker_addr = "", std::string key = "",
+                     std::string custom_addr = "", std::string work_dir = "", bool silent = true);
+}  // namespace runtime
+}  // namespace tvm
+#endif  // TVM_APPS_CPP_RPC_SERVER_H_

apps/cpp_rpc/rpc_tracker_client.h

@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file rpc_tracker_client.h
+ * \brief RPC Tracker client to report resources.
+ */
+#ifndef TVM_APPS_CPP_RPC_TRACKER_CLIENT_H_
+#define TVM_APPS_CPP_RPC_TRACKER_CLIENT_H_
+
+#include <chrono>
+#include <iostream>
+#include <random>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "../../src/runtime/rpc/rpc_endpoint.h"
+#include "../../src/support/socket.h"
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief TrackerClient Tracker client class.
+ * \param tracker The address of RPC tracker in host:port format e.g. 10.77.1.234:9190 Default=""
+ * \param key The key used to identify the device type in tracker. Default=""
+ * \param custom_addr Custom IP Address to Report to RPC Tracker. Default=""
+ */
+class TrackerClient {
+ public:
+  /*!
+   * \brief Constructor.
+   */
+  TrackerClient(const std::string& tracker_addr, const std::string& key,
+                const std::string& custom_addr, int port)
+      : tracker_addr_(tracker_addr),
+        key_(key),
+        custom_addr_(custom_addr),
+        port_(port),
+        gen_(std::random_device{}()),
+        dis_(0.0, 1.0) {
+    if (custom_addr_.empty()) {
+      custom_addr_ = "null";
+    } else {
+      // Since custom_addr_ can be either the json value null which is not surrounded by quotes
+      // or a string containing the custom value which json does required to be quoted then we
+      // need to set custom_addr_ to be a string containing the quotes here.
+      custom_addr_ = "\"" + custom_addr_ + "\"";
+    }
+  }
+  /*!
+   * \brief Destructor.
+   */
+  ~TrackerClient() {
+    // Free the resources
+    Close();
+  }
+  /*!
+   * \brief IsValid Check tracker is valid.
+   */
+  bool IsValid() { return (!tracker_addr_.empty() && !tracker_sock_.IsClosed()); }
+  /*!
+   * \brief TryConnect Connect to tracker if the tracker address is valid.
+   */
+  void TryConnect() {
+    if (!tracker_addr_.empty() && (tracker_sock_.IsClosed())) {
+      tracker_sock_ = ConnectWithRetry();
+
+      int code = kRPCTrackerMagic;
+      TVM_FFI_ICHECK_EQ(tracker_sock_.SendAll(&code, sizeof(code)), sizeof(code));
+      TVM_FFI_ICHECK_EQ(tracker_sock_.RecvAll(&code, sizeof(code)), sizeof(code));
+      TVM_FFI_ICHECK_EQ(code, kRPCTrackerMagic) << tracker_addr_.c_str() << " is not RPC Tracker";
+
+      std::ostringstream ss;
+      ss << "[" << static_cast<int>(TrackerCode::kUpdateInfo) << ", {\"key\": \"server:" << key_
+         << "\", \"addr\": [" << custom_addr_ << ", \"" << port_ << "\"]}]";
+      tracker_sock_.SendBytes(ss.str());
+
+      // Receive status and validate
+      std::string remote_status = tracker_sock_.RecvBytes();
+      TVM_FFI_ICHECK_EQ(std::stoi(remote_status), static_cast<int>(TrackerCode::kSuccess));
+    }
+  }
+  /*!
+   * \brief Close Clean up tracker resources.
+   */
+  void Close() {
+    // close tracker resource
+    if (!tracker_sock_.IsClosed()) {
+      tracker_sock_.Close();
+    }
+  }
+  /*!
+   * \brief ReportResourceAndGetKey Report resource to tracker.
+   * \param port listening port.
+   * \param matchkey Random match key output.
+   */
+  void ReportResourceAndGetKey(int port, std::string* matchkey) {
+    if (!tracker_sock_.IsClosed()) {
+      *matchkey = RandomKey(key_ + ":", old_keyset_);
+
+      std::ostringstream ss;
+      ss << "[" << static_cast<int>(TrackerCode::kPut) << ", \"" << key_ << "\", [" << port
+         << ", \"" << *matchkey << "\"], " << custom_addr_ << "]";
+
+      tracker_sock_.SendBytes(ss.str());
+
+      // Receive status and validate
+      std::string remote_status = tracker_sock_.RecvBytes();
+      TVM_FFI_ICHECK_EQ(std::stoi(remote_status), static_cast<int>(TrackerCode::kSuccess));
+    } else {
+      *matchkey = key_;
+    }
+  }
+
+  /*!
+   * \brief ReportResourceAndGetKey Report resource to tracker.
+   * \param listen_sock Listen socket details for select.
+   * \param port listening port.
+   * \param ping_period Select wait time.
+   * \param matchkey Random match key output.
+   */
+  void WaitConnectionAndUpdateKey(support::TCPSocket listen_sock, int port, int ping_period,
+                                  std::string* matchkey) {
+    int unmatch_period_count = 0;
+    int unmatch_timeout = 4;
+    while (true) {
+      if (!tracker_sock_.IsClosed()) {
+        support::PollHelper poller;
+        poller.WatchRead(listen_sock.sockfd);
+        poller.Poll(ping_period * 1000);
+        if (!poller.CheckRead(listen_sock.sockfd)) {
+          std::ostringstream ss;
+          ss << "[" << int(TrackerCode::kGetPendingMatchKeys) << "]";
+          tracker_sock_.SendBytes(ss.str());
+
+          // Receive status and validate
+          std::string pending_keys = tracker_sock_.RecvBytes();
+          old_keyset_.insert(*matchkey);
+
+          // if match key not in pending key set
+          // it means the key is acquired by a client but not used.
+          if (pending_keys.find(*matchkey) == std::string::npos) {
+            unmatch_period_count += 1;
+          } else {
+            unmatch_period_count = 0;
+          }
+          // regenerate match key if key is acquired but not used for a while
+          if (unmatch_period_count * ping_period > unmatch_timeout + ping_period) {
+            LOG(INFO) << "no incoming connections, regenerate key ...";
+
+            *matchkey = RandomKey(key_ + ":", old_keyset_);
+
+            std::ostringstream ss;
+            ss << "[" << static_cast<int>(TrackerCode::kPut) << ", \"" << key_ << "\", [" << port
+               << ", \"" << *matchkey << "\"], " << custom_addr_ << "]";
+            tracker_sock_.SendBytes(ss.str());
+
+            std::string remote_status = tracker_sock_.RecvBytes();
+            TVM_FFI_ICHECK_EQ(std::stoi(remote_status), static_cast<int>(TrackerCode::kSuccess));
+            unmatch_period_count = 0;
+          }
+          continue;
+        }
+      }
+      break;
+    }
+  }
+
+ private:
+  /*!
+   * \brief Connect to a RPC address with retry.
+            This function is only reliable to short period of server restart.
+   * \param timeout Timeout during retry
+   * \param retry_period Number of seconds before we retry again.
+   * \return TCPSocket The socket information if connect is success.
+   */
+  support::TCPSocket ConnectWithRetry(int timeout = 60, int retry_period = 5) {
+    auto tbegin = std::chrono::system_clock::now();
+    while (true) {
+      support::SockAddr addr(tracker_addr_);
+      support::TCPSocket sock;
+      sock.Create();
+      if (sock.Connect(addr)) {
+        LOG(INFO) << "Connected to tracker " << addr.AsString();
+        return sock;
+      }
+
+      auto period = (std::chrono::duration_cast<std::chrono::seconds>(
+                         std::chrono::system_clock::now() - tbegin))
+                        .count();
+      TVM_FFI_ICHECK(period < timeout) << "Failed to connect to tracker " << addr.AsString();
+      LOG(WARNING) << "Cannot connect to tracker " << addr.AsString() << " retry in "
+                   << retry_period << " seconds.";
+      std::this_thread::sleep_for(std::chrono::seconds(retry_period));
+    }
+  }
+  /*!
+   * \brief Random Generate a random number between 0 and 1.
+   * \return random float value.
+   */
+  float Random() { return dis_(gen_); }
+  /*!
+   * \brief Generate a random key.
+   * \param prefix The string prefix.
+   * \return cmap The conflict map set.
+   */
+  std::string RandomKey(const std::string& prefix, const std::set<std::string>& cmap) {
+    if (!cmap.empty()) {
+      while (true) {
+        std::string key = prefix + std::to_string(Random());
+        if (cmap.find(key) == cmap.end()) {
+          return key;
+        }
+      }
+    }
+    return prefix + std::to_string(Random());
+  }
+
+  std::string tracker_addr_;
+  std::string key_;
+  std::string custom_addr_;
+  int port_;
+  support::TCPSocket tracker_sock_;
+  std::set<std::string> old_keyset_;
+  std::mt19937 gen_;
+  std::uniform_real_distribution<float> dis_;
+};
+}  // namespace runtime
+}  // namespace tvm
+#endif  // TVM_APPS_CPP_RPC_TRACKER_CLIENT_H_

apps/cpp_rpc/win32_process.cc

@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include "win32_process.h"
+
+#include <conio.h>
+#include <tvm/runtime/logging.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#include <cstdio>
+#include <memory>
+#include <stdexcept>
+#include <string>
+
+#include "rpc_server.h"
+
+using namespace std::chrono;
+using namespace tvm::runtime;
+
+namespace {
+// The prefix path for the memory mapped file used to store IPC information
+const std::string kMemoryMapPrefix = "/MAPPED_FILE/TVM_RPC";
+// Used to construct unique names for named resources in the parent process
+const std::string kParent = "parent";
+// Used to construct unique names for named resources in the child process
+const std::string kChild = "child";
+// The timeout of the WIN32 events, in the parent and the child
+const milliseconds kEventTimeout(2000);
+
+// Used to create unique WIN32 mmap paths and event names
+int child_counter_ = 0;
+
+/*!
+ * \brief HandleDeleter Deleter for UniqueHandle smart pointer
+ * \param handle The WIN32 HANDLE to manage
+ */
+struct HandleDeleter {
+  void operator()(HANDLE handle) const {
+    if (handle != INVALID_HANDLE_VALUE && handle != nullptr) {
+      CloseHandle(handle);
+    }
+  }
+};
+
+/*!
+ * \brief UniqueHandle Smart pointer to manage a WIN32 HANDLE
+ */
+using UniqueHandle = std::unique_ptr<void, HandleDeleter>;
+
+/*!
+ * \brief MakeUniqueHandle Helper method to construct a UniqueHandle
+ * \param handle The WIN32 HANDLE to manage
+ */
+UniqueHandle MakeUniqueHandle(HANDLE handle) {
+  if (handle == INVALID_HANDLE_VALUE || handle == nullptr) {
+    return nullptr;
+  }
+
+  return UniqueHandle(handle);
+}
+
+/*!
+ * \brief GetSocket Gets the socket info from the parent process and duplicates the socket
+ * \param mmap_path The path to the memory mapped info set by the parent
+ */
+SOCKET GetSocket(const std::string& mmap_path) {
+  WSAPROTOCOL_INFO protocol_info;
+
+  const std::string parent_event_name = mmap_path + kParent;
+  const std::string child_event_name = mmap_path + kChild;
+
+  // Open the events
+  UniqueHandle parent_file_mapping_event;
+  if ((parent_file_mapping_event = MakeUniqueHandle(
+           OpenEventA(SYNCHRONIZE, false, parent_event_name.c_str()))) == nullptr) {
+    TVM_FFI_THROW(InternalError) << "OpenEvent() failed: " << GetLastError();
+  }
+
+  UniqueHandle child_file_mapping_event;
+  if ((child_file_mapping_event = MakeUniqueHandle(
+           OpenEventA(EVENT_MODIFY_STATE, false, child_event_name.c_str()))) == nullptr) {
+    TVM_FFI_THROW(InternalError) << "OpenEvent() failed: " << GetLastError();
+  }
+
+  // Wait for the parent to set the event, notifying WSAPROTOCOL_INFO is ready to be read
+  if (WaitForSingleObject(parent_file_mapping_event.get(), uint32_t(kEventTimeout.count())) !=
+      WAIT_OBJECT_0) {
+    TVM_FFI_THROW(InternalError) << "WaitForSingleObject() failed: " << GetLastError();
+  }
+
+  const UniqueHandle file_map =
+      MakeUniqueHandle(OpenFileMappingA(FILE_MAP_READ | FILE_MAP_WRITE, false, mmap_path.c_str()));
+  if (!file_map) {
+    LOG(INFO) << "CreateFileMapping() failed: " << GetLastError();
+  }
+
+  void* map_view = MapViewOfFile(file_map.get(), FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
+
+  SOCKET sock_duplicated = INVALID_SOCKET;
+
+  if (map_view != nullptr) {
+    memcpy(&protocol_info, map_view, sizeof(WSAPROTOCOL_INFO));
+    UnmapViewOfFile(map_view);
+
+    // Creates the duplicate socket, that was created in the parent
+    sock_duplicated =
+        WSASocket(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, &protocol_info, 0, 0);
+
+    // Let the parent know we are finished duplicating the socket
+    SetEvent(child_file_mapping_event.get());
+  } else {
+    TVM_FFI_THROW(InternalError) << "MapViewOfFile() failed: " << GetLastError();
+  }
+
+  return sock_duplicated;
+}
+}  // Anonymous namespace
+
+namespace tvm {
+namespace runtime {
+/*!
+ * \brief SpawnRPCChild Spawns a child process with a given timeout to run
+ * \param fd The client socket to duplicate in the child
+ * \param timeout The time in seconds to wait for the child to complete before termination
+ */
+void SpawnRPCChild(SOCKET fd, seconds timeout) {
+  STARTUPINFOA startup_info;
+
+  memset(&startup_info, 0, sizeof(startup_info));
+  startup_info.cb = sizeof(startup_info);
+
+  std::string file_map_path = kMemoryMapPrefix + std::to_string(child_counter_++);
+
+  const std::string parent_event_name = file_map_path + kParent;
+  const std::string child_event_name = file_map_path + kChild;
+
+  // Create an event to let the child know the socket info was set to the mmap file
+  UniqueHandle parent_file_mapping_event;
+  if ((parent_file_mapping_event = MakeUniqueHandle(
+           CreateEventA(nullptr, true, false, parent_event_name.c_str()))) == nullptr) {
+    TVM_FFI_THROW(InternalError) << "CreateEvent for parent file mapping failed";
+  }
+
+  UniqueHandle child_file_mapping_event;
+  // An event to let the parent know the socket info was read from the mmap file
+  if ((child_file_mapping_event = MakeUniqueHandle(
+           CreateEventA(nullptr, true, false, child_event_name.c_str()))) == nullptr) {
+    TVM_FFI_THROW(InternalError) << "CreateEvent for child file mapping failed";
+  }
+
+  char current_executable[MAX_PATH];
+
+  // Get the full path of the current executable
+  GetModuleFileNameA(nullptr, current_executable, MAX_PATH);
+
+  std::string child_command_line = current_executable;
+  child_command_line += " server --child_proc=";
+  child_command_line += file_map_path;
+
+  // CreateProcessA requires a non const char*, so we copy our std::string
+  auto command_line_ptr = std::make_unique<char[]>(child_command_line.size() + 1);
+  strcpy(command_line_ptr.get(), child_command_line.c_str());
+
+  PROCESS_INFORMATION child_process_info;
+  if (CreateProcessA(nullptr, command_line_ptr.get(), nullptr, nullptr, false, CREATE_NO_WINDOW,
+                     nullptr, nullptr, &startup_info, &child_process_info)) {
+    // Child process and thread handles must be closed, so wrapped in RAII
+    auto child_process_handle = MakeUniqueHandle(child_process_info.hProcess);
+    auto child_process_thread_handle = MakeUniqueHandle(child_process_info.hThread);
+
+    WSAPROTOCOL_INFO protocol_info;
+    // Get info needed to duplicate the socket
+    if (WSADuplicateSocket(fd, child_process_info.dwProcessId, &protocol_info) == SOCKET_ERROR) {
+      TVM_FFI_THROW(InternalError) << "WSADuplicateSocket(): failed. Error =" << WSAGetLastError();
+    }
+
+    // Create a mmap file to store the info needed for duplicating the SOCKET in the child proc
+    UniqueHandle file_map =
+        MakeUniqueHandle(CreateFileMappingA(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0,
+                                            sizeof(WSAPROTOCOL_INFO), file_map_path.c_str()));
+    if (!file_map) {
+      LOG(INFO) << "CreateFileMapping() failed: " << GetLastError();
+    }
+
+    if (GetLastError() == ERROR_ALREADY_EXISTS) {
+      TVM_FFI_THROW(InternalError) << "CreateFileMapping(): mapping file already exists";
+    } else {
+      void* map_view = MapViewOfFile(file_map.get(), FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0);
+
+      if (map_view != nullptr) {
+        memcpy(map_view, &protocol_info, sizeof(WSAPROTOCOL_INFO));
+        UnmapViewOfFile(map_view);
+
+        // Let child proc know the mmap file is ready to be read
+        SetEvent(parent_file_mapping_event.get());
+
+        // Wait for the child to finish reading mmap file
+        if (WaitForSingleObject(child_file_mapping_event.get(), uint32_t(kEventTimeout.count())) !=
+            WAIT_OBJECT_0) {
+          TerminateProcess(child_process_handle.get(), 0);
+          TVM_FFI_THROW(InternalError)
+              << "WaitForSingleObject for child file mapping timed out.  Terminating child "
+                 "process.";
+        }
+      } else {
+        TerminateProcess(child_process_handle.get(), 0);
+        TVM_FFI_THROW(InternalError) << "MapViewOfFile() failed: " << GetLastError();
+      }
+    }
+
+    const DWORD process_timeout =
+        timeout.count() ? uint32_t(duration_cast<milliseconds>(timeout).count()) : INFINITE;
+
+    // Wait for child process to exit, or hit configured timeout
+    if (WaitForSingleObject(child_process_handle.get(), process_timeout) != WAIT_OBJECT_0) {
+      LOG(INFO) << "Child process timeout.  Terminating.";
+      TerminateProcess(child_process_handle.get(), 0);
+    }
+  } else {
+    LOG(INFO) << "Create child process failed: " << GetLastError();
+  }
+}
+/*!
+ * \brief ChildProcSocketHandler Ran from the child process and runs server to handle the client
+ * socket \param mmap_path The memory mapped file path that will contain the information to
+ * duplicate the client socket from the parent
+ */
+void ChildProcSocketHandler(const std::string& mmap_path) {
+  SOCKET socket;
+
+  // Set high thread priority to avoid the thread scheduler from
+  // interfering with any measurements in the RPC server.
+  SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL);
+
+  if ((socket = GetSocket(mmap_path)) != INVALID_SOCKET) {
+    tvm::runtime::ServerLoopFromChild(socket);
+  } else {
+    TVM_FFI_THROW(InternalError) << "GetSocket() failed";
+  }
+}
+}  // namespace runtime
+}  // namespace tvm

apps/cpp_rpc/win32_process.h

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file win32_process.h
+ * \brief Win32 process code to mimic a POSIX fork()
+ */
+#ifndef TVM_APPS_CPP_RPC_WIN32_PROCESS_H_
+#define TVM_APPS_CPP_RPC_WIN32_PROCESS_H_
+
+#include <chrono>
+#include <string>
+
+#include "../../src/support/socket.h"
+
+namespace tvm {
+namespace runtime {
+/*!
+ * \brief SpawnRPCChild Spawns a child process with a given timeout to run
+ * \param fd The client socket to duplicate in the child
+ * \param timeout The time in seconds to wait for the child to complete before termination
+ */
+void SpawnRPCChild(SOCKET fd, std::chrono::seconds timeout);
+/*!
+ * \brief ChildProcSocketHandler Ran from the child process and runs server to handle the client
+ * socket \param mmap_path The memory mapped file path that will contain the information to
+ * duplicate the client socket from the parent
+ */
+void ChildProcSocketHandler(const std::string& mmap_path);
+}  // namespace runtime
+}  // namespace tvm
+#endif  // TVM_APPS_CPP_RPC_WIN32_PROCESS_H_

apps/ios_rpc/.gitignore

@@ -0,0 +1 @@
+rpc_config.txt

apps/ios_rpc/CMakeLists.txt

@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+include(ExternalProject)
+
+# Check if xcodebuild tool is available and configured.
+# Otherwise will skip all iOS specific targets.
+execute_process(COMMAND xcodebuild -version
+        RESULT_VARIABLE XCBUILD_AVAILABLE
+        OUTPUT_QUIET
+        ERROR_QUIET
+        )
+
+if (NOT XCBUILD_AVAILABLE EQUAL 0)
+    message(WARNING
+            "The build tool xcodebuild is not properly configured. Please install Xcode app and specify "
+            "path to it via DEVELOPER_DIR env var or \"sudo xcode-select -switch <path-to-xcode-dev-dir>\".\n"
+            "iOS RPC application target is switched off."
+            )
+    return()
+endif()
+
+# External project with custom mach-o dynamic loader
+# It is required to load unsigned shared modules on real iOS devices
+ExternalProject_Add(custom_dso_loader
+        GIT_REPOSITORY https://github.com/octoml/macho-dyld.git
+        GIT_TAG d1f7032e7882bc060b49a4fb058f50a23668b074
+        PREFIX custom_dso_loader
+        LOG_DOWNLOAD TRUE
+        LOG_CONFIGURE TRUE
+        CMAKE_ARGS
+            -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>  # to install into local build dir
+            -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+            -DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}
+            -DCMAKE_SYSTEM_VERSION=${CMAKE_SYSTEM_VERSION}
+            -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT}
+            -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}
+            -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=${CMAKE_BUILD_WITH_INSTALL_NAME_DIR}
+        )
+
+if(NOT CMAKE_IOS_RPC_BUNDLE)
+    set(CMAKE_IOS_RPC_BUNDLE org.apache.tvmrpc)
+endif()
+
+# iOS RPC Xcode project wrapper to integrate into CMake
+ExternalProject_Add(ios_rpc
+        PREFIX ios_rpc
+        DEPENDS custom_dso_loader tvm_runtime
+        SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
+        CONFIGURE_COMMAND ""
+        INSTALL_COMMAND ""
+        BUILD_COMMAND xcodebuild
+            -target tvmrpc
+            -configuration ${CMAKE_BUILD_TYPE}
+            -project <SOURCE_DIR>/tvmrpc.xcodeproj
+            -sdk ${CMAKE_OSX_SYSROOT}
+            -arch ${CMAKE_OSX_ARCHITECTURES}
+            -hideShellScriptEnvironment
+            -allowProvisioningUpdates
+            build
+                SYMROOT=<BINARY_DIR>
+                IPHONEOS_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}
+                DEVELOPMENT_TEAM=${CMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM}
+                TVM_BUILD_DIR=${CMAKE_BINARY_DIR}
+                USE_CUSTOM_DSO_LOADER=1
+                PRODUCT_BUNDLE_IDENTIFIER=${CMAKE_IOS_RPC_BUNDLE}
+        )

apps/ios_rpc/README.md

@@ -0,0 +1,256 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# iOS TVM RPC
+
+This folder contains iOS RPC app that allows us to launch an rpc server on a iOS
+device. You will need XCode and an iOS device to use this.
+
+## Table of Contents
+* [Building](#building)
+    * [Building TVM runtime and custom DSO loader plugin](#building-tvm-runtime-and-custom-dso-loader-plugin)
+    * [Building iOS TVM RPC application](#building-ios-tvm-rpc-application)
+* [Workflow](#workflow)
+    * [Standalone RPC](#standalone-rpc)
+    * [iOS RPC App with proxy](#ios-rpc-app-with-proxy)
+    * [iOS RPC App with tracker](#ios-rpc-app-with-tracker)
+* [Communication without Wi-Fi and speed up in case of slow Wi-Fi](#communication-without-wi-fi-and-speed-up-in-case-of-slow-wi-fi)
+
+## Building
+### Building TVM runtime and custom DSO loader plugin
+While iOS platform itself doesn't allow us to run an unsigned binary, there is a
+partial ability to run JIT code on real iOS devices. While application is
+running under debug session, system allows allocating memory with write and
+execute permissions (a debugger requirement). So we can use this feature to
+implement the `tvm.rpc.server.load_module` PackedFunc, used to load code over
+RPC. For this purpose we use custom version of `dlopen` function which doesn't
+check signature and permissions for module loading.  This custom `dlopen`
+mechanic is integrated into TVM RPC as plugin and registered to execution only
+inside iOS RPC application.
+
+The custom implementation of `dlopen` and other functions from `dlfcn.h` header are placed in separate repository,
+and will be downloaded automatically during CMake build for iOS.
+
+Also, it is necessary to build `libtvm_runtime.dylib` for our iOS device. The
+iOS TVM RPC application will be linked with this library.
+
+Run the build using the following commands:
+```shell
+export DEVELOPER_DIR=/Applications/Xcode.app  # iOS SDK is part of Xcode bundle. Have to set it as default Dev Env
+cmake ..
+  -DCMAKE_BUILD_TYPE=Debug
+  -DCMAKE_SYSTEM_NAME=iOS
+  -DCMAKE_SYSTEM_VERSION=14.0
+  -DCMAKE_OSX_SYSROOT=iphoneos
+  -DCMAKE_OSX_ARCHITECTURES=arm64
+  -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0
+  -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON
+  -DUSE_IOS_RPC=ON  # to enable build iOS RPC application from TVM project tree
+  -DUSE_METAL=ON    # to enable Metal runtime
+
+cmake --build . --target custom_dso_loader tvm_runtime
+```
+
+### Building iOS TVM RPC application
+Before start, please run [init_proj.py](./init_proj.py) to update XCode developer metadata:
+```shell
+python3 init_proj.py --team_id XXXXXXXXXX --tvm_build_dir "/path/to/tvm/ios/build/folder"
+```
+You can get value of your `team_id` in the following ways:
+- **You have registered Apple Developer Profile**. In this case you developer
+  Team ID available at https://developer.apple.com/account/#/membership
+- You are using your local developer profile. In this case, leave `XXXXXXXXXX`
+  in the command instead of substituting a Team ID. Then open `tvmrpc.xcodeproj`
+  by using XCode, click on the project name (`tvmrpc`) on the left panel. Then
+  select target `tvmrpc`. At the bottom of this panel go to `Signing &
+  Capabilities` tab and in the field `Team` select your local developer profile
+  (`Your Name (Personal Team)`).
+
+  On the first run of the application you may see message `Could not launch
+  "tvmrpc"` in the XCode and message `Untrusted Developer` on your device. In
+  this case it will be necessary to check the certificate. Open
+  `Settings -> General -> Device Management -> Apple Development: <your_email>
+  -> Trust "Apple Development: <your_email>"` and click `Trust`. After than you
+  should rerun your application in the XCode.
+
+After this step, open `tvmrpc.xcodeproj` by using XCode, build the App and
+install the App on the phone.
+
+## Workflow
+Due to security restriction of iOS10. We cannot upload dynamic libraries to the
+App and load it from sandbox.  Instead, we need to build a list of libraries,
+pack them into the app bundle, launch the RPC server and connect to test the
+bundled libraries.  For more on the approach we use to work around this
+limitation, please take a look into section
+[Building TVM runtime and custom DSO loader integration](#building-tvm-runtime-and-custom-DSO-loader-plugin).
+
+The test script [tests/ios_rpc_test.py](tests/ios_rpc_test.py) and
+[tests/ios_rpc_mobilenet.py](tests/ios_rpc_mobilenet.py) are good templates for
+demonstrating the workflow.
+
+We have three different modes for iOS RPC server:
+- [Standalone RPC](#standalone-rpc): In this mode RPC server open port on the device and listening. Then
+  client connects to the server directly without any mediators.
+- [iOS RPC application with Proxy](#ios-rpc-app-with-proxy): RPC server and RPC client communicates through
+  `rpc_proxy`. The RPC server on iOS device notify `rpc_proxy` which was run on
+  host machine about itself and wait for incoming connections. Communications
+  between client and server works through `rpc_proxy`.
+- [iOS RPC application with Tracker](#ios-rpc-app-with-tracker): RPC server registered in the `rpc_tracker`
+  and client connects to the RPC server through `rpc_tracker`.
+
+### Standalone RPC
+Start RPC server on your iOS device:
+- Push on the `Connect` button.
+
+After that you supposed to see something like this in the app on the device:
+```
+IP: <device_ip>
+Port: <rpc_server_port>
+```
+
+Printed `IP` is the IP address of your device and `PORT` is the number of port
+which was open for RPC connection. Next you should use them for connect your RPC
+client to the server.
+
+Let's check that direct RPC connection works and we can upload a library with
+model and execute it on the device. For this purpose we will use
+[ios_rpc_test.py](tests/ios_rpc_test.py). Run it:
+```shell
+python3 tests/ios_rpc_test.py --host <device_ip> --port <rpc_server_port> --mode "standalone"
+```
+This will compile TVM IR to shared libraries (CPU and Metal) and run vector
+addition on your iOS device. You are supposed to see something like this:
+```
+Metal: 0.000338692 secs/op
+CPU: 0.000219308 secs/op
+```
+
+### iOS RPC App with proxy
+Start the RPC proxy by running in a terminal:
+```shell
+python3 -m tvm.exec.rpc_proxy --host 0.0.0.0 --port 9090
+```
+
+On success, you should see something like this:
+```
+INFO:root:RPCProxy: client port bind to 0.0.0.0:9090
+INFO:root:RPCProxy: Websock port bind to 8888
+```
+Connect your iOS device to the RPC proxy via the iOS TVM RPC application. Set
+the `Address` and `Port` fields to the address and port of the RPC tracker
+respectively. Select mode `Proxy` and push `Connect` button. In success the
+text on the button will be changed to `Disconnect` and `Disconnected` in the top
+of the screen will be changed to `Connected`.
+On RPC proxy side you can see the next message in a log:
+```
+INFO:root:Handler ready TCPSocketProxy:<iPhone IP address>:server:iphone
+```
+Then we can check that RPC connection works and we can upload a library with
+model and execute it on the target device. For this purpose we will use
+[ios_rpc_test.py](tests/ios_rpc_test.py). Run it:
+```shell
+python3 tests/ios_rpc_test.py --host <host_ip_address> --port 9090 --mode "proxy"
+```
+The output should be the same as it was in previous section.
+
+### iOS RPC App with tracker
+First start an RPC tracker using
+```shell
+python3 -m tvm.exec.rpc_tracker --host 0.0.0.0 --port 9190
+```
+On success, you should see something like this:
+```
+INFO:RPCTracker:bind to 0.0.0.0:9190
+```
+Connect your iOS device to the RPC tracker via the iOS TVM RPC applcation. Set
+the `Address` and `Port` fields to the address and port of the RPC tracker
+respectively. Select mode `Tracker` and push `Connect` button. In success the
+text on the button will be changed to `Disconnect` and `Disconnected` in the top
+of the screen will be changed to `Connected`. On the host side you can check the
+connect by the following command:
+```shell
+python3 -m tvm.exec.query_rpc_tracker --port 9190
+```
+You are supposed to see something like this:
+```
+Tracker address 127.0.0.1:9190
+
+Server List
+----------------------------
+server-address  key
+----------------------------
+192.168.1.57:9190       server:iphone
+----------------------------
+
+Queue Status
+------------------------------
+key      total  free  pending
+------------------------------
+iphone   1      1     0
+------------------------------
+```
+
+Then we can check that RPC connection works and we can upload a library with
+model and execute it on the target device. For this purpose we will use
+[ios_rpc_test.py](tests/ios_rpc_test.py). Run it:
+```shell
+python3 tests/ios_rpc_test.py --host <host_ip_address> --port 9190 --mode "tracker"
+```
+The output will be the same as in section
+[Standalone RPC](#standalone-rpc).
+
+## Communication without Wi-Fi and speed up in case of slow Wi-Fi
+Connection to the RPC server through `usbmux` can be used then you have slow,
+unstable or don't have any Wi-Fi connection. `usbmux` is used for binding local
+TCP port to port on the device and transfer packages between these ports by USB
+cable.
+
+First of all you should install `usbmux` to your system. You can do it with
+brew:
+```shell
+brew install usbmuxd
+```
+After that you can use `iproxy` program for binding ports. You can use it for
+all described workflows. Let's take a look how it works for
+[Standalone RPC](#standalone-rpc).
+
+First, start RPC server on your iOS device. You may see something like this in
+the app on the device:
+```
+IP: unknown
+Port: <rpc_server_port>
+```
+**Note.** Here `IP: unknown` because there was no Internet connection on the iOS
+device.
+Printed `Port` is the port of the RPC server on your iOS device. We will use it
+in binding ports. Run `iproxy`, specify local port which should be used for
+communication with device and the printed port on the device:
+```shell
+iproxy <local_port>:<rpc_server_port>
+```
+After this command you should see something like this:
+```
+Creating listening port <local_port> for device port <rpc_server_port>
+waiting for connection
+```
+Now we can check that RPC connection through `usbmux` works and we can upload a
+library with model and execute it on the device. For this purpose we will use
+[ios_rpc_test.py](tests/ios_rpc_test.py). Run it:
+```shell
+python3 tests/ios_rpc_test.py --host 0.0.0.0 --port <local_port> --mode standalone
+```
+The output should be the same as in all previous runs.

apps/ios_rpc/init_proj.py

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import argparse
+
+default_team_id = "3FR42MXLK9"
+default_tvm_build_dir = "path-to-tvm-ios-build-folder"
+
+parser = argparse.ArgumentParser(
+    description="Update tvmrpc.xcodeproj\
+ developer information"
+)
+parser.add_argument(
+    "--team_id",
+    type=str,
+    required=True,
+    help=f"Apple Developer Team ID.\n\
+                    Can be found here:\n\
+                    \n\
+                    https://developer.apple.com/account/#/membership\n\
+                    (example: {default_team_id})",
+)
+
+parser.add_argument(
+    "--tvm_build_dir",
+    type=str,
+    required=True,
+    help="Path to directory with libtvm_runtime.dylib",
+)
+
+args = parser.parse_args()
+team_id = args.team_id
+tvm_build_dir = args.tvm_build_dir
+
+fi = open("tvmrpc.xcodeproj/project.pbxproj")
+proj_config = fi.read()
+fi.close()
+
+proj_config = proj_config.replace(default_team_id, team_id)
+proj_config = proj_config.replace(default_tvm_build_dir, tvm_build_dir)
+fo = open("tvmrpc.xcodeproj/project.pbxproj", "w")
+fo.write(proj_config)
+fo.close()

apps/ios_rpc/tests/ios_rpc_test.py

@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Testcode for iOS RPC.
+
+To use it, start a rpc proxy with "python -m tvm.exec.rpc_proxy".
+And configure the proxy host field as commented.
+"""
+
+import argparse
+
+import numpy as np
+
+import tvm
+from tvm import rpc, te
+from tvm.support import utils, xcode
+
+# Change target configuration, this is setting for iphone6s
+arch = "arm64"
+sdk = "iphoneos"
+target = {"kind": "llvm", "mtriple": f"{arch}-apple-darwin"}
+
+MODES = {"proxy": rpc.connect, "tracker": rpc.connect_tracker, "standalone": rpc.connect}
+
+
+# override metal compiler to compile to iphone
+@tvm.register_global_func("tvm_callback_metal_compile")
+def compile_metal(src, target):
+    return xcode.compile_metal(src, sdk=sdk)
+
+
+def test_rpc_module(host, port, key, mode):
+    # graph
+    n = tvm.runtime.convert(1024)
+    A = te.placeholder((n,), name="A")
+    B = te.compute(A.shape, lambda *i: A(*i) + 1.0, name="B")
+    temp = utils.tempdir()
+    mod = tvm.IRModule.from_expr(te.create_prim_func([A, B]).with_attr("global_symbol", "myadd"))
+    sch = tvm.s_tir.Schedule(mod)
+    (i,) = sch.get_loops(block=sch.get_sblock("B"))
+    i0, i1 = sch.split(i, [None, 32])
+    sch.bind(i0, "blockIdx.x")
+    sch.bind(i1, "threadIdx.x")
+
+    # Build the dynamic lib.
+    # If we don't want to do metal and only use cpu, just set target to be target
+    f = tvm.compile(sch.mod, target=tvm.target.Target("metal", host=target))
+    path_dso1 = temp.relpath("dev_lib.dylib")
+    f.export_library(path_dso1, fcompile=xcode.create_dylib, arch=arch, sdk=sdk)
+
+    # connect to the proxy
+    if mode == "tracker":
+        remote = MODES[mode](host, port).request(key)
+    else:
+        remote = MODES[mode](host, port, key=key)
+    remote.upload(path_dso1)
+    dev = remote.metal(0)
+    f1 = remote.load_module("dev_lib.dylib")
+    a_np = np.random.uniform(size=1024).astype(A.dtype)
+    a = tvm.runtime.tensor(a_np, dev)
+    b = tvm.runtime.tensor(np.zeros(1024, dtype=A.dtype), dev)
+    time_f = f1.time_evaluator(f1.entry_name, dev, number=10)
+    cost = time_f(a, b).mean
+    print(f"Metal: {cost:g} secs/op")
+    np.testing.assert_equal(b.numpy(), a.numpy() + 1)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Demo app demonstrates how ios_rpc works.")
+    parser.add_argument("--host", required=True, type=str, help="Address of rpc server")
+    parser.add_argument("--port", type=int, default=9090, help="rpc port (default: 9090)")
+    parser.add_argument("--key", type=str, default="iphone", help="device key (default: iphone)")
+    parser.add_argument(
+        "--mode",
+        type=str,
+        default="tracker",
+        help="type of RPC connection (default: tracker), possible values: {}".format(
+            ", ".join(MODES.keys())
+        ),
+    )
+
+    args = parser.parse_args()
+    assert args.mode in MODES.keys()
+    test_rpc_module(args.host, args.port, args.key, args.mode)

apps/ios_rpc/tvmrpc.xcodeproj/project.pbxproj

@@ -0,0 +1,442 @@
+// !$*UTF8*$!
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 46;
+	objects = {
+
+/* Begin PBXBuildFile section */
+		016B19C22657B390002E1719 /* RPCServer.mm in Sources */ = {isa = PBXBuildFile; fileRef = 016B19C12657B390002E1719 /* RPCServer.mm */; };
+		01A1DB432652CBA700655BBC /* RPCArgs.mm in Sources */ = {isa = PBXBuildFile; fileRef = 01A1DB412652CBA700655BBC /* RPCArgs.mm */; };
+		01A9B7B3265BD1FD000D092F /* libtvm_runtime.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = 01A9B7B2265BD1FD000D092F /* libtvm_runtime.dylib */; };
+		01A9B7B8265BD307000D092F /* libtvm_runtime.dylib in Embed Libraries */ = {isa = PBXBuildFile; fileRef = 01A9B7B2265BD1FD000D092F /* libtvm_runtime.dylib */; settings = {ATTRIBUTES = (CodeSignOnCopy, ); }; };
+		C02637501F1C25E8007247A9 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = C026374F1F1C25E8007247A9 /* main.m */; };
+		C02637531F1C25E8007247A9 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = C02637521F1C25E8007247A9 /* AppDelegate.m */; };
+		C02637591F1C25E8007247A9 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = C02637571F1C25E8007247A9 /* Main.storyboard */; };
+		C026375B1F1C25E8007247A9 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C026375A1F1C25E8007247A9 /* Assets.xcassets */; };
+		C026375E1F1C25E8007247A9 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = C026375C1F1C25E8007247A9 /* LaunchScreen.storyboard */; };
+		C02637661F1C2690007247A9 /* TVMRuntime.mm in Sources */ = {isa = PBXBuildFile; fileRef = C02637651F1C2690007247A9 /* TVMRuntime.mm */; };
+		C02637691F1C26AF007247A9 /* ViewController.mm in Sources */ = {isa = PBXBuildFile; fileRef = C02637681F1C26AF007247A9 /* ViewController.mm */; };
+		D7685AD324390EAE00D1469C /* CoreML.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D7685AD224390EAD00D1469C /* CoreML.framework */; };
+/* End PBXBuildFile section */
+
+/* Begin PBXCopyFilesBuildPhase section */
+		01A9B7B9265BD307000D092F /* Embed Libraries */ = {
+			isa = PBXCopyFilesBuildPhase;
+			buildActionMask = 2147483647;
+			dstPath = "";
+			dstSubfolderSpec = 10;
+			files = (
+				01A9B7B8265BD307000D092F /* libtvm_runtime.dylib in Embed Libraries */,
+			);
+			name = "Embed Libraries";
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXCopyFilesBuildPhase section */
+
+/* Begin PBXFileReference section */
+		016B19C02657B390002E1719 /* RPCServer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RPCServer.h; sourceTree = "<group>"; };
+		016B19C12657B390002E1719 /* RPCServer.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = RPCServer.mm; sourceTree = "<group>"; };
+		01A1DB402652CBA700655BBC /* RPCArgs.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RPCArgs.h; sourceTree = "<group>"; };
+		01A1DB412652CBA700655BBC /* RPCArgs.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = RPCArgs.mm; sourceTree = "<group>"; };
+		01A9B7B2265BD1FD000D092F /* libtvm_runtime.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libtvm_runtime.dylib; path = "${TVM_BUILD_DIR}/libtvm_runtime.dylib"; sourceTree = "<group>"; };
+		C026374B1F1C25E8007247A9 /* tvmrpc.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = tvmrpc.app; sourceTree = BUILT_PRODUCTS_DIR; };
+		C026374F1F1C25E8007247A9 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = "<group>"; };
+		C02637511F1C25E8007247A9 /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = "<group>"; };
+		C02637521F1C25E8007247A9 /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = "<group>"; };
+		C02637541F1C25E8007247A9 /* ViewController.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ViewController.h; sourceTree = "<group>"; };
+		C02637581F1C25E8007247A9 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
+		C026375A1F1C25E8007247A9 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
+		C026375D1F1C25E8007247A9 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
+		C026375F1F1C25E8007247A9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+		C02637651F1C2690007247A9 /* TVMRuntime.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = TVMRuntime.mm; sourceTree = "<group>"; };
+		C02637681F1C26AF007247A9 /* ViewController.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = ViewController.mm; sourceTree = "<group>"; };
+		D7685AD224390EAD00D1469C /* CoreML.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreML.framework; path = System/Library/Frameworks/CoreML.framework; sourceTree = SDKROOT; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		C02637481F1C25E8007247A9 /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				01A9B7B3265BD1FD000D092F /* libtvm_runtime.dylib in Frameworks */,
+				D7685AD324390EAE00D1469C /* CoreML.framework in Frameworks */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		C02637421F1C25E8007247A9 = {
+			isa = PBXGroup;
+			children = (
+				C026374D1F1C25E8007247A9 /* tvmrpc */,
+				C026374C1F1C25E8007247A9 /* Products */,
+				D7685AD124390EAD00D1469C /* Frameworks */,
+			);
+			indentWidth = 2;
+			sourceTree = "<group>";
+			tabWidth = 2;
+		};
+		C026374C1F1C25E8007247A9 /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				C026374B1F1C25E8007247A9 /* tvmrpc.app */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		C026374D1F1C25E8007247A9 /* tvmrpc */ = {
+			isa = PBXGroup;
+			children = (
+				016B19C02657B390002E1719 /* RPCServer.h */,
+				016B19C12657B390002E1719 /* RPCServer.mm */,
+				01A1DB402652CBA700655BBC /* RPCArgs.h */,
+				01A1DB412652CBA700655BBC /* RPCArgs.mm */,
+				C02637651F1C2690007247A9 /* TVMRuntime.mm */,
+				C02637511F1C25E8007247A9 /* AppDelegate.h */,
+				C02637521F1C25E8007247A9 /* AppDelegate.m */,
+				C02637541F1C25E8007247A9 /* ViewController.h */,
+				C02637681F1C26AF007247A9 /* ViewController.mm */,
+				C02637571F1C25E8007247A9 /* Main.storyboard */,
+				C026375A1F1C25E8007247A9 /* Assets.xcassets */,
+				C026375C1F1C25E8007247A9 /* LaunchScreen.storyboard */,
+				C026375F1F1C25E8007247A9 /* Info.plist */,
+				C026374E1F1C25E8007247A9 /* Supporting Files */,
+			);
+			path = tvmrpc;
+			sourceTree = "<group>";
+		};
+		C026374E1F1C25E8007247A9 /* Supporting Files */ = {
+			isa = PBXGroup;
+			children = (
+				C026374F1F1C25E8007247A9 /* main.m */,
+			);
+			name = "Supporting Files";
+			sourceTree = "<group>";
+		};
+		D7685AD124390EAD00D1469C /* Frameworks */ = {
+			isa = PBXGroup;
+			children = (
+				01A9B7B2265BD1FD000D092F /* libtvm_runtime.dylib */,
+				D7685AD224390EAD00D1469C /* CoreML.framework */,
+			);
+			name = Frameworks;
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXNativeTarget section */
+		C026374A1F1C25E8007247A9 /* tvmrpc */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = C02637621F1C25E8007247A9 /* Build configuration list for PBXNativeTarget "tvmrpc" */;
+			buildPhases = (
+				C02637471F1C25E8007247A9 /* Sources */,
+				C02637481F1C25E8007247A9 /* Frameworks */,
+				C02637491F1C25E8007247A9 /* Resources */,
+				01A9B7B9265BD307000D092F /* Embed Libraries */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = tvmrpc;
+			productName = tvmrpc;
+			productReference = C026374B1F1C25E8007247A9 /* tvmrpc.app */;
+			productType = "com.apple.product-type.application";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		C02637431F1C25E8007247A9 /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				LastUpgradeCheck = 0830;
+				ORGANIZATIONNAME = dmlc;
+				TargetAttributes = {
+					C026374A1F1C25E8007247A9 = {
+						CreatedOnToolsVersion = 8.3.3;
+						DevelopmentTeam = 3FR42MXLK9;
+						ProvisioningStyle = Automatic;
+					};
+				};
+			};
+			buildConfigurationList = C02637461F1C25E8007247A9 /* Build configuration list for PBXProject "tvmrpc" */;
+			compatibilityVersion = "Xcode 3.2";
+			developmentRegion = English;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				English,
+				en,
+				Base,
+			);
+			mainGroup = C02637421F1C25E8007247A9;
+			productRefGroup = C026374C1F1C25E8007247A9 /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				C026374A1F1C25E8007247A9 /* tvmrpc */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXResourcesBuildPhase section */
+		C02637491F1C25E8007247A9 /* Resources */ = {
+			isa = PBXResourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				C026375E1F1C25E8007247A9 /* LaunchScreen.storyboard in Resources */,
+				C026375B1F1C25E8007247A9 /* Assets.xcassets in Resources */,
+				C02637591F1C25E8007247A9 /* Main.storyboard in Resources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXResourcesBuildPhase section */
+
+/* Begin PBXSourcesBuildPhase section */
+		C02637471F1C25E8007247A9 /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				C02637691F1C26AF007247A9 /* ViewController.mm in Sources */,
+				01A1DB432652CBA700655BBC /* RPCArgs.mm in Sources */,
+				016B19C22657B390002E1719 /* RPCServer.mm in Sources */,
+				C02637531F1C25E8007247A9 /* AppDelegate.m in Sources */,
+				C02637661F1C2690007247A9 /* TVMRuntime.mm in Sources */,
+				C02637501F1C25E8007247A9 /* main.m in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin PBXVariantGroup section */
+		C02637571F1C25E8007247A9 /* Main.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				C02637581F1C25E8007247A9 /* Base */,
+			);
+			name = Main.storyboard;
+			sourceTree = "<group>";
+		};
+		C026375C1F1C25E8007247A9 /* LaunchScreen.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				C026375D1F1C25E8007247A9 /* Base */,
+			);
+			name = LaunchScreen.storyboard;
+			sourceTree = "<group>";
+		};
+/* End PBXVariantGroup section */
+
+/* Begin XCBuildConfiguration section */
+		C02637601F1C25E8007247A9 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				ENABLE_BITCODE = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				ENABLE_TESTABILITY = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu99;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+					"TVM_LOG_CUSTOMIZE=1",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 10.3;
+				MTL_ENABLE_DEBUG_INFO = YES;
+				ONLY_ACTIVE_ARCH = YES;
+				SDKROOT = iphoneos;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Debug;
+		};
+		C02637611F1C25E8007247A9 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				"CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				ENABLE_BITCODE = NO;
+				ENABLE_NS_ASSERTIONS = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu99;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"TVM_LOG_CUSTOMIZE=1",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 10.3;
+				MTL_ENABLE_DEBUG_INFO = NO;
+				SDKROOT = iphoneos;
+				TARGETED_DEVICE_FAMILY = "1,2";
+				VALIDATE_PRODUCT = YES;
+			};
+			name = Release;
+		};
+		C02637631F1C25E8007247A9 /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = NO;
+				DEVELOPMENT_TEAM = 3FR42MXLK9;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"USE_CUSTOM_DSO_LOADER=${USE_CUSTOM_DSO_LOADER}",
+				);
+				GCC_SYMBOLS_PRIVATE_EXTERN = NO;
+				HEADER_SEARCH_PATHS = (
+					../../include,
+					../../3rdparty/dlpack/include,
+					"${TVM_BUILD_DIR}/apps/ios_rpc/custom_dso_loader/include",
+				);
+				INFOPLIST_FILE = tvmrpc/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
+				LIBRARY_SEARCH_PATHS = (
+					"${TVM_BUILD_DIR}/apps/ios_rpc/custom_dso_loader/lib",
+					"${TVM_BUILD_DIR}",
+				);
+				OTHER_LDFLAGS = "${_DSO_LOADER_NAME_${USE_CUSTOM_DSO_LOADER}}";
+				PRODUCT_BUNDLE_IDENTIFIER = org.apache.tvmiosrpc;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				TVM_BUILD_DIR = "path-to-tvm-ios-build-folder";
+				USE_CUSTOM_DSO_LOADER = 1;
+				WARNING_CFLAGS = "-Wno-shorten-64-to-32";
+				_DSO_LOADER_NAME_0 = "";
+				_DSO_LOADER_NAME_1 = "-lmacho_dyld";
+			};
+			name = Debug;
+		};
+		C02637641F1C25E8007247A9 /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = NO;
+				DEVELOPMENT_TEAM = 3FR42MXLK9;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"USE_CUSTOM_DSO_LOADER=${USE_CUSTOM_DSO_LOADER}",
+				);
+				GCC_SYMBOLS_PRIVATE_EXTERN = NO;
+				HEADER_SEARCH_PATHS = (
+					../../include,
+					../../3rdparty/dlpack/include,
+					"${TVM_BUILD_DIR}/apps/ios_rpc/custom_dso_loader/include",
+				);
+				INFOPLIST_FILE = tvmrpc/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
+				LIBRARY_SEARCH_PATHS = (
+					"${TVM_BUILD_DIR}/apps/ios_rpc/custom_dso_loader/lib",
+					"${TVM_BUILD_DIR}",
+				);
+				OTHER_LDFLAGS = "${_DSO_LOADER_NAME_${USE_CUSTOM_DSO_LOADER}}";
+				PRODUCT_BUNDLE_IDENTIFIER = org.apache.tvmiosrpc;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				TVM_BUILD_DIR = "path-to-tvm-ios-build-folder";
+				USE_CUSTOM_DSO_LOADER = 1;
+				WARNING_CFLAGS = "-Wno-shorten-64-to-32";
+				_DSO_LOADER_NAME_0 = "";
+				_DSO_LOADER_NAME_1 = "-lmacho_dyld";
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		C02637461F1C25E8007247A9 /* Build configuration list for PBXProject "tvmrpc" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				C02637601F1C25E8007247A9 /* Debug */,
+				C02637611F1C25E8007247A9 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		C02637621F1C25E8007247A9 /* Build configuration list for PBXNativeTarget "tvmrpc" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				C02637631F1C25E8007247A9 /* Debug */,
+				C02637641F1C25E8007247A9 /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = C02637431F1C25E8007247A9 /* Project object */;
+}

apps/ios_rpc/tvmrpc.xcodeproj/project.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:tvmrpc.xcodeproj">
+   </FileRef>
+</Workspace>

apps/ios_rpc/tvmrpc.xcodeproj/xcshareddata/xcschemes/tvmrpc.xcscheme

@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+<Scheme
+   LastUpgradeVersion = "1250"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+      <BuildActionEntries>
+         <BuildActionEntry
+            buildForTesting = "YES"
+            buildForRunning = "YES"
+            buildForProfiling = "YES"
+            buildForArchiving = "YES"
+            buildForAnalyzing = "YES">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "C026374A1F1C25E8007247A9"
+               BuildableName = "tvmrpc.app"
+               BlueprintName = "tvmrpc"
+               ReferencedContainer = "container:tvmrpc.xcodeproj">
+            </BuildableReference>
+         </BuildActionEntry>
+      </BuildActionEntries>
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <Testables>
+      </Testables>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "C026374A1F1C25E8007247A9"
+            BuildableName = "tvmrpc.app"
+            BlueprintName = "tvmrpc"
+            ReferencedContainer = "container:tvmrpc.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+      <BuildableProductRunnable
+         runnableDebuggingMode = "0">
+         <BuildableReference
+            BuildableIdentifier = "primary"
+            BlueprintIdentifier = "C026374A1F1C25E8007247A9"
+            BuildableName = "tvmrpc.app"
+            BlueprintName = "tvmrpc"
+            ReferencedContainer = "container:tvmrpc.xcodeproj">
+         </BuildableReference>
+      </BuildableProductRunnable>
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

apps/ios_rpc/tvmrpc/AppDelegate.h

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file AppDelegate.h
+ */
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow* window;
+
+@end

apps/ios_rpc/tvmrpc/AppDelegate.m

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file AppDelegate.mm
+ */
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+- (BOOL)application:(UIApplication*)application
+    didFinishLaunchingWithOptions:(NSDictionary*)launchOptions {
+  // Override point for customization after application launch.
+  return YES;
+}
+
+- (void)applicationWillResignActive:(UIApplication*)application {
+  // Sent when the application is about to move from active to inactive state. This can occur for
+  // certain types of temporary interruptions (such as an incoming phone call or SMS message) or
+  // when the user quits the application and it begins the transition to the background state. Use
+  // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering
+  // callbacks. Games should use this method to pause the game.
+}
+
+- (void)applicationDidEnterBackground:(UIApplication*)application {
+  // Use this method to release shared resources, save user data, invalidate timers, and store
+  // enough application state information to restore your application to its current state in case
+  // it is terminated later. If your application supports background execution, this method is
+  // called instead of applicationWillTerminate: when the user quits.
+}
+
+- (void)applicationWillEnterForeground:(UIApplication*)application {
+  // Called as part of the transition from the background to the active state; here you can undo
+  // many of the changes made on entering the background.
+}
+
+- (void)applicationDidBecomeActive:(UIApplication*)application {
+  // Restart any tasks that were paused (or not yet started) while the application was inactive. If
+  // the application was previously in the background, optionally refresh the user interface.
+}
+
+- (void)applicationWillTerminate:(UIApplication*)application {
+  // Called when the application is about to terminate. Save data if appropriate. See also
+  // applicationDidEnterBackground:.
+}
+
+@end

apps/ios_rpc/tvmrpc/Base.lproj/LaunchScreen.storyboard

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="11134" systemVersion="15F34" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="11106"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
+                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>

apps/ios_rpc/tvmrpc/Base.lproj/Main.storyboard

@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="17701" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina5_5" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="17703"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="ViewController" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
+                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="414" height="736"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <textField opaque="NO" clipsSubviews="YES" contentMode="scaleToFill" fixedFrame="YES" contentHorizontalAlignment="left" contentVerticalAlignment="center" text="9090" borderStyle="roundedRect" textAlignment="natural" minimumFontSize="17" translatesAutoresizingMaskIntoConstraints="NO" id="eLf-oe-8c9">
+                                <rect key="frame" x="110" y="186" width="234" height="30"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits"/>
+                            </textField>
+                            <textField opaque="NO" clipsSubviews="YES" contentMode="scaleToFill" fixedFrame="YES" contentHorizontalAlignment="left" contentVerticalAlignment="center" text="iphone" borderStyle="roundedRect" textAlignment="natural" minimumFontSize="17" translatesAutoresizingMaskIntoConstraints="NO" id="o4Q-bP-8Cn">
+                                <rect key="frame" x="110" y="224" width="234" height="30"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits"/>
+                            </textField>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Address" textAlignment="right" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="Vkh-HN-g3u">
+                                <rect key="frame" x="39" y="148" width="63" height="27"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Port" textAlignment="right" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="vtu-2M-JLP">
+                                <rect key="frame" x="61" y="186" width="41" height="30"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Key" textAlignment="right" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="5MW-dM-69p">
+                                <rect key="frame" x="61" y="224" width="41" height="30"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                            <textField opaque="NO" clipsSubviews="YES" contentMode="scaleToFill" fixedFrame="YES" contentHorizontalAlignment="left" contentVerticalAlignment="center" borderStyle="roundedRect" textAlignment="natural" minimumFontSize="17" translatesAutoresizingMaskIntoConstraints="NO" id="7yQ-tQ-Qqc">
+                                <rect key="frame" x="110" y="148" width="234" height="30"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits"/>
+                            </textField>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" fixedFrame="YES" editable="NO" textAlignment="natural" translatesAutoresizingMaskIntoConstraints="NO" id="8jt-IO-eTb">
+                                <rect key="frame" x="69" y="374" width="254" height="207"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <color key="backgroundColor" white="1" alpha="1" colorSpace="calibratedWhite"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                            <segmentedControl opaque="NO" contentMode="scaleToFill" fixedFrame="YES" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="plain" selectedSegmentIndex="0" translatesAutoresizingMaskIntoConstraints="NO" id="ocx-rT-xCk">
+                                <rect key="frame" x="110" y="262" width="234" height="32"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <segments>
+                                    <segment title="Tracker"/>
+                                    <segment title="Proxy"/>
+                                    <segment title="RPC"/>
+                                </segments>
+                            </segmentedControl>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Disconnected" textAlignment="natural" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="9Zy-zO-r1A">
+                                <rect key="frame" x="143" y="99" width="106" height="20.5"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                            <button opaque="NO" contentMode="scaleToFill" fixedFrame="YES" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="system" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="m1Y-7m-LeH">
+                                <rect key="frame" x="110" y="306" width="234" height="60"/>
+                                <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                <state key="normal" title="Connect"/>
+                                <connections>
+                                    <action selector="connect:" destination="BYZ-38-t0r" eventType="touchUpInside" id="1cx-nV-onb"/>
+                                </connections>
+                            </button>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                    </view>
+                    <connections>
+                        <outlet property="ConnectButton" destination="m1Y-7m-LeH" id="Pc5-Cl-uxz"/>
+                        <outlet property="ModeSelector" destination="ocx-rT-xCk" id="5r7-XQ-tMk"/>
+                        <outlet property="infoText" destination="8jt-IO-eTb" id="TmG-H2-eJY"/>
+                        <outlet property="proxyKey" destination="o4Q-bP-8Cn" id="XFh-Qz-86e"/>
+                        <outlet property="proxyPort" destination="eLf-oe-8c9" id="f1g-tu-Q5U"/>
+                        <outlet property="proxyURL" destination="7yQ-tQ-Qqc" id="KBr-zC-3sD"/>
+                        <outlet property="statusLabel" destination="9Zy-zO-r1A" id="gHN-jG-4ii"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="24.800000000000001" y="36.431784107946029"/>
+        </scene>
+    </scenes>
+</document>

apps/ios_rpc/tvmrpc/Info.plist

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>UILaunchStoryboardName</key>
+	<string>LaunchScreen</string>
+	<key>UIMainStoryboardFile</key>
+	<string>Main</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>armv7</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+</dict>
+</plist>

apps/ios_rpc/tvmrpc/RPCArgs.h

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef TVM_APPS_IOS_RPC_ARGS_H_
+#define TVM_APPS_IOS_RPC_ARGS_H_
+
+#import "RPCServer.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*!
+ * \brief Struct representing arguments of iOS RPC app
+ */
+typedef struct RPCArgs_t {
+  /// Tracker or Proxy address (actually ip)
+  const char* host_url;
+
+  /// Tracker or Proxy port
+  int host_port;
+
+  /// device key to report
+  const char* key;
+
+  /// custom address to report into Tracker. Ignored for other server modes.
+  const char* custom_addr;
+
+  /// Verbose mode. Will print status messages to std out.
+  /// 0 - no prints , 1 - print state to output
+  bool verbose;
+
+  /// Immediate server launch. No UI interaction.
+  /// 0 - UI interaction, 1 - automatically connect on launch
+  bool immediate_connect;
+
+  /// Server mode
+  RPCServerMode server_mode;
+} RPCArgs;
+
+/*!
+ * \brief Get current global RPC args
+ */
+RPCArgs get_current_rpc_args(void);
+
+/*!
+ * \brief Set current global RPC args and update values in app cache
+ */
+void set_current_rpc_args(RPCArgs args);
+
+/*!
+ * \brief Pars command line args and update current global RPC args
+ * Also updates values in app cache
+ */
+void update_rpc_args(int argc, char* argv[]);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // TVM_APPS_IOS_RPC_ARGS_H_

apps/ios_rpc/tvmrpc/RPCArgs.mm

@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#import "RPCArgs.h"
+
+#import <Foundation/Foundation.h>
+
+#import "../../../src/support/socket.h"
+#import "../../../src/support/utils.h"
+
+#import <string>
+
+using std::string;
+
+const char* kUsage =
+    "\n"
+    "iOS tvmrpc application supported flags:\n"
+    "--host_url      The tracker/proxy address, Default=0.0.0.0\n"
+    "--host_port     The tracker/proxy port, Default=9190\n"
+    "--key           The key used to identify the device type in tracker. Default=\"\"\n"
+    "--custom_addr   Custom IP Address to Report to RPC Tracker. Default=\"\"\n"
+    "--immediate_connect   No UI interconnection, connect to tracker immediately. Default=False\n"
+    "--verbose       Allow to print status info to std out. Default=False\n"
+    "--server_mode   Server mode. Can be \"standalone\", \"proxy\" or \"tracker\". "
+    "Default=standalone \n"
+    "\n";
+
+struct RPCArgs_cpp {
+  string host_url = "0.0.0.0";
+  int host_port = 9190;
+
+  string key;
+  string custom_addr = "null";
+
+  bool immediate_connect = false;
+  bool verbose = false;
+  RPCServerMode server_mode = RPCServerMode_Tracker;
+
+  operator RPCArgs() const {
+    return RPCArgs{.host_url = host_url.c_str(),
+                   .host_port = host_port,
+                   .key = key.c_str(),
+                   .custom_addr = custom_addr.c_str(),
+                   .verbose = verbose,
+                   .immediate_connect = immediate_connect,
+                   .server_mode = server_mode};
+  };
+
+  RPCArgs_cpp& operator=(const RPCArgs& args) {
+    host_url = args.host_url;
+    host_port = args.host_port;
+    key = args.key;
+    custom_addr = args.custom_addr;
+    verbose = args.verbose;
+    immediate_connect = args.immediate_connect;
+    server_mode = args.server_mode;
+    return *this;
+  }
+};
+
+struct RPCArgs_cpp g_rpc_args;
+
+static void restore_from_cache() {
+  NSUserDefaults* defaults = [NSUserDefaults standardUserDefaults];
+
+  auto get_string_from_cache = [defaults](const char* key) {
+    NSString* ns_key = [NSString stringWithUTF8String:key];
+    NSString* ns_val = [defaults stringForKey:ns_key];
+    return std::string(ns_val != nil ? [ns_val UTF8String] : "");
+  };
+
+  auto get_int_from_cache = [defaults](const char* key) {
+    NSString* ns_key = [NSString stringWithUTF8String:key];
+    return static_cast<int>([defaults integerForKey:ns_key]);
+  };
+
+  g_rpc_args.host_url = get_string_from_cache("RPCArgs_url");
+  g_rpc_args.host_port = get_int_from_cache("RPCArgs_port");
+  g_rpc_args.key = get_string_from_cache("RPCArgs_key");
+}
+
+static void update_in_cache() {
+  NSUserDefaults* defaults = [NSUserDefaults standardUserDefaults];
+
+  [defaults setObject:[NSString stringWithUTF8String:g_rpc_args.host_url.c_str()]
+               forKey:@"RPCArgs_url"];
+  [defaults setInteger:g_rpc_args.host_port forKey:@"RPCArgs_port"];
+  [defaults setObject:[NSString stringWithUTF8String:g_rpc_args.key.c_str()] forKey:@"RPCArgs_key"];
+}
+
+string GetCmdOption(int argc, char* argv[], string option, bool key = false) {
+  string cmd;
+  for (int i = 1; i < argc; ++i) {
+    string arg = argv[i];
+    if (arg.find(option) == 0) {
+      if (key) {
+        cmd = argv[i];
+        return cmd;
+      }
+      // We assume "=" is the end of option.
+      TVM_FFI_ICHECK_EQ(*option.rbegin(), '=');
+      cmd = arg.substr(arg.find('=') + 1);
+      return cmd;
+    }
+  }
+  return cmd;
+}
+
+void update_rpc_args(int argc, char* argv[]) {
+  restore_from_cache();
+  RPCArgs_cpp& args = g_rpc_args;
+
+  using tvm::support::IsNumber;
+  using tvm::support::ValidateIP;
+  constexpr int MAX_PORT_NUM = 65535;
+
+  const string immediate_connect = GetCmdOption(argc, argv, "--immediate_connect", true);
+  args.immediate_connect = !immediate_connect.empty();
+
+  const string verbose = GetCmdOption(argc, argv, "--verbose", true);
+  args.verbose = !verbose.empty();
+
+  const string server_mode = GetCmdOption(argc, argv, "--server_mode=", false);
+  if (!server_mode.empty()) {
+    if (server_mode == "tracker") {
+      args.server_mode = RPCServerMode_Tracker;
+    } else if (server_mode == "proxy") {
+      args.server_mode = RPCServerMode_Proxy;
+    } else if (server_mode == "standalone") {
+      args.server_mode = RPCServerMode_Standalone;
+    } else {
+      LOG(WARNING) << "Wrong server_mode value.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+  }
+
+  const string host_url = GetCmdOption(argc, argv, "--host_url=");
+  if (!host_url.empty()) {
+    if (!ValidateIP(host_url)) {
+      LOG(WARNING) << "Wrong tracker address format.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.host_url = host_url;
+  }
+
+  const string host_port = GetCmdOption(argc, argv, "--host_port=");
+  if (!host_port.empty()) {
+    if (!IsNumber(host_port) || stoi(host_port) > MAX_PORT_NUM) {
+      LOG(WARNING) << "Wrong trackerport number.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.host_port = stoi(host_port);
+  }
+
+  const string key = GetCmdOption(argc, argv, "--key=");
+  if (!key.empty()) {
+    args.key = key;
+  }
+
+  const string custom_addr = GetCmdOption(argc, argv, "--custom_addr=");
+  if (!custom_addr.empty()) {
+    if (!ValidateIP(custom_addr)) {
+      LOG(WARNING) << "Wrong custom address format.";
+      LOG(INFO) << kUsage;
+      exit(1);
+    }
+    args.custom_addr = '"' + custom_addr + '"';
+  }
+
+  update_in_cache();
+}
+
+RPCArgs get_current_rpc_args(void) { return g_rpc_args; }
+
+void set_current_rpc_args(RPCArgs args) {
+  g_rpc_args = args;
+  update_in_cache();
+}

apps/ios_rpc/tvmrpc/RPCServer.h

@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file Provide interfaces to launch and control RPC Service routine
+ */
+
+#import <Foundation/Foundation.h>
+
+/*!
+ * \brief Enum with possible status of RPC server
+ * Used to report state to listener
+ */
+typedef enum {
+  RPCServerStatus_Launched,           // Worker thread is launched
+  RPCServerStatus_Stopped,            // Worker thread stopped
+  RPCServerStatus_Connected,          // Connected to Proxy/Tracker
+  RPCServerStatus_Disconnected,       // Disconnected from Proxy/Tracker
+  RPCServerStatus_RPCSessionStarted,  // RPC session is started
+  RPCServerStatus_RPCSessionFinished  // RPC session is finished
+} RPCServerStatus;
+
+/*!
+ * \brief Enum with modes of servicing supported by RPCServer
+ */
+typedef enum {
+  /// Tracker mode. Same as Standalone Server plus register it into Tracker.
+  RPCServerMode_Tracker,
+  /// Proxy mode. Connect to proxy server and wait response.
+  RPCServerMode_Proxy,
+  /// Standalone RPC server mode. Open port with RPC server and wait incoming connection.
+  RPCServerMode_Standalone
+} RPCServerMode;
+
+/*!
+ * \brief Listener for events happened with RPCServer
+ */
+@protocol RPCServerEventListener <NSObject>
+/// Callback to notifying about new status
+- (void)onError:(NSString*)msg;
+/// Callback to notifying about error
+- (void)onStatusChanged:(RPCServerStatus)status;
+@end
+
+/*!
+ * \brief RPC Server instance
+ * Contains internal worker thread plus
+ */
+@interface RPCServer : NSObject <NSStreamDelegate>
+
+/// Event listener delegate to set
+@property(retain) id<RPCServerEventListener> delegate;
+/// Device key to report during RPC session
+@property(retain) NSString* key;
+/// Host address of Proxy/Tracker server (generally IPv4). Ignored for Standalone mode.
+@property(retain) NSString* host;
+/// Port of Proxy/Tracker server. Ignored for Standalone mode.
+@property int port;
+/// Custom address to report into tracker server (optional). Ignored for Standalone/Proxy modes
+@property(retain) NSString* custom_addr;
+/// Trigger to enable printing of server state info
+@property BOOL verbose;
+/// RPC port opened on the device. Ignored for Proxy/Tracker modes
+@property int actual_port;
+/// IP address of the device. Ignored for Proxy/Tracker modes
+@property(retain) NSString* device_addr;
+
+/*!
+ * \brief Create server with specified servicing mode
+ * \param mode Mode of server
+ */
++ (instancetype)serverWithMode:(RPCServerMode)mode;
+
+/*!
+ * \brief Start RPC server with options. Non blocking method
+ */
+- (void)start;
+
+/*!
+ * \brief Stop RPC server. Non blocking method
+ */
+- (void)stop;
+
+@end

apps/ios_rpc/tvmrpc/RPCServer.mm

@@ -0,0 +1,814 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file ViewController.mm
+ */
+
+#import "RPCServer.h"
+
+#include <tvm/ffi/function.h>
+
+#include <random>
+#include <string>
+
+// To get device WiFi IP
+#include <arpa/inet.h>
+#include <ifaddrs.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+
+// TVM internal header to access Magic keys like kRPCMagic and others
+#include "../../../src/runtime/rpc/rpc_endpoint.h"
+
+namespace tvm {
+namespace runtime {
+
+/*!
+ * \brief Message handling function for event driven server.
+ *
+ * \param in_bytes The incoming bytes.
+ * \param event_flag  1: read_available, 2: write_avaiable.
+ * \return State flag.
+ *     1: continue running, no need to write,
+ *     2: need to write
+ *     0: shutdown
+ */
+using FEventHandler = ffi::Function;
+
+/*!
+ * \brief Create a server event handler.
+ *
+ * \param outputStream The output stream used to send outputs.
+ * \param name The name of the server.
+ * \param remote_key The remote key
+ * \return The event handler.
+ */
+FEventHandler CreateServerEventHandler(NSOutputStream* outputStream, std::string name,
+                                       std::string remote_key) {
+  auto event_handler_factory = tvm::ffi::Function::GetGlobal("rpc.CreateEventDrivenServer");
+  TVM_FFI_ICHECK(event_handler_factory.has_value())
+      << "You are using tvm_runtime module built without RPC support. "
+      << "Please rebuild it with USE_RPC flag.";
+
+  ffi::Function writer_func([outputStream](ffi::PackedArgs args, ffi::Any* rv) {
+    TVMByteArray* data = args[0].ptr<TVMByteArray>();
+    int64_t nbytes = [outputStream write:reinterpret_cast<const uint8_t*>(data->data)
+                               maxLength:data->size];
+    if (nbytes < 0) {
+      NSLog(@"%@", [outputStream streamError].localizedDescription);
+      throw tvm::Error("Stream error");
+    }
+    *rv = nbytes;
+  });
+
+  return (*event_handler_factory)(writer_func, name, remote_key);
+}
+
+/*!
+ * \brief Helper function to query real IP of device in WiFi network
+ * \return string with IPv4 in format "192.168.0.1" or "unknown" if cannot detect
+ */
+static std::string getWiFiAddress() {
+  std::string address = "unknown";
+  ifaddrs* interfaces = nullptr;
+
+  int success = getifaddrs(&interfaces);
+  if (success == 0) {
+    ifaddrs* temp_addr = interfaces;
+    while (temp_addr != NULL) {
+      if (temp_addr->ifa_addr->sa_family == AF_INET) {
+        // Check if interface is en0 which is the wifi connection on the iPhone
+        if (std::string(temp_addr->ifa_name) == "en0") {
+          address = inet_ntoa(((sockaddr_in*)temp_addr->ifa_addr)->sin_addr);
+        }
+      }
+      temp_addr = temp_addr->ifa_next;
+    }
+  }
+
+  freeifaddrs(interfaces);
+  return address;
+}
+
+}  // namespace runtime
+}  // namespace tvm
+
+// Base class for any type of RPC servicing
+@interface RPCServerBase : RPCServer
+
+/*!
+ * Methods should be implemented in inherited classes
+ */
+- (bool)onReadHandler;   // return true - continue feeding, false - stop, try to drain output buffer
+- (bool)onWriteHandler;  // return true - continue draining, false - no data to write
+- (void)onEndEncountered;  // called on disconnect or session decided that it's shutdown time
+- (void)open;              // Initiate listening objects like i/o streams and other resources
+- (void)close;             // Deinitialize resources opend in "open" method
+@end
+
+@implementation RPCServerBase {
+  // Worker thread
+  NSThread* worker_thread_;
+  // Trigger to continue RunLoop processing inside worker_thread_
+  BOOL shouldKeepRunning;
+  // Input socket stream
+ @protected
+  NSInputStream* inputStream_;
+  // Output socket stream
+  NSOutputStream* outputStream_;
+  // Temporal buffer with data to send
+  std::string sendBuffer_;
+  // Temporal receive buffer
+  std::string recvBuffer_;
+  // Requested data size to accumulate in recvBuffer_ before continue processing
+  int requiredToRecv_;
+}
+
+/*!
+ * Start internal worker thread with RunLoop and submit corresponding open handlers into it
+ * Not blocking
+ */
+- (void)start {
+  worker_thread_ = [[NSThread alloc] initWithBlock:^{
+    @autoreleasepool {
+      [self notifyState:RPCServerStatus_Launched];
+      [self open];
+      shouldKeepRunning = YES;
+      while (shouldKeepRunning && [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode
+                                                           beforeDate:[NSDate distantFuture]]);
+      [self notifyState:RPCServerStatus_Stopped];
+    }
+  }];
+  [worker_thread_ start];
+}
+
+/*!
+ * Send message to workel thread runloop to finish processing
+ * Not blocking
+ */
+- (void)stop {
+  if (worker_thread_ == nil) return;
+
+  [self performSelector:@selector(stop_) onThread:worker_thread_ withObject:nil waitUntilDone:NO];
+  worker_thread_ = nil;  // TODO: is it valid? may be better to do that inside NSThread?
+}
+
+- (void)stop_ {
+  [self close];
+  shouldKeepRunning = NO;
+}
+
+/*!
+ * Base implementation to setup i/o streams
+ * Will connect to host and port specified in corresponding properties
+ */
+- (void)open {
+  CFReadStreamRef readStream;
+  CFWriteStreamRef writeStream;
+  CFStreamCreatePairWithSocketToHost(NULL, (__bridge CFStringRef)self.host, self.port, &readStream,
+                                     &writeStream);
+  inputStream_ = (__bridge NSInputStream*)readStream;
+  outputStream_ = (__bridge NSOutputStream*)writeStream;
+  [inputStream_ setDelegate:self];
+  [outputStream_ setDelegate:self];
+  [inputStream_ scheduleInRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+  [outputStream_ scheduleInRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+  [outputStream_ open];
+  [inputStream_ open];
+}
+
+/*!
+ * Base implementation to setup i/o streams
+ * Will assign i/o streams to provided socket connection.
+ */
+- (void)openWithSocket:(CFSocketNativeHandle)sock {
+  CFReadStreamRef readStream;
+  CFWriteStreamRef writeStream;
+  CFStreamCreatePairWithSocket(NULL, sock, &readStream, &writeStream);
+  inputStream_ = (__bridge NSInputStream*)readStream;
+  outputStream_ = (__bridge NSOutputStream*)writeStream;
+  [inputStream_ setDelegate:self];
+  [outputStream_ setDelegate:self];
+  [inputStream_ scheduleInRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+  [outputStream_ scheduleInRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+  [outputStream_ open];
+  [inputStream_ open];
+}
+
+/*!
+ * Close i/o streams associated with connection
+ */
+- (void)close {
+  [inputStream_ close];
+  [outputStream_ close];
+  [inputStream_ removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+  [outputStream_ removeFromRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+  [inputStream_ setDelegate:nil];
+  [outputStream_ setDelegate:nil];
+  inputStream_ = nil;
+  outputStream_ = nil;
+}
+
+/// Unimplemented stubs
+- (bool)onReadHandler {
+  return false;
+}
+- (bool)onWriteHandler {
+  return false;
+}
+- (void)onEndEncountered {
+}
+
+/*!
+ * Try to read data from stream and call processing handler
+ */
+- (void)tryToRead {
+  const int kBufferSize = 4 << 10;  // 4kB buffer
+  const int prev_size = recvBuffer_.size();
+  recvBuffer_.resize(kBufferSize);
+  size_t nbytes = [inputStream_ read:(uint8_t*)recvBuffer_.data() + prev_size
+                           maxLength:recvBuffer_.size() - prev_size];
+  recvBuffer_.resize(nbytes + prev_size);
+
+  // feed while it accept or requested particulat buffer size
+  while (!recvBuffer_.empty() && requiredToRecv_ <= recvBuffer_.size() && [self onReadHandler]);
+}
+
+/*!
+ * Try to write remaining data to stream and call processing handler
+ */
+- (void)tryToWrite {
+  if (!sendBuffer_.empty()) {
+    size_t nbytes = [outputStream_ write:(uint8_t*)sendBuffer_.data() maxLength:sendBuffer_.size()];
+    sendBuffer_.erase(0, nbytes);
+  }
+  // call write handler while it want be called and space is available
+  while (sendBuffer_.empty() && [outputStream_ hasSpaceAvailable] && [self onWriteHandler]);
+}
+
+/*!
+ * Main event handler of socket stream events
+ */
+- (void)stream:(NSStream*)strm handleEvent:(NSStreamEvent)event {
+  std::string buffer;
+  switch (event) {
+    case NSStreamEventOpenCompleted: {
+      // Nothing
+      break;
+    }
+    case NSStreamEventHasBytesAvailable:
+      if (strm == inputStream_) {
+        [self tryToRead];
+        if ([outputStream_ hasSpaceAvailable]) [self tryToWrite];
+      }
+      break;
+    case NSStreamEventHasSpaceAvailable: {
+      if (strm == outputStream_) {
+        [self tryToWrite];
+        if ([inputStream_ hasBytesAvailable]) [self tryToRead];
+      }
+      break;
+    }
+    case NSStreamEventErrorOccurred: {
+      [self notifyError:[strm streamError].localizedDescription];
+      break;
+    }
+    case NSStreamEventEndEncountered: {
+      [self onEndEncountered];
+      break;
+    }
+    default: {
+      NSLog(@"Unknown event");
+    }
+  }
+}
+
+#pragma mark - Helpers
+
+/*!
+ * Set buffer to send into stream. Try to send immediately or submit to lazy sending
+ * Non blocking operation
+ */
+- (void)toSend:(NSData*)data {
+  sendBuffer_.append(static_cast<const char*>(data.bytes), data.length);
+
+  // try to flush buffer
+  NSInteger sent_size = [outputStream_ write:(uint8_t*)sendBuffer_.data()
+                                   maxLength:sendBuffer_.size()];
+  sendBuffer_.erase(0, sent_size);
+}
+
+/*!
+ * Set buffer to send  in packet format [size, data]. Behaviour is same as for toSend.
+ */
+- (void)toSendPacked:(NSData*)data {
+  uint32_t packet_size = data.length;
+  [self toSend:[NSData dataWithBytes:&packet_size length:sizeof(packet_size)]];
+  [self toSend:data];
+}
+
+/*!
+ */
+- (NSData*)requestInputDataWithSize:(NSInteger)size {
+  if (recvBuffer_.size() < size) {
+    requiredToRecv_ = size;
+    return nil;
+  }
+  NSData* res = [NSData dataWithBytes:recvBuffer_.data() length:size];
+  recvBuffer_.erase(0, size);
+  return res;
+}
+
+/*!
+ */
+- (NSData*)requestInputDataPacked {
+  uint32_t size;
+  if (recvBuffer_.size() < sizeof(size)) {
+    requiredToRecv_ = sizeof(size);
+    return nil;
+  }
+  size = *(uint32_t*)recvBuffer_.data();
+  if (recvBuffer_.size() < sizeof(size) + size) {
+    requiredToRecv_ = sizeof(size) + size;
+    return nil;
+  }
+  NSData* res = [NSData dataWithBytes:recvBuffer_.data() + sizeof(size) length:size];
+  recvBuffer_.erase(0, sizeof(size) + size);
+  return res;
+};
+
+#pragma mark - Notifiers
+
+/*!
+ * Notify external listener about error.
+ * Also print error message to std out in case of Verbose mode
+ */
+- (void)notifyError:(NSString*)msg {
+  // Duplicate error message in std output. Host launcher script may listen it.
+  if (self.verbose) NSLog(@"[IOS-RPC] ERROR: %@", msg);
+  if (self.delegate) [self.delegate onError:msg];
+}
+
+/*!
+ * Notify external listener about server state changes.
+ * Also print information to std out in case of Verbose mode
+ */
+- (void)notifyState:(RPCServerStatus)state {
+  // Duplicate sattus changing in std output. Host launcher script may listen it.
+  if (self.verbose) NSLog(@"[IOS-RPC] STATE: %d", state);
+  if (self.delegate != nil) [self.delegate onStatusChanged:state];
+}
+
+@end
+
+@interface RPCServerProxy : RPCServerBase
+@end
+
+typedef enum {
+  RPCServerProxyState_Idle,
+  RPCServerProxyState_HandshakeToSend,
+  RPCServerProxyState_HandshakeToRecv,
+  RPCServerProxyState_Processing,
+} RPCServerProxyState;
+
+@implementation RPCServerProxy {
+  /// Original TVM RPC event handler
+  tvm::runtime::FEventHandler handler_;
+ @protected
+  /// Sate of Proxy client implementation
+  RPCServerProxyState state_;
+}
+
+- (instancetype)init {
+  if (self = [super init]) {
+    handler_ = nullptr;
+    state_ = RPCServerProxyState_Idle;
+  }
+  return self;
+}
+
+/*!
+ * Implement matching of internat state on state available for outside users
+ */
+- (void)setState:(RPCServerProxyState)new_state {
+  // Send Connected notification because Proxy doesn't response until client connected.
+  if (new_state == RPCServerProxyState_HandshakeToRecv)
+    [self notifyState:RPCServerStatus_Connected];
+  if (new_state == RPCServerProxyState_Idle) [self notifyState:RPCServerStatus_Disconnected];
+  if (state_ == RPCServerProxyState_HandshakeToRecv && new_state == RPCServerProxyState_Processing)
+    [self notifyState:RPCServerStatus_RPCSessionStarted];
+  if (state_ == RPCServerProxyState_Processing && new_state == RPCServerProxyState_Idle)
+    [self notifyState:RPCServerStatus_RPCSessionStarted];
+
+  state_ = new_state;
+}
+
+- (bool)onWriteHandler {
+  switch (state_) {
+    case RPCServerProxyState_HandshakeToSend: {
+      // Send together kRPCMagic and server descriptor because of Proxy
+      int32_t code = tvm::runtime::kRPCMagic;
+      [self toSend:[NSData dataWithBytes:&code length:sizeof(code)]];
+
+      std::string full_key = std::string("server:") + self.key.UTF8String;
+      [self toSendPacked:[NSData dataWithBytes:full_key.data() length:full_key.size()]];
+
+      self.state = RPCServerProxyState_HandshakeToRecv;
+      return TRUE;
+    }
+    case RPCServerProxyState_Processing: {
+      try {
+        TVMByteArray dummy{nullptr, 0};
+        int flag = handler_(dummy, 2);
+        if (flag == 0) {
+          [self onEndEncountered];
+        }
+        return flag == 2;
+      } catch (const tvm::Error& e) {
+        [self close];
+      }
+      break;
+    }
+    default:
+      // Nothing
+      break;
+  }
+  return FALSE;
+}
+
+- (bool)onReadHandler {
+  switch (state_) {
+    case RPCServerProxyState_HandshakeToRecv: {
+      int32_t code = tvm::runtime::kRPCMagic;
+      NSData* data = [self requestInputDataWithSize:sizeof(code)];
+      if (data == nil) return FALSE;
+
+      if (*(int32_t*)data.bytes != tvm::runtime::kRPCMagic) {
+        [self notifyError:@"Wrong response, is not RPC client."];
+        [self close];
+        return FALSE;
+        break;
+      }
+
+      handler_ = tvm::runtime::CreateServerEventHandler(outputStream_, "iphone", "%toinit");
+
+      self.state = RPCServerProxyState_Processing;
+      return TRUE;
+      break;
+    }
+    case RPCServerProxyState_Processing: {
+      int flag = 1;
+      if ([outputStream_ hasSpaceAvailable]) {
+        flag |= 2;
+      }
+      // always try to write
+      try {
+        TVMByteArray arr{recvBuffer_.data(), recvBuffer_.size()};
+        flag = handler_(arr, flag);
+        recvBuffer_.clear();
+        if (flag == 0) {
+          [self onEndEncountered];
+        }
+        return flag == 1;
+      } catch (const tvm::Error& e) {
+        [self close];
+      }
+      break;
+    }
+    default:
+      // Nothing
+      break;
+  }
+  return FALSE;
+}
+
+- (void)onEndEncountered {
+  // Automatic reconnection when session is finished.
+  [self close];
+  [self open];
+}
+
+- (void)open {
+  [super open];
+  self.state = RPCServerProxyState_HandshakeToSend;
+}
+
+- (void)close {
+  [super close];
+  handler_ = nullptr;
+  self.state = RPCServerProxyState_Idle;
+}
+
+@end
+
+@interface RPCServerStandalone : RPCServerProxy
+@property(readonly) int rpc_port;
+@end
+
+@implementation RPCServerStandalone {
+  // Socket to listen incoming connections
+  CFSocketRef socket_;
+  /// Current socket connection handler
+  CFSocketNativeHandle connection_;
+  /// Port range to try bind to socket
+  int port_range_start;
+  int port_range_end;
+}
+
+- (instancetype)init {
+  if (self = [super init]) {
+    connection_ = 0;
+    port_range_start = 9090;
+    port_range_end = 9099;
+  }
+  return self;
+}
+
+- (void)setState:(RPCServerProxyState)new_state {
+  if (state_ == RPCServerProxyState_Idle && new_state == RPCServerProxyState_HandshakeToSend) {
+    self.actual_port = _rpc_port;
+    self.device_addr = [NSString stringWithUTF8String:tvm::runtime::getWiFiAddress().c_str()];
+    if (self.verbose) {
+      // Notify host runner script with actual address
+      NSLog(@"[IOS-RPC] IP: %s", tvm::runtime::getWiFiAddress().c_str());
+      NSLog(@"[IOS-RPC] PORT: %d", _rpc_port);
+    }
+    [self notifyState:RPCServerStatus_Connected];
+  }
+  if (new_state == RPCServerProxyState_Idle) [self notifyState:RPCServerStatus_Disconnected];
+  if (state_ == RPCServerProxyState_HandshakeToRecv && new_state == RPCServerProxyState_Processing)
+    [self notifyState:RPCServerStatus_RPCSessionStarted];
+  if (state_ == RPCServerProxyState_Processing && new_state == RPCServerProxyState_HandshakeToSend)
+    [self notifyState:RPCServerStatus_RPCSessionFinished];
+
+  state_ = new_state;
+}
+
+- (void)handleConnect:(CFSocketNativeHandle)hdl {
+  connection_ = hdl;
+  [super openWithSocket:connection_];
+  self.state = RPCServerProxyState_HandshakeToSend;
+}
+
+static void handleConnect(CFSocketRef socket, CFSocketCallBackType type, CFDataRef address,
+                          const void* data, void* info) {
+  RPCServerStandalone* it = (__bridge RPCServerStandalone*)(info);
+  [it handleConnect:*static_cast<const CFSocketNativeHandle*>(data)];
+}
+
+- (void)open {
+  CFSocketContext ctx{};
+  ctx.info = (__bridge void*)self;
+
+  socket_ = CFSocketCreate(kCFAllocatorDefault, PF_INET, SOCK_STREAM, IPPROTO_TCP,
+                           kCFSocketAcceptCallBack, handleConnect, &ctx);
+  self->_rpc_port = 0;
+
+  // Try to bind with range
+  for (int port = port_range_start; port < port_range_end; port++) {
+    struct sockaddr_in sin;
+    memset(&sin, 0, sizeof(sin));
+    sin.sin_len = sizeof(sin);
+    sin.sin_family = AF_INET;
+    sin.sin_port = htons(port);
+    sin.sin_addr.s_addr = INADDR_ANY;
+
+    CFDataRef sincfd = CFDataCreate(kCFAllocatorDefault, (UInt8*)&sin, sizeof(sin));
+    CFSocketError res = CFSocketSetAddress(socket_, sincfd);
+    CFRelease(sincfd);
+    if (res == kCFSocketSuccess) {
+      self->_rpc_port = port;
+      break;
+    }
+  }
+  if (self->_rpc_port == 0) {
+    @throw
+        [NSException exceptionWithName:@"SocketError"
+                                reason:[NSString stringWithFormat:@"Unable bind socket to port"
+                                                                   "in range [%d, %d]",
+                                                                  port_range_start, port_range_end]
+                              userInfo:nil];
+  }
+
+  CFRunLoopSourceRef socketsource = CFSocketCreateRunLoopSource(kCFAllocatorDefault, socket_, 0);
+  CFRunLoopAddSource(CFRunLoopGetCurrent(), socketsource, kCFRunLoopDefaultMode);
+
+  self.state = RPCServerProxyState_HandshakeToSend;
+}
+
+- (void)closeSocket {
+  CFSocketInvalidate(socket_);
+}
+
+- (void)close {
+  [super close];
+  close(connection_);
+}
+
+- (void)onEndEncountered {
+  [self close];
+  [self notifyState:RPCServerStatus_RPCSessionFinished];
+}
+
+@end
+
+@interface RPCServerTracker : RPCServerBase <RPCServerEventListener>
+@end
+
+typedef enum {
+  RPCServerTracker_Idle,
+  RPCServerTracker_HandshakeToSend,
+  RPCServerTracker_HandshakeToRecv,
+  RPCServerTracker_ServerInfoToSend,
+  RPCServerTracker_ServerInfoToRecv,
+  RPCServerTracker_ReportResToSend,
+  RPCServerTracker_ReportResToRecv,
+  RPCServerTracker_UpdateKeyToSend,
+  RPCServerTracker_UpdateKeyToRecv,
+  RPCServerTracker_WaitConnection
+} RPCServerTrackerState;
+
+@implementation RPCServerTracker {
+  RPCServerTrackerState state_;
+  RPCServerStandalone* rpc_server_;
+}
+
+- (void)setState:(RPCServerTrackerState)new_state {
+  if (state_ == RPCServerTracker_ReportResToRecv && new_state == RPCServerTracker_WaitConnection)
+    [self notifyState:RPCServerStatus_Connected];
+  if (state_ == RPCServerTracker_WaitConnection && new_state == RPCServerTracker_Idle)
+    [self notifyState:RPCServerStatus_Disconnected];
+
+  state_ = new_state;
+}
+
+- (bool)onWriteHandler {
+  switch (state_) {
+    case RPCServerTracker_HandshakeToSend: {
+      int32_t code = tvm::runtime::kRPCTrackerMagic;
+      [self toSend:[NSData dataWithBytes:&code length:sizeof(code)]];
+      self.state = RPCServerTracker_HandshakeToRecv;
+      return TRUE;
+      break;
+    }
+    case RPCServerTracker_ServerInfoToSend: {
+      std::ostringstream ss;
+      ss << "[" << static_cast<int>(tvm::runtime::TrackerCode::kUpdateInfo)
+         << ", {\"key\": \"server:" << self.key.UTF8String << "\", \"addr\": ["
+         << self.custom_addr.UTF8String << ", \"" << self.port << "\"]}]";
+      std::string data_s = ss.str();
+      [self toSendPacked:[NSData dataWithBytes:data_s.data() length:data_s.length()]];
+      self.state = RPCServerTracker_ServerInfoToRecv;
+      return TRUE;
+      break;
+    }
+    case RPCServerTracker_ReportResToSend: {
+      std::mt19937 gen(std::random_device{}());
+      std::uniform_real_distribution<float> dis(0.0, 1.0);
+
+      std::string address_to_report = "null";
+      if (self.custom_addr != nil && self.custom_addr.length != 0) {
+        address_to_report = self.custom_addr.UTF8String;
+      }
+
+      std::string matchkey = std::string(self.key.UTF8String) + ":" + std::to_string(dis(gen));
+      std::ostringstream ss;
+      ss << "[" << static_cast<int>(tvm::runtime::TrackerCode::kPut) << ", \""
+         << self.key.UTF8String << "\", [" << rpc_server_.rpc_port << ", \"" << matchkey << "\"], "
+         << address_to_report << "]";
+
+      std::string data_s = ss.str();
+      [self toSendPacked:[NSData dataWithBytes:data_s.data() length:data_s.length()]];
+      self.state = RPCServerTracker_ReportResToRecv;
+      return TRUE;
+      break;
+    }
+    default:
+      // Nothing
+      break;
+  }
+  return FALSE;
+}
+
+- (bool)onReadHandler {
+  static const std::string resp_OK =
+      std::to_string(static_cast<int>(tvm::runtime::TrackerCode::kSuccess));
+  switch (state_) {
+    case RPCServerTracker_HandshakeToRecv: {
+      NSData* data = [self requestInputDataWithSize:sizeof(int)];
+      if (data == nil) return FALSE;
+
+      if (*(int*)data.bytes != tvm::runtime::kRPCTrackerMagic) {
+        [self notifyError:@"Wrong response, is not RPC Tracker."];
+        [self close];
+        return FALSE;
+        break;
+      }
+      self.state = RPCServerTracker_ServerInfoToSend;
+      return TRUE;
+      break;
+    }
+    case RPCServerTracker_ServerInfoToRecv: {
+      NSData* data = [self requestInputDataPacked];
+      if (data == nil) return FALSE;
+
+      if (std::string((char*)data.bytes, data.length) != resp_OK) {
+        [self notifyError:@"Failed to Update info on tracker. Response is not OK."];
+        [self close];
+        return FALSE;
+        break;
+      }
+      self.state = RPCServerTracker_ReportResToSend;
+      return TRUE;
+      break;
+    }
+    case RPCServerTracker_ReportResToRecv: {
+      NSData* data = [self requestInputDataPacked];
+      if (data == nil) return FALSE;
+
+      if (std::string((char*)data.bytes, data.length) != resp_OK) {
+        [self notifyError:@"Failed to Put server into tracker. Response is not OK."];
+        [self close];
+        return FALSE;
+        break;
+      }
+      self.state = RPCServerTracker_WaitConnection;
+      return TRUE;
+      break;
+    }
+    default:
+      // Nothing
+      break;
+  }
+  return FALSE;
+}
+
+- (void)onEndEncountered {
+  [self close];
+}
+
+- (void)close {
+  [rpc_server_ close];
+  [rpc_server_ closeSocket];
+  [super close];
+  self.state = RPCServerTracker_Idle;
+}
+
+- (void)open {
+  // Start internal Standalone RPC server at first
+  rpc_server_ = [[RPCServerStandalone alloc] init];
+  rpc_server_.key = self.key;
+  rpc_server_.delegate = self;
+  [rpc_server_ open];
+
+  [super open];
+  self.state = RPCServerTracker_HandshakeToSend;
+}
+
+- (void)onError:(NSString*)msg {
+  // transfer error form internal rpc_server_ to real delegate
+  [self notifyError:msg];
+}
+
+- (void)onStatusChanged:(RPCServerStatus)status {
+  if (status == RPCServerStatus_RPCSessionFinished) {
+    [self notifyState:status];
+    self.state = RPCServerTracker_ReportResToSend;
+    [self tryToWrite];
+  }
+}
+@end
+
+@implementation RPCServer
+
++ (instancetype)serverWithMode:(RPCServerMode)mode {
+  if (mode == RPCServerMode_Standalone) return [[RPCServerStandalone alloc] init];
+  if (mode == RPCServerMode_Proxy) return [[RPCServerProxy alloc] init];
+  if (mode == RPCServerMode_Tracker) return [[RPCServerTracker alloc] init];
+  return nil;
+}
+
+/// Unimplemented stubs
+- (void)start {
+}
+- (void)stop {
+}
+
+@end

apps/ios_rpc/tvmrpc/TVMRuntime.mm

@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file TVMRuntime.mm
+ */
+
+#import <Foundation/Foundation.h>
+
+#include <tvm/ffi/function.h>
+#include <tvm/ffi/reflection/registry.h>
+
+#include "RPCArgs.h"
+
+// internal TVM header
+#include <../../../src/runtime/file_utils.h>
+
+#if defined(USE_CUSTOM_DSO_LOADER) && USE_CUSTOM_DSO_LOADER == 1
+// internal TVM header to achieve Library class
+#include <../../../3rdparty/tvm-ffi/src/ffi/extra/library_module.h>
+#include <custom_dlfcn.h>
+#endif
+
+namespace tvm {
+namespace runtime {
+namespace detail {
+
+// Override logging mechanism
+[[noreturn]] void LogFatalImpl(const std::string& file, int lineno, const std::string& message) {
+  throw tvm::runtime::InternalError(file, lineno, message);
+}
+
+void LogMessageImpl(const std::string& file, int lineno, int level, const std::string& message) {
+  NSLog(@"%s:%d: %s", file.c_str(), lineno, message.c_str());
+}
+
+}  // namespace detail
+
+TVM_FFI_STATIC_INIT_BLOCK() {
+  namespace refl = tvm::ffi::reflection;
+  refl::GlobalDef()
+      .def_packed("tvm.rpc.server.workpath",
+                  [](ffi::PackedArgs args, ffi::Any* rv) {
+                    static const std::string base_ = NSTemporaryDirectory().UTF8String;
+                    const auto path = args[0].cast<std::string>();
+                    *rv = base_ + "/" + path;
+                  })
+      .def_packed("tvm.rpc.server.load_module", [](ffi::PackedArgs args, ffi::Any* rv) {
+        auto name = args[0].cast<std::string>();
+        std::string fmt = GetFileFormat(name, "");
+        NSString* base;
+        if (fmt == "dylib") {
+          // only load dylib from frameworks.
+          NSBundle* bundle = [NSBundle mainBundle];
+          base = [[bundle privateFrameworksPath] stringByAppendingPathComponent:@"tvm"];
+
+          if (tvm::ffi::Function::GetGlobal("ffi.Module.load_from_file.dylib_custom")) {
+            // Custom dso loader is present. Will use it.
+            base = NSTemporaryDirectory();
+            fmt = "dylib_custom";
+          }
+        } else {
+          // Load other modules in tempdir.
+          base = NSTemporaryDirectory();
+        }
+        NSString* path =
+            [base stringByAppendingPathComponent:[NSString stringWithUTF8String:name.c_str()]];
+        name = [path UTF8String];
+        *rv = Module::LoadFromFile(name, fmt);
+        LOG(INFO) << "Load module from " << name << " ...";
+      });
+}
+
+#if defined(USE_CUSTOM_DSO_LOADER) && USE_CUSTOM_DSO_LOADER == 1
+
+// Custom dynamic library loader. Supports unsigned binary
+class UnsignedDSOLoader final : public Library {
+ public:
+  ~UnsignedDSOLoader() {
+    if (lib_handle_) {
+      custom_dlclose(lib_handle_);
+      lib_handle_ = nullptr;
+    };
+  }
+  void Init(const std::string& name) {
+    lib_handle_ = custom_dlopen(name.c_str(), RTLD_NOW | RTLD_LOCAL);
+    TVM_FFI_ICHECK(lib_handle_ != nullptr)
+        << "Failed to load dynamic shared library " << name << " " << custom_dlerror();
+  }
+
+  void* GetSymbol(const char* name) final { return custom_dlsym(lib_handle_, name); }
+
+ private:
+  // Library handle
+  void* lib_handle_{nullptr};
+};
+
+// Add UnsignedDSOLoader plugin in global registry
+TVM_FFI_STATIC_INIT_BLOCK() {
+  namespace refl = tvm::ffi::reflection;
+  refl::GlobalDef().def_packed("ffi.Module.load_from_file.dylib_custom",
+                               [](ffi::PackedArgs args, ffi::Any* rv) {
+                                 auto n = ffi::make_object<UnsignedDSOLoader>();
+                                 n->Init(args[0]);
+                                 *rv = tvm::ffi::CreateLibraryModule(n);
+                               });
+}
+
+#endif
+
+}  // namespace runtime
+}  // namespace tvm

apps/ios_rpc/tvmrpc/ViewController.h

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file ViewController.h
+ */
+
+#import <UIKit/UIKit.h>
+#import "RPCServer.h"
+
+@interface ViewController : UIViewController <RPCServerEventListener, UITextFieldDelegate>
+
+@property(weak, nonatomic) IBOutlet UITextField* proxyURL;
+@property(weak, nonatomic) IBOutlet UITextField* proxyPort;
+@property(weak, nonatomic) IBOutlet UITextField* proxyKey;
+@property(weak, nonatomic) IBOutlet UILabel* statusLabel;
+@property(weak, nonatomic) IBOutlet UITextView* infoText;
+
+- (IBAction)connect:(id)sender;
+@property(retain, nonatomic) IBOutlet UIButton* ConnectButton;
+@property(retain, nonatomic) IBOutlet UISegmentedControl* ModeSelector;
+
+@end

apps/ios_rpc/tvmrpc/ViewController.mm

@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file ViewController.mm
+ */
+
+#import "ViewController.h"
+#import "RPCArgs.h"
+
+@implementation ViewController {
+  // server implementation
+  RPCServer* server_;
+  // Button state. True - push will start connection, false - push will disconnect
+  bool to_connect_;
+}
+
+- (void)viewDidLoad {
+  // To handle end editing events
+  self.proxyURL.delegate = self;
+  self.proxyPort.delegate = self;
+  self.proxyKey.delegate = self;
+
+  RPCArgs args = get_current_rpc_args();
+  self.proxyURL.text = @(args.host_url);
+  self.proxyPort.text = @(args.host_port).stringValue;
+  self.proxyKey.text = @(args.key);
+
+  self.ModeSelector.selectedSegmentIndex = args.server_mode;
+  self->to_connect_ = true;
+
+  // Add border to button
+  void (^addBorder)(UIButton* btn) = ^(UIButton* btn) {
+    btn.layer.borderWidth = 2.0f;
+    btn.layer.borderColor = self.ConnectButton.currentTitleColor.CGColor;
+    btn.layer.cornerRadius = 10;
+  };
+  addBorder(self.ConnectButton);
+
+  // Connect to tracker immediately
+  if (args.immediate_connect) {
+    [self disableUIInteraction];
+    [self open];
+  }
+}
+
+/*!
+ * \brief Disable all UI elements
+ */
+- (void)disableUIInteraction {
+  void (^disable)(UITextField* field) = ^(UITextField* field) {
+    field.enabled = NO;
+    field.backgroundColor = [UIColor lightGrayColor];
+  };
+
+  void (^disableButton)(UIButton* btn) = ^(UIButton* btn) {
+    btn.enabled = NO;
+    btn.layer.borderColor = btn.currentTitleColor.CGColor;
+  };
+
+  disable(self.proxyURL);
+  disable(self.proxyPort);
+  disable(self.proxyKey);
+  disableButton(self.ConnectButton);
+  self.ModeSelector.enabled = NO;
+}
+
+/*!
+ * \brief Start RPC server
+ */
+- (void)open {
+  RPCArgs args = get_current_rpc_args();
+
+  RPCServerMode server_mode = static_cast<RPCServerMode>(self.ModeSelector.selectedSegmentIndex);
+
+  server_ = [RPCServer serverWithMode:server_mode];
+  server_.host = self.proxyURL.text;
+  server_.port = self.proxyPort.text.intValue;
+  server_.key = self.proxyKey.text;
+  server_.custom_addr = [NSString stringWithUTF8String:args.custom_addr];
+  server_.verbose = args.verbose;
+  server_.delegate = self;
+
+  [server_ start];
+
+  self.infoText.text = @"";
+  self.statusLabel.text = @"Connecting...";
+}
+
+/*!
+ * \brief Stop RPC server
+ */
+- (void)close {
+  [server_ stop];
+  self.statusLabel.text = @"Disconnecting...";
+}
+
+#pragma mark - Button responders
+/*!
+ * \brief Connect/disconnect button handler
+ */
+- (IBAction)connect:(id)sender {
+  [[self view] endEditing:YES];  // to hide keyboard
+  (to_connect_ ^= true) ? [self close] : [self open];
+  [self.ConnectButton setTitle:to_connect_ ? @"Connect" : @"Disconenct"
+                      forState:UIControlStateNormal];
+}
+
+#pragma mark - UITextFieldDelegate
+
+- (BOOL)textFieldShouldReturn:(UITextField*)textField {
+  [[self view] endEditing:YES];  // to hide keyboard on ret key
+  return FALSE;
+}
+
+- (void)textFieldDidEndEditing:(UITextField*)textField {
+  // Update values in app arg cache
+  RPCArgs args = get_current_rpc_args();
+  args.host_url = [self.proxyURL.text UTF8String];
+  args.host_port = [self.proxyPort.text intValue];
+  args.key = [self.proxyKey.text UTF8String];
+  set_current_rpc_args(args);
+}
+
+#pragma mark - RPCServerEvenlListener
+
+- (void)onError:(NSString*)msg {
+  dispatch_sync(dispatch_get_main_queue(), ^{
+    self.infoText.text = [NSString stringWithFormat:@"Error: %@", msg];
+  });
+}
+
+- (void)onStatusChanged:(RPCServerStatus)status {
+  dispatch_sync(dispatch_get_main_queue(), ^{
+    switch (status) {
+      case RPCServerStatus_Connected:
+        if (self.ModeSelector.selectedSegmentIndex == RPCServerMode_Standalone) {
+          self.infoText.text = [NSString
+              stringWithFormat:@"IP: %@\nPort: %d", server_.device_addr, server_.actual_port];
+        }
+        self.statusLabel.text = @"Connected";
+        break;
+      case RPCServerStatus_Disconnected:
+        self.statusLabel.text = @"Disconnected";
+        break;
+      default:
+        // Nothing
+        break;
+    }
+  });
+}
+
+@end

apps/ios_rpc/tvmrpc/main.m

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \file main.m
+ */
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+#import "RPCArgs.h"
+
+int main(int argc, char* argv[]) {
+  update_rpc_args(argc, argv);
+  @autoreleasepool {
+    return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
+  }
+}