chore: import upstream snapshot with attribution

.clang-format

@@ -0,0 +1,8 @@
+# Run the following command to reformat a file:
+# clang-format -i -style=Google <file>
+# Or use clang-format-diff to only reformat the changed lines:
+# https://clang.llvm.org/docs/ClangFormat.html
+BasedOnStyle: Google
+DerivePointerAlignment: false
+ColumnLimit: 100
+PointerAlignment: Left

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,43 @@
+---
+name: "🐛 Bug Report"
+about: Submit a bug report to help us improve MLC-LLM
+title: '[Bug] '
+labels: ['bug']
+assignees: ''
+
+---
+
+## 🐛 Bug
+
+<!-- A clear and concise description of what the bug is. -->
+
+## To Reproduce
+
+Steps to reproduce the behavior:
+
+1.
+1.
+1.
+
+<!-- If you have a code sample, error messages, stack traces, please provide it here as well -->
+
+## Expected behavior
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+## Environment
+
+ - Platform (e.g. WebGPU/Vulkan/IOS/Android/CUDA):
+ - Operating system (e.g. Ubuntu/Windows/MacOS/...):
+ - Device (e.g. iPhone 12 Pro, PC+RTX 3090, ...)
+ - How you installed MLC-LLM (`conda`, source):
+ - How you installed TVM (`pip`, source):
+ - Python version (e.g. 3.10):
+ - GPU driver version (if applicable):
+ - CUDA/cuDNN version (if applicable):
+ - TVM Hash Tag (`python -c "import tvm; print('\n'.join(f'{k}: {v}' for k, v in tvm.support.libinfo().items()))"`, applicable if you compile models):
+ - Any other relevant information:
+
+## Additional context
+
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,9 @@
+blank_issues_enabled: false
+
+contact_links:
+  - name: Check the MLC-LLM Documentation
+    url: https://llm.mlc.ai/docs/
+    about: Our documentation might provide answers to your questions.
+  - name: Chat on Discord
+    url: https://discord.gg/9Xpy2HGBuD
+    about: Join the Discord Server to live chat with the community.

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,17 @@
+---
+name: "\U0001F4DA Documentation"
+about: Report an issue related to https://llm.mlc.ai/docs/
+title: '[Doc] '
+labels: ['documentation']
+assignees: ''
+
+---
+
+## 📚 Documentation
+
+### Suggestion
+<!-- Please leave your general suggestion to our documentation here. -->
+
+### Bug
+- Link to the buggy documentation/tutorial:
+- Description of the bug:

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,23 @@
+---
+name: "\U0001F680 Feature Request"
+about: Submit a proposal/request for a new MLC-LLM feature, or an enhancement on existing features.
+title: '[Feature Request] '
+labels: ['feature request']
+assignees: ''
+
+---
+
+## 🚀 Feature
+<!-- A brief description of the feature proposal -->
+
+## Motivation
+
+<!-- Please outline the motivation for the proposal, and how could this feature benefit the MLC-LLM project/community. -->
+
+## Alternatives
+
+<!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->
+
+## Additional context
+
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/general.md

@@ -0,0 +1,12 @@
+---
+name: "❓ General Questions"
+about: General questions you have about MLC-LLM.
+title: '[Question] '
+labels: ['question']
+assignees: ''
+
+---
+
+## ❓ General Questions
+
+<!-- Describe your questions -->

.github/ISSUE_TEMPLATE/model-request.md

@@ -0,0 +1,17 @@
+---
+name: "️️⚙️  Model Request"
+about: Request a new model in MLC-LLM
+title: '[Model Request] '
+labels: ['new-models']
+assignees: ''
+
+---
+
+## ⚙️  Request New Models
+
+- Link to an existing implementation (e.g. Hugging Face/Github): <!-- Link to the model -->
+- Is this model architecture supported by MLC-LLM? (the list of [supported models](https://llm.mlc.ai/docs/prebuilt_models.html)) <!-- Yes/No -->
+
+## Additional context
+
+<!-- Add any other context that you think would be helpful for the community to add this model -->

.github/ISSUE_TEMPLATE/speed-report.md

@@ -0,0 +1,24 @@
+---
+name: " 🏎️  Speed Report"
+about: Submit a speed report of an model running in MLC-LLM
+title: '[Speed] '
+labels: ['performance']
+assignees: ''
+
+---
+
+# 🏎️  Speed Report
+
+<!-- Please search if there are existing issues discuss the speed of the model you are using, if there are, we encourage you reply in the existed issue instead of creating a new one. -->
+
+- The model code: <!-- e.g. vicuna-7b-1.1 -->
+
+
+- The model configuration (e.g. quantization mode, running data type, etc.):
+- Device (e.g. MacBook Pro M2, PC+RTX 3080):
+- OS (if applicable):
+- Encode speed (Token/s):
+- Decode speed (Token/s):
+- Memory usage (if applicable):
+
+<!-- Note that the measured speed might reflect peak performance if the prompt/chat history is short. -->

.github/ISSUE_TEMPLATE/tracking.md

@@ -0,0 +1,40 @@
+---
+name: "Tracking"
+about: A tracking issue that tracks ongoing item in the project
+title: '[Tracking] '
+labels: ['status: tracking']
+assignees: ''
+
+---
+
+<!--
+
+A tracking issue contains a list of action items
+that can be executed to complete a feature or fix.
+
+We use tracking issues when we have a clear list of action items
+related to feature items as they provide fine-grained
+view of action items and provide clarity on what it takes to implement a feature.
+
+When to open a tracking issue: Open a new tracking issue when you have
+clear, actionable items (as a rule of thumb, make sure action items
+items can be carried through if you are assigned to work on it and
+you can provide enough guides to others who plan to work on these actions).
+-->
+
+
+## Overview
+<!-- A brief overview of the task  -->
+
+
+
+## Action Items
+<!-- Please list set of action items to complete -->
+
+- [ ]
+
+
+## Links to Related Issues and PRs
+
+<!-- Cross link feature requests bug report issues related to the tracking item -->
+<!-- When there are new PRs, open up new PRs -->

.github/workflows/documentation.yaml

@@ -0,0 +1,39 @@
+name: Build Docs
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  test_linux:
+    name: Deploy Docs
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Configuring build Environment
+      run: |
+        sudo apt-get update
+        python -m pip install -U pip wheel
+
+    - name: Setup Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.0'
+
+    - name: Installing dependencies
+      run: |
+        python -m pip install -r docs/requirements.txt
+        gem install jekyll jekyll-remote-theme
+
+    - name: Deploying on GitHub Pages
+      if: github.ref == 'refs/heads/main'
+      run: |
+        git remote set-url origin https://x-access-token:${{ secrets.MLC_GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
+        git config --global user.email "mlc-gh-actions-bot@nomail"
+        git config --global user.name "mlc-gh-actions-bot"
+        ./scripts/gh_deploy_site.sh

.github/workflows/lint.yml

@@ -0,0 +1,25 @@
+name: Lint
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: Lint-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Set up uv
+        uses: astral-sh/setup-uv@b75a909f75acd358c2196fb9a5f1299a9a8868a4  # v6.7.0
+      - name: Set up Python environment
+        run: uv sync --group lint --no-install-project
+      - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd  # v3.0.1

.github/workflows/update-relax.yaml

@@ -0,0 +1,32 @@
+name: 'Relax Submodule Sync'
+
+on:
+  workflow_dispatch:
+
+jobs:
+  sync:
+    name: 'Relax Submodule Sync'
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Git Sumbodule Update
+      run: |
+        git submodule update --remote 3rdparty/tvm
+
+    - name: Commit update
+      env:
+        GITHUB_TOKEN: ${{ secrets.MLC_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name 'Git bot'
+        git config --global user.email 'bot@noreply.github.com'
+        git remote set-url origin https://$GITHUB_TOKEN@github.com/mlc-ai/mlc-llm
+        git commit -am "Auto updated submodule references" && git push || echo "No changes to commit"

.github/workflows/windows-build.yaml

@@ -0,0 +1,49 @@
+# GH actions.
+# We use it to cover windows builds
+# Jenkins is still the primary CI
+name: Windows CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  Windows:
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: 'cmd /C call {0}'
+
+    steps:
+    - name: Git config
+      run: >-
+        git config --system core.longpaths true
+    - uses: actions/checkout@v3
+      with:
+        submodules: 'recursive'
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: mlc-llm-build
+        channel-priority: strict
+        environment-file: ci/build-environment.yaml
+        auto-activate-base: false
+    - name: Conda info
+      run: |
+        conda info
+        conda list
+        python --version
+    # The wheel is built with scikit-build-core, which uses the Ninja
+    # generator on Windows. Ninja needs cl.exe on PATH, so activate the MSVC
+    # developer environment before building (otherwise CMake aborts with
+    # "CMAKE_C_COMPILER not set, after EnableLanguage").
+    - name: Set up MSVC developer environment
+      uses: ilammy/msvc-dev-cmd@v1
+      with:
+        arch: x64
+    - name: Build MLC-LLM
+      run: >-
+        ci/task/build_win.bat

.gitignore

@@ -0,0 +1,324 @@
+tmp/
+dist/
+params/
+debug/
+*.bak
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+.DS_Store
+
+*.S
+# C extensions
+*.so
+
+build/
+
+*.ll
+.npm
+# Distribution / packaging
+.Python
+env/
+build/
+build-*/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+.conda/
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Generated by python/gen_requirements.py
+python/requirements/*.txt
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/_staging/
+
+# PyBuilder
+target/
+/target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+*~
+*.pyc
+*~
+config.mk
+config.cmake
+Win32
+*.dir
+perf
+*.wasm
+.emscripten
+
+## IOS
+DerivedData/
+
+## Java
+*.class
+jvm/*/target/
+jvm/*/*/target/
+jvm/native/*/generated
+jvm/native/src/main/native/org_apache_tvm_native_c_api.h
+*.worksheet
+*.idea
+*.iml
+*.classpath
+*.project
+*.settings
+*/node_modules/
+
+## Various settings
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+xcuserdata/
+.pkl_memoize_*
+
+.emscripten*
+.m2
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+## Other
+*.moved-aside
+*.xccheckout
+*.xcscmblueprint
+.DS_Store
+tags
+cscope*
+*.lock
+
+# vim temporary files
+*.swp
+*.swo
+
+# TVM generated code
+perf
+.bash_history
+# *.json
+*.params
+*.ro
+*.onnx
+*.h5
+synset.txt
+cat.jpg
+cat.png
+docs.tgz
+cat.png
+*.mlmodel
+tvm_u.*
+tvm_t.*
+# Mac OS X
+.DS_Store
+
+# Jetbrain
+.idea
+.ipython
+.jupyter
+.nv
+.pylint.d
+.python_history
+.pytest_cache
+.local
+cmake-build-debug
+
+# Visual Studio
+.vs
+
+# Visual Studio Code
+.vscode
+
+# tmp file
+.nfs*
+
+# keys
+*.pem
+*.p12
+*.pfx
+*.cer
+*.crt
+*.der
+
+# patch sentinel
+patched.txt
+
+# Python type checking
+.mypy_cache/
+.pyre/
+
+# pipenv files
+Pipfile
+Pipfile.lock
+
+# conda package artifacts
+conda/Dockerfile.cuda*
+conda/pkg
+.node_repl_history
+# nix files
+.envrc
+*.nix
+
+# Docker files
+.sudo_as_admin_successful
+
+# Downloaded models/datasets
+.tvm_test_data
+.dgl
+.caffe2
+
+# Local docs build
+_docs/
+jvm/target
+.config/configstore/
+.ci-py-scripts/
+
+# Generated Hexagon files
+src/runtime/hexagon/rpc/hexagon_rpc.h
+src/runtime/hexagon/rpc/hexagon_rpc_skel.c
+src/runtime/hexagon/rpc/hexagon_rpc_stub.c
+
+# Local tvm-site checkout
+tvm-site/
+
+# Generated docs files
+gallery/how_to/work_with_microtvm/micro_tvmc.py
+
+# Test sample data files
+!tests/python/ci/sample_prs/*.json
+
+# Used in CI to communicate between Python and Jenkins
+.docker-image-names/
+
+# Printed TIR code on disk
+*.tir
+
+# GDB history file
+.gdb_history
+
+dist

.gitmodules

@@ -0,0 +1,18 @@
+[submodule "3rdparty/argparse"]
+	path = 3rdparty/argparse
+	url = https://github.com/p-ranav/argparse
+[submodule "3rdparty/tokenizers-cpp"]
+	path = 3rdparty/tokenizers-cpp
+	url = https://github.com/mlc-ai/tokenizers-cpp
+[submodule "3rdparty/googletest"]
+	path = 3rdparty/googletest
+	url = https://github.com/google/googletest.git
+[submodule "3rdparty/tvm"]
+	path = 3rdparty/tvm
+	url = https://github.com/mlc-ai/relax.git
+[submodule "3rdparty/stb"]
+	path = 3rdparty/stb
+	url = https://github.com/nothings/stb.git
+[submodule "3rdparty/xgrammar"]
+	path = 3rdparty/xgrammar
+	url = https://github.com/mlc-ai/xgrammar.git

.pre-commit-config.yaml

@@ -0,0 +1,68 @@
+# To use:
+#
+#     pre-commit run -a
+#
+# Or:
+#
+#     pre-commit install  # (runs every time you commit in git)
+#
+# To update this file:
+#
+#     pre-commit autoupdate
+#
+# See https://github.com/pre-commit/pre-commit
+default_install_hook_types:
+  - pre-commit
+repos:
+  # ---------- General file checks ----------
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: end-of-file-fixer
+        exclude: ^3rdparty/
+      - id: mixed-line-ending
+        exclude: ^3rdparty/
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+        exclude: ^3rdparty/
+      - id: check-yaml
+      - id: check-toml
+
+  # ---------- YAML lint ----------
+  - repo: https://github.com/adrienverge/yamllint
+    rev: v1.35.1
+    hooks:
+      - id: yamllint
+        args:
+          - --config-file
+          - .yamllint.yaml
+        exclude: ^\.github/
+
+  # ---------- TOML format ----------
+  - repo: https://github.com/ComPWA/taplo-pre-commit
+    rev: v0.9.3
+    hooks:
+      - id: taplo-format
+
+  # ---------- Python: ruff ----------
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.12.3
+    hooks:
+      - id: ruff-check
+        types_or: [python, pyi, jupyter]
+        args: [--fix]
+      - id: ruff-format
+        types_or: [python, pyi, jupyter]
+
+  # ---------- C/C++: clang-format ----------
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: "v20.1.8"
+    hooks:
+      - id: clang-format
+        types_or: [c, c++, cuda, objective-c++]
+        exclude: |
+          (?x)^(.*cubin\.cpp$|.*fmha_cubin\.h|3rdparty/.*)$

.yamllint.yaml

@@ -0,0 +1,15 @@
+extends: default
+
+rules:
+  document-start: disable
+  line-length:
+    max: 120
+    level: warning
+  truthy:
+    allowed-values:
+      - "on"
+      - "off"
+      - "yes"
+      - "no"
+      - "true"
+      - "false"

CMakeLists.txt

@@ -0,0 +1,218 @@
+cmake_minimum_required(VERSION 3.18)
+project(mlc_llm C CXX)
+
+include(CheckCXXCompilerFlag)
+if(MSVC)
+  set(CMAKE_CXX_FLAGS "/fp:fast ${CMAKE_CXX_FLAGS}")
+else()
+  set(CMAKE_CXX_FLAGS "-ffast-math ${CMAKE_CXX_FLAGS}")
+endif()
+
+if(EXISTS ${CMAKE_BINARY_DIR}/config.cmake)
+  include(${CMAKE_BINARY_DIR}/config.cmake)
+else()
+  if(EXISTS ${CMAKE_SOURCE_DIR}/config.cmake)
+    include(${CMAKE_SOURCE_DIR}/config.cmake)
+  endif()
+endif()
+
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE
+      RelWithDebInfo
+      CACHE STRING "Build type" FORCE)
+  message(STATUS "Setting default build type to " ${CMAKE_BUILD_TYPE})
+endif(NOT CMAKE_BUILD_TYPE)
+
+option(MLC_HIDE_PRIVATE_SYMBOLS "Hide private symbols" ON)
+option(MLC_LLM_BUILD_PYTHON_MODULE "Build Python module with scikit-build-core"
+       OFF)
+
+if(MLC_LLM_INSTALL_STATIC_LIB)
+  set(BUILD_STATIC_RUNTIME ON)
+endif()
+
+set(MLC_VISIBILITY_FLAG "")
+if(MLC_HIDE_PRIVATE_SYMBOLS)
+  set(HIDE_PRIVATE_SYMBOLS ON)
+  if(NOT MSVC)
+    set(MLC_VISIBILITY_FLAG "-fvisibility=hidden")
+  endif()
+  message(STATUS "Hide private symbols")
+endif()
+
+option(BUILD_CPP_TEST "Build cpp unittests" OFF)
+
+set(CMAKE_CUDA_STANDARD 17)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+# tvm runtime config: minimize runtime components
+set(USE_RPC OFF)
+set(USE_MICRO OFF)
+set(USE_GRAPH_EXECUTOR OFF)
+set(USE_GRAPH_EXECUTOR_DEBUG OFF)
+set(USE_AOT_EXECUTOR OFF)
+set(USE_PROFILER OFF)
+set(USE_GTEST OFF)
+set(USE_LIBBACKTRACE OFF)
+set(BUILD_DUMMY_LIBTVM ON)
+if(NOT DEFINED TVM_SOURCE_DIR)
+  if(DEFINED ENV{TVM_SOURCE_DIR})
+    set(TVM_SOURCE_DIR "$ENV{TVM_SOURCE_DIR}")
+  else()
+    set(TVM_SOURCE_DIR 3rdparty/tvm)
+  endif(DEFINED ENV{TVM_SOURCE_DIR})
+endif(NOT DEFINED TVM_SOURCE_DIR)
+message(STATUS "TVM_SOURCE_DIR: ${TVM_SOURCE_DIR}")
+add_subdirectory(${TVM_SOURCE_DIR} tvm EXCLUDE_FROM_ALL)
+
+set(MLC_LLM_RUNTIME_LINKER_LIB "")
+set(TOKENZIER_CPP_PATH 3rdparty/tokenizers-cpp)
+add_subdirectory(${TOKENZIER_CPP_PATH} tokenizers EXCLUDE_FROM_ALL)
+
+set(XGRAMMAR_PATH 3rdparty/xgrammar)
+tvm_file_glob(GLOB_RECURSE MLC_LLM_SRCS cpp/*.cc)
+tvm_file_glob(GLOB_RECURSE XGRAMMAR_SRCS ${XGRAMMAR_PATH}/cpp/*.cc)
+list(FILTER XGRAMMAR_SRCS EXCLUDE REGEX "${XGRAMMAR_PATH}/cpp/pybind/.*\\.cc")
+list(APPEND MLC_LLM_SRCS ${XGRAMMAR_SRCS})
+add_library(mlc_llm_objs OBJECT ${MLC_LLM_SRCS})
+
+set(MLC_LLM_INCLUDES
+    ${TVM_SOURCE_DIR}/include ${TVM_SOURCE_DIR}/3rdparty/dlpack/include)
+set(MLC_LLM_COMPILE_DEFS ${MLC_LLM_COMPILE_DEFS} __STDC_FORMAT_MACROS=1)
+set(MLC_LLM_COMPILE_DEFS ${MLC_LLM_COMPILE_DEFS} XGRAMMAR_ENABLE_LOG_DEBUG=0)
+
+target_compile_definitions(mlc_llm_objs PRIVATE ${MLC_LLM_COMPILE_DEFS})
+target_compile_definitions(mlc_llm_objs PRIVATE -DMLC_LLM_EXPORTS)
+target_include_directories(mlc_llm_objs PRIVATE ${MLC_LLM_INCLUDES})
+target_include_directories(mlc_llm_objs PRIVATE 3rdparty/stb)
+target_include_directories(mlc_llm_objs PRIVATE ${TOKENZIER_CPP_PATH}/include)
+target_include_directories(mlc_llm_objs PRIVATE ${XGRAMMAR_PATH}/include)
+# xgrammar still depends on picojson - use its bundled copy
+target_include_directories(mlc_llm_objs
+                           PRIVATE ${XGRAMMAR_PATH}/3rdparty/picojson)
+target_link_libraries(mlc_llm_objs PRIVATE tvm_ffi_header)
+
+add_library(mlc_llm SHARED $<TARGET_OBJECTS:mlc_llm_objs>)
+add_library(mlc_llm_static STATIC $<TARGET_OBJECTS:mlc_llm_objs>)
+add_dependencies(mlc_llm_static tokenizers_cpp sentencepiece-static
+                 tokenizers_c tvm_runtime tvm_runtime_extra)
+set_target_properties(mlc_llm_static PROPERTIES OUTPUT_NAME mlc_llm)
+
+target_link_libraries(mlc_llm PUBLIC tvm_runtime)
+target_link_libraries(mlc_llm PRIVATE tvm_runtime_extra)
+target_link_libraries(mlc_llm PRIVATE tokenizers_cpp)
+
+find_library(FLASH_ATTN_LIBRARY flash_attn
+             HINTS ${TVM_SOURCE_DIR}/*/3rdparty/libflash_attn/src)
+
+if(FLASH_ATTN_LIBRARY STREQUAL "FLASH_ATTN_LIBRARY-NOTFOUND")
+  message(
+    WARNING
+      "Cannot find libflash_attn. The model must not have been built with --use-flash-attn-mqa option."
+  )
+else()
+  target_link_libraries(mlc_llm PUBLIC -Wl,--no-as-needed ${FLASH_ATTN_LIBRARY})
+endif()
+
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+  target_compile_definitions(mlc_llm PRIVATE "TVM_LOG_DEBUG")
+  target_compile_definitions(mlc_llm_objs PRIVATE "TVM_LOG_DEBUG")
+  target_compile_definitions(mlc_llm_static PRIVATE "TVM_LOG_DEBUG")
+endif()
+
+if(BUILD_CPP_TEST)
+  message(STATUS "Building cpp unittests")
+  add_subdirectory(3rdparty/googletest)
+  file(GLOB_RECURSE MLC_LLM_TEST_SRCS
+       ${PROJECT_SOURCE_DIR}/tests/cpp/*unittest.cc)
+  add_executable(mlc_llm_cpp_tests ${MLC_LLM_TEST_SRCS})
+  target_include_directories(mlc_llm_cpp_tests PRIVATE ${MLC_LLM_INCLUDES})
+  target_include_directories(mlc_llm_cpp_tests
+                             PRIVATE ${PROJECT_SOURCE_DIR}/cpp)
+  target_include_directories(
+    mlc_llm_cpp_tests PRIVATE ${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR})
+  target_link_libraries(mlc_llm_cpp_tests PUBLIC mlc_llm gtest gtest_main)
+endif(BUILD_CPP_TEST)
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+  target_link_libraries(mlc_llm PRIVATE log)
+  target_link_libraries(tokenizers_cpp PRIVATE log)
+endif()
+
+add_library(mlc_llm_module SHARED $<TARGET_OBJECTS:mlc_llm_objs>)
+target_link_libraries(mlc_llm_module PUBLIC tvm_runtime)
+target_link_libraries(mlc_llm_module PRIVATE tvm_runtime_extra)
+target_link_libraries(mlc_llm_module PRIVATE tokenizers_cpp)
+
+set_property(
+  TARGET mlc_llm_module
+  APPEND
+  PROPERTY LINK_OPTIONS "${MLC_VISIBILITY_FLAG}")
+set_property(
+  TARGET mlc_llm
+  APPEND
+  PROPERTY LINK_OPTIONS "${MLC_VISIBILITY_FLAG}")
+
+find_program(CARGO_EXECUTABLE cargo)
+
+if(NOT CARGO_EXECUTABLE)
+  message(FATAL_ERROR "Cargo is not found! Please install cargo.")
+endif()
+
+# when this option is on, we install all static lib deps into lib
+if(MLC_LLM_INSTALL_STATIC_LIB)
+  install(TARGETS mlc_llm_static tokenizers_cpp sentencepiece-static tvm_runtime
+                  tvm_runtime_extra
+          LIBRARY DESTINATION lib${LIB_SUFFIX})
+  # tokenizers need special handling as it builds from rust
+  if(MSVC)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/libtokenizers_c.lib
+            DESTINATION lib${LIB_SUFFIX})
+  else()
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/libtokenizers_c.a
+            DESTINATION lib${LIB_SUFFIX})
+  endif()
+else()
+  install(
+    TARGETS tvm_runtime
+            tvm_runtime_extra
+            mlc_llm
+            mlc_llm_module
+            mlc_llm_static
+            tokenizers_cpp
+            sentencepiece-static
+            RUNTIME_DEPENDENCY_SET
+            tokenizers_c
+    RUNTIME DESTINATION bin
+    LIBRARY DESTINATION lib${LIB_SUFFIX})
+endif()
+
+# Python package installation configuration This section ensures that all
+# necessary files are installed for the Python wheel
+if(MLC_LLM_BUILD_PYTHON_MODULE)
+  message(STATUS "Configuring Python package installation")
+
+  # Set RPATH for mlc_llm and mlc_llm_module to find other libraries relatively
+  if(APPLE)
+    # macOS uses @loader_path
+    set_target_properties(mlc_llm PROPERTIES INSTALL_RPATH "@loader_path")
+    set_target_properties(mlc_llm_module PROPERTIES INSTALL_RPATH
+                                                    "@loader_path")
+  elseif(LINUX)
+    # Linux uses $ORIGIN
+    set_target_properties(mlc_llm PROPERTIES INSTALL_RPATH "\$ORIGIN")
+    set_target_properties(mlc_llm_module PROPERTIES INSTALL_RPATH "\$ORIGIN")
+  endif()
+
+  # Install compiled shared libraries
+  install(TARGETS mlc_llm DESTINATION ".")
+  install(TARGETS mlc_llm_module DESTINATION ".")
+  install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/cpp/" DESTINATION "cpp/")
+  install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/web/" DESTINATION "web/")
+  install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/README.md"
+                "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE"
+                "${CMAKE_CURRENT_SOURCE_DIR}/NOTICE" DESTINATION ".")
+
+  message(STATUS "Python package installation configured")
+endif()

CONTRIBUTORS.md

@@ -0,0 +1,6 @@
+MLC LLM Contributors
+====================
+
+
+## List of Contributors
+- [Full List of Contributors](https://github.com/mlc-ai/mlc-llm/graphs/contributors)

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

NOTICE

@@ -0,0 +1,3 @@
+MLC LLM
+
+Copyright (c) 2023-2025 by MLC LLM Contributors

README.md

@@ -0,0 +1,133 @@
+<div align="center">
+
+# MLC LLM
+
+[![Installation](https://img.shields.io/badge/docs-latest-green)](https://llm.mlc.ai/docs/)
+[![License](https://img.shields.io/badge/license-apache_2-blue)](https://github.com/mlc-ai/mlc-llm/blob/main/LICENSE)
+[![Join Discoard](https://img.shields.io/badge/Join-Discord-7289DA?logo=discord&logoColor=white)](https://discord.gg/9Xpy2HGBuD)
+[![Related Repository: WebLLM](https://img.shields.io/badge/Related_Repo-WebLLM-fafbfc?logo=github)](https://github.com/mlc-ai/web-llm/)
+
+**Universal LLM Deployment Engine with ML Compilation**
+
+[Get Started](https://llm.mlc.ai/docs/get_started/quick_start) | [Documentation](https://llm.mlc.ai/docs) | [Blog](https://blog.mlc.ai/)
+
+</div>
+
+## About
+
+MLC LLM is a machine learning compiler and high-performance deployment engine for large language models.  The mission of this project is to enable everyone to develop, optimize, and deploy AI models natively on everyone's platforms. 
+
+<div align="center">
+<table style="width:100%">
+  <thead>
+    <tr>
+      <th style="width:15%"> </th>
+      <th style="width:20%">AMD GPU</th>
+      <th style="width:20%">NVIDIA GPU</th>
+      <th style="width:20%">Apple GPU</th>
+      <th style="width:24%">Intel GPU</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>Linux / Win</td>
+      <td>✅ Vulkan, ROCm</td>
+      <td>✅ Vulkan, CUDA</td>
+      <td>N/A</td>
+      <td>✅ Vulkan</td>
+    </tr>
+    <tr>
+      <td>macOS</td>
+      <td>✅ Metal (dGPU)</td>
+      <td>N/A</td>
+      <td>✅ Metal</td>
+      <td>✅ Metal (iGPU)</td>
+    </tr>
+    <tr>
+      <td>Web Browser</td>
+      <td colspan=4>✅ WebGPU and WASM </td>
+    </tr>
+    <tr>
+      <td>iOS / iPadOS</td>
+      <td colspan=4>✅ Metal on Apple A-series GPU</td>
+    </tr>
+    <tr>
+      <td>Android</td>
+      <td colspan=2>✅ OpenCL on Adreno GPU</td>
+      <td colspan=2>✅ OpenCL on Mali GPU</td>
+    </tr>
+  </tbody>
+</table>
+</div>
+
+MLC LLM compiles and runs code on MLCEngine -- a unified high-performance LLM inference engine across the above platforms. MLCEngine provides OpenAI-compatible API available through REST server, python, javascript, iOS, Android, all backed by the same engine and compiler that we keep improving with the community.
+
+## Get Started
+
+Please visit our [documentation](https://llm.mlc.ai/docs/) to get started with MLC LLM.
+- [Installation](https://llm.mlc.ai/docs/install/mlc_llm)
+- [Quick start](https://llm.mlc.ai/docs/get_started/quick_start)
+- [Introduction](https://llm.mlc.ai/docs/get_started/introduction)
+
+## Citation
+
+Please consider citing our project if you find it useful:
+
+```bibtex
+@software{mlc-llm,
+    author = {{MLC team}},
+    title = {{MLC-LLM}},
+    url = {https://github.com/mlc-ai/mlc-llm},
+    year = {2023-2025}
+}
+```
+
+The underlying techniques of MLC LLM include:
+
+<details>
+  <summary>References (Click to expand)</summary>
+
+  ```bibtex
+  @inproceedings{tensorir,
+      author = {Feng, Siyuan and Hou, Bohan and Jin, Hongyi and Lin, Wuwei and Shao, Junru and Lai, Ruihang and Ye, Zihao and Zheng, Lianmin and Yu, Cody Hao and Yu, Yong and Chen, Tianqi},
+      title = {TensorIR: An Abstraction for Automatic Tensorized Program Optimization},
+      year = {2023},
+      isbn = {9781450399166},
+      publisher = {Association for Computing Machinery},
+      address = {New York, NY, USA},
+      url = {https://doi.org/10.1145/3575693.3576933},
+      doi = {10.1145/3575693.3576933},
+      booktitle = {Proceedings of the 28th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2},
+      pages = {804–817},
+      numpages = {14},
+      keywords = {Tensor Computation, Machine Learning Compiler, Deep Neural Network},
+      location = {Vancouver, BC, Canada},
+      series = {ASPLOS 2023}
+  }
+
+  @inproceedings{metaschedule,
+      author = {Shao, Junru and Zhou, Xiyou and Feng, Siyuan and Hou, Bohan and Lai, Ruihang and Jin, Hongyi and Lin, Wuwei and Masuda, Masahiro and Yu, Cody Hao and Chen, Tianqi},
+      booktitle = {Advances in Neural Information Processing Systems},
+      editor = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh},
+      pages = {35783--35796},
+      publisher = {Curran Associates, Inc.},
+      title = {Tensor Program Optimization with Probabilistic Programs},
+      url = {https://proceedings.neurips.cc/paper_files/paper/2022/file/e894eafae43e68b4c8dfdacf742bcbf3-Paper-Conference.pdf},
+      volume = {35},
+      year = {2022}
+  }
+
+  @inproceedings{tvm,
+      author = {Tianqi Chen and Thierry Moreau and Ziheng Jiang and Lianmin Zheng and Eddie Yan and Haichen Shen and Meghan Cowan and Leyuan Wang and Yuwei Hu and Luis Ceze and Carlos Guestrin and Arvind Krishnamurthy},
+      title = {{TVM}: An Automated {End-to-End} Optimizing Compiler for Deep Learning},
+      booktitle = {13th USENIX Symposium on Operating Systems Design and Implementation (OSDI 18)},
+      year = {2018},
+      isbn = {978-1-939133-08-3},
+      address = {Carlsbad, CA},
+      pages = {578--594},
+      url = {https://www.usenix.org/conference/osdi18/presentation/chen},
+      publisher = {USENIX Association},
+      month = oct,
+  }
+  ```
+</details>

README.wehub.md

@@ -0,0 +1,7 @@
+# WeHub 来源说明
+
+- 原始项目：`mlc-ai/mlc-llm`
+- 原始仓库：https://github.com/mlc-ai/mlc-llm
+- 导入方式：上游默认分支的最新快照
+- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
+- 本文件仅用于记录来源，不代表 WeHub 是原项目作者

android/.gitignore

@@ -0,0 +1,19 @@
+app/src/main/jni/*.h
+app/src/main/jni/*.cc
+app/src/main/obj
+
+*.iml
+.gradle
+/local.properties
+/.idea/caches
+/.idea/libraries
+/.idea/modules.xml
+/.idea/workspace.xml
+/.idea/navEditor.xml
+/.idea/assetWizardSettings.xml
+.DS_Store
+/build
+/captures
+.externalNativeBuild
+.cxx
+local.properties

android/MLCChat/README.md

@@ -0,0 +1,6 @@
+# MLC-LLM Android
+
+Checkout [Documentation page](https://llm.mlc.ai/docs/deploy/android.html) for more information.
+
+- run `mlc_llm package`
+- open this `MLCChat/` folder as a project in Android Studio

android/MLCChat/app/.gitignore

@@ -0,0 +1,2 @@
+/build
+/src/main/libs

android/MLCChat/app/build.gradle

@@ -0,0 +1,74 @@
+plugins {
+    id 'com.android.application'
+    id 'org.jetbrains.kotlin.android'
+}
+
+android {
+    namespace 'ai.mlc.mlcchat'
+    compileSdk 35
+
+    defaultConfig {
+        applicationId "ai.mlc.mlcchat"
+        minSdk 26
+        targetSdk 33
+        versionCode 1
+        versionName "1.0"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+        vectorDrawables {
+            useSupportLibrary true
+        }
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+    buildFeatures {
+        compose true
+    }
+    composeOptions {
+        kotlinCompilerExtensionVersion '1.4.3'
+    }
+    packagingOptions {
+        resources {
+            excludes += '/META-INF/{AL2.0,LGPL2.1}'
+        }
+    }
+}
+
+dependencies {
+    implementation project(":mlc4j")
+    implementation 'androidx.core:core-ktx:1.10.1'
+    implementation 'androidx.lifecycle:lifecycle-runtime-ktx:2.6.1'
+    implementation 'com.github.jeziellago:compose-markdown:0.5.2'
+    implementation 'androidx.activity:activity-compose:1.7.1'
+    implementation platform('androidx.compose:compose-bom:2022.10.00')
+    implementation 'androidx.lifecycle:lifecycle-viewmodel-compose:2.6.1'
+    implementation 'androidx.compose.ui:ui'
+    implementation 'androidx.compose.ui:ui-graphics'
+    implementation 'androidx.compose.ui:ui-tooling-preview'
+    implementation 'androidx.compose.material3:material3:1.1.0'
+    implementation 'androidx.compose.material:material-icons-extended'
+    implementation 'androidx.appcompat:appcompat:1.6.1'
+    implementation 'androidx.navigation:navigation-compose:2.5.3'
+    implementation 'com.google.code.gson:gson:2.10.1'
+    implementation fileTree(dir: 'src/main/libs', include: ['*.aar', '*.jar'], exclude: [])
+    testImplementation 'junit:junit:4.13.2'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.5'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
+    androidTestImplementation platform('androidx.compose:compose-bom:2022.10.00')
+    androidTestImplementation 'androidx.compose.ui:ui-test-junit4'
+    debugImplementation 'androidx.compose.ui:ui-tooling'
+    debugImplementation 'androidx.compose.ui:ui-test-manifest'
+
+}

android/MLCChat/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

android/MLCChat/app/src/main/AndroidManifest.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    package="ai.mlc.mlcchat">
+
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.READ_MEDIA_IMAGES" />
+    <uses-permission
+        android:name="android.permission.WRITE_EXTERNAL_STORAGE"
+        android:maxSdkVersion="32"
+        tools:ignore="ScopedStorage" />
+
+    <application
+        android:allowBackup="true"
+        android:dataExtractionRules="@xml/data_extraction_rules"
+        android:fullBackupContent="@xml/backup_rules"
+        android:icon="@drawable/mlc_logo_108"
+        android:label="@string/app_name"
+        android:roundIcon="@drawable/mlc_logo_108"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.MLCChat"
+        tools:targetApi="31">
+        <uses-native-library
+            android:name="libOpenCL.so"
+            android:required="false"/>
+
+        <uses-native-library
+            android:name="libOpenCL-pixel.so"
+            android:required="false" />
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:label="@string/app_name"
+            android:theme="@android:style/Theme.Material.NoActionBar">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

android/MLCChat/app/src/main/java/ai/mlc/mlcchat/AppViewModel.kt

@@ -0,0 +1,852 @@
+package ai.mlc.mlcchat
+
+import ai.mlc.mlcllm.MLCEngine
+import ai.mlc.mlcllm.OpenAIProtocol
+import android.app.Application
+import android.content.ClipData
+import android.content.ClipboardManager
+import android.content.Context
+import android.os.Environment
+import android.widget.Toast
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.toMutableStateList
+import androidx.lifecycle.AndroidViewModel
+import androidx.lifecycle.viewModelScope
+import com.google.gson.Gson
+import com.google.gson.annotations.SerializedName
+import kotlinx.coroutines.launch
+import java.io.File
+import java.io.FileOutputStream
+import java.net.URL
+import java.nio.channels.Channels
+import java.util.UUID
+import java.util.concurrent.Executors
+import kotlin.concurrent.thread
+import ai.mlc.mlcllm.OpenAIProtocol.ChatCompletionMessage
+import ai.mlc.mlcllm.OpenAIProtocol.ChatCompletionMessageContent
+import android.app.Activity
+import kotlinx.coroutines.*
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.net.Uri
+import java.io.ByteArrayOutputStream
+import android.util.Base64
+import android.util.Log
+
+class AppViewModel(application: Application) : AndroidViewModel(application) {
+    val modelList = emptyList<ModelState>().toMutableStateList()
+    val chatState = ChatState()
+    val modelSampleList = emptyList<ModelRecord>().toMutableStateList()
+    private var showAlert = mutableStateOf(false)
+    private var alertMessage = mutableStateOf("")
+    private var appConfig = AppConfig(
+        emptyList<String>().toMutableList(),
+        emptyList<ModelRecord>().toMutableList()
+    )
+    private val application = getApplication<Application>()
+    private val appDirFile = application.getExternalFilesDir("")
+    private val gson = Gson()
+    private val modelIdSet = emptySet<String>().toMutableSet()
+
+    companion object {
+        const val AppConfigFilename = "mlc-app-config.json"
+        const val ModelConfigFilename = "mlc-chat-config.json"
+        const val ParamsConfigFilename = "tensor-cache.json"
+        const val ModelUrlSuffix = "resolve/main/"
+    }
+
+    init {
+        loadAppConfig()
+    }
+
+    fun isShowingAlert(): Boolean {
+        return showAlert.value
+    }
+
+    fun errorMessage(): String {
+        return alertMessage.value
+    }
+
+    fun dismissAlert() {
+        require(showAlert.value)
+        showAlert.value = false
+    }
+
+    fun copyError() {
+        require(showAlert.value)
+        val clipboard =
+            application.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
+        clipboard.setPrimaryClip(ClipData.newPlainText("MLCChat", errorMessage()))
+    }
+
+    private fun issueAlert(error: String) {
+        showAlert.value = true
+        alertMessage.value = error
+    }
+
+    fun requestDeleteModel(modelId: String) {
+        deleteModel(modelId)
+        issueAlert("Model: $modelId has been deleted")
+    }
+
+
+    private fun loadAppConfig() {
+        val appConfigFile = File(appDirFile, AppConfigFilename)
+        val jsonString: String = if (!appConfigFile.exists()) {
+            application.assets.open(AppConfigFilename).bufferedReader().use { it.readText() }
+        } else {
+            appConfigFile.readText()
+        }
+        appConfig = gson.fromJson(jsonString, AppConfig::class.java)
+        appConfig.modelLibs = emptyList<String>().toMutableList()
+        modelList.clear()
+        modelIdSet.clear()
+        modelSampleList.clear()
+        for (modelRecord in appConfig.modelList) {
+            appConfig.modelLibs.add(modelRecord.modelLib)
+            val modelDirFile = File(appDirFile, modelRecord.modelId)
+            val modelConfigFile = File(modelDirFile, ModelConfigFilename)
+            if (modelConfigFile.exists()) {
+                val modelConfigString = modelConfigFile.readText()
+                val modelConfig = gson.fromJson(modelConfigString, ModelConfig::class.java)
+                modelConfig.modelId = modelRecord.modelId
+                modelConfig.modelLib = modelRecord.modelLib
+                modelConfig.estimatedVramBytes = modelRecord.estimatedVramBytes
+                addModelConfig(modelConfig, modelRecord.modelUrl, true)
+            } else {
+                downloadModelConfig(
+                    if (modelRecord.modelUrl.endsWith("/")) modelRecord.modelUrl else "${modelRecord.modelUrl}/",
+                    modelRecord,
+                    true
+                )
+            }
+        }
+    }
+
+    private fun updateAppConfig(action: () -> Unit) {
+        action()
+        val jsonString = gson.toJson(appConfig)
+        val appConfigFile = File(appDirFile, AppConfigFilename)
+        appConfigFile.writeText(jsonString)
+    }
+
+    private fun addModelConfig(modelConfig: ModelConfig, modelUrl: String, isBuiltin: Boolean) {
+        require(!modelIdSet.contains(modelConfig.modelId))
+        modelIdSet.add(modelConfig.modelId)
+        modelList.add(
+            ModelState(
+                modelConfig,
+                modelUrl + if (modelUrl.endsWith("/")) "" else "/",
+                File(appDirFile, modelConfig.modelId)
+            )
+        )
+        if (!isBuiltin) {
+            updateAppConfig {
+                appConfig.modelList.add(
+                    ModelRecord(
+                        modelUrl,
+                        modelConfig.modelId,
+                        modelConfig.estimatedVramBytes,
+                        modelConfig.modelLib
+                    )
+                )
+            }
+        }
+    }
+
+    private fun deleteModel(modelId: String) {
+        val modelDirFile = File(appDirFile, modelId)
+        modelDirFile.deleteRecursively()
+        require(!modelDirFile.exists())
+        modelIdSet.remove(modelId)
+        modelList.removeIf { modelState -> modelState.modelConfig.modelId == modelId }
+        updateAppConfig {
+            appConfig.modelList.removeIf { modelRecord -> modelRecord.modelId == modelId }
+        }
+    }
+
+    private fun isModelConfigAllowed(modelConfig: ModelConfig): Boolean {
+        if (appConfig.modelLibs.contains(modelConfig.modelLib)) return true
+        viewModelScope.launch {
+            issueAlert("Model lib ${modelConfig.modelLib} is not supported.")
+        }
+        return false
+    }
+
+
+    private fun downloadModelConfig(
+        modelUrl: String,
+        modelRecord: ModelRecord,
+        isBuiltin: Boolean
+    ) {
+        thread(start = true) {
+            try {
+                val url = URL("${modelUrl}${ModelUrlSuffix}${ModelConfigFilename}")
+                val tempId = UUID.randomUUID().toString()
+                val tempFile = File(
+                    application.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS),
+                    tempId
+                )
+                url.openStream().use {
+                    Channels.newChannel(it).use { src ->
+                        FileOutputStream(tempFile).use { fileOutputStream ->
+                            fileOutputStream.channel.transferFrom(src, 0, Long.MAX_VALUE)
+                        }
+                    }
+                }
+                require(tempFile.exists())
+                viewModelScope.launch {
+                    try {
+                        val modelConfigString = tempFile.readText()
+                        val modelConfig = gson.fromJson(modelConfigString, ModelConfig::class.java)
+                        modelConfig.modelId = modelRecord.modelId
+                        modelConfig.modelLib = modelRecord.modelLib
+                        modelConfig.estimatedVramBytes = modelRecord.estimatedVramBytes
+                        if (modelIdSet.contains(modelConfig.modelId)) {
+                            tempFile.delete()
+                            issueAlert("${modelConfig.modelId} has been used, please consider another local ID")
+                            return@launch
+                        }
+                        if (!isModelConfigAllowed(modelConfig)) {
+                            tempFile.delete()
+                            return@launch
+                        }
+                        val modelDirFile = File(appDirFile, modelConfig.modelId)
+                        val modelConfigFile = File(modelDirFile, ModelConfigFilename)
+                        tempFile.copyTo(modelConfigFile, overwrite = true)
+                        tempFile.delete()
+                        require(modelConfigFile.exists())
+                        addModelConfig(modelConfig, modelUrl, isBuiltin)
+                    } catch (e: Exception) {
+                        viewModelScope.launch {
+                            issueAlert("Add model failed: ${e.localizedMessage}")
+                        }
+                    }
+                }
+            } catch (e: Exception) {
+                viewModelScope.launch {
+                    issueAlert("Download model config failed: ${e.localizedMessage}")
+                }
+            }
+
+        }
+    }
+
+    inner class ModelState(
+        val modelConfig: ModelConfig,
+        private val modelUrl: String,
+        private val modelDirFile: File
+    ) {
+        var modelInitState = mutableStateOf(ModelInitState.Initializing)
+        private var paramsConfig = ParamsConfig(emptyList())
+        val progress = mutableStateOf(0)
+        val total = mutableStateOf(1)
+        val id: UUID = UUID.randomUUID()
+        private val remainingTasks = emptySet<DownloadTask>().toMutableSet()
+        private val downloadingTasks = emptySet<DownloadTask>().toMutableSet()
+        private val maxDownloadTasks = 3
+        private val gson = Gson()
+
+
+        init {
+            switchToInitializing()
+        }
+
+        private fun switchToInitializing() {
+            val paramsConfigFile = File(modelDirFile, ParamsConfigFilename)
+            if (paramsConfigFile.exists()) {
+                loadParamsConfig()
+                switchToIndexing()
+            } else {
+                downloadParamsConfig()
+            }
+        }
+
+        private fun loadParamsConfig() {
+            val paramsConfigFile = File(modelDirFile, ParamsConfigFilename)
+            require(paramsConfigFile.exists())
+            val jsonString = paramsConfigFile.readText()
+            paramsConfig = gson.fromJson(jsonString, ParamsConfig::class.java)
+        }
+
+        private fun downloadParamsConfig() {
+            thread(start = true) {
+                val url = URL("${modelUrl}${ModelUrlSuffix}${ParamsConfigFilename}")
+                val tempId = UUID.randomUUID().toString()
+                val tempFile = File(modelDirFile, tempId)
+                url.openStream().use {
+                    Channels.newChannel(it).use { src ->
+                        FileOutputStream(tempFile).use { fileOutputStream ->
+                            fileOutputStream.channel.transferFrom(src, 0, Long.MAX_VALUE)
+                        }
+                    }
+                }
+                require(tempFile.exists())
+                val paramsConfigFile = File(modelDirFile, ParamsConfigFilename)
+                tempFile.renameTo(paramsConfigFile)
+                require(paramsConfigFile.exists())
+                viewModelScope.launch {
+                    loadParamsConfig()
+                    switchToIndexing()
+                }
+            }
+        }
+
+        fun handleStart() {
+            switchToDownloading()
+        }
+
+        fun handlePause() {
+            switchToPausing()
+        }
+
+        fun handleClear() {
+            require(
+                modelInitState.value == ModelInitState.Downloading ||
+                        modelInitState.value == ModelInitState.Paused ||
+                        modelInitState.value == ModelInitState.Finished
+            )
+            switchToClearing()
+        }
+
+        private fun switchToClearing() {
+            if (modelInitState.value == ModelInitState.Paused) {
+                modelInitState.value = ModelInitState.Clearing
+                clear()
+            } else if (modelInitState.value == ModelInitState.Finished) {
+                modelInitState.value = ModelInitState.Clearing
+                if (chatState.modelName.value == modelConfig.modelId) {
+                    chatState.requestTerminateChat { clear() }
+                } else {
+                    clear()
+                }
+            } else {
+                modelInitState.value = ModelInitState.Clearing
+            }
+        }
+
+        fun handleDelete() {
+            require(
+                modelInitState.value == ModelInitState.Downloading ||
+                        modelInitState.value == ModelInitState.Paused ||
+                        modelInitState.value == ModelInitState.Finished
+            )
+            switchToDeleting()
+        }
+
+        private fun switchToDeleting() {
+            if (modelInitState.value == ModelInitState.Paused) {
+                modelInitState.value = ModelInitState.Deleting
+                delete()
+            } else if (modelInitState.value == ModelInitState.Finished) {
+                modelInitState.value = ModelInitState.Deleting
+                if (chatState.modelName.value == modelConfig.modelId) {
+                    chatState.requestTerminateChat { delete() }
+                } else {
+                    delete()
+                }
+            } else {
+                modelInitState.value = ModelInitState.Deleting
+            }
+        }
+
+        private fun switchToIndexing() {
+            modelInitState.value = ModelInitState.Indexing
+            progress.value = 0
+            total.value = modelConfig.tokenizerFiles.size + paramsConfig.paramsRecords.size
+            for (tokenizerFilename in modelConfig.tokenizerFiles) {
+                val file = File(modelDirFile, tokenizerFilename)
+                if (file.exists()) {
+                    ++progress.value
+                } else {
+                    remainingTasks.add(
+                        DownloadTask(
+                            URL("${modelUrl}${ModelUrlSuffix}${tokenizerFilename}"),
+                            file
+                        )
+                    )
+                }
+            }
+            for (paramsRecord in paramsConfig.paramsRecords) {
+                val file = File(modelDirFile, paramsRecord.dataPath)
+                if (file.exists()) {
+                    ++progress.value
+                } else {
+                    remainingTasks.add(
+                        DownloadTask(
+                            URL("${modelUrl}${ModelUrlSuffix}${paramsRecord.dataPath}"),
+                            file
+                        )
+                    )
+                }
+            }
+            if (progress.value < total.value) {
+                switchToPaused()
+            } else {
+                switchToFinished()
+            }
+        }
+
+        private fun switchToDownloading() {
+            modelInitState.value = ModelInitState.Downloading
+            for (downloadTask in remainingTasks) {
+                if (downloadingTasks.size < maxDownloadTasks) {
+                    handleNewDownload(downloadTask)
+                } else {
+                    return
+                }
+            }
+        }
+
+        private fun handleNewDownload(downloadTask: DownloadTask) {
+            require(modelInitState.value == ModelInitState.Downloading)
+            require(!downloadingTasks.contains(downloadTask))
+            downloadingTasks.add(downloadTask)
+            thread(start = true) {
+                val tempId = UUID.randomUUID().toString()
+                val tempFile = File(modelDirFile, tempId)
+                downloadTask.url.openStream().use {
+                    Channels.newChannel(it).use { src ->
+                        FileOutputStream(tempFile).use { fileOutputStream ->
+                            fileOutputStream.channel.transferFrom(src, 0, Long.MAX_VALUE)
+                        }
+                    }
+                }
+                require(tempFile.exists())
+                tempFile.renameTo(downloadTask.file)
+                require(downloadTask.file.exists())
+                viewModelScope.launch {
+                    handleFinishDownload(downloadTask)
+                }
+            }
+        }
+
+        private fun handleNextDownload() {
+            require(modelInitState.value == ModelInitState.Downloading)
+            for (downloadTask in remainingTasks) {
+                if (!downloadingTasks.contains(downloadTask)) {
+                    handleNewDownload(downloadTask)
+                    break
+                }
+            }
+        }
+
+        private fun handleFinishDownload(downloadTask: DownloadTask) {
+            remainingTasks.remove(downloadTask)
+            downloadingTasks.remove(downloadTask)
+            ++progress.value
+            require(
+                modelInitState.value == ModelInitState.Downloading ||
+                        modelInitState.value == ModelInitState.Pausing ||
+                        modelInitState.value == ModelInitState.Clearing ||
+                        modelInitState.value == ModelInitState.Deleting
+            )
+            if (modelInitState.value == ModelInitState.Downloading) {
+                if (remainingTasks.isEmpty()) {
+                    if (downloadingTasks.isEmpty()) {
+                        switchToFinished()
+                    }
+                } else {
+                    handleNextDownload()
+                }
+            } else if (modelInitState.value == ModelInitState.Pausing) {
+                if (downloadingTasks.isEmpty()) {
+                    switchToPaused()
+                }
+            } else if (modelInitState.value == ModelInitState.Clearing) {
+                if (downloadingTasks.isEmpty()) {
+                    clear()
+                }
+            } else if (modelInitState.value == ModelInitState.Deleting) {
+                if (downloadingTasks.isEmpty()) {
+                    delete()
+                }
+            }
+        }
+
+        private fun clear() {
+            val files = modelDirFile.listFiles { dir, name ->
+                !(dir == modelDirFile && name == ModelConfigFilename)
+            }
+            require(files != null)
+            for (file in files) {
+                file.deleteRecursively()
+                require(!file.exists())
+            }
+            val modelConfigFile = File(modelDirFile, ModelConfigFilename)
+            require(modelConfigFile.exists())
+            switchToIndexing()
+        }
+
+        private fun delete() {
+            modelDirFile.deleteRecursively()
+            require(!modelDirFile.exists())
+            requestDeleteModel(modelConfig.modelId)
+        }
+
+        private fun switchToPausing() {
+            modelInitState.value = ModelInitState.Pausing
+        }
+
+        private fun switchToPaused() {
+            modelInitState.value = ModelInitState.Paused
+        }
+
+
+        private fun switchToFinished() {
+            modelInitState.value = ModelInitState.Finished
+        }
+
+        fun startChat() {
+            chatState.requestReloadChat(
+                modelConfig,
+                modelDirFile.absolutePath,
+            )
+        }
+
+    }
+
+    inner class ChatState {
+        val messages = emptyList<MessageData>().toMutableStateList()
+        val report = mutableStateOf("")
+        val modelName = mutableStateOf("")
+        private var modelChatState = mutableStateOf(ModelChatState.Ready)
+            @Synchronized get
+            @Synchronized set
+        private val engine = MLCEngine()
+        private var historyMessages = mutableListOf<ChatCompletionMessage>()
+        private var modelLib = ""
+        private var modelPath = ""
+        private val executorService = Executors.newSingleThreadExecutor()
+        private val viewModelScope = CoroutineScope(Dispatchers.Main + Job())
+        private var imageUri: Uri? = null
+        private fun mainResetChat() {
+            imageUri = null
+            executorService.submit {
+                callBackend { engine.reset() }
+                historyMessages = mutableListOf<ChatCompletionMessage>()
+                viewModelScope.launch {
+                    clearHistory()
+                    switchToReady()
+                }
+            }
+        }
+
+        private fun clearHistory() {
+            messages.clear()
+            report.value = ""
+            historyMessages.clear()
+        }
+
+
+        private fun switchToResetting() {
+            modelChatState.value = ModelChatState.Resetting
+        }
+
+        private fun switchToGenerating() {
+            modelChatState.value = ModelChatState.Generating
+        }
+
+        private fun switchToReloading() {
+            modelChatState.value = ModelChatState.Reloading
+        }
+
+        private fun switchToReady() {
+            modelChatState.value = ModelChatState.Ready
+        }
+
+        private fun switchToFailed() {
+            modelChatState.value = ModelChatState.Falied
+        }
+
+        private fun callBackend(callback: () -> Unit): Boolean {
+            try {
+                callback()
+            } catch (e: Exception) {
+                viewModelScope.launch {
+                    val stackTrace = e.stackTraceToString()
+                    val errorMessage = e.localizedMessage
+                    appendMessage(
+                        MessageRole.Assistant,
+                        "MLCChat failed\n\nStack trace:\n$stackTrace\n\nError message:\n$errorMessage"
+                    )
+                    switchToFailed()
+                }
+                return false
+            }
+            return true
+        }
+
+        fun requestResetChat() {
+            require(interruptable())
+            interruptChat(
+                prologue = {
+                    switchToResetting()
+                },
+                epilogue = {
+                    mainResetChat()
+                }
+            )
+        }
+
+        private fun interruptChat(prologue: () -> Unit, epilogue: () -> Unit) {
+            // prologue runs before interruption
+            // epilogue runs after interruption
+            require(interruptable())
+            if (modelChatState.value == ModelChatState.Ready) {
+                prologue()
+                epilogue()
+            } else if (modelChatState.value == ModelChatState.Generating) {
+                prologue()
+                executorService.submit {
+                    viewModelScope.launch { epilogue() }
+                }
+            } else {
+                require(false)
+            }
+        }
+
+        fun requestTerminateChat(callback: () -> Unit) {
+            require(interruptable())
+            interruptChat(
+                prologue = {
+                    switchToTerminating()
+                },
+                epilogue = {
+                    mainTerminateChat(callback)
+                }
+            )
+        }
+
+        private fun mainTerminateChat(callback: () -> Unit) {
+            executorService.submit {
+                callBackend { engine.unload() }
+                viewModelScope.launch {
+                    clearHistory()
+                    switchToReady()
+                    callback()
+                }
+            }
+        }
+
+        private fun switchToTerminating() {
+            modelChatState.value = ModelChatState.Terminating
+        }
+
+
+        fun requestReloadChat(modelConfig: ModelConfig, modelPath: String) {
+
+            if (this.modelName.value == modelConfig.modelId && this.modelLib == modelConfig.modelLib && this.modelPath == modelPath) {
+                return
+            }
+            require(interruptable())
+            interruptChat(
+                prologue = {
+                    switchToReloading()
+                },
+                epilogue = {
+                    mainReloadChat(modelConfig, modelPath)
+                }
+            )
+        }
+
+        private fun mainReloadChat(modelConfig: ModelConfig, modelPath: String) {
+            clearHistory()
+            this.modelName.value = modelConfig.modelId
+            this.modelLib = modelConfig.modelLib
+            this.modelPath = modelPath
+            executorService.submit {
+                viewModelScope.launch {
+                    Toast.makeText(application, "Initialize...", Toast.LENGTH_SHORT).show()
+                }
+                if (!callBackend {
+                        engine.unload()
+                        engine.reload(modelPath, modelConfig.modelLib)
+                    }) return@submit
+                viewModelScope.launch {
+                    Toast.makeText(application, "Ready to chat", Toast.LENGTH_SHORT).show()
+                    switchToReady()
+                }
+            }
+        }
+
+        fun requestImageBitmap(uri: Uri?) {
+            require(chatable())
+            switchToGenerating()
+            executorService.submit {
+                imageUri = uri
+                viewModelScope.launch {
+                    report.value = "Image process is done, ask any question."
+                    if (modelChatState.value == ModelChatState.Generating) switchToReady()
+                }
+            }
+        }
+
+        fun bitmapToURL(bm: Bitmap): String {
+            val targetSize = 336
+            val scaledBitmap = Bitmap.createScaledBitmap(bm, targetSize, targetSize, true)
+
+            val outputStream = ByteArrayOutputStream()
+            scaledBitmap.compress(Bitmap.CompressFormat.JPEG, 100, outputStream)
+            scaledBitmap.recycle()
+
+            val imageBytes = outputStream.toByteArray()
+            val imageBase64 = Base64.encodeToString(imageBytes, Base64.NO_WRAP)
+            return "data:image/jpg;base64,$imageBase64"
+        }
+
+        fun requestGenerate(prompt: String, activity: Activity) {
+            require(chatable())
+            switchToGenerating()
+            appendMessage(MessageRole.User, prompt)
+            appendMessage(MessageRole.Assistant, "")
+            var content = ChatCompletionMessageContent(text=prompt)
+            if (imageUri != null) {
+                val uri = imageUri
+                val bitmap = uri?.let {
+                    activity.contentResolver.openInputStream(it)?.use { input ->
+                        BitmapFactory.decodeStream(input)
+                    }
+                }
+                val imageBase64URL = bitmapToURL(bitmap!!)
+                Log.v("requestGenerate", "image base64 url: $imageBase64URL")
+                val parts = listOf(
+                    mapOf("type" to "text", "text" to prompt),
+                    mapOf("type" to "image_url", "image_url" to imageBase64URL)
+                )
+                content = ChatCompletionMessageContent(parts=parts)
+                imageUri = null
+            }
+
+            executorService.submit {
+                historyMessages.add(ChatCompletionMessage(
+                    role = OpenAIProtocol.ChatCompletionRole.user,
+                    content = content
+                ))
+
+                viewModelScope.launch {
+                    val responses = engine.chat.completions.create(
+                        messages = historyMessages,
+                        stream_options = OpenAIProtocol.StreamOptions(include_usage = true)
+                    )
+
+                    var finishReasonLength = false
+                    var streamingText = ""
+
+                    for (res in responses) {
+                        if (!callBackend {
+                            for (choice in res.choices) {
+                                choice.delta.content?.let { content ->
+                                    streamingText += content.asText()
+                                }
+                                choice.finish_reason?.let { finishReason ->
+                                    if (finishReason == "length") {
+                                        finishReasonLength = true
+                                    }
+                                }
+                            }
+                            updateMessage(MessageRole.Assistant, streamingText)
+                            res.usage?.let { finalUsage ->
+                                report.value = finalUsage.extra?.asTextLabel() ?: ""
+                            }
+                            if (finishReasonLength) {
+                                streamingText += " [output truncated due to context length limit...]"
+                                updateMessage(MessageRole.Assistant, streamingText)
+                            }
+                        });
+                    }
+                    if (streamingText.isNotEmpty()) {
+                        historyMessages.add(ChatCompletionMessage(
+                            role = OpenAIProtocol.ChatCompletionRole.assistant,
+                            content = streamingText
+                        ))
+                        streamingText = ""
+                    } else {
+                        if (historyMessages.isNotEmpty()) {
+                            historyMessages.removeAt(historyMessages.size - 1)
+                        }
+                    }
+
+                    if (modelChatState.value == ModelChatState.Generating) switchToReady()
+                }
+            }
+        }
+
+        private fun appendMessage(role: MessageRole, text: String) {
+            messages.add(MessageData(role, text))
+        }
+
+
+        private fun updateMessage(role: MessageRole, text: String) {
+            messages[messages.size - 1] = MessageData(role, text)
+        }
+
+        fun chatable(): Boolean {
+            return modelChatState.value == ModelChatState.Ready
+        }
+
+        fun interruptable(): Boolean {
+            return modelChatState.value == ModelChatState.Ready
+                    || modelChatState.value == ModelChatState.Generating
+                    || modelChatState.value == ModelChatState.Falied
+        }
+    }
+}
+
+enum class ModelInitState {
+    Initializing,
+    Indexing,
+    Paused,
+    Downloading,
+    Pausing,
+    Clearing,
+    Deleting,
+    Finished
+}
+
+enum class ModelChatState {
+    Generating,
+    Resetting,
+    Reloading,
+    Terminating,
+    Ready,
+    Falied
+}
+
+enum class MessageRole {
+    Assistant,
+    User
+}
+
+data class DownloadTask(val url: URL, val file: File)
+
+data class MessageData(val role: MessageRole, val text: String, val id: UUID = UUID.randomUUID(), var imageUri: Uri? = null)
+
+data class AppConfig(
+    @SerializedName("model_libs") var modelLibs: MutableList<String>,
+    @SerializedName("model_list") val modelList: MutableList<ModelRecord>,
+)
+
+data class ModelRecord(
+    @SerializedName("model_url") val modelUrl: String,
+    @SerializedName("model_id") val modelId: String,
+    @SerializedName("estimated_vram_bytes") val estimatedVramBytes: Long?,
+    @SerializedName("model_lib") val modelLib: String
+)
+
+data class ModelConfig(
+    @SerializedName("model_lib") var modelLib: String,
+    @SerializedName("model_id") var modelId: String,
+    @SerializedName("estimated_vram_bytes") var estimatedVramBytes: Long?,
+    @SerializedName("tokenizer_files") val tokenizerFiles: List<String>,
+    @SerializedName("context_window_size") val contextWindowSize: Int,
+    @SerializedName("prefill_chunk_size") val prefillChunkSize: Int,
+)
+
+data class ParamsRecord(
+    @SerializedName("dataPath") val dataPath: String
+)
+
+data class ParamsConfig(
+    @SerializedName("records") val paramsRecords: List<ParamsRecord>
+)

android/MLCChat/app/src/main/java/ai/mlc/mlcchat/ChatView.kt

@@ -0,0 +1,343 @@
+package ai.mlc.mlcchat
+
+import android.app.Activity
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import androidx.compose.foundation.Image
+import androidx.compose.foundation.background
+import androidx.compose.foundation.gestures.detectTapGestures
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.IntrinsicSize
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.aspectRatio
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.height
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.widthIn
+import androidx.compose.foundation.layout.wrapContentHeight
+import androidx.compose.foundation.layout.wrapContentWidth
+import androidx.compose.foundation.lazy.LazyColumn
+import androidx.compose.foundation.lazy.items
+import androidx.compose.foundation.lazy.rememberLazyListState
+import androidx.compose.foundation.shape.RoundedCornerShape
+import androidx.compose.foundation.text.selection.SelectionContainer
+import androidx.compose.material.icons.Icons
+import androidx.compose.material.icons.filled.AddAPhoto
+import androidx.compose.material.icons.filled.ArrowBack
+import androidx.compose.material.icons.filled.Photo
+import androidx.compose.material.icons.filled.Replay
+import androidx.compose.material.icons.filled.Send
+import androidx.compose.material3.Divider
+import androidx.compose.material3.ExperimentalMaterial3Api
+import androidx.compose.material3.Icon
+import androidx.compose.material3.IconButton
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.OutlinedTextField
+import androidx.compose.material3.Scaffold
+import androidx.compose.material3.Switch
+import androidx.compose.material3.Text
+import androidx.compose.material3.TopAppBar
+import androidx.compose.material3.TopAppBarDefaults
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.rememberCoroutineScope
+import androidx.compose.runtime.saveable.rememberSaveable
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.graphics.asImageBitmap
+import androidx.compose.ui.input.pointer.pointerInput
+import androidx.compose.ui.platform.LocalFocusManager
+import androidx.compose.ui.text.style.TextAlign
+import androidx.compose.ui.tooling.preview.Preview
+import androidx.compose.ui.unit.dp
+import androidx.navigation.NavController
+import dev.jeziellago.compose.markdowntext.MarkdownText
+import kotlinx.coroutines.launch
+
+@ExperimentalMaterial3Api
+@Composable
+fun ChatView(
+    navController: NavController, chatState: AppViewModel.ChatState, activity: Activity
+) {
+    val localFocusManager = LocalFocusManager.current
+    (activity as MainActivity).chatState = chatState
+    Scaffold(topBar = {
+        TopAppBar(
+            title = {
+                Text(
+                    text = "MLCChat: " + chatState.modelName.value.split("-")[0],
+                    color = MaterialTheme.colorScheme.onPrimary
+                )
+            },
+            colors = TopAppBarDefaults.topAppBarColors(containerColor = MaterialTheme.colorScheme.primary),
+            navigationIcon = {
+                IconButton(
+                    onClick = { navController.popBackStack() },
+                    enabled = chatState.interruptable()
+                ) {
+                    Icon(
+                        imageVector = Icons.Filled.ArrowBack,
+                        contentDescription = "back home page",
+                        tint = MaterialTheme.colorScheme.onPrimary
+                    )
+                }
+            },
+            actions = {
+                IconButton(
+                    onClick = {
+                        chatState.requestResetChat()
+                        activity.hasImage = false },
+                    enabled = chatState.interruptable()
+                ) {
+                    Icon(
+                        imageVector = Icons.Filled.Replay,
+                        contentDescription = "reset the chat",
+                        tint = MaterialTheme.colorScheme.onPrimary
+                    )
+                }
+            })
+    }, modifier = Modifier.pointerInput(Unit) {
+        detectTapGestures(onTap = {
+            localFocusManager.clearFocus()
+        })
+    }) { paddingValues ->
+        Column(
+            modifier = Modifier
+                .fillMaxSize()
+                .padding(paddingValues)
+                .padding(horizontal = 10.dp)
+        ) {
+            val lazyColumnListState = rememberLazyListState()
+            val coroutineScope = rememberCoroutineScope()
+            Text(
+                text = chatState.report.value,
+                textAlign = TextAlign.Center,
+                modifier = Modifier
+                    .fillMaxWidth()
+                    .wrapContentHeight()
+                    .padding(top = 5.dp)
+            )
+            Divider(thickness = 1.dp, modifier = Modifier.padding(vertical = 5.dp))
+            LazyColumn(
+                modifier = Modifier.weight(9f),
+                verticalArrangement = Arrangement.spacedBy(5.dp, alignment = Alignment.Bottom),
+                state = lazyColumnListState
+            ) {
+                coroutineScope.launch {
+                    lazyColumnListState.animateScrollToItem(chatState.messages.size)
+                }
+                items(
+                    items = chatState.messages,
+                    key = { message -> message.id },
+                ) { message ->
+                    MessageView(messageData = message, activity)
+                }
+                item {
+                    // place holder item for scrolling to the bottom
+                }
+            }
+            Divider(thickness = 1.dp, modifier = Modifier.padding(top = 5.dp))
+            SendMessageView(chatState = chatState, activity)
+        }
+    }
+}
+
+@Composable
+fun MessageView(messageData: MessageData, activity: Activity?) {
+    // default render the Assistant text as MarkdownText
+    var useMarkdown by remember { mutableStateOf(true) }
+    var localActivity : MainActivity = activity as MainActivity
+    SelectionContainer {
+        if (messageData.role == MessageRole.Assistant) {
+            Column {
+                if (messageData.text.isNotEmpty()) {
+                    Row(
+                        verticalAlignment = Alignment.CenterVertically,
+                    ) {
+                        Text(
+                            text = "Show as Markdown",
+                            color = MaterialTheme.colorScheme.onSecondaryContainer,
+                            modifier = Modifier
+                                .wrapContentWidth()
+                                .padding(end = 8.dp)
+                                .widthIn(max = 300.dp)
+                        )
+                        Switch(
+                            checked = useMarkdown,
+                            onCheckedChange = { useMarkdown = it }
+                        )
+                    }
+                }
+                Row(
+                    horizontalArrangement = Arrangement.Start,
+                    modifier = Modifier.fillMaxWidth()
+                ) {
+                    if (useMarkdown) {
+                        MarkdownText(
+                            isTextSelectable = true,
+                            modifier = Modifier
+                                .wrapContentWidth()
+                                .background(
+                                    color = MaterialTheme.colorScheme.secondaryContainer,
+                                    shape = RoundedCornerShape(5.dp)
+                                )
+                                .padding(5.dp)
+                                .widthIn(max = 300.dp),
+                            markdown = messageData.text,
+                        )
+                    } else {
+                        Text(
+                            text = messageData.text,
+                            textAlign = TextAlign.Left,
+                            color = MaterialTheme.colorScheme.onSecondaryContainer,
+                            modifier = Modifier
+                                .wrapContentWidth()
+                                .background(
+                                    color = MaterialTheme.colorScheme.secondaryContainer,
+                                    shape = RoundedCornerShape(5.dp)
+                                )
+                                .padding(5.dp)
+                                .widthIn(max = 300.dp)
+                        )
+                    }
+                }
+            }
+        } else {
+            Row(
+                horizontalArrangement = Arrangement.End,
+                modifier = Modifier.fillMaxWidth()
+            ) {
+                if (messageData.imageUri != null) {
+                    val uri = messageData.imageUri
+                    val bitmap = uri?.let {
+                        activity.contentResolver.openInputStream(it)?.use { input ->
+                            BitmapFactory.decodeStream(input)
+                        }
+                    }
+                    val displayBitmap = bitmap?.let { Bitmap.createScaledBitmap(it, 224, 224, true) }
+                    if (displayBitmap != null) {
+                        Image(
+                            displayBitmap.asImageBitmap(),
+                            "",
+                            modifier = Modifier
+                                .wrapContentWidth()
+                                .background(
+                                    color = MaterialTheme.colorScheme.secondaryContainer,
+                                    shape = RoundedCornerShape(5.dp)
+                                )
+                                .padding(5.dp)
+                                .widthIn(max = 300.dp)
+                        )
+                    }
+                    if (!localActivity.hasImage) {
+                        localActivity.chatState.requestImageBitmap(messageData.imageUri)
+                    }
+                    localActivity.hasImage = true
+                } else {
+                    Text(
+                        text = messageData.text,
+                        textAlign = TextAlign.Right,
+                        color = MaterialTheme.colorScheme.onPrimaryContainer,
+                        modifier = Modifier
+                            .wrapContentWidth()
+                            .background(
+                                color = MaterialTheme.colorScheme.primaryContainer,
+                                shape = RoundedCornerShape(5.dp)
+                            )
+                            .padding(5.dp)
+                            .widthIn(max = 300.dp)
+                    )
+                }
+
+            }
+        }
+    }
+}
+
+@ExperimentalMaterial3Api
+@Composable
+fun SendMessageView(chatState: AppViewModel.ChatState, activity: Activity) {
+    val localFocusManager = LocalFocusManager.current
+    val localActivity : MainActivity = activity as MainActivity
+    Row(
+        horizontalArrangement = Arrangement.spacedBy(5.dp),
+        verticalAlignment = Alignment.CenterVertically,
+        modifier = Modifier
+            .height(IntrinsicSize.Max)
+            .fillMaxWidth()
+            .padding(bottom = 5.dp)
+    ) {
+        var text by rememberSaveable { mutableStateOf("") }
+        OutlinedTextField(
+            value = text,
+            onValueChange = { text = it },
+            label = { Text(text = "Input") },
+            modifier = Modifier
+                .weight(9f),
+        )
+        IconButton(
+            onClick = {
+                activity.takePhoto()
+            },
+            modifier = Modifier
+                .aspectRatio(1f)
+                .weight(1f),
+            enabled = (chatState.chatable() && !localActivity.hasImage)
+        ) {
+            Icon(
+                imageVector = Icons.Filled.AddAPhoto,
+                contentDescription = "use camera",
+            )
+        }
+        IconButton(
+            onClick = {
+                activity.pickImageFromGallery()
+            },
+            modifier = Modifier
+                .aspectRatio(1f)
+                .weight(1f),
+            enabled = (chatState.chatable() && !localActivity.hasImage)
+        ) {
+            Icon(
+                imageVector = Icons.Filled.Photo,
+                contentDescription = "select image",
+            )
+        }
+        IconButton(
+            onClick = {
+                localFocusManager.clearFocus()
+                chatState.requestGenerate(text, activity)
+                text = ""
+            },
+            modifier = Modifier
+                .aspectRatio(1f)
+                .weight(1f),
+            enabled = (text != "" && chatState.chatable())
+        ) {
+            Icon(
+                imageVector = Icons.Filled.Send,
+                contentDescription = "send message",
+            )
+        }
+    }
+}
+
+@Preview
+@Composable
+fun MessageViewPreviewWithMarkdown() {
+    MessageView(
+        messageData = MessageData(
+            role = MessageRole.Assistant, text = """
+# Sample  Header
+* Markdown
+* [Link](https://example.com)
+<a href="https://www.google.com/">Google</a>
+"""
+        ), null
+    )
+}

android/MLCChat/app/src/main/java/ai/mlc/mlcchat/MainActivity.kt

@@ -0,0 +1,157 @@
+package ai.mlc.mlcchat
+
+import android.Manifest
+import android.content.ContentValues
+import android.content.pm.PackageManager
+import android.net.Uri
+import android.os.Build
+import android.os.Bundle
+import android.provider.MediaStore
+import android.util.Log
+import androidx.activity.ComponentActivity
+import androidx.activity.compose.setContent
+import androidx.activity.result.contract.ActivityResultContracts
+import androidx.annotation.RequiresApi
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.material3.ExperimentalMaterial3Api
+import androidx.compose.material3.Surface
+import androidx.compose.ui.Modifier
+import androidx.core.content.ContextCompat
+import ai.mlc.mlcchat.ui.theme.MLCChatTheme
+import java.text.SimpleDateFormat
+import java.util.Date
+import java.util.Locale
+import java.util.UUID
+
+class MainActivity : ComponentActivity() {
+    var hasImage = false
+
+    private val pickImageLauncher = registerForActivityResult(
+        ActivityResultContracts.GetContent()
+    ) { uri: Uri? ->
+        uri?.let {
+            Log.v("pickImageLauncher", "Selected image uri: $it")
+            chatState.messages.add(
+                MessageData(
+                    role = MessageRole.User,
+                    text = "",
+                    id = UUID.randomUUID(),
+                    imageUri = it
+                )
+            )
+        }
+    }
+
+    private var cameraImageUri: Uri? = null
+    private val takePictureLauncher = registerForActivityResult(
+        ActivityResultContracts.TakePicture()
+    ) { success: Boolean ->
+        if (success && cameraImageUri != null) {
+            Log.v("takePictureLauncher", "Camera image uri: $cameraImageUri")
+            chatState.messages.add(
+                MessageData(
+                    role = MessageRole.User,
+                    text = "",
+                    id = UUID.randomUUID(),
+                    imageUri = cameraImageUri
+                )
+            )
+        }
+    }
+
+    private val requestPermissionLauncher =
+        registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { permissions ->
+            permissions.entries.forEach {
+                Log.d("Permissions", "${it.key} = ${it.value}")
+            }
+        }
+
+    lateinit var chatState: AppViewModel.ChatState
+
+    @RequiresApi(Build.VERSION_CODES.TIRAMISU)
+    @ExperimentalMaterial3Api
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+
+        chatState = AppViewModel(this.application).ChatState()
+        requestNeededPermissions()
+
+        setContent {
+            Surface(
+                modifier = Modifier.fillMaxSize()
+            ) {
+                MLCChatTheme {
+                    NavView(this)
+                }
+            }
+        }
+    }
+
+    private fun requestNeededPermissions() {
+        val permissionsToRequest = mutableListOf<String>()
+
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+            if (ContextCompat.checkSelfPermission(
+                    this,
+                    Manifest.permission.READ_MEDIA_IMAGES
+                ) != PackageManager.PERMISSION_GRANTED
+            ) {
+                permissionsToRequest.add(Manifest.permission.READ_MEDIA_IMAGES)
+            }
+            if (ContextCompat.checkSelfPermission(
+                    this,
+                    Manifest.permission.CAMERA
+                ) != PackageManager.PERMISSION_GRANTED
+            ) {
+                permissionsToRequest.add(Manifest.permission.CAMERA)
+            }
+        } else {
+            if (ContextCompat.checkSelfPermission(
+                    this,
+                    Manifest.permission.READ_EXTERNAL_STORAGE
+                ) != PackageManager.PERMISSION_GRANTED
+            ) {
+                permissionsToRequest.add(Manifest.permission.READ_EXTERNAL_STORAGE)
+            }
+            if (ContextCompat.checkSelfPermission(
+                    this,
+                    Manifest.permission.WRITE_EXTERNAL_STORAGE
+                ) != PackageManager.PERMISSION_GRANTED
+            ) {
+                permissionsToRequest.add(Manifest.permission.WRITE_EXTERNAL_STORAGE)
+            }
+            if (ContextCompat.checkSelfPermission(
+                    this,
+                    Manifest.permission.CAMERA
+                ) != PackageManager.PERMISSION_GRANTED
+            ) {
+                permissionsToRequest.add(Manifest.permission.CAMERA)
+            }
+        }
+
+        if (permissionsToRequest.isNotEmpty()) {
+            requestPermissionLauncher.launch(permissionsToRequest.toTypedArray())
+        }
+    }
+
+    fun pickImageFromGallery() {
+        pickImageLauncher.launch("image/*")
+    }
+
+    fun takePhoto() {
+        val contentValues = ContentValues().apply {
+            val timeFormatter = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.getDefault())
+            val fileName = "IMG_${timeFormatter.format(Date())}.jpg"
+            put(MediaStore.Images.Media.DISPLAY_NAME, fileName)
+            put(MediaStore.Images.Media.MIME_TYPE, "image/jpeg")
+            put(MediaStore.Images.Media.DATE_ADDED, System.currentTimeMillis() / 1000)
+        }
+
+        cameraImageUri = contentResolver.insert(
+            MediaStore.Images.Media.EXTERNAL_CONTENT_URI,
+            contentValues
+        )
+
+        takePictureLauncher.launch(cameraImageUri)
+    }
+}

android/MLCChat/app/src/main/java/ai/mlc/mlcchat/NavView.kt

@@ -0,0 +1,19 @@
+package ai.mlc.mlcchat
+
+import android.app.Activity
+import androidx.compose.material3.ExperimentalMaterial3Api
+import androidx.compose.runtime.Composable
+import androidx.lifecycle.viewmodel.compose.viewModel
+import androidx.navigation.compose.NavHost
+import androidx.navigation.compose.composable
+import androidx.navigation.compose.rememberNavController
+
+@ExperimentalMaterial3Api
+@Composable
+fun NavView(activity: Activity, appViewModel: AppViewModel = viewModel()) {
+    val navController = rememberNavController()
+    NavHost(navController = navController, startDestination = "home") {
+        composable("home") { StartView(navController, appViewModel) }
+        composable("chat") { ChatView(navController, appViewModel.chatState, activity) }
+    }
+}

android/MLCChat/app/src/main/java/ai/mlc/mlcchat/StartView.kt

@@ -0,0 +1,250 @@
+package ai.mlc.mlcchat
+
+import androidx.compose.foundation.gestures.detectTapGestures
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.aspectRatio
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.height
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.width
+import androidx.compose.foundation.layout.wrapContentHeight
+import androidx.compose.foundation.lazy.LazyColumn
+import androidx.compose.foundation.lazy.items
+import androidx.compose.foundation.text.selection.SelectionContainer
+import androidx.compose.material.icons.Icons
+import androidx.compose.material.icons.outlined.Chat
+import androidx.compose.material.icons.outlined.Delete
+import androidx.compose.material.icons.outlined.Download
+import androidx.compose.material.icons.outlined.Pause
+import androidx.compose.material.icons.outlined.Schedule
+import androidx.compose.material3.AlertDialog
+import androidx.compose.material3.Divider
+import androidx.compose.material3.ExperimentalMaterial3Api
+import androidx.compose.material3.Icon
+import androidx.compose.material3.IconButton
+import androidx.compose.material3.LinearProgressIndicator
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.OutlinedTextField
+import androidx.compose.material3.Scaffold
+import androidx.compose.material3.Text
+import androidx.compose.material3.TextButton
+import androidx.compose.material3.TopAppBar
+import androidx.compose.material3.TopAppBarDefaults
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.saveable.rememberSaveable
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.input.pointer.pointerInput
+import androidx.compose.ui.platform.LocalFocusManager
+import androidx.compose.ui.text.style.TextAlign
+import androidx.compose.ui.unit.dp
+import androidx.navigation.NavController
+
+
+@ExperimentalMaterial3Api
+@Composable
+fun StartView(
+    navController: NavController,
+    appViewModel: AppViewModel
+) {
+    val localFocusManager = LocalFocusManager.current
+    Scaffold(
+        topBar = {
+            TopAppBar(
+                title = { Text(text = "MLCChat", color = MaterialTheme.colorScheme.onPrimary) },
+                colors = TopAppBarDefaults.topAppBarColors(containerColor = MaterialTheme.colorScheme.primary)
+            )
+        },
+        modifier = Modifier.pointerInput(Unit) {
+            detectTapGestures(onTap = {
+                localFocusManager.clearFocus()
+            })
+        }
+    )
+    { paddingValues ->
+        Column(
+            modifier = Modifier
+                .fillMaxSize()
+                .padding(paddingValues)
+                .padding(horizontal = 10.dp)
+        ) {
+            Text(text = "Model List", modifier = Modifier.padding(top = 10.dp))
+            LazyColumn() {
+                items(items = appViewModel.modelList,
+                    key = { modelState -> modelState.id }
+                ) { modelState ->
+                    ModelView(
+                        navController = navController,
+                        modelState = modelState,
+                        appViewModel = appViewModel
+                    )
+                }
+            }
+        }
+        if (appViewModel.isShowingAlert()) {
+            AlertDialog(
+                onDismissRequest = { appViewModel.dismissAlert() },
+                onConfirmation = { appViewModel.copyError() },
+                error = appViewModel.errorMessage()
+            )
+        }
+    }
+}
+
+@ExperimentalMaterial3Api
+@Composable
+fun AlertDialog(
+    onDismissRequest: () -> Unit,
+    onConfirmation: () -> Unit,
+    error: String,
+) {
+    AlertDialog(
+        title = { Text(text = "Error") },
+        text = { Text(text = error) },
+        onDismissRequest = { onDismissRequest() },
+        confirmButton = {
+            TextButton(onClick = { onConfirmation() }) { Text("Copy") }
+        },
+        dismissButton = {
+            TextButton(onClick = { onDismissRequest() }) { Text("Dismiss") }
+        }
+    )
+}
+
+@Composable
+fun ModelView(
+    navController: NavController,
+    modelState: AppViewModel.ModelState,
+    appViewModel: AppViewModel
+) {
+    var isDeletingModel by rememberSaveable { mutableStateOf(false) }
+    Column(
+        verticalArrangement = Arrangement.SpaceBetween,
+        modifier = Modifier
+            .wrapContentHeight()
+    ) {
+        Row(
+            horizontalArrangement = Arrangement.spacedBy(5.dp),
+            verticalAlignment = Alignment.CenterVertically,
+            modifier = Modifier
+                .fillMaxWidth()
+                .wrapContentHeight()
+        ) {
+            Text(
+                text = modelState.modelConfig.modelId,
+                textAlign = TextAlign.Left,
+                modifier = Modifier
+                    .wrapContentHeight()
+                    .weight(8f)
+            )
+            Divider(
+                modifier = Modifier
+                    .height(20.dp)
+                    .width(1.dp)
+            )
+            if (modelState.modelInitState.value == ModelInitState.Paused) {
+                IconButton(
+                    onClick = { modelState.handleStart() }, modifier = Modifier
+                        .aspectRatio(1f)
+                        .weight(1f)
+                ) {
+                    Icon(
+                        imageVector = Icons.Outlined.Download,
+                        contentDescription = "start downloading",
+                    )
+                }
+
+            } else if (modelState.modelInitState.value == ModelInitState.Downloading) {
+                IconButton(
+                    onClick = { modelState.handlePause() }, modifier = Modifier
+                        .aspectRatio(1f)
+                        .weight(1f)
+                ) {
+                    Icon(
+                        imageVector = Icons.Outlined.Pause,
+                        contentDescription = "pause downloading",
+                    )
+                }
+            } else if (modelState.modelInitState.value == ModelInitState.Finished) {
+                IconButton(
+                    onClick = {
+                        modelState.startChat()
+                        navController.navigate("chat")
+                    },
+                    enabled = appViewModel.chatState.interruptable(),
+                    modifier = Modifier
+                        .aspectRatio(1f)
+                        .weight(1f)
+                ) {
+                    Icon(
+                        imageVector = Icons.Outlined.Chat,
+                        contentDescription = "start chatting",
+                    )
+                }
+            } else {
+                IconButton(
+                    enabled = false, onClick = {}, modifier = Modifier
+                        .aspectRatio(1f)
+                        .weight(1f)
+                ) {
+                    Icon(
+                        imageVector = Icons.Outlined.Schedule,
+                        contentDescription = "pending",
+                    )
+                }
+            }
+            if (modelState.modelInitState.value == ModelInitState.Downloading ||
+                modelState.modelInitState.value == ModelInitState.Paused ||
+                modelState.modelInitState.value == ModelInitState.Finished
+            ) {
+                IconButton(
+                    onClick = { isDeletingModel = true },
+                    modifier = Modifier
+                        .aspectRatio(1f)
+                        .weight(1f)
+                ) {
+                    Icon(
+                        imageVector = Icons.Outlined.Delete,
+                        contentDescription = "start downloading",
+                        tint = MaterialTheme.colorScheme.error
+                    )
+                }
+            }
+        }
+        LinearProgressIndicator(
+            progress = modelState.progress.value.toFloat() / modelState.total.value,
+            modifier = Modifier.fillMaxWidth()
+        )
+        if (isDeletingModel) {
+            Row(
+                horizontalArrangement = Arrangement.End,
+                verticalAlignment = Alignment.CenterVertically,
+                modifier = Modifier
+                    .fillMaxWidth()
+                    .wrapContentHeight()
+            ) {
+                TextButton(onClick = { isDeletingModel = false }) {
+                    Text(text = "cancel")
+                }
+                TextButton(onClick = {
+                    isDeletingModel = false
+                    modelState.handleClear()
+                }) {
+                    Text(text = "clear data", color = MaterialTheme.colorScheme.error)
+                }
+                TextButton(onClick = {
+                    isDeletingModel = false
+                    modelState.handleDelete()
+                }) {
+                    Text(text = "delete model", color = MaterialTheme.colorScheme.error)
+                }
+            }
+        }
+    }
+}

android/MLCChat/app/src/main/java/ai/mlc/mlcchat/ui/theme/Color.kt

@@ -0,0 +1,44 @@
+package ai.mlc.mlcchat.ui.theme
+
+import androidx.compose.ui.graphics.Color
+
+val Blue10 = Color(0xFF000F5E)
+val Blue20 = Color(0xFF001E92)
+val Blue30 = Color(0xFF002ECC)
+val Blue40 = Color(0xFF1546F6)
+val Blue80 = Color(0xFFB8C3FF)
+val Blue90 = Color(0xFFDDE1FF)
+
+val DarkBlue10 = Color(0xFF00036B)
+val DarkBlue20 = Color(0xFF000BA6)
+val DarkBlue30 = Color(0xFF1026D3)
+val DarkBlue40 = Color(0xFF3648EA)
+val DarkBlue80 = Color(0xFFBBC2FF)
+val DarkBlue90 = Color(0xFFDEE0FF)
+
+val Yellow10 = Color(0xFF261900)
+val Yellow20 = Color(0xFF402D00)
+val Yellow30 = Color(0xFF5C4200)
+val Yellow40 = Color(0xFF7A5900)
+val Yellow80 = Color(0xFFFABD1B)
+val Yellow90 = Color(0xFFFFDE9C)
+
+val Red10 = Color(0xFF410001)
+val Red20 = Color(0xFF680003)
+val Red30 = Color(0xFF930006)
+val Red40 = Color(0xFFBA1B1B)
+val Red80 = Color(0xFFFFB4A9)
+val Red90 = Color(0xFFFFDAD4)
+
+val Grey10 = Color(0xFF191C1D)
+val Grey20 = Color(0xFF2D3132)
+val Grey80 = Color(0xFFC4C7C7)
+val Grey90 = Color(0xFFE0E3E3)
+val Grey95 = Color(0xFFEFF1F1)
+val Grey99 = Color(0xFFFBFDFD)
+
+val BlueGrey30 = Color(0xFF45464F)
+val BlueGrey50 = Color(0xFF767680)
+val BlueGrey60 = Color(0xFF90909A)
+val BlueGrey80 = Color(0xFFC6C5D0)
+val BlueGrey90 = Color(0xFFE2E1EC)

android/MLCChat/app/src/main/java/ai/mlc/mlcchat/ui/theme/Theme.kt

@@ -0,0 +1,107 @@
+package ai.mlc.mlcchat.ui.theme
+
+import android.app.Activity
+import android.os.Build
+import androidx.compose.foundation.isSystemInDarkTheme
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.darkColorScheme
+import androidx.compose.material3.dynamicDarkColorScheme
+import androidx.compose.material3.dynamicLightColorScheme
+import androidx.compose.material3.lightColorScheme
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.SideEffect
+import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.toArgb
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalView
+import androidx.core.view.WindowCompat
+
+private val DarkColorScheme = darkColorScheme(
+    primary = Blue80,
+    onPrimary = Blue20,
+    primaryContainer = Blue30,
+    onPrimaryContainer = Blue90,
+    inversePrimary = Blue40,
+    secondary = DarkBlue80,
+    onSecondary = DarkBlue20,
+    secondaryContainer = DarkBlue30,
+    onSecondaryContainer = DarkBlue90,
+    tertiary = Yellow80,
+    onTertiary = Yellow20,
+    tertiaryContainer = Yellow30,
+    onTertiaryContainer = Yellow90,
+    error = Red80,
+    onError = Red20,
+    errorContainer = Red30,
+    onErrorContainer = Red90,
+    background = Grey10,
+    onBackground = Grey90,
+    surface = Grey10,
+    onSurface = Grey80,
+    inverseSurface = Grey90,
+    inverseOnSurface = Grey20,
+    surfaceVariant = BlueGrey30,
+    onSurfaceVariant = BlueGrey80,
+    outline = BlueGrey60
+)
+
+private val LightColorScheme = lightColorScheme(
+    primary = Blue40,
+    onPrimary = Color.White,
+    primaryContainer = Blue90,
+    onPrimaryContainer = Blue10,
+    inversePrimary = Blue80,
+    secondary = DarkBlue40,
+    onSecondary = Color.White,
+    secondaryContainer = DarkBlue90,
+    onSecondaryContainer = DarkBlue10,
+    tertiary = Yellow40,
+    onTertiary = Color.White,
+    tertiaryContainer = Yellow90,
+    onTertiaryContainer = Yellow10,
+    error = Red40,
+    onError = Color.White,
+    errorContainer = Red90,
+    onErrorContainer = Red10,
+    background = Grey99,
+    onBackground = Grey10,
+    surface = Grey99,
+    onSurface = Grey10,
+    inverseSurface = Grey20,
+    inverseOnSurface = Grey95,
+    surfaceVariant = BlueGrey90,
+    onSurfaceVariant = BlueGrey30,
+    outline = BlueGrey50
+)
+
+@Composable
+fun MLCChatTheme(
+    darkTheme: Boolean = isSystemInDarkTheme(),
+    // Dynamic color is available on Android 12+
+    dynamicColor: Boolean = true,
+    content: @Composable () -> Unit
+) {
+    val colorScheme = when {
+        dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> {
+            val context = LocalContext.current
+            if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
+        }
+
+        darkTheme -> DarkColorScheme
+        else -> LightColorScheme
+    }
+    val view = LocalView.current
+    if (!view.isInEditMode) {
+        SideEffect {
+            val window = (view.context as Activity).window
+            window.statusBarColor = colorScheme.primary.toArgb()
+            WindowCompat.getInsetsController(window, view).isAppearanceLightStatusBars = darkTheme
+        }
+    }
+
+    MaterialTheme(
+        colorScheme = colorScheme,
+        typography = Typography,
+        content = content
+    )
+}

android/MLCChat/app/src/main/java/ai/mlc/mlcchat/ui/theme/Type.kt

@@ -0,0 +1,34 @@
+package ai.mlc.mlcchat.ui.theme
+
+import androidx.compose.material3.Typography
+import androidx.compose.ui.text.TextStyle
+import androidx.compose.ui.text.font.FontFamily
+import androidx.compose.ui.text.font.FontWeight
+import androidx.compose.ui.unit.sp
+
+// Set of Material typography styles to start with
+val Typography = Typography(
+    bodyLarge = TextStyle(
+        fontFamily = FontFamily.Default,
+        fontWeight = FontWeight.Normal,
+        fontSize = 16.sp,
+        lineHeight = 24.sp,
+        letterSpacing = 0.5.sp
+    )
+    /* Other default text styles to override
+    titleLarge = TextStyle(
+        fontFamily = FontFamily.Default,
+        fontWeight = FontWeight.Normal,
+        fontSize = 22.sp,
+        lineHeight = 28.sp,
+        letterSpacing = 0.sp
+    ),
+    labelSmall = TextStyle(
+        fontFamily = FontFamily.Default,
+        fontWeight = FontWeight.Medium,
+        fontSize = 11.sp,
+        lineHeight = 16.sp,
+        letterSpacing = 0.5.sp
+    )
+    */
+)

android/MLCChat/app/src/main/res/drawable/ic_android_black_24dp.xml

@@ -0,0 +1,5 @@
+<vector android:height="24dp" android:tint="#000000"
+    android:viewportHeight="24" android:viewportWidth="24"
+    android:width="24dp" xmlns:android="http://schemas.android.com/apk/res/android">
+    <path android:fillColor="#FF000000" android:pathData="M17.6,11.48 L19.44,8.3a0.63,0.63 0,0 0,-1.09 -0.63l-1.88,3.24a11.43,11.43 0,0 0,-8.94 0L5.65,7.67a0.63,0.63 0,0 0,-1.09 0.63L6.4,11.48A10.81,10.81 0,0 0,1 20L23,20A10.81,10.81 0,0 0,17.6 11.48ZM7,17.25A1.25,1.25 0,1 1,8.25 16,1.25 1.25,0 0,1 7,17.25ZM17,17.25A1.25,1.25 0,1 1,18.25 16,1.25 1.25,0 0,1 17,17.25Z"/>
+</vector>

android/MLCChat/app/src/main/res/values/colors.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="purple_200">#FFBB86FC</color>
+    <color name="purple_500">#FF6200EE</color>
+    <color name="purple_700">#FF3700B3</color>
+    <color name="teal_200">#FF03DAC5</color>
+    <color name="teal_700">#FF018786</color>
+    <color name="black">#FF000000</color>
+    <color name="white">#FFFFFFFF</color>
+</resources>

android/MLCChat/app/src/main/res/values/strings.xml

@@ -0,0 +1,3 @@
+<resources>
+    <string name="app_name">MLCChat</string>
+</resources>

android/MLCChat/app/src/main/res/values/themes.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+
+    <style name="Theme.MLCChat" parent="android:Theme.Material.Light" />
+
+</resources>

android/MLCChat/app/src/main/res/xml/backup_rules.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+   Sample backup rules file; uncomment and customize as necessary.
+   See https://developer.android.com/guide/topics/data/autobackup
+   for details.
+   Note: This file is ignored for devices older that API 31
+   See https://developer.android.com/about/versions/12/backup-restore
+-->
+<full-backup-content>
+    <!--
+   <include domain="sharedpref" path="."/>
+   <exclude domain="sharedpref" path="device.xml"/>
+-->
+</full-backup-content>

android/MLCChat/app/src/main/res/xml/data_extraction_rules.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+   Sample data extraction rules file; uncomment and customize as necessary.
+   See https://developer.android.com/about/versions/12/backup-restore#xml-changes
+   for details.
+-->
+<data-extraction-rules>
+    <cloud-backup>
+        <!-- TODO: Use <include> and <exclude> to control what is backed up.
+        <include .../>
+        <exclude .../>
+        -->
+    </cloud-backup>
+    <!--
+    <device-transfer>
+        <include .../>
+        <exclude .../>
+    </device-transfer>
+    -->
+</data-extraction-rules>

android/MLCChat/build.gradle

@@ -0,0 +1,5 @@
+plugins {
+    id 'com.android.application' version '8.2.0' apply false
+    id 'com.android.library' version '8.2.0' apply false
+    id 'org.jetbrains.kotlin.android' version '1.8.10' apply false
+}

android/MLCChat/bundle_weight.py

@@ -0,0 +1,65 @@
+import argparse
+import os
+import subprocess
+from pathlib import Path
+
+from mlc_llm.support import logging
+
+logging.enable_logging()
+logger = logging.getLogger(__name__)
+
+
+def main(apk_path: Path, package_output_path: Path):
+    """Push weights to the android device with adb"""
+    # - Install the apk on device.
+    logger.info('Install apk "%s" to device', str(apk_path.absolute()))
+    subprocess.run(["adb", "install", str(apk_path)], check=True, env=os.environ)
+    # - Create the weight directory for the app.
+    device_weihgt_dir = "/storage/emulated/0/Android/data/ai.mlc.mlcchat/files/"
+    logger.info('Creating directory "%s" on device', device_weihgt_dir)
+    subprocess.run(
+        ["adb", "shell", "mkdir", "-p", device_weihgt_dir],
+        check=True,
+        env=os.environ,
+    )
+    for model_weight_dir in (package_output_path / "bundle").iterdir():
+        if model_weight_dir.is_dir():
+            src_path = str(model_weight_dir.absolute())
+            dst_path = "/data/local/tmp/" + model_weight_dir.name
+            logger.info('Pushing local weights "%s" to device location "%s"', src_path, dst_path)
+            subprocess.run(["adb", "push", src_path, dst_path], check=True, env=os.environ)
+
+            src_path = dst_path
+            dst_path = "/storage/emulated/0/Android/data/ai.mlc.mlcchat/files/"
+            logger.info('Move weights from "%s" to "%s"', src_path, dst_path)
+            subprocess.run(["adb", "shell", "mv", src_path, dst_path], check=True, env=os.environ)
+    logger.info("All finished.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("MLC LLM Android Weight Bundle")
+
+    def _parse_apk_path(path: str) -> Path:
+        path = Path(path)
+        if not path.exists():
+            raise ValueError(
+                f"Path {str(path)} is expected to be an apk file, but the file does not exist."
+            )
+        if not path.is_file():
+            raise ValueError(f"Path {str(path)} is expected to be an apk file.")
+        return path
+
+    parser.add_argument(
+        "--apk-path",
+        type=_parse_apk_path,
+        default="app/release/app-release.apk",
+        help="The path to generated MLCChat apk file.",
+    )
+    parser.add_argument(
+        "--package-output-path",
+        type=Path,
+        default="dist",
+        help='The path to the output directory of "mlc_llm package".',
+    )
+    args = parser.parse_args()
+    main(args.apk_path, args.package_output_path)

android/MLCChat/gradle.properties

@@ -0,0 +1,23 @@
+# Project-wide Gradle settings.
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. More details, visit
+# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
+# org.gradle.parallel=true
+# AndroidX package structure to make it clearer which packages are bundled with the
+# Android operating system, and which are packaged with your app's APK
+# https://developer.android.com/topic/libraries/support-library/androidx-rn
+android.useAndroidX=true
+# Kotlin code style for this project: "official" or "obsolete":
+kotlin.code.style=official
+# Enables namespacing of each library's R class so that its R class includes only the
+# resources declared in the library itself and none from the library's dependencies,
+# thereby reducing the size of the R class for that library
+android.nonTransitiveRClass=true

android/MLCChat/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Thu Jan 25 10:19:50 EST 2024
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

android/MLCChat/gradlew

@@ -0,0 +1,185 @@
+#!/usr/bin/env sh
+
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=`expr $i + 1`
+    done
+    case $i in
+        0) set -- ;;
+        1) set -- "$args0" ;;
+        2) set -- "$args0" "$args1" ;;
+        3) set -- "$args0" "$args1" "$args2" ;;
+        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=`save "$@"`
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+exec "$JAVACMD" "$@"

android/MLCChat/gradlew.bat

@@ -0,0 +1,89 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

android/MLCChat/mlc-package-config.json

@@ -0,0 +1,50 @@
+{
+    "device": "android",
+    "model_list": [
+        {
+            "model": "HF://mlc-ai/Phi-3.5-mini-instruct-q4f16_0-MLC",
+            "estimated_vram_bytes": 4250586449,
+            "model_id": "Phi-3.5-mini-instruct-q4f16_0-MLC",
+            "overrides": {
+                "prefill_chunk_size": 128
+            }
+        },
+        {
+            "model": "HF://mlc-ai/Qwen3-0.6B-q0f16-MLC",
+            "model_id": "Qwen3-0.6B-q0f16-MLC",
+            "estimated_vram_bytes": 3000000000,
+            "overrides": {
+                "prefill_chunk_size": 128,
+                "context_window_size": 2048
+            }
+        },
+        {
+            "model": "HF://mlc-ai/Qwen3-1.7B-q4f16_1-MLC",
+            "model_id": "Qwen3-1.7B-q4f16_1-MLC",
+            "estimated_vram_bytes": 3000000000,
+            "overrides": {
+                "prefill_chunk_size": 128,
+                "context_window_size": 2048
+            }
+        },
+        {
+            "model": "HF://mlc-ai/gemma-2-2b-it-q4f16_1-MLC",
+            "model_id": "gemma-2-2b-it-q4f16_1-MLC",
+            "estimated_vram_bytes": 3000000000
+        },
+        {
+            "model": "HF://mlc-ai/Llama-3.2-3B-Instruct-q4f16_0-MLC",
+            "estimated_vram_bytes": 4679979417,
+            "model_id": "Llama-3.2-3B-Instruct-q4f16_0-MLC"
+        },
+        {
+            "model": "HF://mlc-ai/Mistral-7B-Instruct-v0.3-q4f16_1-MLC",
+            "estimated_vram_bytes": 4115131883,
+            "model_id": "Mistral-7B-Instruct-v0.3-q4f16_1-MLC",
+            "overrides": {
+                "sliding_window_size": 768,
+                "prefill_chunk_size": 256
+            }
+        }
+    ]
+}

android/MLCChat/settings.gradle

@@ -0,0 +1,20 @@
+pluginManagement {
+    repositories {
+        google()
+        mavenCentral()
+        gradlePluginPortal()
+    }
+}
+dependencyResolutionManagement {
+    repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
+    repositories {
+        google()
+        mavenCentral()
+        maven { url "https://jitpack.io" }
+    }
+}
+rootProject.name = "MLCChat"
+include ':app'
+include ':mlc4j'
+project(':mlc4j').projectDir = file('dist/lib/mlc4j')
+include ':mlcengineexample'

android/MLCEngineExample/README.md

@@ -0,0 +1,6 @@
+# MLC-LLM Android
+
+Checkout [Documentation page](https://llm.mlc.ai/docs/deploy/android.html) for more information.
+
+- run `mlc_llm package`
+- open this `MLCEngineExample/` folder as a project in Android Studio

android/MLCEngineExample/app/.gitignore

@@ -0,0 +1,2 @@
+/build
+/src/main/libs

android/MLCEngineExample/app/build.gradle

@@ -0,0 +1,73 @@
+plugins {
+    id 'com.android.application'
+    id 'org.jetbrains.kotlin.android'
+}
+
+android {
+    namespace 'ai.mlc.mlcengineexample'
+    compileSdk 34
+
+    defaultConfig {
+        applicationId "ai.mlc.mlcengineexample"
+        minSdk 26
+        targetSdk 33
+        versionCode 1
+        versionName "1.0"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+        vectorDrawables {
+            useSupportLibrary true
+        }
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+    buildFeatures {
+        compose true
+    }
+    composeOptions {
+        kotlinCompilerExtensionVersion '1.4.3'
+    }
+    packagingOptions {
+        resources {
+            excludes += '/META-INF/{AL2.0,LGPL2.1}'
+        }
+    }
+}
+
+dependencies {
+    implementation project(":mlc4j")
+    implementation 'androidx.core:core-ktx:1.10.1'
+    implementation 'androidx.lifecycle:lifecycle-runtime-ktx:2.6.1'
+    implementation 'androidx.activity:activity-compose:1.7.1'
+    implementation platform('androidx.compose:compose-bom:2022.10.00')
+    implementation 'androidx.lifecycle:lifecycle-viewmodel-compose:2.6.1'
+    implementation 'androidx.compose.ui:ui'
+    implementation 'androidx.compose.ui:ui-graphics'
+    implementation 'androidx.compose.ui:ui-tooling-preview'
+    implementation 'androidx.compose.material3:material3:1.1.0'
+    implementation 'androidx.compose.material:material-icons-extended'
+    implementation 'androidx.appcompat:appcompat:1.6.1'
+    implementation 'androidx.navigation:navigation-compose:2.5.3'
+    implementation 'com.google.code.gson:gson:2.10.1'
+    implementation fileTree(dir: 'src/main/libs', include: ['*.aar', '*.jar'], exclude: [])
+    testImplementation 'junit:junit:4.13.2'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.5'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
+    androidTestImplementation platform('androidx.compose:compose-bom:2022.10.00')
+    androidTestImplementation 'androidx.compose.ui:ui-test-junit4'
+    debugImplementation 'androidx.compose.ui:ui-tooling'
+    debugImplementation 'androidx.compose.ui:ui-test-manifest'
+
+}

android/MLCEngineExample/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

android/MLCEngineExample/app/src/main/AndroidManifest.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    package="ai.mlc.mlcengineexample">
+
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission
+        android:name="android.permission.WRITE_EXTERNAL_STORAGE"
+        android:maxSdkVersion="32"
+        tools:ignore="ScopedStorage" />
+
+    <application
+        android:allowBackup="true"
+        android:dataExtractionRules="@xml/data_extraction_rules"
+        android:fullBackupContent="@xml/backup_rules"
+        android:icon="@drawable/mlc_logo_108"
+        android:label="@string/app_name"
+        android:roundIcon="@drawable/mlc_logo_108"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.MLCEngineExample"
+        tools:targetApi="31">
+        <uses-native-library
+            android:name="libOpenCL.so"
+            android:required="false"/>
+
+        <uses-native-library
+            android:name="libOpenCL-pixel.so"
+            android:required="false" />
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:label="@string/app_name"
+            android:theme="@android:style/Theme.Material.NoActionBar">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>

android/MLCEngineExample/app/src/main/java/ai/mlc/mlcengineexample/MainActivity.kt

@@ -0,0 +1,81 @@
+package ai.mlc.mlcengineexample
+
+import ai.mlc.mlcengineexample.ui.theme.MLCEngineExampleTheme
+import ai.mlc.mlcllm.MLCEngine
+import ai.mlc.mlcllm.OpenAIProtocol
+import ai.mlc.mlcllm.OpenAIProtocol.*
+import android.annotation.SuppressLint
+import android.os.Bundle
+import android.util.Log
+import androidx.activity.ComponentActivity
+import androidx.activity.compose.setContent
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.material3.ExperimentalMaterial3Api
+import androidx.compose.material3.Surface
+import androidx.compose.material3.Text
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.rememberCoroutineScope
+import androidx.compose.ui.Modifier
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.channels.ReceiveChannel
+import kotlinx.coroutines.launch
+import java.io.File
+
+
+class MainActivity : ComponentActivity() {
+    @SuppressLint("CoroutineCreationDuringComposition")
+    @ExperimentalMaterial3Api
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+
+        val modelName = "phi-2-q4f16_1-MLC"
+        var modelPath = File(application.getExternalFilesDir(""), modelName).toString()
+        Log.i("MLC", "model path: $modelPath")
+        // need to be changed to the custom system lib prefix used while compiling the model
+        val modelLib = "phi_msft_q4f16_1_686d8979c6ebf05d142d9081f1b87162"
+        Log.i("MLC", "engine loaded")
+
+        setContent {
+            val responseText = remember { mutableStateOf("") }
+            val coroutineScope = rememberCoroutineScope()
+            val engine = MLCEngine()
+            engine.unload()
+            engine.reload(modelPath, modelLib)
+            coroutineScope.launch {
+                var channel = engine.chat.completions.create(
+                    messages = listOf(
+                        ChatCompletionMessage(
+                            role = OpenAIProtocol.ChatCompletionRole.user,
+                            content = "What is the meaning of life?"
+                        )
+                    ),
+                    stream_options = OpenAIProtocol.StreamOptions(include_usage = true)
+                )
+
+
+                for (response in channel) {
+                    val finalusage = response.usage
+                    if (finalusage != null) {
+                        responseText.value += "\n" + (finalusage.extra?.asTextLabel() ?: "")
+                    } else {
+                        if (response.choices.size > 0) {
+                            responseText.value += response.choices[0].delta.content?.asText()
+                                .orEmpty()
+                        }
+                    }
+
+                }
+            }
+
+            Surface(
+                modifier = Modifier
+                    .fillMaxSize()
+            ) {
+                MLCEngineExampleTheme {
+                    Text(text = responseText.value)
+                }
+            }
+        }
+    }
+}

android/MLCEngineExample/app/src/main/java/ai/mlc/mlcengineexample/ui/theme/Color.kt

@@ -0,0 +1,44 @@
+package ai.mlc.mlcengineexample.ui.theme
+
+import androidx.compose.ui.graphics.Color
+
+val Blue10 = Color(0xFF000F5E)
+val Blue20 = Color(0xFF001E92)
+val Blue30 = Color(0xFF002ECC)
+val Blue40 = Color(0xFF1546F6)
+val Blue80 = Color(0xFFB8C3FF)
+val Blue90 = Color(0xFFDDE1FF)
+
+val DarkBlue10 = Color(0xFF00036B)
+val DarkBlue20 = Color(0xFF000BA6)
+val DarkBlue30 = Color(0xFF1026D3)
+val DarkBlue40 = Color(0xFF3648EA)
+val DarkBlue80 = Color(0xFFBBC2FF)
+val DarkBlue90 = Color(0xFFDEE0FF)
+
+val Yellow10 = Color(0xFF261900)
+val Yellow20 = Color(0xFF402D00)
+val Yellow30 = Color(0xFF5C4200)
+val Yellow40 = Color(0xFF7A5900)
+val Yellow80 = Color(0xFFFABD1B)
+val Yellow90 = Color(0xFFFFDE9C)
+
+val Red10 = Color(0xFF410001)
+val Red20 = Color(0xFF680003)
+val Red30 = Color(0xFF930006)
+val Red40 = Color(0xFFBA1B1B)
+val Red80 = Color(0xFFFFB4A9)
+val Red90 = Color(0xFFFFDAD4)
+
+val Grey10 = Color(0xFF191C1D)
+val Grey20 = Color(0xFF2D3132)
+val Grey80 = Color(0xFFC4C7C7)
+val Grey90 = Color(0xFFE0E3E3)
+val Grey95 = Color(0xFFEFF1F1)
+val Grey99 = Color(0xFFFBFDFD)
+
+val BlueGrey30 = Color(0xFF45464F)
+val BlueGrey50 = Color(0xFF767680)
+val BlueGrey60 = Color(0xFF90909A)
+val BlueGrey80 = Color(0xFFC6C5D0)
+val BlueGrey90 = Color(0xFFE2E1EC)

android/MLCEngineExample/app/src/main/java/ai/mlc/mlcengineexample/ui/theme/Theme.kt

@@ -0,0 +1,107 @@
+package ai.mlc.mlcengineexample.ui.theme
+
+import android.app.Activity
+import android.os.Build
+import androidx.compose.foundation.isSystemInDarkTheme
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.darkColorScheme
+import androidx.compose.material3.dynamicDarkColorScheme
+import androidx.compose.material3.dynamicLightColorScheme
+import androidx.compose.material3.lightColorScheme
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.SideEffect
+import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.toArgb
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalView
+import androidx.core.view.WindowCompat
+
+private val DarkColorScheme = darkColorScheme(
+    primary = Blue80,
+    onPrimary = Blue20,
+    primaryContainer = Blue30,
+    onPrimaryContainer = Blue90,
+    inversePrimary = Blue40,
+    secondary = DarkBlue80,
+    onSecondary = DarkBlue20,
+    secondaryContainer = DarkBlue30,
+    onSecondaryContainer = DarkBlue90,
+    tertiary = Yellow80,
+    onTertiary = Yellow20,
+    tertiaryContainer = Yellow30,
+    onTertiaryContainer = Yellow90,
+    error = Red80,
+    onError = Red20,
+    errorContainer = Red30,
+    onErrorContainer = Red90,
+    background = Grey10,
+    onBackground = Grey90,
+    surface = Grey10,
+    onSurface = Grey80,
+    inverseSurface = Grey90,
+    inverseOnSurface = Grey20,
+    surfaceVariant = BlueGrey30,
+    onSurfaceVariant = BlueGrey80,
+    outline = BlueGrey60
+)
+
+private val LightColorScheme = lightColorScheme(
+    primary = Blue40,
+    onPrimary = Color.White,
+    primaryContainer = Blue90,
+    onPrimaryContainer = Blue10,
+    inversePrimary = Blue80,
+    secondary = DarkBlue40,
+    onSecondary = Color.White,
+    secondaryContainer = DarkBlue90,
+    onSecondaryContainer = DarkBlue10,
+    tertiary = Yellow40,
+    onTertiary = Color.White,
+    tertiaryContainer = Yellow90,
+    onTertiaryContainer = Yellow10,
+    error = Red40,
+    onError = Color.White,
+    errorContainer = Red90,
+    onErrorContainer = Red10,
+    background = Grey99,
+    onBackground = Grey10,
+    surface = Grey99,
+    onSurface = Grey10,
+    inverseSurface = Grey20,
+    inverseOnSurface = Grey95,
+    surfaceVariant = BlueGrey90,
+    onSurfaceVariant = BlueGrey30,
+    outline = BlueGrey50
+)
+
+@Composable
+fun MLCEngineExampleTheme(
+    darkTheme: Boolean = isSystemInDarkTheme(),
+    // Dynamic color is available on Android 12+
+    dynamicColor: Boolean = true,
+    content: @Composable () -> Unit
+) {
+    val colorScheme = when {
+        dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> {
+            val context = LocalContext.current
+            if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
+        }
+
+        darkTheme -> DarkColorScheme
+        else -> LightColorScheme
+    }
+    val view = LocalView.current
+    if (!view.isInEditMode) {
+        SideEffect {
+            val window = (view.context as Activity).window
+            window.statusBarColor = colorScheme.primary.toArgb()
+            WindowCompat.getInsetsController(window, view).isAppearanceLightStatusBars = darkTheme
+        }
+    }
+
+    MaterialTheme(
+        colorScheme = colorScheme,
+        typography = Typography,
+        content = content
+    )
+}

android/MLCEngineExample/app/src/main/java/ai/mlc/mlcengineexample/ui/theme/Type.kt

@@ -0,0 +1,34 @@
+package ai.mlc.mlcengineexample.ui.theme
+
+import androidx.compose.material3.Typography
+import androidx.compose.ui.text.TextStyle
+import androidx.compose.ui.text.font.FontFamily
+import androidx.compose.ui.text.font.FontWeight
+import androidx.compose.ui.unit.sp
+
+// Set of Material typography styles to start with
+val Typography = Typography(
+    bodyLarge = TextStyle(
+        fontFamily = FontFamily.Default,
+        fontWeight = FontWeight.Normal,
+        fontSize = 16.sp,
+        lineHeight = 24.sp,
+        letterSpacing = 0.5.sp
+    )
+    /* Other default text styles to override
+    titleLarge = TextStyle(
+        fontFamily = FontFamily.Default,
+        fontWeight = FontWeight.Normal,
+        fontSize = 22.sp,
+        lineHeight = 28.sp,
+        letterSpacing = 0.sp
+    ),
+    labelSmall = TextStyle(
+        fontFamily = FontFamily.Default,
+        fontWeight = FontWeight.Medium,
+        fontSize = 11.sp,
+        lineHeight = 16.sp,
+        letterSpacing = 0.5.sp
+    )
+    */
+)

android/MLCEngineExample/app/src/main/res/drawable/ic_android_black_24dp.xml

@@ -0,0 +1,5 @@
+<vector android:height="24dp" android:tint="#000000"
+    android:viewportHeight="24" android:viewportWidth="24"
+    android:width="24dp" xmlns:android="http://schemas.android.com/apk/res/android">
+    <path android:fillColor="#FF000000" android:pathData="M17.6,11.48 L19.44,8.3a0.63,0.63 0,0 0,-1.09 -0.63l-1.88,3.24a11.43,11.43 0,0 0,-8.94 0L5.65,7.67a0.63,0.63 0,0 0,-1.09 0.63L6.4,11.48A10.81,10.81 0,0 0,1 20L23,20A10.81,10.81 0,0 0,17.6 11.48ZM7,17.25A1.25,1.25 0,1 1,8.25 16,1.25 1.25,0 0,1 7,17.25ZM17,17.25A1.25,1.25 0,1 1,18.25 16,1.25 1.25,0 0,1 17,17.25Z"/>
+</vector>

android/MLCEngineExample/app/src/main/res/values/colors.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="purple_200">#FFBB86FC</color>
+    <color name="purple_500">#FF6200EE</color>
+    <color name="purple_700">#FF3700B3</color>
+    <color name="teal_200">#FF03DAC5</color>
+    <color name="teal_700">#FF018786</color>
+    <color name="black">#FF000000</color>
+    <color name="white">#FFFFFFFF</color>
+</resources>

android/MLCEngineExample/app/src/main/res/values/strings.xml

@@ -0,0 +1,3 @@
+<resources>
+    <string name="app_name">MLCEngineExample</string>
+</resources>

android/MLCEngineExample/app/src/main/res/values/themes.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+
+    <style name="Theme.MLCEngineExample" parent="android:Theme.Material.Light" />
+
+</resources>

android/MLCEngineExample/app/src/main/res/xml/backup_rules.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+   Sample backup rules file; uncomment and customize as necessary.
+   See https://developer.android.com/guide/topics/data/autobackup
+   for details.
+   Note: This file is ignored for devices older that API 31
+   See https://developer.android.com/about/versions/12/backup-restore
+-->
+<full-backup-content>
+    <!--
+   <include domain="sharedpref" path="."/>
+   <exclude domain="sharedpref" path="device.xml"/>
+-->
+</full-backup-content>

android/MLCEngineExample/app/src/main/res/xml/data_extraction_rules.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+   Sample data extraction rules file; uncomment and customize as necessary.
+   See https://developer.android.com/about/versions/12/backup-restore#xml-changes
+   for details.
+-->
+<data-extraction-rules>
+    <cloud-backup>
+        <!-- TODO: Use <include> and <exclude> to control what is backed up.
+        <include .../>
+        <exclude .../>
+        -->
+    </cloud-backup>
+    <!--
+    <device-transfer>
+        <include .../>
+        <exclude .../>
+    </device-transfer>
+    -->
+</data-extraction-rules>

android/MLCEngineExample/build.gradle

@@ -0,0 +1,5 @@
+plugins {
+    id 'com.android.application' version '8.2.0' apply false
+    id 'com.android.library' version '8.2.0' apply false
+    id 'org.jetbrains.kotlin.android' version '1.8.10' apply false
+}

android/MLCEngineExample/bundle_weight.py

@@ -0,0 +1,65 @@
+import argparse
+import os
+import subprocess
+from pathlib import Path
+
+from mlc_llm.support import logging
+
+logging.enable_logging()
+logger = logging.getLogger(__name__)
+
+
+def main(apk_path: Path, package_output_path: Path):
+    """Push weights to the android device with adb"""
+    # - Install the apk on device.
+    logger.info('Install apk "%s" to device', str(apk_path.absolute()))
+    subprocess.run(["adb", "install", str(apk_path)], check=True, env=os.environ)
+    # - Create the weight directory for the app.
+    device_weihgt_dir = "/storage/emulated/0/Android/data/ai.mlc.mlcengineexample/files/"
+    logger.info('Creating directory "%s" on device', device_weihgt_dir)
+    subprocess.run(
+        ["adb", "shell", "mkdir", "-p", device_weihgt_dir],
+        check=True,
+        env=os.environ,
+    )
+    for model_weight_dir in (package_output_path / "bundle").iterdir():
+        if model_weight_dir.is_dir():
+            src_path = str(model_weight_dir.absolute())
+            dst_path = "/data/local/tmp/" + model_weight_dir.name
+            logger.info('Pushing local weights "%s" to device location "%s"', src_path, dst_path)
+            subprocess.run(["adb", "push", src_path, dst_path], check=True, env=os.environ)
+
+            src_path = dst_path
+            dst_path = "/storage/emulated/0/Android/data/ai.mlc.mlcengineexample/files/"
+            logger.info('Move weights from "%s" to "%s"', src_path, dst_path)
+            subprocess.run(["adb", "shell", "mv", src_path, dst_path], check=True, env=os.environ)
+    logger.info("All finished.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("MLC LLM Android Weight Bundle")
+
+    def _parse_apk_path(path: str) -> Path:
+        path = Path(path)
+        if not path.exists():
+            raise ValueError(
+                f"Path {str(path)} is expected to be an apk file, but the file does not exist."
+            )
+        if not path.is_file():
+            raise ValueError(f"Path {str(path)} is expected to be an apk file.")
+        return path
+
+    parser.add_argument(
+        "--apk-path",
+        type=_parse_apk_path,
+        default="app/release/app-release.apk",
+        help="The path to generated MLCEngineExample apk file.",
+    )
+    parser.add_argument(
+        "--package-output-path",
+        type=Path,
+        default="dist",
+        help='The path to the output directory of "mlc_llm package".',
+    )
+    args = parser.parse_args()
+    main(args.apk_path, args.package_output_path)

android/MLCEngineExample/gradle.properties

@@ -0,0 +1,23 @@
+# Project-wide Gradle settings.
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. More details, visit
+# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
+# org.gradle.parallel=true
+# AndroidX package structure to make it clearer which packages are bundled with the
+# Android operating system, and which are packaged with your app's APK
+# https://developer.android.com/topic/libraries/support-library/androidx-rn
+android.useAndroidX=true
+# Kotlin code style for this project: "official" or "obsolete":
+kotlin.code.style=official
+# Enables namespacing of each library's R class so that its R class includes only the
+# resources declared in the library itself and none from the library's dependencies,
+# thereby reducing the size of the R class for that library
+android.nonTransitiveRClass=true

android/MLCEngineExample/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Thu Jan 25 10:19:50 EST 2024
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

android/MLCEngineExample/gradlew

@@ -0,0 +1,185 @@
+#!/usr/bin/env sh
+
+#
+# Copyright 2015 the original author or authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=`expr $i + 1`
+    done
+    case $i in
+        0) set -- ;;
+        1) set -- "$args0" ;;
+        2) set -- "$args0" "$args1" ;;
+        3) set -- "$args0" "$args1" "$args2" ;;
+        4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=`save "$@"`
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+exec "$JAVACMD" "$@"

android/MLCEngineExample/gradlew.bat

@@ -0,0 +1,89 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

android/MLCEngineExample/mlc-package-config.json

@@ -0,0 +1,13 @@
+{
+    "device": "android",
+    "model_list": [
+        {
+            "model": "HF://mlc-ai/phi-2-q4f16_1-MLC",
+            "estimated_vram_bytes": 2036816936,
+            "model_id": "phi-2-q4f16_1-MLC",
+            "overrides": {
+                "prefill_chunk_size": 1024
+            }
+        }
+    ]
+}

android/MLCEngineExample/settings.gradle

@@ -0,0 +1,18 @@
+pluginManagement {
+    repositories {
+        google()
+        mavenCentral()
+        gradlePluginPortal()
+    }
+}
+dependencyResolutionManagement {
+    repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
+    repositories {
+        google()
+        mavenCentral()
+    }
+}
+rootProject.name = "MLCEngineExample"
+include ':app'
+include ':mlc4j'
+project(':mlc4j').projectDir = file('dist/lib/mlc4j')

android/README.md

@@ -0,0 +1,3 @@
+# MLC-LLM Android
+
+[Documentation page](https://llm.mlc.ai/docs/deploy/android.html)

android/mlc4j/.gitignore

@@ -0,0 +1 @@
+/build

android/mlc4j/CMakeLists.txt

@@ -0,0 +1,90 @@
+cmake_minimum_required(VERSION 3.18)
+
+project(mlc-chat C CXX)
+
+set(ANDROID_DIR ${CMAKE_CURRENT_LIST_DIR})
+set(ANDROID_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+set(MLC_LLM_DIR ${ANDROID_DIR}/../..)
+set(MLC_LLM_BINARY_DIR mlc_llm)
+set(MLC_LLM_COMPILE_DEFS TVM_LOG_CUSTOMIZE=1)
+add_subdirectory(${MLC_LLM_DIR} ${MLC_LLM_BINARY_DIR} EXCLUDE_FROM_ALL)
+
+if(NOT DEFINED TVM_SOURCE_DIR)
+  set(TVM_SOURCE_DIR ${MLC_LLM_DIR}/3rdparty/tvm)
+endif(NOT DEFINED TVM_SOURCE_DIR)
+message(STATUS "TVM_SOURCE_DIR: ${TVM_SOURCE_DIR}")
+
+find_package(Java REQUIRED)
+include(UseJava)
+
+find_package(JNI)
+if(JNI_FOUND)
+  message(STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
+else()
+  message(STATUS "Try to find jni directly from android env")
+  # try to find JNI_LIBRARY
+  find_path(JNI_INCLUDE_DIRS NAMES "jni.h")
+  message(STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
+endif()
+
+file(GLOB_RECURSE javasources
+     ${TVM_SOURCE_DIR}/jvm/core/src/main/java/org/apache/tvm/*.java
+     ${ANDROID_DIR}/src/java/*.java)
+set(JNI_HEADER ${CMAKE_BINARY_DIR}/jni_header)
+add_jar(tvm4j_core ${javasources} GENERATE_NATIVE_HEADERS tvm4jheaders
+        DESTINATION ${JNI_HEADER})
+
+add_custom_command(
+  TARGET tvm4j_core
+  POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy ${JNI_HEADER}/org_apache_tvm_LibInfo.h
+          ${JNI_HEADER}/org_apache_tvm_native_c_api.h)
+
+add_library(model_android STATIC IMPORTED)
+set_target_properties(
+  model_android PROPERTIES IMPORTED_LOCATION
+                           ${ANDROID_BIN_DIR}/lib/libmodel_android.a)
+
+add_library(
+  tvm4j_runtime_packed SHARED
+  ${TVM_SOURCE_DIR}/jvm/native/src/main/native/org_apache_tvm_native_c_api.cc)
+set(MLC_LLM_COMPILE_DEFS ${MLC_LLM_COMPILE_DEFS}
+                         TVM_SOURCE_DIR=${TVM_SOURCE_DIR})
+
+target_include_directories(
+  tvm4j_runtime_packed
+  PUBLIC ${JNI_INCLUDE_DIRS}
+         ${JNI_HEADER}
+         ${ANDROID_DIR}/src/cpp
+         ${TVM_SOURCE_DIR}/3rdparty/tvm-ffi/3rdparty/dlpack/include
+         ${TVM_SOURCE_DIR}/3rdparty/OpenCL-Headers
+         ${TVM_SOURCE_DIR}/include
+         ${TVM_SOURCE_DIR}/src
+         ${TVM_SOURCE_DIR}/3rdparty/tvm-ffi/include
+         ${TVM_SOURCE_DIR}/3rdparty/tvm-ffi/src)
+target_compile_definitions(tvm4j_runtime_packed PUBLIC ${MLC_LLM_COMPILE_DEFS})
+target_compile_definitions(
+  tvm4j_runtime_packed
+  PUBLIC TVM_VM_ENABLE_PROFILER=0
+  PUBLIC TVM_FFI_USE_LIBBACKTRACE=0
+  PUBLIC TVM_FFI_BACKTRACE_ON_SEGFAULT=0)
+
+set(MLC_ENABLE_SENTENCEPIECE_TOKENIZER OFF)
+target_link_libraries(
+  tvm4j_runtime_packed
+  tokenizers_c
+  tokenizers_cpp
+  log
+  -Wl,--whole-archive
+  mlc_llm_static
+  model_android
+  -Wl,--no-whole-archive)
+
+target_compile_definitions(tvm4j_runtime_packed PUBLIC TVM4J_ANDROID)
+add_dependencies(tvm4j_runtime_packed tvm4j_core)
+
+target_compile_definitions(mlc_llm_objs PUBLIC MLC_SINGLE_GPU_ONLY)
+
+install_jar(tvm4j_core output)
+install(TARGETS tvm4j_runtime_packed LIBRARY DESTINATION output/${ANDROID_ABI})

android/mlc4j/build.gradle

@@ -0,0 +1,34 @@
+plugins {
+    id 'com.android.library'
+    id 'org.jetbrains.kotlin.android'
+    id 'org.jetbrains.kotlin.plugin.serialization' version '1.8.0'
+}
+
+android {
+    namespace 'ai.mlc.mlcllm'
+    compileSdk 34
+
+    defaultConfig {
+        minSdk 22
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+    sourceSets {
+        main {
+            jniLibs.srcDirs = ['output']
+        }
+    }
+}
+
+dependencies {
+    implementation fileTree(dir: 'output', include: ['*.jar'])
+    implementation 'androidx.core:core-ktx:1.9.0'
+    implementation 'androidx.appcompat:appcompat:1.6.1'
+    implementation 'com.google.android.material:material:1.10.0'
+    implementation 'org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3'
+}

android/mlc4j/prepare_libs.py

@@ -0,0 +1,122 @@
+"""The build script for mlc4j (MLC LLM and tvm4j)"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from mlc_llm.support import logging
+
+logging.enable_logging()
+logger = logging.getLogger(__name__)
+
+
+def run_cmake(mlc4j_path: Path):
+    if "ANDROID_NDK" not in os.environ:
+        raise ValueError(
+            'Environment variable "ANDROID_NDK" is required but not found.'
+            "Please follow https://llm.mlc.ai/docs/deploy/android.html to properly "
+            'specify "ANDROID_NDK".'
+        )
+    logger.info("Running cmake")
+    # use pathlib so it is cross platform
+    android_ndk_path = (
+        Path(os.environ["ANDROID_NDK"]) / "build" / "cmake" / "android.toolchain.cmake"
+    )
+    cmd = [
+        "cmake",
+        str(mlc4j_path),
+        "-DCMAKE_BUILD_TYPE=Release",
+        f"-DCMAKE_TOOLCHAIN_FILE={str(android_ndk_path)}",
+        "-DCMAKE_INSTALL_PREFIX=.",
+        '-DCMAKE_CXX_FLAGS="-O3"',
+        "-DANDROID_ABI=arm64-v8a",
+        "-DANDROID_NATIVE_API_LEVEL=android-24",
+        "-DANDROID_PLATFORM=android-24",
+        "-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON",
+        "-DANDROID_STL=c++_static",
+        "-DUSE_HEXAGON_SDK=OFF",
+        "-DMLC_LLM_INSTALL_STATIC_LIB=ON",
+        "-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON",
+        "-DUSE_OPENCL=ON",
+        "-DUSE_OPENCL_ENABLE_HOST_PTR=ON",
+        "-DUSE_CUSTOM_LOGGING=ON",
+        "-DTVM_FFI_USE_LIBBACKTRACE=OFF",
+        "-DTVM_FFI_BACKTRACE_ON_SEGFAULT=OFF",
+    ]
+
+    if sys.platform == "win32":
+        logger.info("Using ninja in windows, make sure you installed ninja in conda")
+        cmd += ["-G", "Ninja"]
+    subprocess.run(cmd, check=True, env=os.environ)
+
+
+def run_cmake_build():
+    logger.info("Running cmake build")
+    cmd = [
+        "cmake",
+        "--build",
+        ".",
+        "--target",
+        "tvm4j_runtime_packed",
+        "--config",
+        "release",
+        f"-j{os.cpu_count()}",
+    ]
+    subprocess.run(cmd, check=True, env=os.environ)
+
+
+def run_cmake_install():
+    logger.info("Running cmake install")
+    cmd = [
+        "cmake",
+        "--build",
+        ".",
+        "--target",
+        "install",
+        "--config",
+        "release",
+        f"-j{os.cpu_count()}",
+    ]
+    subprocess.run(cmd, check=True, env=os.environ)
+
+
+def main(mlc_llm_source_dir: Path):
+    # - Setup rust.
+    subprocess.run(["rustup", "target", "add", "aarch64-linux-android"], check=True, env=os.environ)
+
+    # - Build MLC LLM and tvm4j.
+    build_path = Path("build")
+    os.makedirs(build_path / "lib", exist_ok=True)
+    logger.info('Entering "%s" for MLC LLM and tvm4j build.', os.path.abspath(build_path))
+    os.chdir(build_path)
+    # Generate config.cmake if TVM Home is set.
+    if "TVM_SOURCE_DIR" in os.environ:
+        logger.info('Set TVM_SOURCE_DIR to "%s"', os.environ["TVM_SOURCE_DIR"])
+        with open("config.cmake", "w", encoding="utf-8") as file:
+            # We use "json.dumps" to escape backslashes and quotation marks
+            tvm_source_dir_str_with_escape = json.dumps(os.environ["TVM_SOURCE_DIR"])
+            print("set(TVM_SOURCE_DIR %s)" % tvm_source_dir_str_with_escape, file=file)
+
+    # - Run cmake, build and install
+    run_cmake(mlc_llm_source_dir / "android" / "mlc4j")
+    run_cmake_build()
+    run_cmake_install()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("MLC LLM Android Lib Preparation")
+
+    parser.add_argument(
+        "--mlc-llm-source-dir",
+        type=Path,
+        default=os.environ.get("MLC_LLM_SOURCE_DIR", None),
+        help="The path to MLC LLM source",
+    )
+    parsed = parser.parse_args()
+    if parsed.mlc_llm_source_dir is None:
+        parsed.mlc_llm_source_dir = Path(os.path.abspath(os.path.curdir)).parent.parent
+    os.environ["MLC_LLM_SOURCE_DIR"] = str(parsed.mlc_llm_source_dir)
+    main(parsed.mlc_llm_source_dir)

android/mlc4j/src/cpp/tvm_runtime.h

@@ -0,0 +1,65 @@
+#define TVM_USE_LIBBACKTRACE 0
+
+#include <android/log.h>
+#include <dlfcn.h>
+#include <tvm/runtime/logging.h>
+
+#include <ffi/backtrace.cc>
+#include <ffi/container.cc>
+#include <ffi/dtype.cc>
+#include <ffi/error.cc>
+#include <ffi/extra/env_c_api.cc>
+#include <ffi/extra/env_context.cc>
+#include <ffi/extra/json_parser.cc>
+#include <ffi/extra/json_writer.cc>
+#include <ffi/extra/library_module.cc>
+#include <ffi/extra/library_module_dynamic_lib.cc>
+#include <ffi/extra/library_module_system_lib.cc>
+#include <ffi/extra/module.cc>
+#include <ffi/function.cc>
+#include <ffi/object.cc>
+#include <runtime/cpu_device_api.cc>
+#include <runtime/device_api.cc>
+#include <runtime/file_utils.cc>
+#include <runtime/logging.cc>
+#include <runtime/memory/memory_manager.cc>
+#include <runtime/module.cc>
+#include <runtime/nvtx.cc>
+#include <runtime/opencl/opencl_device_api.cc>
+#include <runtime/opencl/opencl_module.cc>
+#include <runtime/opencl/opencl_wrapper/opencl_wrapper.cc>
+#include <runtime/profiling.cc>
+#include <runtime/source_utils.cc>
+#include <runtime/tensor.cc>
+#include <runtime/thread_pool.cc>
+#include <runtime/threading_backend.cc>
+#include <runtime/vm/attn_backend.cc>
+#include <runtime/vm/builtin.cc>
+#include <runtime/vm/bytecode.cc>
+#include <runtime/vm/executable.cc>
+#include <runtime/vm/kv_state.cc>
+#include <runtime/vm/paged_kv_cache.cc>
+#include <runtime/vm/rnn_state.cc>
+#include <runtime/vm/tensor_cache_support.cc>
+#include <runtime/vm/vm.cc>
+#include <runtime/workspace_pool.cc>
+
+static_assert(TVM_LOG_CUSTOMIZE == 1, "TVM_LOG_CUSTOMIZE must be 1");
+
+namespace tvm {
+namespace runtime {
+namespace detail {
+// Override logging mechanism
+[[noreturn]] void LogFatalImpl(const std::string& file, int lineno, const std::string& message) {
+  std::string m = file + ":" + std::to_string(lineno) + ": " + message;
+  __android_log_write(ANDROID_LOG_FATAL, "TVM_RUNTIME", m.c_str());
+  throw InternalError(file, lineno, message);
+}
+void LogMessageImpl(const std::string& file, int lineno, int level, const std::string& message) {
+  std::string m = file + ":" + std::to_string(lineno) + ": " + message;
+  __android_log_write(ANDROID_LOG_DEBUG + level, "TVM_RUNTIME", m.c_str());
+}
+
+}  // namespace detail
+}  // namespace runtime
+}  // namespace tvm

android/mlc4j/src/main/AndroidManifest.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+</manifest>

android/mlc4j/src/main/java/ai/mlc/mlcllm/JSONFFIEngine.java

@@ -0,0 +1,87 @@
+package ai.mlc.mlcllm;
+
+import org.apache.tvm.Device;
+import org.apache.tvm.Function;
+import org.apache.tvm.Module;
+import org.apache.tvm.TVMValue;
+import android.util.Log;
+
+public class JSONFFIEngine {
+    private Module jsonFFIEngine;
+    private Function initBackgroundEngineFunc;
+    private Function reloadFunc;
+    private Function unloadFunc;
+    private Function resetFunc;
+    private Function chatCompletionFunc;
+    private Function abortFunc;
+    private Function getLastErrorFunc;
+    private Function runBackgroundLoopFunc;
+    private Function runBackgroundStreamBackLoopFunc;
+    private Function exitBackgroundLoopFunc;
+    private Function requestStreamCallback;
+
+    public JSONFFIEngine() {
+        Function createFunc = Function.getFunction("mlc.json_ffi.CreateJSONFFIEngine");
+        assert createFunc != null;
+        jsonFFIEngine = createFunc.invoke().asModule();
+        initBackgroundEngineFunc = jsonFFIEngine.getFunction("init_background_engine");
+        reloadFunc = jsonFFIEngine.getFunction("reload");
+        unloadFunc = jsonFFIEngine.getFunction("unload");
+        resetFunc = jsonFFIEngine.getFunction("reset");
+        chatCompletionFunc = jsonFFIEngine.getFunction("chat_completion");
+        abortFunc = jsonFFIEngine.getFunction("abort");
+        getLastErrorFunc = jsonFFIEngine.getFunction("get_last_error");
+        runBackgroundLoopFunc = jsonFFIEngine.getFunction("run_background_loop");
+        runBackgroundStreamBackLoopFunc = jsonFFIEngine.getFunction("run_background_stream_back_loop");
+        exitBackgroundLoopFunc = jsonFFIEngine.getFunction("exit_background_loop");
+    }
+
+    public void initBackgroundEngine(KotlinFunction callback) {
+        Device device = Device.opencl();
+
+        requestStreamCallback = Function.convertFunc(new Function.Callback() {
+            @Override
+            public Object invoke(TVMValue... args) {
+                final String chatCompletionStreamResponsesJSONStr = args[0].asString();
+                callback.invoke(chatCompletionStreamResponsesJSONStr);
+                return 1;
+            }
+        });
+
+        initBackgroundEngineFunc.pushArg(device.deviceType).pushArg(device.deviceId).pushArg(requestStreamCallback)
+                .invoke();
+    }
+
+    public void reload(String engineConfigJSONStr) {
+        reloadFunc.pushArg(engineConfigJSONStr).invoke();
+    }
+
+    public void chatCompletion(String requestJSONStr, String requestId) {
+        chatCompletionFunc.pushArg(requestJSONStr).pushArg(requestId).invoke();
+    }
+
+    public void runBackgroundLoop() {
+        runBackgroundLoopFunc.invoke();
+    }
+
+    public void runBackgroundStreamBackLoop() {
+        runBackgroundStreamBackLoopFunc.invoke();
+    }
+
+    public void exitBackgroundLoop() {
+        exitBackgroundLoopFunc.invoke();
+    }
+
+    public void unload() {
+        unloadFunc.invoke();
+    }
+
+    public interface KotlinFunction {
+        void invoke(String arg);
+    }
+
+    public void reset() {
+        resetFunc.invoke();
+    }
+
+}

android/mlc4j/src/main/java/ai/mlc/mlcllm/MLCEngine.kt

@@ -0,0 +1,196 @@
+package ai.mlc.mlcllm
+
+import ai.mlc.mlcllm.OpenAIProtocol.*
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.channels.ReceiveChannel
+import kotlinx.coroutines.launch
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.decodeFromString
+import kotlin.concurrent.thread
+import java.util.UUID
+import java.util.logging.Logger
+
+class BackgroundWorker(private val task: () -> Unit) {
+
+    fun start() {
+        thread(start = true) {
+            task()
+        }
+    }
+}
+
+class MLCEngine {
+
+    private val state: EngineState
+    private val jsonFFIEngine: JSONFFIEngine
+    val chat: Chat
+    private val threads = mutableListOf<BackgroundWorker>()
+
+    init {
+        state = EngineState()
+        jsonFFIEngine = JSONFFIEngine()
+        chat = Chat(jsonFFIEngine, state)
+
+        jsonFFIEngine.initBackgroundEngine { result ->
+            state.streamCallback(result)
+        }
+
+        val backgroundWorker = BackgroundWorker {
+            Thread.currentThread().priority = Thread.MAX_PRIORITY
+            jsonFFIEngine.runBackgroundLoop()
+        }
+
+        val backgroundStreamBackWorker = BackgroundWorker {
+            jsonFFIEngine.runBackgroundStreamBackLoop()
+        }
+
+        threads.add(backgroundWorker)
+        threads.add(backgroundStreamBackWorker)
+
+        backgroundWorker.start()
+        backgroundStreamBackWorker.start()
+    }
+
+    fun reload(modelPath: String, modelLib: String) {
+        val engineConfig = """
+            {
+                "model": "$modelPath",
+                "model_lib": "system://$modelLib",
+                "mode": "interactive"
+            }
+        """
+        jsonFFIEngine.reload(engineConfig)
+    }
+
+    fun reset() {
+        jsonFFIEngine.reset()
+    }
+
+    fun unload() {
+        jsonFFIEngine.unload()
+    }
+}
+
+data class RequestState(
+    val request: ChatCompletionRequest,
+    val continuation: Channel<ChatCompletionStreamResponse>
+)
+
+class EngineState {
+
+    private val logger = Logger.getLogger(EngineState::class.java.name)
+    private val requestStateMap = mutableMapOf<String, RequestState>()
+
+    suspend fun chatCompletion(
+        jsonFFIEngine: JSONFFIEngine,
+        request: ChatCompletionRequest
+    ): ReceiveChannel<ChatCompletionStreamResponse> {
+        val json = Json { encodeDefaults = true }
+        val jsonRequest = json.encodeToString(request)
+        val requestID = UUID.randomUUID().toString()
+        val channel = Channel<ChatCompletionStreamResponse>(Channel.UNLIMITED)
+
+        requestStateMap[requestID] = RequestState(request, channel)
+
+        jsonFFIEngine.chatCompletion(jsonRequest, requestID)
+
+        return channel
+    }
+
+    fun streamCallback(result: String?) {
+        val json = Json { ignoreUnknownKeys = true }
+        try {
+            val responses: List<ChatCompletionStreamResponse> = json.decodeFromString(result ?: return)
+
+            responses.forEach { res ->
+                val requestState = requestStateMap[res.id] ?: return@forEach
+                GlobalScope.launch {
+
+                    res.usage?.let { finalUsage ->
+                        requestState.request.stream_options?.include_usage?.let { includeUsage ->
+                            if (includeUsage) {
+                                requestState.continuation.send(res)
+                            }
+                        }
+                        requestState.continuation.close()
+                        requestStateMap.remove(res.id)
+                    } ?: run {
+                        val sendResult = requestState.continuation.trySend(res)
+                        if (sendResult.isFailure) {
+                            // Handle the failure case if needed
+                            logger.severe("Failed to send the response: ${sendResult.exceptionOrNull()}")
+                        }
+                    }
+                }
+            }
+        } catch (e: Exception) {
+            logger.severe("Kotlin JSON parsing error: $e, jsonsrc=$result")
+        }
+    }
+}
+
+class Chat(
+    private val jsonFFIEngine: JSONFFIEngine,
+    private val state: EngineState
+) {
+    val completions = Completions(jsonFFIEngine, state)
+}
+
+class Completions(
+    private val jsonFFIEngine: JSONFFIEngine,
+    private val state: EngineState
+) {
+
+    suspend fun create(request: ChatCompletionRequest): ReceiveChannel<ChatCompletionStreamResponse> {
+        return state.chatCompletion(jsonFFIEngine, request)
+    }
+
+    suspend fun create(
+        messages: List<ChatCompletionMessage>,
+        model: String? = null,
+        frequency_penalty: Float? = null,
+        presence_penalty: Float? = null,
+        logprobs: Boolean = false,
+        top_logprobs: Int = 0,
+        logit_bias: Map<Int, Float>? = null,
+        max_tokens: Int? = null,
+        n: Int = 1,
+        seed: Int? = null,
+        stop: List<String>? = null,
+        stream: Boolean = true,
+        stream_options: StreamOptions? = null,
+        temperature: Float? = null,
+        top_p: Float? = null,
+        tools: List<ChatTool>? = null,
+        user: String? = null,
+        response_format: ResponseFormat? = null
+    ): ReceiveChannel<ChatCompletionStreamResponse> {
+        if (!stream) {
+            throw IllegalArgumentException("Only stream=true is supported in MLCKotlin")
+        }
+
+        val request = ChatCompletionRequest(
+            messages = messages,
+            model = model,
+            frequency_penalty = frequency_penalty,
+            presence_penalty = presence_penalty,
+            logprobs = logprobs,
+            top_logprobs = top_logprobs,
+            logit_bias = logit_bias,
+            max_tokens = max_tokens,
+            n = n,
+            seed = seed,
+            stop = stop,
+            stream = stream,
+            stream_options = stream_options,
+            temperature = temperature,
+            top_p = top_p,
+            tools = tools,
+            user = user,
+            response_format = response_format
+        )
+        return create(request)
+    }
+}

android/mlc4j/src/main/java/ai/mlc/mlcllm/OpenAIProtocol.kt

@@ -0,0 +1,225 @@
+package ai.mlc.mlcllm
+
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.builtins.ListSerializer
+import kotlinx.serialization.builtins.MapSerializer
+import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.descriptors.buildClassSerialDescriptor
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.jsonPrimitive
+import java.util.*
+
+// Data classes for v1/chat/completions
+// API reference: https://platform.openai.com/docs/api-reference/chat/create
+
+class OpenAIProtocol {
+    @Serializable
+    data class TopLogProbs(
+        val token: String,
+        val logprob: Float,
+        val bytes: List<Int>? = null
+    )
+
+    @Serializable
+    data class LogProbsContent(
+        val token: String,
+        val logprob: Float,
+        var bytes: List<Int>? = null,
+        var top_logprobs: List<TopLogProbs> = listOf()
+    )
+
+    @Serializable
+    data class LogProbs(
+        var content: List<LogProbsContent> = listOf()
+    )
+
+    @Serializable
+    data class ChatFunction(
+        val name: String,
+        var description: String? = null,
+        val parameters: Map<String, String>
+    )
+
+    @Serializable
+    data class ChatTool(
+        val type: String = "function",
+        val function: ChatFunction
+    )
+
+    @Serializable
+    data class ChatFunctionCall(
+        val name: String,
+        // NOTE: arguments should be dict str to any codable
+        // for now only allow string output due to typing issues
+        var arguments: Map<String, String>? = null
+    )
+
+    @Serializable
+    data class ChatToolCall(
+        val id: String = UUID.randomUUID().toString(),
+        val type: String = "function",
+        val function: ChatFunctionCall
+    )
+
+    @Serializable
+    enum class ChatCompletionRole {
+        system,
+        user,
+        assistant,
+        tool
+    }
+
+    @Serializable(with = ChatCompletionMessageContentSerializer::class)
+    data class ChatCompletionMessageContent(
+        val text: String? = null,
+        val parts: List<Map<String, String>>? = null
+    ) {
+        constructor(text: String) : this(text, null)
+        constructor(parts: List<Map<String, String>>) : this(null, parts)
+
+        fun isText(): Boolean {
+            return text != null
+        }
+
+        fun isParts(): Boolean {
+            return parts != null
+        }
+
+        fun asText(): String {
+            return text ?: (parts?.filter { it["type"] == "text" }?.joinToString("") { it["text"] ?: "" } ?: "")
+        }
+    }
+
+    object ChatCompletionMessageContentSerializer : KSerializer<ChatCompletionMessageContent> {
+        override val descriptor: SerialDescriptor = buildClassSerialDescriptor("ChatCompletionMessageContent") {
+            element("text", String.serializer().descriptor)
+            element("parts", ListSerializer(MapSerializer(String.serializer(), String.serializer())).descriptor)
+        }
+
+        override fun serialize(encoder: Encoder, value: ChatCompletionMessageContent) {
+            if (value.isText()) {
+                encoder.encodeString(value.text!!)
+            } else {
+                encoder.encodeSerializableValue(ListSerializer(MapSerializer(String.serializer(), String.serializer())), value.parts ?: listOf())
+            }
+        }
+
+        override fun deserialize(decoder: Decoder): ChatCompletionMessageContent {
+            return when (val element = decoder.decodeSerializableValue(JsonElement.serializer())) {
+                is JsonArray -> {
+                    val parts = element.map { (it as JsonObject).map { entry -> entry.key to entry.value.jsonPrimitive.content }.toMap() }
+                    ChatCompletionMessageContent(parts)
+                }
+                is JsonPrimitive -> {
+                    ChatCompletionMessageContent(element.content)
+                }
+                else -> throw IllegalStateException("Unexpected JsonElement type")
+            }
+        }
+    }
+
+    @Serializable
+    data class ChatCompletionMessage(
+        val role: ChatCompletionRole,
+        var content: ChatCompletionMessageContent? = null,
+        var name: String? = null,
+        var tool_calls: List<ChatToolCall>? = null,
+        var tool_call_id: String? = null
+    ) {
+        constructor(
+            role: ChatCompletionRole,
+            content: String,
+            name: String? = null,
+            tool_calls: List<ChatToolCall>? = null,
+            tool_call_id: String? = null
+        ) : this(role, ChatCompletionMessageContent(content), name, tool_calls, tool_call_id)
+    }
+
+    @Serializable
+    data class CompletionUsageExtra(
+        val prefill_tokens_per_s: Float? = null,
+        val decode_tokens_per_s: Float? = null,
+        val num_prefill_tokens: Int? = null
+    ) {
+        fun asTextLabel(): String {
+            var outputText = ""
+            if (prefill_tokens_per_s != null) {
+                outputText += "prefill: ${String.format("%.1f", prefill_tokens_per_s)} tok/s"
+            }
+            if (decode_tokens_per_s != null) {
+                if (outputText.isNotEmpty()) {
+                    outputText += ", "
+                }
+                outputText += "decode: ${String.format("%.1f", decode_tokens_per_s)} tok/s"
+            }
+            return outputText
+        }
+    }
+
+    @Serializable
+    data class CompletionUsage(
+        val prompt_tokens: Int,
+        val completion_tokens: Int,
+        val total_tokens: Int,
+        val extra: CompletionUsageExtra? = null
+    )
+
+    @Serializable
+    data class StreamOptions(
+        val include_usage: Boolean = false
+    )
+
+    @Serializable
+    data class ChatCompletionStreamResponseChoice(
+        var finish_reason: String? = null,
+        val index: Int,
+        val delta: ChatCompletionMessage,
+        var lobprobs: LogProbs? = null
+    )
+
+    @Serializable
+    data class ChatCompletionStreamResponse(
+        val id: String,
+        var choices: List<ChatCompletionStreamResponseChoice> = listOf(),
+        var created: Int? = null,
+        var model: String? = null,
+        val system_fingerprint: String,
+        var `object`: String? = null,
+        val usage: CompletionUsage? = null
+    )
+
+    @Serializable
+    data class ChatCompletionRequest(
+        val messages: List<ChatCompletionMessage>,
+        val model: String? = null,
+        val frequency_penalty: Float? = null,
+        val presence_penalty: Float? = null,
+        val logprobs: Boolean = false,
+        val top_logprobs: Int = 0,
+        val logit_bias: Map<Int, Float>? = null,
+        val max_tokens: Int? = null,
+        val n: Int = 1,
+        val seed: Int? = null,
+        val stop: List<String>? = null,
+        val stream: Boolean = true,
+        val stream_options: StreamOptions? = null,
+        val temperature: Float? = null,
+        val top_p: Float? = null,
+        val tools: List<ChatTool>? = null,
+        val user: String? = null,
+        val response_format: ResponseFormat? = null
+    )
+
+    @Serializable
+    data class ResponseFormat(
+        val type: String,
+        val schema: String? = null
+    )
+}

ci/bash.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+if [ "$#" -lt 1 ]; then
+    echo "Usage: ci/bash.sh <CONTAINER_NAME> -e key value -v key value [COMMAND]"
+    exit -1
+fi
+
+DOCKER_IMAGE_NAME=("$1")
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORKSPACE="$(pwd)"
+DOCKER_BINARY="docker"
+DOCKER_ENV="-e ENV_USER_ID=$(id -u) -e ENV_GROUP_ID=$(id -g)"
+DOCKER_VOLUMNS="-v ${WORKSPACE}:/workspace -v ${SCRIPT_DIR}:/docker"
+
+shift 1
+while [[ $# -gt 0 ]]; do
+    cmd="$1"
+    if [[ $cmd == "-e" ]]; then
+        env_key=$2
+        env_value=$3
+        shift 3
+        DOCKER_ENV="${DOCKER_ENV} -e ${env_key}=${env_value}"
+    elif [[ $cmd == "-v" ]]; then
+        volumn_key=$2
+        volumn_value=$3
+        shift 3
+        DOCKER_VOLUMNS="${DOCKER_VOLUMNS} -v ${volumn_key}:${volumn_value}"
+    elif [[ $cmd == "-j" ]]; then
+        num_threads=$2
+        shift 2
+        DOCKER_ENV="${DOCKER_ENV} -e NUM_THREADS=${num_threads} --cpus ${num_threads}"
+    else
+        break
+    fi
+done
+
+if [ "$#" -eq 0 ]; then
+    COMMAND="bash"
+    if [[ $(uname) == "Darwin" ]]; then
+        # Docker's host networking driver isn't supported on macOS.
+        # Use default bridge network and expose port for jupyter notebook.
+        DOCKER_EXTRA_PARAMS=("-it -p 8888:8888")
+    else
+        DOCKER_EXTRA_PARAMS=("-it --net=host")
+    fi
+else
+    COMMAND=("$@")
+fi
+
+if [[ -n ${MLC_CI_SETUP_DEPS:-} ]]; then
+    DOCKER_ENV="${DOCKER_ENV} -e MLC_CI_SETUP_DEPS=${MLC_CI_SETUP_DEPS}"
+fi
+
+# Use nvidia-docker if the container is GPU.
+if [[ -n ${CUDA_VISIBLE_DEVICES:-} ]]; then
+    DOCKER_ENV="${DOCKER_ENV} -e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}"
+    if type nvidia-docker 1> /dev/null 2> /dev/null; then
+        DOCKER_BINARY=nvidia-docker
+    else
+        DOCKER_BINARY=docker
+        DOCKER_ENV="${DOCKER_ENV} --gpus all"
+    fi
+
+    # nvidia-docker treats Vulkan as a graphics API, so we need to
+    # request passthrough of graphics APIs.  This could also be set in
+    # the Dockerfile.
+    DOCKER_ENV="${DOCKER_ENV} -e NVIDIA_DRIVER_CAPABILITIES=compute,graphics,utility"
+
+    # vulkan comaptibility
+    ICD_SEARCH_LOCATIONS=(
+        # https://github.com/KhronosGroup/Vulkan-Loader/blob/master/loader/LoaderAndLayerInterface.md#icd-discovery-on-linux
+        /usr/local/etc/vulkan/icd.d
+        /usr/local/share/vulkan/icd.d
+        /etc/vulkan/icd.d
+        /usr/share/vulkan/icd.d
+        /etc/glvnd/egl_vendor.d
+        /usr/share/glvnd/egl_vendor.d
+    )
+    for filename in $(find "${ICD_SEARCH_LOCATIONS[@]}" -name "*nvidia*.json" 2> /dev/null); do
+    DOCKER_VOLUMNS="${DOCKER_VOLUMNS} -v ${filename}:${filename}:ro"
+    done
+fi
+
+# Print arguments.
+echo "DOCKER_BINARY ${DOCKER_BINARY}"
+echo "WORKSPACE: ${WORKSPACE}"
+echo "IMAGE NAME: ${DOCKER_IMAGE_NAME}"
+echo "ENV VARIABLES: ${DOCKER_ENV}"
+echo "VOLUMES: ${DOCKER_VOLUMNS}"
+echo "COMMANDS: '${COMMAND[@]}'"
+
+# By default we cleanup - remove the container once it finish running (--rm)
+# and share the PID namespace (--pid=host) so the process inside does not have
+# pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
+
+${DOCKER_BINARY} run --rm --pid=host \
+    -w /workspace \
+    ${DOCKER_VOLUMNS} \
+    ${DOCKER_ENV} \
+    ${DOCKER_EXTRA_PARAMS[@]} \
+    ${DOCKER_IMAGE_NAME} \
+    ${COMMAND[@]}

ci/build-environment.yaml

@@ -0,0 +1,18 @@
+name: mlc-llm-build
+
+channels:
+  - conda-forge
+
+dependencies:
+  # Pin Python: the unpinned env resolved to 3.10, but core deps (numpy,
+  # pandas, networkx, ...) have dropped 3.10 wheels.
+  - python=3.11
+  - conda-build
+  - anaconda-client
+  - libvulkan-headers
+  - libvulkan-loader
+  - spirv-tools
+  - spirv-headers
+  - git
+  - cmake<4.0
+  - bzip2

ci/jenkinsfile.groovy

@@ -0,0 +1,217 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
+
+run_cpu = "bash ci/bash.sh mlcaidev/ci-cpu:26d65cc -e GPU cpu -e MLC_CI_SETUP_DEPS 1"
+run_cuda = "bash ci/bash.sh mlcaidev/ci-cu128:26d65cc -e GPU cuda-12.8 -e MLC_CI_SETUP_DEPS 1"
+// run_rocm = "bash ci/bash.sh mlcaidev/ci-rocm57:26d65cc -e GPU rocm-5.7 -e MLC_CI_SETUP_DEPS 1"
+
+pkg_cpu = "bash ci/bash.sh mlcaidev/package-rocm61:a86c2a7 -e GPU cpu -e MLC_CI_SETUP_DEPS 1"
+pkg_cuda = "bash ci/bash.sh mlcaidev/package-cu128:a86c2a7 -e GPU cuda-12.8 -e MLC_CI_SETUP_DEPS 1"
+pkg_rocm = "bash ci/bash.sh mlcaidev/package-rocm61:a86c2a7 -e GPU rocm-6.1 -e MLC_CI_SETUP_DEPS 1"
+
+
+def per_exec_ws(folder) {
+  return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
+}
+
+def pack_lib(name, libs) {
+  sh """
+     echo "Packing ${libs} into ${name}"
+     echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+     """
+  stash includes: libs, name: name
+}
+
+def unpack_lib(name, libs) {
+  unstash name
+  sh """
+     echo "Unpacked ${libs} from ${name}"
+     echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+     """
+}
+
+def init_git(submodule = false) {
+  cleanWs()
+  // add retry in case checkout timeouts
+  retry(5) {
+    checkout scm
+  }
+  if (submodule) {
+    retry(5) {
+      timeout(time: 10, unit: 'MINUTES') {
+        sh(script: 'git submodule update --init --recursive -f', label: 'Update git submodules')
+      }
+    }
+  }
+}
+
+stage('Build') {
+  parallel(
+    'CUDA': {
+      node('CPU-SMALL') {
+        ws(per_exec_ws('mlc-llm-build-cuda')) {
+          init_git(true)
+          sh(script: "ls -alh", label: 'Show work directory')
+          sh(script: "${pkg_cuda} conda env export --name py313", label: 'Checkout version')
+          sh(script: "${pkg_cuda} -j 8 -v \$HOME/.ccache /ccache conda run -n py313 ./ci/task/build_lib.sh", label: 'Build MLC LLM runtime')
+          sh(script: "${pkg_cuda} -j 1 conda run -n py313 ./ci/task/build_clean.sh", label: 'Clean up after build')
+          sh(script: "ls -alh ./wheels/", label: 'Build artifact')
+          pack_lib('mlc_wheel_cuda', 'wheels/*.whl')
+        }
+      }
+    },
+    // 'ROCm': {
+    //   node('CPU-SMALL') {
+    //     ws(per_exec_ws('mlc-llm-build-rocm')) {
+    //       init_git(true)
+    //       sh(script: "ls -alh", label: 'Show work directory')
+    //       sh(script: "${pkg_rocm} conda env export --name py38", label: 'Checkout version')
+    //       sh(script: "${pkg_rocm} -j 8 conda run -n py38 ./ci/task/build_lib.sh", label: 'Build MLC LLM runtime')
+    //       sh(script: "${pkg_rocm} -j 1 conda run -n py38 ./ci/task/build_clean.sh", label: 'Clean up after build')
+    //       sh(script: "ls -alh ./wheels/", label: 'Build artifact')
+    //       pack_lib('mlc_wheel_rocm', 'wheels/*.whl')
+    //     }
+    //   }
+    // },
+    'Metal': {
+      node('MAC') {
+        ws(per_exec_ws('mlc-llm-build-metal')) {
+          init_git(true)
+          sh(script: "ls -alh", label: 'Show work directory')
+          sh(script: "conda env export --name mlc-llm-ci", label: 'Checkout version')
+          sh(script: "NUM_THREADS=6 GPU=metal conda run -n mlc-llm-ci ./ci/task/build_lib.sh", label: 'Build MLC LLM runtime')
+          sh(script: "NUM_THREADS=6 GPU=metal conda run -n mlc-llm-ci ./ci/task/build_clean.sh", label: 'Clean up after build')
+          sh(script: "ls -alh ./wheels/", label: 'Build artifact')
+          pack_lib('mlc_wheel_metal', 'wheels/*.whl')
+        }
+      }
+    },
+    'Vulkan': {
+      node('CPU-SMALL') {
+        ws(per_exec_ws('mlc-llm-build-vulkan')) {
+          init_git(true)
+          sh(script: "ls -alh", label: 'Show work directory')
+          sh(script: "${pkg_cpu} conda env export --name py313", label: 'Checkout version')
+          sh(script: "${pkg_cpu} -j 8 conda run -n py313 ./ci/task/build_lib.sh", label: 'Build MLC LLM runtime')
+          sh(script: "${pkg_cpu} -j 1 conda run -n py313 ./ci/task/build_clean.sh", label: 'Clean up after build')
+          sh(script: "ls -alh ./wheels/", label: 'Build artifact')
+          pack_lib('mlc_wheel_vulkan', 'wheels/*.whl')
+        }
+      }
+    }
+  )
+}
+
+stage('Unittest') {
+  parallel(
+    'CUDA': {
+      node('GPU') {
+        ws(per_exec_ws('mlc-llm-unittest')) {
+          init_git(false)
+          sh(script: "ls -alh", label: 'Show work directory')
+          unpack_lib('mlc_wheel_cuda', 'wheels/*.whl')
+          sh(script: "${run_cuda} conda env export --name ci-unittest", label: 'Checkout version')
+          sh(script: "${run_cuda} conda run -n ci-unittest ./ci/task/test_unittest.sh", label: 'Testing')
+        }
+      }
+    }
+  )
+}
+
+stage('Model Compilation') {
+  parallel(
+    'CUDA': {
+      node('CPU-SMALL') {
+        ws(per_exec_ws('mlc-llm-compile-cuda')) {
+          init_git(false)
+          sh(script: "ls -alh", label: 'Show work directory')
+          unpack_lib('mlc_wheel_cuda', 'wheels/*.whl')
+          sh(script: "${run_cuda} conda env export --name ci-unittest", label: 'Checkout version')
+          sh(script: "${run_cuda} -j 4 conda run -n ci-unittest ./ci/task/test_model_compile.sh", label: 'Testing')
+        }
+      }
+    },
+    // 'ROCm': {
+    //   node('CPU-SMALL') {
+    //     ws(per_exec_ws('mlc-llm-compile-rocm')) {
+    //       init_git(false)
+    //       sh(script: "ls -alh", label: 'Show work directory')
+    //       unpack_lib('mlc_wheel_rocm', 'wheels/*.whl')
+    //       sh(script: "${run_rocm} conda env export --name ci-unittest", label: 'Checkout version')
+    //       sh(script: "${run_rocm} -j 4 conda run -n ci-unittest ./ci/task/test_model_compile.sh", label: 'Testing')
+    //     }
+    //   }
+    // },
+    'Metal': {
+      node('MAC') {
+        ws(per_exec_ws('mlc-llm-compile-metal')) {
+          init_git(false)
+          sh(script: "ls -alh", label: 'Show work directory')
+          unpack_lib('mlc_wheel_metal', 'wheels/*.whl')
+          sh(script: "conda env export --name mlc-llm-ci", label: 'Checkout version')
+          sh(script: "NUM_THREADS=6 GPU=metal conda run -n mlc-llm-ci ./ci/task/test_model_compile.sh", label: 'Testing')
+        }
+      }
+    },
+    'Vulkan': {
+      node('CPU-SMALL') {
+        ws(per_exec_ws('mlc-llm-compile-vulkan')) {
+          init_git(false)
+          sh(script: "ls -alh", label: 'Show work directory')
+          unpack_lib('mlc_wheel_vulkan', 'wheels/*.whl')
+          sh(script: "${run_cpu} conda env export --name ci-unittest", label: 'Checkout version')
+          // sh(script: "${run_cpu} -j 4 conda run -n ci-unittest ./ci/task/test_model_compile.sh", label: 'Testing')
+        }
+      }
+    },
+    'WASM': {
+      node('CPU-SMALL') {
+        ws(per_exec_ws('mlc-llm-compile-wasm')) {
+          init_git(true)
+          sh(script: "ls -alh", label: 'Show work directory')
+          unpack_lib('mlc_wheel_vulkan', 'wheels/*.whl')
+          sh(script: "${run_cpu} conda env export --name ci-unittest", label: 'Checkout version')
+          sh(script: "${run_cpu} -j 8 -e GPU wasm conda run -n ci-unittest ./ci/task/test_model_compile.sh", label: 'Testing')
+        }
+      }
+    },
+    'iOS': {
+      node('MAC') {
+        ws(per_exec_ws('mlc-llm-compile-ios')) {
+          init_git(false)
+          sh(script: "ls -alh", label: 'Show work directory')
+          unpack_lib('mlc_wheel_metal', 'wheels/*.whl')
+          sh(script: "conda env export --name mlc-llm-ci", label: 'Checkout version')
+          sh(script: "NUM_THREADS=6 GPU=ios conda run -n mlc-llm-ci ./ci/task/test_model_compile.sh", label: 'Testing')
+        }
+      }
+    },
+    'Android-OpenCL': {
+      node('CPU-SMALL') {
+        ws(per_exec_ws('mlc-llm-compile-android')) {
+          init_git(false)
+          sh(script: "ls -alh", label: 'Show work directory')
+          unpack_lib('mlc_wheel_vulkan', 'wheels/*.whl')
+          sh(script: "${run_cpu} conda env export --name ci-unittest", label: 'Checkout version')
+          sh(script: "${run_cpu} -j 4 -e GPU android conda run -n ci-unittest ./ci/task/test_model_compile.sh", label: 'Testing')
+        }
+      }
+    }
+  )
+}

ci/task/build_clean.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+: ${NUM_THREADS:=$(nproc)}
+: ${WORKSPACE_CWD:=$(pwd)}
+: ${GPU:="cpu"}
+
+rm -rf ${WORKSPACE_CWD}/build/ \
+    ${WORKSPACE_CWD}/dist/

ci/task/build_lib.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+: ${NUM_THREADS:=$(nproc)}
+: ${WORKSPACE_CWD:=$(pwd)}
+: ${GPU:="cpu"}
+export CCACHE_COMPILERCHECK=content
+export CCACHE_NOHASHDIR=1
+export CCACHE_DIR=/ccache
+
+# Temporary workaround to install ccache.
+if [[ ${GPU} != metal ]]; then
+    conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
+    conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
+fi
+conda install -c conda-forge ccache
+
+if [[ ${GPU} != metal ]]; then
+    source /multibuild/manylinux_utils.sh
+    source /opt/rh/gcc-toolset-11/enable # GCC-11 is the hightest GCC version compatible with NVCC < 12
+fi
+
+mkdir -p $WORKSPACE_CWD/build
+if [[ ${GPU} == rocm* ]]; then
+    echo set\(USE_VULKAN ON\) >>config.cmake
+    echo set\(USE_ROCM ON\) >>config.cmake
+    echo set\(USE_RCCL /opt/rocm/rccl/ \) >>config.cmake
+elif [[ ${GPU} == cuda* ]]; then
+    echo set\(USE_VULKAN ON\) >>config.cmake
+    echo set\(CMAKE_CUDA_COMPILER_LAUNCHER ccache\) >>config.cmake
+    echo set\(CMAKE_CUDA_ARCHITECTURES "80;90;100;120"\) >>config.cmake
+    echo set\(CMAKE_CUDA_FLAGS \"\$\{CMAKE_CUDA_FLAGS\} -t $NUM_THREADS\"\) >>config.cmake
+    echo set\(USE_CUDA ON\) >>config.cmake
+    echo set\(USE_CUBLAS ON\) >>config.cmake
+    echo set\(USE_NCCL ON\) >>config.cmake
+elif [[ ${GPU} == metal ]]; then
+    export CCACHE_DIR=$HOME/ci/ccache
+    echo set\(USE_METAL ON\) >>config.cmake
+else
+    echo set\(USE_VULKAN ON\) >>config.cmake
+fi
+
+cat config.cmake
+
+AUDITWHEEL_OPTS="--plat ${AUDITWHEEL_PLAT} -w repaired_wheels/"
+AUDITWHEEL_OPTS="--exclude libtvm --exclude libtvm_runtime --exclude libtvm_ffi --exclude libvulkan ${AUDITWHEEL_OPTS}"
+if [[ ${GPU} == rocm* ]]; then
+    AUDITWHEEL_OPTS="--exclude libamdhip64 --exclude libhsa-runtime64 --exclude librocm_smi64 --exclude librccl ${AUDITWHEEL_OPTS}"
+elif [[ ${GPU} == cuda* ]]; then
+    AUDITWHEEL_OPTS="--exclude libcuda --exclude libcudart --exclude libnvrtc --exclude libcublas --exclude libcublasLt ${AUDITWHEEL_OPTS}"
+fi
+
+rm -rf ${WORKSPACE_CWD}/dist
+cd ${WORKSPACE_CWD} && pip wheel --no-deps -w dist . -v
+
+rm -rf ${WORKSPACE_CWD}/wheels/
+if [[ ${GPU} != metal ]]; then
+    mkdir -p ${WORKSPACE_CWD}/repaired_wheels
+    rm -rf ${WORKSPACE_CWD}/repaired_wheels/*
+    auditwheel repair ${AUDITWHEEL_OPTS} dist/*.whl
+    mv ${WORKSPACE_CWD}/repaired_wheels/ ${WORKSPACE_CWD}/wheels/
+else
+    mkdir ${WORKSPACE_CWD}/wheels/
+    mv dist/*.whl ${WORKSPACE_CWD}/wheels/
+fi
+
+chown -R $ENV_USER_ID:$ENV_GROUP_ID ${WORKSPACE_CWD}/wheels/

ci/task/build_win.bat

@@ -0,0 +1,14 @@
+cd mlc-llm
+rd /s /q build
+mkdir build
+
+echo set(USE_VULKAN ON) >> config.cmake
+
+REM Conda ships a GNU coreutils link.exe in %CONDA_PREFIX%\Library\usr\bin that
+REM shadows the MSVC linker on PATH and breaks the Rust (cargo) build of the
+REM tokenizers crate. Remove it so cargo/rustc pick up the MSVC link.exe.
+if exist "%CONDA_PREFIX%\Library\usr\bin\link.exe" del /f /q "%CONDA_PREFIX%\Library\usr\bin\link.exe"
+
+pip install . -v
+
+if %errorlevel% neq 0 exit %errorlevel%

ci/task/test_model_compile.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+: ${NUM_THREADS:=$(nproc)}
+: ${WORKSPACE_CWD:=$(pwd)}
+: ${GPU:="cpu"}
+
+pip install --force-reinstall wheels/*.whl
+pip install --quiet pytest
+
+if [[ ${GPU} == cuda* ]]; then
+    TARGET=cuda
+    pip install --pre -U --no-index -f https://mlc.ai/wheels mlc-ai-nightly-cu128
+    export LD_LIBRARY_PATH=/usr/local/cuda/compat/:$LD_LIBRARY_PATH
+elif [[ ${GPU} == rocm* ]]; then
+    TARGET=rocm
+    pip install --pre -U --no-index -f https://mlc.ai/wheels mlc-ai-nightly-rocm57
+elif [[ ${GPU} == metal ]]; then
+    TARGET=metal
+    pip install --pre -U --force-reinstall -f https://mlc.ai/wheels mlc-ai-nightly-cpu
+elif [[ ${GPU} == wasm* ]]; then
+    TARGET=wasm
+    # Clone a copy a tvm source code to build tvm web runtime
+    git clone https://github.com/mlc-ai/relax.git /tmp/tvm --recursive
+    export TVM_SOURCE_DIR=/tmp/tvm
+    # Pip install tvm so that `import tvm` in Python works
+    pip install --pre -U --no-index -f https://mlc.ai/wheels mlc-ai-nightly-cpu
+    export TVM_HOME=${TVM_SOURCE_DIR}
+    export MLC_LLM_SOURCE_DIR=$(pwd)
+    cd $TVM_SOURCE_DIR/web/ && make -j${NUM_THREADS} && cd -
+    cd $MLC_LLM_SOURCE_DIR/web/ && make -j${NUM_THREADS} && cd -
+elif [[ ${GPU} == ios ]]; then
+    TARGET=ios
+    pip install --pre -U --force-reinstall -f https://mlc.ai/wheels mlc-ai-nightly-cpu
+elif [[ ${GPU} == android* ]]; then
+    TARGET=android
+    pip install --pre -U --no-index -f https://mlc.ai/wheels mlc-ai-nightly-cpu
+else
+    TARGET=vulkan
+    pip install --pre -U --no-index -f https://mlc.ai/wheels mlc-ai-nightly-cpu
+fi
+
+python tests/python/integration/test_model_compile.py $TARGET $NUM_THREADS

ci/task/test_unittest.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eo pipefail
+set -x
+
+# this scripts only triggers in CI_ENV where these environment variable are passed
+if [[ -n ${MLC_CI_SETUP_DEPS:-} ]]; then
+    echo "MLC_CI_SETUP_DEPS=1 start setup deps.."
+    # Install dependency
+    pip install --force-reinstall wheels/*.whl
+    pip install --quiet pytest
+    pip install --pre -U --no-index -f https://mlc.ai/wheels mlc-ai-nightly-cu128
+    export LD_LIBRARY_PATH=/usr/local/cuda/compat/:$LD_LIBRARY_PATH
+fi
+
+# run all tests that are categorized as "unittest"
+# add pytestmarker = [pytest.mark.unittest] in the test file
+# so they will be run here
+python -m pytest -v tests/python/ -m unittest

cmake/gen_cmake_config.py

@@ -0,0 +1,59 @@
+from collections import namedtuple
+
+Backend = namedtuple("Backend", ["name", "cmake_config_name", "prompt_str", "parent"])
+
+if __name__ == "__main__":
+    tvm_home = ""
+
+    tvm_home = input(
+        "Enter TVM_SOURCE_DIR in absolute path. If not specified, 3rdparty/tvm will be used by default: "  # noqa: E501
+    )
+    if len(tvm_home) == 0:
+        tvm_home = "3rdparty/tvm"
+
+    cmake_config_str = f"set(TVM_SOURCE_DIR {tvm_home})\n"
+    cmake_config_str += "set(CMAKE_BUILD_TYPE RelWithDebInfo)\n"
+    cuda_backend = Backend("CUDA", "USE_CUDA", "Use CUDA? (y/n): ", None)
+    opencl_backend = Backend("OpenCL", "USE_OPENCL", "Use OpenCL? (y/n) ", None)
+    backends = [
+        cuda_backend,
+        Backend("CUTLASS", "USE_CUTLASS", "Use CUTLASS? (y/n): ", cuda_backend),
+        Backend("CUBLAS", "USE_CUBLAS", "Use CUBLAS? (y/n): ", cuda_backend),
+        Backend("ROCm", "USE_ROCM", "Use ROCm? (y/n): ", None),
+        Backend("Vulkan", "USE_VULKAN", "Use Vulkan? (y/n): ", None),
+        Backend("Metal", "USE_METAL", "Use Metal (Apple M1/M2 GPU) ? (y/n): ", None),
+        opencl_backend,
+        Backend(
+            "OpenCLHostPtr",
+            "USE_OPENCL_ENABLE_HOST_PTR",
+            "Use OpenCLHostPtr? (y/n): ",
+            opencl_backend,
+        ),
+    ]
+
+    enabled_backends = set()
+
+    for backend in backends:
+        if backend.parent is not None and backend.parent.name not in enabled_backends:
+            cmake_config_str += f"set({backend.cmake_config_name} OFF)\n"
+        else:
+            while True:
+                use_backend = input(backend.prompt_str)
+                if use_backend in ["yes", "Y", "y"]:
+                    cmake_config_str += f"set({backend.cmake_config_name} ON)\n"
+                    enabled_backends.add(backend.name)
+                    break
+                elif use_backend in ["no", "N", "n"]:
+                    cmake_config_str += f"set({backend.cmake_config_name} OFF)\n"
+                    break
+                else:
+                    print(f"Invalid input: {use_backend}. Please input again.")
+
+    if "CUDA" in enabled_backends:
+        cmake_config_str += "set(USE_THRUST ON)\n"
+
+    print("\nWriting the following configuration to config.cmake...")
+    print(cmake_config_str)
+
+    with open("config.cmake", "w") as f:
+        f.write(cmake_config_str)

cpp/base.h

@@ -0,0 +1,16 @@
+/*!
+ *  Copyright (c) 2023-2025 by Contributors
+ * \file base.h
+ */
+
+#ifndef MLC_LLM_DLL
+#ifdef _WIN32
+#ifdef MLC_LLM_EXPORTS
+#define MLC_LLM_DLL __declspec(dllexport)
+#else
+#define MLC_LLM_DLL __declspec(dllimport)
+#endif
+#else
+#define MLC_LLM_DLL __attribute__((visibility("default")))
+#endif
+#endif