chore: import upstream snapshot with attribution

tests/python/serve/server/conftest.py

@@ -0,0 +1,34 @@
+import os
+from typing import Tuple  # noqa: UP035
+
+import pytest
+
+from mlc_llm.serve import PopenServer
+
+
+@pytest.fixture(scope="session")
+def served_model() -> Tuple[str, str]:  # noqa: UP006
+    model_lib = os.environ.get("MLC_SERVE_MODEL_LIB")
+    if model_lib is None:
+        raise ValueError(
+            'Environment variable "MLC_SERVE_MODEL_LIB" not found. '
+            "Please set it to model lib compiled by MLC LLM "
+            "(e.g., `dist/Llama-2-7b-chat-hf-q0f16-MLC/Llama-2-7b-chat-hf-q0f16-MLC-cuda.so`)."
+        )
+    model = os.path.dirname(model_lib)
+    return model, model_lib
+
+
+@pytest.fixture(scope="session")
+def launch_server(served_model):
+    """A pytest session-level fixture which launches the server in a subprocess."""
+    server = PopenServer(
+        model=served_model[0],
+        model_lib=served_model[1],
+        enable_tracing=True,
+        enable_debug=True,
+        port=8000,
+    )
+
+    with server:
+        yield

tests/python/serve/server/test_embedding_server.py

@@ -0,0 +1,369 @@
+"""Embedding server endpoint tests in MLC LLM.
+
+Tests the /v1/embeddings endpoint via HTTP using the OpenAI client,
+following the same patterns as test_server.py.
+
+Reuses MLC LLM test infrastructure:
+  - Pytest markers (endpoint)
+  - expect_error() response validation pattern from test_server.py
+  - OpenAI client usage pattern from test_server.py
+  - Session-scoped server fixture pattern from conftest.py
+
+Run (launches its own embedding-only server):
+  MLC_SERVE_EMBEDDING_MODEL_LIB="path/to/model.dylib" \
+    pytest -m endpoint tests/python/serve/server/test_embedding_server.py -v
+
+Environment variables:
+  MLC_SERVE_EMBEDDING_MODEL_LIB  Path to compiled embedding model library (required)
+  MLC_SERVE_EMBEDDING_MODEL      Path to embedding model weight directory
+                                  (optional, defaults to dirname of model lib)
+"""
+
+import json
+import os
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Dict, Optional  # noqa: UP035
+
+import numpy as np
+import pytest
+import requests
+from openai import OpenAI
+
+# Reuse MLC LLM marker system
+pytestmark = [pytest.mark.endpoint]
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+EMBEDDING_MODEL_LIB = os.environ.get("MLC_SERVE_EMBEDDING_MODEL_LIB")
+EMBEDDING_MODEL_DIR = os.environ.get(
+    "MLC_SERVE_EMBEDDING_MODEL",
+    os.path.dirname(EMBEDDING_MODEL_LIB) if EMBEDDING_MODEL_LIB else None,
+)
+EMBEDDING_SERVER_HOST = "127.0.0.1"
+EMBEDDING_SERVER_PORT = 8321
+EMBEDDING_BASE_URL = f"http://{EMBEDDING_SERVER_HOST}:{EMBEDDING_SERVER_PORT}/v1"
+EMBEDDING_MODEL_NAME = "embedding"
+
+
+def _skip_if_no_model():
+    if EMBEDDING_MODEL_LIB is None:
+        pytest.skip(
+            'Environment variable "MLC_SERVE_EMBEDDING_MODEL_LIB" not found. '
+            "Set it to a compiled embedding model library."
+        )
+    if not os.path.isfile(EMBEDDING_MODEL_LIB):
+        pytest.skip(f"Embedding model library not found at: {EMBEDDING_MODEL_LIB}")
+    if EMBEDDING_MODEL_DIR is None or not os.path.isdir(EMBEDDING_MODEL_DIR):
+        pytest.skip(f"Embedding model directory not found at: {EMBEDDING_MODEL_DIR}")
+
+
+# ---------------------------------------------------------------------------
+# Response validation helpers — adapted from test_server.py patterns
+# ---------------------------------------------------------------------------
+
+
+def check_embedding_response(
+    response: Dict,  # noqa: UP006
+    *,
+    model: str,
+    num_embeddings: int,
+    expected_dim: Optional[int] = None,
+    check_unit_norm: bool = True,
+):
+    """Validate an OpenAI-compatible embedding response.
+
+    Adapted from check_openai_nonstream_response() in test_server.py,
+    specialized for embedding responses.
+    """
+    assert response["object"] == "list"
+    assert response["model"] == model
+
+    data = response["data"]
+    assert isinstance(data, list)
+    assert len(data) == num_embeddings
+
+    for item in data:
+        assert item["object"] == "embedding"
+        assert isinstance(item["index"], int)
+        emb = item["embedding"]
+        assert isinstance(emb, list)
+        assert len(emb) > 0
+
+        if expected_dim is not None:
+            assert len(emb) == expected_dim, f"Expected dim={expected_dim}, got {len(emb)}"
+
+        if check_unit_norm:
+            norm = float(np.linalg.norm(emb))
+            assert abs(norm - 1.0) < 1e-3, f"Expected unit norm, got {norm}"
+
+    # Usage validation — same pattern as test_server.py
+    usage = response["usage"]
+    assert isinstance(usage, dict)
+    assert usage["prompt_tokens"] > 0
+    assert usage["total_tokens"] == usage["prompt_tokens"]
+
+
+def expect_error(response_str: str, msg_prefix: Optional[str] = None):
+    """Validate error response — reused directly from test_server.py."""
+    response = json.loads(response_str)
+    assert response["object"] == "error"
+    assert isinstance(response["message"], str)
+    if msg_prefix is not None:
+        assert response["message"].startswith(msg_prefix)
+
+
+# ---------------------------------------------------------------------------
+# Server fixture — follows PopenServer/launch_server pattern from conftest.py
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture(scope="module")
+def launch_embedding_server():
+    """Launch an embedding-only server as a subprocess.
+
+    Follows the same lifecycle pattern as the launch_server fixture
+    in serve/server/conftest.py, but uses a lightweight embedding-only
+    server since PopenServer doesn't support --embedding-model yet.
+    """
+    _skip_if_no_model()
+
+    mlc_llm_path = str(Path(__file__).resolve().parents[4] / "python")
+    server_code = f"""
+import sys
+sys.path.insert(0, "{mlc_llm_path}")
+
+import fastapi
+import uvicorn
+from mlc_llm.serve.embedding_engine import AsyncEmbeddingEngine
+from mlc_llm.serve.server import ServerContext
+from mlc_llm.serve.entrypoints import openai_entrypoints
+
+app = fastapi.FastAPI()
+app.include_router(openai_entrypoints.app)
+
+engine = AsyncEmbeddingEngine(
+    model="{EMBEDDING_MODEL_DIR}",
+    model_lib="{EMBEDDING_MODEL_LIB}",
+    device="auto",
+)
+ctx = ServerContext()
+ServerContext.server_context = ctx
+ctx.add_embedding_engine("{EMBEDDING_MODEL_NAME}", engine)
+
+uvicorn.run(app, host="{EMBEDDING_SERVER_HOST}", port={EMBEDDING_SERVER_PORT}, log_level="info")
+"""
+    with subprocess.Popen(
+        [sys.executable, "-c", server_code],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    ) as proc:
+        # Wait for server readiness — same polling pattern as PopenServer.start()
+        timeout = 120
+        attempts = 0.0
+        ready = False
+        while attempts < timeout:
+            try:
+                response = requests.get(f"{EMBEDDING_BASE_URL}/models", timeout=2)
+                if response.status_code == 200:
+                    ready = True
+                    break
+            except requests.RequestException:
+                pass
+            attempts += 0.5
+            time.sleep(0.5)
+
+        if not ready:
+            stderr = proc.stderr.read().decode() if proc.stderr else ""
+            proc.kill()
+            raise RuntimeError(f"Embedding server failed to start in {timeout}s.\nStderr: {stderr}")
+
+        yield proc
+
+        # Cleanup — same pattern as PopenServer.terminate()
+        proc.send_signal(signal.SIGINT)
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+
+@pytest.fixture(scope="module")
+def client(launch_embedding_server):
+    """OpenAI client connected to the embedding server."""
+    assert launch_embedding_server is not None
+    return OpenAI(base_url=EMBEDDING_BASE_URL, api_key="none")
+
+
+# ===================================================================
+# /v1/models
+# ===================================================================
+
+
+@pytest.mark.usefixtures("client")
+def test_models_endpoint():
+    """The /v1/models endpoint lists the embedding model."""
+    resp = requests.get(f"{EMBEDDING_BASE_URL}/models", timeout=5)
+    assert resp.status_code == 200
+    data = resp.json()
+    assert isinstance(data["data"], list)
+
+
+# ===================================================================
+# Single input
+# ===================================================================
+
+
+def test_single_string_input(client):
+    """Single string input returns one embedding."""
+    resp = client.embeddings.create(input="What is machine learning?", model=EMBEDDING_MODEL_NAME)
+    raw = resp.model_dump()
+    check_embedding_response(raw, model=EMBEDDING_MODEL_NAME, num_embeddings=1)
+
+
+# ===================================================================
+# Batch input
+# ===================================================================
+
+BATCH_INPUTS = [
+    "What is machine learning?",
+    "How to brew coffee?",
+    "ML is a subset of AI.",
+]
+
+
+def test_batch_string_input(client):
+    """List of strings returns one embedding per input."""
+    resp = client.embeddings.create(input=BATCH_INPUTS, model=EMBEDDING_MODEL_NAME)
+    raw = resp.model_dump()
+    check_embedding_response(raw, model=EMBEDDING_MODEL_NAME, num_embeddings=len(BATCH_INPUTS))
+
+
+def test_batch_index_ordering(client):
+    """Embedding indices are sequential."""
+    resp = client.embeddings.create(input=BATCH_INPUTS, model=EMBEDDING_MODEL_NAME)
+    indices = [d.index for d in resp.data]
+    assert indices == list(range(len(BATCH_INPUTS)))
+
+
+# ===================================================================
+# Cosine similarity — semantic quality via endpoint
+# ===================================================================
+
+
+def test_cosine_similarity_via_endpoint(client):
+    """Related texts have higher similarity than unrelated (end-to-end)."""
+    resp = client.embeddings.create(
+        input=[
+            "What is machine learning?",
+            "Explain deep learning",
+            "Order a pizza",
+        ],
+        model=EMBEDDING_MODEL_NAME,
+    )
+    e0, e1, e2 = [np.array(d.embedding) for d in resp.data]
+    sim_related = float(np.dot(e0, e1))
+    sim_unrelated = float(np.dot(e0, e2))
+    assert sim_related > sim_unrelated, (
+        f"Related ({sim_related:.4f}) should > unrelated ({sim_unrelated:.4f})"
+    )
+
+
+# ===================================================================
+# Dimension truncation (Matryoshka)
+# ===================================================================
+
+
+def test_dimension_truncation(client):
+    """dimensions parameter truncates and re-normalizes output."""
+    target_dim = 256
+    resp = client.embeddings.create(
+        input="Hello world", model=EMBEDDING_MODEL_NAME, dimensions=target_dim
+    )
+    raw = resp.model_dump()
+    check_embedding_response(
+        raw,
+        model=EMBEDDING_MODEL_NAME,
+        num_embeddings=1,
+        expected_dim=target_dim,
+    )
+
+
+# ===================================================================
+# Encoding format
+# ===================================================================
+
+
+@pytest.mark.usefixtures("launch_embedding_server")
+def test_base64_encoding():
+    """base64 encoding format returns base64-encoded embeddings."""
+    resp = requests.post(
+        f"{EMBEDDING_BASE_URL}/embeddings",
+        json={
+            "input": "Hello world",
+            "model": EMBEDDING_MODEL_NAME,
+            "encoding_format": "base64",
+        },
+        timeout=5,
+    )
+    assert resp.status_code == 200
+    data = resp.json()
+    assert data["data"][0]["object"] == "embedding"
+    # base64 string should be a non-empty string (not a list)
+    emb = data["data"][0]["embedding"]
+    assert isinstance(emb, str) and len(emb) > 0
+
+
+# ===================================================================
+# Error handling — reuses expect_error() pattern from test_server.py
+# ===================================================================
+
+
+@pytest.mark.usefixtures("launch_embedding_server")
+def test_any_model_name_works_with_single_engine():
+    """When only one embedding engine is served, any model name works.
+
+    This mirrors ServerContext.get_engine() behavior: a single served
+    model is returned regardless of the requested model name.
+    """
+    resp = requests.post(
+        f"{EMBEDDING_BASE_URL}/embeddings",
+        json={"input": "test", "model": "any-name-works"},
+        timeout=5,
+    )
+    assert resp.status_code == 200
+    data = resp.json()
+    assert len(data["data"]) == 1
+
+
+# ===================================================================
+# Standalone runner (same pattern as test_server.py __main__)
+# ===================================================================
+
+if __name__ == "__main__":
+    _skip_if_no_model()
+
+    print(f"Using model: {EMBEDDING_MODEL_DIR}")
+    print(f"Using model lib: {EMBEDDING_MODEL_LIB}")
+    print(f"Server URL: {EMBEDDING_BASE_URL}")
+    print(
+        "\nMake sure the embedding server is running, or set env vars "
+        "and use pytest to auto-launch."
+    )
+
+    # Allow running against an already-running server
+    c = OpenAI(base_url=EMBEDDING_BASE_URL, api_key="none")
+    test_models_endpoint()
+    test_single_string_input(c)
+    test_batch_string_input(c)
+    test_batch_index_ordering(c)
+    test_cosine_similarity_via_endpoint(c)
+    test_dimension_truncation(c)
+    test_base64_encoding()
+    test_any_model_name_works_with_single_engine()
+    print("\nAll embedding server tests passed!")

tests/python/serve/server/test_server_function_call.py

@@ -0,0 +1,208 @@
+"""
+Test script for function call in chat completion. To run this script, use the following command:
+MLC_SERVE_MODEL_LIB=dist/gorilla-openfunctions-v1-q4f16_1_MLC/gorilla-openfunctions-v1-q4f16_1-cuda.so
+MLC_SERVE_MODEL_LIB=${MLC_SERVE_MODEL_LIB} python -m pytest -x tests/python/serve/server/test_server_function_call.py
+"""  # noqa: E501
+
+import json
+import os
+from typing import Dict, List, Optional, Tuple  # noqa: UP035
+
+import pytest
+import requests
+
+OPENAI_V1_CHAT_COMPLETION_URL = "http://127.0.0.1:8000/v1/chat/completions"
+
+
+def check_openai_nonstream_response(
+    response: Dict,  # noqa: UP006
+    *,
+    model: str,
+    object_str: str,
+    num_choices: int,
+    finish_reason: List[str],  # noqa: UP006
+    completion_tokens: Optional[int] = None,
+):
+    print(response)
+    assert response["model"] == model
+    assert response["object"] == object_str
+
+    choices = response["choices"]
+    assert isinstance(choices, list)
+    assert len(choices) == num_choices
+    for idx, choice in enumerate(choices):
+        assert choice["index"] == idx
+        assert choice["finish_reason"] in finish_reason
+
+        # text: str
+        message = choice["message"]
+        assert message["role"] == "assistant"
+        if choice["finish_reason"] == "tool_calls":
+            assert message["content"] is None
+            assert isinstance(message["tool_calls"], list)
+        else:
+            assert message["tool_calls"] is None
+            assert message["content"] is not None
+
+    usage = response["usage"]
+    assert isinstance(usage, dict)
+    assert usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"]
+    assert usage["prompt_tokens"] > 0
+
+    if completion_tokens is not None:
+        assert usage["completion_tokens"] == completion_tokens
+
+
+def check_openai_stream_response(
+    responses: List[Dict],  # noqa: UP006
+    *,
+    model: str,
+    object_str: str,
+    num_choices: int,
+    finish_reason: str,
+    echo_prompt: Optional[str] = None,
+    suffix: Optional[str] = None,
+    stop: Optional[List[str]] = None,  # noqa: UP006
+    require_substr: Optional[List[str]] = None,  # noqa: UP006
+):
+    assert len(responses) > 0
+
+    finished = [False for _ in range(num_choices)]
+    outputs = ["" for _ in range(num_choices)]
+    for response in responses:
+        assert response["model"] == model
+        assert response["object"] == object_str
+
+        choices = response["choices"]
+        assert isinstance(choices, list)
+        assert len(choices) == num_choices
+        for idx, choice in enumerate(choices):
+            assert choice["index"] == idx
+
+            delta = choice["delta"]
+            assert delta["role"] == "assistant"
+            assert isinstance(delta["content"], str)
+            outputs[idx] += delta["content"]
+
+            if finished[idx]:
+                assert choice["finish_reason"] == finish_reason
+            elif choice["finish_reason"] is not None:
+                assert choice["finish_reason"] == finish_reason
+                finished[idx] = True
+
+    for output in outputs:
+        if echo_prompt is not None:
+            assert output.startswith(echo_prompt)
+        if suffix is not None:
+            assert output.endswith(suffix)
+        if stop is not None:
+            for stop_str in stop:
+                assert stop_str not in output
+        if require_substr is not None:
+            for substr in require_substr:
+                assert substr in output
+
+
+tools = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_current_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {
+                        "type": "string",
+                        "description": "The city and state, e.g. San Francisco, CA",
+                    },
+                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+                },
+                "required": ["location"],
+            },
+        },
+    }
+]
+
+
+CHAT_COMPLETION_MESSAGES = [
+    # messages #0
+    [
+        {
+            "role": "user",
+            "content": "What is the current weather in Pittsburgh, PA?",
+        }
+    ],
+    # messages #1
+    [
+        {
+            "role": "user",
+            "content": "What is the current weather in Pittsburgh, PA and Tokyo, JP?",
+        }
+    ],
+    # messages #2
+    [
+        {
+            "role": "user",
+            "content": "What is the current weather in Pittsburgh, PA in fahrenheit?",
+        }
+    ],
+]
+
+
+@pytest.mark.parametrize("stream", [False, True])
+@pytest.mark.parametrize("messages", CHAT_COMPLETION_MESSAGES)
+def test_openai_v1_chat_completion_function_call(
+    served_model: Tuple[str, str],  # noqa: UP006
+    launch_server,
+    stream: bool,
+    messages: List[Dict[str, str]],  # noqa: UP006
+):
+    # `served_model` and `launch_server` are pytest fixtures
+    # defined in conftest.py.
+
+    payload = {
+        "model": served_model[0],
+        "messages": messages,
+        "stream": stream,
+        "tools": tools,
+    }
+
+    response = requests.post(OPENAI_V1_CHAT_COMPLETION_URL, json=payload, timeout=60)
+    if not stream:
+        check_openai_nonstream_response(
+            response.json(),
+            model=served_model[0],
+            object_str="chat.completion",
+            num_choices=1,
+            finish_reason=["tool_calls", "error"],
+        )
+    else:
+        responses = []
+        for chunk in response.iter_lines(chunk_size=512):
+            if not chunk or chunk == b"data: [DONE]":
+                continue
+            responses.append(json.loads(chunk.decode("utf-8")[6:]))
+        check_openai_stream_response(
+            responses,
+            model=served_model[0],
+            object_str="chat.completion.chunk",
+            num_choices=1,
+            finish_reason="tool_calls",
+        )
+
+
+if __name__ == "__main__":
+    model_lib = os.environ.get("MLC_SERVE_MODEL_LIB")
+    if model_lib is None:
+        raise ValueError(
+            'Environment variable "MLC_SERVE_MODEL_LIB" not found. '
+            "Please set it to model lib compiled by MLC LLM "
+            "(e.g., `./dist/gorilla-openfunctions-v1-q4f16_1_MLC/gorilla-openfunctions-v1-q4f16_1-cuda.so`) "  # noqa: E501
+            "which supports function calls."
+        )
+    MODEL = (os.path.dirname(model_lib), model_lib)
+
+    for msg in CHAT_COMPLETION_MESSAGES:
+        test_openai_v1_chat_completion_function_call(MODEL, None, stream=False, messages=msg)
+        test_openai_v1_chat_completion_function_call(MODEL, None, stream=True, messages=msg)

tests/python/serve/server/test_server_image.py

@@ -0,0 +1,257 @@
+import json
+import os
+from typing import Dict, List, Optional, Tuple  # noqa: UP035
+
+import pytest
+import regex
+import requests
+
+OPENAI_V1_CHAT_COMPLETION_URL = "http://127.0.0.1:8001/v1/chat/completions"
+
+JSON_TOKEN_PATTERN = (
+    r"((-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?)|null|true|false|"
+    r'("((\\["\\\/bfnrt])|(\\u[0-9a-fA-F]{4})|[^"\\\x00-\x1f])*")'
+)
+JSON_TOKEN_RE = regex.compile(JSON_TOKEN_PATTERN)
+
+
+def is_json_or_json_prefix(s: str) -> bool:
+    try:
+        json.loads(s)
+        return True
+    except json.JSONDecodeError as e:
+        # If the JSON decoder reaches the end of s, it is a prefix of a JSON string.
+        if e.pos == len(s):
+            return True
+        # Since json.loads is token-based instead of char-based, there may remain half a token after
+        # the matching position.
+        # If the left part is a prefix of a valid JSON token, the output is also valid
+        regex_match = JSON_TOKEN_RE.fullmatch(s[e.pos :], partial=True)
+        return regex_match is not None
+
+
+def check_openai_nonstream_response(
+    response: Dict,  # noqa: UP006
+    *,
+    is_chat_completion: bool,
+    model: str,
+    object_str: str,
+    num_choices: int,
+    finish_reasons: List[str],  # noqa: UP006
+    completion_tokens: Optional[int] = None,
+    echo_prompt: Optional[str] = None,
+    suffix: Optional[str] = None,
+    stop: Optional[List[str]] = None,  # noqa: UP006
+    require_substr: Optional[List[str]] = None,  # noqa: UP006
+    json_mode: bool = False,
+):
+    assert response["model"] == model
+    assert response["object"] == object_str
+
+    choices = response["choices"]
+    assert isinstance(choices, list)
+    assert len(choices) <= num_choices
+    texts: List[str] = ["" for _ in range(num_choices)]  # noqa: UP006
+    for choice in choices:
+        idx = choice["index"]
+        assert choice["finish_reason"] in finish_reasons
+
+        if not is_chat_completion:
+            assert isinstance(choice["text"], str)
+            texts[idx] = choice["text"]
+            if echo_prompt is not None:
+                assert texts[idx]
+            if suffix is not None:
+                assert texts[idx]
+        else:
+            message = choice["message"]
+            assert message["role"] == "assistant"
+            assert isinstance(message["content"], str)
+            texts[idx] = message["content"]
+
+        if stop is not None:
+            for stop_str in stop:
+                assert stop_str not in texts[idx]
+        if require_substr is not None:
+            for substr in require_substr:
+                assert substr in texts[idx]
+        if json_mode:
+            assert is_json_or_json_prefix(texts[idx])
+
+    usage = response["usage"]
+    assert isinstance(usage, dict)
+    assert usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"]
+    assert usage["prompt_tokens"] > 0
+    if completion_tokens is not None:
+        assert usage["completion_tokens"] == completion_tokens
+
+
+def check_openai_stream_response(
+    responses: List[Dict],  # noqa: UP006
+    *,
+    is_chat_completion: bool,
+    model: str,
+    object_str: str,
+    num_choices: int,
+    finish_reasons: List[str],  # noqa: UP006
+    completion_tokens: Optional[int] = None,
+    echo_prompt: Optional[str] = None,
+    suffix: Optional[str] = None,
+    stop: Optional[List[str]] = None,  # noqa: UP006
+    require_substr: Optional[List[str]] = None,  # noqa: UP006
+    json_mode: bool = False,
+):
+    assert len(responses) > 0
+
+    finished = [False for _ in range(num_choices)]
+    outputs = ["" for _ in range(num_choices)]
+    for response in responses:
+        assert response["model"] == model
+        assert response["object"] == object_str
+
+        choices = response["choices"]
+        assert isinstance(choices, list)
+        assert len(choices) <= num_choices
+        for choice in choices:
+            idx = choice["index"]
+
+            if not is_chat_completion:
+                assert isinstance(choice["text"], str)
+                outputs[idx] += choice["text"]
+            else:
+                delta = choice["delta"]
+                assert delta["role"] == "assistant"
+                assert isinstance(delta["content"], str)
+                outputs[idx] += delta["content"]
+
+            if finished[idx]:
+                assert choice["finish_reason"] in finish_reasons
+            elif choice["finish_reason"] is not None:
+                assert choice["finish_reason"] in finish_reasons
+                finished[idx] = True
+
+        if not is_chat_completion:
+            usage = response["usage"]
+            assert isinstance(usage, dict)
+            assert usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"]
+            assert usage["prompt_tokens"] > 0
+            if completion_tokens is not None:
+                assert usage["completion_tokens"] <= completion_tokens
+
+    if not is_chat_completion:
+        if completion_tokens is not None:
+            assert responses[-1]["usage"]["completion_tokens"] == completion_tokens
+
+    for i, output in enumerate(outputs):
+        if echo_prompt is not None:
+            assert output.startswith(echo_prompt)
+        if suffix is not None:
+            assert output.endswith(suffix)
+        if stop is not None:
+            for stop_str in stop:
+                assert stop_str not in output
+        if require_substr is not None:
+            for substr in require_substr:
+                assert substr in output
+        if json_mode:
+            assert is_json_or_json_prefix(output)
+
+
+CHAT_COMPLETION_MESSAGES = [
+    # messages #0
+    [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": "https://llava-vl.github.io/static/images/view.jpg",
+                },
+                {"type": "text", "text": "What does this image represent?"},
+            ],
+        },
+    ],
+    # messages #1
+    [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": "https://llava-vl.github.io/static/images/view.jpg",
+                },
+                {"type": "text", "text": "What does this image represent?"},
+            ],
+        },
+        {
+            "role": "assistant",
+            "content": "The image represents a serene and peaceful scene of a pier extending over a body of water, such as a lake or a river.er. The pier is made of wood and has a bench on it, providing a place for people to sit and enjoy the view. The pier is situated in a natural environment, surrounded by trees and mountains in the background. This setting creates a tranquil atmosphere, inviting visitors to relax and appreciate the beauty of the landscape.",  # noqa: E501
+        },
+        {
+            "role": "user",
+            "content": "What country is the image set in? Give me 10 ranked guesses and reasons why.",  # noqa: E501
+        },
+    ],
+]
+
+
+@pytest.mark.parametrize("stream", [False, True])
+@pytest.mark.parametrize("messages", CHAT_COMPLETION_MESSAGES)
+def test_openai_v1_chat_completions(
+    served_model: Tuple[str, str],  # noqa: UP006
+    launch_server,
+    stream: bool,
+    messages: List[Dict[str, str]],  # noqa: UP006
+):
+    # `served_model` and `launch_server` are pytest fixtures
+    # defined in conftest.py.
+
+    payload = {
+        "model": served_model[0],
+        "messages": messages,
+        "stream": stream,
+    }
+    response = requests.post(OPENAI_V1_CHAT_COMPLETION_URL, json=payload, timeout=180)
+    if not stream:
+        check_openai_nonstream_response(
+            response.json(),
+            is_chat_completion=True,
+            model=served_model[0],
+            object_str="chat.completion",
+            num_choices=1,
+            finish_reasons=["stop"],
+        )
+    else:
+        responses = []
+        for chunk in response.iter_lines(chunk_size=512):
+            if not chunk or chunk == b"data: [DONE]":
+                continue
+            responses.append(json.loads(chunk.decode("utf-8")[6:]))
+        check_openai_stream_response(
+            responses,
+            is_chat_completion=True,
+            model=served_model[0],
+            object_str="chat.completion.chunk",
+            num_choices=1,
+            finish_reasons=["stop"],
+        )
+
+
+if __name__ == "__main__":
+    model_lib = os.environ.get("MLC_SERVE_MODEL_LIB")
+    if model_lib is None:
+        raise ValueError(
+            'Environment variable "MLC_SERVE_MODEL_LIB" not found. '
+            "Please set it to model lib compiled by MLC LLM "
+            "(e.g., `dist/Llama-2-7b-chat-hf-q0f16-MLC/Llama-2-7b-chat-hf-q0f16-MLC-cuda.so`)."
+        )
+
+    model = os.environ.get("MLC_SERVE_MODEL")
+    if model is None:
+        MODEL = (os.path.dirname(model_lib), model_lib)
+    else:
+        MODEL = (model, model_lib)
+
+    for msg in CHAT_COMPLETION_MESSAGES:
+        test_openai_v1_chat_completions(MODEL, None, stream=False, messages=msg)
+        test_openai_v1_chat_completions(MODEL, None, stream=True, messages=msg)