wehub-resource-sync
35 lines
991 B
Python
35 lines
991 B
Python
"""The request class in MLC LLM serving"""
|
|
|
|
from typing import List # noqa: UP035
|
|
|
|
import tvm_ffi
|
|
from tvm.runtime import Object
|
|
|
|
from mlc_llm.protocol.generation_config import GenerationConfig
|
|
|
|
from . import _ffi_api
|
|
from .data import Data
|
|
|
|
|
|
@tvm_ffi.register_object("mlc.serve.Request")
|
|
class Request(Object):
|
|
"""The user submitted text-generation request, which contains
|
|
a unique request id, a list of multi-modal inputs, a set of
|
|
generation configuration parameters.
|
|
|
|
Note
|
|
----
|
|
Do not explicitly construct this class.
|
|
Construct this object via engine.create_request functions.
|
|
"""
|
|
|
|
@property
|
|
def inputs(self) -> List[Data]: # noqa: UP006
|
|
"""The inputs of the request."""
|
|
return _ffi_api.RequestGetInputs(self)
|
|
|
|
@property
|
|
def generation_config(self) -> GenerationConfig:
|
|
"""The generation config of the request."""
|
|
return GenerationConfig.model_validate_json(_ffi_api.RequestGetGenerationConfigJSON(self))
|