wehub-resource-sync
34 lines
898 B
Python
34 lines
898 B
Python
"""Single-node SGLang serving example using Ray Serve LLM.
|
|
|
|
Usage:
|
|
RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=0 serve run serve_sglang_example:app
|
|
"""
|
|
|
|
from ray import serve
|
|
from ray.llm._internal.serve.engines.sglang import SGLangServer
|
|
from ray.serve.llm import LLMConfig, build_openai_app
|
|
|
|
llm_config = LLMConfig(
|
|
model_loading_config={
|
|
"model_id": "Llama-3.1-8B-Instruct",
|
|
"model_source": "unsloth/Llama-3.1-8B-Instruct",
|
|
},
|
|
deployment_config={
|
|
"autoscaling_config": {
|
|
"min_replicas": 1,
|
|
"max_replicas": 2,
|
|
}
|
|
},
|
|
server_cls=SGLangServer,
|
|
engine_kwargs={
|
|
"trust_remote_code": True,
|
|
"model_path": "unsloth/Llama-3.1-8B-Instruct",
|
|
"tp_size": 1,
|
|
"mem_fraction_static": 0.8,
|
|
},
|
|
)
|
|
|
|
app = build_openai_app({"llm_configs": [llm_config]})
|
|
serve.start()
|
|
serve.run(app, blocking=True)
|