wehub-resource-sync
28 lines
645 B
YAML
28 lines
645 B
YAML
model_loading_config:
|
|
model_id: FAKE_MODEL_2_UNDER_TEST
|
|
|
|
# Overriding the engine class to only focus on testing the components around the engine
|
|
runtime_env:
|
|
env_vars:
|
|
RAYLLM_VLLM_ENGINE_CLS: "ray.llm.tests.serve.mocks.mock_vllm_engine.MockVLLMEngine"
|
|
|
|
llm_engine: vLLM
|
|
|
|
engine_kwargs:
|
|
max_model_len: 4096
|
|
|
|
accelerator_type: A10G
|
|
|
|
deployment_config:
|
|
autoscaling_config:
|
|
min_replicas: 4
|
|
initial_replicas: 4
|
|
max_replicas: 10
|
|
target_ongoing_requests: 20
|
|
metrics_interval_s: 10.0
|
|
look_back_period_s: 30.0
|
|
smoothing_factor: 0.6
|
|
downscale_delay_s: 300.0
|
|
upscale_delay_s: 15.0
|
|
max_ongoing_requests: 48
|