chore: import upstream snapshot with attribution

python/ray/llm/examples/sglang/batch_sglang_example.py

@@ -0,0 +1,49 @@
+"""Batch inference with SGLang using Ray Data.
+
+Usage:
+    RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=0 python batch_sglang_example.py
+"""
+
+import ray
+from ray.data.llm import SGLangEngineProcessorConfig, build_processor
+
+config = SGLangEngineProcessorConfig(
+    model_source="unsloth/Llama-3.1-8B-Instruct",
+    engine_kwargs=dict(
+        dtype="half",
+        mem_fraction_static=0.8,
+    ),
+    batch_size=32,
+    concurrency=1,
+)
+
+processor = build_processor(
+    config,
+    preprocess=lambda row: dict(
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": row["prompt"]},
+        ],
+        sampling_params=dict(
+            temperature=0.7,
+            max_new_tokens=256,
+        ),
+    ),
+    postprocess=lambda row: dict(
+        prompt=row["prompt"],
+        response=row["generated_text"],
+    ),
+)
+
+ds = ray.data.from_items(
+    [
+        {"prompt": "What is the capital of France?"},
+        {"prompt": "Explain photosynthesis in one sentence."},
+        {"prompt": "Write a haiku about programming."},
+    ]
+)
+ds = processor(ds)
+
+for row in ds.take_all():
+    print(f"Prompt: {row['prompt']}")
+    print(f"Response: {row['response']}\n")

python/ray/llm/examples/sglang/query_example.py

@@ -0,0 +1,35 @@
+"""Query client for an SGLang model served via Ray Serve LLM.
+
+Prerequisites:
+    Start a serving example first, e.g.:
+    RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=0 serve run serve_sglang_example:app
+
+Usage:
+    python query_example.py
+"""
+
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8000/v1", api_key="fake-key")
+
+# Chat completions
+print("=== Chat Completions ===")
+chat_response = client.chat.completions.create(
+    model="Llama-3.1-8B-Instruct",
+    messages=[
+        {"role": "user", "content": "List 3 countries and their capitals."},
+    ],
+    temperature=0,
+    max_tokens=64,
+)
+print(chat_response.choices[0].message.content)
+
+# Text completions
+print("\n=== Text Completions ===")
+completion_response = client.completions.create(
+    model="Llama-3.1-8B-Instruct",
+    prompt="San Francisco is a",
+    temperature=0,
+    max_tokens=30,
+)
+print(completion_response.choices[0].text)

python/ray/llm/examples/sglang/readme.md

@@ -0,0 +1,27 @@
+# SGLang on Ray Serve LLM
+
+This directory contains example scripts for using SGLang with Ray Serve LLM.
+
+## Examples
+
+| File | Description |
+|------|-------------|
+| `serve_sglang_example.py` | Single-node SGLang serving with autoscaling |
+| `serve_sglang_multinode_example.py` | Multi-node serving with tensor and pipeline parallelism |
+| `batch_sglang_example.py` | Batch inference using Ray Data |
+| `query_example.py` | OpenAI client for querying a running deployment |
+
+## Prerequisites
+
+```bash
+pip install ray[serve,llm] "sglang[all,ray]"
+```
+
+Set the environment variable before running:
+
+- **CUDA:** `RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=0`
+- **ROCm:** `RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES=0`
+
+## Engine implementation
+
+The `SGLangServer` class is located at `ray.llm._internal.serve.engines.sglang` and wraps SGLang's in-process engine with the Ray Serve LLM server protocol.

python/ray/llm/examples/sglang/serve_sglang_example.py

@@ -0,0 +1,33 @@
+"""Single-node SGLang serving example using Ray Serve LLM.
+
+Usage:
+    RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=0 serve run serve_sglang_example:app
+"""
+
+from ray import serve
+from ray.llm._internal.serve.engines.sglang import SGLangServer
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "Llama-3.1-8B-Instruct",
+        "model_source": "unsloth/Llama-3.1-8B-Instruct",
+    },
+    deployment_config={
+        "autoscaling_config": {
+            "min_replicas": 1,
+            "max_replicas": 2,
+        }
+    },
+    server_cls=SGLangServer,
+    engine_kwargs={
+        "trust_remote_code": True,
+        "model_path": "unsloth/Llama-3.1-8B-Instruct",
+        "tp_size": 1,
+        "mem_fraction_static": 0.8,
+    },
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+serve.start()
+serve.run(app, blocking=True)

python/ray/llm/examples/sglang/serve_sglang_multinode_example.py

@@ -0,0 +1,46 @@
+"""Multi-node SGLang serving example with tensor and pipeline parallelism.
+
+Requirements:
+    - 2 nodes with 4 GPUs each (8 GPUs total for tp_size=4, pp_size=2)
+    - pip install ray[serve,llm] "sglang[all,ray]"
+    - Set RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=0
+
+Usage:
+    RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=0 serve run serve_sglang_multinode_example:app
+"""
+
+from ray import serve
+from ray.llm._internal.serve.engines.sglang import SGLangServer
+from ray.serve.llm import LLMConfig, build_openai_app
+
+llm_config = LLMConfig(
+    model_loading_config={
+        "model_id": "Llama-3.1-70B-Instruct",
+        "model_source": "meta-llama/Llama-3.1-70B-Instruct",
+    },
+    deployment_config={
+        "autoscaling_config": {
+            "min_replicas": 1,
+            "max_replicas": 2,
+            "target_ongoing_requests": 4,
+        }
+    },
+    # PACK fills GPUs on each node before moving to the next.
+    # With 8 bundles across 2 nodes (4 GPUs each), each node gets 4 bundles.
+    placement_group_config={
+        "placement_group_bundles": [{"GPU": 1}] * 8,
+        "placement_group_strategy": "PACK",
+    },
+    server_cls=SGLangServer,
+    engine_kwargs={
+        "model_path": "meta-llama/Llama-3.1-70B-Instruct",
+        "tp_size": 4,
+        "pp_size": 2,
+        "mem_fraction_static": 0.8,
+    },
+)
+
+app = build_openai_app({"llm_configs": [llm_config]})
+
+if __name__ == "__main__":
+    serve.run(app, blocking=True)