chore: import upstream snapshot with attribution

python/sglang/test/ascend/disaggregation_utils.py

@@ -0,0 +1,153 @@
+import logging
+import os
+import time
+import warnings
+from urllib.parse import urlparse
+
+import requests
+
+from sglang.srt.environ import envs
+from sglang.srt.utils import kill_process_tree
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_with_error_check,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class TestDisaggregationBase(CustomTestCase):
+    @classmethod
+    def setUpClass(cls):
+        parsed_url = urlparse(DEFAULT_URL_FOR_TEST)
+        cls.base_host = parsed_url.hostname
+        base_port = str(parsed_url.port)
+        cls.lb_port = base_port
+        cls.prefill_port = f"{int(base_port) + 100}"
+        cls.decode_port = f"{int(base_port) + 200}"
+        cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}"
+        cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}"
+        cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}"
+        print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}")
+        cls.process_lb, cls.process_decode, cls.process_prefill = None, None, None
+
+        # config transfer backend and rdma devices
+        cls.transfer_backend = [
+            "--disaggregation-transfer-backend",
+            envs.SGLANG_TEST_PD_DISAGG_BACKEND.get(),
+        ]
+        cls.rdma_devices = [
+            "--disaggregation-ib-device",
+            envs.SGLANG_TEST_PD_DISAGG_DEVICES.get(),
+        ]
+        if cls.rdma_devices[1] is None:
+            cls.rdma_devices = []
+            msg = "No RDMA devices specified for disaggregation test, using default settings."
+            warnings.warn(msg)
+
+    @classmethod
+    def launch_lb(cls):
+        lb_command = [
+            "python3",
+            "-m",
+            "sglang_router.launch_router",
+            "--pd-disaggregation",
+            "--mini-lb",
+            "--prefill",
+            cls.prefill_url,
+            "--decode",
+            cls.decode_url,
+            "--host",
+            cls.base_host,
+            "--port",
+            cls.lb_port,
+        ]
+        print("Starting load balancer:", " ".join(lb_command))
+        cls.process_lb = popen_with_error_check(lb_command)
+        cls.wait_server_ready(cls.lb_url + "/health")
+
+    @classmethod
+    def wait_server_ready(cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH):
+        start_time = time.perf_counter()
+        while True:
+            try:
+                response = requests.get(url)
+                if response.status_code == 200:
+                    print(f"Server {url} is ready")
+                    return
+            except Exception:
+                pass
+
+            if time.perf_counter() - start_time > timeout:
+                raise RuntimeError(f"Server {url} failed to start in {timeout}s")
+            time.sleep(1)
+
+    @classmethod
+    def tearDownClass(cls):
+        for process in [cls.process_lb, cls.process_decode, cls.process_prefill]:
+            if process:
+                try:
+                    kill_process_tree(process.pid)
+                except Exception as e:
+                    print(f"Error killing process {process.pid}: {e}")
+
+        # wait for 5 seconds
+        time.sleep(5)
+
+
+def get_rdma_devices_args():
+    def _parse_list_env(var_name: str):
+        val = os.getenv(var_name)
+        if not val:
+            return None
+        items = [x.strip() for x in val.split(",") if x.strip()]
+        return items or None
+
+    def _pick_default_pair(rdma_all_devices):
+        return [rdma_all_devices[0], rdma_all_devices[len(rdma_all_devices) // 2]]
+
+    rdma_all_devices = _parse_list_env("SGLANG_CI_RDMA_ALL_DEVICES") or [
+        f"mlx5_roce{i}" for i in range(8)
+    ]
+    logger.info("Resolved rdma_all_devices=%s", rdma_all_devices)
+
+    n_rdma = len(rdma_all_devices)
+
+    # 1. Get visible GPU indices
+    cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES")
+    if not cuda_visible_devices:
+        warnings.warn("CUDA_VISIBLE_DEVICES is not set. Using default RDMA devices.")
+        return ",".join(_pick_default_pair(rdma_all_devices))
+
+    try:
+        # Convert to list of integers (handling possible spaces and empty strings)
+        gpu_indices = [
+            int(idx.strip()) for idx in cuda_visible_devices.split(",") if idx.strip()
+        ]
+        if not gpu_indices or len(gpu_indices) > 4:
+            return ",".join(_pick_default_pair(rdma_all_devices))
+    except ValueError:
+        warnings.warn(f"Invalid CUDA_VISIBLE_DEVICES format: {cuda_visible_devices}")
+        return ",".join(_pick_default_pair(rdma_all_devices))
+
+    # 2. Calculate base RDMA index group (each group of 4 GPUs uses consecutive devices)
+    base_rdma_group = (min(gpu_indices) // 4) * 4
+    for gpu_idx in gpu_indices:
+        if not (base_rdma_group <= gpu_idx < base_rdma_group + 4):
+            warnings.warn(
+                f"GPU index {gpu_idx} is outside expected group "
+                f"{base_rdma_group}-{base_rdma_group+3}"
+            )
+
+    # 3. Generate RDMA device names
+    rdma_devices = []
+    for gpu_idx in gpu_indices:
+        nic_index = gpu_idx // (8 // n_rdma)
+        rdma_devices.append(rdma_all_devices[nic_index])
+
+    if not rdma_devices:
+        return ",".join(_pick_default_pair(rdma_all_devices))
+
+    return ",".join(rdma_devices)

python/sglang/test/ascend/e2e/gen_dataset_fixed_len.py

@@ -0,0 +1,521 @@
+import json
+import os
+import random
+import string
+
+import numpy as np
+from PIL import Image
+from transformers import AutoTokenizer
+
+
+def load_jsonl(path):
+    """Load data from a JSONL file, one JSON object per line."""
+    data = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                data.append(json.loads(line))
+    return data
+
+
+def save_jsonl(data, file_path):
+    """Save a list of dicts to a JSONL file, one JSON object per line."""
+    file_dir = os.path.dirname(file_path)
+    if file_dir:
+        os.makedirs(file_dir, exist_ok=True)
+    with open(file_path, "w", encoding="utf-8") as f:
+        for item in data:
+            f.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+
+def format_qa(item):
+    """Format a GSM8K data entry into QA text for the few-shot pool."""
+    question = item["question"]
+    answer = item["answer"]
+    return f"Question: {question}\nLet's think step by step\nAnswer:\n{answer}\n\n"
+
+
+def pad_to_target_tokens(
+    question,
+    few_shot_pool_token_ids,
+    tokenizer,
+    target_tokens,
+    test_template="Question: {question}\nLet's think step by step\nAnswer:\n",
+):
+    """Pad a question text to the target token length.
+
+    Tokenizes the question using the test_template, calculates the remaining tokens
+    needed, and prepends randomly sampled few-shot token ids from the pool to reach
+    target_tokens. If the few-shot pool is insufficient, repeats the first sample
+    to fill the remaining gap.
+
+    Args:
+        question: The test question text.
+        few_shot_pool_token_ids: List of token id lists from the few-shot training pool.
+        tokenizer: The tokenizer instance.
+        target_tokens: Target input token length.
+        test_template: Question template string, defaults to GSM8K format.
+    """
+    test_prompt = test_template.format(question=question)
+    test_token_ids = tokenizer.encode(test_prompt, add_special_tokens=False)
+
+    remaining_tokens = target_tokens - len(test_token_ids)
+    if remaining_tokens <= 0:
+        return tokenizer.decode(
+            test_token_ids[:target_tokens], skip_special_tokens=True
+        )
+
+    shuffled_ids = list(range(len(few_shot_pool_token_ids)))
+    random.shuffle(shuffled_ids)
+
+    prefix_ids = []
+    for idx in shuffled_ids:
+        fs_ids = few_shot_pool_token_ids[idx]
+        if len(prefix_ids) + len(fs_ids) <= remaining_tokens:
+            prefix_ids.extend(fs_ids)
+        else:
+            partial_gap = remaining_tokens - len(prefix_ids)
+            if partial_gap > 0:
+                prefix_ids.extend(fs_ids[:partial_gap])
+            break
+
+    if len(prefix_ids) < remaining_tokens and few_shot_pool_token_ids:
+        padding_source_ids = few_shot_pool_token_ids[shuffled_ids[0]]
+        repeat_count = (remaining_tokens // len(padding_source_ids)) + 1
+        padding_ids = (padding_source_ids * repeat_count)[
+            : remaining_tokens - len(prefix_ids)
+        ]
+        prefix_ids.extend(padding_ids)
+
+    full_ids = prefix_ids + test_token_ids
+    return tokenizer.decode(full_ids[:target_tokens], skip_special_tokens=True)
+
+
+def generate_custom_dataset(
+    train_path,
+    test_path,
+    tokenizer_path,
+    target_tokens,
+    num_prompts,
+    trust_remote_code=False,
+    test_template="Question: {question}\nLet's think step by step\nAnswer:\n",
+):
+    """Generate a custom dataset with a fixed input token length.
+
+    Builds a few-shot pool from the training set and pads test questions to the
+    specified token length. If the test set has fewer samples than num_prompts,
+    it cycles and repeats to fill the required count.
+
+    Args:
+        train_path: Path to the GSM8K training JSONL file.
+        test_path: Path to the GSM8K test JSONL file.
+        tokenizer_path: Path to the tokenizer.
+        target_tokens: Target input token length.
+        num_prompts: Number of prompts to generate; 0 means use all test samples.
+        trust_remote_code: Whether to trust remote code when loading the tokenizer.
+        test_template: Question template string.
+
+    Returns:
+        list[dict]: Each item contains fields defined in test_template.
+    """
+    tokenizer = AutoTokenizer.from_pretrained(
+        tokenizer_path, trust_remote_code=trust_remote_code
+    )
+
+    train_data = load_jsonl(train_path)
+    test_data = load_jsonl(test_path)
+    if num_prompts > 0 and num_prompts > len(test_data):
+        multiplier = (num_prompts // len(test_data)) + 1
+        test_data = (test_data * multiplier)[:num_prompts]
+    elif num_prompts > 0:
+        test_data = test_data[:num_prompts]
+
+    few_shot_pool = [format_qa(item) for item in train_data]
+    few_shot_pool_token_ids = [
+        tokenizer.encode(fs, add_special_tokens=False) for fs in few_shot_pool
+    ]
+
+    output_data = []
+    for i, test_item in enumerate(test_data):
+        padded_question = pad_to_target_tokens(
+            question=test_item["question"],
+            few_shot_pool_token_ids=few_shot_pool_token_ids,
+            tokenizer=tokenizer,
+            target_tokens=target_tokens,
+            test_template=test_template,
+        )
+        output_data.append(
+            {
+                "question": padded_question,
+                "answer": test_item["answer"],
+            }
+        )
+        if (i + 1) % 100 == 0:
+            actual_tokens = len(
+                tokenizer.encode(padded_question, add_special_tokens=False)
+            )
+            print(
+                f"Processed {i + 1}/{len(test_data)}, last item tokens: {actual_tokens}"
+            )
+
+    token_counts = [
+        len(tokenizer.encode(item["question"], add_special_tokens=False))
+        for item in output_data
+    ]
+    print(
+        f"Token count stats: min={min(token_counts)}, max={max(token_counts)}, avg={sum(token_counts)/len(token_counts):.1f}"
+    )
+
+    return output_data
+
+
+def generate_random_images(mm_dataset_data, size):
+    """Generate random image files for a multimodal dataset.
+
+    Creates random RGB images at the specified resolution for each image path
+    listed in the dataset entries.
+
+    Args:
+        mm_dataset_data: List of multimodal data entries, each with a "path" field
+            containing a list of image file paths.
+        size: Image size tuple (width, height), e.g. (1080, 1920).
+    """
+    total_image_num = len(mm_dataset_data)
+    print(f"begin to generate images, total {total_image_num}")
+
+    file_count = 0
+    for item in mm_dataset_data:
+        image_paths = item.get("path")
+
+        for image_path in image_paths:
+            if not image_path:
+                print("Error: The image path is none.")
+                continue
+
+            dir_name = os.path.dirname(image_path)
+            if dir_name and not os.path.exists(dir_name):
+                os.makedirs(dir_name, exist_ok=True)
+
+            random_array = np.random.randint(
+                0, 256, (size[1], size[0], 3), dtype=np.uint8
+            )
+
+            img = Image.fromarray(random_array)
+            img.save(image_path, quality=95)
+            if os.path.isfile(image_path):
+                file_count += 1
+
+    print(f"Finish images generation. Image num: {file_count}")
+
+
+def generate_mm_dataset(
+    train_path,
+    test_path,
+    tokenizer_path,
+    target_tokens=3500,
+    num_prompts=1024,
+    trust_remote_code=False,
+    test_template="Question: {question}\nLet's think step by step\nAnswer:\n",
+    image_dir="/tmp/datasets/image",
+    size=None,
+):
+    """Generate a multimodal (text + image) dataset.
+
+    First generates fixed-length text data via generate_fixed_len_dataset, then
+    attaches random image paths and type labels to each entry, and generates
+    the corresponding random image files.
+
+    Args:
+        train_path: Path to the GSM8K training JSONL file.
+        test_path: Path to the GSM8K test JSONL file.
+        tokenizer_path: Path to the tokenizer.
+        target_tokens: Target input token length.
+        num_prompts: Number of prompts to generate.
+        trust_remote_code: Whether to trust remote code when loading the tokenizer.
+        test_template: Question template string.
+        image_dir: Directory to save generated image files.
+        size: Image size string in "widthxheight" format, e.g. "1080x1920".
+
+    Returns:
+        list[dict]: Each item contains "question", "answer", "type", and "path" fields.
+    """
+    output_data = []
+    text_data = generate_custom_dataset(
+        train_path,
+        test_path,
+        tokenizer_path,
+        target_tokens,
+        num_prompts,
+        trust_remote_code,
+        test_template,
+    )
+
+    for item in text_data:
+        random_string = "".join(
+            random.choices(string.ascii_letters + string.digits, k=10)
+        )
+        item["type"] = "image"
+        item["path"] = [f"{image_dir}/{random_string}.jpg"]
+        output_data.append(item)
+
+    size = tuple(map(int, size.split("x")))
+    generate_random_images(output_data, size)
+    return output_data
+
+
+def generate_gsm8k_dataset(
+    model_path, source_dataset_path, batch_size, input_len, output_file
+):
+    """Generate a dataset with a fixed input token length from GSM8K (JSONL format).
+
+    Reads GSM8K source data, repeats or truncates each question's tokens to input_len,
+    then trims or replicates the dataset to batch_size entries, shuffles, and writes
+    to the output file.
+
+    Args:
+        model_path: Model path used to load the tokenizer.
+        source_dataset_path: Path to the GSM8K source JSONL file.
+        batch_size: Number of samples to generate.
+        input_len: Target input token length.
+        output_file: Output JSONL file path.
+    """
+    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    dataset = []
+    with open(source_dataset_path, "r", encoding="utf-8") as f:
+        for line in f:
+            data = json.loads(line)
+            dataset.append(data["question"])
+
+    dataset_new = []
+    for sentence in dataset:
+        words = tokenizer.tokenize(sentence)
+        len_num = len(words) // input_len
+        if len_num == 0:
+            multiplier = (input_len // len(words)) + 1
+            repeated_len = words * multiplier
+            words = repeated_len[:input_len]
+            decoded_text = tokenizer.convert_tokens_to_string(words)
+            if len(words) != input_len:
+                print(
+                    f"Generate DataSet Error: the length of new input is {len(words)}, not {input_len}"
+                )
+            dataset_new.append(decoded_text)
+
+    batch_num = len(dataset_new) // batch_size
+    if batch_num == 0:
+        multiplier = (batch_size // len(dataset_new)) + 1
+        repeated_batch = dataset_new * multiplier
+        dataset_new = repeated_batch[:batch_size]
+    else:
+        dataset_new = dataset_new[:batch_size]
+
+    random.shuffle(dataset_new)
+
+    if len(dataset_new) != batch_size:
+        print(
+            f"Generate DataSet Error: the size of new dataset is {len(dataset_new)}, not {batch_size}"
+        )
+
+    output_dir = os.path.dirname(output_file)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+
+    with open(output_file, "w", encoding="utf-8") as f:
+        for i in range(len(dataset_new)):
+            f.write(
+                json.dumps(
+                    {"question": f"{dataset_new[i]}", "answer": "none"},
+                    ensure_ascii=False,
+                )
+            )
+            f.write("\n")
+
+
+def generate_random_dataset(
+    model_path,
+    source_dataset_path,
+    batch_size,
+    input_len,
+    output_file,
+    output_len=1024,
+    range_ratio=1,
+):
+    """Generate a random dataset with logic matching bench_serving's --dataset-name random.
+
+    Samples real conversation text from the ShareGPT dataset as prompts, adjusting
+    to the target token length via truncation or repetition. Input/output lengths
+    are randomly sampled from [target*range_ratio, target]. Output format is a
+    JSON array compatible with ais_bench's ShareGPTDataset.
+
+    If source_dataset_path is not a valid JSON file, automatically downloads the
+    ShareGPT dataset from HuggingFace (anon8231489123/ShareGPT_Vicuna_unfiltered).
+
+    Args:
+        model_path: Model path used to load the tokenizer.
+        source_dataset_path: Path to the ShareGPT JSON file; auto-downloaded if invalid.
+        batch_size: Number of samples to generate.
+        input_len: Target input token length.
+        output_file: Output JSON file path.
+        output_len: Target output token length, default 1024.
+        range_ratio: Random range ratio for input/output lengths. Actual lengths are
+            uniformly sampled from [target*range_ratio, target]. Default 1 (fixed length).
+    """
+    SHAREGPT_REPO_ID = "anon8231489123/ShareGPT_Vicuna_unfiltered"
+    SHAREGPT_FILENAME = "ShareGPT_V3_unfiltered_cleaned_split.json"
+
+    def _is_file_valid_json(path):
+        """Check if the path points to a valid JSON file (exists and parseable)."""
+        if not os.path.isfile(path):
+            return False
+        try:
+            with open(path, encoding="utf-8") as f:
+                json.load(f)
+            return True
+        except json.JSONDecodeError:
+            return False
+
+    def _download_and_cache_hf_file(repo_id, filename, repo_type="dataset"):
+        """Download and cache a file from HuggingFace Hub."""
+        from huggingface_hub import hf_hub_download
+
+        return hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+    # Randomly sample input/output lengths per request in [target*range_ratio, target]
+    input_lens = np.random.randint(
+        max(int(input_len * range_ratio), 1),
+        input_len + 1,
+        size=batch_size,
+    ).tolist()
+    output_lens = np.random.randint(
+        max(int(output_len * range_ratio), 1),
+        output_len + 1,
+        size=batch_size,
+    ).tolist()
+
+    # Subtract special tokens to ensure the actual encoded length does not exceed target
+    num_special_tokens = int(tokenizer.num_special_tokens_to_add())
+    for i in range(batch_size):
+        input_lens[i] = max(1, input_lens[i] - num_special_tokens)
+
+    # Auto-download ShareGPT dataset from HuggingFace if local file is invalid
+    if not _is_file_valid_json(source_dataset_path):
+        print(
+            f"source_dataset_path '{source_dataset_path}' is not a valid file, downloading from HuggingFace..."
+        )
+        source_dataset_path = _download_and_cache_hf_file(
+            repo_id=SHAREGPT_REPO_ID,
+            filename=SHAREGPT_FILENAME,
+        )
+
+    # Load ShareGPT dataset, filter for >=2 turns, take the first turn (human) as prompt
+    with open(source_dataset_path, "r", encoding="utf-8") as f:
+        dataset = json.load(f)
+
+    dataset = [
+        data
+        for data in dataset
+        if len(data.get("conversations", data.get("conversation", []))) >= 2
+    ]
+    dataset = [
+        (
+            data.get("conversations", data.get("conversation", []))[0]["value"],
+            data.get("conversations", data.get("conversation", []))[1]["value"],
+        )
+        for data in dataset
+    ]
+    random.shuffle(dataset)
+
+    # Sample prompts, truncating or repeating tokens to reach target input length
+    input_requests = []
+    for data in dataset:
+        i = len(input_requests)
+        if i == batch_size:
+            break
+
+        prompt = data[0]
+        prompt_token_ids = tokenizer.encode(prompt)
+        prompt_len = len(prompt_token_ids)
+
+        if prompt_len == 0:
+            continue
+
+        if prompt_len > input_lens[i]:
+            input_ids = prompt_token_ids[: input_lens[i]]
+        else:
+            ratio = (input_lens[i] + prompt_len - 1) // prompt_len
+            input_ids = (prompt_token_ids * ratio)[: input_lens[i]]
+        input_content = tokenizer.decode(input_ids)
+        # Output format compatible with ais_bench ShareGPTDataset
+        input_requests.append(
+            {
+                "id": str(i),
+                "conversations": [
+                    {"from": "human", "value": input_content},
+                    {"from": "gpt", "value": "none"},
+                ],
+            }
+        )
+
+    print(f"#Input tokens: {np.sum(input_lens[:len(input_requests)])}")
+    print(f"#Output tokens: {np.sum(output_lens[:len(input_requests)])}")
+
+    output_dir = os.path.dirname(output_file)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+
+    # Output as JSON array format, compatible with ais_bench's json.load()
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(input_requests, f, ensure_ascii=False, indent=2)
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Generate GSM8K dataset with exact input token length"
+    )
+    parser.add_argument(
+        "--train_path", type=str, required=True, help="Path to GSM8K train.jsonl"
+    )
+    parser.add_argument(
+        "--test_path", type=str, required=True, help="Path to GSM8K test.jsonl"
+    )
+    parser.add_argument(
+        "--output_path", type=str, required=True, help="Output jsonl path"
+    )
+    parser.add_argument(
+        "--tokenizer_path", type=str, required=True, help="Path to model tokenizer"
+    )
+    parser.add_argument(
+        "--target_tokens", type=int, default=3500, help="Target input token length"
+    )
+    parser.add_argument(
+        "--trust_remote_code",
+        action="store_true",
+        help="Trust remote code for tokenizer",
+    )
+    parser.add_argument(
+        "--num_prompts",
+        type=int,
+        default=0,
+        help="Number of prompts to generate, 0 means all",
+    )
+    args = parser.parse_args()
+
+    output_data = generate_custom_dataset(
+        train_path=args.train_path,
+        test_path=args.test_path,
+        tokenizer_path=args.tokenizer_path,
+        target_tokens=args.target_tokens,
+        num_prompts=args.num_prompts,
+        trust_remote_code=args.trust_remote_code,
+    )
+    save_jsonl(output_data, args.output_path)
+    print(f"Done! Output {len(output_data)} items to {args.output_path}")
+
+
+if __name__ == "__main__":
+    main()

python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2

@@ -0,0 +1,134 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kube_config_map }}
+  namespace: {{ name_space }}
+data: {}
+
+---
+apiVersion: batch.volcano.sh/v1alpha1
+kind: Job
+metadata:
+  name: {{ kube_job_name }}
+  namespace: {{ name_space }}
+  labels:
+    ring-controller.atlas: ascend-1980
+    fault-scheduling: "force"
+spec:
+  minAvailable: {{ node_size }}
+  schedulerName: volcano
+  policies:
+    - event: PodEvicted
+      action: RestartJob
+  queue: default
+  tasks:
+  - name: "sglang-node"
+    replicas: {{ node_size }}
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        hostNetwork: True
+        containers:
+        - image: {{ image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: NAMESPACE
+            value: "{{ name_space }}"
+          - name: KUBE_CONFIG_MAP
+            value: "{{ kube_config_map }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               {% if env in ["ci", "debug"] %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+               {% endif %}
+               {% if env == "green" %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }}
+               {% endif %}
+          resources:
+            requests:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+            limits:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure

python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2

@@ -0,0 +1,107 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kube_config_map }}
+  namespace: {{ name_space }}
+data: {}
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ kube_job_name }}-sglang-node
+  namespace: {{ name_space }}
+spec:
+  replicas: {{ node_size }}
+  serviceName: {{ kube_job_name }}-sglang-pd-mix-headless
+  selector:
+    matchLabels:
+      app: sgl-ascend
+      task: pd-mix
+  template:
+    metadata:
+      labels:
+        app: sgl-ascend
+        task: pd-mix
+    spec:
+      hostNetwork: True
+      containers:
+      - image: {{ image }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        name: sgl-ascend
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: INSTALL_SGLANG_FROM_SOURCE
+          value: "{{ install_sglang_from_source }}"
+        - name: KUBECONFIG
+          value: "{{ kube_config }}"
+        - name: NAMESPACE
+          value: "{{ name_space }}"
+        - name: KUBE_CONFIG_MAP
+          value: "{{ kube_config_map }}"
+        - name: HF_ENDPOINT
+          value: "https://hf-mirror.com"
+        - name: METRICS_DATA_FILE
+          value: "{{ metrics_data_file }}"
+        - name: SGLANG_IS_IN_CI
+          value: "{{ sglang_is_in_ci }}"
+        - name: TRANSFORMERS_VERSION_FOR_SGLANG
+          value: "{{ transformers_version }}"
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+             bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+        resources:
+          requests:
+            huawei.com/Ascend910: 16
+          limits:
+            huawei.com/Ascend910: 16
+        volumeMounts:
+        - name: ascend-driver
+          mountPath: /usr/local/Ascend/driver
+        - name: localtime
+          mountPath: /etc/localtime
+        - name: share
+          mountPath: /data/ascend-ci-share-pkking-sglang
+        - name: share
+          mountPath: /root/.cache
+        - name: share
+          mountPath: /root/sglang
+          subPath: {{ sglang_source_relative_path }}
+      volumes:
+      - name: share
+        hostPath:
+          path: /home
+      - name: ascend-driver
+        hostPath:
+          path: /usr/local/Ascend/driver
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      nodeSelector:
+        accelerator: huawei-Ascend910
+        accelerator-type: module-a3-16
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ kube_job_name }}-sglang-pd-mix-headless
+  namespace: {{ name_space }}
+spec:
+  clusterIP: None
+  selector:
+    app: sgl-ascend
+    task: pd-mix
+  ports:
+  - port: 80
+    name: dummy-port

python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2

@@ -0,0 +1,350 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kube_config_map }}
+  namespace: {{ name_space }}
+data: {}
+
+---
+apiVersion: batch.volcano.sh/v1alpha1
+kind: Job
+metadata:
+  name: {{ kube_job_name }}
+  namespace: {{ name_space }}
+  labels:
+    ring-controller.atlas: ascend-1980
+    fault-scheduling: "force"
+spec:
+  minAvailable: {{ prefill_size + decode_size + router_size }}
+  schedulerName: volcano
+  policies:
+    - event: PodEvicted
+      action: RestartJob
+  queue: default
+  tasks:
+  - name: "sglang-prefill"
+    replicas: {{ prefill_size }}
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        hostNetwork: True
+        containers:
+        - image: {{ image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: NAMESPACE
+            value: "{{ name_space }}"
+          - name: KUBE_CONFIG_MAP
+            value: "{{ kube_config_map }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               {% if env in ["ci", "debug"] %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+               {% endif %}
+               {% if env == "green" %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }}
+               {% endif %}
+          resources:
+            requests:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+            limits:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure
+  - name: "sglang-decode"
+    replicas: {{ decode_size }}
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        hostNetwork: True
+        containers:
+        - image: {{ image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: NAMESPACE
+            value: "{{ name_space }}"
+          - name: KUBE_CONFIG_MAP
+            value: "{{ kube_config_map }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               {% if env in ["ci", "debug"] %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+               {% endif %}
+               {% if env == "green" %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }}
+               {% endif %}
+          resources:
+            requests:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+            limits:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure
+  - name: "sglang-router"
+    replicas: {{ router_size }}
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        hostNetwork: True
+        containers:
+        - image: {{ image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: NAMESPACE
+            value: "{{ name_space }}"
+          - name: KUBE_CONFIG_MAP
+            value: "{{ kube_config_map }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               {% if env in ["ci", "debug"] %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+               {% endif %}
+               {% if env == "green" %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }}
+               {% endif %}
+          resources:
+            requests:
+              cpu: "4"
+            limits:
+              cpu: "4"
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure

python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2

@@ -0,0 +1,307 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kube_config_map }}
+  namespace: {{ name_space }}
+data: {}
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ kube_job_name }}-sglang-prefill
+  namespace: {{ name_space }}
+spec:
+  replicas: {{ prefill_size }}
+  serviceName: {{ kube_job_name }}-sglang-prefill-headless
+  selector:
+    matchLabels:
+      app: sgl-ascend
+      task: prefill
+  template:
+    metadata:
+      labels:
+        app: sgl-ascend
+        task: prefill
+    spec:
+      restartPolicy: Always
+      hostNetwork: True
+      containers:
+      - image: {{ image }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        name: sgl-ascend
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: INSTALL_SGLANG_FROM_SOURCE
+          value: "{{ install_sglang_from_source }}"
+        - name: KUBECONFIG
+          value: "{{ kube_config }}"
+        - name: NAMESPACE
+          value: "{{ name_space }}"
+        - name: KUBE_CONFIG_MAP
+          value: "{{ kube_config_map }}"
+        - name: HF_ENDPOINT
+          value: "https://hf-mirror.com"
+        - name: METRICS_DATA_FILE
+          value: "{{ metrics_data_file }}"
+        - name: SGLANG_IS_IN_CI
+          value: "{{ sglang_is_in_ci }}"
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+             bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+             exit $?
+        resources:
+          requests:
+            huawei.com/Ascend910: 16
+          limits:
+            huawei.com/Ascend910: 16
+        volumeMounts:
+        - name: ascend-driver
+          mountPath: /usr/local/Ascend/driver
+        - name: localtime
+          mountPath: /etc/localtime
+        - name: share
+          mountPath: /data/ascend-ci-share-pkking-sglang
+        - name: share
+          mountPath: /root/.cache
+        - name: share
+          mountPath: /root/sglang
+          subPath: {{ sglang_source_relative_path }}
+      volumes:
+      - name: share
+        hostPath:
+          path: /home
+      - name: ascend-driver
+        hostPath:
+          path: /usr/local/Ascend/driver
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      nodeSelector:
+        accelerator: huawei-Ascend910
+        accelerator-type: module-a3-16
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ kube_job_name }}-sglang-prefill-headless
+  namespace: {{ name_space }}
+spec:
+  clusterIP: None
+  selector:
+    app: sgl-ascend
+    task: prefill
+  ports:
+  - port: 80
+    name: dummy-port
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ kube_job_name }}-sglang-decode
+  namespace: {{ name_space }}
+spec:
+  replicas: {{ decode_size }}
+  serviceName: {{ kube_job_name }}-sglang-decode-headless
+  selector:
+    matchLabels:
+      app: sgl-ascend
+      task: decode
+  template:
+    metadata:
+      labels:
+        app: sgl-ascend
+        task: decode
+    spec:
+      restartPolicy: Always
+      hostNetwork: True
+      containers:
+      - image: {{ image }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        name: sgl-ascend
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: INSTALL_SGLANG_FROM_SOURCE
+          value: "{{ install_sglang_from_source }}"
+        - name: KUBECONFIG
+          value: "{{ kube_config }}"
+        - name: NAMESPACE
+          value: "{{ name_space }}"
+        - name: KUBE_CONFIG_MAP
+          value: "{{ kube_config_map }}"
+        - name: HF_ENDPOINT
+          value: "https://hf-mirror.com"
+        - name: METRICS_DATA_FILE
+          value: "{{ metrics_data_file }}"
+        - name: SGLANG_IS_IN_CI
+          value: "{{ sglang_is_in_ci }}"
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+             bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+             exit $?
+        resources:
+          requests:
+            huawei.com/Ascend910: 16
+          limits:
+            huawei.com/Ascend910: 16
+        volumeMounts:
+        - name: ascend-driver
+          mountPath: /usr/local/Ascend/driver
+        - name: localtime
+          mountPath: /etc/localtime
+        - name: share
+          mountPath: /data/ascend-ci-share-pkking-sglang
+        - name: share
+          mountPath: /root/.cache
+        - name: share
+          mountPath: /root/sglang
+          subPath: {{ sglang_source_relative_path }}
+      volumes:
+      - name: share
+        hostPath:
+          path: /home
+      - name: ascend-driver
+        hostPath:
+          path: /usr/local/Ascend/driver
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      nodeSelector:
+        accelerator: huawei-Ascend910
+        accelerator-type: module-a3-16
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ kube_job_name }}-sglang-decode-headless
+  namespace: {{ name_space }}
+spec:
+  clusterIP: None
+  selector:
+    app: sgl-ascend
+    task: decode
+  ports:
+  - port: 80
+    name: dummy-port
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ kube_job_name }}-sglang-router
+  namespace: {{ name_space }}
+spec:
+  replicas: {{ router_size }}
+  serviceName: {{ kube_job_name }}-sglang-router-headless
+  selector:
+    matchLabels:
+      app: sgl-ascend
+      task: router
+  template:
+    metadata:
+      labels:
+        app: sgl-ascend
+        task: router
+    spec:
+      hostNetwork: True
+      containers:
+      - image: {{ image }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        name: sgl-ascend
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: INSTALL_SGLANG_FROM_SOURCE
+          value: "{{ install_sglang_from_source }}"
+        - name: KUBECONFIG
+          value: "{{ kube_config }}"
+        - name: NAMESPACE
+          value: "{{ name_space }}"
+        - name: KUBE_CONFIG_MAP
+          value: "{{ kube_config_map }}"
+        - name: HF_ENDPOINT
+          value: "https://hf-mirror.com"
+        - name: METRICS_DATA_FILE
+          value: "{{ metrics_data_file }}"
+        - name: SGLANG_IS_IN_CI
+          value: "{{ sglang_is_in_ci }}"
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+             bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+        resources:
+          requests:
+            cpu: 4
+          limits:
+            cpu: 4
+        volumeMounts:
+        - name: ascend-driver
+          mountPath: /usr/local/Ascend/driver
+        - name: localtime
+          mountPath: /etc/localtime
+        - name: share
+          mountPath: /data/ascend-ci-share-pkking-sglang
+        - name: share
+          mountPath: /root/.cache
+        - name: share
+          mountPath: /root/sglang
+          subPath: {{ sglang_source_relative_path }}
+      volumes:
+      - name: share
+        hostPath:
+          path: /home
+      - name: ascend-driver
+        hostPath:
+          path: /usr/local/Ascend/driver
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      nodeSelector:
+        accelerator: huawei-Ascend910
+        accelerator-type: module-a3-16
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ kube_job_name }}-sglang-router-headless
+  namespace: {{ name_space }}
+spec:
+  clusterIP: None
+  selector:
+    app: sgl-ascend
+    task: router
+  ports:
+  - port: 80
+    name: dummy-port

python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2

@@ -0,0 +1,134 @@
+apiVersion: batch.volcano.sh/v1alpha1
+kind: Job
+metadata:
+  name: {{ kube_job_name }}
+  namespace: {{ name_space }}
+  labels:
+    ring-controller.atlas: ascend-1980
+    fault-scheduling: "force"
+spec:
+  minAvailable: 1
+  schedulerName: volcano
+  policies:
+    - event: PodEvicted
+      action: RestartJob
+  {% if env == "green" %}
+  queue: default
+  {% endif %}
+  tasks:
+  - name: "pod"
+    replicas: 1
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        containers:
+        - image: {{ image }}
+          {% if env == "green" %}
+          imagePullPolicy: IfNotPresent
+          {% else %}
+          imagePullPolicy: Always
+          {% endif %}
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+          resources:
+            requests:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: {{ npu_size }}
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: {{ npu_size }}
+              memory: 128Gi
+              {% endif %}
+              cpu: "46"
+            limits:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: {{ npu_size }}
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: {{ npu_size }}
+              memory: 128Gi
+              {% endif %}
+              cpu: "46"
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: shm-volume
+            mountPath: /dev/shm
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: shm-volume
+          emptyDir:
+            medium: Memory
+            sizeLimit: "16Gi"
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure

python/sglang/test/ascend/e2e/run_evalscope.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+PYTHON_ENV_FOR_EVALSCOPE=test_env_evalscope
+PIP_FOR_EVALSCOPE=${PYTHON_ENV_FOR_EVALSCOPE}/bin/pip
+EVALSCOPE_SOURCE_PATH=/root/.cache/.cache/evalscope
+pip_mirror_source="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
+
+if [ -d "${PYTHON_ENV_FOR_EVALSCOPE}" ]; then
+    echo "Virtual env ${PYTHON_ENV_FOR_EVALSCOPE} already exists, skip installation."
+    exit 0
+fi
+
+echo "===== Install evalscope in virtual env - Begin ====="
+python -m venv ${PYTHON_ENV_FOR_EVALSCOPE}
+
+if [ ! -d "${EVALSCOPE_SOURCE_PATH}" ]; then
+    echo "The evalscope source does not exist: ${EVALSCOPE_SOURCE_PATH}."
+    echo "Install evalscope online."
+    ${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source}
+    ${PIP_FOR_EVALSCOPE} install evalscope -i ${pip_mirror_source}
+else
+    echo "Install evalscope from local source: ${EVALSCOPE_SOURCE_PATH}"
+    ${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source}
+    ${PIP_FOR_EVALSCOPE} install -e ${EVALSCOPE_SOURCE_PATH} -i ${pip_mirror_source}
+fi
+echo "===== Install evalscope in virtual env - End ====="

python/sglang/test/ascend/e2e/run_npu_e2e_test.py

@@ -0,0 +1,890 @@
+import argparse
+import json
+import logging
+import os
+import random
+import re
+import string
+import subprocess
+import time
+import uuid
+
+import psutil
+import yaml
+from jinja2 import Template
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
+KUBE_CONFIG = os.environ.get("KUBECONFIG")
+logger.info(f"KUBE_CONFIG: {KUBE_CONFIG}")
+config.load_kube_config(KUBE_CONFIG)
+core_api = client.CoreV1Api()
+custom_api = client.CustomObjectsApi()
+batch_api = client.BatchV1Api()
+rbac_api = client.RbacAuthorizationV1Api()
+
+LOCAL_TIMEOUT = 10800
+
+script_path = os.path.dirname(os.path.abspath(__file__))
+
+KUBE_JOB_SINGLE = "single"
+KUBE_JOB_MULTI_PD_MIX = "multi-pd-mix"
+KUBE_JOB_MULTI_PD_SEPARATION = "multi-pd-separation"
+KUBE_JOB_MULTI_PD_MIX_GREEN = "multi-pd-mix-green"
+KUBE_JOB_MULTI_PD_SEPARATION_GREEN = "multi-pd-separation-green"
+KUBE_YAML_TEMPLATE = {
+    KUBE_JOB_SINGLE: f"{script_path}/k8s_single.yaml.jinja2",
+    KUBE_JOB_MULTI_PD_MIX: f"{script_path}/k8s_multi_pd_mix.yaml.jinja2",
+    KUBE_JOB_MULTI_PD_MIX_GREEN: f"{script_path}/k8s_multi_pd_mix_green.yaml.jinja2",
+    KUBE_JOB_MULTI_PD_SEPARATION: f"{script_path}/k8s_multi_pd_separation.yaml.jinja2",
+    KUBE_JOB_MULTI_PD_SEPARATION_GREEN: f"{script_path}/k8s_multi_pd_separation_green.yaml.jinja2",
+}
+
+
+def get_unique_random_string(length: int = 16, add_random: bool = True) -> str:
+    """Generate a random string."""
+    uuid_str = str(uuid.uuid4()).replace("-", "")
+
+    if add_random:
+        if length < 8:
+            raise ValueError("length can not be smaller than 8")
+        random_length = length - 8
+        char_pool = string.ascii_lowercase + string.digits
+        random_chars = "".join([random.choice(char_pool) for _ in range(random_length)])
+        result = uuid_str[:8] + random_chars
+    else:
+        result = uuid_str[:length]
+
+    return result
+
+
+def create_kube_yaml(kube_yaml_template, output_yaml, pod_context):
+    """Create a k8s config yaml file"""
+    with open(kube_yaml_template, "r") as f:
+        template = Template(f.read())
+    kube_pod_yaml = template.render(pod_context)
+    with open(output_yaml, "w") as f:
+        f.write(kube_pod_yaml)
+    logger.info(f"Pod YAML written to {output_yaml}")
+
+
+def create_pod(yaml_file, namespace):
+    """Create a pod by k8s config yaml file"""
+    with open(yaml_file, "r", encoding="utf-8") as f:
+        yaml_docs = list(yaml.safe_load_all(f))
+
+    for doc in yaml_docs:
+        if not doc:
+            continue
+
+        kind = doc.get("kind")
+        api_version = doc.get("apiVersion")
+
+        try:
+            if kind == "Pod" and api_version == "v1":
+                core_api.create_namespaced_pod(namespace=namespace, body=doc)
+                logger.info(f"Pod {doc['metadata']['name']} created")
+
+            elif kind == "Job" and api_version == "batch/v1":
+                batch_api.create_namespaced_job(namespace=namespace, body=doc)
+                logger.info(f"Job {doc['metadata']['name']} is created")
+
+            elif kind == "Job" and api_version == "batch.volcano.sh/v1alpha1":
+                response = custom_api.create_namespaced_custom_object(
+                    group="batch.volcano.sh",
+                    version="v1alpha1",
+                    namespace=namespace,
+                    plural="jobs",
+                    body=doc,
+                )
+                logger.info(f"Volcano job {doc['metadata']['name']} is created")
+                logger.debug(response)
+
+            elif kind == "ConfigMap" and api_version == "v1":
+                core_api.create_namespaced_config_map(namespace=namespace, body=doc)
+                logger.info(f"ConfigMap {doc['metadata']['name']} is created")
+
+            elif kind == "Role" and api_version == "rbac.authorization.k8s.io/v1":
+                rbac_api.create_namespaced_role(namespace=namespace, body=doc)
+                logger.info(f"Role {doc['metadata']['name']} is created")
+
+            elif (
+                kind == "RoleBinding" and api_version == "rbac.authorization.k8s.io/v1"
+            ):
+                rbac_api.create_namespaced_role_binding(namespace=namespace, body=doc)
+                logger.info(f"RoleBinding {doc['metadata']['name']} is created")
+
+            elif kind == "Deployment" and api_version == "apps/v1":
+                apps_api = client.AppsV1Api()
+                apps_api.create_namespaced_deployment(namespace=namespace, body=doc)
+                logger.info(f"Deployment {doc['metadata']['name']} is created")
+
+            elif kind == "StatefulSet" and api_version == "apps/v1":
+                apps_api = client.AppsV1Api()
+                apps_api.create_namespaced_stateful_set(namespace=namespace, body=doc)
+                logger.info(f"StatefulSet {doc['metadata']['name']} is created")
+
+            elif kind == "Service" and api_version == "v1":
+                core_api.create_namespaced_service(namespace=namespace, body=doc)
+                logger.info(f"Service {doc['metadata']['name']} is created")
+
+            else:
+                raise f"Unrecognized kind: {kind}/{api_version}"
+        except ApiException as e:
+            print(f"create resource {kind} error: {e}")
+            raise
+
+
+def delete_pod(yaml_file, namespace):
+    """Delete k8s pod by config yaml file"""
+    with open(yaml_file, "r", encoding="utf-8") as f:
+        yaml_docs = list(yaml.safe_load_all(f))
+    for doc in yaml_docs:
+        if not doc:
+            continue
+
+        kind = doc.get("kind")
+        api_version = doc.get("apiVersion")
+        try:
+            if kind == "Job" and api_version == "batch.volcano.sh/v1alpha1":
+                job_name = doc["metadata"]["name"]
+                response = custom_api.delete_namespaced_custom_object(
+                    group="batch.volcano.sh",
+                    version="v1alpha1",
+                    namespace=namespace,
+                    plural="jobs",
+                    name=job_name,
+                    body=client.V1DeleteOptions(
+                        grace_period_seconds=0, propagation_policy="Foreground"
+                    ),
+                )
+                logger.info(f"Deleted job {job_name}")
+                logger.info(f"Response status: {response.get('status')}")
+            elif kind == "ConfigMap" and api_version == "v1":
+                config_map_name = doc["metadata"]["name"]
+                core_api.delete_namespaced_config_map(
+                    name=config_map_name, namespace=namespace
+                )
+                print(f"ConfigMap {config_map_name} is deleted.")
+            elif kind == "Deployment" and api_version == "apps/v1":
+                deployment_name = doc["metadata"]["name"]
+                apps_api = client.AppsV1Api()
+                apps_api.delete_namespaced_deployment(
+                    name=deployment_name,
+                    namespace=namespace,
+                    body=client.V1DeleteOptions(
+                        grace_period_seconds=0, propagation_policy="Foreground"
+                    ),
+                )
+                logger.info(f"Deployment {deployment_name} is deleted.")
+
+            elif kind == "StatefulSet" and api_version == "apps/v1":
+                statefulset_name = doc["metadata"]["name"]
+                apps_api = client.AppsV1Api()
+                apps_api.delete_namespaced_stateful_set(
+                    name=statefulset_name,
+                    namespace=namespace,
+                    body=client.V1DeleteOptions(
+                        grace_period_seconds=0, propagation_policy="Foreground"
+                    ),
+                )
+                logger.info(f"StatefulSet {statefulset_name} is deleted.")
+
+            elif kind == "Service" and api_version == "v1":
+                service_name = doc["metadata"]["name"]
+                core_api.delete_namespaced_service(
+                    name=service_name,
+                    namespace=namespace,
+                    body=client.V1DeleteOptions(
+                        grace_period_seconds=0, propagation_policy="Foreground"
+                    ),
+                )
+                logger.info(f"Service {service_name} is deleted.")
+
+            else:
+                raise f"Unrecognized kind: {kind}/{api_version}"
+        except ApiException as e:
+            raise f"delete resource {kind} error: {e}"
+
+
+def check_parent_process():
+    """Check parent process is alive or not."""
+    try:
+        parent_pid = os.getppid()
+        psutil.Process(parent_pid)
+        return True
+    except psutil.NoSuchProcess:
+        return False
+
+
+def check_pods_ready(namespace, pod_name_key_str, timeout=300):
+    """Waiting for all k8s pods are ready"""
+    logger.info("Waiting all pods to running...")
+    start_time = time.time()
+
+    while time.time() - start_time < timeout:
+        if not check_parent_process():
+            raise Exception("Parent process exited.")
+
+        pods = core_api.list_namespaced_pod(namespace=namespace)
+
+        if len(pods.items) == 0:
+            time.sleep(5)
+            continue
+
+        all_running = True
+        sglang_pods_found = False
+        for pod in pods.items:
+            pod_name = pod.metadata.name
+            if pod_name_key_str not in pod_name:
+                continue
+
+            sglang_pods_found = True
+            status = pod.status
+            phase = status.phase
+            logger.info(f"Pod: {pod_name}, status: {phase}")
+            if phase != "Running":
+                all_running = False
+                break
+
+            containers_ready = True
+            for condition in status.conditions:
+                if condition.type == "Ready" and condition.status != "True":
+                    containers_ready = False
+                    break
+
+            if not containers_ready:
+                all_running = False
+                break
+
+        if not sglang_pods_found:
+            logger.info("No sglang pod, waiting...")
+            time.sleep(5)
+            continue
+        if all_running:
+            logger.info("All sglang Pod is Running !")
+            return True
+
+        time.sleep(5)
+
+    logger.info(f"timeout in {timeout}s")
+    return False
+
+
+def create_or_update_configmap(cm_name: str, data: dict, namespace: str):
+    """Create a k8s configmap or update it if already exists"""
+    cm_metadata = client.V1ObjectMeta(name=cm_name, namespace=namespace)
+    configmap = client.V1ConfigMap(
+        api_version="v1", kind="ConfigMap", metadata=cm_metadata, data=data
+    )
+
+    try:
+        response = core_api.create_namespaced_config_map(
+            namespace=namespace, body=configmap
+        )
+        logger.info(f"ConfigMap '{cm_name}' create successfully!")
+        logger.info(f"data: {list(data.keys())}")
+        return response
+    except ApiException as e:
+        if e.status == 409:
+            logger.info(f"ConfigMap {cm_name} already exists. Updating...")
+            response = core_api.replace_namespaced_config_map(
+                namespace=namespace, name=cm_name, body=configmap
+            )
+            logger.info(f"ConfigMap {cm_name} updated successfully.")
+            return response
+        else:
+            error_msg = f"ConfigMap create failed: {e.reason}"
+            if e.body:
+                error_msg += f" | details: {e.body}"
+            logger.info(error_msg)
+            raise
+
+
+def prepare_cm_data(namespace, pod_string):
+    """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information."""
+    pods = core_api.list_namespaced_pod(namespace=namespace)
+    data = {}
+    for pod in pods.items:
+        pod_name = pod.metadata.name
+        if pod_string in pod_name:
+            pod_ip = pod.status.pod_ip
+            data[pod_name] = pod_ip
+    return data
+
+
+def monitor_pod_logs(
+    kube_job_type, kube_job_prefix_name, namespace, timeout=LOCAL_TIMEOUT
+):
+    """Monitor the logs of the specified pod until the special pattern is matched or reaches its timeout."""
+    monitor_pod_name = {
+        KUBE_JOB_SINGLE: f"{kube_job_prefix_name}-pod-0",
+        KUBE_JOB_MULTI_PD_MIX: f"{kube_job_prefix_name}-sglang-node-0",
+        KUBE_JOB_MULTI_PD_SEPARATION: f"{kube_job_prefix_name}-sglang-router-0",
+    }
+    pod_name = monitor_pod_name.get(kube_job_type)
+
+    # Build kubectl command
+    cmd = ["kubectl", "logs", "-f", "-n", namespace, pod_name]
+
+    # Define multiline pattern to match
+    pattern_lines = [
+        r"^-{70,}$",
+        r"^Ran \d+ tests? in [\d.]+s$",
+        r"^$",
+        r"^(OK|FAILED \(errors=\d+\))$",
+    ]
+    patterns = [re.compile(line_pattern) for line_pattern in pattern_lines]
+    pattern_ok = re.compile(r"^OK$")
+
+    process = None
+    try:
+        # Start kubectl logs process
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+            bufsize=1,
+        )
+
+        logger.info(f"Starting to monitor logs for Pod: {pod_name}")
+        match_state = 0
+        is_success = False
+
+        # Use two threads: one for reading logs, one for checking pod status
+        import threading
+
+        # Shared variables
+        match_event = threading.Event()
+        pod_error_event = threading.Event()
+
+        def read_logs():
+            """Thread function to read logs continuously"""
+            nonlocal is_success, match_state
+
+            while process.poll() is None and not match_event.is_set():
+                line = process.stdout.readline()
+                if line:
+                    line = line.rstrip("\n")
+                    print(line)
+                    # Check if current line matches expected pattern
+                    if match_state < len(patterns) and patterns[match_state].match(
+                        line
+                    ):
+                        match_state += 1
+                        if match_state == len(patterns):
+                            if pattern_ok.match(line):
+                                is_success = True
+                            logger.info("Detected complete test completion pattern!")
+                            match_event.set()
+                    else:
+                        match_state = 0
+                        if patterns[0].match(line):
+                            match_state = 1
+
+            # Read remaining output after process exits
+            if not match_event.is_set():
+                remaining_output, stderr_output = process.communicate()
+                if remaining_output:
+                    print(remaining_output)
+                if stderr_output:
+                    logger.error(f"kubectl command error: {stderr_output}")
+                    pod_error_event.set()
+
+        def check_pods_running(namespace, pod_name_key_str):
+            """check pods are running"""
+            pods = core_api.list_namespaced_pod(namespace=namespace)
+            if len(pods.items) == 0:
+                logger.warning(f"No pods found in the namespace {namespace}")
+                return False
+
+            for pod in pods.items:
+                pod_name = pod.metadata.name
+                if pod_name_key_str not in pod_name:
+                    continue
+                status = pod.status
+                phase = status.phase
+                if phase != "Running":
+                    logger.error(f"Pod {pod_name} is not running, status: {phase}")
+                    return False
+
+            return True
+
+        def check_pod_status():
+            """Thread function to check pod status periodically"""
+            start_time = time.time()
+            while not match_event.is_set() and not pod_error_event.is_set():
+                if time.time() - start_time > timeout:
+                    pod_error_event.set()
+                    break
+
+                if not check_parent_process():
+                    logger.error(f"Parent process exited. Exiting...")
+                    pod_error_event.set()
+                    break
+
+                if not check_pods_running(
+                    namespace=namespace, pod_name_key_str=kube_job_prefix_name
+                ):
+                    logger.error(
+                        f"Some pods are not running properly. Please check the sglang logs on these pods. Exiting..."
+                    )
+                    pod_error_event.set()
+                    break
+
+                # Sleep for a short time before next check
+                time.sleep(0.5)
+
+        # Start threads
+        log_thread = threading.Thread(target=read_logs)
+        status_thread = threading.Thread(target=check_pod_status)
+
+        log_thread.daemon = True
+        status_thread.daemon = True
+
+        log_thread.start()
+        status_thread.start()
+
+        # Wait for either match event or error event
+        start_time = time.time()
+        while not match_event.is_set() and not pod_error_event.is_set():
+            if time.time() - start_time > timeout:
+                raise Exception(
+                    f"Timeout exceeded, the thread is {timeout} seconds long."
+                )
+            time.sleep(0.1)
+
+        # Check if pattern was successfully matched
+        if not match_event.is_set():
+            if process.poll() is not None:
+                remaining_output, stderr_output = process.communicate()
+                if remaining_output:
+                    logger.info(remaining_output)
+                if stderr_output:
+                    raise Exception(f"kubectl command error: {stderr_output}")
+                else:
+                    raise Exception(
+                        "Pod logs ended but target pattern was not detected"
+                    )
+            else:
+                raise Exception("Monitoring ended but target pattern was not detected")
+        elif not is_success:
+            raise Exception("The test result was FAILED!")
+        else:
+            logger.info("The test result was OK!")
+    finally:
+        if process and process.poll() is None:
+            process.terminate()
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                process.kill()
+
+
+def generate_metrics_json(metrics_data_file, test_case, status):
+    log_file = os.path.join(metrics_data_file, "test_output.log")
+
+    metrics = {}
+    baselines = {}
+
+    if os.path.exists(log_file):
+        with open(log_file, "r") as f:
+            for line in f:
+                m = re.match(r"\[METRIC\] (\S+)=(\S+)", line.strip())
+                if m:
+                    key = m.group(1)
+                    value = m.group(2)
+                    try:
+                        value = float(value)
+                    except ValueError:
+                        pass
+                    if key.endswith("_baseline"):
+                        baselines[key[:-9]] = value
+                    else:
+                        metrics[key] = value
+    else:
+        logger.warning(f"Metrics log file not found: {log_file}")
+
+    tc_name = test_case.rsplit("/", 1)[-1].rsplit(".", 1)[0]
+
+    test_type = "unknown"
+    parts = metrics_data_file.split("/")
+    for i, part in enumerate(parts):
+        if part == "output" and i + 1 < len(parts):
+            test_type = parts[i + 1]
+            break
+
+    output = {
+        "test_case": tc_name,
+        "test_type": test_type,
+        "status": status,
+        "metrics": metrics,
+        "baselines": baselines,
+    }
+
+    output_path = os.path.join(metrics_data_file, "metrics.json")
+    with open(output_path, "w") as f:
+        json.dump(output, f, indent=2)
+    logger.info(f"Metrics JSON written to {output_path}")
+
+    with open("/tmp/metrics.json", "w") as f:
+        json.dump(output, f, indent=2)
+    logger.info("Metrics JSON written to /tmp/metrics.json")
+
+
+def run_npu_e2e_test_case(
+    docker_image_url: str,
+    kube_name_space: str,
+    kube_job_type: str,
+    kube_job_name_prefix: str,
+    resource_info: dict,
+    sglang_source_relative_path: str,
+    metrics_data_file: str,
+    test_case: str,
+    sglang_is_in_ci=False,
+    install_sglang_from_source=False,
+    env="debug",
+    trouble_shotting=False,
+    transformers_version="",
+):
+    """The method for running a npu e2e test case.
+    Args:
+        docker_image_url (str): the url of docker image for creating k8s pods.
+        kube_name_space (str): the namespace of the k8s.
+        kube_job_name_prefix (str): the prefix of the k8s job name which will be set as the prefix of the pod name.
+        resource_info (dict): the number of k8s nodes used by the testcase.
+            for pd-separation as: {"prefill_size": 1, "decode_size": 1, "router_size": 1};
+            for pd-mix as: {"node_size": 2; single: {"npu_size": 4}
+        sglang_source_relative_path (str): the relative path of the sglang source on shared-disk.
+        metrics_data_file (str): the output path of the metrics data file, only for performance testing.
+        test_case (str): the test case relative path in sglang source root path. like test/registered/...
+        sglang_is_in_ci (bool): whether running in CI environment.
+        install_sglang_from_source (bool): whether installing sglang from source or use docker image directly.
+        env (str): the environment to run the test on.  Choose one in ["debug", "ci"]
+    """
+    random_str = get_unique_random_string(16, True)
+
+    kube_config_map = f"sglang-configmap-{random_str}"
+    final_kube_job_name = f"{kube_job_name_prefix}-{random_str}"
+
+    kube_yaml_file_dict = {
+        KUBE_JOB_SINGLE: f"k8s_single_{random_str}.yaml",
+        KUBE_JOB_MULTI_PD_MIX: f"k8s_multi_pd_mix_{random_str}.yaml",
+        KUBE_JOB_MULTI_PD_SEPARATION: f"k8s_multi_pd_separation_{random_str}.yaml",
+    }
+    kube_yaml_file = kube_yaml_file_dict.get(kube_job_type)
+
+    try:
+        logger.info(
+            f"Apply k8s yaml... KUBE_NAME_SPACE:{kube_name_space}, KUBE_CONFIG_MAP:{kube_config_map}, "
+            f"KUBE_JOB_TYPE:{kube_job_type}, KUBE_YAML_FILE:{kube_yaml_file}"
+        )
+
+        if kube_job_type == KUBE_JOB_SINGLE:
+            k8s_context = {
+                "image": docker_image_url,
+                "name_space": kube_name_space,
+                "kube_job_name": final_kube_job_name,
+                "kube_config": KUBE_CONFIG,
+                "npu_size": resource_info["npu_size"],
+                "sglang_source_relative_path": sglang_source_relative_path,
+                "metrics_data_file": metrics_data_file,
+                "test_case": test_case,
+                "sglang_is_in_ci": sglang_is_in_ci,
+                "install_sglang_from_source": install_sglang_from_source,
+                "env": env,
+                "trouble_shotting": trouble_shotting,
+                "transformers_version": transformers_version,
+            }
+            create_kube_yaml(
+                kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type),
+                output_yaml=kube_yaml_file,
+                pod_context=k8s_context,
+            )
+        elif kube_job_type == KUBE_JOB_MULTI_PD_MIX:
+            k8s_context = {
+                "image": docker_image_url,
+                "name_space": kube_name_space,
+                "kube_job_name": final_kube_job_name,
+                "kube_config": KUBE_CONFIG,
+                "kube_config_map": kube_config_map,
+                "node_size": resource_info["node_size"],
+                "sglang_source_relative_path": sglang_source_relative_path,
+                "metrics_data_file": metrics_data_file,
+                "test_case": test_case,
+                "sglang_is_in_ci": sglang_is_in_ci,
+                "install_sglang_from_source": install_sglang_from_source,
+                "env": env,
+                "trouble_shotting": trouble_shotting,
+                "transformers_version": transformers_version,
+            }
+            template_key = (
+                KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type
+            )
+            create_kube_yaml(
+                kube_yaml_template=KUBE_YAML_TEMPLATE.get(template_key),
+                output_yaml=kube_yaml_file,
+                pod_context=k8s_context,
+            )
+        elif kube_job_type == KUBE_JOB_MULTI_PD_SEPARATION:
+            k8s_context = {
+                "image": docker_image_url,
+                "name_space": kube_name_space,
+                "kube_job_name": final_kube_job_name,
+                "kube_config": KUBE_CONFIG,
+                "kube_config_map": kube_config_map,
+                "prefill_size": resource_info["prefill_size"],
+                "decode_size": resource_info["decode_size"],
+                "router_size": resource_info["router_size"],
+                "sglang_source_relative_path": sglang_source_relative_path,
+                "metrics_data_file": metrics_data_file,
+                "test_case": test_case,
+                "sglang_is_in_ci": sglang_is_in_ci,
+                "install_sglang_from_source": install_sglang_from_source,
+                "env": env,
+                "trouble_shotting": trouble_shotting,
+                "transformers_version": transformers_version,
+            }
+            template_key = (
+                KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type
+            )
+            create_kube_yaml(
+                kube_yaml_template=KUBE_YAML_TEMPLATE.get(template_key),
+                output_yaml=kube_yaml_file,
+                pod_context=k8s_context,
+            )
+        else:
+            raise Exception(f"Unknown k8s job type: {kube_job_type}")
+
+        create_pod(yaml_file=kube_yaml_file, namespace=kube_name_space)
+
+        if check_pods_ready(
+            kube_name_space, final_kube_job_name, timeout=LOCAL_TIMEOUT
+        ):
+            if kube_job_type != "single":
+                matching_pod_string = final_kube_job_name
+                cm_data = prepare_cm_data(kube_name_space, matching_pod_string)
+                if not cm_data:
+                    logger.info(
+                        f"No sglang pod found while matching {matching_pod_string}"
+                    )
+
+                response = create_or_update_configmap(
+                    cm_name=kube_config_map, data=cm_data, namespace=kube_name_space
+                )
+                logger.info(response)
+        else:
+            logger.info("Pod not ready, maybe not enough resource")
+
+        monitor_success = False
+        try:
+            monitor_pod_logs(
+                kube_job_type, final_kube_job_name, kube_name_space, LOCAL_TIMEOUT
+            )
+            monitor_success = True
+        except Exception:
+            logger.error(f"Test case failed: {test_case}", exc_info=True)
+            raise
+        finally:
+            if metrics_data_file:
+                status = "pass" if monitor_success else "fail"
+                try:
+                    generate_metrics_json(metrics_data_file, test_case, status)
+                except Exception as e:
+                    logger.error(f"Failed to generate metrics JSON: {e}", exc_info=True)
+    finally:
+        if os.path.exists(kube_yaml_file):
+            # Don't delete pod when trouble_shotting is enabled
+            if not trouble_shotting:
+                delete_pod(yaml_file=kube_yaml_file, namespace=kube_name_space)
+                os.remove(kube_yaml_file)
+            else:
+                logger.info(
+                    f"Trouble shooting mode enabled, keeping pod {final_kube_job_name} alive"
+                )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Apply k8s yaml", formatter_class=argparse.RawTextHelpFormatter
+    )
+
+    parser.add_argument(
+        "--image",
+        type=str,
+        required=True,
+        help="Docker image to use",
+    )
+
+    parser.add_argument(
+        "--prefill-size",
+        type=int,
+        required=False,
+        default=1,
+        help="Number of prefill nodes",
+    )
+
+    parser.add_argument(
+        "--decode-size",
+        type=int,
+        required=False,
+        default=1,
+        help="Number of decode nodes",
+    )
+
+    parser.add_argument(
+        "--router-size",
+        type=int,
+        required=False,
+        default=1,
+        help="Number of router nodes",
+    )
+
+    parser.add_argument(
+        "--node-size",
+        type=int,
+        required=False,
+        default=2,
+        help="Number of nodes for multi-node-pd-mix scenario",
+    )
+
+    parser.add_argument(
+        "--npu-size",
+        type=int,
+        required=False,
+        default=0,
+        help="Number of npu for single-node scenario",
+    )
+
+    parser.add_argument(
+        "--sglang-source-relative-path",
+        type=str,
+        required=True,
+        help="Sglang source code relative path on shared-disk(NFS_ROOT_PATH: /data/ascend-ci-share-pkking-sglang/)",
+    )
+
+    parser.add_argument(
+        "--metrics-data-file",
+        type=str,
+        required=False,
+        default="",
+        help="Metrics data file",
+    )
+
+    parser.add_argument(
+        "--test-case",
+        type=str,
+        required=True,
+        help="Test case path",
+    )
+
+    parser.add_argument(
+        "--sglang-is-in-ci",
+        action="store_true",
+        help="Used to set env var SGLANG_IS_IN_CI in pod",
+    )
+
+    parser.add_argument(
+        "--install-sglang-from-source",
+        action="store_true",
+        help="Used to set env var INSTALL_SGLANG_FROM_SOURCE in pod",
+    )
+
+    parser.add_argument(
+        "--kube-name-space",
+        type=str,
+        required=True,
+        help="K8s name space",
+    )
+
+    parser.add_argument(
+        "--kube-job-type",
+        type=str,
+        choices=[KUBE_JOB_SINGLE, KUBE_JOB_MULTI_PD_MIX, KUBE_JOB_MULTI_PD_SEPARATION],
+        required=True,
+        help=f"K8s job type [{KUBE_JOB_SINGLE}, {KUBE_JOB_MULTI_PD_MIX}, {KUBE_JOB_MULTI_PD_SEPARATION}]",
+    )
+
+    parser.add_argument(
+        "--kube-job-name-prefix",
+        type=str,
+        required=True,
+        help="K8s job name prefix",
+    )
+
+    parser.add_argument(
+        "--env",
+        type=str,
+        choices=["debug", "ci", "green"],
+        required=True,
+        help="Environment type",
+    )
+
+    parser.add_argument(
+        "--trouble-shotting",
+        action="store_true",
+        help="Used for troubleshotting issues, such as retaining pods",
+    )
+
+    parser.add_argument(
+        "--transformers-version",
+        type=str,
+        required=False,
+        default="",
+        help="The transformers version number for running sglang. Use default version in image if keep empty.",
+    )
+
+    args = parser.parse_args()
+
+    docker_image_url = args.image
+    npu_size = int(args.npu_size)
+    node_size = int(args.node_size)
+    prefill_size = int(args.prefill_size)
+    decode_size = int(args.decode_size)
+    router_size = int(args.router_size)
+    sglang_source_relative_path = args.sglang_source_relative_path
+    metrics_data_file = args.metrics_data_file
+    test_case = args.test_case
+    sglang_is_in_ci = args.sglang_is_in_ci
+    install_sglang_from_source = args.install_sglang_from_source
+    env = args.env
+    trouble_shotting = args.trouble_shotting
+    transformers_version = args.transformers_version
+
+    kube_name_space = args.kube_name_space
+    kube_job_type = args.kube_job_type
+    kube_job_name_prefix = args.kube_job_name_prefix
+
+    resource_info_dict = {
+        KUBE_JOB_SINGLE: {"npu_size": npu_size},
+        KUBE_JOB_MULTI_PD_MIX: {"node_size": node_size},
+        KUBE_JOB_MULTI_PD_SEPARATION: {
+            "prefill_size": prefill_size,
+            "decode_size": decode_size,
+            "router_size": router_size,
+        },
+    }
+
+    run_npu_e2e_test_case(
+        docker_image_url=docker_image_url,
+        kube_name_space=kube_name_space,
+        kube_job_type=kube_job_type,
+        kube_job_name_prefix=kube_job_name_prefix,
+        resource_info=resource_info_dict.get(kube_job_type),
+        sglang_source_relative_path=sglang_source_relative_path,
+        metrics_data_file=metrics_data_file,
+        test_case=test_case,
+        sglang_is_in_ci=sglang_is_in_ci,
+        install_sglang_from_source=install_sglang_from_source,
+        env=env,
+        trouble_shotting=trouble_shotting,
+        transformers_version=transformers_version,
+    )

python/sglang/test/ascend/e2e/run_npu_testcase.sh

@@ -0,0 +1,151 @@
+test_case=$1
+
+sglang_source_path=/root/sglang
+if [ ! -f "${sglang_source_path}/${test_case}" ];then
+  echo "The test case file is not exist: $test_case"
+  exit 0
+fi
+
+echo "NPU info:"
+npu-smi info
+
+echo "===== Install kubernetes - Begin ====="
+KUBERNETES_PKG_PATH_SOURCE=/root/.cache/.cache/kubernetes
+if [ ! -d "${KUBERNETES_PKG_PATH_SOURCE}" ]; then
+  echo "Install kubernetes online."
+  pip install kubernetes -i -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+else
+  echo "Install kubernetes locally."
+  cp -r ${KUBERNETES_PKG_PATH_SOURCE} /tmp/
+  pip install --no-index --find-links=/tmp/kubernetes/ kubernetes
+fi
+echo "===== Install kubernetes - End ====="
+
+PYTHON_FOR_SGLANG="python"
+PIP_FOR_SGLANG="pip"
+if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
+  echo "===== Install transformers for sglang - Begin ====="
+  TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
+  if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
+    echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
+    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
+    pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+  else
+    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
+    TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
+    mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
+    cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
+    pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
+  fi
+  echo "===== Install transformers for sglang in virtual env - End ====="
+fi
+
+if [ -n "${TRANSFORMERS_VERSION_FOR_TEST_TOOL}" ]; then
+  # Example: TRANSFORMERS_VERSION_FOR_TEST_TOOL=4.57.6
+  echo "===== Install transformers in virtual env for test tools - Begin ====="
+  PYTHON_ENV_FOR_TEST_TOOL=python_venv_for_test_tool
+  PIP_FOR_TEST_TOOL=${PYTHON_ENV_FOR_TEST_TOOL}/bin/pip
+  python -m venv ${PYTHON_ENV_FOR_TEST_TOOL} --system-site-packages
+  TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_TEST_TOOL}
+  if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
+    echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
+    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_TEST_TOOL} online."
+    ${PIP_FOR_TEST_TOOL} install transformers==${TRANSFORMERS_VERSION_FOR_TEST_TOOL} -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+  else
+    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_TEST_TOOL} locally."
+    TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_TEST_TOOL}
+    mkdir -p ${TRANSFORMERS_PKG_PATH_TARGET}
+    cp ${TRANSFORMERS_PKG_PATH_SOURCE}/* ${TRANSFORMERS_PKG_PATH_TARGET}/
+    ${PIP_FOR_TEST_TOOL} install --no-index --find-links=${TRANSFORMERS_PKG_PATH_TARGET} transformers==${TRANSFORMERS_VERSION_FOR_TEST_TOOL}
+  fi
+  echo "===== Install transformers in virtual env for test tools - End ====="
+  echo "Transformers version for test tools: $(${PIP_FOR_TEST_TOOL} show transformers | grep Version | cut -d: -f2)"
+fi
+
+echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
+
+# copy or download required file
+cp /root/.cache/huggingface/hub/datasets--anon8231489123--ShareGPT_Vicuna_unfiltered/snapshots/192ab2185289094fc556ec8ce5ce1e8e587154ca/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+#curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
+cp /root/.cache/modelscope/hub/datasets/grade_school_math/test.jsonl /tmp
+
+echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+sysctl -w vm.swappiness=0
+sysctl -w kernel.numa_balancing=0
+sysctl -w kernel.sched_migration_cost_ns=50000
+
+export SGLANG_TEST_MAX_RETRY=0
+export SGLANG_SET_CPU_AFFINITY=1
+export HCCL_HOST_SOCKET_PORT_RANGE="auto"
+export HCCL_NPU_SOCKET_PORT_RANGE="auto"
+
+visibe_devices=$ASCEND_VISIBLE_DEVICES
+echo "ASCEND_VISIBLE_DEVICES=$ASCEND_VISIBLE_DEVICES"
+if [ "${visibe_devices}" != "" ];then
+    ASCEND_RT_VISIBLE_DEVICES=$(echo "$ASCEND_VISIBLE_DEVICES" | tr ',' '\n' | sort -n | tr '\n' ',')
+    export ASCEND_RT_VISIBLE_DEVICES=${ASCEND_RT_VISIBLE_DEVICES%,}
+    echo "ASCEND_RT_VISIBLE_DEVICES=$ASCEND_RT_VISIBLE_DEVICES"
+    export ASCEND_VISIBLE_DEVICES=""
+fi
+
+unset https_proxy
+unset http_proxy
+unset HTTPS_PROXY
+unset HTTP_PROXY
+unset ASCEND_LAUNCH_BLOCKING
+
+# use sglang from source or from image
+if [ "${INSTALL_SGLANG_FROM_SOURCE}" = "true" ] || [ "${INSTALL_SGLANG_FROM_SOURCE}" = "True" ];then
+    echo "Use sglang from source: ${sglang_source_path}"
+    export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH
+else
+    echo "Use sglang from docker image"
+    sglang_pkg_path=/sgl-workspace/sglang/python
+    ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
+    mkdir -p "${ascend_test_util_path}"
+    mv "${ascend_test_util_path}" "${ascend_test_util_path}_bak"
+    cp -r ${sglang_source_path}/python/sglang/test/ascend "${ascend_test_util_path}"
+fi
+
+# set environment of cann
+. /usr/local/Ascend/cann/set_env.sh
+. /usr/local/Ascend/nnal/atb/set_env.sh
+
+echo "Running test case ${test_case}"
+tc_name=${test_case##*/}
+tc_name=${tc_name%.*}
+current_date=$(date +%Y%m%d)
+log_path="/root/sglang/debug/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
+    log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+fi
+rm -rf "${log_path}"
+mkdir -p "${log_path}"
+echo "Log path: ${log_path}"
+
+if [ "${TROUBLE_SHOTTING}" = "true" ] || [ "${TROUBLE_SHOTTING}" = "True" ];then
+    echo "TROUBLE_SHOTTING=true, the pod will keep alive for four hour."
+    ( ${PYTHON_FOR_SGLANG} -u "${sglang_source_path}/${test_case}" 2>&1 || true ) | tee -a "${log_path}/${tc_name}.log"
+    sleep 14400
+else
+    ${PYTHON_FOR_SGLANG} -u "${sglang_source_path}/${test_case}" 2>&1 | tee -a "${log_path}/${tc_name}.log"
+fi
+echo "Finished test case ${test_case}"
+
+if [ -n "${METRICS_DATA_FILE}" ]; then
+    mkdir -p "${METRICS_DATA_FILE}"
+    cp "${log_path}/${tc_name}.log" "${METRICS_DATA_FILE}/test_output.log"
+    echo "Metrics log saved to ${METRICS_DATA_FILE}/test_output.log"
+fi
+
+source_plog_path="/root/ascend/log/debug/plog"
+if [ -d "$source_plog_path" ];then
+    echo "Plog files found. Begin to backup them."
+    target_plog_path="/root/sglang/debug/logs/plog/${tc_name}/${HOSTNAME}"
+    if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
+        target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
+    fi
+    rm -rf "${target_plog_path}"
+    mkdir -p "${target_plog_path}"
+    cp ${source_plog_path}/* "${target_plog_path}"
+fi

python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py

@@ -0,0 +1,637 @@
+import json
+import logging
+import os
+import re
+import subprocess
+import threading
+import time
+from urllib.parse import urlparse
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ascend.e2e.test_npu_multi_node_utils import (
+    SERVICE_PORT,
+    check_role,
+    launch_pd_mix_node,
+    launch_pd_separation_node,
+    launch_router,
+    wait_server_ready,
+)
+from sglang.test.test_utils import (
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    dump_metric,
+    popen_launch_server,
+)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
+EVALSCOPE = "evalscope"
+BENCHMARK_TOOL_DEFAULT = EVALSCOPE
+
+PYTHON_FOR_TEST_TOOL = "test_env_transformers_tool/bin/python"
+if not os.path.exists(PYTHON_FOR_TEST_TOOL) or not os.access(
+    PYTHON_FOR_TEST_TOOL, os.X_OK
+):
+    PYTHON_FOR_TEST_TOOL = "python3"
+logger.info(f"PYTHON_FOR_TEST_TOOL: {PYTHON_FOR_TEST_TOOL}")
+
+DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600
+MAX_SERVER_KEEP_ALIVE_TIME = 3600
+
+ACCURACY_TOLERANCE = 0.99
+
+# Dataset total question counts and allowed fluctuation (in questions)
+DATASET_QUESTION_COUNTS = {
+    "aime25": 30,
+    "aime26": 30,
+    "gpqa_diamond": 198,
+}
+
+DATASET_FLUCTUATION = {
+    "aime25": 2,
+    "aime26": 2,
+    "gpqa_diamond": 5,
+}
+
+MAX_RETRY_COUNT = 3
+
+SERVER_INITIALIZATION_DELAY = 120
+
+if os.environ.get("ASCEND_RT_VISIBLE_DEVICES"):
+    DEFAULT_SERVER_PORT_FOR_TEST = (
+        20000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100
+    )
+else:
+    DEFAULT_SERVER_PORT_FOR_TEST = (
+        20000 + int(os.environ.get("ASCEND_VISIBLE_DEVICES", "0")[0]) * 100
+    )
+DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_SERVER_PORT_FOR_TEST + 66}"
+
+
+def get_accuracy_threshold(datasets, baseline_accuracy):
+    """Calculate accuracy threshold based on dataset fluctuation tolerance.
+
+    For datasets with defined fluctuation (aime*, gpqa_diamond), use absolute
+    question count tolerance. For others (e.g. mmmu), use percentage tolerance.
+    """
+    dataset = datasets[0] if datasets else None
+    if dataset in DATASET_FLUCTUATION and dataset in DATASET_QUESTION_COUNTS:
+        fluctuation = DATASET_FLUCTUATION[dataset] / DATASET_QUESTION_COUNTS[dataset]
+        return baseline_accuracy - fluctuation
+    return baseline_accuracy * ACCURACY_TOLERANCE
+
+
+def get_max_retries(datasets):
+    """Return max retry count for accuracy tests.
+
+    gpqa and aime datasets support up to MAX_RETRY_COUNT retries.
+    mmmu and others use 1 attempt (no retry).
+    """
+    dataset = datasets[0] if datasets else None
+    if dataset in DATASET_FLUCTUATION:
+        return MAX_RETRY_COUNT
+    return 1
+
+
+def run_evalscope(
+    host,
+    port,
+    model,
+    datasets,
+    dataset_args=None,
+    eval_batch_size=16,
+    limit=100000,
+    generation_config=None,
+    dataset_dir=None,
+    timeout=60000,
+    stream=True,
+    eval_type="openai_api",
+):
+
+    metrics_path = os.getenv("METRICS_DATA_FILE")
+    result_path = "./evalscope_result" if not metrics_path else metrics_path
+    logger.info(f"The metrics result file: {result_path}")
+
+    api_url = f"http://{host}:{port}/v1/chat/completions"
+
+    if generation_config is None:
+        generation_config = {"max_tokens": 512}
+
+    config_dict = {
+        "model": model,
+        "api_url": api_url,
+        "eval_type": eval_type,
+        "datasets": datasets,
+        "eval_batch_size": eval_batch_size,
+        "generation_config": generation_config,
+        "timeout": timeout,
+        "stream": stream,
+        "limit": limit,
+        "work_dir": result_path,
+    }
+    if dataset_args:
+        config_dict["dataset_args"] = dataset_args
+    if dataset_dir:
+        config_dict["dataset_dir"] = dataset_dir
+
+    config_json = json.dumps(config_dict, ensure_ascii=False, indent=2)
+    config_json_escaped = config_json.replace("\\", "\\\\").replace("'''", "\\'\\'\\'")
+
+    script_content = "import json\n"
+    script_content += "from evalscope import TaskConfig, run_task\n\n"
+    script_content += f"config = json.loads('''{config_json_escaped}''')\n"
+    script_content += "task_cfg = TaskConfig(**config)\n"
+    script_content += "run_task(task_cfg=task_cfg)\n"
+
+    script_path = f"/tmp/evalscope_run_{model}_{'_'.join(datasets)}.py"
+    with open(script_path, "w") as f:
+        f.write(script_content)
+
+    logger.info(f"Generated evalscope script: {script_path}")
+
+    install_cmd = (
+        "/bin/bash /root/sglang/python/sglang/test/ascend/e2e/run_evalscope.sh"
+    )
+    subprocess.run(install_cmd, shell=True, check=True)
+
+    python_bin = "test_env_evalscope/bin/python"
+    cmd = f"{python_bin} {script_path}"
+
+    logger.info(f"Command: {cmd}")
+
+    process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+        shell=True,
+    )
+
+    output_lines = []
+    try:
+        for line in iter(process.stdout.readline, ""):
+            if line.strip():
+                print(line, end="")
+            output_lines.append(line.strip())
+
+        process.wait()
+
+        if process.returncode != 0:
+            logger.error(f"Command failed with return code: {process.returncode}")
+            raise subprocess.CalledProcessError(process.returncode, cmd)
+
+        logger.info("Command executed successfully")
+
+        metrics = {}
+        full_output = "\n".join(output_lines)
+
+        report_match = re.search(r"Dump report to:\s*(\S+)", full_output)
+        if report_match:
+            report_path = report_match.group(1)
+            logger.info(f"Found evalscope report file: {report_path}")
+            try:
+                with open(report_path, "r") as rf:
+                    report_data = json.load(rf)
+                for item in report_data:
+                    score = item.get("score")
+                    if score is not None:
+                        metrics["accuracy"] = float(score)
+                        logger.info(f"The Final Accuracy from report: {score}")
+                        break
+            except Exception as e:
+                logger.warning(f"Failed to read report file {report_path}: {e}")
+
+        if "accuracy" not in metrics:
+            accuracy_patterns = [
+                r"mean_acc\s*.*?│\s*\d+\s*│\s*([\d.]+)\s*│",
+                r"│\s+([\d.]+)\s+│\s+\S+\s+│\s*$",
+                r"accuracy\s*[:=]?\s*([\d.]+)",
+                r"Accuracy\s*[:=]?\s*([\d.]+)",
+                r"score\s*[:=]?\s*([\d.]+)",
+            ]
+
+            for pattern in accuracy_patterns:
+                matches = re.findall(pattern, full_output)
+                if matches:
+                    final_accuracy = float(matches[-1])
+                    metrics["accuracy"] = final_accuracy
+                    logger.info(f"The Final Accuracy from output: {final_accuracy}")
+                    break
+
+        if "accuracy" not in metrics:
+            logger.info("Can Not Find The Accuracy in evalscope output")
+
+        return metrics
+
+    except KeyboardInterrupt:
+        logger.info("Keyboard interrupt received, terminating process...")
+        process.terminate()
+        try:
+            process.wait(timeout=5)
+            logger.info("Process terminated")
+        except subprocess.TimeoutExpired:
+            logger.warning("Process did not terminate gracefully, killing it...")
+            process.kill()
+            logger.info("Process killed")
+        raise
+    except Exception as e:
+        logger.error(f"Error executing command: {e}")
+        process.terminate()
+        process.wait(timeout=5)
+        raise
+
+
+def assert_metrics(self, metrics):
+    if not metrics:
+        raise Exception("No metrics obtained from benchmark")
+
+    if self.accuracy is not None:
+        threshold = get_accuracy_threshold(self.datasets, self.accuracy)
+        dump_metric(
+            "accuracy",
+            float(metrics["accuracy"]),
+            labels={"test_case": self.__class__.__name__, "type": "accuracy"},
+        )
+        dump_metric(
+            "accuracy_baseline",
+            float(self.accuracy),
+            labels={"test_case": self.__class__.__name__, "type": "accuracy"},
+        )
+        self.assertGreaterEqual(
+            float(metrics["accuracy"]),
+            threshold,
+            f"Accuracy check failed. Expected >= {threshold}, Got: {metrics['accuracy']}",
+        )
+
+
+class TestNpuAccuracyTestCaseBase(CustomTestCase):
+    model = None
+    benchmark_tool = BENCHMARK_TOOL_DEFAULT
+    backend = "sglang"
+    datasets = ["gsm8k"]
+    dataset_args = None
+    eval_batch_size = 16
+    limit = 100000
+    generation_config = None
+    dataset_dir = None
+    stream = True
+    timeout = 60000
+    eval_type = "openai_api"
+    other_args = None
+    server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    envs = None
+    max_attempts = 2
+    n_runs = 3
+    accuracy = 0.1
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        env = os.environ.copy()
+        for key, value in env.items():
+            logger.info(f"ENV_VAR_SYS {key}:{value}")
+        if cls.envs:
+            for key, value in cls.envs.items():
+                logger.info(f"ENV_VAR_CASE {key}:{value}")
+                env[key] = value
+
+        other_args = list(cls.other_args)
+
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=cls.server_timeout,
+            other_args=other_args,
+            env=env,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        if hasattr(cls, "process") and cls.process:
+            try:
+                kill_process_tree(cls.process.pid)
+            except Exception as e:
+                logger.error(f"Error during tearDown: {e}")
+
+    def run_accuracy(self):
+        parsed_url = urlparse(self.base_url)
+        host = parsed_url.hostname
+        port = parsed_url.port
+        if self.benchmark_tool == EVALSCOPE:
+            model_name = os.path.basename(self.model)
+            max_retries = get_max_retries(self.datasets)
+            best_metrics = None
+            for attempt in range(max_retries):
+                metrics = run_evalscope(
+                    host=host,
+                    port=port,
+                    model=model_name,
+                    datasets=self.datasets,
+                    dataset_args=self.dataset_args,
+                    eval_batch_size=self.eval_batch_size,
+                    limit=self.limit,
+                    generation_config=self.generation_config,
+                    dataset_dir=self.dataset_dir,
+                    stream=self.stream,
+                    timeout=self.timeout,
+                    eval_type=self.eval_type,
+                )
+                if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
+                    best_metrics.get("accuracy", 0)
+                ):
+                    best_metrics = metrics
+                threshold = get_accuracy_threshold(self.datasets, self.accuracy)
+                if float(best_metrics.get("accuracy", 0)) >= threshold:
+                    break
+                if attempt < max_retries - 1:
+                    logger.info(
+                        f"Accuracy {best_metrics.get('accuracy')} below threshold "
+                        f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
+                    )
+            assert_metrics(self, best_metrics)
+
+    def run_accuracy_multiple(self, n_runs=None):
+        if n_runs is None:
+            n_runs = self.n_runs
+
+        parsed_url = urlparse(self.base_url)
+        host = parsed_url.hostname
+        port = parsed_url.port
+
+        if self.benchmark_tool != EVALSCOPE:
+            raise Exception(
+                "run_accuracy_multiple only supports evalscope benchmark tool"
+            )
+
+        model_name = os.path.basename(self.model)
+        all_metrics = []
+
+        for i in range(n_runs):
+            logger.info(f"=== Accuracy run {i + 1}/{n_runs} ===")
+            metrics = run_evalscope(
+                host=host,
+                port=port,
+                model=model_name,
+                datasets=self.datasets,
+                dataset_args=self.dataset_args,
+                eval_batch_size=self.eval_batch_size,
+                limit=self.limit,
+                generation_config=self.generation_config,
+                dataset_dir=self.dataset_dir,
+                stream=self.stream,
+                timeout=self.timeout,
+                eval_type=self.eval_type,
+            )
+            all_metrics.append(metrics)
+            if metrics and "accuracy" in metrics:
+                logger.info(f"Run {i + 1} accuracy: {metrics['accuracy']}")
+            else:
+                logger.warning(f"Run {i + 1} failed to get accuracy metric")
+
+        valid_metrics = [m for m in all_metrics if m and "accuracy" in m]
+        if not valid_metrics:
+            raise Exception("No valid accuracy metrics obtained from any run")
+
+        avg_accuracy = sum(float(m["accuracy"]) for m in valid_metrics) / len(
+            valid_metrics
+        )
+
+        logger.info("=" * 60)
+        logger.info("Multiple Run Accuracy Results:")
+        for i, m in enumerate(valid_metrics):
+            logger.info(f"  Run {i + 1}: {m['accuracy']}")
+        logger.info(f"  Average: {avg_accuracy}")
+        logger.info("=" * 60)
+
+        avg_metrics = {"accuracy": avg_accuracy}
+        dump_metric(
+            "accuracy_avg",
+            avg_accuracy,
+            labels={"test_case": self.__class__.__name__, "type": "accuracy"},
+        )
+        assert_metrics(self, avg_metrics)
+
+
+class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase):
+    model_config = None
+    benchmark_tool = BENCHMARK_TOOL_DEFAULT
+    backend = "sglang"
+    datasets = ["gsm8k"]
+    dataset_args = None
+    eval_batch_size = 16
+    limit = 100000
+    generation_config = None
+    dataset_dir = None
+    stream = True
+    timeout = 60000
+    eval_type = "openai_api"
+    other_args = None
+    server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    envs = None
+    max_attempts = 2
+    accuracy = 0.1
+
+    @classmethod
+    def setUpClass(cls):
+        cls.local_ip = "127.0.0.1"
+        cls.host = os.getenv("POD_IP")
+        cls.port = SERVICE_PORT
+        cls.base_url = f"http://{cls.host}:{cls.port}"
+        cls.hostname = os.getenv("HOSTNAME")
+        cls.role = "master" if cls.hostname.endswith("sglang-node-0") else "worker"
+        logger.info(f"Init {cls.host} {cls.role=}!")
+
+        cls.start_pd_mix_master_node()
+        cls.start_pd_mix_worker_node()
+
+    @classmethod
+    def tearDownClass(cls):
+        pass
+
+    @classmethod
+    @check_role(allowed_roles=["master"])
+    def start_pd_mix_master_node(cls):
+        sglang_thread = threading.Thread(
+            target=launch_pd_mix_node, args=(cls.model_config,)
+        )
+        sglang_thread.start()
+
+        wait_server_ready(f"{cls.base_url}/health")
+
+        logger.info(
+            f"Wait {SERVER_INITIALIZATION_DELAY}s, starting run benchmark ......"
+        )
+        time.sleep(SERVER_INITIALIZATION_DELAY)
+
+    @classmethod
+    @check_role(allowed_roles=["worker"])
+    def start_pd_mix_worker_node(cls):
+        sglang_thread = threading.Thread(
+            target=launch_pd_mix_node, args=(cls.model_config,)
+        )
+        sglang_thread.start()
+
+        logger.info(
+            f"{cls.role} node started, keeping test alive for {MAX_SERVER_KEEP_ALIVE_TIME} seconds"
+        )
+        time.sleep(MAX_SERVER_KEEP_ALIVE_TIME)
+
+    @check_role(allowed_roles=["master", "worker"])
+    def run_accuracy(self):
+        parsed_url = urlparse(self.base_url)
+        host = parsed_url.hostname
+        port = parsed_url.port
+        if self.benchmark_tool == EVALSCOPE:
+            model_name = os.path.basename(self.model_config.get("model_path"))
+            max_retries = get_max_retries(self.datasets)
+            best_metrics = None
+            for attempt in range(max_retries):
+                metrics = run_evalscope(
+                    host=self.host,
+                    port=self.port,
+                    model=model_name,
+                    datasets=self.datasets,
+                    dataset_args=self.dataset_args,
+                    eval_batch_size=self.eval_batch_size,
+                    limit=self.limit,
+                    generation_config=self.generation_config,
+                    dataset_dir=self.dataset_dir,
+                    stream=self.stream,
+                    timeout=self.timeout,
+                    eval_type=self.eval_type,
+                )
+                if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
+                    best_metrics.get("accuracy", 0)
+                ):
+                    best_metrics = metrics
+                threshold = get_accuracy_threshold(self.datasets, self.accuracy)
+                if float(best_metrics.get("accuracy", 0)) >= threshold:
+                    break
+                if attempt < max_retries - 1:
+                    logger.info(
+                        f"Accuracy {best_metrics.get('accuracy')} below threshold "
+                        f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
+                    )
+            assert_metrics(self, best_metrics)
+
+
+class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase):
+    model_config = None
+    benchmark_tool = BENCHMARK_TOOL_DEFAULT
+    backend = "sglang"
+    datasets = ["gsm8k"]
+    dataset_args = None
+    eval_batch_size = 16
+    limit = 100000
+    generation_config = None
+    dataset_dir = None
+    stream = True
+    timeout = 60000
+    eval_type = "openai_api"
+    other_args = None
+    server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    max_attempts = 2
+    accuracy = 0.1
+
+    @classmethod
+    def setUpClass(cls):
+        cls.process = None
+        cls.local_ip = "127.0.0.1"
+        cls.host = os.getenv("POD_IP")
+        cls.port = SERVICE_PORT
+        cls.base_url = f"http://{cls.host}:{cls.port}"
+        cls.hostname = os.getenv("HOSTNAME")
+        cls.role = (
+            "router"
+            if "router" in cls.hostname
+            else "prefill" if "prefill" in cls.hostname else "decode"
+        )
+        logger.info(f"Init {cls.host} {cls.role=}!")
+
+        cls.start_pd_server()
+        cls.start_router_server()
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.process:
+            try:
+                kill_process_tree(cls.process.pid)
+            except Exception as e:
+                logger.error(f"Error during tearDown: {e}")
+
+    @classmethod
+    @check_role(allowed_roles=["router"])
+    def start_router_server(cls):
+        logger.info(f"Starting router in thread...")
+        sglang_thread = threading.Thread(target=launch_router, args=(cls.model_config,))
+        sglang_thread.daemon = True
+        sglang_thread.start()
+
+        health_check_url = f"{cls.base_url}/health"
+        logger.info(f"Waiting for router to be ready at {health_check_url}")
+        wait_server_ready(health_check_url)
+
+        logger.info(
+            f"Waiting {SERVER_INITIALIZATION_DELAY} seconds for the server to fully initialize..."
+        )
+        time.sleep(SERVER_INITIALIZATION_DELAY)
+
+    @classmethod
+    @check_role(allowed_roles=["prefill", "decode"])
+    def start_pd_server(cls):
+        logger.info(f"Starting pd separation node...")
+        cls.process = launch_pd_separation_node(cls.model_config)
+        logger.info(f"Pd separation node started with PID: {cls.process.pid}")
+
+        while True:
+            if cls.process.poll() is None:
+                time.sleep(30)
+            else:
+                exit_code = cls.process.poll()
+                raise Exception(
+                    f"Sglang process exited on node {cls.host} {cls.hostname} with exit code: {exit_code}"
+                )
+
+    @check_role(allowed_roles=["router"])
+    def run_accuracy(self):
+        parsed_url = urlparse(self.base_url)
+        host = parsed_url.hostname
+        port = parsed_url.port
+        if self.benchmark_tool == EVALSCOPE:
+            model_name = os.path.basename(self.model_config.get("model_path"))
+            max_retries = get_max_retries(self.datasets)
+            best_metrics = None
+            for attempt in range(max_retries):
+                metrics = run_evalscope(
+                    host=host,
+                    port=port,
+                    model=model_name,
+                    datasets=self.datasets,
+                    dataset_args=self.dataset_args,
+                    eval_batch_size=self.eval_batch_size,
+                    limit=self.limit,
+                    generation_config=self.generation_config,
+                    dataset_dir=self.dataset_dir,
+                    stream=self.stream,
+                    timeout=self.timeout,
+                    eval_type=self.eval_type,
+                )
+                if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
+                    best_metrics.get("accuracy", 0)
+                ):
+                    best_metrics = metrics
+                threshold = get_accuracy_threshold(self.datasets, self.accuracy)
+                if float(best_metrics.get("accuracy", 0)) >= threshold:
+                    break
+                if attempt < max_retries - 1:
+                    logger.info(
+                        f"Accuracy {best_metrics.get('accuracy')} below threshold "
+                        f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
+                    )
+            assert_metrics(self, best_metrics)

python/sglang/test/ascend/gsm8k_ascend_mixin.py

@@ -0,0 +1,108 @@
+import os
+import subprocess
+from abc import ABC
+from types import SimpleNamespace
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    popen_launch_server,
+    write_github_step_summary,
+)
+
+
+class GSM8KAscendMixin(ABC):
+    model = ""
+
+    timeout_for_server_launch = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    other_args = [
+        "--trust-remote-code",
+        "--mem-fraction-static",
+        "0.8",
+        "--attention-backend",
+        "ascend",
+        "--disable-cuda-graph",
+    ]
+    server_cmd = ""
+    gsm8k_num_shots = 5
+    num_questions = 200
+    gsm8k_parallel = 128
+
+    env = {
+        **os.environ,
+        "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
+        "ASCEND_MF_STORE_URL": "tcp://127.0.0.1:24666",
+        "HCCL_BUFFSIZE": "200",
+        "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "24",
+        "USE_VLLM_CUSTOM_ALLREDUCE": "1",
+        "HCCL_EXEC_TIMEOUT": "200",
+        "STREAMS_PER_DEVICE": "32",
+        "SGLANG_ENBLE_TORCH_COMILE": "1",
+        "AUTO_USE_UC_MEMORY": "0",
+        "P2P_HCCL_BUFFSIZE": "20",
+    }
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        try:
+            cls.process = popen_launch_server(
+                cls.model,
+                cls.base_url,
+                timeout=cls.timeout_for_server_launch,
+                other_args=cls.other_args,
+                env=cls.env,
+            )
+            cls.server_cmd = subprocess.list2cmdline(cls.process.args)
+        except Exception as e:
+            write_github_step_summary(f"Failed to launch server for {cls.model}: {e}")
+            raise AssertionError(f"Test failed for {cls.model}: {e}")
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        accuracy_threshold = getattr(self, "accuracy", 0.00)
+        output_throughput_threshold = getattr(self, "output_throughput", 0.00)
+
+        model_metrics = {
+            "server": self.server_cmd,
+            "client": "few_shot_gsm8k",
+            "accuracy_threshold": getattr(self, "accuracy", "N/A"),
+            "output_throughput_threshold": getattr(self, "output_throughput", "N/A"),
+        }
+
+        try:
+            args = SimpleNamespace(
+                max_tokens=512,
+                base_url=self.base_url,
+                model=self.model,
+                eval_name="gsm8k",
+                api="completion",
+                num_examples=self.num_questions,
+                num_threads=128,
+                num_shots=self.gsm8k_num_shots,
+            )
+            metrics = run_eval(args)
+            model_metrics["accuracy"] = metrics["score"]
+            model_metrics["output_throughput"] = metrics["output_throughput"]
+            model_metrics["latency"] = metrics["latency"]
+            self.assertGreaterEqual(
+                metrics["score"],
+                accuracy_threshold,
+                f'Accuracy of {self.model} is {str(metrics["score"])}, is lower than {accuracy_threshold}',
+            )
+            self.assertGreaterEqual(
+                metrics["output_throughput"],
+                output_throughput_threshold,
+                f'Output throughput of {self.model} is {str(metrics["output_throughput"])}, is lower than {output_throughput_threshold}',
+            )
+        except Exception as e:
+            model_metrics["error"] = e
+            self.fail(f"Test failed for {self.model}: {e}")
+        finally:
+            write_results_to_github_step_summary({self.model: model_metrics})

python/sglang/test/ascend/output_capturer.py

@@ -0,0 +1,117 @@
+import os
+import select
+import threading
+
+
+class OutputCapturer:
+    """Capture all console print information
+
+    Class Description:
+        Capture console output using low-level file descriptor redirection.
+        Used to obtain print information from child processes, NPU processes,
+        and underlying C/C++ modules that are not logged in sglang logs for test assertion.
+        All captured output will be displayed normally in the console in real-time.
+    """
+
+    def __init__(self):
+        """Initialize all member variables of the capturer"""
+        self.old_stdout = None
+        self.old_stderr = None
+        self.pipe_out = None
+        self.pipe_in = None
+        self.pipe_err_out = None
+        self.pipe_err_in = None
+        self.captured_stdout = []
+        self.captured_stderr = []
+        self.stop_thread = False
+        self.thread = None
+
+    def start(self):
+        """Start console output capture"""
+        # Duplicate and save original stdout/stderr file descriptors
+        self.old_stdout = os.dup(1)
+        self.old_stderr = os.dup(2)
+
+        # Create anonymous pipes for output redirection
+        self.pipe_out, self.pipe_in = os.pipe()
+        self.pipe_err_out, self.pipe_err_in = os.pipe()
+
+        # Redirect system stdout/stderr to the write end of pipes
+        os.dup2(self.pipe_in, 1)
+        os.dup2(self.pipe_err_in, 2)
+
+        # Close unused pipe write ends
+        os.close(self.pipe_in)
+        os.close(self.pipe_err_in)
+
+        # Start daemon thread to read output in real time
+        self.stop_thread = False
+        self.thread = threading.Thread(target=self._read_loop, daemon=True)
+        self.thread.start()
+
+    def _read_loop(self):
+        """The background process reads and prints pipeline data records in a loop."""
+        read_fds = [self.pipe_out, self.pipe_err_out]
+        while not self.stop_thread:
+            try:
+                # select listens to multiple file descriptors simultaneously, waiting for data in a non-blocking manner
+                readable, _, exceptional = select.select(read_fds, [], read_fds, 0.01)
+
+                # Processing file descriptors containing data
+                for fd in readable:
+                    if fd == self.pipe_out:
+                        data = os.read(fd, 4096)
+                        if data:
+                            self.captured_stdout.append(data)
+                            os.write(self.old_stdout, data)
+                    elif fd == self.pipe_err_out:
+                        err_data = os.read(fd, 4096)
+                        if err_data:
+                            self.captured_stderr.append(err_data)
+                            os.write(self.old_stderr, err_data)
+
+                for fd in exceptional:
+                    if fd in read_fds:
+                        self.stop()
+
+            except OSError:
+                self.stop()
+                break
+
+    def get_all(self):
+        """Get all captured stdout and stderr as UTF-8 string
+
+        Return: Decoded stdout and stderr string (ignore decoding errors)
+        """
+        return self.get_output() + self.get_error()
+
+    def get_output(self):
+        """Get all captured stdout as UTF-8 string
+
+        Return: Decoded stdout string (ignore decoding errors)
+        """
+        return b"".join(self.captured_stdout).decode("utf-8", errors="ignore")
+
+    def get_error(self):
+        """Get all captured stderr as UTF-8 string
+
+        Return: Decoded stderr string (ignore decoding errors)
+        """
+        return b"".join(self.captured_stderr).decode("utf-8", errors="ignore")
+
+    def stop(self):
+        """Stop capture and restore system environment"""
+        self.stop_thread = True
+        if self.thread:
+            self.thread.join(timeout=0.5)
+
+        # Restore original output
+        os.dup2(self.old_stdout, 1)
+        os.dup2(self.old_stderr, 2)
+
+        # Close all file descriptors
+        for fd in [self.pipe_out, self.pipe_err_out, self.old_stdout, self.old_stderr]:
+            try:
+                os.close(fd)
+            except (OSError, IOError):
+                pass

python/sglang/test/ascend/run_eval.py

@@ -0,0 +1,122 @@
+import json
+import os
+
+from sglang.test.run_eval import run_eval_once
+from sglang.test.simple_eval_common import (
+    make_report,
+    set_ulimit,
+)
+
+
+def run_eval(args):
+    # Lazy import to avoid circular dependency with test_utils
+    from sglang.test.test_utils import dump_metric
+
+    set_ulimit()
+
+    if "OPENAI_API_KEY" not in os.environ:
+        os.environ["OPENAI_API_KEY"] = "EMPTY"
+
+    base_url = (
+        f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
+    )
+
+    if args.eval_name == "mmlu":
+        from sglang.test.ascend.simple_eval_mmlu import MMLUEval
+
+        filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
+        eval_obj = MMLUEval(
+            filename, args.num_examples, args.num_threads, getattr(args, "num_shots", 0)
+        )
+    else:
+        raise ValueError(f"Invalid eval name: {args.eval_name}")
+
+    if getattr(args, "repeat", 1) == 1:
+        result, latency, sampler = run_eval_once(args, base_url, eval_obj)
+        metrics = result.metrics | {"score": result.score}
+        metrics["latency"] = latency
+        print(f"Total latency: {latency:.3f} s")
+        print(f"Score: {metrics['score']:.3f}")
+
+        # Compute output throughput from accumulated completion tokens
+        total_completion_tokens = sum(sampler._completion_tokens)
+        if total_completion_tokens > 0 and latency > 0:
+            metrics["output_throughput"] = total_completion_tokens / latency
+            print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
+
+        # Report metrics to unified collection framework
+        dump_metric(
+            f"{args.eval_name}_score",
+            metrics["score"],
+            labels={"model": sampler.model, "eval": args.eval_name},
+        )
+        dump_metric(
+            f"{args.eval_name}_latency",
+            latency,
+            labels={"model": sampler.model, "eval": args.eval_name},
+        )
+    else:
+        from concurrent.futures import ThreadPoolExecutor
+
+        executor = ThreadPoolExecutor(max_workers=args.repeat)
+
+        futures = [
+            executor.submit(run_eval_once, args, base_url, eval_obj)
+            for _ in range(args.repeat)
+        ]
+
+        scores_repeat = []
+        latencies = []
+        total_completion_tokens = 0
+
+        for f in futures:
+            result, latency, sampler = f.result()
+            scores_repeat.append(result.score)
+            latencies.append(latency)
+            total_completion_tokens += sum(sampler._completion_tokens)
+
+        mean_score = sum(scores_repeat) / len(scores_repeat)
+        mean_latency = sum(latencies) / len(latencies)
+        total_latency = sum(latencies)
+        scores_repeat = [f"{s:.3f}" for s in scores_repeat]
+        print("=" * 20)
+        print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}")
+        print(f"Scores: {scores_repeat}")
+        print(f"Mean latency: {mean_latency:.3f} s")
+        print("=" * 20)
+        metrics = result.metrics | {"scores": scores_repeat}
+        metrics = metrics | {"mean_score": mean_score}
+        metrics["latency"] = mean_latency
+
+        if total_completion_tokens > 0 and total_latency > 0:
+            metrics["output_throughput"] = total_completion_tokens / total_latency
+            print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
+
+        # Report metrics to unified collection framework
+        dump_metric(
+            f"{args.eval_name}_mean_score",
+            mean_score,
+            labels={
+                "model": sampler.model,
+                "eval": args.eval_name,
+                "repeat": args.repeat,
+            },
+        )
+
+        executor.shutdown()
+
+    # Dump reports
+    file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}"
+    report_filename = f"/tmp/{file_stem}.html"
+    print(f"Writing report to {report_filename}")
+    with open(report_filename, "w") as fh:
+        fh.write(make_report(result))
+    print(metrics)
+    result_filename = f"/tmp/{file_stem}.json"
+    with open(result_filename, "w") as f:
+        f.write(json.dumps(metrics, indent=2))
+    print(f"Writing results to {result_filename}")
+
+    if getattr(args, "return_latency", False):
+        return metrics, latency
+    return metrics

python/sglang/test/ascend/simple_eval_mmlu.py

@@ -0,0 +1,125 @@
+# Adapted from https://github.com/openai/simple-evals/
+
+"""
+Measuring Massive Multitask Language Understanding
+Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
+https://arxiv.org/abs/2009.03300
+"""
+
+import random
+import re
+from typing import Optional
+
+import pandas
+
+from sglang.test import simple_eval_common as common
+from sglang.test.simple_eval_common import (
+    ANSWER_PATTERN_MULTICHOICE,
+    HTML_JINJA,
+    Eval,
+    EvalResult,
+    SamplerBase,
+    SingleEvalResult,
+    format_multichoice_question,
+)
+from sglang.test.simple_eval_mmlu import subject2category
+
+
+def format_multichoice_question_example(row):
+    return QUERY_TEMPLATE_MULTICHOICE.format(**row)
+
+
+QUERY_TEMPLATE_MULTICHOICE = """
+Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
+
+{Question}
+
+A) {A}
+B) {B}
+C) {C}
+D) {D}
+""".strip()
+
+TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN = """
+Answer the multiple-choice questions following the examples below. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD.
+
+"""
+
+TEMPLATE_MULTICHOICE_EXAMPLE = """
+Example question:
+{Question}
+
+A {A}
+B {B}
+C {C}
+D {D}
+
+The last line of your response should be
+Answer: {Answer}
+""".strip()
+
+
+class MMLUEval(Eval):
+    def __init__(
+        self,
+        filename: str,
+        num_examples: Optional[int],
+        num_threads: int,
+        num_shots: int,
+    ):
+        if "://" in filename:
+            df = pandas.read_csv(filename, storage_options={"timeout": 30})
+        else:
+            df = pandas.read_csv(filename)
+        examples = [row.to_dict() for _, row in df.iterrows()]
+        if num_shots:
+            example_questions = "".join(
+                format_multichoice_question_example(row) + "\n\n"
+                for row in examples[:num_shots]
+            )
+            self.template = (
+                TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN
+                + example_questions
+                + QUERY_TEMPLATE_MULTICHOICE
+            )
+            examples = examples[num_shots:]
+        if num_examples:
+            examples = random.Random(0).sample(examples, num_examples)
+        self.examples = examples
+        self.num_threads = num_threads
+        self.num_shots = num_shots
+
+    def __call__(self, sampler: SamplerBase) -> EvalResult:
+        def fn(row: dict):
+            if self.num_shots:
+                prompt_messages = [
+                    sampler._pack_message(
+                        content=self.template.format(**row), role="user"
+                    )
+                ]
+            else:
+                prompt_messages = [
+                    sampler._pack_message(
+                        content=format_multichoice_question(row), role="user"
+                    )
+                ]
+            response_text = sampler(prompt_messages)
+            response_text = response_text or ""
+            match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
+            extracted_answer = match.group(1) if match else None
+            score = 1.0 if extracted_answer == row["Answer"] else 0.0
+            html = common.jinja_env.from_string(HTML_JINJA).render(
+                prompt_messages=prompt_messages,
+                next_message=dict(content=response_text, role="assistant"),
+                score=score,
+                correct_answer=row["Answer"],
+                extracted_answer=extracted_answer,
+            )
+            convo = prompt_messages + [dict(content=response_text, role="assistant")]
+            category = subject2category.get(row["Subject"], "other")
+            return SingleEvalResult(
+                html=html, score=score, metrics={category: score}, convo=convo
+            )
+
+        results = common.map_with_progress(fn, self.examples, self.num_threads)
+        return common.aggregate_results(results)

python/sglang/test/ascend/test_ascend_utils.py

@@ -0,0 +1,960 @@
+"""
+Common utilities for testing and benchmarking on NPU.
+
+This file contains the following weight path categories:
+- LLM model weights path
+- VLM model weights path
+- Embedding model weights path
+- Rerank model weights path
+- Reward model weights path
+
+Please remember to sort by variable name within each section.
+"""
+
+import asyncio
+import copy
+import logging
+import os
+import random
+import subprocess
+import threading
+import time
+from types import SimpleNamespace
+from typing import Awaitable, Callable, List, NamedTuple, Optional
+
+import requests
+
+from sglang.benchmark.serving import run_benchmark
+from sglang.srt.utils import kill_process_tree
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    auto_config_device,
+    is_in_ci,
+    popen_launch_server,
+    write_github_step_summary,
+)
+
+STDERR_FILENAME = "/tmp/stderr.txt"
+STDOUT_FILENAME = "/tmp/stdout.txt"
+
+# Model weights storage directory
+MODEL_WEIGHTS_DIR = "/root/.cache/modelscope/hub/models/"
+HF_MODEL_WEIGHTS_DIR = "/root/.cache/huggingface/hub/"
+IMAGES_DIR = "/root/.cache/modelscope/hub/datasets/images/"
+VIDEO_DIR = "/root/.cache/modelscope/hub/datasets/video/"
+
+# LLM model weights path
+AFM_4_5B_BASE_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "arcee-ai/AFM-4.5B-Base")
+BAICHUAN2_13B_CHAT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "baichuan-inc/Baichuan2-13B-Chat"
+)
+C4AI_COMMAND_R_V01_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "CohereForAI/c4ai-command-r-v01"
+)
+C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH = "/__w/sglang/sglang/test/registered/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja"
+CHATGLM2_6B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "ZhipuAI/chatglm2-6b")
+DBRX_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "AI-ModelScope/dbrx-instruct"
+)
+DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "vllm-ascend/DeepSeek-R1-0528-W8A8"
+)
+DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "DeepSeek-V3.2-Exp-W8A8"
+)
+DEEPSEEK_V3_2_W8A8_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "vllm-ascend/DeepSeek-V3.2-W8A8"
+)
+DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"
+)
+DEEPSEEK_CODER_1_3_B_BASE_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-coder-1.3b-base"
+)
+DOTS_OCR_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "rednote-hilab/dots.ocr")
+ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-32B-w4a4-LAOS"
+)
+ERNIE_4_5_21B_A3B_PT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "baidu/ERNIE-4.5-21B-A3B-PT"
+)
+EXAONE_3_5_7_8B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct"
+)
+GEMMA_3_4B_IT_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "google/gemma-3-4b-it")
+GEMMA_4_E2B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "google/gemma-4-E2B-it")
+GEMMA_4_E4B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "google/gemma-4-E4B-it")
+GEMMA_4_26B_A4B_IT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "google/gemma-4-26B-A4B-it"
+)
+GEMMA_4_31B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "google/gemma-4-31B-it")
+GLM_4_9B_CHAT_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "ZhipuAI/glm-4-9b-chat")
+GLM_5_1_W4A8_MODEL_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Eco-Tech/GLM-5.1-w4a8")
+GPT_OSS_120B_BF16_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "eigen-ai-labs/gpt-oss-120b-bf16"
+)
+GRANITE_3_0_3B_A800M_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "ibm-granite/granite-3.0-3b-a800m-instruct"
+)
+GRANITE_3_1_8B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "ibm-granite/granite-3.1-8b-instruct"
+)
+GROK_2_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "huihui-ai/grok-2")
+GROK_2_WEIGHTS_TOKENIZER_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "huihui-ai/grok-2/tokenizer.tok.json"
+)
+INTERNLM2_7B_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Shanghai_AI_Laboratory/internlm2-7b"
+)
+KIMI_K2_THINKING_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Kimi/Kimi-K2-Thinking")
+KIMI_K2_5_W4A8_MODEL_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Eco-Tech/Kimi-K2.5-w4a8")
+KIMI_K2_5_EAGLE3_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "lightseekorg/kimi-k2.5-eagle3"
+)
+LING_LITE_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "inclusionAI/Ling-lite")
+LLAMA_2_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "LLM-Research/Llama-2-7B")
+LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "AI-ModelScope/Llama-3.1-8B-Instruct"
+)
+LLAMA_3_2_1B_INSTRUCT_TOOL_CALLING_LORA_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "codelion/Llama-3.2-1B-Instruct-tool-calling-lora"
+)
+LLAMA_3_2_1B_INSTRUCT_TOOL_FAST_LORA_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "suayptalha/FastLlama-3.2-LoRA"
+)
+LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "LLM-Research/Llama-3.2-1B-Instruct"
+)
+LLAMA_3_2_1B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "LLM-Research/Llama-3.2-1B")
+LLAMA_3_8B_EAGLE_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "lmsys/sglang-EAGLE-LLaMA3-Instruct-8B"
+)
+LLAMA_3_8B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "LLM-Research/Meta-Llama-3-8B-Instruct"
+)
+
+LLAMA_4_SCOUT_17B_16E_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "meta-llama/Llama-4-Scout-17B-16E-Instruct"
+)
+LLaDA2_0_MINI_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "inclusionAI/LLaDA2.0-mini"
+)
+META_LLAMA_3_1_8B_INSTRUCT = os.path.join(
+    MODEL_WEIGHTS_DIR, "LLM-Research/Meta-Llama-3.1-8B-Instruct"
+)
+MIMO_7B_RL_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "XiaomiMiMo/MiMo-7B-RL")
+MIMO_V2_FLASH_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "XiaomiMiMo/MiMo-V2-Flash")
+MINICPM3_4B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "OpenBMB/MiniCPM3-4B")
+MISTRAL_7B_INSTRUCT_V0_2_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "mistralai/Mistral-7B-Instruct-v0.2"
+)
+OLMO_2_1124_7B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "allenai/OLMo-2-1124-7B-Instruct"
+)
+OLMOE_1B_7B_0924_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "allenai/OLMoE-1B-7B-0924"
+)
+PERSIMMON_8B_CHAT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Howeee/persimmon-8b-chat"
+)
+PHI_4_MULTIMODAL_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "microsoft/Phi-4-multimodal-instruct"
+)
+QWEN2_5_7B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen2.5-7B-Instruct"
+)
+QWEN3_0_6B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-0.6B")
+QWEN3_5_27B_MODEL_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3.5-27B")
+QWEN3_1_7B_GPTQ_INT8_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-1.7B-GPTQ-Int8"
+)
+QWEN3_235B_A22B_W8A8_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "vllm-ascend/Qwen3-235B-A22B-W8A8"
+)
+QWEN3_235B_A22B_EAGLE_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-235B-A22B-Eagle3"
+)
+QWEN3_30B_A3B_GPTQ_2507_INT4_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
+)
+QWEN3_30B_A3B_GGUF_Q4_K_M_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-30B-A3B-GGUF/Qwen3-30B-A3B-Q4_K_M.gguf"
+)
+QWEN3_30B_A3B_INSTRUCT_2507_INT4_AUTOROUND_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Intel/Qwen3-30B-A3B-Instruct-2507-int4-AutoRound"
+)
+QWEN3_30B_A3B_INSTRUCT_2507_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-30B-A3B-Instruct-2507"
+)
+QWEN3_4B_GGUF_Q4_K_M_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-4B-GGUF/Qwen3-4B-Q4_K_M.gguf"
+)
+QWEN3_8B_INT4_AUTOROUND_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Intel/Qwen3-8B-int4-AutoRound"
+)
+QWEN3_8B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-8B")
+QWEN3_8B_EAGLE3_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-8B_eagle3")
+QWEN3_8B_DECRYPTED_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "YZY/Qwen3-8B")
+QWEN3_8B_EAGLE3_DECRYPTED_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "YZY/Qwen3-8B_eagle3"
+)
+QWEN3_32B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-32B")
+QWEN3_CODER_480B_A35B_INSTRUCT_W8A8_QUAROT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen3-Coder-480B-A35B-Instruct-w8a8-QuaRot"
+)
+QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-Next-80B-A3B-Instruct"
+)
+QWEN3_32B_EAGLE3_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Zjcxy-SmartAI/Qwen3-32B-Eagle3"
+)
+QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "aleoyang/Qwen3-32B-w8a8-MindIE"
+)
+QWQ_32B_W8A8_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "vllm-ascend/QWQ-32B-W8A8")
+SMOLLM_1_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "HuggingFaceTB/SmolLM-1.7B")
+SOLAR_10_7B_INSTRUCT_V1_0_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "upstage/SOLAR-10.7B-Instruct-v1.0"
+)
+STABLELM_2_1_6B_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "stabilityai/stablelm-2-1_6b"
+)
+STARCODER2_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "bigcode/starcoder2-7b")
+TRINITY_MINI_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "arcee-ai/Trinity-Mini")
+XVERSE_MOE_A36B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "xverse/XVERSE-MoE-A36B")
+MINIMAX_M2_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "cyankiwi/MiniMax-M2-BF16")
+MINIMAX_M2_5_W8A8_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Eco-Tech/MiniMax-M2.5-w8a8-QuaRot"
+)
+MINIMAX_M2_5_EAGLE3_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "sgl-npu/MiniMax-M2.5-eagel-model-0318"
+)
+EAGLE3_LLAMA3_1_INSTRUCT_8B_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "sglang-EAGLE3-LLaMA3.1-Instruct-8B"
+)
+
+# VLM model weights path
+DEEPSEEK_VL2_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-vl2")
+GLM_4_5V_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "ZhipuAI/GLM-4.5V")
+JANUS_PRO_1B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "deepseek-ai/Janus-Pro-1B")
+JANUS_PRO_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "deepseek-ai/Janus-Pro-7B")
+KIMI_VL_A3B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "moonshotai/Kimi-VL-A3B-Instruct"
+)
+LLAMA_3_2_11B_VISION_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "LLM-Research/Llama-3.2-11B-Vision-Instruct"
+)
+LLAVA_NEXT_72B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "lmms-lab/llava-next-72b")
+LLAVA_ONEVISION_QWEN2_7B_OV_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "lmms-lab/llava-onevision-qwen2-7b-ov"
+)
+LLAVA_V1_6_34B_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "AI-ModelScope/llava-v1.6-34b"
+)
+LLAVA_V1_6_34B_TOKENIZER_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "AI-ModelScope/llava-v1.6-34b/llava-1.6v-34b-tokenizer"
+)
+MIMO_VL_7B_RL_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "XiaomiMiMo/MiMo-VL-7B-RL")
+MINICPM_O_2_6_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "openbmb/MiniCPM-o-2_6")
+MINICPM_V_2_6_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "openbmb/MiniCPM-V-2_6")
+MISTRAL_SMALL_3_1_24B_INSTRUCT_2503_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
+)
+QWEN2_5_VL_3B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen2.5-VL-3B-Instruct"
+)
+QWEN2_5_VL_72B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen2.5-VL-72B-Instruct"
+)
+QWEN3_VL_4B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-VL-4B-Instruct"
+)
+QWEN3_VL_8B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-VL-8B-Instruct"
+)
+QWEN3_VL_30B_A3B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-VL-30B-A3B-Instruct"
+)
+QWEN3_VL_235B_A22B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-VL-235B-A22B-Instruct"
+)
+QWEN2_0_5B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen2-0.5B-Instruct"
+)
+
+QWEN3_30B_A3B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Qwen/Qwen3-30B-A3B")
+QWEN3_30B_A3B_W8A8_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-30B-A3B-w8a8"
+)
+DEEPSEEK_V2_LITE_W8A8_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "vllm-ascend/DeepSeek-V2-Lite-W8A8"
+)
+
+DEEPSEEK_R1_DISTILL_QWEN_7B_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
+)
+DEEPSEEK_R1_0528_W4A8_PER_CHANNEL_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "DeepSeek-R1-0528-w4a8-per-channel"
+)
+DEEPSEEK_R1_0528_W8A8_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "vllm-ascend/DeepSeek-R1-0528-W8A8"
+)
+QWEN3_30B_MODELSLIM_INT4_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-30B-A3B-w4a4-LAOS"
+)
+QWEN3_5_397B_W4A8_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp"
+)
+QWEN3_5_397B_W8A8_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
+)
+
+# Embedding model weights path
+BGE_LARGE_EN_V1_5_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "bge-large-en-v1.5")
+CLIP_VIT_LARGE_PATCH14_336_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "AI-ModelScope/clip-vit-large-patch14-336"
+)
+E5_MISTRAL_7B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "intfloat/e5-mistral-7b-instruct"
+)
+GME_QWEN2_VL_2B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Alibaba-NLP/gme-Qwen2-VL-2B-Instruct"
+)
+GTE_QWEN2_1_5B_INSTRUCT_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "iic/gte_Qwen2-1.5B-instruct"
+)
+QWEN3_EMBEDDING_8B_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-Embedding-8B"
+)
+
+# Rerank model weights path
+BGE_RERANKER_V2_M3_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "BAAI/bge-reranker-v2-m3"
+)
+
+# Reward model weights path
+INTERNLM2_7B_REWARD_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Shanghai_AI_Laboratory/internlm2-7b-reward"
+)
+QWEN2_5_1_5B_APEACH_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Howeee/Qwen2.5-1.5B-apeach"
+)
+QWEN2_5_MATH_RM_72B_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen2.5-Math-RM-72B"
+)
+SKYWORK_REWARD_GEMMA_2_27B_V0_2_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "AI-ModelScope/Skywork-Reward-Gemma-2-27B-v0.2"
+)
+SKYWORK_REWARD_LLAMA_3_1_8B_V0_2_WEIGHTS_PATH = os.path.join(
+    HF_MODEL_WEIGHTS_DIR,
+    "models--Skywork--Skywork-Reward-Llama-3.1-8B-v0.2/snapshots/d4117fbfd81b72f41b96341238baa1e3e90a4ce1",
+)
+KIMI_K2_6_W4A8_MODEL_PATH = os.path.join(MODEL_WEIGHTS_DIR, "Eco-Tech/Kimi-K2.6-w4a8")
+KIMI_K2_6_EAGLE3_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "lightseekorg/kimi-k2.6-eagle3"
+)
+GLM_4_6V_FLASH_MODEL_PATH = os.path.join(MODEL_WEIGHTS_DIR, "ZhipuAI/GLM-4.6V-Flash")
+QWEN3_VL_8B_THINKING_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-VL-8B-Thinking"
+)
+QWEN3_VL_30B_A3B_THINKING_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-VL-30B-A3B-Thinking"
+)
+QWEN3_OMNI_30B_A3B_THINKING_MODEL_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "Qwen/Qwen3-Omni-30B-A3B-Thinking"
+)
+# Images path
+IMAGES_EXAMPLE_PATH = os.path.join(IMAGES_DIR, "example_image.png")
+IMAGES_023_PATH = os.path.join(IMAGES_DIR, "023.jpg")
+IMAGES_MAN_PATH = os.path.join(IMAGES_DIR, "man.png")
+IMAGES_LOGO_PATH = os.path.join(IMAGES_DIR, "logo.png")
+VIDEO_JOBS_PATH = os.path.join(VIDEO_DIR, "jobs.mp4")
+INVOICE_WITH_BARCODE_LOGO_IMAGES_PATH = os.path.join(
+    IMAGES_DIR, "invoice_with_barcode_logo.jpeg"
+)
+# fmt: on
+
+# Other
+DEEPSEEK_CODER_JSON_PATH = "/__w/sglang/sglang/test/registered/ascend/basic_function/parameter/deepseek_coder.json"
+FR_SPEC_TOKEN_MAP_PATH = "/root/.cache/sglang/FR-Spec/freq_32768.pt"
+CONFIG_YAML_PATH = (
+    "/__w/sglang/sglang/test/registered/ascend/basic_function/config/config.yaml"
+)
+
+
+class ModelTestConfig(NamedTuple):
+    """
+    Configuration for model testing.
+
+    Attributes:
+        model_path: Path to the model weights directory
+        mmlu_score: Weight for MMLU benchmark score
+        gsm8k_accuracy: Weight for GSM8K benchmark score
+        mmmu_accuracy: Weight for MMMU benchmark score
+    """
+
+    model_path: str
+    mmlu_score: Optional[float] = None
+    gsm8k_accuracy: Optional[float] = None
+    mmmu_accuracy: Optional[float] = None
+
+
+LLAMA_3_2_1B_INSTRUCT_WEIGHTS_FOR_TEST = ModelTestConfig(
+    model_path=LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH, mmlu_score=0.2
+)
+
+QWEN3_30B_A3B_INSTRUCT_2507_WEIGHTS_FOR_TEST = ModelTestConfig(
+    model_path=QWEN3_30B_A3B_INSTRUCT_2507_WEIGHTS_PATH, gsm8k_accuracy=0.9
+)
+
+QWEN3_32B_WEIGHTS_FOR_TEST = ModelTestConfig(
+    model_path=QWEN3_32B_WEIGHTS_PATH, gsm8k_accuracy=0.82
+)
+
+QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_FOR_TEST = ModelTestConfig(
+    model_path=QWEN3_NEXT_80B_A3B_INSTRUCT_WEIGHTS_PATH, gsm8k_accuracy=0.92
+)
+
+QWQ_32B_W8A8_WEIGHTS_FOR_TEST = ModelTestConfig(
+    model_path=QWQ_32B_W8A8_WEIGHTS_PATH, gsm8k_accuracy=0.59
+)
+
+# Default configuration for testing
+DEFAULT_WEIGHTS_FOR_TEST = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_FOR_TEST
+
+
+def run_command(cmd, shell=True):
+    """Execute system command and return stdout
+
+    parameter:
+        cmd: command to execute
+        shell:
+        True, Execute command in shell
+        False, Commands are invoked directly without shell parsing
+    return:
+        The result of executing the command
+    """
+    try:
+        result = subprocess.run(
+            cmd, shell=shell, capture_output=True, text=True, check=True
+        )
+        return result.stdout
+    except subprocess.CalledProcessError as e:
+        print(f"execute command error: {e}")
+        return None
+
+
+def get_benchmark_args(
+    base_url="",
+    backend="sglang",
+    dataset_name="",
+    dataset_path="",
+    tokenizer="",
+    num_prompts=500,
+    sharegpt_output_len=None,
+    random_input_len=4096,
+    random_output_len=2048,
+    sharegpt_context_len=None,
+    request_rate=float("inf"),
+    disable_stream=False,
+    disable_ignore_eos=False,
+    seed: int = 0,
+    device="auto",
+    pd_separated: bool = False,
+    lora_name=None,
+    lora_request_distribution="uniform",
+    lora_zipf_alpha=1.5,
+    gsp_num_groups=4,
+    gsp_prompts_per_group=4,
+    gsp_system_prompt_len=128,
+    gsp_question_len=32,
+    gsp_output_len=32,
+    gsp_num_turns=1,
+    header=None,
+    max_concurrency=None,
+):
+    return SimpleNamespace(
+        backend=backend,
+        base_url=base_url,
+        host=None,
+        port=None,
+        dataset_name=dataset_name,
+        dataset_path=dataset_path,
+        model=None,
+        tokenizer=tokenizer,
+        num_prompts=num_prompts,
+        sharegpt_output_len=sharegpt_output_len,
+        sharegpt_context_len=sharegpt_context_len,
+        random_input_len=random_input_len,
+        random_output_len=random_output_len,
+        random_range_ratio=0.0,
+        request_rate=request_rate,
+        multi=None,
+        output_file=None,
+        disable_tqdm=False,
+        disable_stream=disable_stream,
+        return_logprob=False,
+        return_routed_experts=False,
+        seed=seed,
+        disable_ignore_eos=disable_ignore_eos,
+        extra_request_body=None,
+        apply_chat_template=False,
+        profile=None,
+        lora_name=lora_name,
+        lora_request_distribution=lora_request_distribution,
+        lora_zipf_alpha=lora_zipf_alpha,
+        prompt_suffix="",
+        device=device,
+        pd_separated=pd_separated,
+        gsp_num_groups=gsp_num_groups,
+        gsp_prompts_per_group=gsp_prompts_per_group,
+        gsp_system_prompt_len=gsp_system_prompt_len,
+        gsp_question_len=gsp_question_len,
+        gsp_output_len=gsp_output_len,
+        gsp_num_turns=gsp_num_turns,
+        header=header,
+        max_concurrency=max_concurrency,
+        ready_check_timeout_sec=0,
+    )
+
+
+def run_bench_serving(
+    model,
+    num_prompts,
+    request_rate,
+    other_server_args,
+    dataset_name="random",
+    dataset_path="",
+    tokenizer=None,
+    random_input_len=4096,
+    random_output_len=2048,
+    sharegpt_context_len=None,
+    disable_stream=False,
+    disable_ignore_eos=False,
+    need_warmup=False,
+    seed: int = 0,
+    device="auto",
+    gsp_num_groups=None,
+    gsp_prompts_per_group=None,
+    gsp_system_prompt_len=None,
+    gsp_question_len=None,
+    gsp_output_len=None,
+    max_concurrency=None,
+    background_task: Optional[Callable[[str, asyncio.Event], Awaitable[None]]] = None,
+    lora_name: Optional[str] = None,
+    timeout_for_server_launch=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+):
+    """Start the service and obtain the inference results.
+
+    Parameters:
+        model: Model name
+        num_prompts: Total number of test requests
+        request_rate: Request rate
+        other_server_args: Additional configuration when starting the service
+        dataset_name: Data set name
+        dataset_path: Dataset path
+        tokenizer: tokenizer
+        random_input_len: The length of the randomly generated input prompt
+        random_output_len: The length of the randomly generated output prompt
+        sharegpt_context_len: Sharegpt dataset context length
+        disable_stream: Disable streaming output
+        disable_ignore_eos: Should eos_token be ignored?
+        need_warmup: Preheating required
+        seed: random seed
+        device: Device type
+        gsp_num_groups: Grouped Sequence Parallelism
+        gsp_prompts_per_group: Number of parallel prompts within each group
+        gsp_system_prompt_len: GSP system prompts length
+        gsp_question_len: GSP question length
+        gsp_output_len: GSP output length
+        max_concurrency: Maximum number of concurrent requests
+        background_task: Background tasks
+        lora_name: LoRA fine-tuning model path
+        timeout_for_server_launch: Raise the service timeout period
+    Returns:
+        res: Number of requests successfully completed
+
+    """
+
+    if device == "auto":
+        device = auto_config_device()
+    # Launch the server
+    base_url = DEFAULT_URL_FOR_TEST
+    process = popen_launch_server(
+        model,
+        base_url,
+        timeout=timeout_for_server_launch,
+        other_args=other_server_args,
+    )
+
+    # Run benchmark
+    args = get_benchmark_args(
+        base_url=base_url,
+        dataset_name=dataset_name,
+        dataset_path=dataset_path,
+        tokenizer=tokenizer,
+        num_prompts=num_prompts,
+        random_input_len=random_input_len,
+        random_output_len=random_output_len,
+        sharegpt_context_len=sharegpt_context_len,
+        request_rate=request_rate,
+        disable_stream=disable_stream,
+        disable_ignore_eos=disable_ignore_eos,
+        seed=seed,
+        device=device,
+        lora_name=lora_name,
+        gsp_num_groups=gsp_num_groups,
+        gsp_prompts_per_group=gsp_prompts_per_group,
+        gsp_system_prompt_len=gsp_system_prompt_len,
+        gsp_question_len=gsp_question_len,
+        gsp_output_len=gsp_output_len,
+        max_concurrency=max_concurrency,
+    )
+
+    async def _run():
+        if need_warmup:
+            warmup_args = copy.deepcopy(args)
+            warmup_args.num_prompts = 16
+            await asyncio.to_thread(run_benchmark, warmup_args)
+
+        start_event = asyncio.Event()
+        stop_event = asyncio.Event()
+        task_handle = (
+            asyncio.create_task(background_task(base_url, start_event, stop_event))
+            if background_task
+            else None
+        )
+
+        try:
+            start_event.set()
+            result = await asyncio.to_thread(run_benchmark, args)
+        finally:
+            if task_handle:
+                stop_event.set()
+                await task_handle
+
+        return result
+
+    try:
+        res = asyncio.run(_run())
+    finally:
+        kill_process_tree(process.pid)
+
+    assert res["completed"] == num_prompts
+    return res
+
+
+# hook factory
+def create_attention_monitor_hook_factory(config):
+    """
+    Factory function to create a forward hook for monitoring self-attention layer states.
+    This hook records input/output statistics during model forward propagation.
+
+    Args:
+        config (dict): Configuration dictionary containing hook parameters
+            layer_index (int): Index of the target attention layer to monitor
+
+    Returns:
+        function: Forward hook function to be registered on the target module
+    """
+    # Get target layer index from config, default to 0 if not specified
+    layer_index = config.get("layer_index", 0)
+
+    # Initialize logging configuration if no handlers are set
+    if not logging.root.handlers:
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s - %(levelname)s - %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+
+    def attention_monitor_hook(module, inputs, output):
+        """
+        Forward hook function that monitors and logs the internal states of a self-attention layer.
+        Executed automatically during the forward pass of the module it is registered to.
+
+        Args:
+            module (torch.nn.Module): The module this hook is attached to
+            inputs (tuple): Input tensors passed to the module's forward method
+            output (torch.Tensor): Output tensor returned by the module's forward method
+
+        Returns:
+            torch.Tensor: Unmodified output tensor to preserve model computation flow
+        """
+        # Record current timestamp for time-series tracking
+        timestamp = time.time()
+
+        # Extract hidden states from inputs (second input tensor of attention layer)
+        hidden_states = inputs[1] if inputs and len(inputs) > 1 else None
+
+        # Construct monitoring record with key statistics
+        monitor_record = {
+            "timestamp": timestamp,
+            "layer_index": layer_index,
+            "module_type": type(module).__name__,
+            # Compute sum of hidden states across last dim, take first 5 elements for logging
+            "inputs": hidden_states.sum(-1)[:5] if hidden_states is not None else None,
+            # Compute sum of output across last dim, take first 5 elements for logging
+            "outputs": output.sum(-1)[:5],
+        }
+
+        # Log the monitoring record
+        logging.info(f"hook effect: {monitor_record}")
+
+        # Return the original output to maintain normal model forward propagation
+        return output
+
+    return attention_monitor_hook
+
+
+def read_output(output_lines: List[str], filename: str = STDERR_FILENAME):
+    """Print the output in real time with another thread."""
+    while not os.path.exists(filename):
+        time.sleep(0.01)
+
+    pt = 0
+    while pt >= 0:
+        if pt > 0 and not os.path.exists(filename):
+            break
+        try:
+            lines = open(filename).readlines()
+        except FileNotFoundError:
+            print(f"{pt=}, {os.path.exists(filename)=}")
+            raise
+        for line in lines[pt:]:
+            print(line, end="", flush=True)
+            output_lines.append(line)
+            pt += 1
+        time.sleep(0.1)
+
+
+def run_and_check_memory_leak(
+    workload_func,
+    disable_radix_cache,
+    enable_mixed_chunk,
+    disable_overlap,
+    chunked_prefill_size,
+    assert_has_abort,
+    api_key: Optional[str] = None,
+):
+    other_args = [
+        "--chunked-prefill-size",
+        str(chunked_prefill_size),
+        "--log-level",
+        "debug",
+    ]
+    if disable_radix_cache:
+        other_args += ["--disable-radix-cache"]
+    if enable_mixed_chunk:
+        other_args += ["--enable-mixed-chunk"]
+    if disable_overlap:
+        other_args += ["--disable-overlap-schedule"]
+
+    model = LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH
+    port = random.randint(4000, 5000)
+    base_url = f"http://127.0.0.1:{port}"
+
+    # Create files and launch the server
+    stdout = open(STDOUT_FILENAME, "w")
+    stderr = open(STDERR_FILENAME, "w")
+    process = popen_launch_server(
+        model,
+        base_url,
+        timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+        other_args=other_args,
+        return_stdout_stderr=(stdout, stderr),
+        api_key=api_key,
+    )
+
+    # Launch a thread to stream the output
+    output_lines = []
+    t = threading.Thread(target=read_output, args=(output_lines,))
+    t.start()
+
+    # Run the workload
+    workload_func(base_url, model)
+
+    # Clean up everything
+    kill_process_tree(process.pid)
+    stdout.close()
+    stderr.close()
+    if os.path.exists(STDOUT_FILENAME):
+        os.remove(STDOUT_FILENAME)
+    if os.path.exists(STDERR_FILENAME):
+        os.remove(STDERR_FILENAME)
+    kill_process_tree(process.pid)
+    t.join()
+
+    # Assert success
+    has_new_server = False
+    has_leak = False
+    has_abort = False
+    for line in output_lines:
+        if "Uvicorn running" in line:
+            has_new_server = True
+        if "leak" in line:
+            has_leak = True
+        if "Abort" in line:
+            has_abort = True
+
+    assert has_new_server
+    assert not has_leak
+    if assert_has_abort:
+        assert has_abort
+
+
+def run_mmlu_test(
+    disable_radix_cache=False,
+    enable_mixed_chunk=False,
+    disable_overlap=False,
+    chunked_prefill_size=32,
+):
+    def workload_func(base_url, model):
+        # Run the eval
+        args = SimpleNamespace(
+            base_url=base_url,
+            model=model,
+            eval_name="mmlu",
+            num_examples=128,
+            num_threads=128,
+        )
+
+        try:
+            metrics = run_eval(args)
+            assert metrics["score"] >= 0.65, f"{metrics=}"
+        finally:
+            pass
+
+    run_and_check_memory_leak(
+        workload_func,
+        disable_radix_cache,
+        enable_mixed_chunk,
+        disable_overlap,
+        chunked_prefill_size,
+        assert_has_abort=False,
+    )
+
+
+def send_concurrent_requests(
+    base_url: str,
+    num_requests: int,
+    num_concurrent: int = 8,
+    input_text: str = "The capital of France is",
+    max_new_tokens: int = 32,
+    temperature: float = 0.0,
+    request_timeout: int = 60,
+) -> list:
+    """Send multiple concurrent HTTP POST requests to the /generate endpoint.
+
+    Uses threading (NOT asyncio + blocking calls) to achieve true concurrency.
+    asyncio.gather() combined with synchronous requests.post() does not produce
+    real parallelism; threading is required for concurrent blocking I/O.
+
+    Parameters:
+        base_url: Server base URL, e.g. "http://127.0.0.1:30000"
+        num_requests: Total number of requests to send
+        num_concurrent: Maximum in-flight requests at any given time (semaphore)
+        input_text: Text prompt sent to every request
+        max_new_tokens: Maximum new tokens to generate per request
+        temperature: Sampling temperature (0 = greedy / deterministic)
+        request_timeout: Per-request HTTP timeout in seconds; raises on exceed
+
+    Returns:
+        Unsorted list of result dicts, one per request, each with:
+          task_id (int)    -- zero-based request index
+          status_code (int)-- HTTP status code, or -1 on exception
+          text (str)       -- response body, or exception message on failure
+    """
+
+    results: list = []
+    lock = threading.Lock()
+    semaphore = threading.Semaphore(num_concurrent)
+
+    def _send_one(task_id: int) -> None:
+        semaphore.acquire()
+        try:
+            response = requests.post(
+                f"{base_url}/generate",
+                json={
+                    "text": input_text,
+                    "sampling_params": {
+                        "temperature": temperature,
+                        "max_new_tokens": max_new_tokens,
+                    },
+                },
+                timeout=request_timeout,
+            )
+            with lock:
+                results.append(
+                    {
+                        "task_id": task_id,
+                        "status_code": response.status_code,
+                        "text": response.text,
+                    }
+                )
+        except Exception as exc:
+            with lock:
+                results.append(
+                    {
+                        "task_id": task_id,
+                        "status_code": -1,
+                        "text": str(exc),
+                    }
+                )
+        finally:
+            semaphore.release()
+
+    threads = [
+        threading.Thread(target=_send_one, args=(i,)) for i in range(num_requests)
+    ]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    return results
+
+
+HEADER = """
+### Models
+| Model | Server | Client | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Expected Accuracy | Status |
+| ----- | ------ | ------ | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ |
+"""
+
+
+def write_results_to_github_step_summary(results: dict):
+    if not is_in_ci():
+        return
+
+    write_github_step_summary_once(HEADER)
+
+    get_float = lambda metrics, item, precision: (
+        f"{metrics[item]:.{precision}f}"
+        if isinstance(metrics.get(item, "-"), (int, float))
+        else metrics.get(item, "-")
+    )
+
+    summary = ""
+    for model, metrics in results.items():
+        model = model.replace(MODEL_WEIGHTS_DIR, "").replace(HF_MODEL_WEIGHTS_DIR, "")
+        output_throughput = get_float(metrics, "output_throughput", 2)
+        output_throughput_threshold = metrics.get("output_throughput_threshold", "N/A")
+        accuracy = get_float(metrics, "accuracy", 4)
+        accuracy_threshold = metrics.get("accuracy_threshold", "N/A")
+        latency = get_float(metrics, "latency", 4)
+        latency_threshold = metrics.get("latency_threshold", "N/A")
+        server = metrics.get("server", "N/A")
+        client = metrics.get("client", "N/A")
+        error = metrics.get("error", "")
+        status = "✅" if error == "" else "❌ " + str(error)
+        summary += f"| {model} | {server} | {client} | {output_throughput} | {output_throughput_threshold} | {latency} | {latency_threshold} | {accuracy} | {accuracy_threshold} | {status} |\n"
+    write_github_step_summary(summary)
+
+
+def write_github_step_summary_once(summary: str):
+    if getattr(write_github_step_summary_once, "has_written", False):
+        return
+    write_github_step_summary_once.has_written = True
+    write_github_step_summary(summary)

python/sglang/test/ascend/test_embedding_base.py

@@ -0,0 +1,102 @@
+import multiprocessing as mp
+from abc import ABC
+from typing import List, Optional, Tuple
+
+import torch
+from transformers import AutoConfig, AutoTokenizer
+
+from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner
+from sglang.test.test_utils import get_similarities
+
+
+class BaseEmbeddingTest(ABC):
+    """Base test class for embedding model tests"""
+
+    MODELS: List[
+        Tuple[str, int, float]
+    ]  # [(model_path, tp_size, prefill_tolerance), ...]
+    TORCH_DTYPES: List[torch.dtype] = [torch.float16]
+    DEFAULT_PROMPTS: List[str] = DEFAULT_PROMPTS
+    DEFAULT_MAX_LENGTH: int = 2048
+
+    @classmethod
+    def setUpClass(cls):
+        mp.set_start_method("spawn", force=True)
+
+    def _truncate_prompts(self, prompts, model_path):
+        """Truncate prompts to model's max length"""
+        config = AutoConfig.from_pretrained(model_path)
+        max_length = getattr(config, "max_position_embeddings", self.DEFAULT_MAX_LENGTH)
+
+        tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+        truncated_prompts = []
+        for prompt in prompts:
+            tokens = tokenizer(prompt, return_tensors="pt", truncation=False)
+            if len(tokens.input_ids[0]) > max_length:
+                truncated_text = tokenizer.decode(
+                    tokens.input_ids[0][: max_length - 1], skip_special_tokens=True
+                )
+                truncated_prompts.append(truncated_text)
+            else:
+                truncated_prompts.append(prompt)
+        return truncated_prompts
+
+    def assert_close_prefill_logits(
+        self,
+        prompts,
+        model_path,
+        tp_size,
+        torch_dtype,
+        prefill_tolerance,
+        matryoshka_dim: Optional[int] = None,
+    ) -> None:
+        """Assert embeddings from HF and SRT are within tolerance"""
+        truncated_prompts = self._truncate_prompts(prompts, model_path)
+
+        with HFRunner(
+            model_path,
+            torch_dtype=torch_dtype,
+            model_type="embedding",
+            matryoshka_dim=matryoshka_dim,
+        ) as hf_runner:
+            hf_outputs = hf_runner.forward(truncated_prompts)
+
+        attention_backend = "ascend"
+        json_model_override_args = (
+            {"matryoshka_dimensions": [matryoshka_dim]} if matryoshka_dim else None
+        )
+
+        with SRTRunner(
+            model_path,
+            tp_size=tp_size,
+            torch_dtype=torch_dtype,
+            model_type="embedding",
+            attention_backend=attention_backend,
+            json_model_override_args=json_model_override_args,
+        ) as srt_runner:
+            srt_outputs = srt_runner.forward(
+                truncated_prompts, dimensions=matryoshka_dim
+            )
+
+        for i in range(len(prompts)):
+            hf_logits = torch.Tensor(hf_outputs.embed_logits[i])
+            srt_logits = torch.Tensor(srt_outputs.embed_logits[i])
+
+            similarity = torch.tensor(get_similarities(hf_logits, srt_logits))
+            print("similarity diff", abs(similarity - 1))
+
+            if len(prompts[i]) <= 1000:
+                assert torch.all(
+                    abs(similarity - 1) < prefill_tolerance
+                ), "embeddings are not all close"
+
+    def test_prefill_logits(self):
+        """Main test method to run for all models and dtypes"""
+        models_to_test = self.MODELS
+
+        for model, tp_size, prefill_tolerance in models_to_test:
+            for torch_dtype in self.TORCH_DTYPES:
+                self.assert_close_prefill_logits(
+                    self.DEFAULT_PROMPTS, model, tp_size, torch_dtype, prefill_tolerance
+                )

python/sglang/test/ascend/test_mmlu.py

@@ -0,0 +1,41 @@
+import subprocess
+from types import SimpleNamespace
+
+from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary
+from sglang.test.run_eval import run_eval
+
+
+class TestMMLU:
+
+    mmlu_num_examples = 128
+
+    def test_mmlu(self):
+        accuracy_mmlu_threshold = getattr(self, "accuracy_mmlu", 0.00)
+
+        model_metrics = {
+            "server": getattr(
+                self, "server_cmd", subprocess.list2cmdline(map(str, self.other_args))
+            ),
+            "client": "simple_eval_mmlu",
+            "accuracy_threshold": getattr(self, "accuracy_mmlu", "N/A"),
+        }
+
+        try:
+            args = SimpleNamespace(
+                base_url=self.base_url,
+                model=self.model,
+                eval_name="mmlu",
+                num_examples=self.mmlu_num_examples,
+                num_threads=32,
+            )
+            print("Starting mmlu test...")
+            metrics = run_eval(args)
+            model_metrics["accuracy"] = metrics["score"]
+            model_metrics["latency"] = metrics.get("latency", "-")
+            model_metrics["output_throughput"] = metrics.get("output_throughput", "-")
+            self.assertGreater(metrics["score"], accuracy_mmlu_threshold)
+        except Exception as e:
+            model_metrics["error"] = e
+            self.fail(f"Test failed for {self.model}: {e}")
+        finally:
+            write_results_to_github_step_summary({self.model: model_metrics})

python/sglang/test/ascend/test_no_hf_reward_base.py

@@ -0,0 +1,59 @@
+import multiprocessing as mp
+from abc import ABC
+
+import torch
+
+from sglang.test.runners import SRTRunner
+
+PROMPT = (
+    "What is the range of the numeric output of a sigmoid node in a neural network?"
+)
+RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1."
+RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1."
+
+CONVS = [
+    [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}],
+    [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}],
+]
+
+
+class BaseNoHFRewardModelTest(ABC):
+    """Base test class for reward model testing that doesn't compare with HF.
+
+    This is for models that only need to verify SGLang can run them successfully.
+    """
+
+    # Required attributes for subclasses
+    model_path: str
+
+    # Optional attributes with defaults
+    torch_dtype: torch.dtype = torch.float16
+    tp_size: int = 4
+    trust_remote_code: bool = True
+    disable_cuda_graph: bool = True
+    mem_fraction_static: float = 0.8
+
+    @classmethod
+    def setUpClass(cls):
+        mp.set_start_method("spawn", force=True)
+
+    def test_assert_close_reward_scores(self):
+        """Test that the model can generate reward scores."""
+        srt_runner_kwargs = {
+            "trust_remote_code": self.trust_remote_code,
+            "disable_cuda_graph": self.disable_cuda_graph,
+            "tp_size": self.tp_size,
+            "mem_fraction_static": self.mem_fraction_static,
+        }
+
+        with SRTRunner(
+            self.model_path,
+            torch_dtype=self.torch_dtype,
+            model_type="reward",
+            **srt_runner_kwargs,
+        ) as srt_runner:
+            prompts = srt_runner.tokenizer.apply_chat_template(CONVS, tokenize=False)
+            srt_outputs = srt_runner.forward(prompts)
+        srt_scores = torch.tensor(srt_outputs.scores)
+        print(f"accuracy: {srt_scores}")
+        self.assertIsInstance(srt_scores, torch.Tensor)

python/sglang/test/ascend/test_npu_logging.py

@@ -0,0 +1,153 @@
+import os
+import re
+import tempfile
+import time
+
+import requests
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ascend.test_ascend_utils import LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+
+class TestNPULoggingBase(CustomTestCase):
+    """Testcase：Test base class to verify whether the parameters in the logging function are correct.
+
+    Description:
+        Includes methods for initializing data and methods for verifying the correctness of the logging function.
+
+    [Test Category] Parameter
+    [Test Target] --log-requests; --log-requests-level; --log-requests-target; --uvicorn-access-log-exclude-prefixes;
+    --enable-metrics; --enable-metrics-for-all-scheduler;
+    --bucket-time-to-first-token; --bucket-inter-token-latency; --bucket-e2e-request-latency;
+    --collect-tokens-histogram; --prompt-tokens-buckets; --generation-tokens-buckets;
+    --tokenizer-metrics-custom-labels-header; --tokenizer-metrics-allowed-custom-labels;
+    --gc-warning-threshold-secs
+    """
+
+    @staticmethod
+    def get_lines_with_keyword(filename, keyword):
+        """Find and return lines matching a regex keyword from a specified file, with line numbers and content.
+
+        Function Description:
+            Reads the target file line by line, uses the input keyword as a regular expression pattern to match each line's content.
+            For each line that matches the regex pattern, encapsulates the line number (1-indexed) and content into a dictionary,
+            and finally returns a list of dictionaries containing all matched lines.
+
+        Args:
+            filename (str): Path to the file to be read
+            keyword (str): Regular expression pattern for matching
+
+        Returns:
+            List[Dict[str, Union[str, int]]]
+                List of dictionaries for matched lines, each dictionary contains two key-value pairs:
+                - "line_number": int - Line number of the matched line (starts from 1)
+                - "content": str - Full text content of the matched line
+        """
+        results = []
+        try:
+            with open(filename, "r", encoding="utf-8") as file:
+                for line_num, line in enumerate(file, 1):
+                    if re.match(keyword, line):
+                        results.append(
+                            {
+                                "line_number": line_num,
+                                "content": line.strip(),
+                            }
+                        )
+            return results
+        except Exception as e:
+            print(f"error:{e}")
+            return []
+
+    @classmethod
+    def setUpClass(cls):
+        cls.model = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_PATH
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.other_args = [
+            "--trust-remote-code",
+            "--mem-fraction-static",
+            "0.8",
+            "--attention-backend",
+            "ascend",
+            "--disable-cuda-graph",
+            "--log-requests",
+        ]
+        cls.out_log_file_obj = tempfile.NamedTemporaryFile(
+            mode="w+", encoding="utf-8", delete=False, suffix=".txt"
+        )
+        cls.out_log_name = cls.out_log_file_obj.name
+        cls.out_log_file = cls.out_log_file_obj
+        cls.err_log_file_obj = tempfile.NamedTemporaryFile(
+            mode="w+", encoding="utf-8", delete=False, suffix=".txt"
+        )
+        cls.err_log_name = cls.err_log_file_obj.name
+        cls.err_log_file = cls.err_log_file_obj
+        cls.process = None
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.process:
+            kill_process_tree(cls.process.pid)
+        cls.out_log_file.close()
+        os.remove(cls.out_log_name)
+        cls.err_log_file.close()
+        os.remove(cls.err_log_name)
+
+    @classmethod
+    def launch_server(cls):
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            other_args=cls.other_args,
+            return_stdout_stderr=(cls.out_log_file, cls.err_log_file),
+        )
+
+    def inference_once(self, max_tokens=32):
+        response = requests.post(
+            f"{self.base_url}/generate",
+            json={
+                "text": "The capital of France is",
+                "sampling_params": {
+                    "temperature": 0,
+                    "max_new_tokens": max_tokens,
+                },
+            },
+        )
+
+        self.assertEqual(response.status_code, 200, "Failed to call generate API")
+        self.assertIn("Paris", response.text, "Inference out error.")
+
+    def wait_for_log_content(self, timeout=30):
+        """Wait for and return the content of the specified log file, with timeout handling.
+
+        Function Description:
+            Continuously reads the target log file in a loop within the set timeout period,
+            avoids assertion failures caused by reading the log too early before log writing is completed.
+            Returns the log content immediately once the file has non-empty content,
+            otherwise waits and retries reading at intervals until the timeout is reached.
+
+        Args:
+            timeout (int, optional): Maximum waiting time in seconds, default value is 30 seconds.
+
+        Returns:
+            str
+                Full text content read from the log file:
+                - Non-empty string if log content is detected within the timeout period
+                - Empty string if no log content is found after the timeout expires
+        """
+        start_time = time.time()
+        content = ""
+        while time.time() - start_time < timeout:
+            with open(self.out_log_file.name, "r", encoding="utf-8") as f:
+                content = f.read()
+            if content:
+                break
+            time.sleep(0.5)
+        return content

python/sglang/test/ascend/vlm_utils.py

@@ -0,0 +1,97 @@
+import os
+import warnings
+from types import SimpleNamespace
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    popen_launch_server,
+)
+
+
+class TestVLMModels(CustomTestCase):
+    model = ""
+    mmmu_accuracy = 0.00
+    other_args = [
+        "--trust-remote-code",
+        "--cuda-graph-max-bs-decode",
+        "32",
+        "--enable-multimodal",
+        "--mem-fraction-static",
+        0.35,
+        "--log-level",
+        "info",
+        "--attention-backend",
+        "ascend",
+        "--disable-cuda-graph",
+        "--tp-size",
+        4,
+    ]
+    timeout_for_server_launch = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+
+    @classmethod
+    def setUpClass(cls):
+        # Removed argument parsing from here
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.api_key = "sk-123456"
+
+        # Set OpenAI API key and base URL environment variables. Needed for lmm-evals to work.
+        os.environ["OPENAI_API_KEY"] = cls.api_key
+        os.environ["OPENAI_API_BASE"] = f"{cls.base_url}/v1"
+
+        # Prepare environment variables
+        process_env = os.environ.copy()
+
+        cls.process = popen_launch_server(
+            cls.model,
+            base_url=cls.base_url,
+            timeout=cls.timeout_for_server_launch,
+            api_key=cls.api_key,
+            other_args=cls.other_args,
+            env=process_env,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.process and cls.process.poll() is None:
+            print(f"Cleaning up server process {cls.process.pid}")
+            try:
+                kill_process_tree(cls.process.pid)
+            except Exception as e:
+                print(f"Error killing server process: {e}")
+
+    def _run_vlm_mmmu_test(self, test_name=""):
+        warnings.filterwarnings(
+            "ignore", category=ResourceWarning, message="unclosed.*socket"
+        )
+
+        args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="mmmu",
+            num_examples=100,
+            num_threads=64,
+            max_tokens=30,
+            return_latency=True,
+        )
+
+        metrics, latency = run_eval(args)
+
+        metrics["score"] = round(metrics["score"], 4)
+        metrics["latency"] = round(latency, 4)
+
+        print(
+            f"\n{'=' * 42}\n"
+            f"{self.model} - metrics={metrics} score={metrics['score']}\n"
+            f"{'=' * 42}\n"
+        )
+
+        self.assertGreaterEqual(
+            metrics["score"],
+            self.mmmu_accuracy,
+            f"Model {self.model} accuracy ({metrics['score']}) "
+            f"below expected threshold ({self.mmmu_accuracy:.4f}){test_name}",
+        )