chore: import upstream snapshot with attribution

python/sglang/test/ascend/e2e/gen_dataset_fixed_len.py

@@ -0,0 +1,521 @@
+import json
+import os
+import random
+import string
+
+import numpy as np
+from PIL import Image
+from transformers import AutoTokenizer
+
+
+def load_jsonl(path):
+    """Load data from a JSONL file, one JSON object per line."""
+    data = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                data.append(json.loads(line))
+    return data
+
+
+def save_jsonl(data, file_path):
+    """Save a list of dicts to a JSONL file, one JSON object per line."""
+    file_dir = os.path.dirname(file_path)
+    if file_dir:
+        os.makedirs(file_dir, exist_ok=True)
+    with open(file_path, "w", encoding="utf-8") as f:
+        for item in data:
+            f.write(json.dumps(item, ensure_ascii=False) + "\n")
+
+
+def format_qa(item):
+    """Format a GSM8K data entry into QA text for the few-shot pool."""
+    question = item["question"]
+    answer = item["answer"]
+    return f"Question: {question}\nLet's think step by step\nAnswer:\n{answer}\n\n"
+
+
+def pad_to_target_tokens(
+    question,
+    few_shot_pool_token_ids,
+    tokenizer,
+    target_tokens,
+    test_template="Question: {question}\nLet's think step by step\nAnswer:\n",
+):
+    """Pad a question text to the target token length.
+
+    Tokenizes the question using the test_template, calculates the remaining tokens
+    needed, and prepends randomly sampled few-shot token ids from the pool to reach
+    target_tokens. If the few-shot pool is insufficient, repeats the first sample
+    to fill the remaining gap.
+
+    Args:
+        question: The test question text.
+        few_shot_pool_token_ids: List of token id lists from the few-shot training pool.
+        tokenizer: The tokenizer instance.
+        target_tokens: Target input token length.
+        test_template: Question template string, defaults to GSM8K format.
+    """
+    test_prompt = test_template.format(question=question)
+    test_token_ids = tokenizer.encode(test_prompt, add_special_tokens=False)
+
+    remaining_tokens = target_tokens - len(test_token_ids)
+    if remaining_tokens <= 0:
+        return tokenizer.decode(
+            test_token_ids[:target_tokens], skip_special_tokens=True
+        )
+
+    shuffled_ids = list(range(len(few_shot_pool_token_ids)))
+    random.shuffle(shuffled_ids)
+
+    prefix_ids = []
+    for idx in shuffled_ids:
+        fs_ids = few_shot_pool_token_ids[idx]
+        if len(prefix_ids) + len(fs_ids) <= remaining_tokens:
+            prefix_ids.extend(fs_ids)
+        else:
+            partial_gap = remaining_tokens - len(prefix_ids)
+            if partial_gap > 0:
+                prefix_ids.extend(fs_ids[:partial_gap])
+            break
+
+    if len(prefix_ids) < remaining_tokens and few_shot_pool_token_ids:
+        padding_source_ids = few_shot_pool_token_ids[shuffled_ids[0]]
+        repeat_count = (remaining_tokens // len(padding_source_ids)) + 1
+        padding_ids = (padding_source_ids * repeat_count)[
+            : remaining_tokens - len(prefix_ids)
+        ]
+        prefix_ids.extend(padding_ids)
+
+    full_ids = prefix_ids + test_token_ids
+    return tokenizer.decode(full_ids[:target_tokens], skip_special_tokens=True)
+
+
+def generate_custom_dataset(
+    train_path,
+    test_path,
+    tokenizer_path,
+    target_tokens,
+    num_prompts,
+    trust_remote_code=False,
+    test_template="Question: {question}\nLet's think step by step\nAnswer:\n",
+):
+    """Generate a custom dataset with a fixed input token length.
+
+    Builds a few-shot pool from the training set and pads test questions to the
+    specified token length. If the test set has fewer samples than num_prompts,
+    it cycles and repeats to fill the required count.
+
+    Args:
+        train_path: Path to the GSM8K training JSONL file.
+        test_path: Path to the GSM8K test JSONL file.
+        tokenizer_path: Path to the tokenizer.
+        target_tokens: Target input token length.
+        num_prompts: Number of prompts to generate; 0 means use all test samples.
+        trust_remote_code: Whether to trust remote code when loading the tokenizer.
+        test_template: Question template string.
+
+    Returns:
+        list[dict]: Each item contains fields defined in test_template.
+    """
+    tokenizer = AutoTokenizer.from_pretrained(
+        tokenizer_path, trust_remote_code=trust_remote_code
+    )
+
+    train_data = load_jsonl(train_path)
+    test_data = load_jsonl(test_path)
+    if num_prompts > 0 and num_prompts > len(test_data):
+        multiplier = (num_prompts // len(test_data)) + 1
+        test_data = (test_data * multiplier)[:num_prompts]
+    elif num_prompts > 0:
+        test_data = test_data[:num_prompts]
+
+    few_shot_pool = [format_qa(item) for item in train_data]
+    few_shot_pool_token_ids = [
+        tokenizer.encode(fs, add_special_tokens=False) for fs in few_shot_pool
+    ]
+
+    output_data = []
+    for i, test_item in enumerate(test_data):
+        padded_question = pad_to_target_tokens(
+            question=test_item["question"],
+            few_shot_pool_token_ids=few_shot_pool_token_ids,
+            tokenizer=tokenizer,
+            target_tokens=target_tokens,
+            test_template=test_template,
+        )
+        output_data.append(
+            {
+                "question": padded_question,
+                "answer": test_item["answer"],
+            }
+        )
+        if (i + 1) % 100 == 0:
+            actual_tokens = len(
+                tokenizer.encode(padded_question, add_special_tokens=False)
+            )
+            print(
+                f"Processed {i + 1}/{len(test_data)}, last item tokens: {actual_tokens}"
+            )
+
+    token_counts = [
+        len(tokenizer.encode(item["question"], add_special_tokens=False))
+        for item in output_data
+    ]
+    print(
+        f"Token count stats: min={min(token_counts)}, max={max(token_counts)}, avg={sum(token_counts)/len(token_counts):.1f}"
+    )
+
+    return output_data
+
+
+def generate_random_images(mm_dataset_data, size):
+    """Generate random image files for a multimodal dataset.
+
+    Creates random RGB images at the specified resolution for each image path
+    listed in the dataset entries.
+
+    Args:
+        mm_dataset_data: List of multimodal data entries, each with a "path" field
+            containing a list of image file paths.
+        size: Image size tuple (width, height), e.g. (1080, 1920).
+    """
+    total_image_num = len(mm_dataset_data)
+    print(f"begin to generate images, total {total_image_num}")
+
+    file_count = 0
+    for item in mm_dataset_data:
+        image_paths = item.get("path")
+
+        for image_path in image_paths:
+            if not image_path:
+                print("Error: The image path is none.")
+                continue
+
+            dir_name = os.path.dirname(image_path)
+            if dir_name and not os.path.exists(dir_name):
+                os.makedirs(dir_name, exist_ok=True)
+
+            random_array = np.random.randint(
+                0, 256, (size[1], size[0], 3), dtype=np.uint8
+            )
+
+            img = Image.fromarray(random_array)
+            img.save(image_path, quality=95)
+            if os.path.isfile(image_path):
+                file_count += 1
+
+    print(f"Finish images generation. Image num: {file_count}")
+
+
+def generate_mm_dataset(
+    train_path,
+    test_path,
+    tokenizer_path,
+    target_tokens=3500,
+    num_prompts=1024,
+    trust_remote_code=False,
+    test_template="Question: {question}\nLet's think step by step\nAnswer:\n",
+    image_dir="/tmp/datasets/image",
+    size=None,
+):
+    """Generate a multimodal (text + image) dataset.
+
+    First generates fixed-length text data via generate_fixed_len_dataset, then
+    attaches random image paths and type labels to each entry, and generates
+    the corresponding random image files.
+
+    Args:
+        train_path: Path to the GSM8K training JSONL file.
+        test_path: Path to the GSM8K test JSONL file.
+        tokenizer_path: Path to the tokenizer.
+        target_tokens: Target input token length.
+        num_prompts: Number of prompts to generate.
+        trust_remote_code: Whether to trust remote code when loading the tokenizer.
+        test_template: Question template string.
+        image_dir: Directory to save generated image files.
+        size: Image size string in "widthxheight" format, e.g. "1080x1920".
+
+    Returns:
+        list[dict]: Each item contains "question", "answer", "type", and "path" fields.
+    """
+    output_data = []
+    text_data = generate_custom_dataset(
+        train_path,
+        test_path,
+        tokenizer_path,
+        target_tokens,
+        num_prompts,
+        trust_remote_code,
+        test_template,
+    )
+
+    for item in text_data:
+        random_string = "".join(
+            random.choices(string.ascii_letters + string.digits, k=10)
+        )
+        item["type"] = "image"
+        item["path"] = [f"{image_dir}/{random_string}.jpg"]
+        output_data.append(item)
+
+    size = tuple(map(int, size.split("x")))
+    generate_random_images(output_data, size)
+    return output_data
+
+
+def generate_gsm8k_dataset(
+    model_path, source_dataset_path, batch_size, input_len, output_file
+):
+    """Generate a dataset with a fixed input token length from GSM8K (JSONL format).
+
+    Reads GSM8K source data, repeats or truncates each question's tokens to input_len,
+    then trims or replicates the dataset to batch_size entries, shuffles, and writes
+    to the output file.
+
+    Args:
+        model_path: Model path used to load the tokenizer.
+        source_dataset_path: Path to the GSM8K source JSONL file.
+        batch_size: Number of samples to generate.
+        input_len: Target input token length.
+        output_file: Output JSONL file path.
+    """
+    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    dataset = []
+    with open(source_dataset_path, "r", encoding="utf-8") as f:
+        for line in f:
+            data = json.loads(line)
+            dataset.append(data["question"])
+
+    dataset_new = []
+    for sentence in dataset:
+        words = tokenizer.tokenize(sentence)
+        len_num = len(words) // input_len
+        if len_num == 0:
+            multiplier = (input_len // len(words)) + 1
+            repeated_len = words * multiplier
+            words = repeated_len[:input_len]
+            decoded_text = tokenizer.convert_tokens_to_string(words)
+            if len(words) != input_len:
+                print(
+                    f"Generate DataSet Error: the length of new input is {len(words)}, not {input_len}"
+                )
+            dataset_new.append(decoded_text)
+
+    batch_num = len(dataset_new) // batch_size
+    if batch_num == 0:
+        multiplier = (batch_size // len(dataset_new)) + 1
+        repeated_batch = dataset_new * multiplier
+        dataset_new = repeated_batch[:batch_size]
+    else:
+        dataset_new = dataset_new[:batch_size]
+
+    random.shuffle(dataset_new)
+
+    if len(dataset_new) != batch_size:
+        print(
+            f"Generate DataSet Error: the size of new dataset is {len(dataset_new)}, not {batch_size}"
+        )
+
+    output_dir = os.path.dirname(output_file)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+
+    with open(output_file, "w", encoding="utf-8") as f:
+        for i in range(len(dataset_new)):
+            f.write(
+                json.dumps(
+                    {"question": f"{dataset_new[i]}", "answer": "none"},
+                    ensure_ascii=False,
+                )
+            )
+            f.write("\n")
+
+
+def generate_random_dataset(
+    model_path,
+    source_dataset_path,
+    batch_size,
+    input_len,
+    output_file,
+    output_len=1024,
+    range_ratio=1,
+):
+    """Generate a random dataset with logic matching bench_serving's --dataset-name random.
+
+    Samples real conversation text from the ShareGPT dataset as prompts, adjusting
+    to the target token length via truncation or repetition. Input/output lengths
+    are randomly sampled from [target*range_ratio, target]. Output format is a
+    JSON array compatible with ais_bench's ShareGPTDataset.
+
+    If source_dataset_path is not a valid JSON file, automatically downloads the
+    ShareGPT dataset from HuggingFace (anon8231489123/ShareGPT_Vicuna_unfiltered).
+
+    Args:
+        model_path: Model path used to load the tokenizer.
+        source_dataset_path: Path to the ShareGPT JSON file; auto-downloaded if invalid.
+        batch_size: Number of samples to generate.
+        input_len: Target input token length.
+        output_file: Output JSON file path.
+        output_len: Target output token length, default 1024.
+        range_ratio: Random range ratio for input/output lengths. Actual lengths are
+            uniformly sampled from [target*range_ratio, target]. Default 1 (fixed length).
+    """
+    SHAREGPT_REPO_ID = "anon8231489123/ShareGPT_Vicuna_unfiltered"
+    SHAREGPT_FILENAME = "ShareGPT_V3_unfiltered_cleaned_split.json"
+
+    def _is_file_valid_json(path):
+        """Check if the path points to a valid JSON file (exists and parseable)."""
+        if not os.path.isfile(path):
+            return False
+        try:
+            with open(path, encoding="utf-8") as f:
+                json.load(f)
+            return True
+        except json.JSONDecodeError:
+            return False
+
+    def _download_and_cache_hf_file(repo_id, filename, repo_type="dataset"):
+        """Download and cache a file from HuggingFace Hub."""
+        from huggingface_hub import hf_hub_download
+
+        return hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+    # Randomly sample input/output lengths per request in [target*range_ratio, target]
+    input_lens = np.random.randint(
+        max(int(input_len * range_ratio), 1),
+        input_len + 1,
+        size=batch_size,
+    ).tolist()
+    output_lens = np.random.randint(
+        max(int(output_len * range_ratio), 1),
+        output_len + 1,
+        size=batch_size,
+    ).tolist()
+
+    # Subtract special tokens to ensure the actual encoded length does not exceed target
+    num_special_tokens = int(tokenizer.num_special_tokens_to_add())
+    for i in range(batch_size):
+        input_lens[i] = max(1, input_lens[i] - num_special_tokens)
+
+    # Auto-download ShareGPT dataset from HuggingFace if local file is invalid
+    if not _is_file_valid_json(source_dataset_path):
+        print(
+            f"source_dataset_path '{source_dataset_path}' is not a valid file, downloading from HuggingFace..."
+        )
+        source_dataset_path = _download_and_cache_hf_file(
+            repo_id=SHAREGPT_REPO_ID,
+            filename=SHAREGPT_FILENAME,
+        )
+
+    # Load ShareGPT dataset, filter for >=2 turns, take the first turn (human) as prompt
+    with open(source_dataset_path, "r", encoding="utf-8") as f:
+        dataset = json.load(f)
+
+    dataset = [
+        data
+        for data in dataset
+        if len(data.get("conversations", data.get("conversation", []))) >= 2
+    ]
+    dataset = [
+        (
+            data.get("conversations", data.get("conversation", []))[0]["value"],
+            data.get("conversations", data.get("conversation", []))[1]["value"],
+        )
+        for data in dataset
+    ]
+    random.shuffle(dataset)
+
+    # Sample prompts, truncating or repeating tokens to reach target input length
+    input_requests = []
+    for data in dataset:
+        i = len(input_requests)
+        if i == batch_size:
+            break
+
+        prompt = data[0]
+        prompt_token_ids = tokenizer.encode(prompt)
+        prompt_len = len(prompt_token_ids)
+
+        if prompt_len == 0:
+            continue
+
+        if prompt_len > input_lens[i]:
+            input_ids = prompt_token_ids[: input_lens[i]]
+        else:
+            ratio = (input_lens[i] + prompt_len - 1) // prompt_len
+            input_ids = (prompt_token_ids * ratio)[: input_lens[i]]
+        input_content = tokenizer.decode(input_ids)
+        # Output format compatible with ais_bench ShareGPTDataset
+        input_requests.append(
+            {
+                "id": str(i),
+                "conversations": [
+                    {"from": "human", "value": input_content},
+                    {"from": "gpt", "value": "none"},
+                ],
+            }
+        )
+
+    print(f"#Input tokens: {np.sum(input_lens[:len(input_requests)])}")
+    print(f"#Output tokens: {np.sum(output_lens[:len(input_requests)])}")
+
+    output_dir = os.path.dirname(output_file)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+
+    # Output as JSON array format, compatible with ais_bench's json.load()
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(input_requests, f, ensure_ascii=False, indent=2)
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Generate GSM8K dataset with exact input token length"
+    )
+    parser.add_argument(
+        "--train_path", type=str, required=True, help="Path to GSM8K train.jsonl"
+    )
+    parser.add_argument(
+        "--test_path", type=str, required=True, help="Path to GSM8K test.jsonl"
+    )
+    parser.add_argument(
+        "--output_path", type=str, required=True, help="Output jsonl path"
+    )
+    parser.add_argument(
+        "--tokenizer_path", type=str, required=True, help="Path to model tokenizer"
+    )
+    parser.add_argument(
+        "--target_tokens", type=int, default=3500, help="Target input token length"
+    )
+    parser.add_argument(
+        "--trust_remote_code",
+        action="store_true",
+        help="Trust remote code for tokenizer",
+    )
+    parser.add_argument(
+        "--num_prompts",
+        type=int,
+        default=0,
+        help="Number of prompts to generate, 0 means all",
+    )
+    args = parser.parse_args()
+
+    output_data = generate_custom_dataset(
+        train_path=args.train_path,
+        test_path=args.test_path,
+        tokenizer_path=args.tokenizer_path,
+        target_tokens=args.target_tokens,
+        num_prompts=args.num_prompts,
+        trust_remote_code=args.trust_remote_code,
+    )
+    save_jsonl(output_data, args.output_path)
+    print(f"Done! Output {len(output_data)} items to {args.output_path}")
+
+
+if __name__ == "__main__":
+    main()

python/sglang/test/ascend/e2e/k8s_multi_pd_mix.yaml.jinja2

@@ -0,0 +1,134 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kube_config_map }}
+  namespace: {{ name_space }}
+data: {}
+
+---
+apiVersion: batch.volcano.sh/v1alpha1
+kind: Job
+metadata:
+  name: {{ kube_job_name }}
+  namespace: {{ name_space }}
+  labels:
+    ring-controller.atlas: ascend-1980
+    fault-scheduling: "force"
+spec:
+  minAvailable: {{ node_size }}
+  schedulerName: volcano
+  policies:
+    - event: PodEvicted
+      action: RestartJob
+  queue: default
+  tasks:
+  - name: "sglang-node"
+    replicas: {{ node_size }}
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        hostNetwork: True
+        containers:
+        - image: {{ image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: NAMESPACE
+            value: "{{ name_space }}"
+          - name: KUBE_CONFIG_MAP
+            value: "{{ kube_config_map }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               {% if env in ["ci", "debug"] %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+               {% endif %}
+               {% if env == "green" %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }}
+               {% endif %}
+          resources:
+            requests:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+            limits:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure

python/sglang/test/ascend/e2e/k8s_multi_pd_mix_green.yaml.jinja2

@@ -0,0 +1,107 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kube_config_map }}
+  namespace: {{ name_space }}
+data: {}
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ kube_job_name }}-sglang-node
+  namespace: {{ name_space }}
+spec:
+  replicas: {{ node_size }}
+  serviceName: {{ kube_job_name }}-sglang-pd-mix-headless
+  selector:
+    matchLabels:
+      app: sgl-ascend
+      task: pd-mix
+  template:
+    metadata:
+      labels:
+        app: sgl-ascend
+        task: pd-mix
+    spec:
+      hostNetwork: True
+      containers:
+      - image: {{ image }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        name: sgl-ascend
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: INSTALL_SGLANG_FROM_SOURCE
+          value: "{{ install_sglang_from_source }}"
+        - name: KUBECONFIG
+          value: "{{ kube_config }}"
+        - name: NAMESPACE
+          value: "{{ name_space }}"
+        - name: KUBE_CONFIG_MAP
+          value: "{{ kube_config_map }}"
+        - name: HF_ENDPOINT
+          value: "https://hf-mirror.com"
+        - name: METRICS_DATA_FILE
+          value: "{{ metrics_data_file }}"
+        - name: SGLANG_IS_IN_CI
+          value: "{{ sglang_is_in_ci }}"
+        - name: TRANSFORMERS_VERSION_FOR_SGLANG
+          value: "{{ transformers_version }}"
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+             bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+        resources:
+          requests:
+            huawei.com/Ascend910: 16
+          limits:
+            huawei.com/Ascend910: 16
+        volumeMounts:
+        - name: ascend-driver
+          mountPath: /usr/local/Ascend/driver
+        - name: localtime
+          mountPath: /etc/localtime
+        - name: share
+          mountPath: /data/ascend-ci-share-pkking-sglang
+        - name: share
+          mountPath: /root/.cache
+        - name: share
+          mountPath: /root/sglang
+          subPath: {{ sglang_source_relative_path }}
+      volumes:
+      - name: share
+        hostPath:
+          path: /home
+      - name: ascend-driver
+        hostPath:
+          path: /usr/local/Ascend/driver
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      nodeSelector:
+        accelerator: huawei-Ascend910
+        accelerator-type: module-a3-16
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ kube_job_name }}-sglang-pd-mix-headless
+  namespace: {{ name_space }}
+spec:
+  clusterIP: None
+  selector:
+    app: sgl-ascend
+    task: pd-mix
+  ports:
+  - port: 80
+    name: dummy-port

python/sglang/test/ascend/e2e/k8s_multi_pd_separation.yaml.jinja2

@@ -0,0 +1,350 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kube_config_map }}
+  namespace: {{ name_space }}
+data: {}
+
+---
+apiVersion: batch.volcano.sh/v1alpha1
+kind: Job
+metadata:
+  name: {{ kube_job_name }}
+  namespace: {{ name_space }}
+  labels:
+    ring-controller.atlas: ascend-1980
+    fault-scheduling: "force"
+spec:
+  minAvailable: {{ prefill_size + decode_size + router_size }}
+  schedulerName: volcano
+  policies:
+    - event: PodEvicted
+      action: RestartJob
+  queue: default
+  tasks:
+  - name: "sglang-prefill"
+    replicas: {{ prefill_size }}
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        hostNetwork: True
+        containers:
+        - image: {{ image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: NAMESPACE
+            value: "{{ name_space }}"
+          - name: KUBE_CONFIG_MAP
+            value: "{{ kube_config_map }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               {% if env in ["ci", "debug"] %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+               {% endif %}
+               {% if env == "green" %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }}
+               {% endif %}
+          resources:
+            requests:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+            limits:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure
+  - name: "sglang-decode"
+    replicas: {{ decode_size }}
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        hostNetwork: True
+        containers:
+        - image: {{ image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: NAMESPACE
+            value: "{{ name_space }}"
+          - name: KUBE_CONFIG_MAP
+            value: "{{ kube_config_map }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               {% if env in ["ci", "debug"] %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+               {% endif %}
+               {% if env == "green" %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }}
+               {% endif %}
+          resources:
+            requests:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+            limits:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: 16
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: 16
+              {% endif %}
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure
+  - name: "sglang-router"
+    replicas: {{ router_size }}
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        hostNetwork: True
+        containers:
+        - image: {{ image }}
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: NAMESPACE
+            value: "{{ name_space }}"
+          - name: KUBE_CONFIG_MAP
+            value: "{{ kube_config_map }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: SGLANG_IS_IN_CI
+            value: "{{ sglang_is_in_ci }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               {% if env in ["ci", "debug"] %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+               {% endif %}
+               {% if env == "green" %}
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase_green.sh {{ test_case }}
+               {% endif %}
+          resources:
+            requests:
+              cpu: "4"
+            limits:
+              cpu: "4"
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure

python/sglang/test/ascend/e2e/k8s_multi_pd_separation_green.yaml.jinja2

@@ -0,0 +1,307 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ kube_config_map }}
+  namespace: {{ name_space }}
+data: {}
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ kube_job_name }}-sglang-prefill
+  namespace: {{ name_space }}
+spec:
+  replicas: {{ prefill_size }}
+  serviceName: {{ kube_job_name }}-sglang-prefill-headless
+  selector:
+    matchLabels:
+      app: sgl-ascend
+      task: prefill
+  template:
+    metadata:
+      labels:
+        app: sgl-ascend
+        task: prefill
+    spec:
+      restartPolicy: Always
+      hostNetwork: True
+      containers:
+      - image: {{ image }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        name: sgl-ascend
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: INSTALL_SGLANG_FROM_SOURCE
+          value: "{{ install_sglang_from_source }}"
+        - name: KUBECONFIG
+          value: "{{ kube_config }}"
+        - name: NAMESPACE
+          value: "{{ name_space }}"
+        - name: KUBE_CONFIG_MAP
+          value: "{{ kube_config_map }}"
+        - name: HF_ENDPOINT
+          value: "https://hf-mirror.com"
+        - name: METRICS_DATA_FILE
+          value: "{{ metrics_data_file }}"
+        - name: SGLANG_IS_IN_CI
+          value: "{{ sglang_is_in_ci }}"
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+             bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+             exit $?
+        resources:
+          requests:
+            huawei.com/Ascend910: 16
+          limits:
+            huawei.com/Ascend910: 16
+        volumeMounts:
+        - name: ascend-driver
+          mountPath: /usr/local/Ascend/driver
+        - name: localtime
+          mountPath: /etc/localtime
+        - name: share
+          mountPath: /data/ascend-ci-share-pkking-sglang
+        - name: share
+          mountPath: /root/.cache
+        - name: share
+          mountPath: /root/sglang
+          subPath: {{ sglang_source_relative_path }}
+      volumes:
+      - name: share
+        hostPath:
+          path: /home
+      - name: ascend-driver
+        hostPath:
+          path: /usr/local/Ascend/driver
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      nodeSelector:
+        accelerator: huawei-Ascend910
+        accelerator-type: module-a3-16
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ kube_job_name }}-sglang-prefill-headless
+  namespace: {{ name_space }}
+spec:
+  clusterIP: None
+  selector:
+    app: sgl-ascend
+    task: prefill
+  ports:
+  - port: 80
+    name: dummy-port
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ kube_job_name }}-sglang-decode
+  namespace: {{ name_space }}
+spec:
+  replicas: {{ decode_size }}
+  serviceName: {{ kube_job_name }}-sglang-decode-headless
+  selector:
+    matchLabels:
+      app: sgl-ascend
+      task: decode
+  template:
+    metadata:
+      labels:
+        app: sgl-ascend
+        task: decode
+    spec:
+      restartPolicy: Always
+      hostNetwork: True
+      containers:
+      - image: {{ image }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        name: sgl-ascend
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: INSTALL_SGLANG_FROM_SOURCE
+          value: "{{ install_sglang_from_source }}"
+        - name: KUBECONFIG
+          value: "{{ kube_config }}"
+        - name: NAMESPACE
+          value: "{{ name_space }}"
+        - name: KUBE_CONFIG_MAP
+          value: "{{ kube_config_map }}"
+        - name: HF_ENDPOINT
+          value: "https://hf-mirror.com"
+        - name: METRICS_DATA_FILE
+          value: "{{ metrics_data_file }}"
+        - name: SGLANG_IS_IN_CI
+          value: "{{ sglang_is_in_ci }}"
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+             bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+             exit $?
+        resources:
+          requests:
+            huawei.com/Ascend910: 16
+          limits:
+            huawei.com/Ascend910: 16
+        volumeMounts:
+        - name: ascend-driver
+          mountPath: /usr/local/Ascend/driver
+        - name: localtime
+          mountPath: /etc/localtime
+        - name: share
+          mountPath: /data/ascend-ci-share-pkking-sglang
+        - name: share
+          mountPath: /root/.cache
+        - name: share
+          mountPath: /root/sglang
+          subPath: {{ sglang_source_relative_path }}
+      volumes:
+      - name: share
+        hostPath:
+          path: /home
+      - name: ascend-driver
+        hostPath:
+          path: /usr/local/Ascend/driver
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      nodeSelector:
+        accelerator: huawei-Ascend910
+        accelerator-type: module-a3-16
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ kube_job_name }}-sglang-decode-headless
+  namespace: {{ name_space }}
+spec:
+  clusterIP: None
+  selector:
+    app: sgl-ascend
+    task: decode
+  ports:
+  - port: 80
+    name: dummy-port
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ kube_job_name }}-sglang-router
+  namespace: {{ name_space }}
+spec:
+  replicas: {{ router_size }}
+  serviceName: {{ kube_job_name }}-sglang-router-headless
+  selector:
+    matchLabels:
+      app: sgl-ascend
+      task: router
+  template:
+    metadata:
+      labels:
+        app: sgl-ascend
+        task: router
+    spec:
+      hostNetwork: True
+      containers:
+      - image: {{ image }}
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        name: sgl-ascend
+        env:
+        - name: HOSTNAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: INSTALL_SGLANG_FROM_SOURCE
+          value: "{{ install_sglang_from_source }}"
+        - name: KUBECONFIG
+          value: "{{ kube_config }}"
+        - name: NAMESPACE
+          value: "{{ name_space }}"
+        - name: KUBE_CONFIG_MAP
+          value: "{{ kube_config_map }}"
+        - name: HF_ENDPOINT
+          value: "https://hf-mirror.com"
+        - name: METRICS_DATA_FILE
+          value: "{{ metrics_data_file }}"
+        - name: SGLANG_IS_IN_CI
+          value: "{{ sglang_is_in_ci }}"
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+             bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+        resources:
+          requests:
+            cpu: 4
+          limits:
+            cpu: 4
+        volumeMounts:
+        - name: ascend-driver
+          mountPath: /usr/local/Ascend/driver
+        - name: localtime
+          mountPath: /etc/localtime
+        - name: share
+          mountPath: /data/ascend-ci-share-pkking-sglang
+        - name: share
+          mountPath: /root/.cache
+        - name: share
+          mountPath: /root/sglang
+          subPath: {{ sglang_source_relative_path }}
+      volumes:
+      - name: share
+        hostPath:
+          path: /home
+      - name: ascend-driver
+        hostPath:
+          path: /usr/local/Ascend/driver
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      nodeSelector:
+        accelerator: huawei-Ascend910
+        accelerator-type: module-a3-16
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ kube_job_name }}-sglang-router-headless
+  namespace: {{ name_space }}
+spec:
+  clusterIP: None
+  selector:
+    app: sgl-ascend
+    task: router
+  ports:
+  - port: 80
+    name: dummy-port

python/sglang/test/ascend/e2e/k8s_single.yaml.jinja2

@@ -0,0 +1,134 @@
+apiVersion: batch.volcano.sh/v1alpha1
+kind: Job
+metadata:
+  name: {{ kube_job_name }}
+  namespace: {{ name_space }}
+  labels:
+    ring-controller.atlas: ascend-1980
+    fault-scheduling: "force"
+spec:
+  minAvailable: 1
+  schedulerName: volcano
+  policies:
+    - event: PodEvicted
+      action: RestartJob
+  {% if env == "green" %}
+  queue: default
+  {% endif %}
+  tasks:
+  - name: "pod"
+    replicas: 1
+    template:
+      metadata:
+        labels:
+          app: sgl-ascend
+          ring-controller.atlas: ascend-1980
+      spec:
+        containers:
+        - image: {{ image }}
+          {% if env == "green" %}
+          imagePullPolicy: IfNotPresent
+          {% else %}
+          imagePullPolicy: Always
+          {% endif %}
+          securityContext:
+            privileged: true
+          name: sgl-ascend
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: INSTALL_SGLANG_FROM_SOURCE
+            value: "{{ install_sglang_from_source }}"
+          - name: METRICS_DATA_FILE
+            value: "{{ metrics_data_file }}"
+          - name: KUBECONFIG
+            value: "{{ kube_config }}"
+          - name: HF_ENDPOINT
+            value: "https://hf-mirror.com"
+          - name: TROUBLE_SHOTTING
+            value: "{{ trouble_shotting }}"
+          - name: TRANSFORMERS_VERSION_FOR_SGLANG
+            value: "{{ transformers_version }}"
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+               bash /root/sglang/python/sglang/test/ascend/e2e/run_npu_testcase.sh {{ test_case }}
+          resources:
+            requests:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: {{ npu_size }}
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: {{ npu_size }}
+              memory: 128Gi
+              {% endif %}
+              cpu: "46"
+            limits:
+              {% if env in ["ci", "debug"] %}
+              huawei.com/ascend-1980: {{ npu_size }}
+              {% endif %}
+              {% if env == "green" %}
+              huawei.com/Ascend910: {{ npu_size }}
+              memory: 128Gi
+              {% endif %}
+              cpu: "46"
+          volumeMounts:
+          - name: ascend-driver
+            mountPath: /usr/local/Ascend/driver
+          - name: shm-volume
+            mountPath: /dev/shm
+          - name: localtime
+            mountPath: /etc/localtime
+          - name: share
+            mountPath: /data/ascend-ci-share-pkking-sglang
+          - name: share
+            mountPath: /root/.cache
+          - name: share
+            mountPath: /root/sglang
+            subPath: {{ sglang_source_relative_path }}
+        volumes:
+        {% if env in ["ci", "debug"] %}
+        - name: share
+          persistentVolumeClaim:
+            claimName: sglang-guiyang004
+        {% endif %}
+        {% if env == "green" %}
+        - name: share
+          hostPath:
+            path: /home
+        {% endif %}
+        - name: ascend-driver
+          hostPath:
+            path: /usr/local/Ascend/driver
+        - name: shm-volume
+          emptyDir:
+            medium: Memory
+            sizeLimit: "16Gi"
+        - name: localtime
+          hostPath:
+            path: /etc/localtime
+        {% if env == "debug" %}
+        tolerations:
+          - key: "instance"
+            operator: "Equal"
+            value: "npu-class-service"
+            effect: "NoSchedule"
+        {% endif %}
+        nodeSelector:
+          {% if env in ["ci", "debug"] %}
+          accelerator/huawei-npu: ascend-snt9c
+          {% endif %}
+          {% if env == "debug" %}
+          node-status: debug
+          {% endif %}
+          {% if env == "green" %}
+          accelerator: huawei-Ascend910
+          accelerator-type: module-a3-16
+          {% endif %}
+        restartPolicy: OnFailure

python/sglang/test/ascend/e2e/run_evalscope.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+PYTHON_ENV_FOR_EVALSCOPE=test_env_evalscope
+PIP_FOR_EVALSCOPE=${PYTHON_ENV_FOR_EVALSCOPE}/bin/pip
+EVALSCOPE_SOURCE_PATH=/root/.cache/.cache/evalscope
+pip_mirror_source="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
+
+if [ -d "${PYTHON_ENV_FOR_EVALSCOPE}" ]; then
+    echo "Virtual env ${PYTHON_ENV_FOR_EVALSCOPE} already exists, skip installation."
+    exit 0
+fi
+
+echo "===== Install evalscope in virtual env - Begin ====="
+python -m venv ${PYTHON_ENV_FOR_EVALSCOPE}
+
+if [ ! -d "${EVALSCOPE_SOURCE_PATH}" ]; then
+    echo "The evalscope source does not exist: ${EVALSCOPE_SOURCE_PATH}."
+    echo "Install evalscope online."
+    ${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source}
+    ${PIP_FOR_EVALSCOPE} install evalscope -i ${pip_mirror_source}
+else
+    echo "Install evalscope from local source: ${EVALSCOPE_SOURCE_PATH}"
+    ${PIP_FOR_EVALSCOPE} install -U pip -i ${pip_mirror_source}
+    ${PIP_FOR_EVALSCOPE} install -e ${EVALSCOPE_SOURCE_PATH} -i ${pip_mirror_source}
+fi
+echo "===== Install evalscope in virtual env - End ====="

python/sglang/test/ascend/e2e/run_npu_e2e_test.py

@@ -0,0 +1,890 @@
+import argparse
+import json
+import logging
+import os
+import random
+import re
+import string
+import subprocess
+import time
+import uuid
+
+import psutil
+import yaml
+from jinja2 import Template
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
+KUBE_CONFIG = os.environ.get("KUBECONFIG")
+logger.info(f"KUBE_CONFIG: {KUBE_CONFIG}")
+config.load_kube_config(KUBE_CONFIG)
+core_api = client.CoreV1Api()
+custom_api = client.CustomObjectsApi()
+batch_api = client.BatchV1Api()
+rbac_api = client.RbacAuthorizationV1Api()
+
+LOCAL_TIMEOUT = 10800
+
+script_path = os.path.dirname(os.path.abspath(__file__))
+
+KUBE_JOB_SINGLE = "single"
+KUBE_JOB_MULTI_PD_MIX = "multi-pd-mix"
+KUBE_JOB_MULTI_PD_SEPARATION = "multi-pd-separation"
+KUBE_JOB_MULTI_PD_MIX_GREEN = "multi-pd-mix-green"
+KUBE_JOB_MULTI_PD_SEPARATION_GREEN = "multi-pd-separation-green"
+KUBE_YAML_TEMPLATE = {
+    KUBE_JOB_SINGLE: f"{script_path}/k8s_single.yaml.jinja2",
+    KUBE_JOB_MULTI_PD_MIX: f"{script_path}/k8s_multi_pd_mix.yaml.jinja2",
+    KUBE_JOB_MULTI_PD_MIX_GREEN: f"{script_path}/k8s_multi_pd_mix_green.yaml.jinja2",
+    KUBE_JOB_MULTI_PD_SEPARATION: f"{script_path}/k8s_multi_pd_separation.yaml.jinja2",
+    KUBE_JOB_MULTI_PD_SEPARATION_GREEN: f"{script_path}/k8s_multi_pd_separation_green.yaml.jinja2",
+}
+
+
+def get_unique_random_string(length: int = 16, add_random: bool = True) -> str:
+    """Generate a random string."""
+    uuid_str = str(uuid.uuid4()).replace("-", "")
+
+    if add_random:
+        if length < 8:
+            raise ValueError("length can not be smaller than 8")
+        random_length = length - 8
+        char_pool = string.ascii_lowercase + string.digits
+        random_chars = "".join([random.choice(char_pool) for _ in range(random_length)])
+        result = uuid_str[:8] + random_chars
+    else:
+        result = uuid_str[:length]
+
+    return result
+
+
+def create_kube_yaml(kube_yaml_template, output_yaml, pod_context):
+    """Create a k8s config yaml file"""
+    with open(kube_yaml_template, "r") as f:
+        template = Template(f.read())
+    kube_pod_yaml = template.render(pod_context)
+    with open(output_yaml, "w") as f:
+        f.write(kube_pod_yaml)
+    logger.info(f"Pod YAML written to {output_yaml}")
+
+
+def create_pod(yaml_file, namespace):
+    """Create a pod by k8s config yaml file"""
+    with open(yaml_file, "r", encoding="utf-8") as f:
+        yaml_docs = list(yaml.safe_load_all(f))
+
+    for doc in yaml_docs:
+        if not doc:
+            continue
+
+        kind = doc.get("kind")
+        api_version = doc.get("apiVersion")
+
+        try:
+            if kind == "Pod" and api_version == "v1":
+                core_api.create_namespaced_pod(namespace=namespace, body=doc)
+                logger.info(f"Pod {doc['metadata']['name']} created")
+
+            elif kind == "Job" and api_version == "batch/v1":
+                batch_api.create_namespaced_job(namespace=namespace, body=doc)
+                logger.info(f"Job {doc['metadata']['name']} is created")
+
+            elif kind == "Job" and api_version == "batch.volcano.sh/v1alpha1":
+                response = custom_api.create_namespaced_custom_object(
+                    group="batch.volcano.sh",
+                    version="v1alpha1",
+                    namespace=namespace,
+                    plural="jobs",
+                    body=doc,
+                )
+                logger.info(f"Volcano job {doc['metadata']['name']} is created")
+                logger.debug(response)
+
+            elif kind == "ConfigMap" and api_version == "v1":
+                core_api.create_namespaced_config_map(namespace=namespace, body=doc)
+                logger.info(f"ConfigMap {doc['metadata']['name']} is created")
+
+            elif kind == "Role" and api_version == "rbac.authorization.k8s.io/v1":
+                rbac_api.create_namespaced_role(namespace=namespace, body=doc)
+                logger.info(f"Role {doc['metadata']['name']} is created")
+
+            elif (
+                kind == "RoleBinding" and api_version == "rbac.authorization.k8s.io/v1"
+            ):
+                rbac_api.create_namespaced_role_binding(namespace=namespace, body=doc)
+                logger.info(f"RoleBinding {doc['metadata']['name']} is created")
+
+            elif kind == "Deployment" and api_version == "apps/v1":
+                apps_api = client.AppsV1Api()
+                apps_api.create_namespaced_deployment(namespace=namespace, body=doc)
+                logger.info(f"Deployment {doc['metadata']['name']} is created")
+
+            elif kind == "StatefulSet" and api_version == "apps/v1":
+                apps_api = client.AppsV1Api()
+                apps_api.create_namespaced_stateful_set(namespace=namespace, body=doc)
+                logger.info(f"StatefulSet {doc['metadata']['name']} is created")
+
+            elif kind == "Service" and api_version == "v1":
+                core_api.create_namespaced_service(namespace=namespace, body=doc)
+                logger.info(f"Service {doc['metadata']['name']} is created")
+
+            else:
+                raise f"Unrecognized kind: {kind}/{api_version}"
+        except ApiException as e:
+            print(f"create resource {kind} error: {e}")
+            raise
+
+
+def delete_pod(yaml_file, namespace):
+    """Delete k8s pod by config yaml file"""
+    with open(yaml_file, "r", encoding="utf-8") as f:
+        yaml_docs = list(yaml.safe_load_all(f))
+    for doc in yaml_docs:
+        if not doc:
+            continue
+
+        kind = doc.get("kind")
+        api_version = doc.get("apiVersion")
+        try:
+            if kind == "Job" and api_version == "batch.volcano.sh/v1alpha1":
+                job_name = doc["metadata"]["name"]
+                response = custom_api.delete_namespaced_custom_object(
+                    group="batch.volcano.sh",
+                    version="v1alpha1",
+                    namespace=namespace,
+                    plural="jobs",
+                    name=job_name,
+                    body=client.V1DeleteOptions(
+                        grace_period_seconds=0, propagation_policy="Foreground"
+                    ),
+                )
+                logger.info(f"Deleted job {job_name}")
+                logger.info(f"Response status: {response.get('status')}")
+            elif kind == "ConfigMap" and api_version == "v1":
+                config_map_name = doc["metadata"]["name"]
+                core_api.delete_namespaced_config_map(
+                    name=config_map_name, namespace=namespace
+                )
+                print(f"ConfigMap {config_map_name} is deleted.")
+            elif kind == "Deployment" and api_version == "apps/v1":
+                deployment_name = doc["metadata"]["name"]
+                apps_api = client.AppsV1Api()
+                apps_api.delete_namespaced_deployment(
+                    name=deployment_name,
+                    namespace=namespace,
+                    body=client.V1DeleteOptions(
+                        grace_period_seconds=0, propagation_policy="Foreground"
+                    ),
+                )
+                logger.info(f"Deployment {deployment_name} is deleted.")
+
+            elif kind == "StatefulSet" and api_version == "apps/v1":
+                statefulset_name = doc["metadata"]["name"]
+                apps_api = client.AppsV1Api()
+                apps_api.delete_namespaced_stateful_set(
+                    name=statefulset_name,
+                    namespace=namespace,
+                    body=client.V1DeleteOptions(
+                        grace_period_seconds=0, propagation_policy="Foreground"
+                    ),
+                )
+                logger.info(f"StatefulSet {statefulset_name} is deleted.")
+
+            elif kind == "Service" and api_version == "v1":
+                service_name = doc["metadata"]["name"]
+                core_api.delete_namespaced_service(
+                    name=service_name,
+                    namespace=namespace,
+                    body=client.V1DeleteOptions(
+                        grace_period_seconds=0, propagation_policy="Foreground"
+                    ),
+                )
+                logger.info(f"Service {service_name} is deleted.")
+
+            else:
+                raise f"Unrecognized kind: {kind}/{api_version}"
+        except ApiException as e:
+            raise f"delete resource {kind} error: {e}"
+
+
+def check_parent_process():
+    """Check parent process is alive or not."""
+    try:
+        parent_pid = os.getppid()
+        psutil.Process(parent_pid)
+        return True
+    except psutil.NoSuchProcess:
+        return False
+
+
+def check_pods_ready(namespace, pod_name_key_str, timeout=300):
+    """Waiting for all k8s pods are ready"""
+    logger.info("Waiting all pods to running...")
+    start_time = time.time()
+
+    while time.time() - start_time < timeout:
+        if not check_parent_process():
+            raise Exception("Parent process exited.")
+
+        pods = core_api.list_namespaced_pod(namespace=namespace)
+
+        if len(pods.items) == 0:
+            time.sleep(5)
+            continue
+
+        all_running = True
+        sglang_pods_found = False
+        for pod in pods.items:
+            pod_name = pod.metadata.name
+            if pod_name_key_str not in pod_name:
+                continue
+
+            sglang_pods_found = True
+            status = pod.status
+            phase = status.phase
+            logger.info(f"Pod: {pod_name}, status: {phase}")
+            if phase != "Running":
+                all_running = False
+                break
+
+            containers_ready = True
+            for condition in status.conditions:
+                if condition.type == "Ready" and condition.status != "True":
+                    containers_ready = False
+                    break
+
+            if not containers_ready:
+                all_running = False
+                break
+
+        if not sglang_pods_found:
+            logger.info("No sglang pod, waiting...")
+            time.sleep(5)
+            continue
+        if all_running:
+            logger.info("All sglang Pod is Running !")
+            return True
+
+        time.sleep(5)
+
+    logger.info(f"timeout in {timeout}s")
+    return False
+
+
+def create_or_update_configmap(cm_name: str, data: dict, namespace: str):
+    """Create a k8s configmap or update it if already exists"""
+    cm_metadata = client.V1ObjectMeta(name=cm_name, namespace=namespace)
+    configmap = client.V1ConfigMap(
+        api_version="v1", kind="ConfigMap", metadata=cm_metadata, data=data
+    )
+
+    try:
+        response = core_api.create_namespaced_config_map(
+            namespace=namespace, body=configmap
+        )
+        logger.info(f"ConfigMap '{cm_name}' create successfully!")
+        logger.info(f"data: {list(data.keys())}")
+        return response
+    except ApiException as e:
+        if e.status == 409:
+            logger.info(f"ConfigMap {cm_name} already exists. Updating...")
+            response = core_api.replace_namespaced_config_map(
+                namespace=namespace, name=cm_name, body=configmap
+            )
+            logger.info(f"ConfigMap {cm_name} updated successfully.")
+            return response
+        else:
+            error_msg = f"ConfigMap create failed: {e.reason}"
+            if e.body:
+                error_msg += f" | details: {e.body}"
+            logger.info(error_msg)
+            raise
+
+
+def prepare_cm_data(namespace, pod_string):
+    """Prepare a configmap data: {pod_name: pod_ip} by the running pod's information."""
+    pods = core_api.list_namespaced_pod(namespace=namespace)
+    data = {}
+    for pod in pods.items:
+        pod_name = pod.metadata.name
+        if pod_string in pod_name:
+            pod_ip = pod.status.pod_ip
+            data[pod_name] = pod_ip
+    return data
+
+
+def monitor_pod_logs(
+    kube_job_type, kube_job_prefix_name, namespace, timeout=LOCAL_TIMEOUT
+):
+    """Monitor the logs of the specified pod until the special pattern is matched or reaches its timeout."""
+    monitor_pod_name = {
+        KUBE_JOB_SINGLE: f"{kube_job_prefix_name}-pod-0",
+        KUBE_JOB_MULTI_PD_MIX: f"{kube_job_prefix_name}-sglang-node-0",
+        KUBE_JOB_MULTI_PD_SEPARATION: f"{kube_job_prefix_name}-sglang-router-0",
+    }
+    pod_name = monitor_pod_name.get(kube_job_type)
+
+    # Build kubectl command
+    cmd = ["kubectl", "logs", "-f", "-n", namespace, pod_name]
+
+    # Define multiline pattern to match
+    pattern_lines = [
+        r"^-{70,}$",
+        r"^Ran \d+ tests? in [\d.]+s$",
+        r"^$",
+        r"^(OK|FAILED \(errors=\d+\))$",
+    ]
+    patterns = [re.compile(line_pattern) for line_pattern in pattern_lines]
+    pattern_ok = re.compile(r"^OK$")
+
+    process = None
+    try:
+        # Start kubectl logs process
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+            bufsize=1,
+        )
+
+        logger.info(f"Starting to monitor logs for Pod: {pod_name}")
+        match_state = 0
+        is_success = False
+
+        # Use two threads: one for reading logs, one for checking pod status
+        import threading
+
+        # Shared variables
+        match_event = threading.Event()
+        pod_error_event = threading.Event()
+
+        def read_logs():
+            """Thread function to read logs continuously"""
+            nonlocal is_success, match_state
+
+            while process.poll() is None and not match_event.is_set():
+                line = process.stdout.readline()
+                if line:
+                    line = line.rstrip("\n")
+                    print(line)
+                    # Check if current line matches expected pattern
+                    if match_state < len(patterns) and patterns[match_state].match(
+                        line
+                    ):
+                        match_state += 1
+                        if match_state == len(patterns):
+                            if pattern_ok.match(line):
+                                is_success = True
+                            logger.info("Detected complete test completion pattern!")
+                            match_event.set()
+                    else:
+                        match_state = 0
+                        if patterns[0].match(line):
+                            match_state = 1
+
+            # Read remaining output after process exits
+            if not match_event.is_set():
+                remaining_output, stderr_output = process.communicate()
+                if remaining_output:
+                    print(remaining_output)
+                if stderr_output:
+                    logger.error(f"kubectl command error: {stderr_output}")
+                    pod_error_event.set()
+
+        def check_pods_running(namespace, pod_name_key_str):
+            """check pods are running"""
+            pods = core_api.list_namespaced_pod(namespace=namespace)
+            if len(pods.items) == 0:
+                logger.warning(f"No pods found in the namespace {namespace}")
+                return False
+
+            for pod in pods.items:
+                pod_name = pod.metadata.name
+                if pod_name_key_str not in pod_name:
+                    continue
+                status = pod.status
+                phase = status.phase
+                if phase != "Running":
+                    logger.error(f"Pod {pod_name} is not running, status: {phase}")
+                    return False
+
+            return True
+
+        def check_pod_status():
+            """Thread function to check pod status periodically"""
+            start_time = time.time()
+            while not match_event.is_set() and not pod_error_event.is_set():
+                if time.time() - start_time > timeout:
+                    pod_error_event.set()
+                    break
+
+                if not check_parent_process():
+                    logger.error(f"Parent process exited. Exiting...")
+                    pod_error_event.set()
+                    break
+
+                if not check_pods_running(
+                    namespace=namespace, pod_name_key_str=kube_job_prefix_name
+                ):
+                    logger.error(
+                        f"Some pods are not running properly. Please check the sglang logs on these pods. Exiting..."
+                    )
+                    pod_error_event.set()
+                    break
+
+                # Sleep for a short time before next check
+                time.sleep(0.5)
+
+        # Start threads
+        log_thread = threading.Thread(target=read_logs)
+        status_thread = threading.Thread(target=check_pod_status)
+
+        log_thread.daemon = True
+        status_thread.daemon = True
+
+        log_thread.start()
+        status_thread.start()
+
+        # Wait for either match event or error event
+        start_time = time.time()
+        while not match_event.is_set() and not pod_error_event.is_set():
+            if time.time() - start_time > timeout:
+                raise Exception(
+                    f"Timeout exceeded, the thread is {timeout} seconds long."
+                )
+            time.sleep(0.1)
+
+        # Check if pattern was successfully matched
+        if not match_event.is_set():
+            if process.poll() is not None:
+                remaining_output, stderr_output = process.communicate()
+                if remaining_output:
+                    logger.info(remaining_output)
+                if stderr_output:
+                    raise Exception(f"kubectl command error: {stderr_output}")
+                else:
+                    raise Exception(
+                        "Pod logs ended but target pattern was not detected"
+                    )
+            else:
+                raise Exception("Monitoring ended but target pattern was not detected")
+        elif not is_success:
+            raise Exception("The test result was FAILED!")
+        else:
+            logger.info("The test result was OK!")
+    finally:
+        if process and process.poll() is None:
+            process.terminate()
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                process.kill()
+
+
+def generate_metrics_json(metrics_data_file, test_case, status):
+    log_file = os.path.join(metrics_data_file, "test_output.log")
+
+    metrics = {}
+    baselines = {}
+
+    if os.path.exists(log_file):
+        with open(log_file, "r") as f:
+            for line in f:
+                m = re.match(r"\[METRIC\] (\S+)=(\S+)", line.strip())
+                if m:
+                    key = m.group(1)
+                    value = m.group(2)
+                    try:
+                        value = float(value)
+                    except ValueError:
+                        pass
+                    if key.endswith("_baseline"):
+                        baselines[key[:-9]] = value
+                    else:
+                        metrics[key] = value
+    else:
+        logger.warning(f"Metrics log file not found: {log_file}")
+
+    tc_name = test_case.rsplit("/", 1)[-1].rsplit(".", 1)[0]
+
+    test_type = "unknown"
+    parts = metrics_data_file.split("/")
+    for i, part in enumerate(parts):
+        if part == "output" and i + 1 < len(parts):
+            test_type = parts[i + 1]
+            break
+
+    output = {
+        "test_case": tc_name,
+        "test_type": test_type,
+        "status": status,
+        "metrics": metrics,
+        "baselines": baselines,
+    }
+
+    output_path = os.path.join(metrics_data_file, "metrics.json")
+    with open(output_path, "w") as f:
+        json.dump(output, f, indent=2)
+    logger.info(f"Metrics JSON written to {output_path}")
+
+    with open("/tmp/metrics.json", "w") as f:
+        json.dump(output, f, indent=2)
+    logger.info("Metrics JSON written to /tmp/metrics.json")
+
+
+def run_npu_e2e_test_case(
+    docker_image_url: str,
+    kube_name_space: str,
+    kube_job_type: str,
+    kube_job_name_prefix: str,
+    resource_info: dict,
+    sglang_source_relative_path: str,
+    metrics_data_file: str,
+    test_case: str,
+    sglang_is_in_ci=False,
+    install_sglang_from_source=False,
+    env="debug",
+    trouble_shotting=False,
+    transformers_version="",
+):
+    """The method for running a npu e2e test case.
+    Args:
+        docker_image_url (str): the url of docker image for creating k8s pods.
+        kube_name_space (str): the namespace of the k8s.
+        kube_job_name_prefix (str): the prefix of the k8s job name which will be set as the prefix of the pod name.
+        resource_info (dict): the number of k8s nodes used by the testcase.
+            for pd-separation as: {"prefill_size": 1, "decode_size": 1, "router_size": 1};
+            for pd-mix as: {"node_size": 2; single: {"npu_size": 4}
+        sglang_source_relative_path (str): the relative path of the sglang source on shared-disk.
+        metrics_data_file (str): the output path of the metrics data file, only for performance testing.
+        test_case (str): the test case relative path in sglang source root path. like test/registered/...
+        sglang_is_in_ci (bool): whether running in CI environment.
+        install_sglang_from_source (bool): whether installing sglang from source or use docker image directly.
+        env (str): the environment to run the test on.  Choose one in ["debug", "ci"]
+    """
+    random_str = get_unique_random_string(16, True)
+
+    kube_config_map = f"sglang-configmap-{random_str}"
+    final_kube_job_name = f"{kube_job_name_prefix}-{random_str}"
+
+    kube_yaml_file_dict = {
+        KUBE_JOB_SINGLE: f"k8s_single_{random_str}.yaml",
+        KUBE_JOB_MULTI_PD_MIX: f"k8s_multi_pd_mix_{random_str}.yaml",
+        KUBE_JOB_MULTI_PD_SEPARATION: f"k8s_multi_pd_separation_{random_str}.yaml",
+    }
+    kube_yaml_file = kube_yaml_file_dict.get(kube_job_type)
+
+    try:
+        logger.info(
+            f"Apply k8s yaml... KUBE_NAME_SPACE:{kube_name_space}, KUBE_CONFIG_MAP:{kube_config_map}, "
+            f"KUBE_JOB_TYPE:{kube_job_type}, KUBE_YAML_FILE:{kube_yaml_file}"
+        )
+
+        if kube_job_type == KUBE_JOB_SINGLE:
+            k8s_context = {
+                "image": docker_image_url,
+                "name_space": kube_name_space,
+                "kube_job_name": final_kube_job_name,
+                "kube_config": KUBE_CONFIG,
+                "npu_size": resource_info["npu_size"],
+                "sglang_source_relative_path": sglang_source_relative_path,
+                "metrics_data_file": metrics_data_file,
+                "test_case": test_case,
+                "sglang_is_in_ci": sglang_is_in_ci,
+                "install_sglang_from_source": install_sglang_from_source,
+                "env": env,
+                "trouble_shotting": trouble_shotting,
+                "transformers_version": transformers_version,
+            }
+            create_kube_yaml(
+                kube_yaml_template=KUBE_YAML_TEMPLATE.get(kube_job_type),
+                output_yaml=kube_yaml_file,
+                pod_context=k8s_context,
+            )
+        elif kube_job_type == KUBE_JOB_MULTI_PD_MIX:
+            k8s_context = {
+                "image": docker_image_url,
+                "name_space": kube_name_space,
+                "kube_job_name": final_kube_job_name,
+                "kube_config": KUBE_CONFIG,
+                "kube_config_map": kube_config_map,
+                "node_size": resource_info["node_size"],
+                "sglang_source_relative_path": sglang_source_relative_path,
+                "metrics_data_file": metrics_data_file,
+                "test_case": test_case,
+                "sglang_is_in_ci": sglang_is_in_ci,
+                "install_sglang_from_source": install_sglang_from_source,
+                "env": env,
+                "trouble_shotting": trouble_shotting,
+                "transformers_version": transformers_version,
+            }
+            template_key = (
+                KUBE_JOB_MULTI_PD_MIX_GREEN if env == "green" else kube_job_type
+            )
+            create_kube_yaml(
+                kube_yaml_template=KUBE_YAML_TEMPLATE.get(template_key),
+                output_yaml=kube_yaml_file,
+                pod_context=k8s_context,
+            )
+        elif kube_job_type == KUBE_JOB_MULTI_PD_SEPARATION:
+            k8s_context = {
+                "image": docker_image_url,
+                "name_space": kube_name_space,
+                "kube_job_name": final_kube_job_name,
+                "kube_config": KUBE_CONFIG,
+                "kube_config_map": kube_config_map,
+                "prefill_size": resource_info["prefill_size"],
+                "decode_size": resource_info["decode_size"],
+                "router_size": resource_info["router_size"],
+                "sglang_source_relative_path": sglang_source_relative_path,
+                "metrics_data_file": metrics_data_file,
+                "test_case": test_case,
+                "sglang_is_in_ci": sglang_is_in_ci,
+                "install_sglang_from_source": install_sglang_from_source,
+                "env": env,
+                "trouble_shotting": trouble_shotting,
+                "transformers_version": transformers_version,
+            }
+            template_key = (
+                KUBE_JOB_MULTI_PD_SEPARATION_GREEN if env == "green" else kube_job_type
+            )
+            create_kube_yaml(
+                kube_yaml_template=KUBE_YAML_TEMPLATE.get(template_key),
+                output_yaml=kube_yaml_file,
+                pod_context=k8s_context,
+            )
+        else:
+            raise Exception(f"Unknown k8s job type: {kube_job_type}")
+
+        create_pod(yaml_file=kube_yaml_file, namespace=kube_name_space)
+
+        if check_pods_ready(
+            kube_name_space, final_kube_job_name, timeout=LOCAL_TIMEOUT
+        ):
+            if kube_job_type != "single":
+                matching_pod_string = final_kube_job_name
+                cm_data = prepare_cm_data(kube_name_space, matching_pod_string)
+                if not cm_data:
+                    logger.info(
+                        f"No sglang pod found while matching {matching_pod_string}"
+                    )
+
+                response = create_or_update_configmap(
+                    cm_name=kube_config_map, data=cm_data, namespace=kube_name_space
+                )
+                logger.info(response)
+        else:
+            logger.info("Pod not ready, maybe not enough resource")
+
+        monitor_success = False
+        try:
+            monitor_pod_logs(
+                kube_job_type, final_kube_job_name, kube_name_space, LOCAL_TIMEOUT
+            )
+            monitor_success = True
+        except Exception:
+            logger.error(f"Test case failed: {test_case}", exc_info=True)
+            raise
+        finally:
+            if metrics_data_file:
+                status = "pass" if monitor_success else "fail"
+                try:
+                    generate_metrics_json(metrics_data_file, test_case, status)
+                except Exception as e:
+                    logger.error(f"Failed to generate metrics JSON: {e}", exc_info=True)
+    finally:
+        if os.path.exists(kube_yaml_file):
+            # Don't delete pod when trouble_shotting is enabled
+            if not trouble_shotting:
+                delete_pod(yaml_file=kube_yaml_file, namespace=kube_name_space)
+                os.remove(kube_yaml_file)
+            else:
+                logger.info(
+                    f"Trouble shooting mode enabled, keeping pod {final_kube_job_name} alive"
+                )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Apply k8s yaml", formatter_class=argparse.RawTextHelpFormatter
+    )
+
+    parser.add_argument(
+        "--image",
+        type=str,
+        required=True,
+        help="Docker image to use",
+    )
+
+    parser.add_argument(
+        "--prefill-size",
+        type=int,
+        required=False,
+        default=1,
+        help="Number of prefill nodes",
+    )
+
+    parser.add_argument(
+        "--decode-size",
+        type=int,
+        required=False,
+        default=1,
+        help="Number of decode nodes",
+    )
+
+    parser.add_argument(
+        "--router-size",
+        type=int,
+        required=False,
+        default=1,
+        help="Number of router nodes",
+    )
+
+    parser.add_argument(
+        "--node-size",
+        type=int,
+        required=False,
+        default=2,
+        help="Number of nodes for multi-node-pd-mix scenario",
+    )
+
+    parser.add_argument(
+        "--npu-size",
+        type=int,
+        required=False,
+        default=0,
+        help="Number of npu for single-node scenario",
+    )
+
+    parser.add_argument(
+        "--sglang-source-relative-path",
+        type=str,
+        required=True,
+        help="Sglang source code relative path on shared-disk(NFS_ROOT_PATH: /data/ascend-ci-share-pkking-sglang/)",
+    )
+
+    parser.add_argument(
+        "--metrics-data-file",
+        type=str,
+        required=False,
+        default="",
+        help="Metrics data file",
+    )
+
+    parser.add_argument(
+        "--test-case",
+        type=str,
+        required=True,
+        help="Test case path",
+    )
+
+    parser.add_argument(
+        "--sglang-is-in-ci",
+        action="store_true",
+        help="Used to set env var SGLANG_IS_IN_CI in pod",
+    )
+
+    parser.add_argument(
+        "--install-sglang-from-source",
+        action="store_true",
+        help="Used to set env var INSTALL_SGLANG_FROM_SOURCE in pod",
+    )
+
+    parser.add_argument(
+        "--kube-name-space",
+        type=str,
+        required=True,
+        help="K8s name space",
+    )
+
+    parser.add_argument(
+        "--kube-job-type",
+        type=str,
+        choices=[KUBE_JOB_SINGLE, KUBE_JOB_MULTI_PD_MIX, KUBE_JOB_MULTI_PD_SEPARATION],
+        required=True,
+        help=f"K8s job type [{KUBE_JOB_SINGLE}, {KUBE_JOB_MULTI_PD_MIX}, {KUBE_JOB_MULTI_PD_SEPARATION}]",
+    )
+
+    parser.add_argument(
+        "--kube-job-name-prefix",
+        type=str,
+        required=True,
+        help="K8s job name prefix",
+    )
+
+    parser.add_argument(
+        "--env",
+        type=str,
+        choices=["debug", "ci", "green"],
+        required=True,
+        help="Environment type",
+    )
+
+    parser.add_argument(
+        "--trouble-shotting",
+        action="store_true",
+        help="Used for troubleshotting issues, such as retaining pods",
+    )
+
+    parser.add_argument(
+        "--transformers-version",
+        type=str,
+        required=False,
+        default="",
+        help="The transformers version number for running sglang. Use default version in image if keep empty.",
+    )
+
+    args = parser.parse_args()
+
+    docker_image_url = args.image
+    npu_size = int(args.npu_size)
+    node_size = int(args.node_size)
+    prefill_size = int(args.prefill_size)
+    decode_size = int(args.decode_size)
+    router_size = int(args.router_size)
+    sglang_source_relative_path = args.sglang_source_relative_path
+    metrics_data_file = args.metrics_data_file
+    test_case = args.test_case
+    sglang_is_in_ci = args.sglang_is_in_ci
+    install_sglang_from_source = args.install_sglang_from_source
+    env = args.env
+    trouble_shotting = args.trouble_shotting
+    transformers_version = args.transformers_version
+
+    kube_name_space = args.kube_name_space
+    kube_job_type = args.kube_job_type
+    kube_job_name_prefix = args.kube_job_name_prefix
+
+    resource_info_dict = {
+        KUBE_JOB_SINGLE: {"npu_size": npu_size},
+        KUBE_JOB_MULTI_PD_MIX: {"node_size": node_size},
+        KUBE_JOB_MULTI_PD_SEPARATION: {
+            "prefill_size": prefill_size,
+            "decode_size": decode_size,
+            "router_size": router_size,
+        },
+    }
+
+    run_npu_e2e_test_case(
+        docker_image_url=docker_image_url,
+        kube_name_space=kube_name_space,
+        kube_job_type=kube_job_type,
+        kube_job_name_prefix=kube_job_name_prefix,
+        resource_info=resource_info_dict.get(kube_job_type),
+        sglang_source_relative_path=sglang_source_relative_path,
+        metrics_data_file=metrics_data_file,
+        test_case=test_case,
+        sglang_is_in_ci=sglang_is_in_ci,
+        install_sglang_from_source=install_sglang_from_source,
+        env=env,
+        trouble_shotting=trouble_shotting,
+        transformers_version=transformers_version,
+    )

python/sglang/test/ascend/e2e/run_npu_testcase.sh

@@ -0,0 +1,151 @@
+test_case=$1
+
+sglang_source_path=/root/sglang
+if [ ! -f "${sglang_source_path}/${test_case}" ];then
+  echo "The test case file is not exist: $test_case"
+  exit 0
+fi
+
+echo "NPU info:"
+npu-smi info
+
+echo "===== Install kubernetes - Begin ====="
+KUBERNETES_PKG_PATH_SOURCE=/root/.cache/.cache/kubernetes
+if [ ! -d "${KUBERNETES_PKG_PATH_SOURCE}" ]; then
+  echo "Install kubernetes online."
+  pip install kubernetes -i -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+else
+  echo "Install kubernetes locally."
+  cp -r ${KUBERNETES_PKG_PATH_SOURCE} /tmp/
+  pip install --no-index --find-links=/tmp/kubernetes/ kubernetes
+fi
+echo "===== Install kubernetes - End ====="
+
+PYTHON_FOR_SGLANG="python"
+PIP_FOR_SGLANG="pip"
+if [ -n "${TRANSFORMERS_VERSION_FOR_SGLANG}" ];then
+  echo "===== Install transformers for sglang - Begin ====="
+  TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
+  if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
+    echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
+    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} online."
+    pip install transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}" -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+  else
+    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_SGLANG} locally."
+    TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_SGLANG}
+    mkdir -p "${TRANSFORMERS_PKG_PATH_TARGET}"
+    cp "${TRANSFORMERS_PKG_PATH_SOURCE}/*" "${TRANSFORMERS_PKG_PATH_TARGET}/"
+    pip install --no-index --find-links="${TRANSFORMERS_PKG_PATH_TARGET}" transformers=="${TRANSFORMERS_VERSION_FOR_SGLANG}"
+  fi
+  echo "===== Install transformers for sglang in virtual env - End ====="
+fi
+
+if [ -n "${TRANSFORMERS_VERSION_FOR_TEST_TOOL}" ]; then
+  # Example: TRANSFORMERS_VERSION_FOR_TEST_TOOL=4.57.6
+  echo "===== Install transformers in virtual env for test tools - Begin ====="
+  PYTHON_ENV_FOR_TEST_TOOL=python_venv_for_test_tool
+  PIP_FOR_TEST_TOOL=${PYTHON_ENV_FOR_TEST_TOOL}/bin/pip
+  python -m venv ${PYTHON_ENV_FOR_TEST_TOOL} --system-site-packages
+  TRANSFORMERS_PKG_PATH_SOURCE=/root/.cache/.cache/transformers/${TRANSFORMERS_VERSION_FOR_TEST_TOOL}
+  if [ ! -d "${TRANSFORMERS_PKG_PATH_SOURCE}" ]; then
+    echo "The dependent transformers package does not exist: ${TRANSFORMERS_PKG_PATH_SOURCE}."
+    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_TEST_TOOL} online."
+    ${PIP_FOR_TEST_TOOL} install transformers==${TRANSFORMERS_VERSION_FOR_TEST_TOOL} -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
+  else
+    echo "Install transformers ${TRANSFORMERS_VERSION_FOR_TEST_TOOL} locally."
+    TRANSFORMERS_PKG_PATH_TARGET=/tmp/transformers/${TRANSFORMERS_VERSION_FOR_TEST_TOOL}
+    mkdir -p ${TRANSFORMERS_PKG_PATH_TARGET}
+    cp ${TRANSFORMERS_PKG_PATH_SOURCE}/* ${TRANSFORMERS_PKG_PATH_TARGET}/
+    ${PIP_FOR_TEST_TOOL} install --no-index --find-links=${TRANSFORMERS_PKG_PATH_TARGET} transformers==${TRANSFORMERS_VERSION_FOR_TEST_TOOL}
+  fi
+  echo "===== Install transformers in virtual env for test tools - End ====="
+  echo "Transformers version for test tools: $(${PIP_FOR_TEST_TOOL} show transformers | grep Version | cut -d: -f2)"
+fi
+
+echo "Transformers version for sglang: $(${PIP_FOR_SGLANG} show transformers | grep Version | cut -d: -f2)"
+
+# copy or download required file
+cp /root/.cache/huggingface/hub/datasets--anon8231489123--ShareGPT_Vicuna_unfiltered/snapshots/192ab2185289094fc556ec8ce5ce1e8e587154ca/ShareGPT_V3_unfiltered_cleaned_split.json /tmp
+#curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl
+cp /root/.cache/modelscope/hub/datasets/grade_school_math/test.jsonl /tmp
+
+echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+sysctl -w vm.swappiness=0
+sysctl -w kernel.numa_balancing=0
+sysctl -w kernel.sched_migration_cost_ns=50000
+
+export SGLANG_TEST_MAX_RETRY=0
+export SGLANG_SET_CPU_AFFINITY=1
+export HCCL_HOST_SOCKET_PORT_RANGE="auto"
+export HCCL_NPU_SOCKET_PORT_RANGE="auto"
+
+visibe_devices=$ASCEND_VISIBLE_DEVICES
+echo "ASCEND_VISIBLE_DEVICES=$ASCEND_VISIBLE_DEVICES"
+if [ "${visibe_devices}" != "" ];then
+    ASCEND_RT_VISIBLE_DEVICES=$(echo "$ASCEND_VISIBLE_DEVICES" | tr ',' '\n' | sort -n | tr '\n' ',')
+    export ASCEND_RT_VISIBLE_DEVICES=${ASCEND_RT_VISIBLE_DEVICES%,}
+    echo "ASCEND_RT_VISIBLE_DEVICES=$ASCEND_RT_VISIBLE_DEVICES"
+    export ASCEND_VISIBLE_DEVICES=""
+fi
+
+unset https_proxy
+unset http_proxy
+unset HTTPS_PROXY
+unset HTTP_PROXY
+unset ASCEND_LAUNCH_BLOCKING
+
+# use sglang from source or from image
+if [ "${INSTALL_SGLANG_FROM_SOURCE}" = "true" ] || [ "${INSTALL_SGLANG_FROM_SOURCE}" = "True" ];then
+    echo "Use sglang from source: ${sglang_source_path}"
+    export PYTHONPATH=${sglang_source_path}/python:$PYTHONPATH
+else
+    echo "Use sglang from docker image"
+    sglang_pkg_path=/sgl-workspace/sglang/python
+    ascend_test_util_path=${sglang_pkg_path}/sglang/test/ascend
+    mkdir -p "${ascend_test_util_path}"
+    mv "${ascend_test_util_path}" "${ascend_test_util_path}_bak"
+    cp -r ${sglang_source_path}/python/sglang/test/ascend "${ascend_test_util_path}"
+fi
+
+# set environment of cann
+. /usr/local/Ascend/cann/set_env.sh
+. /usr/local/Ascend/nnal/atb/set_env.sh
+
+echo "Running test case ${test_case}"
+tc_name=${test_case##*/}
+tc_name=${tc_name%.*}
+current_date=$(date +%Y%m%d)
+log_path="/root/sglang/debug/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
+    log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}"
+fi
+rm -rf "${log_path}"
+mkdir -p "${log_path}"
+echo "Log path: ${log_path}"
+
+if [ "${TROUBLE_SHOTTING}" = "true" ] || [ "${TROUBLE_SHOTTING}" = "True" ];then
+    echo "TROUBLE_SHOTTING=true, the pod will keep alive for four hour."
+    ( ${PYTHON_FOR_SGLANG} -u "${sglang_source_path}/${test_case}" 2>&1 || true ) | tee -a "${log_path}/${tc_name}.log"
+    sleep 14400
+else
+    ${PYTHON_FOR_SGLANG} -u "${sglang_source_path}/${test_case}" 2>&1 | tee -a "${log_path}/${tc_name}.log"
+fi
+echo "Finished test case ${test_case}"
+
+if [ -n "${METRICS_DATA_FILE}" ]; then
+    mkdir -p "${METRICS_DATA_FILE}"
+    cp "${log_path}/${tc_name}.log" "${METRICS_DATA_FILE}/test_output.log"
+    echo "Metrics log saved to ${METRICS_DATA_FILE}/test_output.log"
+fi
+
+source_plog_path="/root/ascend/log/debug/plog"
+if [ -d "$source_plog_path" ];then
+    echo "Plog files found. Begin to backup them."
+    target_plog_path="/root/sglang/debug/logs/plog/${tc_name}/${HOSTNAME}"
+    if [ "${SGLANG_IS_IN_CI}" = "true" ] || [ "${SGLANG_IS_IN_CI}" = "True" ];then
+        target_plog_path="/root/.cache/tests/logs/plog/${tc_name}/${HOSTNAME}"
+    fi
+    rm -rf "${target_plog_path}"
+    mkdir -p "${target_plog_path}"
+    cp ${source_plog_path}/* "${target_plog_path}"
+fi

python/sglang/test/ascend/e2e/test_npu_accuracy_utils.py

@@ -0,0 +1,637 @@
+import json
+import logging
+import os
+import re
+import subprocess
+import threading
+import time
+from urllib.parse import urlparse
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ascend.e2e.test_npu_multi_node_utils import (
+    SERVICE_PORT,
+    check_role,
+    launch_pd_mix_node,
+    launch_pd_separation_node,
+    launch_router,
+    wait_server_ready,
+)
+from sglang.test.test_utils import (
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    dump_metric,
+    popen_launch_server,
+)
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
+EVALSCOPE = "evalscope"
+BENCHMARK_TOOL_DEFAULT = EVALSCOPE
+
+PYTHON_FOR_TEST_TOOL = "test_env_transformers_tool/bin/python"
+if not os.path.exists(PYTHON_FOR_TEST_TOOL) or not os.access(
+    PYTHON_FOR_TEST_TOOL, os.X_OK
+):
+    PYTHON_FOR_TEST_TOOL = "python3"
+logger.info(f"PYTHON_FOR_TEST_TOOL: {PYTHON_FOR_TEST_TOOL}")
+
+DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3600
+MAX_SERVER_KEEP_ALIVE_TIME = 3600
+
+ACCURACY_TOLERANCE = 0.99
+
+# Dataset total question counts and allowed fluctuation (in questions)
+DATASET_QUESTION_COUNTS = {
+    "aime25": 30,
+    "aime26": 30,
+    "gpqa_diamond": 198,
+}
+
+DATASET_FLUCTUATION = {
+    "aime25": 2,
+    "aime26": 2,
+    "gpqa_diamond": 5,
+}
+
+MAX_RETRY_COUNT = 3
+
+SERVER_INITIALIZATION_DELAY = 120
+
+if os.environ.get("ASCEND_RT_VISIBLE_DEVICES"):
+    DEFAULT_SERVER_PORT_FOR_TEST = (
+        20000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100
+    )
+else:
+    DEFAULT_SERVER_PORT_FOR_TEST = (
+        20000 + int(os.environ.get("ASCEND_VISIBLE_DEVICES", "0")[0]) * 100
+    )
+DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_SERVER_PORT_FOR_TEST + 66}"
+
+
+def get_accuracy_threshold(datasets, baseline_accuracy):
+    """Calculate accuracy threshold based on dataset fluctuation tolerance.
+
+    For datasets with defined fluctuation (aime*, gpqa_diamond), use absolute
+    question count tolerance. For others (e.g. mmmu), use percentage tolerance.
+    """
+    dataset = datasets[0] if datasets else None
+    if dataset in DATASET_FLUCTUATION and dataset in DATASET_QUESTION_COUNTS:
+        fluctuation = DATASET_FLUCTUATION[dataset] / DATASET_QUESTION_COUNTS[dataset]
+        return baseline_accuracy - fluctuation
+    return baseline_accuracy * ACCURACY_TOLERANCE
+
+
+def get_max_retries(datasets):
+    """Return max retry count for accuracy tests.
+
+    gpqa and aime datasets support up to MAX_RETRY_COUNT retries.
+    mmmu and others use 1 attempt (no retry).
+    """
+    dataset = datasets[0] if datasets else None
+    if dataset in DATASET_FLUCTUATION:
+        return MAX_RETRY_COUNT
+    return 1
+
+
+def run_evalscope(
+    host,
+    port,
+    model,
+    datasets,
+    dataset_args=None,
+    eval_batch_size=16,
+    limit=100000,
+    generation_config=None,
+    dataset_dir=None,
+    timeout=60000,
+    stream=True,
+    eval_type="openai_api",
+):
+
+    metrics_path = os.getenv("METRICS_DATA_FILE")
+    result_path = "./evalscope_result" if not metrics_path else metrics_path
+    logger.info(f"The metrics result file: {result_path}")
+
+    api_url = f"http://{host}:{port}/v1/chat/completions"
+
+    if generation_config is None:
+        generation_config = {"max_tokens": 512}
+
+    config_dict = {
+        "model": model,
+        "api_url": api_url,
+        "eval_type": eval_type,
+        "datasets": datasets,
+        "eval_batch_size": eval_batch_size,
+        "generation_config": generation_config,
+        "timeout": timeout,
+        "stream": stream,
+        "limit": limit,
+        "work_dir": result_path,
+    }
+    if dataset_args:
+        config_dict["dataset_args"] = dataset_args
+    if dataset_dir:
+        config_dict["dataset_dir"] = dataset_dir
+
+    config_json = json.dumps(config_dict, ensure_ascii=False, indent=2)
+    config_json_escaped = config_json.replace("\\", "\\\\").replace("'''", "\\'\\'\\'")
+
+    script_content = "import json\n"
+    script_content += "from evalscope import TaskConfig, run_task\n\n"
+    script_content += f"config = json.loads('''{config_json_escaped}''')\n"
+    script_content += "task_cfg = TaskConfig(**config)\n"
+    script_content += "run_task(task_cfg=task_cfg)\n"
+
+    script_path = f"/tmp/evalscope_run_{model}_{'_'.join(datasets)}.py"
+    with open(script_path, "w") as f:
+        f.write(script_content)
+
+    logger.info(f"Generated evalscope script: {script_path}")
+
+    install_cmd = (
+        "/bin/bash /root/sglang/python/sglang/test/ascend/e2e/run_evalscope.sh"
+    )
+    subprocess.run(install_cmd, shell=True, check=True)
+
+    python_bin = "test_env_evalscope/bin/python"
+    cmd = f"{python_bin} {script_path}"
+
+    logger.info(f"Command: {cmd}")
+
+    process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+        shell=True,
+    )
+
+    output_lines = []
+    try:
+        for line in iter(process.stdout.readline, ""):
+            if line.strip():
+                print(line, end="")
+            output_lines.append(line.strip())
+
+        process.wait()
+
+        if process.returncode != 0:
+            logger.error(f"Command failed with return code: {process.returncode}")
+            raise subprocess.CalledProcessError(process.returncode, cmd)
+
+        logger.info("Command executed successfully")
+
+        metrics = {}
+        full_output = "\n".join(output_lines)
+
+        report_match = re.search(r"Dump report to:\s*(\S+)", full_output)
+        if report_match:
+            report_path = report_match.group(1)
+            logger.info(f"Found evalscope report file: {report_path}")
+            try:
+                with open(report_path, "r") as rf:
+                    report_data = json.load(rf)
+                for item in report_data:
+                    score = item.get("score")
+                    if score is not None:
+                        metrics["accuracy"] = float(score)
+                        logger.info(f"The Final Accuracy from report: {score}")
+                        break
+            except Exception as e:
+                logger.warning(f"Failed to read report file {report_path}: {e}")
+
+        if "accuracy" not in metrics:
+            accuracy_patterns = [
+                r"mean_acc\s*.*?│\s*\d+\s*│\s*([\d.]+)\s*│",
+                r"│\s+([\d.]+)\s+│\s+\S+\s+│\s*$",
+                r"accuracy\s*[:=]?\s*([\d.]+)",
+                r"Accuracy\s*[:=]?\s*([\d.]+)",
+                r"score\s*[:=]?\s*([\d.]+)",
+            ]
+
+            for pattern in accuracy_patterns:
+                matches = re.findall(pattern, full_output)
+                if matches:
+                    final_accuracy = float(matches[-1])
+                    metrics["accuracy"] = final_accuracy
+                    logger.info(f"The Final Accuracy from output: {final_accuracy}")
+                    break
+
+        if "accuracy" not in metrics:
+            logger.info("Can Not Find The Accuracy in evalscope output")
+
+        return metrics
+
+    except KeyboardInterrupt:
+        logger.info("Keyboard interrupt received, terminating process...")
+        process.terminate()
+        try:
+            process.wait(timeout=5)
+            logger.info("Process terminated")
+        except subprocess.TimeoutExpired:
+            logger.warning("Process did not terminate gracefully, killing it...")
+            process.kill()
+            logger.info("Process killed")
+        raise
+    except Exception as e:
+        logger.error(f"Error executing command: {e}")
+        process.terminate()
+        process.wait(timeout=5)
+        raise
+
+
+def assert_metrics(self, metrics):
+    if not metrics:
+        raise Exception("No metrics obtained from benchmark")
+
+    if self.accuracy is not None:
+        threshold = get_accuracy_threshold(self.datasets, self.accuracy)
+        dump_metric(
+            "accuracy",
+            float(metrics["accuracy"]),
+            labels={"test_case": self.__class__.__name__, "type": "accuracy"},
+        )
+        dump_metric(
+            "accuracy_baseline",
+            float(self.accuracy),
+            labels={"test_case": self.__class__.__name__, "type": "accuracy"},
+        )
+        self.assertGreaterEqual(
+            float(metrics["accuracy"]),
+            threshold,
+            f"Accuracy check failed. Expected >= {threshold}, Got: {metrics['accuracy']}",
+        )
+
+
+class TestNpuAccuracyTestCaseBase(CustomTestCase):
+    model = None
+    benchmark_tool = BENCHMARK_TOOL_DEFAULT
+    backend = "sglang"
+    datasets = ["gsm8k"]
+    dataset_args = None
+    eval_batch_size = 16
+    limit = 100000
+    generation_config = None
+    dataset_dir = None
+    stream = True
+    timeout = 60000
+    eval_type = "openai_api"
+    other_args = None
+    server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    envs = None
+    max_attempts = 2
+    n_runs = 3
+    accuracy = 0.1
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        env = os.environ.copy()
+        for key, value in env.items():
+            logger.info(f"ENV_VAR_SYS {key}:{value}")
+        if cls.envs:
+            for key, value in cls.envs.items():
+                logger.info(f"ENV_VAR_CASE {key}:{value}")
+                env[key] = value
+
+        other_args = list(cls.other_args)
+
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=cls.server_timeout,
+            other_args=other_args,
+            env=env,
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        if hasattr(cls, "process") and cls.process:
+            try:
+                kill_process_tree(cls.process.pid)
+            except Exception as e:
+                logger.error(f"Error during tearDown: {e}")
+
+    def run_accuracy(self):
+        parsed_url = urlparse(self.base_url)
+        host = parsed_url.hostname
+        port = parsed_url.port
+        if self.benchmark_tool == EVALSCOPE:
+            model_name = os.path.basename(self.model)
+            max_retries = get_max_retries(self.datasets)
+            best_metrics = None
+            for attempt in range(max_retries):
+                metrics = run_evalscope(
+                    host=host,
+                    port=port,
+                    model=model_name,
+                    datasets=self.datasets,
+                    dataset_args=self.dataset_args,
+                    eval_batch_size=self.eval_batch_size,
+                    limit=self.limit,
+                    generation_config=self.generation_config,
+                    dataset_dir=self.dataset_dir,
+                    stream=self.stream,
+                    timeout=self.timeout,
+                    eval_type=self.eval_type,
+                )
+                if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
+                    best_metrics.get("accuracy", 0)
+                ):
+                    best_metrics = metrics
+                threshold = get_accuracy_threshold(self.datasets, self.accuracy)
+                if float(best_metrics.get("accuracy", 0)) >= threshold:
+                    break
+                if attempt < max_retries - 1:
+                    logger.info(
+                        f"Accuracy {best_metrics.get('accuracy')} below threshold "
+                        f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
+                    )
+            assert_metrics(self, best_metrics)
+
+    def run_accuracy_multiple(self, n_runs=None):
+        if n_runs is None:
+            n_runs = self.n_runs
+
+        parsed_url = urlparse(self.base_url)
+        host = parsed_url.hostname
+        port = parsed_url.port
+
+        if self.benchmark_tool != EVALSCOPE:
+            raise Exception(
+                "run_accuracy_multiple only supports evalscope benchmark tool"
+            )
+
+        model_name = os.path.basename(self.model)
+        all_metrics = []
+
+        for i in range(n_runs):
+            logger.info(f"=== Accuracy run {i + 1}/{n_runs} ===")
+            metrics = run_evalscope(
+                host=host,
+                port=port,
+                model=model_name,
+                datasets=self.datasets,
+                dataset_args=self.dataset_args,
+                eval_batch_size=self.eval_batch_size,
+                limit=self.limit,
+                generation_config=self.generation_config,
+                dataset_dir=self.dataset_dir,
+                stream=self.stream,
+                timeout=self.timeout,
+                eval_type=self.eval_type,
+            )
+            all_metrics.append(metrics)
+            if metrics and "accuracy" in metrics:
+                logger.info(f"Run {i + 1} accuracy: {metrics['accuracy']}")
+            else:
+                logger.warning(f"Run {i + 1} failed to get accuracy metric")
+
+        valid_metrics = [m for m in all_metrics if m and "accuracy" in m]
+        if not valid_metrics:
+            raise Exception("No valid accuracy metrics obtained from any run")
+
+        avg_accuracy = sum(float(m["accuracy"]) for m in valid_metrics) / len(
+            valid_metrics
+        )
+
+        logger.info("=" * 60)
+        logger.info("Multiple Run Accuracy Results:")
+        for i, m in enumerate(valid_metrics):
+            logger.info(f"  Run {i + 1}: {m['accuracy']}")
+        logger.info(f"  Average: {avg_accuracy}")
+        logger.info("=" * 60)
+
+        avg_metrics = {"accuracy": avg_accuracy}
+        dump_metric(
+            "accuracy_avg",
+            avg_accuracy,
+            labels={"test_case": self.__class__.__name__, "type": "accuracy"},
+        )
+        assert_metrics(self, avg_metrics)
+
+
+class TestNpuAccuracyMultiNodePdMixTestCaseBase(CustomTestCase):
+    model_config = None
+    benchmark_tool = BENCHMARK_TOOL_DEFAULT
+    backend = "sglang"
+    datasets = ["gsm8k"]
+    dataset_args = None
+    eval_batch_size = 16
+    limit = 100000
+    generation_config = None
+    dataset_dir = None
+    stream = True
+    timeout = 60000
+    eval_type = "openai_api"
+    other_args = None
+    server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    envs = None
+    max_attempts = 2
+    accuracy = 0.1
+
+    @classmethod
+    def setUpClass(cls):
+        cls.local_ip = "127.0.0.1"
+        cls.host = os.getenv("POD_IP")
+        cls.port = SERVICE_PORT
+        cls.base_url = f"http://{cls.host}:{cls.port}"
+        cls.hostname = os.getenv("HOSTNAME")
+        cls.role = "master" if cls.hostname.endswith("sglang-node-0") else "worker"
+        logger.info(f"Init {cls.host} {cls.role=}!")
+
+        cls.start_pd_mix_master_node()
+        cls.start_pd_mix_worker_node()
+
+    @classmethod
+    def tearDownClass(cls):
+        pass
+
+    @classmethod
+    @check_role(allowed_roles=["master"])
+    def start_pd_mix_master_node(cls):
+        sglang_thread = threading.Thread(
+            target=launch_pd_mix_node, args=(cls.model_config,)
+        )
+        sglang_thread.start()
+
+        wait_server_ready(f"{cls.base_url}/health")
+
+        logger.info(
+            f"Wait {SERVER_INITIALIZATION_DELAY}s, starting run benchmark ......"
+        )
+        time.sleep(SERVER_INITIALIZATION_DELAY)
+
+    @classmethod
+    @check_role(allowed_roles=["worker"])
+    def start_pd_mix_worker_node(cls):
+        sglang_thread = threading.Thread(
+            target=launch_pd_mix_node, args=(cls.model_config,)
+        )
+        sglang_thread.start()
+
+        logger.info(
+            f"{cls.role} node started, keeping test alive for {MAX_SERVER_KEEP_ALIVE_TIME} seconds"
+        )
+        time.sleep(MAX_SERVER_KEEP_ALIVE_TIME)
+
+    @check_role(allowed_roles=["master", "worker"])
+    def run_accuracy(self):
+        parsed_url = urlparse(self.base_url)
+        host = parsed_url.hostname
+        port = parsed_url.port
+        if self.benchmark_tool == EVALSCOPE:
+            model_name = os.path.basename(self.model_config.get("model_path"))
+            max_retries = get_max_retries(self.datasets)
+            best_metrics = None
+            for attempt in range(max_retries):
+                metrics = run_evalscope(
+                    host=self.host,
+                    port=self.port,
+                    model=model_name,
+                    datasets=self.datasets,
+                    dataset_args=self.dataset_args,
+                    eval_batch_size=self.eval_batch_size,
+                    limit=self.limit,
+                    generation_config=self.generation_config,
+                    dataset_dir=self.dataset_dir,
+                    stream=self.stream,
+                    timeout=self.timeout,
+                    eval_type=self.eval_type,
+                )
+                if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
+                    best_metrics.get("accuracy", 0)
+                ):
+                    best_metrics = metrics
+                threshold = get_accuracy_threshold(self.datasets, self.accuracy)
+                if float(best_metrics.get("accuracy", 0)) >= threshold:
+                    break
+                if attempt < max_retries - 1:
+                    logger.info(
+                        f"Accuracy {best_metrics.get('accuracy')} below threshold "
+                        f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
+                    )
+            assert_metrics(self, best_metrics)
+
+
+class TestNpuAccuracyMultiNodePdSepTestCaseBase(CustomTestCase):
+    model_config = None
+    benchmark_tool = BENCHMARK_TOOL_DEFAULT
+    backend = "sglang"
+    datasets = ["gsm8k"]
+    dataset_args = None
+    eval_batch_size = 16
+    limit = 100000
+    generation_config = None
+    dataset_dir = None
+    stream = True
+    timeout = 60000
+    eval_type = "openai_api"
+    other_args = None
+    server_timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    max_attempts = 2
+    accuracy = 0.1
+
+    @classmethod
+    def setUpClass(cls):
+        cls.process = None
+        cls.local_ip = "127.0.0.1"
+        cls.host = os.getenv("POD_IP")
+        cls.port = SERVICE_PORT
+        cls.base_url = f"http://{cls.host}:{cls.port}"
+        cls.hostname = os.getenv("HOSTNAME")
+        cls.role = (
+            "router"
+            if "router" in cls.hostname
+            else "prefill" if "prefill" in cls.hostname else "decode"
+        )
+        logger.info(f"Init {cls.host} {cls.role=}!")
+
+        cls.start_pd_server()
+        cls.start_router_server()
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.process:
+            try:
+                kill_process_tree(cls.process.pid)
+            except Exception as e:
+                logger.error(f"Error during tearDown: {e}")
+
+    @classmethod
+    @check_role(allowed_roles=["router"])
+    def start_router_server(cls):
+        logger.info(f"Starting router in thread...")
+        sglang_thread = threading.Thread(target=launch_router, args=(cls.model_config,))
+        sglang_thread.daemon = True
+        sglang_thread.start()
+
+        health_check_url = f"{cls.base_url}/health"
+        logger.info(f"Waiting for router to be ready at {health_check_url}")
+        wait_server_ready(health_check_url)
+
+        logger.info(
+            f"Waiting {SERVER_INITIALIZATION_DELAY} seconds for the server to fully initialize..."
+        )
+        time.sleep(SERVER_INITIALIZATION_DELAY)
+
+    @classmethod
+    @check_role(allowed_roles=["prefill", "decode"])
+    def start_pd_server(cls):
+        logger.info(f"Starting pd separation node...")
+        cls.process = launch_pd_separation_node(cls.model_config)
+        logger.info(f"Pd separation node started with PID: {cls.process.pid}")
+
+        while True:
+            if cls.process.poll() is None:
+                time.sleep(30)
+            else:
+                exit_code = cls.process.poll()
+                raise Exception(
+                    f"Sglang process exited on node {cls.host} {cls.hostname} with exit code: {exit_code}"
+                )
+
+    @check_role(allowed_roles=["router"])
+    def run_accuracy(self):
+        parsed_url = urlparse(self.base_url)
+        host = parsed_url.hostname
+        port = parsed_url.port
+        if self.benchmark_tool == EVALSCOPE:
+            model_name = os.path.basename(self.model_config.get("model_path"))
+            max_retries = get_max_retries(self.datasets)
+            best_metrics = None
+            for attempt in range(max_retries):
+                metrics = run_evalscope(
+                    host=host,
+                    port=port,
+                    model=model_name,
+                    datasets=self.datasets,
+                    dataset_args=self.dataset_args,
+                    eval_batch_size=self.eval_batch_size,
+                    limit=self.limit,
+                    generation_config=self.generation_config,
+                    dataset_dir=self.dataset_dir,
+                    stream=self.stream,
+                    timeout=self.timeout,
+                    eval_type=self.eval_type,
+                )
+                if best_metrics is None or float(metrics.get("accuracy", 0)) > float(
+                    best_metrics.get("accuracy", 0)
+                ):
+                    best_metrics = metrics
+                threshold = get_accuracy_threshold(self.datasets, self.accuracy)
+                if float(best_metrics.get("accuracy", 0)) >= threshold:
+                    break
+                if attempt < max_retries - 1:
+                    logger.info(
+                        f"Accuracy {best_metrics.get('accuracy')} below threshold "
+                        f"{threshold}, retrying ({attempt + 1}/{max_retries - 1})..."
+                    )
+            assert_metrics(self, best_metrics)