chore: import upstream snapshot with attribution

examples/ascend/activation_cpu_offload/fsdp2.json

@@ -0,0 +1,26 @@
+{
+    "_description": "FSDP2 configuration for distributed training (PyTorch native FSDP v2)",
+    "_requires": "torch>=2.4.0",
+    "_note": "This is the recommended configuration for multi-GPU training without CPU offloading. NOTE: When using FSDP2, do NOT use --gradient_checkpointing, use activation_checkpointing in fsdp_config instead.",
+
+    "_param_docs": {
+        "fsdp": "FSDP strategy string. Options: 'full_shard' (ZeRO-3 style, shards params+grads+optimizer), 'shard_grad_op' (ZeRO-2 style, shards grads+optimizer only). Add 'auto_wrap' to enable automatic layer wrapping. Add 'offload' to enable CPU offloading.",
+        "fsdp_version": "FSDP version. Use 2 for PyTorch native FSDP2 (recommended). FSDP2 uses DTensor for per-parameter sharding, supports LoRA/QLoRA natively.",
+        "auto_wrap_policy": "How to wrap model layers. 'TRANSFORMER_BASED_WRAP' wraps transformer decoder layers (from model._no_split_modules). 'SIZE_BASED_WRAP' wraps modules exceeding min_num_params.",
+        "cpu_ram_efficient_loading": "If true, only rank 0 loads full model weights, then broadcasts to other ranks. Reduces CPU RAM usage during initialization.",
+        "state_dict_type": "'SHARDED_STATE_DICT' (recommended): each rank saves its own shard without extra communication. 'FULL_STATE_DICT': gathers full model on rank 0 (higher memory, slower).",
+        "reshard_after_forward": "true = FULL_SHARD (ZeRO-3), reshards params after forward pass. false = SHARD_GRAD_OP (ZeRO-2), keeps params gathered during forward/backward.",
+        "activation_checkpointing": "Use FSDP's native activation checkpointing instead of gradient_checkpointing. This is the correct way to save memory with FSDP.",
+        "activation_cpu_offload": "true = offload activations to CPU. false = keep activations on GPU,can enable when using activation_checkpointing."
+    },
+    "fsdp": "full_shard auto_wrap",
+    "fsdp_config": {
+        "fsdp_version": 2,
+        "reshard_after_forward": true,
+        "auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
+        "cpu_ram_efficient_loading": true,
+        "state_dict_type": "SHARDED_STATE_DICT",
+        "activation_checkpointing": false,
+        "activation_cpu_offload": true
+    }
+}

examples/ascend/activation_cpu_offload/train.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+ASCEND_RT_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=2 \
+swift sft \
+    --model 'Qwen/Qwen3-8B' \
+    --tuner_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_checkpointing false \
+    --weight_decay 0.1 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 5 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --system You\ are\ a\ helpful\ assistant. \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --fsdp './examples/ascend/activation_cpu_offload/fsdp2.json'
+
+#  --dataset AI-ModelScope/LongAlpaca-12k
+# activation_cpu_offload=false
+
+# {'loss': 2.93329144, 'grad_norm': 2.44835496, 'learning_rate': 0.0001, 'token_acc': 0.56405613, 'epoch': 0.06, 'global_step/max_steps': '1/16', 'percentage': '6.25%', 'elapsed_time': '8s', 'remaining_time': '2m 6s', 'memory(GiB)': 24.8, 'train_speed(iter/s)': 0.118837}
+# {'loss': 2.93490505, 'grad_norm': 2.63550186, 'learning_rate': 8.346e-05, 'token_acc': 0.58979954, 'epoch': 0.32, 'global_step/max_steps': '5/16', 'percentage': '31.25%', 'elapsed_time': '28s', 'remaining_time': '1m 2s', 'memory(GiB)': 57.91, 'train_speed(iter/s)': 0.175644}
+# Train:  31%|███████████████████████████████████                                                                             | 5/16 [00:28<00:57,  5.22s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v60-20260202-130514/checkpoint-5
+# {'loss': 1.61339226, 'grad_norm': 1.05343676, 'learning_rate': 3.455e-05, 'token_acc': 0.63342983, 'epoch': 0.64, 'global_step/max_steps': '10/16', 'percentage': '62.50%', 'elapsed_time': '51s', 'remaining_time': '31s', 'memory(GiB)': 58.02, 'train_speed(iter/s)': 0.192856}
+# Train:  62%|█████████████████████████████████████████████████████████████████████▍                                         | 10/16 [00:51<00:27,  4.66s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v60-20260202-130514/checkpoint-10
+# {'loss': 1.32472887, 'grad_norm': 0.60581738, 'learning_rate': 1.09e-06, 'token_acc': 0.64779323, 'epoch': 0.96, 'global_step/max_steps': '15/16', 'percentage': '93.75%', 'elapsed_time': '1m 13s', 'remaining_time': '4s', 'memory(GiB)': 58.02, 'train_speed(iter/s)': 0.204973}
+# Train:  94%|████████████████████████████████████████████████████████████████████████████████████████████████████████       | 15/16 [01:13<00:04,  4.12s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v60-20260202-130514/checkpoint-15
+# Train: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [01:17<00:00,  4.25s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v60-20260202-130514/checkpoint-16
+# {'train_runtime': 79.7064, 'train_samples_per_second': 6.311, 'train_steps_per_second': 0.201, 'train_loss': 1.91648413, 'token_acc': 0.68027888, 'epoch': 1.0, 'global_step/max_steps': '16/16', 'percentage': '100.00%', 'elapsed_time': '1m 19s', 'remaining_time': '0s', 'memory(GiB)': 58.02, 'train_speed(iter/s)': 0.200728}
+# Train: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [01:19<00:00,  4.98s/it]
+
+
+#  --dataset AI-ModelScope/LongAlpaca-12k
+# "activation_cpu_offload": true
+
+# {'loss': 2.93329144, 'grad_norm': 2.44853568, 'learning_rate': 0.0001, 'token_acc': 0.56405613, 'epoch': 0.06, 'global_step/max_steps': '1/16', 'percentage': '6.25%', 'elapsed_time': '26s', 'remaining_time': '6m 43s', 'memory(GiB)': 24.62, 'train_speed(iter/s)': 0.037168}
+# {'loss': 2.93512678, 'grad_norm': 2.6212213, 'learning_rate': 8.346e-05, 'token_acc': 0.5895268, 'epoch': 0.32, 'global_step/max_steps': '5/16', 'percentage': '31.25%', 'elapsed_time': '1m 21s', 'remaining_time': '2m 58s', 'memory(GiB)': 26.93, 'train_speed(iter/s)': 0.061631}
+# Train:  31%|███████████████████████████████████                                                                             | 5/16 [01:21<02:30, 13.67s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v59-20260202-125158/checkpoint-5
+# {'loss': 1.61200867, 'grad_norm': 1.05091298, 'learning_rate': 3.455e-05, 'token_acc': 0.63310818, 'epoch': 0.64, 'global_step/max_steps': '10/16', 'percentage': '62.50%', 'elapsed_time': '2m 20s', 'remaining_time': '1m 24s', 'memory(GiB)': 26.93, 'train_speed(iter/s)': 0.0712}
+# Train:  62%|█████████████████████████████████████████████████████████████████████▍                                         | 10/16 [02:20<01:11, 11.97s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v59-20260202-125158/checkpoint-10
+# {'loss': 1.32489185, 'grad_norm': 0.60476321, 'learning_rate': 1.09e-06, 'token_acc': 0.64746468, 'epoch': 0.96, 'global_step/max_steps': '15/16', 'percentage': '93.75%', 'elapsed_time': '3m 11s', 'remaining_time': '12s', 'memory(GiB)': 26.94, 'train_speed(iter/s)': 0.078265}
+# Train:  94%|████████████████████████████████████████████████████████████████████████████████████████████████████████       | 15/16 [03:11<00:10, 10.03s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v59-20260202-125158/checkpoint-15
+# Train: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [03:20<00:00,  9.65s/it][INFO:swift] Saving model checkpoint to /model/ljl/project/ms-swift/output/v59-20260202-125158/checkpoint-16
+# {'train_runtime': 202.2537, 'train_samples_per_second': 2.487, 'train_steps_per_second': 0.079, 'train_loss': 1.91632293, 'token_acc': 0.67729084, 'epoch': 1.0, 'global_step/max_steps': '16/16', 'percentage': '100.00%', 'elapsed_time': '3m 22s', 'remaining_time': '0s', 'memory(GiB)': 26.94, 'train_speed(iter/s)': 0.078996}
+# Train: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [03:22<00:00, 12.66s/it]

examples/ascend/deploy/vllm.sh

@@ -0,0 +1,4 @@
+ASCEND_RT_VISIBLE_DEVICES=0 swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm \
+    --served_model_name Qwen2.5-7B-Instruct

examples/ascend/infer/vllm/dp_tp.sh

@@ -0,0 +1,11 @@
+NPROC_PER_NODE=4 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 \
+swift infer \
+    --model Qwen/Qwen3-8B \
+    --infer_backend vllm \
+    --val_dataset AI-ModelScope/alpaca-gpt4-data-zh#2000 \
+    --vllm_gpu_memory_utilization 0.9 \
+    --vllm_max_model_len 8192 \
+    --vllm_tensor_parallel_size 2 \
+    --max_new_tokens 2048 \
+    --write_batch_size 1000

examples/ascend/megatron/train_sft_full.sh

@@ -0,0 +1,35 @@
+PYTORCH_NPU_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=4 \
+megatron sft \
+    --model Qwen/Qwen3-4B \
+    --save_safetensors true \
+    --dataset 'llm-wizard/alpaca-gpt4-data-zh' \
+    --use_hf true \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 2 \
+    --packing True \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 5 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3-4B \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 8192 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --cross_entropy_loss_fusion true \
+    --cross_entropy_fusion_impl native \
+    --attention_backend flash

examples/ascend/multi-node/megatron/node1.sh

@@ -0,0 +1,33 @@
+# Atlas A2 * 2 nodes * 8 cards per node
+
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NNODES=2 \
+NODE_RANK=0 \
+MASTER_ADDR=127.0.0.1 \
+MASTER_PORT=29500 \
+NPROC_PER_NODE=8 \
+HCCL_SOCKET_IFNAME=xxx \
+megatron sft \
+    --model 'Qwen/Qwen3-8B' \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' \
+    --output_dir './SAVE' \
+    --tuner_type 'lora' \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules 'all-linear' \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 1 \
+    --context_parallel_size 1 \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 64 \
+    --recompute_granularity selective \
+    --recompute_modules core_attn \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --logging_steps 5 \
+    --dataloader_num_workers 4

examples/ascend/multi-node/megatron/node2.sh

@@ -0,0 +1,33 @@
+# Atlas A2 * 2 nodes * 8 cards per node
+
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NNODES=2 \
+NODE_RANK=1 \
+MASTER_ADDR=xxx.xxx.xxx.xxx \
+MASTER_PORT=29500 \
+NPROC_PER_NODE=8 \
+HCCL_SOCKET_IFNAME=xxx \
+megatron sft \
+    --model 'Qwen/Qwen3-8B' \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#1000' \
+    --output_dir './SAVE' \
+    --tuner_type 'lora' \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules 'all-linear' \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 1 \
+    --context_parallel_size 1 \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 64 \
+    --recompute_granularity selective \
+    --recompute_modules core_attn \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --logging_steps 5 \
+    --dataloader_num_workers 4

examples/ascend/train/qwen3/qwen3_lora_deepspeed.sh

@@ -0,0 +1,31 @@
+# hardware: Atlas 900 A2
+export TASK_QUEUE_ENABLE=2
+export CPU_AFFINITY_CONF=2
+nproc_per_node=8
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model 'Qwen/Qwen3-32B' \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 10 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --deepspeed zero3

examples/ascend/train/qwen3/qwen3_lora_fsdp/fsdp.json

@@ -0,0 +1,26 @@
+{
+  "compute_environment": "LOCAL_MACHINE",
+  "debug": false,
+  "distributed_type": "FSDP",
+  "downcast_bf16": "no",
+  "mixed_precision": "bf16",
+  "num_machines": 1,
+  "num_processes": 8,
+  "machine_rank": 0,
+  "rdzv_backend": "static",
+  "same_network": true,
+  "use_cpu": false,
+  "fsdp_config": {
+    "fsdp_auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
+    "fsdp_transformer_cls_names_to_wrap": "Qwen3DecoderLayer",
+    "fsdp_sharding_strategy": "FULL_SHARD",
+    "fsdp_backward_prefetch": "BACKWARD_PRE",
+    "fsdp_forward_prefetch": true,
+    "fsdp_limit_all_gathers": true,
+    "fsdp_state_dict_type": "FULL_STATE_DICT",
+    "fsdp_sync_module_states": true,
+    "fsdp_cpu_ram_efficient_loading": false,
+    "fsdp_use_orig_params": true,
+    "fsdp_offload_params": false
+  }
+}

examples/ascend/train/qwen3/qwen3_lora_fsdp/train.sh

@@ -0,0 +1,29 @@
+# hardware: Atlas 900 A2
+# For NPU, in Transformers versions 5.0 and above, it is recommended to disable
+# cpu_ram_efficient_loading in fsdp.json to avoid timeout issues at the first
+# synchronization point caused by inter-card desynchronization when loading the model.
+export TASK_QUEUE_ENABLE=2
+export CPU_AFFINITY_CONF=2
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+accelerate launch --config_file "./examples/ascend/train/qwen3/qwen3_lora_fsdp/fsdp.json" \
+    swift/cli/sft.py \
+    --model 'Qwen/Qwen3-32B' \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 10 \
+    --gradient_accumulation_steps 2 \
+    --gradient_checkpointing true \
+    --gradient_checkpointing_kwargs '{"use_reentrant": false}' \
+    --max_length 1200 \
+    --num_train_epochs 2 \
+    --eval_strategy no \
+    --save_steps 500 \
+    --logging_steps 1 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output \
+    --attn_impl 'flash_attention_2' \
+    --packing true

examples/ascend/train/qwen3/qwen3_lora_megatron.sh

@@ -0,0 +1,37 @@
+NPROC_PER_NODE=2 \
+ASCEND_RT_VISIBLE_DEVICES=0,1 \
+megatron sft \
+    --model Qwen/Qwen3-8B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 2 \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 2 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-8B \
+    --save_steps 100 \
+    --max_length 2048 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_5/qwen3_lora_megatron_npu.sh

@@ -0,0 +1,58 @@
+# 16 * 64GiB Ascend A3
+
+# NPU stability environment variables
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export MULTI_STREAM_MEMORY_REUSE=1
+# NPU memory management environment variables
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+# NPU performance environment variables
+export TASK_QUEUE_ENABLE=2
+export USE_MCORE_GDN=0
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 16 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    \
+    --micro_batch_size 2 \
+    --global_batch_size 32 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 100 \
+    --max_length 1024 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot \
+    --save_total_limit 3

examples/ascend/train/qwen3_5/qwen3_lora_megatron_npu_a3.sh

@@ -0,0 +1,56 @@
+# 16 * 64GiB Ascend A3
+export USE_MCORE_GDN=0
+
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export MULTI_STREAM_MEMORY_REUSE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export TASK_QUEUE_ENABLE=2
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+MASTER_PORT=29609 \
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors false \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    \
+    --micro_batch_size 1 \
+    --global_batch_size 16 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 32 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_5/qwen3_lora_megatron_npu_a5.sh

@@ -0,0 +1,52 @@
+# 8 * 96GiB Ascend A5
+
+export TASK_QUEUE_ENABLE=2
+export USE_MCORE_GDN=0
+
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    \
+    --micro_batch_size 2 \
+    --global_batch_size 16 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 32 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_next/qwen3_next_megatron.sh

@@ -0,0 +1,39 @@
+export TASK_QUEUE_ENABLE=2
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3-Next-80B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 2 \
+    --export_model_parallel_size 4 \
+    --model_type qwen3_next \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 4 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-Next-Instruct \
+    --save_steps 100 \
+    --max_length 1024 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_omni/qwen3_omni_full_mindspeed.sh

@@ -0,0 +1,50 @@
+# 16 * 64GiB Ascend A3
+# Modified from https://github.com/modelscope/ms-swift/blob/main/examples/megatron/multimodal/omni/moe.sh
+export TASK_QUEUE_ENABLE=2
+PYTORCH_NPU_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=16 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+megatron sft \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite' \
+    --load_from_cache_file true \
+    --sequence_parallel true \
+    --packing true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --split_dataset_ratio 0.01 \
+    --expert_model_parallel_size 8 \
+    --expert_tensor_parallel_size 1 \
+    --tensor_model_parallel_size 1 \
+    --pipeline_model_parallel_size 2 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-3 \
+    --micro_batch_size 1 \
+    --global_batch_size 8 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --cross_entropy_fusion_impl native \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --num_train_epochs 3 \
+    --output_dir megatron_output/Qwen3-Omni-30B-A3B-Instruct \
+    --eval_steps 1000 \
+    --save_steps 10000 \
+    --max_length 1024 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --attention_backend flash \
+    --gradient_accumulation_fusion False \
+    --masked_softmax_fusion False

examples/ascend/train/qwen3_omni/qwen3_omni_lora_megatron_A3.sh

@@ -0,0 +1,54 @@
+# 16 * 64GiB Ascend A3
+
+export USE_MCORE_GDN=0
+
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export HCCL_CONNECT_TIMEOUT=600
+export MULTI_STREAM_MEMORY_REUSE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+
+export TASK_QUEUE_ENABLE=2
+
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+    --save_safetensors false \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 8 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --micro_batch_size 2 \
+    --global_batch_size 32 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-omni \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --attention_backend flash \
+    --padding_free false \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_omni/qwen3_omni_lora_megatron_A5.sh

@@ -0,0 +1,52 @@
+# 8 * 96GiB Ascend A5
+
+export USE_MCORE_GDN=0
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export TASK_QUEUE_ENABLE=2
+export HCCL_CONNECT_TIMEOUT=600
+
+MODEL_PATH=Qwen/Qwen3-Omni-30B-A3B-Instruct
+DATASET_PATH='AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000'
+
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model ${MODEL_PATH} \
+    --save_safetensors false \
+    --dataset ${DATASET_PATH} \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 8 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --micro_batch_size 4 \
+    --global_batch_size 32 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-omni \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --attention_backend flash \
+    --padding_free false \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_vl/moe_full_mindspeed.sh

@@ -0,0 +1,55 @@
+# 16 * 64GiB Ascend A3
+# Modified from https://github.com/modelscope/ms-swift/blob/main/examples/models/qwen3_vl/mcore_full.sh
+export TASK_QUEUE_ENABLE=2
+export COMBINED_ENABLE=1
+export CPU_AFFINITY_CONF=1
+export TORCH_HCCL_ZERO_COPY=1
+
+PYTORCH_NPU_ALLOC_CONF='expandable_segments:True' \
+MULTI_STREAM_MEMORY_REUSE=2 \
+OMP_NUM_THREADS=14 \
+NPROC_PER_NODE=16 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=16 \
+megatron sft \
+    --model Qwen/Qwen3-VL-30B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'swift/VideoChatGPT:Generic#2000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 2 \
+    --expert_model_parallel_size 8 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3-VL-30B-A3B-Instruct \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 4096 \
+    --packing true \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --moe_expert_capacity_factor 2 \
+    --attention_backend flash
+    # --moe_permute_fusion true
+    # --optimizer_cpu_offload true
+    # --use_precision_aware_optimizer true
+    # --optimizer_offload_fraction 0.2