chore: import upstream snapshot with attribution

examples/ascend/train/qwen3/qwen3_lora_deepspeed.sh

@@ -0,0 +1,31 @@
+# hardware: Atlas 900 A2
+export TASK_QUEUE_ENABLE=2
+export CPU_AFFINITY_CONF=2
+nproc_per_node=8
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model 'Qwen/Qwen3-32B' \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 10 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 1 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot \
+    --deepspeed zero3

examples/ascend/train/qwen3/qwen3_lora_fsdp/fsdp.json

@@ -0,0 +1,26 @@
+{
+  "compute_environment": "LOCAL_MACHINE",
+  "debug": false,
+  "distributed_type": "FSDP",
+  "downcast_bf16": "no",
+  "mixed_precision": "bf16",
+  "num_machines": 1,
+  "num_processes": 8,
+  "machine_rank": 0,
+  "rdzv_backend": "static",
+  "same_network": true,
+  "use_cpu": false,
+  "fsdp_config": {
+    "fsdp_auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
+    "fsdp_transformer_cls_names_to_wrap": "Qwen3DecoderLayer",
+    "fsdp_sharding_strategy": "FULL_SHARD",
+    "fsdp_backward_prefetch": "BACKWARD_PRE",
+    "fsdp_forward_prefetch": true,
+    "fsdp_limit_all_gathers": true,
+    "fsdp_state_dict_type": "FULL_STATE_DICT",
+    "fsdp_sync_module_states": true,
+    "fsdp_cpu_ram_efficient_loading": false,
+    "fsdp_use_orig_params": true,
+    "fsdp_offload_params": false
+  }
+}

examples/ascend/train/qwen3/qwen3_lora_fsdp/train.sh

@@ -0,0 +1,29 @@
+# hardware: Atlas 900 A2
+# For NPU, in Transformers versions 5.0 and above, it is recommended to disable
+# cpu_ram_efficient_loading in fsdp.json to avoid timeout issues at the first
+# synchronization point caused by inter-card desynchronization when loading the model.
+export TASK_QUEUE_ENABLE=2
+export CPU_AFFINITY_CONF=2
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+accelerate launch --config_file "./examples/ascend/train/qwen3/qwen3_lora_fsdp/fsdp.json" \
+    swift/cli/sft.py \
+    --model 'Qwen/Qwen3-32B' \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --torch_dtype bfloat16 \
+    --per_device_train_batch_size 10 \
+    --gradient_accumulation_steps 2 \
+    --gradient_checkpointing true \
+    --gradient_checkpointing_kwargs '{"use_reentrant": false}' \
+    --max_length 1200 \
+    --num_train_epochs 2 \
+    --eval_strategy no \
+    --save_steps 500 \
+    --logging_steps 1 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --save_total_limit 2 \
+    --save_only_model true \
+    --output_dir output \
+    --attn_impl 'flash_attention_2' \
+    --packing true

examples/ascend/train/qwen3/qwen3_lora_megatron.sh

@@ -0,0 +1,37 @@
+NPROC_PER_NODE=2 \
+ASCEND_RT_VISIBLE_DEVICES=0,1 \
+megatron sft \
+    --model Qwen/Qwen3-8B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 2 \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 2 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-8B \
+    --save_steps 100 \
+    --max_length 2048 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_5/qwen3_lora_megatron_npu.sh

@@ -0,0 +1,58 @@
+# 16 * 64GiB Ascend A3
+
+# NPU stability environment variables
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export MULTI_STREAM_MEMORY_REUSE=1
+# NPU memory management environment variables
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+# NPU performance environment variables
+export TASK_QUEUE_ENABLE=2
+export USE_MCORE_GDN=0
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 16 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    \
+    --micro_batch_size 2 \
+    --global_batch_size 32 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 100 \
+    --max_length 1024 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot \
+    --save_total_limit 3

examples/ascend/train/qwen3_5/qwen3_lora_megatron_npu_a3.sh

@@ -0,0 +1,56 @@
+# 16 * 64GiB Ascend A3
+export USE_MCORE_GDN=0
+
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export MULTI_STREAM_MEMORY_REUSE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export TASK_QUEUE_ENABLE=2
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+MASTER_PORT=29609 \
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors false \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    \
+    --micro_batch_size 1 \
+    --global_batch_size 16 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 32 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_5/qwen3_lora_megatron_npu_a5.sh

@@ -0,0 +1,52 @@
+# 8 * 96GiB Ascend A5
+
+export TASK_QUEUE_ENABLE=2
+export USE_MCORE_GDN=0
+
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3.5-35B-A3B \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    \
+    --tensor_model_parallel_size 2 \
+    --expert_model_parallel_size 4 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    \
+    --micro_batch_size 2 \
+    --global_batch_size 16 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 32 \
+    \
+    --output_dir output/Qwen3.5-35B-A3B \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    \
+    --attention_backend flash \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_next/qwen3_next_megatron.sh

@@ -0,0 +1,39 @@
+export TASK_QUEUE_ENABLE=2
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model Qwen/Qwen3-Next-80B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 2 \
+    --export_model_parallel_size 4 \
+    --model_type qwen3_next \
+    --sequence_parallel true \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 4 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-Next-Instruct \
+    --save_steps 100 \
+    --max_length 1024 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_omni/qwen3_omni_full_mindspeed.sh

@@ -0,0 +1,50 @@
+# 16 * 64GiB Ascend A3
+# Modified from https://github.com/modelscope/ms-swift/blob/main/examples/megatron/multimodal/omni/moe.sh
+export TASK_QUEUE_ENABLE=2
+PYTORCH_NPU_ALLOC_CONF='expandable_segments:True' \
+NPROC_PER_NODE=16 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=12 \
+megatron sft \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite' \
+    --load_from_cache_file true \
+    --sequence_parallel true \
+    --packing true \
+    --freeze_llm false \
+    --freeze_vit true \
+    --freeze_aligner true \
+    --split_dataset_ratio 0.01 \
+    --expert_model_parallel_size 8 \
+    --expert_tensor_parallel_size 1 \
+    --tensor_model_parallel_size 1 \
+    --pipeline_model_parallel_size 2 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-3 \
+    --micro_batch_size 1 \
+    --global_batch_size 8 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --cross_entropy_fusion_impl native \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --num_train_epochs 3 \
+    --output_dir megatron_output/Qwen3-Omni-30B-A3B-Instruct \
+    --eval_steps 1000 \
+    --save_steps 10000 \
+    --max_length 1024 \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --attention_backend flash \
+    --gradient_accumulation_fusion False \
+    --masked_softmax_fusion False

examples/ascend/train/qwen3_omni/qwen3_omni_lora_megatron_A3.sh

@@ -0,0 +1,54 @@
+# 16 * 64GiB Ascend A3
+
+export USE_MCORE_GDN=0
+
+export HCCL_OP_BASE_FFTS_MODE_ENABLE=TRUE
+export HCCL_CONNECT_TIMEOUT=600
+export MULTI_STREAM_MEMORY_REUSE=1
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+
+export TASK_QUEUE_ENABLE=2
+
+
+export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+NPROC_PER_NODE=16 \
+megatron sft \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
+    --save_safetensors false \
+    --dataset 'AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000' \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 8 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --micro_batch_size 2 \
+    --global_batch_size 32 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-omni \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --attention_backend flash \
+    --padding_free false \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_omni/qwen3_omni_lora_megatron_A5.sh

@@ -0,0 +1,52 @@
+# 8 * 96GiB Ascend A5
+
+export USE_MCORE_GDN=0
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export TASK_QUEUE_ENABLE=2
+export HCCL_CONNECT_TIMEOUT=600
+
+MODEL_PATH=Qwen/Qwen3-Omni-30B-A3B-Instruct
+DATASET_PATH='AI-ModelScope/MAmmoTH-VL-Instruct-12M#1000'
+
+NPROC_PER_NODE=8 \
+ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+megatron sft \
+    --model ${MODEL_PATH} \
+    --save_safetensors false \
+    --dataset ${DATASET_PATH} \
+    --tuner_type lora \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --tensor_model_parallel_size 1 \
+    --expert_model_parallel_size 8 \
+    --moe_permute_fusion true \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --sequence_parallel true \
+    --micro_batch_size 4 \
+    --global_batch_size 32 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-4 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-5 \
+    --num_train_epochs 1 \
+    --output_dir megatron_output/Qwen3-omni \
+    --save_steps 2000 \
+    --max_length 4096 \
+    --system 'You are a helpful assistant.' \
+    --dataloader_num_workers 4 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --dataset_num_proc 4 \
+    --gradient_accumulation_fusion false \
+    --masked_softmax_fusion false \
+    --attention_backend flash \
+    --padding_free false \
+    --model_author swift \
+    --model_name swift-robot

examples/ascend/train/qwen3_vl/moe_full_mindspeed.sh

@@ -0,0 +1,55 @@
+# 16 * 64GiB Ascend A3
+# Modified from https://github.com/modelscope/ms-swift/blob/main/examples/models/qwen3_vl/mcore_full.sh
+export TASK_QUEUE_ENABLE=2
+export COMBINED_ENABLE=1
+export CPU_AFFINITY_CONF=1
+export TORCH_HCCL_ZERO_COPY=1
+
+PYTORCH_NPU_ALLOC_CONF='expandable_segments:True' \
+MULTI_STREAM_MEMORY_REUSE=2 \
+OMP_NUM_THREADS=14 \
+NPROC_PER_NODE=16 \
+IMAGE_MAX_TOKEN_NUM=1024 \
+VIDEO_MAX_TOKEN_NUM=128 \
+FPS_MAX_FRAMES=16 \
+megatron sft \
+    --model Qwen/Qwen3-VL-30B-A3B-Instruct \
+    --save_safetensors true \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#10000' \
+              'AI-ModelScope/LaTeX_OCR:human_handwrite#5000' \
+              'swift/VideoChatGPT:Generic#2000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --tensor_model_parallel_size 2 \
+    --pipeline_model_parallel_size 2 \
+    --expert_model_parallel_size 8 \
+    --moe_grouped_gemm true \
+    --moe_shared_expert_overlap true \
+    --moe_aux_loss_coeff 1e-6 \
+    --micro_batch_size 1 \
+    --global_batch_size 4 \
+    --recompute_granularity full \
+    --recompute_method uniform \
+    --recompute_num_layers 1 \
+    --num_train_epochs 1 \
+    --finetune true \
+    --cross_entropy_loss_fusion true \
+    --lr 1e-5 \
+    --lr_warmup_fraction 0.05 \
+    --min_lr 1e-6 \
+    --output_dir megatron_output/Qwen3-VL-30B-A3B-Instruct \
+    --eval_steps 500 \
+    --save_steps 500 \
+    --max_length 4096 \
+    --packing true \
+    --dataloader_num_workers 8 \
+    --dataset_num_proc 8 \
+    --no_save_optim true \
+    --no_save_rng true \
+    --sequence_parallel true \
+    --moe_expert_capacity_factor 2 \
+    --attention_backend flash
+    # --moe_permute_fusion true
+    # --optimizer_cpu_offload true
+    # --use_precision_aware_optimizer true
+    # --optimizer_offload_fraction 0.2