chore: import upstream snapshot with attribution

examples/train/multi-node/accelerate/multi_node.yaml

@@ -0,0 +1,17 @@
+compute_environment: LOCAL_MACHINE
+deepspeed_config:
+    deepspeed_multinode_launcher: standard
+    gradient_accumulation_steps: 16
+    offload_optimizer_device: none
+    offload_param_device: none
+    zero3_init_flag: false
+    zero_stage: 3
+distributed_type: DEEPSPEED
+main_process_ip: 'xxx.xxx.xxx.xxx'
+main_process_port: 29500
+main_training_function: main
+mixed_precision: bf16
+num_machines: 2
+num_processes: 8  # world size
+rdzv_backend: static
+use_cpu: false

examples/train/multi-node/accelerate/train_node1.sh

@@ -0,0 +1,18 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+accelerate launch --config_file ./examples/train/multi-node/accelerate/multi_node.yaml --machine_rank 0 \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-node/accelerate/train_node2.sh

@@ -0,0 +1,18 @@
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+accelerate launch --config_file ./examples/train/multi-node/accelerate/multi_node.yaml --machine_rank 1 \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-node/deepspeed/README.md

@@ -0,0 +1,42 @@
+# How to run
+
+## 1. Install pdsh in your nodes
+
+```shell
+# https://code.google.com/archive/p/pdsh/downloads
+# For example, download to /root:
+cd /root
+wget https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/pdsh/pdsh-2.29.tar.bz2
+tar -xvf pdsh-2.29.tar.bz2
+cd pdsh-2.29
+./configure --prefix=/root/pdsh-2.29 --with-ssh --without-rsh --with-exec --with-timeout=60 --with-nodeupdown --with-rcmd-rank-list=ssh
+make
+make install
+```
+
+In case of the privilege is correct:
+```shell
+chown root:root /root/pdsh-2.29
+```
+
+## Configure the ssh
+
+vim your ~/.ssh/config and input:
+```text
+Host worker-0
+    HostName your-worker-0-ip-here
+    User root
+Host worker-1
+    HostName your-worker-1-ip-here
+    User root
+```
+Say you have two nodes, when doing this, make sure your other nodes can be logined with `ssh root@worker-x` without password(with ssh-key).
+
+## Clone swift repo and run
+
+```shell
+git clone https://github.com/modelscope/ms-swift.git
+cd ms-swift
+# If your node number is different, edit examples/train/multi-node/deepspeed/host.txt
+sh examples/train/multi-node/deepspeed/train.sh
+```

examples/train/multi-node/deepspeed/host.txt

@@ -0,0 +1,2 @@
+worker-0 slots=2
+worker-1 slots=2

examples/train/multi-node/deepspeed/train.sh

@@ -0,0 +1,20 @@
+# If your need only a part of the GPUs in every node, try:
+# --include="worker-0:0,1@worker-1:2,3"
+deepspeed --hostfile=./examples/train/multi-node/deepspeed/host.txt \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --torch_dtype bfloat16 \
+    --dataset 'swift/self-cognition#1000' \
+    --load_from_cache_file true \
+    --num_train_epochs 1 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/multi-node/dlc/train.sh

@@ -0,0 +1,27 @@
+# For more information, visit: https://www.aliyun.com/activity/bigdata/pai-dlc
+# https://help.aliyun.com/zh/pai/user-guide/general-environment-variables
+NNODES=$WORLD_SIZE \
+NODE_RANK=$RANK \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' \
+              'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2

examples/train/multi-node/ray/sft.sh

@@ -0,0 +1 @@
+swift sft examples/train/multi-node/ray/sft.yaml

examples/train/multi-node/ray/sft.yaml

@@ -0,0 +1,36 @@
+model: Qwen/Qwen2.5-7B-Instruct
+split_dataset_ratio: 0.0
+tuner_type: lora
+target_modules:
+  - q_proj
+  - k_proj
+  - v_proj
+  - o_proj
+torch_dtype: bfloat16
+attn_impl: flash_attn
+num_train_epochs: 5
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 1
+learning_rate: 1e-4
+dataset: swift/self-cognition#1000
+gradient_accumulation_steps: 8
+eval_steps: 1000
+save_steps: 1000
+save_total_limit: 5
+logging_steps: 5
+warmup_ratio: 0.05
+dataloader_num_workers: 0
+dataset_num_proc: 8
+deepspeed: zero3
+model_name: swift-bot
+model_author: swift
+
+use_ray: true
+
+device_groups:
+  nproc_per_node: 4
+  default:
+    device: GPU
+    ranks: list(range(0, 4))
+    workers:
+      - default

examples/train/multi-node/swift/train_node1.sh

@@ -0,0 +1,32 @@
+nnodes=2
+nproc_per_node=4
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NNODES=$nnodes \
+NODE_RANK=0 \
+MASTER_ADDR=127.0.0.1 \
+MASTER_PORT=29500 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' \
+              'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps $(expr 32 / $nproc_per_node / $nnodes) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2

examples/train/multi-node/swift/train_node2.sh

@@ -0,0 +1,32 @@
+nnodes=2
+nproc_per_node=4
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NNODES=$nnodes \
+NODE_RANK=1 \
+MASTER_ADDR=xxx.xxx.xxx.xxx \
+MASTER_PORT=29500 \
+NPROC_PER_NODE=$nproc_per_node \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' \
+              'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps $(expr 32 / $nproc_per_node / $nnodes) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2

examples/train/multi-node/torchrun/train_node1.sh

@@ -0,0 +1,33 @@
+nnodes=2
+nproc_per_node=4
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+torchrun \
+    --master_port 29500 \
+    --nproc_per_node=$nproc_per_node \
+    --nnodes=$nnodes \
+    --node_rank=0 \
+    --master_addr=127.0.0.1 \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' \
+              'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps $(expr 32 / $nproc_per_node / $nnodes) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2

examples/train/multi-node/torchrun/train_node2.sh

@@ -0,0 +1,33 @@
+nnodes=2
+nproc_per_node=4
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+torchrun \
+    --master_port 29500 \
+    --nproc_per_node=$nproc_per_node \
+    --nnodes=$nnodes \
+    --node_rank=1 \
+    --master_addr=xxx.xxx.xxx.xxx \
+    swift/cli/sft.py \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' \
+              'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps $(expr 32 / $nproc_per_node / $nnodes) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2