chore: import upstream snapshot with attribution

examples/train/tuners/adalora/train.sh

@@ -0,0 +1,16 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type adalora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/adapter/train.sh

@@ -0,0 +1,16 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type adapter \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/boft/train.sh

@@ -0,0 +1,16 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type boft \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/bone/train.sh

@@ -0,0 +1,16 @@
+# 17.3GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type bone \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/dora/train.sh

@@ -0,0 +1,19 @@
+# 17.2GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --use_dora true \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/galore/train_galore.sh

@@ -0,0 +1,18 @@
+# 38GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot \
+    --use_galore true \
+    --galore_optim_per_parameter true

examples/train/tuners/galore/train_qgalore.sh

@@ -0,0 +1,22 @@
+# 35GiB
+# pip install bitsandbytes==0.40.0
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --torch_dtype bfloat16 \
+    --dataset 'lvjianjin/AdvertiseGen#1000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot \
+    --use_galore true \
+    --galore_quantization true

examples/train/tuners/lisa/train.sh

@@ -0,0 +1,17 @@
+# 29GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type full \
+    --dataset 'swift/self-cognition#1000' \
+    --lisa_activated_layers 2 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/llamapro/train.sh

@@ -0,0 +1,17 @@
+# 25.4GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type llamapro \
+    --dataset 'swift/self-cognition#1000' \
+    --llamapro_num_new_blocks 4 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/longlora/train.sh

@@ -0,0 +1,18 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model LLM-Research/Meta-Llama-3.1-8B-Instruct \
+    --tuner_type longlora \
+    --dataset 'AI-ModelScope/LongAlpaca-12k#1000' \
+    --load_from_cache_file true \
+    --split_dataset_ratio 0.01 \
+    --num_train_epochs 1 \
+    --learning_rate 1e-4 \
+    --attn_impl flash_attn \
+    --gradient_accumulation_steps 16 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --max_length 10000 \
+    --save_total_limit 2 \
+    --logging_steps 5

examples/train/tuners/lora-ga/train.sh

@@ -0,0 +1,33 @@
+# Train
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2-1.5B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --init_weights lora-ga \
+    --lora_ga_batch_size 2 \
+    --lora_ga_iters 2 \
+    --lora_ga_max_length 1024 \
+    --lora_ga_direction ArB2r \
+    --lora_ga_scale stable \
+    --lora_ga_stable_gamma 16 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot
+
+# Infer
+# swift infer \
+#     --model Qwen/Qwen2-1.5B-Instruct \
+#     --adapters ./output/Qwen2-1.5B-Instruct/v0-20241214-191235/checkpoint-62/converted/default \
+#     --infer_backend transformers \
+#     --stream true \
+#     --max_new_tokens 2048

examples/train/tuners/lora/train.sh

@@ -0,0 +1,18 @@
+# 17.2GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/neftune/train.sh

@@ -0,0 +1,19 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --neftune_noise_alpha 15 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/olora/train.sh

@@ -0,0 +1,19 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --init_weights olora \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/pissa/train.sh

@@ -0,0 +1,19 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --init_weights pissa \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/qlora/train.sh

@@ -0,0 +1,19 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot \
+    --quant_bits 4 \
+    --quant_method bnb

examples/train/tuners/reft/train.sh

@@ -0,0 +1,17 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_type reft \
+    --dataset 'swift/self-cognition#1000' \
+    --reft_intervention_type 'LoreftIntervention' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_checkpointing false \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

examples/train/tuners/unsloth/train.sh

@@ -0,0 +1,19 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --tuner_backend unsloth \
+    --tuner_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot