wehub-resource-sync
67 lines
1.3 KiB
YAML
67 lines
1.3 KiB
YAML
ENV:
|
|
PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'
|
|
MAX_PIXELS: '1003520'
|
|
VIDEO_MAX_PIXELS: '50176'
|
|
FPS_MAX_FRAMES: '12'
|
|
|
|
# model args
|
|
model: "Qwen/Qwen3.5-35B-A3B"
|
|
|
|
tuner_type: lora
|
|
lora_rank: 8
|
|
lora_alpha: 32
|
|
target_modules: all-linear
|
|
|
|
# dataset args
|
|
dataset:
|
|
- 'AI-ModelScope/alpaca-gpt4-data-zh#500'
|
|
- 'AI-ModelScope/alpaca-gpt4-data-en#500'
|
|
- 'swift/self-cognition#500'
|
|
- 'AI-ModelScope/LaTeX_OCR:human_handwrite#2000'
|
|
load_from_cache_file: true
|
|
split_dataset_ratio: 0.01
|
|
max_length: 2048
|
|
dataloader_num_workers: 4
|
|
dataset_num_proc: 4
|
|
model_author: swift
|
|
model_name: swift-bot
|
|
padding_free: true
|
|
packing: true
|
|
|
|
# template args
|
|
loss_scale: ignore_empty_think
|
|
add_non_thinking_prefix: true
|
|
|
|
# training args
|
|
save_safetensors: true
|
|
merge_lora: true
|
|
micro_batch_size: 1
|
|
global_batch_size: 4
|
|
num_train_epochs: 1
|
|
finetune: true
|
|
lr: 1e-4
|
|
lr_warmup_fraction: 0.05
|
|
min_lr: 1e-5
|
|
output_dir: megatron_output/Qwen3.5-35B-A3B
|
|
eval_steps: 200
|
|
save_steps: 200
|
|
no_save_optim: true
|
|
no_save_rng: true
|
|
|
|
freeze_llm: false
|
|
freeze_vit: true
|
|
freeze_aligner: true
|
|
|
|
expert_model_parallel_size: 4
|
|
sequence_parallel: true
|
|
moe_permute_fusion: true
|
|
moe_grouped_gemm: true
|
|
moe_shared_expert_overlap: true
|
|
moe_aux_loss_coeff: 1e-6
|
|
|
|
recompute_granularity: full
|
|
recompute_method: uniform
|
|
recompute_num_layers: 1
|
|
cross_entropy_loss_fusion: true
|
|
attention_backend: flash
|