wehub-resource-sync
221 lines
11 KiB
Python
221 lines
11 KiB
Python
# Copyright (c) ModelScope Contributors. All rights reserved.
|
|
from dataclasses import dataclass, field
|
|
from transformers.utils import strtobool
|
|
from typing import List, Literal, Optional
|
|
|
|
from swift.utils import get_logger
|
|
|
|
logger = get_logger()
|
|
|
|
|
|
@dataclass
|
|
class TunerArguments:
|
|
"""
|
|
TunerArguments is a dataclass that holds configuration for various tuners.
|
|
|
|
Args:
|
|
freeze_parameters (List[str]): A list of prefixes for parameters that should be frozen during training.
|
|
Defaults to an empty list `[]`.
|
|
freeze_parameters_regex (Optional[str]): A regular expression to match the names of parameters that should be
|
|
frozen. Defaults to `None`.
|
|
freeze_parameters_ratio (float): The ratio of parameters to freeze, starting from the bottom layers upwards
|
|
(from 0.0 to 1.0). Setting this to 1.0 freezes all model parameters, which can be useful when selectively
|
|
unfreezing specific parameters with `trainable_parameters`. Defaults to 0.0.
|
|
trainable_parameters (List[str]): A list of prefixes for parameters that should be made explicitly trainable.
|
|
Defaults to an empty list `[]`.
|
|
trainable_parameters_regex (Optional[str]): A regular expression to match the names of parameters that should
|
|
be made explicitly trainable. Defaults to `None`.
|
|
Note on parameter freezing priority: The `trainable_*` arguments have higher priority than the `freeze_*`
|
|
arguments. The freezing logic is applied as follows:
|
|
Firstly, all parameters are set to trainable.
|
|
Then, `freeze_parameters`, `freeze_parameters_regex`, and `freeze_parameters_ratio` are applied to freeze
|
|
parts of the model.
|
|
Finally, `trainable_parameters` and `trainable_parameters_regex` are used to unfreeze specific parameters,
|
|
ensuring they are trainable regardless of the freezing rules.
|
|
|
|
freeze_llm (bool): For multi-modal models only. If `True`, it affects the Large Language Model (LLM) part.
|
|
In full fine-tuning, this freezes the LLM weights. In LoRA training with `target_modules=['all-linear']`,
|
|
this prevents adding LoRA modules to the LLM. Defaults to `False`.
|
|
freeze_vit (bool): For multi-modal models only. If `True`, it affects the Vision/Audio Transformer (ViT) part.
|
|
In full fine-tuning, this freezes the ViT weights. In LoRA training with `target_modules=['all-linear']`,
|
|
this prevents adding LoRA modules to the ViT. Note: 'vit' can refer to `vision_tower` and `audio_tower`.
|
|
Defaults to `True`.
|
|
freeze_aligner (bool): For multi-modal models only. If `True`, it affects the aligner (projector) part.
|
|
In full fine-tuning, this freezes the aligner weights. In LoRA training with
|
|
`target_modules=['all-linear']`, this prevents adding LoRA modules to the aligner. Defaults to `True`.
|
|
|
|
target_modules (List[str]): List of target modules for tuning. Default is ['all-linear'].
|
|
target_regex (Optional[str]): Regular expression to match target modules. Default is None.
|
|
target_parameters (Optional[List[str]]): A list of parameter names to be replaced by LoRA modules. This is
|
|
similar to `target_modules` but targets parameters directly, which is useful for layers like MoE that use
|
|
`nn.Parameter` instead of `nn.Linear`. Requires `peft>=0.17.0`. Defaults to `None`.
|
|
modules_to_save (List[str]): List of modules to save. Default is an empty list.
|
|
|
|
lora_rank (int): Rank for LoRA. Default is 8.
|
|
lora_alpha (int): Alpha value for LoRA. Default is 32.
|
|
lora_dropout (float): Dropout rate for LoRA. Default is 0.05.
|
|
lora_bias (Literal['none', 'all']): The possible values are 'none' and 'all'. If set to 'all', all biases
|
|
will be trainable. Default is 'none'.
|
|
lora_dtype (Literal): Data type for LoRA. Default is 'AUTO'. Allowed values are 'fp16', 'bf16', 'fp32', 'AUTO'.
|
|
lorap_lr_ratio (float): Learning rate ratio for LoRA. Default is None.
|
|
use_rslora (bool): Flag to indicate if RSLora is used. Default is False.
|
|
use_dora (bool): Flag to indicate if Dora is used. Default is False.
|
|
|
|
lora_ga_batch_size (int): Batch size used for estimating gradients during initialization in LoRA-GA. Default
|
|
value is 2.
|
|
lora_ga_iters (int): Number of iterations for estimating gradients during initialization in LoRA-GA. Default
|
|
value is 2.
|
|
lora_ga_max_length (int): Maximum input length for estimating gradients during initialization in LoRA-GA.
|
|
Default value is 1024.
|
|
lora_ga_direction (str): Initial direction used for gradient estimation during initialization in LoRA-GA.
|
|
Default value is `ArB2r`. Allowed: `ArBr`, `A2rBr`, `ArB2r`, and `random`.
|
|
lora_ga_scale (str): The scaling method for initialization in LoRA-GA.
|
|
Default value is `stable`. Allowed values are: `gd`, `unit`, `stable`, and `weightS`.
|
|
lora_ga_stable_gamma (int): The gamma value when choosing `stable` scaling for initialization. Default
|
|
value is 16.
|
|
|
|
init_weights (str): The method for initializing adapter weights. For LoRA, options include 'true', 'false',
|
|
'gaussian', 'pissa', 'pissa_niter_[number of iters]', 'olora', 'loftq', and 'lora-ga'. For BoNE,
|
|
options are 'true', 'false', and 'bat'. Defaults to 'true'.
|
|
|
|
fourier_n_frequency (int): Number of frequencies for FourierFT. Default is 2000.
|
|
fourier_scaling (float): Scaling factor for FourierFT. Default is 300.0.
|
|
|
|
boft_block_size (int): Block size for BOFT. Default is 4.
|
|
boft_block_num (int): Number of blocks for BOFT. Default is 0.
|
|
boft_n_butterfly_factor (int): Butterfly factor for BOFT. Default is 1.
|
|
boft_dropout (float): Dropout rate for BOFT. Default is 0.0.
|
|
|
|
vera_rank (int): Rank for Vera. Default is 256.
|
|
vera_projection_prng_key (int): PRNG key for Vera projection. Default is 0.
|
|
vera_dropout (float): Dropout rate for Vera. Default is 0.0.
|
|
vera_d_initial (float): Initial value for Vera D. Default is 0.1.
|
|
|
|
adapter_act (str): Activation function for adapter. Default is 'gelu'.
|
|
adapter_length (int): Length of the adapter. Default is 128.
|
|
|
|
adalora_target_r (int): Target rank for AdaLoRA. Default is 8.
|
|
adalora_init_r (int): Initial rank for AdaLoRA. Default is 12.
|
|
adalora_tinit (int): Initial T value for AdaLoRA. Default is 100.
|
|
adalora_tfinal (int): Final T value for AdaLoRA. Default is 1000.
|
|
adalora_deltaT (int): Delta T value for AdaLoRA. Default is 10.
|
|
adalora_beta1 (float): Beta1 value for AdaLoRA. Default is 0.85.
|
|
adalora_beta2 (float): Beta2 value for AdaLoRA. Default is 0.85.
|
|
adalora_orth_reg_weight (float): Orthogonal regularization weight for AdaLoRA. Default is 0.5.
|
|
|
|
llamapro_num_new_blocks (int): Number of new blocks for LLaMAPro. Default is 4.
|
|
llamapro_num_groups (Optional[int]): Number of groups for LLaMAPro. Default is None.
|
|
|
|
reft_layer_key (Optional[str]): Key identifier for ReFT layer. Default is None.
|
|
reft_layers (Optional[List[int]]): List of layers involved in ReFT. Default is None.
|
|
reft_rank (int): Rank parameter for ReFT. Default is 4.
|
|
reft_intervention_type (Literal): Type of intervention for ReFT. Default is 'LoreftIntervention'.
|
|
reft_args (Optional[str]): Additional arguments for ReFT. Default is None.
|
|
"""
|
|
# full
|
|
freeze_parameters: List[str] = field(default_factory=list)
|
|
freeze_parameters_regex: Optional[str] = None
|
|
freeze_parameters_ratio: float = 0. # 0 ~ 1
|
|
trainable_parameters: List[str] = field(default_factory=list)
|
|
trainable_parameters_regex: Optional[str] = None
|
|
# lora or full
|
|
freeze_llm: bool = False
|
|
freeze_vit: bool = True
|
|
freeze_aligner: bool = True
|
|
# tuners
|
|
target_modules: List[str] = field(default_factory=lambda: ['all-linear'])
|
|
target_regex: Optional[str] = None
|
|
target_parameters: Optional[List[str]] = None
|
|
# e.g. ['wte', 'ln_1', 'ln_2', 'ln_f', 'lm_head']
|
|
modules_to_save: List[str] = field(default_factory=list)
|
|
|
|
# lora
|
|
lora_rank: int = 8
|
|
lora_alpha: int = 32
|
|
lora_dropout: float = 0.05
|
|
lora_bias: Literal['none', 'all'] = 'none'
|
|
lora_dtype: Literal['float16', 'bfloat16', 'float32', None] = None
|
|
lorap_lr_ratio: Optional[float] = None
|
|
use_rslora: bool = False
|
|
use_dora: bool = False
|
|
|
|
# lora_ga
|
|
lora_ga_batch_size: int = 2
|
|
lora_ga_iters: int = 2
|
|
lora_ga_max_length: int = 1024
|
|
lora_ga_direction: str = 'ArB2r'
|
|
lora_ga_scale: str = 'stable'
|
|
lora_ga_stable_gamma: int = 16
|
|
|
|
# Lora: Literal['gaussian', 'pissa', 'pissa_niter_[number of iters]', 'olora', 'loftq', 'true', 'false', 'lora-ga']
|
|
# Bone: Literal['bat', 'true', 'false']
|
|
init_weights: str = 'true'
|
|
|
|
# fourierft
|
|
fourier_n_frequency: int = 2000
|
|
fourier_scaling: float = 300.0
|
|
|
|
# BOFT
|
|
boft_block_size: int = 4
|
|
boft_block_num: int = 0
|
|
boft_n_butterfly_factor: int = 1
|
|
boft_dropout: float = 0.0
|
|
|
|
# Vera
|
|
vera_rank: int = 256
|
|
vera_projection_prng_key: int = 0
|
|
vera_dropout: float = 0.0
|
|
vera_d_initial: float = 0.1
|
|
|
|
# adapter
|
|
adapter_act: str = 'gelu'
|
|
adapter_length: int = 128
|
|
|
|
# adalora
|
|
adalora_target_r: int = 8
|
|
adalora_init_r: int = 12
|
|
adalora_tinit: int = 0
|
|
adalora_tfinal: int = 0
|
|
adalora_deltaT: int = 1
|
|
adalora_beta1: float = 0.85
|
|
adalora_beta2: float = 0.85
|
|
adalora_orth_reg_weight: float = 0.5
|
|
|
|
# llamapro
|
|
llamapro_num_new_blocks: int = 4
|
|
llamapro_num_groups: Optional[int] = None
|
|
|
|
# reft
|
|
reft_layer_key: Optional[str] = None
|
|
reft_layers: Optional[List[int]] = None
|
|
reft_rank: int = 4
|
|
reft_intervention_type: Literal['NoreftIntervention', 'LoreftIntervention', 'ConsreftIntervention',
|
|
'LobireftIntervention', 'DireftIntervention',
|
|
'NodireftIntervention'] = 'LoreftIntervention'
|
|
reft_args: Optional[str] = None
|
|
|
|
def __post_init__(self):
|
|
if isinstance(self.init_weights, str) and self.init_weights.lower() in {'true', 'false'}:
|
|
self.init_weights = bool(strtobool(self.init_weights))
|
|
self._init_multimodal_full()
|
|
if self.target_regex:
|
|
self.target_modules = self.target_regex
|
|
|
|
def _init_multimodal_full(self):
|
|
model_arch = self.model_meta.model_arch
|
|
if not self.model_meta.is_multimodal or not model_arch or self.tuner_type != 'full':
|
|
return
|
|
if self.freeze_llm:
|
|
self.freeze_parameters += model_arch.language_model
|
|
if self.freeze_vit:
|
|
self.freeze_parameters += model_arch.vision_tower
|
|
if self.freeze_aligner:
|
|
self.freeze_parameters += model_arch.aligner
|
|
else:
|
|
self.trainable_parameters += model_arch.aligner
|
|
self.freeze_parameters += model_arch.generator
|
|
if self.freeze_parameters:
|
|
logger.info(f'freeze_parameters: {self.freeze_parameters}')
|
|
if self.trainable_parameters:
|
|
logger.info(f'additional trainable_parameters: {self.trainable_parameters}')
|