mirrors/modelscope--ms-swift
1
0
Fork 0
Code Issues Pull Requests Actions 1 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Lint test / lint (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:34:58 +08:00
commit a203934033
1368 changed files with 175001 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
swift/ui/llm_train/__init__.py
+15
View File
@@ -0,0 +1,15 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
from .advanced import Advanced
from .dataset import Dataset
from .hyper import Hyper
from .llm_train import LLMTrain
from .lora import LoRA
from .model import Model
from .optimizer import Optimizer
from .quantization import Quantization
from .report_to import ReportTo
from .runtime import Runtime
from .save import Save
from .target import Target
from .tuner import Tuner
from .utils import run_command_in_background_with_popen
swift/ui/llm_train/advanced.py
+114
View File
@@ -0,0 +1,114 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class Advanced(BaseUI):
group = 'llm_train'
locale_dict = {
'advanced_tab': {
'label': {
'zh': '高级参数设置',
'en': 'Advanced settings'
},
},
'tuner_backend': {
'label': {
'zh': 'Tuner backend',
'en': 'Tuner backend'
},
'info': {
'zh': 'Tuner实现框架',
'en': 'The tuner backend'
}
},
'weight_decay': {
'label': {
'zh': '权重衰减',
'en': 'Weight decay'
},
'info': {
'zh': '设置weight decay',
'en': 'Set the weight decay'
}
},
'logging_steps': {
'label': {
'zh': '日志打印步数',
'en': 'Logging steps'
},
'info': {
'zh': '设置日志打印的步数间隔',
'en': 'Set the logging interval'
}
},
'lr_scheduler_type': {
'label': {
'zh': 'LrScheduler类型',
'en': 'The LrScheduler type'
},
'info': {
'zh': '设置LrScheduler类型',
'en': 'Set the LrScheduler type'
}
},
'warmup_ratio': {
'label': {
'zh': '学习率warmup比例',
'en': 'Lr warmup ratio'
},
'info': {
'zh': '设置学习率warmup比例',
'en': 'Set the warmup ratio in total steps'
}
},
'truncation_strategy': {
'label': {
'zh': '数据集超长策略',
'en': 'Dataset truncation strategy'
},
'info': {
'zh': '如果token超长该如何处理',
'en': 'How to deal with the rows exceed the max length'
}
},
'max_steps': {
'label': {
'zh': '最大迭代步数',
'en': 'Max steps',
},
'info': {
'zh': '设置最大迭代步数，该值如果大于零则数据集迭代次数不生效',
'en': 'Set the max steps, if the value > 0 then num_train_epochs has no effects',
}
},
'max_grad_norm': {
'label': {
'zh': '梯度裁剪',
'en': 'Max grad norm',
},
'info': {
'zh': '设置梯度裁剪',
'en': 'Set the max grad norm',
}
}
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.TabItem(elem_id='advanced_tab'):
with gr.Blocks():
with gr.Row():
gr.Dropdown(elem_id='tuner_backend', scale=20)
gr.Textbox(elem_id='weight_decay', lines=1, scale=20)
gr.Textbox(elem_id='logging_steps', lines=1, scale=20)
gr.Textbox(elem_id='lr_scheduler_type', lines=1, scale=20)
with gr.Row():
gr.Dropdown(elem_id='truncation_strategy', value=None, scale=20)
gr.Textbox(elem_id='max_steps', lines=1, scale=20)
gr.Textbox(elem_id='max_grad_norm', lines=1, scale=20)
gr.Slider(elem_id='warmup_ratio', minimum=0.0, maximum=1.0, step=0.05, scale=20)
swift/ui/llm_train/dataset.py
+70
View File
@@ -0,0 +1,70 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from swift.dataset import get_dataset_list
from ..base import BaseUI
class Dataset(BaseUI):
group = 'llm_train'
locale_dict = {
'dataset': {
'label': {
'zh': '数据集名称',
'en': 'Dataset Code'
},
'info': {
'zh': '选择训练的数据集，支持复选/本地路径',
'en': 'The dataset(s) to train the models, support multi select and local folder/files'
}
},
'max_length': {
'label': {
'zh': '句子最大长度',
'en': 'The max length',
},
'info': {
'zh': '设置输入模型的最大长度',
'en': 'Set the max length input to the model',
}
},
'split_dataset_ratio': {
'label': {
'zh': '验证集拆分比例',
'en': 'Split ratio of eval dataset'
},
'info': {
'zh': '表示将总数据的多少拆分到验证集中',
'en': 'Split the datasets by this ratio for eval'
}
},
'padding_free': {
'label': {
'zh': '无填充批处理',
'en': 'Padding-free batching'
},
'info': {
'zh': '将一个batch中的数据进行展平而避免数据padding',
'en': 'Flatten the data in a batch to avoid data padding'
}
},
'dataset_param': {
'label': {
'zh': '数据集设置',
'en': 'Dataset settings'
},
},
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.Accordion(elem_id='dataset_param', open=True):
with gr.Row():
gr.Dropdown(
elem_id='dataset', multiselect=True, choices=get_dataset_list(), scale=20, allow_custom_value=True)
gr.Slider(elem_id='split_dataset_ratio', minimum=0.0, maximum=1.0, step=0.05, scale=10)
gr.Slider(elem_id='max_length', minimum=32, maximum=32768, value=1024, step=1, scale=10)
gr.Checkbox(elem_id='padding_free', scale=10)
swift/ui/llm_train/hyper.py
+149
View File
@@ -0,0 +1,149 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class Hyper(BaseUI):
group = 'llm_train'
locale_dict = {
'hyper_param': {
'label': {
'zh': '超参数设置(更多参数->其他参数设置)',
'en': 'Hyper settings(more params->Extra settings)',
},
},
'per_device_train_batch_size': {
'label': {
'zh': '训练batch size',
'en': 'Train batch size',
},
'info': {
'zh': '训练的batch size',
'en': 'Set the train batch size',
}
},
'per_device_eval_batch_size': {
'label': {
'zh': '验证batch size',
'en': 'Val batch size',
},
'info': {
'zh': '验证的batch size',
'en': 'Set the val batch size',
}
},
'learning_rate': {
'label': {
'zh': '学习率',
'en': 'Learning rate',
},
'info': {
'zh': '设置学习率',
'en': 'Set the learning rate',
}
},
'eval_steps': {
'label': {
'zh': '交叉验证步数',
'en': 'Eval steps',
},
'info': {
'zh': '设置每隔多少步数进行一次验证',
'en': 'Set the step interval to validate',
}
},
'num_train_epochs': {
'label': {
'zh': '数据集迭代轮次',
'en': 'Train epoch',
},
'info': {
'zh': '设置对数据集训练多少轮次',
'en': 'Set the max train epoch',
}
},
'gradient_accumulation_steps': {
'label': {
'zh': '梯度累计步数',
'en': 'Gradient accumulation steps',
},
'info': {
'zh': '设置梯度累计步数以减小显存占用',
'en': 'Set the gradient accumulation steps',
}
},
'attn_impl': {
'label': {
'zh': 'Flash Attention类型',
'en': 'Flash Attention Type',
},
},
'neftune_noise_alpha': {
'label': {
'zh': 'NEFTune噪声系数',
'en': 'NEFTune noise coefficient'
},
'info': {
'zh': '使用NEFTune提升训练效果, 一般设置为5或者10',
'en': 'Use NEFTune to improve performance, normally the value should be 5 or 10'
}
},
'save_steps': {
'label': {
'zh': '存储步数',
'en': 'Save steps',
},
'info': {
'zh': '设置每个多少步数进行存储',
'en': 'Set the save steps',
}
},
'output_dir': {
'label': {
'zh': '存储目录',
'en': 'The output dir',
},
'info': {
'zh': '设置输出模型存储在哪个文件夹下',
'en': 'Set the output folder',
}
},
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.Accordion(elem_id='hyper_param', open=False):
with gr.Blocks():
with gr.Row():
gr.Slider(elem_id='per_device_train_batch_size', minimum=1, maximum=256, step=2, scale=20)
gr.Slider(elem_id='per_device_eval_batch_size', minimum=1, maximum=256, step=2, scale=20)
gr.Textbox(elem_id='learning_rate', value='1e-4', lines=1, scale=20)
gr.Textbox(elem_id='num_train_epochs', lines=1, scale=20)
gr.Slider(
elem_id='gradient_accumulation_steps',
minimum=1,
maximum=256,
step=2,
value=1 if cls.group == 'llm_grpo' else 16,
scale=20)
with gr.Row():
gr.Textbox(elem_id='eval_steps', lines=1, value='500', scale=20)
gr.Textbox(elem_id='save_steps', value='500', lines=1, scale=20)
gr.Textbox(elem_id='output_dir', scale=20)
gr.Dropdown(
elem_id='attn_impl',
value=None,
choices=[None, 'sdpa', 'eager', 'flash_attention_2', 'flash_attention_3', 'flash_attention_4'],
scale=20)
gr.Slider(elem_id='neftune_noise_alpha', minimum=0.0, maximum=20.0, step=0.5, scale=20)
@staticmethod
def update_lr(tuner_type):
if tuner_type == 'full':
return 1e-5
else:
return 1e-4
swift/ui/llm_train/llm_train.py
+579
View File
@@ -0,0 +1,579 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import collections
import gradio as gr
import json
import os
import re
import sys
import time
from copy import deepcopy
from functools import partial
from json import JSONDecodeError
from subprocess import PIPE, STDOUT, Popen
from transformers.utils import is_torch_cuda_available, is_torch_npu_available
from typing import Dict, Type
from swift.arguments import ExportArguments, RLHFArguments, get_supported_tuners
from swift.utils import get_device_count, get_logger
from ..base import BaseUI
from .advanced import Advanced
from .dataset import Dataset
from .hyper import Hyper
from .model import Model
from .optimizer import Optimizer
from .quantization import Quantization
from .report_to import ReportTo
from .runtime import Runtime
from .save import Save
from .self_cog import SelfCog
from .task import Task
from .tuner import Tuner
from .utils import run_command_in_background_with_popen
logger = get_logger()
class LLMTrain(BaseUI):
group = 'llm_train'
sub_ui = [
Model,
Dataset,
Runtime,
Save,
Optimizer,
Task,
Tuner,
Hyper,
Quantization,
SelfCog,
Advanced,
ReportTo,
]
locale_dict: Dict[str, Dict] = {
'llm_train': {
'label': {
'zh': 'LLM预训练/微调',
'en': 'LLM PT/SFT',
}
},
'train_stage': {
'label': {
'zh': '训练Stage',
'en': 'Train Stage'
},
'info': {
'zh': '请注意选择与此匹配的数据集',
'en': 'Please choose matched dataset'
}
},
'submit_alert': {
'value': {
'zh':
'任务已开始，请查看tensorboard或日志记录，请勿关闭终端，否则训练过程将被打断',
'en':
'Task started, please check the tensorboard or log file, '
'do not close the terminal, otherwise the training process will be interrupted'
}
},
'dataset_alert': {
'value': {
'zh': '请选择或填入一个数据集',
'en': 'Please input or select a dataset'
}
},
'submit': {
'value': {
'zh': '🚀 开始训练',
'en': '🚀 Begin'
}
},
'dry_run': {
'label': {
'zh': '仅生成运行命令',
'en': 'Dry-run'
},
'info': {
'zh': '仅生成运行命令，开发者自行运行',
'en': 'Generate run command only, for manually running'
}
},
'gpu_id': {
'label': {
'zh': '选择可用GPU',
'en': 'Choose GPU'
},
'info': {
'zh': '选择训练使用的GPU号，如CUDA不可用只能选择CPU',
'en': 'Select GPU to train'
}
},
'tuner_type': {
'label': {
'zh': '训练方式',
'en': 'Train type'
},
'info': {
'zh': '选择训练的方式',
'en': 'Select the tuner type'
}
},
'seed': {
'label': {
'zh': '随机数种子',
'en': 'Seed'
},
'info': {
'zh': '选择随机数种子',
'en': 'Select a random seed'
}
},
'torch_dtype': {
'label': {
'zh': '训练精度',
'en': 'Training Precision'
},
'info': {
'zh': '选择训练精度',
'en': 'Select the training precision'
}
},
'envs': {
'label': {
'zh': '环境变量',
'en': 'Extra env vars'
},
},
'use_ddp': {
'label': {
'zh': '使用DDP',
'en': 'Use DDP'
},
'info': {
'zh': '是否使用数据并行训练',
'en': 'Use Distributed Data Parallel to train'
}
},
'ddp_num': {
'label': {
'zh': 'DDP分片数量',
'en': 'Number of DDP sharding'
},
'info': {
'zh': '启用多少进程的数据并行',
'en': 'The data parallel size of DDP'
}
},
'use_liger_kernel': {
'label': {
'zh': '使用Liger kernel',
'en': 'Use Liger kernel'
},
'info': {
'zh': 'Liger kernel可以有效降低显存使用',
'en': 'Liger kernel can reduce memory usage'
}
},
'sequence_parallel_size': {
'label': {
'zh': '序列并行大小',
'en': 'Sequence parallel size',
},
'info': {
'zh': '当前支持CPT/SFT/DPO/GRPO',
'en': 'Currently supports CPT/SFT/DPO/GRPO',
}
},
'deepspeed': {
'label': {
'zh': 'DeepSpeed',
'en': 'DeepSpeed',
},
'info': {
'zh': '可以选择下拉列表，也支持传入路径',
'en': 'Choose from the dropbox or fill in a valid path',
}
},
'resume_checkpoint_alert': {
'value': {
'zh': '检测到"args.json"在{}中，将从此检查点开始断点续训',
'en': 'Detected that "args.json" is in {}, will start breakpoint resume training from this checkpoint'
}
},
'resume_only_model_alert': {
'value': {
'zh':
'检测到"args.json"在{}中，但未检测到优化器参数，将仅加载模型参数开始断点续训',
'en':
'"args.json" is detected in {}, but optimizer parameters are not detected. '
'Only model parameters will be loaded to start breakpoint continuation training'
}
},
'more_params': {
'label': {
'zh': '其他高级参数',
'en': 'Other params'
},
'info': {
'zh': '以json格式或--xxx xxx命令行格式填入',
'en': 'Fill in with json format or --xxx xxx cmd format'
}
},
'extra_params': {
'label': {
'zh': '其他参数设置',
'en': 'Extra settings'
},
},
'train_param': {
'label': {
'zh': '训练参数设置',
'en': 'Train settings'
},
},
}
choice_dict = BaseUI.get_choices_from_dataclass(RLHFArguments)
default_dict = BaseUI.get_default_value_from_dataclass(RLHFArguments)
arguments = BaseUI.get_argument_names(RLHFArguments)
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.TabItem(elem_id='llm_train', label=''):
default_device = 'cpu'
device_count = get_device_count()
if device_count > 0:
default_device = '0'
with gr.Blocks():
Model.build_ui(base_tab)
Dataset.build_ui(base_tab)
with gr.Accordion(elem_id='train_param', open=True):
with gr.Row():
gr.Dropdown(elem_id='train_stage', choices=['pt', 'sft'], value='sft', scale=4)
gr.Dropdown(elem_id='tuner_type', scale=4, choices=list(get_supported_tuners()))
gr.Textbox(elem_id='seed', scale=4)
gr.Dropdown(elem_id='torch_dtype', scale=4)
gr.Checkbox(elem_id='use_liger_kernel', scale=4)
with gr.Row():
gr.Dropdown(
elem_id='gpu_id',
multiselect=True,
choices=[str(i) for i in range(device_count)] + ['cpu'],
value=default_device,
scale=4)
gr.Checkbox(elem_id='use_ddp', value=False, scale=4)
gr.Textbox(elem_id='ddp_num', value='1', scale=4)
gr.Dropdown(
elem_id='deepspeed',
scale=4,
allow_custom_value=True,
value=None,
choices=['zero0', 'zero1', 'zero2', 'zero3', 'zero2_offload', 'zero3_offload'])
gr.Textbox(elem_id='sequence_parallel_size', lines=1, scale=4)
Hyper.build_ui(base_tab)
Runtime.build_ui(base_tab)
with gr.Row(equal_height=True):
gr.Textbox(elem_id='envs', scale=12)
gr.Checkbox(elem_id='dry_run', value=False, scale=4)
submit = gr.Button(elem_id='submit', scale=4, variant='primary')
Tuner.build_ui(base_tab)
Optimizer.build_ui(base_tab)
Task.build_ui(base_tab)
with gr.Accordion(elem_id='extra_params', open=False):
with gr.Tabs():
Advanced.build_ui(base_tab)
Quantization.build_ui(base_tab)
SelfCog.build_ui(base_tab)
Save.build_ui(base_tab)
ReportTo.build_ui(base_tab)
with gr.Row():
gr.Textbox(elem_id='more_params', lines=4, scale=20)
cls.element('tuner_type').change(
Hyper.update_lr, inputs=[base_tab.element('tuner_type')], outputs=[cls.element('learning_rate')])
submit.click(cls.train_local, list(cls.valid_elements().values()), [
cls.element('running_cmd'),
cls.element('logging_dir'),
cls.element('runtime_tab'),
cls.element('running_tasks'),
cls.element('train_record'),
])
base_tab.element('gpu_id').change(
cls.update_ddp_num,
[base_tab.element('gpu_id'), base_tab.element('use_ddp')], base_tab.element('ddp_num'))
base_tab.element('use_ddp').change(
cls.update_ddp_num,
[base_tab.element('gpu_id'), base_tab.element('use_ddp')], base_tab.element('ddp_num'))
base_tab.element('running_tasks').change(
partial(Runtime.task_changed, base_tab=base_tab), [base_tab.element('running_tasks')],
list(base_tab.valid_elements().values()) + [cls.element('log')] + Runtime.all_plots)
Runtime.element('kill_task').click(
Runtime.kill_task,
[Runtime.element('running_tasks')],
[Runtime.element('running_tasks')] + [Runtime.element('log')] + Runtime.all_plots,
).then(Runtime.reset, [], [Runtime.element('logging_dir')] + [Hyper.element('output_dir')])
@classmethod
def update_runtime(cls):
return gr.update(open=True), gr.update(visible=True)
@classmethod
def train(cls, *args):
ignore_elements = ('logging_dir', 'more_params', 'train_stage', 'envs')
default_args = cls.get_default_value_from_dataclass(RLHFArguments)
extra_default_args = cls.get_default_value_from_dataclass(ExportArguments)
kwargs = {}
kwargs_is_list = {}
other_kwargs = {}
more_params = {}
more_params_cmd = ''
keys = cls.valid_element_keys()
if cls.group in ('llm_grpo', 'llm_rlhf'):
train_stage = 'rlhf'
else:
train_stage = 'sft'
for key, value in zip(keys, args):
compare_value = default_args.get(key) if key != 'hub_private_repo' else extra_default_args.get(key)
if isinstance(value, str) and re.fullmatch(cls.int_regex, value):
value = int(value)
elif isinstance(value, str) and re.fullmatch(cls.float_regex, value):
value = float(value)
elif isinstance(value, str) and re.fullmatch(cls.bool_regex, value):
value = True if value.lower() == 'true' else False
if compare_value in ('true', 'false'):
value = str(value).lower()
if key not in ignore_elements and key in default_args and compare_value != value and (value or value
in (0, False)):
kwargs[key] = value if not isinstance(value, list) else ' '.join(value)
kwargs_is_list[key] = isinstance(value, list) or getattr(cls.element(key), 'is_list', False)
else:
other_kwargs[key] = value
if key == 'more_params' and value:
try:
more_params = json.loads(value)
except (JSONDecodeError or TypeError):
more_params_cmd = value
if key == 'train_stage':
train_stage = value
model = kwargs.get('model')
if '-merged' not in model and os.path.exists(model) and os.path.exists(os.path.join(model, 'args.json')):
ckpt_dir = kwargs.pop('model')
with open(os.path.join(ckpt_dir, 'args.json'), 'r', encoding='utf-8') as f:
_json = json.load(f)
kwargs['model'] = _json['model_dir']
kwargs['model_type'] = _json['model_type']
kwargs['template'] = _json['template']
if os.path.exists(os.path.join(ckpt_dir, 'scheduler.pt')):
kwargs['resume_from_checkpoint'] = ckpt_dir
gr.Info(cls.locale('resume_checkpoint_alert', cls.lang)['value'].format(ckpt_dir))
else:
kwargs['resume_from_checkpoint'] = ckpt_dir
kwargs['resume_only_model'] = True
gr.Info(cls.locale('resume_only_model_alert', cls.lang)['value'].format(ckpt_dir))
model = kwargs.get('model')
kwargs.update(more_params)
if 'dataset' not in kwargs and 'custom_train_dataset_path' not in kwargs:
raise gr.Error(cls.locale('dataset_alert', cls.lang)['value'])
cmd = train_stage
if kwargs.get('deepspeed'):
more_params_cmd += f' --deepspeed {kwargs.pop("deepspeed")} '
use_liger_kernel = kwargs.get('use_liger_kernel', None)
if use_liger_kernel:
kwargs.pop('use_liger_kernel')
if other_kwargs.get('use_muon'):
kwargs['use_muon'] = other_kwargs.pop('use_muon')
# filter kwargs
tabs_relation_dict = cls.prepare_sub_to_filter()
cls.remove_useless_args(kwargs, tabs_relation_dict)
use_muon = kwargs.pop('use_muon', None)
if cls.group == 'llm_rlhf':
cls.filter_rlhf_args(kwargs)
try:
sft_args = RLHFArguments(
**{
key: value.split(' ') if kwargs_is_list.get(key, False) and isinstance(value, str) else value
for key, value in kwargs.items()
})
except Exception as e:
raise e
params = ''
command = ['swift', cmd]
if cls.group == 'llm_grpo' and sys.platform != 'win32':
params += f'--rlhf_type {cls.quote}grpo{cls.quote} '
command.extend(['--rlhf_type', 'grpo'])
sep = f'{cls.quote} {cls.quote}'
for e in kwargs:
if isinstance(kwargs[e], list):
params += f'--{e} {cls.quote}{sep.join(kwargs[e])}{cls.quote} '
command.extend([f'--{e}'] + kwargs[e])
elif e in kwargs_is_list and kwargs_is_list[e]:
all_args = [arg for arg in kwargs[e].split(' ') if arg.strip()]
params += f'--{e} {cls.quote}{sep.join(all_args)}{cls.quote} '
command.extend([f'--{e}'] + all_args)
else:
params += f'--{e} {cls.quote}{kwargs[e]}{cls.quote} '
command.extend([f'--{e}', f'{kwargs[e]}'])
if use_liger_kernel:
params += f'--use_liger_kernel {cls.quote}{use_liger_kernel}{cls.quote} '
command.extend(['--use_liger_kernel', f'{use_liger_kernel}'])
if use_muon:
params += f'--optimizer {cls.quote}muon{cls.quote} '
command.extend(['--optimizer', 'muon'])
more_params_cmd = more_params_cmd.strip()
if more_params_cmd != '':
params += f'{more_params_cmd} '
more_params_cmd = [param.strip() for param in more_params_cmd.split('--')]
more_params_cmd = [param.split(' ') for param in more_params_cmd if param]
for param in more_params_cmd:
command.extend([f'--{param[0]}'] + param[1:])
params += f'--add_version False --output_dir {sft_args.output_dir} ' \
f'--logging_dir {sft_args.logging_dir} --ignore_args_error True'
command.extend([
'--add_version', 'False', '--output_dir', f'{sft_args.output_dir}', '--logging_dir',
f'{sft_args.logging_dir}', '--ignore_args_error', 'True'
])
all_envs = {}
ddp_param = ''
devices = other_kwargs['gpu_id']
envs = other_kwargs['envs'] or ''
envs = envs.strip()
devices = [d for d in devices if d]
if other_kwargs['use_ddp']:
assert int(other_kwargs['ddp_num']) > 0
ddp_param = f'NPROC_PER_NODE={int(other_kwargs["ddp_num"])}'
all_envs['NPROC_PER_NODE'] = str(other_kwargs['ddp_num'])
assert (len(devices) == 1 or 'cpu' not in devices)
gpus = ','.join(devices)
cuda_param = ''
if gpus != 'cpu':
if is_torch_npu_available():
cuda_param = f'ASCEND_RT_VISIBLE_DEVICES={gpus}'
all_envs['ASCEND_RT_VISIBLE_DEVICES'] = gpus
elif is_torch_cuda_available():
cuda_param = f'CUDA_VISIBLE_DEVICES={gpus}'
all_envs['CUDA_VISIBLE_DEVICES'] = gpus
else:
cuda_param = ''
if envs:
env_list = envs.split(' ')
for env in env_list:
k, v = env.split('=')
all_envs[k] = v
log_file = os.path.join(sft_args.logging_dir, 'run.log')
if sys.platform == 'win32':
if cuda_param:
cuda_param = f'set {cuda_param} && '
if ddp_param:
ddp_param = f'set {ddp_param} && '
if envs:
envs = [env.strip() for env in envs.split(' ') if env.strip()]
_envs = ''
for env in envs:
_envs += f'set {env} && '
envs = _envs
run_command = f'{cuda_param}{ddp_param}{envs}start /b swift sft {params} > {log_file} 2>&1'
else:
run_command = f'{cuda_param} {ddp_param} {envs} nohup swift {cmd} {params} > {log_file} 2>&1 &'
logger.info(f'Run training: {run_command}')
if model:
record = {}
for key, value in zip(keys, args):
if key in default_args or key in ('more_params', 'train_stage', 'use_ddp', 'ddp_num', 'gpu_id', 'envs'):
record[key] = value or None
cls.save_cache(model, record)
return command, all_envs, log_file, run_command, sft_args, other_kwargs
@classmethod
def train_studio(cls, *args):
command, all_envs, log_file, run_command, sft_args, other_kwargs = cls.train(*args)
if not other_kwargs['dry_run']:
lines = collections.deque(maxlen=int(os.environ.get('MAX_LOG_LINES', 50)))
env = deepcopy(os.environ)
if len(all_envs) > 0:
for k, v in all_envs.items():
env[k] = v
process = Popen(command, env=env, stdout=PIPE, stderr=STDOUT)
with process.stdout:
for line in iter(process.stdout.readline, b''):
line = line.decode('utf-8')
lines.append(line)
yield ['\n'.join(lines)] + Runtime.plot(run_command) + [run_command]
else:
yield [
'Current is dryrun mode so you can only view the training cmd, please duplicate this space to '
'do training or use with inference.'
] + [None] * len(Runtime.sft_plot) + [run_command]
@classmethod
def train_local(cls, *args):
command, all_envs, log_file, run_command, sft_args, other_kwargs = cls.train(*args)
if cls.group == 'llm_grpo' and sft_args.vllm_mode == 'server':
host = sft_args.vllm_server_host if sft_args.vllm_server_host else '127.0.0.1'
port = sft_args.vllm_server_port if sft_args.vllm_server_port else '8000'
try:
import requests
headers = {'Accept': 'application/json'}
url = f'http://{host}:{port}/health/'
response = requests.get(url, headers=headers)
res = response.json()
assert res['status'] == 'ok', 'statue must be ok'
except Exception as err:
gr.Info(cls.locale('external_alert', cls.lang)['value'].format(err))
return [None] * 2 + [gr.update(open=False)] + [None] * 2
if not other_kwargs['dry_run']:
os.makedirs(sft_args.logging_dir, exist_ok=True)
run_command_in_background_with_popen(command, all_envs, log_file)
time.sleep(1) # to make sure the log file has been created.
gr.Info(cls.locale('submit_alert', cls.lang)['value'])
return run_command, sft_args.logging_dir, gr.update(open=True), Runtime.refresh_tasks(
sft_args.output_dir, cls.group), gr.update(choices=cls.list_cache(sft_args.model))
@classmethod
def prepare_sub_to_filter(cls):
tabs_relation_dict = {
key: val
for key, val in zip(['tuner_type', 'optimizer', 'task_type'],
[Tuner.tabs_to_filter, Optimizer.tabs_to_filter, Task.tabs_to_filter])
}
return tabs_relation_dict
@classmethod
def remove_useless_args(cls, uncleaned_kwargs, tabs_relation_dict):
for target, tabs_to_filter in tabs_relation_dict.items():
target_value = uncleaned_kwargs.get(target)
if target == 'tuner_type' and target_value is None:
target_value = 'lora'
elif target == 'vllm_mode' and target_value is None:
target_value = 'colocate'
elif target == 'optimizer':
if uncleaned_kwargs.get('use_galore'):
target_value = 'galore'
if uncleaned_kwargs.get('lorap_lr_ratio'):
target_value = 'lorap'
if uncleaned_kwargs.get('vit_lr') or uncleaned_kwargs.get('aligner_lr'):
target_value = 'multimodal'
if uncleaned_kwargs.get('use_muon'):
target_value = 'muon'
for tab_key in tabs_to_filter.keys():
if tab_key == 'lora' and target_value in ('longlora', 'adalora'):
continue
if tab_key == 'lisa' and target_value == 'full' and uncleaned_kwargs.get('lisa_activated_layers'):
continue
if tab_key == 'lora_ga' and target_value == 'lora' and uncleaned_kwargs.get(
'init_weights') == 'lora-ga':
continue
if tab_key != target_value:
for arg in tabs_to_filter[tab_key]:
if uncleaned_kwargs.get(arg) is not None:
uncleaned_kwargs.pop(arg)
swift/ui/llm_train/lora.py
+68
View File
@@ -0,0 +1,68 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class LoRA(BaseUI):
group = 'llm_train'
locale_dict = {
'lora_tab': {
'label': {
'zh': 'LoRA参数设置',
'en': 'LoRA settings'
},
},
'lora_rank': {
'label': {
'zh': 'LoRA的秩',
'en': 'The LoRA rank'
}
},
'lora_alpha': {
'label': {
'zh': 'LoRA的缩放因子',
'en': 'The LoRA alpha'
}
},
'lora_dropout': {
'label': {
'zh': 'LoRA的丢弃概率',
'en': 'The LoRA dropout'
}
},
'use_rslora': {
'label': {
'zh': '使用rsLoRA',
'en': 'Use rsLoRA'
}
},
'use_dora': {
'label': {
'zh': '使用DoRA',
'en': 'Use DoRA'
}
},
'lora_dtype': {
'label': {
'zh': 'LoRA部分的参数类型',
'en': 'The dtype of LoRA'
}
},
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.TabItem(elem_id='lora_tab'):
with gr.Blocks():
with gr.Row():
gr.Slider(elem_id='lora_rank', value=8, minimum=1, maximum=512, step=8, scale=2)
gr.Slider(elem_id='lora_alpha', value=32, minimum=1, maximum=512, step=8, scale=2)
gr.Textbox(elem_id='lora_dropout', scale=2)
with gr.Row():
gr.Dropdown(elem_id='lora_dtype', scale=2, value=None)
gr.Checkbox(elem_id='use_rslora', scale=2)
gr.Checkbox(elem_id='use_dora', scale=2)
swift/ui/llm_train/model.py
+127
View File
@@ -0,0 +1,127 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from functools import partial
from typing import Type
from swift.arguments import RLHFArguments
from swift.model import ModelType, get_model_list
from swift.template import TEMPLATE_MAPPING
from ..base import BaseUI
class Model(BaseUI):
group = 'llm_train'
locale_dict = {
'model_type': {
'label': {
'zh': '模型类型',
'en': 'Select Model Type'
},
'info': {
'zh': 'SWIFT已支持的模型类型',
'en': 'Base model type supported by SWIFT'
}
},
'model': {
'label': {
'zh': '模型id或路径',
'en': 'Model id or path'
},
'info': {
'zh': '实际的模型id',
'en': 'The actual model id or model path'
}
},
'template': {
'label': {
'zh': '模型Prompt模板类型',
'en': 'Prompt template type'
},
'info': {
'zh': '选择匹配模型的Prompt模板',
'en': 'Choose the template type of the model'
}
},
'system': {
'label': {
'zh': 'System字段',
'en': 'System'
},
'info': {
'zh': '选择system字段的内容',
'en': 'Choose the content of the system field'
}
},
'reset': {
'value': {
'zh': '恢复模型初始值',
'en': 'Reset model default'
},
},
'train_record': {
'label': {
'zh': '训练记录',
'en': 'Train record'
},
'info': {
'zh': '展示使用web-ui的历史训练及参数',
'en': 'Show the training history and parameters'
}
},
'clear_cache': {
'value': {
'zh': '删除训练记录',
'en': 'Delete train records'
},
},
'model_param': {
'label': {
'zh': '模型设置',
'en': 'Model settings'
},
},
'checkpoint': {
'value': {
'zh': '训练后的模型',
'en': 'Trained model'
}
},
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.Accordion(elem_id='model_param', open=True):
with gr.Row(equal_height=True):
model = gr.Dropdown(
elem_id='model',
scale=20,
choices=get_model_list(),
value='Qwen/Qwen2.5-7B-Instruct',
allow_custom_value=True)
gr.Dropdown(elem_id='model_type', choices=ModelType.get_model_name_list(), scale=20)
gr.Dropdown(elem_id='template', choices=list(TEMPLATE_MAPPING.keys()), scale=20)
train_record = gr.Dropdown(elem_id='train_record', choices=[], scale=20)
clear_cache = gr.Button(elem_id='clear_cache', scale=2)
with gr.Row():
gr.Textbox(elem_id='system', lines=4 if cls.group == 'llm_grpo' else 1, scale=20)
def clear_record(model):
if model:
cls.clear_cache(model)
return gr.update(choices=[])
return gr.update()
clear_cache.click(clear_record, inputs=[model], outputs=[train_record])
@classmethod
def after_build_ui(cls, base_tab: Type['BaseUI']):
cls.element('model').change(
partial(base_tab.update_input_model, arg_cls=RLHFArguments),
inputs=[cls.element('model')],
outputs=[cls.element('train_record')] + list(base_tab.valid_elements().values()))
cls.element('train_record').change(
partial(base_tab.update_all_settings, base_tab=base_tab),
inputs=[cls.element('model'), cls.element('train_record')],
outputs=list(base_tab.valid_elements().values()))
swift/ui/llm_train/optimizer.py
+139
View File
@@ -0,0 +1,139 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class Optimizer(BaseUI):
group = 'llm_train'
locale_dict = {
'galore_tab': {
'label': {
'zh': 'GaLore参数设置',
'en': 'GaLore Settings'
},
},
'use_galore': {
'label': {
'zh': '使用GaLore',
'en': 'Use GaLore'
},
'info': {
'zh': '使用GaLore来减少全参数训练的显存消耗',
'en': 'Use GaLore to reduce GPU memory usage in full parameter training'
}
},
'galore_rank': {
'label': {
'zh': 'GaLore的秩',
'en': 'The rank of GaLore'
},
},
'galore_update_proj_gap': {
'label': {
'zh': '投影矩阵更新间隔',
'en': 'Projection matrix update interval'
},
'info': {
'zh': 'GaLore分解矩阵的更新间隔',
'en': 'Update interval of GaLore decomposition matrix'
},
},
'galore_with_embedding': {
'label': {
'zh': '对嵌入层应用GaLore',
'en': 'Use GaLore with embedding'
},
'info': {
'zh': '是否对嵌入层应用GaLore',
'en': 'Whether to apply GaLore to embedding'
},
},
'lorap_tab': {
'label': {
'zh': 'LoRA+参数设置',
'en': 'LoRA+ settings'
},
},
'lorap_lr_ratio': {
'label': {
'zh': 'LoRA+学习率比率',
'en': 'LoRA+ lr ratio'
},
'info': {
'zh': '使用LoRA时指定该参数可使用LoRA+，建议值10～16',
'en': 'When using LoRA, specify this parameter to use LoRA+, and the recommended value is 10 to 16'
}
},
'muon_tab': {
'label': {
'zh': 'Muon参数设置',
'en': 'Muon Settings'
},
},
'use_muon': {
'label': {
'zh': '使用Muon',
'en': 'Use Muon'
},
'info': {
'zh': '使用Muon优化器，将在命令行参数中设置`--optimizer muon`',
'en': 'Using the Muon optimizer, set `--optimizer muon` in the command line'
}
},
'multimodal_tab': {
'label': {
'zh': '多模态参数设置',
'en': 'Multimodal Settings'
},
},
'vit_lr': {
'label': {
'zh': 'ViT的学习率',
'en': 'Learning rate of ViT'
},
},
'aligner_lr': {
'label': {
'zh': 'Aligner的学习率',
'en': 'Learning rate of aligner'
},
},
'optimizer_params': {
'label': {
'zh': '优化器参数',
'en': 'Optimizer params'
},
},
}
tabs_to_filter = {
'galore': ['use_galore', 'galore_with_embedding', 'galore_rank', 'galore_update_proj_gap'],
'lorap': ['lorap_lr_ratio'],
'multimodal': ['vit_lr', 'aligner_lr'],
'muon': ['use_muon']
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.Accordion(elem_id='optimizer_params', open=False):
with gr.Tabs():
with gr.TabItem(elem_id='galore_tab'):
with gr.Row():
gr.Checkbox(elem_id='use_galore', scale=4)
gr.Checkbox(elem_id='galore_with_embedding', scale=4)
gr.Slider(elem_id='galore_rank', minimum=8, maximum=256, step=8, scale=4)
gr.Slider(elem_id='galore_update_proj_gap', minimum=10, maximum=1000, step=50, scale=4)
with gr.TabItem(elem_id='lorap_tab'):
with gr.Row():
gr.Textbox(elem_id='lorap_lr_ratio', scale=4)
with gr.TabItem(elem_id='multimodal_tab'):
with gr.Row():
gr.Textbox(elem_id='vit_lr', lines=1, scale=20)
gr.Textbox(elem_id='aligner_lr', lines=1, scale=20)
with gr.TabItem(elem_id='muon_tab'):
with gr.Row():
gr.Checkbox(elem_id='use_muon', scale=4)
swift/ui/llm_train/quantization.py
+67
View File
@@ -0,0 +1,67 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class Quantization(BaseUI):
group = 'llm_train'
locale_dict = {
'quantization_tab': {
'label': {
'zh': '量化参数设置',
'en': 'Quantization settings'
},
},
'quant_method': {
'label': {
'zh': '量化方式',
'en': 'Quantization method'
},
'info': {
'zh': '如果制定了量化位数，本参数默认为bnb',
'en': 'Default is bnb if quantization_bit is specified'
}
},
'quant_bits': {
'label': {
'zh': '量化bit数',
'en': 'Quantization bit'
},
'info': {
'zh': '设置量化bit数, 0代表不进行量化',
'en': 'Set the quantization bit, 0 for no quantization'
}
},
'bnb_4bit_compute_dtype': {
'label': {
'zh': '计算数据类型',
'en': 'Computational data type'
},
},
'bnb_4bit_quant_type': {
'label': {
'zh': '量化数据类型',
'en': 'Quantization data type'
},
},
'bnb_4bit_use_double_quant': {
'label': {
'zh': '使用嵌套量化',
'en': 'Use double quantization'
},
},
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.TabItem(elem_id='quantization_tab'):
with gr.Row():
gr.Dropdown(elem_id='quant_bits', value=None)
gr.Dropdown(elem_id='quant_method', value=None)
gr.Dropdown(elem_id='bnb_4bit_compute_dtype', value=None)
gr.Dropdown(elem_id='bnb_4bit_quant_type', value=None)
gr.Checkbox(elem_id='bnb_4bit_use_double_quant', value=None)
swift/ui/llm_train/report_to.py
+74
View File
@@ -0,0 +1,74 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class ReportTo(BaseUI):
group = 'llm_train'
locale_dict = {
'reporter_tab': {
'label': {
'zh': '训练记录',
'en': 'Training report'
},
},
'report_to': {
'label': {
'zh': '训练记录方式',
'en': 'Report to'
},
},
'swanlab_token': {
'label': {
'zh': 'SwanLab登录token',
'en': 'The login token of SwanLab'
},
},
'swanlab_project': {
'label': {
'zh': 'SwanLab项目名称',
'en': 'Project of SwanLab'
},
},
'swanlab_workspace': {
'label': {
'zh': 'SwanLab工作空间',
'en': 'Workspace of SwanLab'
},
},
'swanlab_exp_name': {
'label': {
'zh': 'SwanLab实验名称',
'en': 'Experiment of SwanLab'
},
},
'swanlab_mode': {
'label': {
'zh': 'SwanLab工作模式',
'en': 'Work mode of SwanLab'
},
},
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.TabItem(elem_id='reporter_tab'):
with gr.Blocks():
with gr.Row():
gr.Dropdown(
elem_id='report_to',
multiselect=True,
is_list=True,
choices=['tensorboard', 'wandb', 'swanlab'],
allow_custom_value=True,
scale=20)
gr.Textbox(elem_id='swanlab_token', lines=1, scale=20)
gr.Textbox(elem_id='swanlab_project', lines=1, scale=20)
with gr.Row():
gr.Textbox(elem_id='swanlab_workspace', lines=1, scale=20)
gr.Textbox(elem_id='swanlab_exp_name', lines=1, scale=20)
gr.Dropdown(elem_id='swanlab_mode', scale=20)
swift/ui/llm_train/runtime.py
+717
View File
@@ -0,0 +1,717 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import collections
import gradio as gr
import json
import matplotlib.pyplot as plt
import os
import psutil
import re
import subprocess
import sys
import time
import webbrowser
from datetime import datetime
from functools import partial
from packaging import version
from transformers import is_tensorboard_available
from typing import Dict, List, Tuple, Type
from swift.utils import TB_COLOR, TB_COLOR_SMOOTH, format_time, get_logger, read_tensorboard_file, tensorboard_smoothing
from ..base import BaseUI
from .utils import close_loop, run_command_in_subprocess
logger = get_logger()
class Runtime(BaseUI):
handlers: Dict[str, Tuple[List, Tuple]] = {}
group = 'llm_train'
all_plots = None
log_event = {}
sft_plot = [
{
'name': 'train/loss',
'smooth': 0.9,
},
{
'name': 'train/acc',
'smooth': None,
},
{
'name': 'train/learning_rate',
'smooth': None,
},
{
'name': 'eval/loss',
'smooth': 0.9,
},
{
'name': 'eval/acc',
'smooth': None,
},
]
dpo_plot = [
{
'name': 'train/loss',
'smooth': 0.9,
},
{
'name': 'train/rewards/accuracies',
'smooth': None,
},
{
'name': 'train/rewards/margins',
'smooth': 0.9,
},
{
'name': 'train/logps/chosen',
'smooth': 0.9,
},
{
'name': 'train/logps/rejected',
'smooth': 0.9,
},
]
kto_plot = [
{
'name': 'kl',
'smooth': None,
},
{
'name': 'rewards/chosen_sum',
'smooth': 0.9,
},
{
'name': 'logps/chosen_sum',
'smooth': 0.9,
},
{
'name': 'rewards/rejected_sum',
'smooth': 0.9,
},
{
'name': 'logps/rejected_sum',
'smooth': 0.9,
},
]
orpo_plot = [
{
'name': 'train/loss',
'smooth': 0.9,
},
{
'name': 'train/rewards/accuracies',
'smooth': None,
},
{
'name': 'train/rewards/margins',
'smooth': 0.9,
},
{
'name': 'train/rewards/chosen',
'smooth': 0.9,
},
{
'name': 'train/log_odds_ratio',
'smooth': 0.9,
},
]
grpo_plot = [
{
'name': 'train/loss',
'smooth': 0.9,
},
{
'name': 'train/reward',
'smooth': 0.9,
},
{
'name': 'train/learning_rate',
'smooth': None,
},
{
'name': 'train/completions/mean_length',
'smooth': 0.9,
},
{
'name': 'train/kl',
'smooth': 0.9,
},
]
locale_dict = {
'runtime_tab': {
'label': {
'zh': '运行时',
'en': 'Runtime'
},
},
'tb_not_found': {
'value': {
'zh': 'TensorBoard未安装,使用`pip install tensorboard`进行安装',
'en': 'TensorBoard not found, install it by `pip install tensorboard`',
}
},
'running_cmd': {
'label': {
'zh': '运行命令',
'en': 'Command line'
},
'info': {
'zh': '执行的实际命令',
'en': 'The actual command'
}
},
'show_running_cmd': {
'value': {
'zh': '展示运行命令',
'en': 'Show running command line'
},
},
'show_sh': {
'label': {
'zh': '展示sh命令行',
'en': 'Show sh command line'
},
},
'cmd_sh': {
'label': {
'zh': '训练命令行',
'en': 'Training command line'
},
'info': {
'zh':
'如果训练命令行没有展示请再次点击"展示运行命令"，点击下方的"保存训练命令"可以保存sh脚本',
'en': ('Please press "Show running command line" if the content is none, '
'click the "Save training command" below to save the sh script')
}
},
'save_cmd_as_sh': {
'value': {
'zh': '保存训练命令',
'en': 'Save training command'
}
},
'save_cmd_alert': {
'value': {
'zh': '训练命令行将被保存在：{}',
'en': 'The training command line will be saved in: {}'
}
},
'close_cmd_show': {
'value': {
'zh': '关闭训练命令展示',
'en': 'Close training command show'
}
},
'show_log': {
'value': {
'zh': '展示运行状态',
'en': 'Show running status'
},
},
'stop_show_log': {
'value': {
'zh': '停止展示运行状态',
'en': 'Stop showing running status'
},
},
'logging_dir': {
'label': {
'zh': '日志路径',
'en': 'Logging dir'
},
'info': {
'zh': '支持手动传入文件路径',
'en': 'Support fill custom path in'
}
},
'log': {
'label': {
'zh': '日志输出',
'en': 'Logging content'
},
'info': {
'zh': '如果日志无更新请再次点击"展示运行状态"',
'en': 'Please press "Show running status" if the log content is not updating'
}
},
'running_tasks': {
'label': {
'zh': '运行中任务',
'en': 'Running Tasks'
},
'info': {
'zh': '运行中的任务（所有的swift sft/pt命令）',
'en': 'All running tasks(started by swift sft/pt)'
}
},
'refresh_tasks': {
'value': {
'zh': '找回运行时任务',
'en': 'Find running tasks'
},
},
'kill_task': {
'value': {
'zh': '杀死任务',
'en': 'Kill running task'
},
},
'tb_url': {
'label': {
'zh': 'Tensorboard链接',
'en': 'Tensorboard URL'
},
'info': {
'zh': '仅展示，不可编辑',
'en': 'Not editable'
}
},
'start_tb': {
'value': {
'zh': '打开TensorBoard',
'en': 'Start TensorBoard'
},
},
'close_tb': {
'value': {
'zh': '关闭TensorBoard',
'en': 'Close TensorBoard'
},
},
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.Accordion(elem_id='runtime_tab', open=False):
with gr.Blocks():
with gr.Row():
with gr.Column(scale=3):
with gr.Row():
gr.Textbox(elem_id='running_cmd', lines=1, scale=3, interactive=False, max_lines=1)
gr.Textbox(elem_id='logging_dir', lines=1, scale=3, max_lines=1)
with gr.Row():
gr.Button(elem_id='show_running_cmd', scale=2, variant='primary')
gr.Button(elem_id='show_log', scale=2, variant='primary')
gr.Button(elem_id='stop_show_log', scale=2)
with gr.Column(scale=2):
with gr.Row():
gr.Textbox(elem_id='tb_url', lines=1, scale=4, interactive=False, max_lines=1)
with gr.Row():
gr.Button(elem_id='start_tb', scale=2, variant='primary')
gr.Button(elem_id='close_tb', scale=2)
with gr.Accordion(elem_id='show_sh', open=False, visible=False):
with gr.Blocks():
gr.Textbox(elem_id='cmd_sh', lines=8)
with gr.Row(equal_height=True):
gr.Button(elem_id='save_cmd_as_sh', variant='primary', scale=2)
gr.Button(elem_id='close_cmd_show', scale=2)
with gr.Row():
gr.Textbox(elem_id='log', lines=6, visible=False)
with gr.Row(equal_height=True):
gr.Dropdown(elem_id='running_tasks', scale=10)
gr.Button(elem_id='refresh_tasks', scale=1)
gr.Button(elem_id='kill_task', scale=1)
with gr.Row():
cls.all_plots = []
plot = Runtime.sft_plot
if base_tab.group == 'llm_rlhf':
plot = Runtime.dpo_plot
elif base_tab.group == 'llm_grpo':
plot = Runtime.grpo_plot
for idx, k in enumerate(plot):
name = k['name']
cls.all_plots.append(gr.Plot(elem_id=str(idx), label=name))
concurrency_limit = {}
if version.parse(gr.__version__) >= version.parse('4.0.0'):
concurrency_limit = {'concurrency_limit': 5}
base_tab.element('show_log').click(
Runtime.update_log, [base_tab.element('running_tasks')], [cls.element('log')] + cls.all_plots).then(
Runtime.wait, [base_tab.element('logging_dir'),
base_tab.element('running_tasks')], [cls.element('log')] + cls.all_plots,
**concurrency_limit)
base_tab.element('stop_show_log').click(cls.break_log_event, [cls.element('running_tasks')], [])
base_tab.element('start_tb').click(
Runtime.start_tb,
[base_tab.element('logging_dir')],
[base_tab.element('tb_url')],
)
base_tab.element('close_tb').click(
Runtime.close_tb,
[base_tab.element('logging_dir')],
[],
)
base_tab.element('refresh_tasks').click(
partial(Runtime.refresh_tasks, group=cls.group),
[base_tab.element('running_tasks')],
[base_tab.element('running_tasks')],
)
@classmethod
def after_build_ui(cls, base_tab: Type['BaseUI']):
cls.element('show_running_cmd').click(Runtime.show_train_sh, cls.element('running_cmd'),
[cls.element('show_sh')] + [cls.element('cmd_sh')])
cls.element('save_cmd_as_sh').click(cls.save_cmd, cls.element('running_cmd'), [])
cls.element('close_cmd_show').click(Runtime.close_cmd_show, [], [cls.element('show_sh')])
@classmethod
def get_plot(cls, task):
if not task or 'swift sft' in task or 'swift pt' in task:
return cls.sft_plot
args: dict = cls.parse_info_from_cmdline(task)[1]
rlhf_type = args.get('rlhf_type', 'dpo')
if rlhf_type in ('dpo', 'cpo', 'simpo'):
return cls.dpo_plot
elif rlhf_type == 'kto':
return cls.kto_plot
elif rlhf_type == 'orpo':
return cls.orpo_plot
elif rlhf_type == 'grpo':
return cls.grpo_plot
@classmethod
def update_log(cls, task):
ret = [gr.update(visible=True)]
plot = Runtime.get_plot(task)
for i in range(len(plot)):
p = plot[i]
ret.append(gr.update(visible=True, label=p['name']))
return ret
@classmethod
def get_initial(cls, line):
tqdm_starts = ['Train:', 'Map:', 'Val:', 'Filter:']
for start in tqdm_starts:
if line.startswith(start):
return start
return None
@classmethod
def wait(cls, logging_dir, task):
if not logging_dir:
return [None] + Runtime.plot(task)
log_file = os.path.join(logging_dir, 'run.log')
cls.log_event[logging_dir] = False
offset = 0
latest_data = ''
lines = collections.deque(maxlen=int(os.environ.get('MAX_LOG_LINES', 100)))
try:
with open(log_file, 'r', encoding='utf-8') as input:
input.seek(offset)
fail_cnt = 0
while True:
try:
latest_data += input.read()
except UnicodeDecodeError:
continue
if not latest_data:
time.sleep(0.5)
fail_cnt += 1
if fail_cnt > 50:
break
if cls.log_event.get(logging_dir, False):
cls.log_event[logging_dir] = False
break
if '\n' not in latest_data:
continue
latest_lines = latest_data.split('\n')
if latest_data[-1] != '\n':
latest_data = latest_lines[-1]
latest_lines = latest_lines[:-1]
else:
latest_data = ''
lines.extend(latest_lines)
start = cls.get_initial(lines[-1])
if start:
i = len(lines) - 2
while i >= 0:
if lines[i].startswith(start):
del lines[i]
i -= 1
else:
break
yield [gr.update(value='\n'.join(lines))] + Runtime.plot(task)
time.sleep(0.5)
except IOError:
pass
@classmethod
def break_log_event(cls, task):
if not task:
return
pid, all_args = Runtime.parse_info_from_cmdline(task)
cls.log_event[all_args['logging_dir']] = True
@classmethod
def show_log(cls, logging_dir):
webbrowser.open('file://' + os.path.join(logging_dir, 'run.log'), new=2)
@classmethod
def start_tb(cls, logging_dir):
if not is_tensorboard_available():
gr.Error(cls.locale('tb_not_found', cls.lang)['value'])
return ''
logging_dir = logging_dir.strip()
logging_dir = logging_dir if not logging_dir.endswith(os.sep) else logging_dir[:-1]
if logging_dir in cls.handlers:
return cls.handlers[logging_dir][1]
handler, lines = run_command_in_subprocess('tensorboard', '--logdir', logging_dir, timeout=2)
localhost_addr = ''
for line in lines:
if 'http://localhost:' in line:
line = line[line.index('http://localhost:'):]
localhost_addr = line[:line.index(' ')]
cls.handlers[logging_dir] = (handler, localhost_addr)
logger.info('===========Tensorboard Log============')
logger.info('\n'.join(lines))
webbrowser.open(localhost_addr, new=2)
return localhost_addr
@staticmethod
def close_tb(logging_dir):
if logging_dir in Runtime.handlers:
close_loop(Runtime.handlers[logging_dir][0])
Runtime.handlers.pop(logging_dir)
@staticmethod
def refresh_tasks(running_task=None, group=None):
output_dir = running_task if not running_task or 'pid:' not in running_task else None
process_name = 'swift'
negative_names = ['swift.exe', 'swift-script.py']
cmd_name = ['pt', 'sft'] if group == 'llm_train' else ['rlhf']
process = []
selected = None
for proc in psutil.process_iter():
try:
cmdlines = proc.cmdline()
except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess):
cmdlines = []
if any([
process_name in cmdline for cmdline in cmdlines # noqa
]) and not any([ # noqa
negative_name in cmdline for negative_name in negative_names # noqa
for cmdline in cmdlines # noqa
]) and any([cmdline in cmd_name for cmdline in cmdlines]): # noqa
if any([group == 'llm_rlhf' and 'grpo' in cmdline for cmdline in cmdlines]):
continue
if group == 'llm_grpo' and all(['grpo' not in cmdline for cmdline in cmdlines]):
continue
process.append(Runtime.construct_running_task(proc))
if output_dir is not None and any( # noqa
[output_dir == cmdline for cmdline in cmdlines]): # noqa
selected = Runtime.construct_running_task(proc)
if not selected:
if running_task and running_task in process:
selected = running_task
if not selected and process:
selected = process[0]
return gr.update(choices=process, value=selected)
@staticmethod
def construct_running_task(proc):
pid = proc.pid
ts = time.time()
create_time = proc.create_time()
create_time_formatted = datetime.fromtimestamp(create_time).strftime('%Y-%m-%d, %H:%M')
return f'pid:{pid}/create:{create_time_formatted}' \
f'/running:{format_time(ts - create_time)}/cmd:{" ".join(proc.cmdline())}'
@staticmethod
def parse_info_from_cmdline(task):
pid = None
if '/cmd:' in task:
for i in range(3):
slash = task.find('/')
if i == 0:
pid = task[:slash].split(':')[1]
task = task[slash + 1:]
if 'swift sft' in task:
args = task.split('swift sft')[1]
elif 'swift pt' in task:
args = task.split('swift pt')[1]
elif 'swift rlhf' in task:
args = task.split('swift rlhf')[1]
else:
raise ValueError(f'Cannot parse cmd line: {task}')
args = [arg.strip() for arg in args.split('--') if arg.strip()]
all_args = {}
for i in range(len(args)):
space = args[i].find(' ')
splits = args[i][:space], args[i][space + 1:]
all_args[splits[0]] = str(splits[1]) if isinstance(splits[1], int) else splits[1]
output_dir = all_args['output_dir']
if os.path.exists(os.path.join(output_dir, 'args.json')):
with open(os.path.join(output_dir, 'args.json'), 'r', encoding='utf-8') as f:
_json = json.load(f)
for key in all_args.keys():
all_args[key] = str(_json.get(key)) if isinstance(_json.get(key), int) else _json.get(key)
if isinstance(all_args[key], list):
if any([' ' in value for value in all_args[key] if isinstance(value, str)]):
all_args[key] = [f'"{value}"' for value in all_args[key]]
if len(all_args[key]) > 0 and isinstance(all_args[key][0], str):
all_args[key] = ' '.join(all_args[key])
return pid, all_args
@staticmethod
def kill_task(task):
if task:
pid, all_args = Runtime.parse_info_from_cmdline(task)
output_dir = all_args['output_dir']
if sys.platform == 'win32':
command = ['taskkill', '/f', '/t', '/pid', pid]
else:
command = ['pkill', '-9', '-f', output_dir]
try:
result = subprocess.run(command, capture_output=True, text=True)
assert result.returncode == 0
except Exception as e:
raise e
Runtime.break_log_event(task)
return [Runtime.refresh_tasks()] + [gr.update(value=None)] * (len(Runtime.get_plot(task)) + 1)
@staticmethod
def reset():
return None, 'output'
@staticmethod
def task_changed(task, base_tab):
if task:
_, all_args = Runtime.parse_info_from_cmdline(task)
else:
all_args = {}
elements = list(base_tab.valid_elements().values())
ret = []
for e in elements:
if e.elem_id in all_args:
if isinstance(e, gr.Dropdown) and e.multiselect:
arg = all_args[e.elem_id].split(' ')
else:
arg = all_args[e.elem_id]
if isinstance(e, gr.Slider) and isinstance(arg, str) and re.fullmatch(base_tab.int_regex, arg):
arg = int(arg)
elif isinstance(e, gr.Slider) and isinstance(arg, str) and re.fullmatch(base_tab.float_regex, arg):
arg = float(arg)
elif isinstance(e, gr.Checkbox) and isinstance(arg, str) and re.fullmatch(base_tab.bool_regex, arg):
arg = True if arg.lower() == 'true' else False
ret.append(gr.update(value=arg))
else:
ret.append(gr.update())
Runtime.break_log_event(task)
return ret + [gr.update(value=None)] * (len(Runtime.get_plot(task)) + 1)
@staticmethod
def plot(task):
plot = Runtime.get_plot(task)
if not task:
return [None] * len(plot)
_, all_args = Runtime.parse_info_from_cmdline(task)
tb_dir = all_args['logging_dir']
if not os.path.exists(tb_dir):
return [None] * len(plot)
fname = [
fname for fname in os.listdir(tb_dir)
if os.path.isfile(os.path.join(tb_dir, fname)) and fname.startswith('events.out')
]
if fname:
fname = fname[0]
else:
return [None] * len(plot)
tb_path = os.path.join(tb_dir, fname)
data = read_tensorboard_file(tb_path)
plots = []
for k in plot:
name = k['name']
smooth = k['smooth']
if name == 'train/acc':
if 'train/token_acc' in data:
name = 'train/token_acc'
if 'train/seq_acc' in data:
name = 'train/seq_acc'
if name == 'eval/acc':
if 'eval/token_acc' in data:
name = 'eval/token_acc'
if 'eval/seq_acc' in data:
name = 'eval/seq_acc'
if name not in data:
plots.append(None)
continue
_data = data[name]
steps = [d['step'] for d in _data]
values = [d['value'] for d in _data]
if len(values) == 0:
continue
plt.close('all')
fig = plt.figure()
ax = fig.add_subplot()
# _, ax = plt.subplots(1, 1, squeeze=True, figsize=(8, 5), dpi=100)
ax.set_title(name)
if len(values) == 1:
ax.scatter(steps, values, color=TB_COLOR_SMOOTH)
elif smooth is not None:
ax.plot(steps, values, color=TB_COLOR)
values_s = tensorboard_smoothing(values, smooth)
ax.plot(steps, values_s, color=TB_COLOR_SMOOTH)
else:
ax.plot(steps, values, color=TB_COLOR_SMOOTH)
plots.append(fig)
return plots
@classmethod
def save_cmd(cls, cmd):
if len(cmd) > 0:
cmd_sh, output_dir = Runtime.cmd_to_sh_format(cmd)
os.makedirs(output_dir, exist_ok=True)
sh_file_path = os.path.join(output_dir, 'train.sh')
gr.Info(cls.locale('save_cmd_alert', cls.lang)['value'].format(sh_file_path))
with open(sh_file_path, 'w', encoding='utf-8') as f:
f.write(cmd_sh)
@staticmethod
def show_train_sh(cmd):
if len(cmd) == 0:
return gr.update(visible=False, open=False), None
cmd_sh, _ = Runtime.cmd_to_sh_format(cmd)
return gr.update(visible=True, open=True), cmd_sh
@staticmethod
def cmd_to_sh_format(cmd):
cmd_sh = ''
params = cmd.split('--')
env_params = params[0].split('nohup')[0].strip()
cmd_sh += (env_params + ' \\\n')
swift_cmd = params[0].split('nohup')[1].strip()
cmd_sh += ('nohup ' + swift_cmd + ' \\\n')
for param in params[1:]:
if param.startswith('output_dir'):
output_dir = param.split(' ')[1].strip()
cmd_sh += ('--' + param.strip() + ' \\\n')
return cmd_sh, output_dir
@staticmethod
def close_cmd_show():
return gr.update(visible=False)
swift/ui/llm_train/save.py
+83
View File
@@ -0,0 +1,83 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class Save(BaseUI):
group = 'llm_train'
locale_dict = {
'save_tab': {
'label': {
'zh': '存储参数设置',
'en': 'Saving settings'
},
},
'push_to_hub': {
'label': {
'zh': '推送魔搭Hub',
'en': 'Push to modelscope hub',
},
'info': {
'zh': '是否推送魔搭的模型库',
'en': 'Whether push the output model to modelscope hub',
}
},
'hub_model_id': {
'label': {
'zh': '魔搭模型id',
'en': 'The model-id in modelscope',
},
'info': {
'zh': '设置魔搭的模型id',
'en': 'Set the model-id of modelscope',
}
},
'hub_private_repo': {
'label': {
'zh': '设置仓库私有',
'en': 'Model is private',
},
'info': {
'zh': '以私有方式推送魔搭hub',
'en': 'Set the model as private',
}
},
'hub_strategy': {
'label': {
'zh': '推送策略',
'en': 'Push strategy',
},
'info': {
'zh': '设置模型推送策略',
'en': 'Set the push strategy',
}
},
'hub_token': {
'label': {
'zh': '仓库token',
'en': 'The hub token',
},
'info': {
'zh': '该token可以在www.modelscope.cn找到',
'en': 'Find the token in www.modelscope.cn',
}
}
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.TabItem(elem_id='save_tab'):
with gr.Blocks():
with gr.Row():
gr.Checkbox(elem_id='push_to_hub', scale=20)
gr.Textbox(elem_id='hub_model_id', lines=1, scale=20)
gr.Checkbox(elem_id='hub_private_repo', scale=20)
gr.Dropdown(
elem_id='hub_strategy',
scale=20,
choices=['end', 'every_save', 'checkpoint', 'all_checkpoints'])
gr.Textbox(elem_id='hub_token', lines=1, scale=20)
swift/ui/llm_train/self_cog.py
+46
View File
@@ -0,0 +1,46 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class SelfCog(BaseUI):
group = 'llm_train'
locale_dict = {
'selfcog_tab': {
'label': {
'zh': '自我认知任务参数设置',
'en': 'Self cognition settings'
},
},
'model_name': {
'label': {
'zh': '模型认知名称',
'en': 'Model name'
},
'info': {
'zh': '设置模型应当认知自己的名字, 格式为:中文名字 英文名字,中间以空格分隔',
'en': 'Set the name of the model think itself of, the format is Chinesename Englishname, split by space'
}
},
'model_author': {
'label': {
'zh': '模型作者',
'en': 'Model author'
},
'info': {
'zh': '设置模型认知的自己的作者, 格式为:中文作者 英文作者,中间以空格分隔',
'en': 'Set the author of the model, the format is Chineseauthor Englishauthor, split by space'
}
},
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.TabItem(elem_id='selfcog_tab'):
with gr.Row():
gr.Textbox(elem_id='model_name', scale=20, is_list=True)
gr.Textbox(elem_id='model_author', scale=20, is_list=True)
swift/ui/llm_train/target.py
+80
View File
@@ -0,0 +1,80 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class Target(BaseUI):
group = 'llm_train'
locale_dict = {
'target_params': {
'label': {
'zh': 'Target模块参数',
'en': 'Tuner modules params'
}
},
'freeze_llm': {
'label': {
'zh': '冻结LLM',
'en': 'Freeze LLM'
},
},
'freeze_aligner': {
'label': {
'zh': '冻结aligner',
'en': 'Freeze aligner'
},
},
'freeze_vit': {
'label': {
'zh': '冻结ViT',
'en': 'Freeze ViT'
},
},
'target_modules': {
'label': {
'zh': '指定tuner模块',
'en': 'Specify the tuner module'
}
},
'target_regex': {
'label': {
'zh': 'Tuner模块regex表达式',
'en': 'Tuner module regex expression'
}
},
'modules_to_save': {
'label': {
'zh': '额外训练和存储的原模型模块',
'en': 'Original model modules to train and save'
}
},
'init_weights': {
'label': {
'zh': 'Tuner初始化方法',
'en': 'Init tuner weights'
},
'info': {
'zh': ('LoRA: gaussian/pissa/pissa_niter_[n]/olora/loftq/lora-ga/true/false,'
'Bone: bat/true/false'),
'en': ('LoRA: gaussian/pissa/pissa_niter_[n]/olora/loftq/lora-ga/true/false,'
'Bone: bat/true/false'),
}
}
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.Blocks():
with gr.Row():
gr.Textbox(elem_id='target_modules', lines=1, value='all-linear', is_list=True, scale=5)
gr.Checkbox(elem_id='freeze_llm', scale=5)
gr.Checkbox(elem_id='freeze_aligner', scale=5)
gr.Checkbox(elem_id='freeze_vit', scale=5)
with gr.Row():
gr.Textbox(elem_id='target_regex', scale=5)
gr.Textbox(elem_id='modules_to_save', scale=5)
gr.Textbox(elem_id='init_weights', scale=5)
swift/ui/llm_train/task.py
+76
View File
@@ -0,0 +1,76 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
class Task(BaseUI):
group = 'llm_train'
locale_dict = {
'embed_tab': {
'label': {
'zh': '文本嵌入',
'en': 'Embedding'
},
},
'loss_type': {
'label': {
'zh': 'Loss类型',
'en': 'Loss type'
}
},
'seq_cls_tab': {
'label': {
'zh': '序列分类',
'en': 'Sequence Classification'
},
},
'num_labels': {
'label': {
'zh': '标签数量',
'en': 'Number of labels'
}
},
'use_chat_template': {
'label': {
'zh': '使用对话模板',
'en': 'use chat template'
},
'info': {
'zh': '使用对话模板或生成模板',
'en': 'Use the chat template or generation template'
}
},
'task_type': {
'label': {
'zh': '任务类型',
'en': 'Task type'
},
},
'task_params': {
'label': {
'zh': '任务参数',
'en': 'Task params'
},
}
}
tabs_to_filter = {'embedding': ['loss_type'], 'seq_cls': ['num_labels', 'use_chat_template']}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.Accordion(elem_id='task_params', open=False):
gr.Dropdown(elem_id='task_type', choices=['causal_lm', 'seq_cls', 'embedding'])
with gr.Tabs():
with gr.TabItem(elem_id='embed_tab'):
with gr.Row():
gr.Dropdown(
elem_id='loss_type',
choices=['cosine_similarity', 'contrastive', 'online_contrastive', 'infonce'])
with gr.TabItem(elem_id='seq_cls_tab'):
with gr.Row():
gr.Textbox(elem_id='num_labels', scale=4)
gr.Checkbox(elem_id='use_chat_template', value=True, scale=4)
swift/ui/llm_train/tuner.py
+366
View File
@@ -0,0 +1,366 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import gradio as gr
from typing import Type
from ..base import BaseUI
from .lora import LoRA
from .target import Target
class Tuner(BaseUI):
group = 'llm_train'
sub_ui = [LoRA, Target]
locale_dict = {
'adalora_tab': {
'label': {
'zh': 'AdaLoRA参数设置',
'en': 'AdaLoRA settings'
},
},
'adalora_target_r': {
'label': {
'zh': 'AdaLoRA的平均秩',
'en': 'Average rank of AdaLoRA'
},
},
'adalora_init_r': {
'label': {
'zh': 'AdaLoRA的初始秩',
'en': 'Initial rank of AdaLoRA'
},
},
'adalora_tinit': {
'label': {
'zh': 'AdaLoRA初始微调预热的步数',
'en': 'Initial fine-tuning warmup steps of AdaLoRA'
},
},
'adalora_tfinal': {
'label': {
'zh': 'AdaLoRA最终微调的步数',
'en': 'Final fine-tuning steps of AdaLoRA'
},
},
'adalora_deltaT': {
'label': {
'zh': 'AdaLoRA两次预算分配间隔',
'en': 'Internval of AdaLoRA two budget allocations'
},
},
'adalora_beta1': {
'label': {
'zh': 'AdaLoRA的EMA参数',
'en': 'AdaLoRA EMA parameters'
},
},
'adalora_beta2': {
'label': {
'zh': 'AdaLoRA的EMA参数',
'en': 'AdaLoRA EMA parameters'
},
},
'adalora_orth_reg_weight': {
'label': {
'zh': 'AdaLoRA的正交正则化参数',
'en': 'Coefficient of AdaLoRA orthogonal regularization'
},
},
'lora_ga_tab': {
'label': {
'zh': 'LoRA-GA参数设置',
'en': 'LoRA-GA settings'
},
},
'lora_ga_batch_size': {
'label': {
'zh': 'LoRA-GA初始化批处理大小',
'en': 'LoRA-GA initialization batch size'
},
},
'lora_ga_iters': {
'label': {
'zh': 'LoRA-GA初始化迭代次数',
'en': 'LoRA-GA initialization iters'
},
},
'lora_ga_max_length': {
'label': {
'zh': 'LoRA-GA初始化最大输入长度',
'en': 'LoRA-GA initialization max length'
},
},
'lora_ga_direction': {
'label': {
'zh': 'LoRA-GA初始化的初始方向',
'en': 'LoRA-GA initialization direction'
},
},
'lora_ga_scale': {
'label': {
'zh': 'LoRA-GA初始化缩放方式',
'en': 'LoRA-GA initialization scaling method'
},
},
'lora_ga_stable_gamma': {
'label': {
'zh': 'Gamma参数值',
'en': 'Gamma value'
},
'info': {
'zh': '当初始化时选择stable缩放时的gamma值',
'en': 'Select the gamma value for stable scaling',
}
},
'longlora': {
'label': {
'zh': 'LongLoRA参数设置',
'en': 'LongLoRA settings'
},
},
'reft_tab': {
'label': {
'zh': 'ReFT参数设置',
'en': 'ReFT settings'
},
},
'reft_layers': {
'label': {
'zh': '应用ReFT的层',
'en': 'ReFT layers'
},
},
'reft_rank': {
'label': {
'zh': 'ReFT矩阵的秩',
'en': 'Rank of the ReFT matrix'
},
},
'reft_intervention_type': {
'label': {
'zh': 'ReFT的类型',
'en': 'ReFT intervention type'
},
},
'vera_tab': {
'label': {
'zh': 'VeRA参数设置',
'en': 'VeRA settings'
},
},
'vera_rank': {
'label': {
'zh': 'VeRA注意力维度',
'en': 'VeRA rank'
},
},
'vera_projection_prng_key': {
'label': {
'zh': 'VeRA PRNG初始化key',
'en': 'VeRA PRNG initialisation key'
},
},
'vera_dropout': {
'label': {
'zh': 'VeRA的丢弃概率',
'en': 'VeRA dropout'
},
},
'vera_d_initial': {
'label': {
'zh': 'VeRA的d矩阵初始值',
'en': 'Initial value of d matrix'
},
},
'boft_tab': {
'label': {
'zh': 'BOFT参数设置',
'en': 'BOFT settings'
},
},
'boft_block_size': {
'label': {
'zh': 'BOFT块大小',
'en': 'BOFT block size'
},
},
'boft_block_num': {
'label': {
'zh': 'BOFT块数量',
'en': 'Number of BOFT blocks'
},
'info': {
'zh': '不能和boft_block_size同时使用',
'en': 'Cannot be used with boft_block_size',
}
},
'boft_dropout': {
'label': {
'zh': 'BOFT丢弃概率',
'en': 'Dropout value of BOFT'
},
},
'fourierft_tab': {
'label': {
'zh': 'FourierFT参数设置',
'en': 'FourierFT settings'
},
},
'fourier_n_frequency': {
'label': {
'zh': 'FourierFT频率数量',
'en': 'Num of FourierFT frequencies'
},
},
'fourier_scaling': {
'label': {
'zh': 'W矩阵缩放值',
'en': 'W matrix scaling value'
},
},
'llamapro_tab': {
'label': {
'zh': 'LLaMA Pro参数设置',
'en': 'LLaMA Pro Settings'
},
},
'llamapro_num_new_blocks': {
'label': {
'zh': 'LLaMA Pro插入层数',
'en': 'LLaMA Pro new layers'
},
},
'llamapro_num_groups': {
'label': {
'zh': 'LLaMA Pro对原模型的分组数',
'en': 'LLaMA Pro groups of model'
}
},
'lisa_tab': {
'label': {
'zh': 'LISA参数设置',
'en': 'LISA settings'
},
},
'lisa_activated_layers': {
'label': {
'zh': 'LISA激活层数',
'en': 'Num of LISA activated layers'
},
'info': {
'zh': 'LISA每次训练的模型层数，调整为正整数代表使用LISA',
'en': 'Num of layers activated each time, a positive value means using LISA'
}
},
'lisa_step_interval': {
'label': {
'zh': 'LISA切换层间隔',
'en': 'The interval of LISA layers switching'
}
},
'tuner_params': {
'label': {
'zh': 'Tuner参数',
'en': 'Tuner params'
}
},
}
tabs_to_filter = {
'lora': ['lora_rank', 'lora_alpha', 'lora_dropout', 'lora_dtype', 'use_rslora', 'use_dora'],
'llamapro': ['llamapro_num_new_blocks', 'llamapro_num_groups'],
'lisa': ['lisa_activated_layers', 'lisa_step_interval'],
'adalora': [
'adalora_target_r', 'adalora_init_r', 'adalora_tinit', 'adalora_tfinal', 'adalora_deltaT', 'adalora_beta1',
'adalora_beta2', 'adalora_orth_reg_weight'
],
'lora_ga': [
'lora_ga_batch_size', 'lora_ga_iters', 'lora_ga_max_length', 'lora_ga_direction', 'lora_ga_scale',
'lora_ga_stable_gamma'
],
'reft': ['reft_layers', 'reft_rank', 'reft_intervention_type'],
'vera': ['vera_rank', 'vera_projection_prng_key', 'vera_dropout', 'vera_d_initial'],
'boft': ['boft_block_size', 'boft_block_num', 'boft_dropout'],
'fourierft': ['fourier_n_frequency', 'fourier_scaling']
}
@classmethod
def do_build_ui(cls, base_tab: Type['BaseUI']):
with gr.Accordion(elem_id='tuner_params', open=False):
with gr.Tabs():
LoRA.set_lang(cls.lang)
LoRA.build_ui(base_tab)
with gr.TabItem(elem_id='llamapro_tab'):
with gr.Blocks():
with gr.Row():
gr.Textbox(elem_id='llamapro_num_new_blocks', scale=2)
gr.Textbox(elem_id='llamapro_num_groups', scale=2)
with gr.TabItem(elem_id='lisa_tab'):
with gr.Blocks():
with gr.Row():
gr.Textbox(elem_id='lisa_activated_layers', value='0', scale=2)
gr.Textbox(elem_id='lisa_step_interval', value='20', scale=2)
with gr.TabItem(elem_id='adalora_tab'):
with gr.Blocks():
with gr.Row():
gr.Textbox(elem_id='adalora_target_r', value='8', scale=2)
gr.Slider(elem_id='adalora_init_r', value=12, minimum=1, maximum=512, step=4, scale=2)
gr.Textbox(elem_id='adalora_tinit', value='0', scale=2)
gr.Textbox(elem_id='adalora_tfinal', value='0', scale=2)
with gr.Row():
gr.Textbox(elem_id='adalora_deltaT', value='1', scale=2)
gr.Textbox(elem_id='adalora_beta1', value='0.85', scale=2)
gr.Textbox(elem_id='adalora_beta2', value='0.85', scale=2)
gr.Textbox(elem_id='adalora_orth_reg_weight', value='0.5', scale=2)
with gr.TabItem(elem_id='lora_ga_tab'):
with gr.Blocks():
with gr.Row():
gr.Slider(elem_id='lora_ga_batch_size', value=2, minimum=1, maximum=256, step=1, scale=20)
gr.Textbox(elem_id='lora_ga_iters', value='2', scale=20)
gr.Textbox(elem_id='lora_ga_max_length', value='2048', scale=20)
gr.Dropdown(
elem_id='lora_ga_direction',
scale=20,
value='ArB2r',
choices=['ArBr', 'A2rBr', 'ArB2r', 'random'])
gr.Dropdown(
elem_id='lora_ga_scale',
scale=20,
value='stable',
choices=['gd', 'unit', 'stable', 'weights'])
gr.Textbox(elem_id='lora_ga_stable_gamma', value='16', scale=20)
with gr.TabItem(elem_id='reft_tab'):
with gr.Blocks():
with gr.Row():
gr.Textbox(elem_id='reft_layers', scale=2)
gr.Slider(elem_id='reft_rank', value=4, minimum=1, maximum=512, step=4, scale=2)
gr.Dropdown(
elem_id='reft_intervention_type',
scale=2,
value='LoreftIntervention',
choices=[
'NoreftIntervention', 'LoreftIntervention', 'ConsreftIntervention',
'LobireftIntervention', 'DireftIntervention', 'NodireftIntervention'
])
with gr.TabItem(elem_id='vera_tab'):
with gr.Blocks():
with gr.Row():
gr.Slider(elem_id='vera_rank', value=256, minimum=1, maximum=512, step=4, scale=2)
gr.Textbox(elem_id='vera_projection_prng_key', value='0', scale=2)
gr.Textbox(elem_id='vera_dropout', value='0.0', scale=2)
gr.Textbox(elem_id='vera_d_initial', value='0.1', scale=2)
with gr.TabItem(elem_id='boft_tab'):
with gr.Blocks():
with gr.Row():
gr.Textbox(elem_id='boft_block_size', value='4', scale=2)
gr.Textbox(elem_id='boft_block_num', scale=2)
gr.Textbox(elem_id='boft_dropout', value='0.0', scale=2)
with gr.TabItem(elem_id='fourierft_tab'):
with gr.Blocks():
with gr.Row():
gr.Textbox(elem_id='fourier_n_frequency', value='2000', scale=2)
gr.Textbox(elem_id='fourier_scaling', value='300.0', scale=2)
Target.set_lang(cls.lang)
Target.build_ui(base_tab)
swift/ui/llm_train/utils.py
+58
View File
@@ -0,0 +1,58 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
import asyncio
import os
import subprocess
import sys
from asyncio.subprocess import PIPE, STDOUT
from copy import deepcopy
async def run_and_get_log(*args, timeout=None):
process = await asyncio.create_subprocess_exec(*args, stdout=PIPE, stderr=STDOUT)
lines = []
while True:
try:
line = await asyncio.wait_for(process.stdout.readline(), timeout)
except asyncio.TimeoutError:
break
else:
if not line:
break
else:
lines.append(str(line))
return process, lines
def run_command_in_subprocess(*args, timeout):
if sys.platform == 'win32':
loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(loop)
else:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
process, lines = loop.run_until_complete(run_and_get_log(*args, timeout=timeout))
return (loop, process), lines
def close_loop(handler):
loop, process = handler
process.kill()
loop.close()
def run_command_in_background_with_popen(command, all_envs, log_file):
env = deepcopy(os.environ)
if len(all_envs) > 0:
for k, v in all_envs.items():
env[k] = v
daemon_kwargs = {}
if sys.platform == 'win32':
from subprocess import CREATE_NO_WINDOW, DETACHED_PROCESS
daemon_kwargs['creationflags'] = DETACHED_PROCESS | CREATE_NO_WINDOW
daemon_kwargs['close_fds'] = True
else:
daemon_kwargs['preexec_fn'] = os.setsid
with open(log_file, 'w', encoding='utf-8') as f:
subprocess.Popen(
command, stdout=f, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, text=True, bufsize=1, env=env)