This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,301 @@
|
||||
{
|
||||
"cmd": "sft",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"eval_requirements": {
|
||||
"gpu": "1"
|
||||
},
|
||||
"eval_dataset": ["ceval", "gsm8k", "arc"],
|
||||
"args": {
|
||||
"model": "Qwen/Qwen-7B-Chat",
|
||||
"dataset": "iic/ms_agent",
|
||||
"per_device_train_batch_size": 1,
|
||||
"max_length": 2048,
|
||||
"loss_scale": "react",
|
||||
"gradient_accumulation_steps": 16,
|
||||
"learning_rate": 5e-5,
|
||||
"attn_impl": "flash_attn",
|
||||
"eval_steps": 2000,
|
||||
"save_steps": 2000,
|
||||
"num_train_epochs": 2,
|
||||
"gradient_checkpointing": true,
|
||||
"weight_decay": 0.01,
|
||||
"warmup_ratio": 0.03,
|
||||
"save_total_limit": 2,
|
||||
"logging_steps": 10
|
||||
},
|
||||
"experiment": [
|
||||
{
|
||||
"name": "lora",
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lora+packing",
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32,
|
||||
"packing": true,
|
||||
"eval_steps": 200,
|
||||
"save_steps": 200
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lora+packing+ddp",
|
||||
"requirements":{
|
||||
"gpu": "2",
|
||||
"ddp": "2"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32,
|
||||
"packing": true,
|
||||
"eval_steps": 100,
|
||||
"save_steps": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lora+packing+lazytokenize",
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32,
|
||||
"packing": true,
|
||||
"lazy_tokenize": true,
|
||||
"eval_steps": 200,
|
||||
"save_steps": 200
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lora+",
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32,
|
||||
"lorap_lr_ratio": 16.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "rslora",
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32,
|
||||
"use_rslora": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "dora",
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32,
|
||||
"use_dora": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lora+neftune",
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32,
|
||||
"neftune_noise_alpha": 15.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "llamapro",
|
||||
"args": {
|
||||
"tuner_type": "llamapro",
|
||||
"llamapro_num_new_blocks": "4"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "full"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "reft",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "reft",
|
||||
"gradient_checkpointing": "false",
|
||||
"loss_scale": "default"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+galore128+quantize",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"use_galore": "true",
|
||||
"galore_rank": "128",
|
||||
"galore_update_proj_gap": "200",
|
||||
"galore_optim_per_parameter": "false",
|
||||
"galore_with_embedding": "false",
|
||||
"galore_quantization": "true"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+galore128+quantize+proj_quant",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"use_galore": "true",
|
||||
"galore_rank": "128",
|
||||
"galore_update_proj_gap": "200",
|
||||
"galore_optim_per_parameter": "false",
|
||||
"galore_with_embedding": "false",
|
||||
"galore_quantization": "true",
|
||||
"galore_proj_quant": "true"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+galore128",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"use_galore": "true",
|
||||
"galore_rank": "128",
|
||||
"galore_update_proj_gap": "200",
|
||||
"galore_optim_per_parameter": "false",
|
||||
"galore_with_embedding": "false"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+galore64",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"use_galore": "true",
|
||||
"galore_rank": "64",
|
||||
"galore_update_proj_gap": "200",
|
||||
"galore_optim_per_parameter": "false",
|
||||
"galore_with_embedding": "false"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+galore32",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"use_galore": "true",
|
||||
"galore_rank": "32",
|
||||
"galore_update_proj_gap": "200",
|
||||
"galore_optim_per_parameter": "false",
|
||||
"galore_with_embedding": "false"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+galore_emb",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"use_galore": "true",
|
||||
"galore_rank": "128",
|
||||
"galore_update_proj_gap": "200",
|
||||
"galore_optim_per_parameter": "false",
|
||||
"galore_with_embedding": "true"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+galore_perparam",
|
||||
"requirements":{
|
||||
"gpu": "1",
|
||||
"ddp": "1"
|
||||
},
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"use_galore": "true",
|
||||
"galore_rank": "128",
|
||||
"galore_update_proj_gap": "200",
|
||||
"galore_optim_per_parameter": "true",
|
||||
"galore_with_embedding": "false"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "adalora",
|
||||
"args": {
|
||||
"tuner_type": "adalora",
|
||||
"lora_rank": 8,
|
||||
"lora_alpha": 32
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "adapter",
|
||||
"args": {
|
||||
"tuner_type": "adapter"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+lisa_2",
|
||||
"info": "lisa 2layers + full",
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"lisa_activated_layers": 2,
|
||||
"lisa_step_interval": 20
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "full+lisa_4",
|
||||
"info": "lisa 4layers + full",
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"lisa_activated_layers": 4,
|
||||
"lisa_step_interval": 20
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "unsloth+lora+q4",
|
||||
"info": "unsloth lora quantization bit 4",
|
||||
"args": {
|
||||
"tuner_type": "lora",
|
||||
"tuner_backend": "unsloth",
|
||||
"quantization_bit": 4,
|
||||
"model": "LLM-Research/Meta-Llama-3-8B-Instruct"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "unsloth+full",
|
||||
"info": "unsloth full",
|
||||
"args": {
|
||||
"tuner_type": "full",
|
||||
"tuner_backend": "unsloth",
|
||||
"model_type": "LLM-Research/Meta-Llama-3-8B-Instruct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import argparse
|
||||
import os
|
||||
import os.path
|
||||
from exp_utils import ExpManager, find_all_config
|
||||
|
||||
from swift.utils import *
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Simple args for swift experiments.')
|
||||
parser.add_argument(
|
||||
'--config',
|
||||
type=str,
|
||||
default=None,
|
||||
required=True,
|
||||
help='The experiment config file',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--save_dir',
|
||||
type=str,
|
||||
default='./experiment',
|
||||
required=False,
|
||||
help='The experiment output folder',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def llm_exp():
|
||||
args = parse_args()
|
||||
config: str = args.config
|
||||
config = config.split(',')
|
||||
os.makedirs(args.save_dir, exist_ok=True)
|
||||
all_configs = []
|
||||
if not isinstance(config, list):
|
||||
config = [config]
|
||||
for dir_or_file in config:
|
||||
all_configs.extend(find_all_config(dir_or_file))
|
||||
args.config = all_configs
|
||||
exp_manager = ExpManager()
|
||||
exp_manager.begin(args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
llm_exp()
|
||||
@@ -0,0 +1,389 @@
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from collections import deque
|
||||
from copy import deepcopy
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from swift.arguments import ExportArguments
|
||||
from swift.utils import find_free_port, get_device_count, get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Experiment:
|
||||
|
||||
name: str
|
||||
|
||||
cmd: str
|
||||
|
||||
group: str
|
||||
|
||||
requirements: Dict = field(default_factory=dict)
|
||||
|
||||
eval_requirements: Dict = field(default_factory=dict)
|
||||
|
||||
eval_dataset: List = field(default_factory=list)
|
||||
|
||||
args: Dict = field(default_factory=dict)
|
||||
|
||||
env: Dict = field(default_factory=dict)
|
||||
|
||||
record: Dict = field(default_factory=dict)
|
||||
|
||||
create_time: float = None
|
||||
|
||||
runtime: Dict = field(default_factory=dict)
|
||||
|
||||
input_args: Any = None
|
||||
|
||||
do_eval = False
|
||||
|
||||
def __init__(self,
|
||||
name,
|
||||
cmd,
|
||||
group,
|
||||
requirements=None,
|
||||
eval_requirements=None,
|
||||
eval_dataset=None,
|
||||
args=None,
|
||||
input_args=None,
|
||||
**kwargs):
|
||||
self.name = name
|
||||
self.cmd = cmd
|
||||
self.group = group
|
||||
self.requirements = requirements or {}
|
||||
self.args = args or {}
|
||||
self.record = {}
|
||||
self.env = {}
|
||||
self.runtime = {}
|
||||
self.input_args = input_args
|
||||
self.eval_requirements = eval_requirements or {}
|
||||
self.eval_dataset = eval_dataset or []
|
||||
if self.cmd == 'eval':
|
||||
self.do_eval = True
|
||||
|
||||
def load(self, _json):
|
||||
self.name = _json['name']
|
||||
self.cmd = _json['cmd']
|
||||
self.requirements = _json['requirements']
|
||||
self.args = _json['args']
|
||||
self.record = _json['record']
|
||||
self.env = _json['env']
|
||||
self.create_time = _json['create_time']
|
||||
|
||||
@property
|
||||
def priority(self):
|
||||
return self.requirements.get('gpu', 0)
|
||||
|
||||
def to_dict(self):
|
||||
_dict = asdict(self)
|
||||
_dict.pop('runtime')
|
||||
_dict.pop('input_args')
|
||||
return _dict
|
||||
|
||||
|
||||
class ExpManager:
|
||||
|
||||
RESULT_FILE = 'result.jsonl'
|
||||
|
||||
def __init__(self):
|
||||
self.exps = []
|
||||
|
||||
def assert_gpu_not_overlap(self):
|
||||
all_gpus = set()
|
||||
for exp in self.exps:
|
||||
gpus = exp.runtime['env']['CUDA_VISIBLE_DEVICES'].split(',')
|
||||
if all_gpus & set(gpus):
|
||||
raise ValueError(f'GPU overlap: {self.exps}!')
|
||||
all_gpus.update(gpus)
|
||||
|
||||
def run(self, exp: Experiment):
|
||||
if os.path.exists(os.path.join(exp.input_args.save_dir, exp.name + '.json')):
|
||||
with open(os.path.join(exp.input_args.save_dir, exp.name + '.json'), 'r', encoding='utf-8') as f:
|
||||
_json = json.load(f)
|
||||
if exp.eval_dataset and 'eval_result' not in _json['record']:
|
||||
if not exp.do_eval:
|
||||
logger.info(f'Experiment {exp.name} need eval, load from file.')
|
||||
exp.load(_json)
|
||||
exp.do_eval = True
|
||||
else:
|
||||
logger.warn(f'Experiment {exp.name} already done, skip')
|
||||
return
|
||||
|
||||
if exp.do_eval:
|
||||
runtime = self._build_eval_cmd(exp)
|
||||
exp.runtime = runtime
|
||||
envs = deepcopy(runtime.get('env', {}))
|
||||
envs.update(os.environ)
|
||||
logger.info(f'Running cmd: {runtime["running_cmd"]}, env: {runtime.get("env", {})}')
|
||||
os.makedirs('exp', exist_ok=True)
|
||||
log_file = os.path.join('exp', f'{exp.name}.eval.log')
|
||||
exp.handler = subprocess.Popen(runtime['running_cmd'] + f' > {log_file} 2>&1', env=envs, shell=True)
|
||||
self.exps.append(exp)
|
||||
self.assert_gpu_not_overlap()
|
||||
return
|
||||
|
||||
if any([exp.name == e.name for e in self.exps]):
|
||||
raise ValueError(f'Why exp name duplicate? {exp.name}')
|
||||
elif exp.cmd == 'export' and any([exp.cmd == 'export' for exp in self.exps]): # noqa
|
||||
raise AssertionError('Cannot run parallel export task.')
|
||||
else:
|
||||
exp.create_time = time.time()
|
||||
runtime = self._build_cmd(exp)
|
||||
exp.runtime = runtime
|
||||
envs = deepcopy(runtime.get('env', {}))
|
||||
envs.update(os.environ)
|
||||
logger.info(f'Running cmd: {runtime["running_cmd"]}, env: {runtime.get("env", {})}')
|
||||
os.makedirs('exp', exist_ok=True)
|
||||
log_file = os.path.join('exp', f'{exp.name}.{exp.cmd}.log')
|
||||
exp.handler = subprocess.Popen(runtime['running_cmd'] + f' > {log_file} 2>&1', env=envs, shell=True)
|
||||
self.exps.append(exp)
|
||||
self.assert_gpu_not_overlap()
|
||||
|
||||
def _build_eval_cmd(self, exp: Experiment):
|
||||
gpu = exp.eval_requirements.get('gpu', None)
|
||||
env = {}
|
||||
allocated = []
|
||||
if gpu:
|
||||
allocated = self._find_free_gpu(int(gpu))
|
||||
assert allocated, 'No free gpu for now!'
|
||||
allocated = [str(gpu) for gpu in allocated]
|
||||
env['CUDA_VISIBLE_DEVICES'] = ','.join(allocated)
|
||||
|
||||
best_model_checkpoint = exp.record.get('best_model_checkpoint')
|
||||
eval_dataset = exp.eval_dataset
|
||||
if best_model_checkpoint is not None:
|
||||
if not os.path.exists(os.path.join(best_model_checkpoint, 'args.json')):
|
||||
cmd = f'swift eval --ckpt_dir {best_model_checkpoint} ' \
|
||||
+ f'--infer_backend transformers --tuner_type full --eval_dataset {" ".join(eval_dataset)}'
|
||||
else:
|
||||
cmd = f'swift eval --model {exp.args.get("model")} --infer_backend transformers ' \
|
||||
f'--eval_dataset {" ".join(eval_dataset)}'
|
||||
|
||||
return {
|
||||
'running_cmd': cmd,
|
||||
'gpu': allocated,
|
||||
'env': env,
|
||||
}
|
||||
|
||||
def _build_cmd(self, exp: Experiment):
|
||||
gpu = exp.requirements.get('gpu', None)
|
||||
env = {}
|
||||
allocated = []
|
||||
if gpu:
|
||||
allocated = self._find_free_gpu(int(gpu))
|
||||
assert allocated, 'No free gpu for now!'
|
||||
allocated = [str(gpu) for gpu in allocated]
|
||||
env['CUDA_VISIBLE_DEVICES'] = ','.join(allocated)
|
||||
if int(exp.requirements.get('ddp', 1)) > 1:
|
||||
env['NPROC_PER_NODE'] = exp.requirements.get('ddp')
|
||||
env['MASTER_PORT'] = str(find_free_port())
|
||||
|
||||
if exp.cmd == 'sft':
|
||||
from swift import SftArguments
|
||||
args = exp.args
|
||||
sft_args = SftArguments(**args)
|
||||
args['output_dir'] = sft_args.output_dir
|
||||
args['logging_dir'] = sft_args.logging_dir
|
||||
args['add_version'] = False
|
||||
os.makedirs(sft_args.output_dir, exist_ok=True)
|
||||
os.makedirs(sft_args.logging_dir, exist_ok=True)
|
||||
cmd = 'swift sft '
|
||||
for key, value in args.items():
|
||||
cmd += f' --{key} {value}'
|
||||
elif exp.cmd == 'rlhf':
|
||||
from swift import RLHFArguments
|
||||
args = exp.args
|
||||
rlhf_args = RLHFArguments(**args)
|
||||
args['output_dir'] = rlhf_args.output_dir
|
||||
args['logging_dir'] = rlhf_args.logging_dir
|
||||
args['add_version'] = False
|
||||
os.makedirs(rlhf_args.output_dir, exist_ok=True)
|
||||
os.makedirs(rlhf_args.logging_dir, exist_ok=True)
|
||||
cmd = 'swift rlhf '
|
||||
for key, value in args.items():
|
||||
cmd += f' --{key} {value}'
|
||||
elif exp.cmd == 'export':
|
||||
args = exp.args
|
||||
cmd = 'swift export '
|
||||
for key, value in args.items():
|
||||
cmd += f' --{key} {value}'
|
||||
else:
|
||||
raise ValueError(f'Unsupported cmd type: {exp.cmd}')
|
||||
return {
|
||||
'running_cmd': cmd,
|
||||
'gpu': allocated,
|
||||
'env': env,
|
||||
'logging_dir': args.get('logging_dir'),
|
||||
'output_dir': args.get('output_dir', args.get('ckpt_dir'))
|
||||
}
|
||||
|
||||
def _find_free_gpu(self, n):
|
||||
all_gpus = set()
|
||||
for exp in self.exps:
|
||||
all_gpus.update(exp.runtime.get('gpu', set()))
|
||||
all_gpus = {int(g) for g in all_gpus}
|
||||
free_gpu = set(range(get_device_count())) - all_gpus
|
||||
if len(free_gpu) < n:
|
||||
return None
|
||||
return list(free_gpu)[:n]
|
||||
|
||||
def prepare_experiments(self, args: Any):
|
||||
experiments = []
|
||||
for config_file in args.config:
|
||||
with open(config_file, 'r', encoding='utf-8') as f:
|
||||
group = os.path.basename(config_file)
|
||||
group = group[:-5]
|
||||
content = json.load(f)
|
||||
exps = content['experiment']
|
||||
for exp in exps:
|
||||
main_cfg = deepcopy(content)
|
||||
name = exp['name']
|
||||
cmd = main_cfg['cmd']
|
||||
run_args = main_cfg['args']
|
||||
env = main_cfg.get('env', {})
|
||||
requirements = main_cfg.get('requirements', {})
|
||||
eval_requirements = main_cfg.get('eval_requirements', {})
|
||||
eval_dataset = main_cfg.get('eval_dataset', {})
|
||||
if 'args' in exp:
|
||||
run_args.update(exp['args'])
|
||||
if 'requirements' in exp:
|
||||
requirements.update(exp['requirements'])
|
||||
if 'env' in exp:
|
||||
env.update(exp['env'])
|
||||
experiments.append(
|
||||
Experiment(
|
||||
group=group,
|
||||
name=name,
|
||||
cmd=cmd,
|
||||
args=run_args,
|
||||
env=env,
|
||||
requirements=requirements,
|
||||
eval_requirements=eval_requirements,
|
||||
eval_dataset=eval_dataset,
|
||||
input_args=args))
|
||||
return experiments
|
||||
|
||||
@staticmethod
|
||||
def _get_metric(exp: Experiment):
|
||||
if exp.do_eval:
|
||||
if os.path.isfile(os.path.join('exp', f'{exp.name}.eval.log')):
|
||||
with open(os.path.join('exp', f'{exp.name}.eval.log'), 'r', encoding='utf-8') as f:
|
||||
for line in f.readlines():
|
||||
if 'Final report:' in line:
|
||||
return json.loads(line.split('Final report:')[1].replace('\'', '"'))
|
||||
elif exp.cmd == 'export':
|
||||
exp_args = ExportArguments(**exp.args)
|
||||
if exp_args.quant_bits > 0:
|
||||
if exp_args.ckpt_dir is None:
|
||||
path = f'{exp_args.model_type}-{exp_args.quant_method}-int{exp_args.quant_bits}'
|
||||
else:
|
||||
ckpt_dir, ckpt_name = os.path.split(exp_args.ckpt_dir)
|
||||
path = os.path.join(ckpt_dir, f'{ckpt_name}-{exp_args.quant_method}-int{exp_args.quant_bits}')
|
||||
else:
|
||||
ckpt_dir, ckpt_name = os.path.split(exp_args.ckpt_dir)
|
||||
path = os.path.join(ckpt_dir, f'{ckpt_name}-merged')
|
||||
if os.path.exists(path):
|
||||
shutil.rmtree(exp.name, ignore_errors=True)
|
||||
os.makedirs(exp.name, exist_ok=True)
|
||||
shutil.move(path, os.path.join(exp.name, path))
|
||||
return {
|
||||
'best_model_checkpoint': os.path.join(exp.name, path),
|
||||
}
|
||||
else:
|
||||
logging_dir = exp.runtime.get('logging_dir')
|
||||
logging_file = os.path.join(logging_dir, '..', 'logging.jsonl')
|
||||
if os.path.isfile(logging_file):
|
||||
with open(logging_file, 'r', encoding='utf-8') as f:
|
||||
for line in f.readlines():
|
||||
if 'model_info' in line:
|
||||
return json.loads(line)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def write_record(exp: Experiment):
|
||||
target_dir = exp.input_args.save_dir
|
||||
file = os.path.join(target_dir, exp.name + '.json')
|
||||
with open(file, 'w', encoding='utf-8') as f:
|
||||
f.write(json.dumps(exp.to_dict()) + '\n')
|
||||
|
||||
def _poll(self):
|
||||
while True:
|
||||
time.sleep(5)
|
||||
|
||||
has_finished = False
|
||||
for exp in self.exps:
|
||||
rt = exp.handler.poll()
|
||||
if rt is None:
|
||||
continue
|
||||
|
||||
has_finished = True
|
||||
if rt == 0:
|
||||
if not exp.do_eval:
|
||||
all_metric = self._get_metric(exp)
|
||||
if all_metric:
|
||||
exp.record.update(all_metric)
|
||||
if exp.eval_dataset:
|
||||
exp.do_eval = True
|
||||
self.exp_queue.appendleft(exp)
|
||||
self.write_record(exp)
|
||||
else:
|
||||
logger.error(f'Running {exp.name} task, but no result found')
|
||||
else:
|
||||
all_metric = self._get_metric(exp)
|
||||
exp.record['eval_result'] = all_metric
|
||||
if all_metric:
|
||||
self.write_record(exp)
|
||||
else:
|
||||
logger.error(f'Running {exp.name} eval task, but no eval result found')
|
||||
logger.info(f'Running {exp.name} finished with return code: {rt}')
|
||||
|
||||
if has_finished:
|
||||
self.exps = [exp for exp in self.exps if exp.handler.poll() is None]
|
||||
break
|
||||
|
||||
def begin(self, args: Any):
|
||||
exps = self.prepare_experiments(args)
|
||||
logger.info(f'all exps: {exps}')
|
||||
exps.sort(key=lambda e: e.priority)
|
||||
self.exp_queue = deque()
|
||||
for exp in exps:
|
||||
self.exp_queue.append(exp)
|
||||
|
||||
while len(self.exp_queue) or len(self.exps) > 0:
|
||||
while len(self.exp_queue):
|
||||
try:
|
||||
logger.info(f'Running exp: {self.exp_queue[0].name}')
|
||||
self.run(self.exp_queue[0])
|
||||
except Exception as e:
|
||||
if not isinstance(e, AssertionError):
|
||||
logger.error(f'Adding exp {self.exp_queue[0].name} error because of:')
|
||||
logger.error(e)
|
||||
self.exp_queue.popleft()
|
||||
else:
|
||||
logger.info(f'Adding exp {self.exp_queue[0].name} error because of:', str(e))
|
||||
if 'no free gpu' in str(e).lower():
|
||||
break
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
self.exp_queue.popleft()
|
||||
self._poll()
|
||||
logger.info(f'Run task finished because of exp queue: {self.exp_queue} and exps: {self.exps}')
|
||||
|
||||
|
||||
def find_all_config(dir_or_file: str):
|
||||
if os.path.isfile(dir_or_file):
|
||||
return [dir_or_file]
|
||||
else:
|
||||
configs = []
|
||||
for dirpath, dirnames, filenames in os.walk(dir_or_file):
|
||||
for name in filenames:
|
||||
if name.endswith('.json') and 'ipynb' not in dirpath:
|
||||
configs.append(os.path.join(dirpath, name))
|
||||
return configs
|
||||
@@ -0,0 +1,432 @@
|
||||
# Copyright (c) ModelScope Contributors. All rights reserved.
|
||||
import dataclasses
|
||||
import json
|
||||
import numpy as np
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from swift.template import split_str_parts_by
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelOutput:
|
||||
|
||||
group: str = None
|
||||
|
||||
name: str = None
|
||||
|
||||
cmd: str = None
|
||||
|
||||
requirements: Dict[str, str] = dataclasses.field(default_factory=dict)
|
||||
|
||||
args: Dict[str, Any] = dataclasses.field(default_factory=dict)
|
||||
|
||||
memory: str = None
|
||||
|
||||
train_time: float = None
|
||||
|
||||
train_samples: int = None
|
||||
|
||||
train_samples_per_second: float = None
|
||||
|
||||
last_model_checkpoint: str = None
|
||||
|
||||
best_model_checkpoint: str = None
|
||||
|
||||
best_metric: Any = None
|
||||
|
||||
global_step: int = None
|
||||
|
||||
num_total_parameters: float = None
|
||||
|
||||
num_trainable_parameters: float = None
|
||||
|
||||
num_buffers: float = None
|
||||
|
||||
trainable_parameters_percentage: float = None
|
||||
|
||||
train_dataset_info: str = None
|
||||
|
||||
val_dataset_info: str = None
|
||||
|
||||
train_create_time: float = None
|
||||
|
||||
eval_tokens: int = None
|
||||
|
||||
eval_time: float = None
|
||||
|
||||
reports: Dict[str, Any] = None
|
||||
|
||||
train_loss: float = None
|
||||
|
||||
@property
|
||||
def tuner_hyper_params(self):
|
||||
hyper_params = ''
|
||||
args = self.args
|
||||
if 'tuner_type' not in args:
|
||||
return ''
|
||||
if args['tuner_type'] in ('lora', 'adalora', 'longlora'):
|
||||
if 'lora_rank' in args:
|
||||
hyper_params += f'rank={args["lora_rank"]}/' \
|
||||
f'target={args["lora_target_modules"]}/' \
|
||||
f'alpha={args["lora_alpha"]}/' \
|
||||
f'lr_ratio={args.get("lora_lr_ratio", None)}/' \
|
||||
f'use_rslora={args.get("use_rslora", False)}/' \
|
||||
f'use_dora={args.get("use_dora", False)}'
|
||||
else:
|
||||
hyper_params = ''
|
||||
if args['tuner_type'] == 'full':
|
||||
if 'use_galore' in args and args['use_galore'] == 'true':
|
||||
hyper_params += f'galore_rank={args["galore_rank"]}/' \
|
||||
f'galore_per_parameter={args["galore_optim_per_parameter"]}/' \
|
||||
f'galore_with_embedding={args["galore_with_embedding"]}/'
|
||||
if args['tuner_type'] == 'llamapro':
|
||||
hyper_params += f'num_blocks={args["llamapro_num_new_blocks"]}/'
|
||||
if 'neftune_noise_alpha' in args and args['neftune_noise_alpha']:
|
||||
hyper_params += f'neftune_noise_alpha={args["neftune_noise_alpha"]}/'
|
||||
|
||||
if hyper_params.endswith('/'):
|
||||
hyper_params = hyper_params[:-1]
|
||||
return hyper_params
|
||||
|
||||
@property
|
||||
def hyper_parameters(self):
|
||||
if 'learning_rate' not in self.args:
|
||||
return ''
|
||||
return f'lr={self.args["learning_rate"]}/' \
|
||||
f'epoch={self.args["num_train_epochs"]}'
|
||||
|
||||
@property
|
||||
def train_speed(self):
|
||||
if self.train_samples_per_second:
|
||||
return f'{self.train_samples_per_second:.2f}({self.train_samples} samples/{self.train_time:.2f} seconds)'
|
||||
else:
|
||||
return ''
|
||||
|
||||
@property
|
||||
def infer_speed(self):
|
||||
if self.eval_tokens:
|
||||
return f'{self.eval_tokens / self.eval_time:.2f}({self.eval_tokens} tokens/{self.eval_time:.2f} seconds)'
|
||||
return ''
|
||||
|
||||
|
||||
def generate_sft_report(outputs: List[ModelOutput]):
|
||||
gsm8k_accs = []
|
||||
arc_accs = []
|
||||
ceval_accs = []
|
||||
for output in outputs:
|
||||
gsm8k_acc = None
|
||||
arc_acc = None
|
||||
ceval_acc = None
|
||||
for report in (output.reports or []):
|
||||
if report['name'] == 'gsm8k':
|
||||
gsm8k_acc = report['score']
|
||||
if report['name'] == 'arc':
|
||||
arc_acc = report['score']
|
||||
if report['name'] == 'ceval':
|
||||
ceval_acc = report['score']
|
||||
gsm8k_accs.append(gsm8k_acc)
|
||||
arc_accs.append(arc_acc)
|
||||
ceval_accs.append(ceval_acc)
|
||||
|
||||
tab = '| exp_name | model_type | dataset | ms-bench mix ratio | tuner | tuner_params | trainable params(M) | flash_attn | gradient_checkpointing | hypers | memory | train speed(samples/s) | infer speed(tokens/s) | train_loss | eval_loss | gsm8k weighted acc | arc weighted acc | ceval weighted acc |\n' \
|
||||
'| -------- | ---------- | ------- | -------------------| ----- | ------------ | ------------------- | -----------| ---------------------- | ------ | ------ | ---------------------- | --------------------- | ---------- | --------- | ------------------ | ---------------- | ------------------ |\n' # noqa
|
||||
min_best_metric = 999.
|
||||
min_train_loss = 999.
|
||||
if outputs:
|
||||
min_best_metric = min([output.best_metric or 999. for output in outputs])
|
||||
min_train_loss = min([output.train_loss or 999. for output in outputs])
|
||||
|
||||
max_gsm8k = 0.0
|
||||
if gsm8k_accs:
|
||||
max_gsm8k = max([gsm8k or 0. for gsm8k in gsm8k_accs])
|
||||
|
||||
max_arc = 0.0
|
||||
if arc_accs:
|
||||
max_arc = max([arc or 0. for arc in arc_accs])
|
||||
|
||||
max_ceval = 0.0
|
||||
if ceval_accs:
|
||||
max_ceval = max([ceval or 0. for ceval in ceval_accs])
|
||||
|
||||
for output, gsm8k_acc, arc_acc, ceval_acc in zip(outputs, gsm8k_accs, arc_accs, ceval_accs):
|
||||
use_flash_attn = output.args.get('use_flash_attn', '')
|
||||
use_gc = output.args.get('gradient_checkpointing', '')
|
||||
memory = output.memory
|
||||
train_speed = output.train_speed
|
||||
infer_speed = output.infer_speed
|
||||
|
||||
is_best_metric = np.isclose(min_best_metric, output.best_metric or 999.0)
|
||||
is_best_loss = np.isclose(min_train_loss, output.train_loss or 999.0)
|
||||
is_best_gsm8k = np.isclose(max_gsm8k, gsm8k_acc or 0.0)
|
||||
is_best_arc = np.isclose(max_arc, arc_acc or 0.0)
|
||||
is_best_ceval = np.isclose(max_ceval, ceval_acc or 0.0)
|
||||
|
||||
if not is_best_metric:
|
||||
best_metric = '' if not output.best_metric else f'{output.best_metric:.2f}'
|
||||
else:
|
||||
best_metric = '' if not output.best_metric else f'**{output.best_metric:.2f}**'
|
||||
|
||||
if not is_best_loss:
|
||||
train_loss = '' if not output.train_loss else f'{output.train_loss:.2f}'
|
||||
else:
|
||||
train_loss = '' if not output.train_loss else f'**{output.train_loss:.2f}**'
|
||||
|
||||
if not is_best_gsm8k:
|
||||
gsm8k_acc = '' if not gsm8k_acc else f'{gsm8k_acc:.3f}'
|
||||
else:
|
||||
gsm8k_acc = '' if not gsm8k_acc else f'**{gsm8k_acc:.3f}**'
|
||||
|
||||
if not is_best_arc:
|
||||
arc_acc = '' if not arc_acc else f'{arc_acc:.3f}'
|
||||
else:
|
||||
arc_acc = '' if not arc_acc else f'**{arc_acc:.3f}**'
|
||||
|
||||
if not is_best_ceval:
|
||||
ceval_acc = '' if not ceval_acc else f'{ceval_acc:.3f}'
|
||||
else:
|
||||
ceval_acc = '' if not ceval_acc else f'**{ceval_acc:.3f}**'
|
||||
|
||||
line = f'|{output.name}|' \
|
||||
f'{output.args["model_type"]}|' \
|
||||
f'{output.args.get("dataset")}|' \
|
||||
f'{output.args.get("train_dataset_mix_ratio", 0.)}|' \
|
||||
f'{output.args.get("tuner_type")}|' \
|
||||
f'{output.tuner_hyper_params}|' \
|
||||
f'{output.num_trainable_parameters}({output.trainable_parameters_percentage})|' \
|
||||
f'{use_flash_attn}|' \
|
||||
f'{use_gc}|' \
|
||||
f'{output.hyper_parameters}|' \
|
||||
f'{memory}|' \
|
||||
f'{train_speed}|' \
|
||||
f'{infer_speed}|' \
|
||||
f'{best_metric}|' \
|
||||
f'{train_loss}|' \
|
||||
f'{gsm8k_acc}|' \
|
||||
f'{arc_acc}|' \
|
||||
f'{ceval_acc}|\n'
|
||||
tab += line
|
||||
return tab
|
||||
|
||||
|
||||
def generate_export_report(outputs: List[ModelOutput]):
|
||||
tab = '| exp_name | model_type | calibration dataset | quantization method | quantization bits | infer speed(tokens/s) | gsm8k weighted acc | arc weighted acc | ceval weighted acc |\n' \
|
||||
'| -------- | ---------- | ------------------- | ------------------- | ----------------- | --------------------- | ------------------ | ---------------- | ------------------ |\n' # noqa
|
||||
|
||||
gsm8k_accs = []
|
||||
arc_accs = []
|
||||
ceval_accs = []
|
||||
for output in outputs:
|
||||
gsm8k_acc = None
|
||||
arc_acc = None
|
||||
ceval_acc = None
|
||||
for report in (output.reports or []):
|
||||
if report['name'] == 'gsm8k':
|
||||
gsm8k_acc = report['score']
|
||||
if report['name'] == 'arc':
|
||||
arc_acc = report['score']
|
||||
if report['name'] == 'ceval':
|
||||
ceval_acc = report['score']
|
||||
gsm8k_accs.append(gsm8k_acc)
|
||||
arc_accs.append(arc_acc)
|
||||
ceval_accs.append(ceval_acc)
|
||||
|
||||
max_gsm8k = 0.0
|
||||
if gsm8k_accs:
|
||||
max_gsm8k = max([gsm8k or 0. for gsm8k in gsm8k_accs])
|
||||
|
||||
max_arc = 0.0
|
||||
if arc_accs:
|
||||
max_arc = max([arc or 0. for arc in arc_accs])
|
||||
|
||||
max_ceval = 0.0
|
||||
if ceval_accs:
|
||||
max_ceval = max([ceval or 0. for ceval in ceval_accs])
|
||||
|
||||
for output, gsm8k_acc, arc_acc, ceval_acc in zip(outputs, gsm8k_accs, arc_accs, ceval_accs):
|
||||
infer_speed = output.infer_speed
|
||||
is_best_gsm8k = np.isclose(max_gsm8k, gsm8k_acc or 0.0)
|
||||
is_best_arc = np.isclose(max_arc, arc_acc or 0.0)
|
||||
is_best_ceval = np.isclose(max_ceval, ceval_acc or 0.0)
|
||||
|
||||
if not is_best_gsm8k:
|
||||
gsm8k_acc = '' if not gsm8k_acc else f'{gsm8k_acc:.3f}'
|
||||
else:
|
||||
gsm8k_acc = '' if not gsm8k_acc else f'**{gsm8k_acc:.3f}**'
|
||||
|
||||
if not is_best_arc:
|
||||
arc_acc = '' if not arc_acc else f'{arc_acc:.3f}'
|
||||
else:
|
||||
arc_acc = '' if not arc_acc else f'**{arc_acc:.3f}**'
|
||||
|
||||
if not is_best_ceval:
|
||||
ceval_acc = '' if not ceval_acc else f'{ceval_acc:.3f}'
|
||||
else:
|
||||
ceval_acc = '' if not ceval_acc else f'**{ceval_acc:.3f}**'
|
||||
|
||||
if output.train_dataset_info:
|
||||
dataset_info = f'{output.args["dataset"]}/{output.train_dataset_info}'
|
||||
else:
|
||||
dataset_info = f'{output.args["dataset"]}'
|
||||
line = f'|{output.name}|' \
|
||||
f'{output.args["model_type"]}|' \
|
||||
f'{dataset_info}|' \
|
||||
f'{output.args["quant_method"]}|' \
|
||||
f'{output.args["quant_bits"]}|' \
|
||||
f'{infer_speed}|' \
|
||||
f'{gsm8k_acc}|' \
|
||||
f'{arc_acc}|' \
|
||||
f'{ceval_acc}|\n'
|
||||
tab += line
|
||||
return tab
|
||||
|
||||
|
||||
def parse_output(file):
|
||||
with open(file, 'r', encoding='utf-8') as f:
|
||||
content = json.load(f)
|
||||
|
||||
name = content['name']
|
||||
group = content['group']
|
||||
cmd = content['cmd']
|
||||
requirements = content['requirements']
|
||||
args = content['args']
|
||||
create_time = float(content.get('create_time') or 0)
|
||||
content = content['record']
|
||||
if cmd == 'export':
|
||||
best_model_checkpoint = content['best_model_checkpoint']
|
||||
eval_tokens = 0
|
||||
eval_time = 0.0
|
||||
eval_result = None
|
||||
if 'eval_result' in content:
|
||||
eval_result = content['eval_result']
|
||||
eval_tokens = eval_result['generation_info']['tokens']
|
||||
eval_time = eval_result['generation_info']['time']
|
||||
eval_result = eval_result['report']
|
||||
return ModelOutput(
|
||||
group=group,
|
||||
name=name,
|
||||
cmd=cmd,
|
||||
requirements=requirements,
|
||||
args=args,
|
||||
best_model_checkpoint=best_model_checkpoint,
|
||||
eval_time=eval_time,
|
||||
eval_tokens=eval_tokens,
|
||||
reports=eval_result,
|
||||
)
|
||||
else:
|
||||
memory = None
|
||||
train_time = None
|
||||
train_samples = None
|
||||
train_samples_per_second = None
|
||||
last_model_checkpoint = None
|
||||
best_model_checkpoint = None
|
||||
best_metric = None
|
||||
global_step = None
|
||||
train_dataset_info = None
|
||||
val_dataset_info = None
|
||||
num_trainable_parameters = None
|
||||
num_buffers = None
|
||||
trainable_parameters_percentage = None
|
||||
num_total_parameters = None
|
||||
train_loss = None
|
||||
if 'memory' in content:
|
||||
memory = content['memory']
|
||||
memory = '/'.join(memory.values())
|
||||
if 'train_time' in content:
|
||||
train_time = content['train_time']['train_runtime']
|
||||
train_samples = content['train_time']['n_train_samples']
|
||||
train_samples_per_second = content['train_time']['train_samples_per_second']
|
||||
if 'last_model_checkpoint' in content:
|
||||
last_model_checkpoint = content['last_model_checkpoint']
|
||||
if 'best_model_checkpoint' in content:
|
||||
best_model_checkpoint = content['best_model_checkpoint']
|
||||
if 'best_metric' in content:
|
||||
best_metric = content['best_metric']
|
||||
if 'log_history' in content:
|
||||
train_loss = content['log_history'][-1]['train_loss']
|
||||
if 'global_step' in content:
|
||||
global_step = content['global_step']
|
||||
if 'dataset_info' in content:
|
||||
train_dataset_info = content['dataset_info'].get('train_dataset')
|
||||
val_dataset_info = content['dataset_info'].get('val_dataset')
|
||||
if 'model_info' in content:
|
||||
# model_info like: SwiftModel: 6758.4041M Params (19.9885M Trainable [0.2958%]), 16.7793M Buffers.
|
||||
str_dict = split_str_parts_by(content['model_info'], [
|
||||
'SwiftModel:', 'CausalLM:', 'Seq2SeqLM:', 'LMHeadModel:', 'M Params (', 'M Trainable [', ']), ',
|
||||
'M Buffers.'
|
||||
])
|
||||
str_dict = {c['key']: c['content'] for c in str_dict}
|
||||
if 'SwiftModel:' in str_dict:
|
||||
num_total_parameters = float(str_dict['SwiftModel:'])
|
||||
elif 'CausalLM:' in str_dict:
|
||||
num_total_parameters = float(str_dict['CausalLM:'])
|
||||
elif 'Seq2SeqLM:' in str_dict:
|
||||
num_total_parameters = float(str_dict['Seq2SeqLM:'])
|
||||
elif 'LMHeadModel:' in str_dict:
|
||||
num_total_parameters = float(str_dict['LMHeadModel:'])
|
||||
num_trainable_parameters = float(str_dict['M Params ('])
|
||||
num_buffers = float(str_dict[']), '])
|
||||
trainable_parameters_percentage = str_dict['M Trainable [']
|
||||
|
||||
eval_tokens = 0
|
||||
eval_time = 0.0
|
||||
eval_result = None
|
||||
if 'eval_result' in content:
|
||||
eval_result = content['eval_result']
|
||||
eval_tokens = eval_result['generation_info']['tokens']
|
||||
eval_time = eval_result['generation_info']['time']
|
||||
eval_result = eval_result['report']
|
||||
|
||||
return ModelOutput(
|
||||
group=group,
|
||||
name=name,
|
||||
cmd=cmd,
|
||||
requirements=requirements,
|
||||
args=args,
|
||||
memory=memory,
|
||||
train_time=train_time,
|
||||
train_samples=train_samples,
|
||||
train_samples_per_second=train_samples_per_second,
|
||||
last_model_checkpoint=last_model_checkpoint,
|
||||
best_model_checkpoint=best_model_checkpoint,
|
||||
best_metric=best_metric,
|
||||
global_step=global_step,
|
||||
train_dataset_info=train_dataset_info,
|
||||
val_dataset_info=val_dataset_info,
|
||||
train_create_time=create_time,
|
||||
num_total_parameters=num_total_parameters,
|
||||
num_trainable_parameters=num_trainable_parameters,
|
||||
num_buffers=num_buffers,
|
||||
trainable_parameters_percentage=trainable_parameters_percentage,
|
||||
eval_time=eval_time,
|
||||
eval_tokens=eval_tokens,
|
||||
reports=eval_result,
|
||||
train_loss=train_loss,
|
||||
)
|
||||
|
||||
|
||||
def generate_reports():
|
||||
outputs = []
|
||||
for dirs, _, files in os.walk('./experiment'):
|
||||
for file in files:
|
||||
abs_file = os.path.join(dirs, file)
|
||||
if not abs_file.endswith('.json') or 'ipynb' in abs_file:
|
||||
continue
|
||||
|
||||
outputs.append(parse_output(abs_file))
|
||||
|
||||
all_groups = set([output.group for output in outputs])
|
||||
for group in all_groups:
|
||||
group_outputs = [output for output in outputs if output.group == group]
|
||||
print(f'=================Printing the sft cmd result of exp {group}==================\n\n')
|
||||
print(generate_sft_report([output for output in group_outputs if output.cmd in ('sft', 'eval')]))
|
||||
# print(f'=================Printing the dpo result of exp {group}==================')
|
||||
# print(generate_dpo_report([output for output in outputs if output.cmd == 'dpo']))
|
||||
print(f'=================Printing the export cmd result of exp {group}==================\n\n')
|
||||
print(generate_export_report([output for output in group_outputs if output.cmd == 'export']))
|
||||
print('=================Printing done==================\n\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
generate_reports()
|
||||
Reference in New Issue
Block a user