chore: import upstream snapshot with attribution

swift/ui/llm_eval/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from .llm_eval import LLMEval

swift/ui/llm_eval/eval.py

@@ -0,0 +1,129 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import gradio as gr
+from typing import Type
+
+from swift.arguments import EvalArguments
+from swift.utils import get_logger
+from ..base import BaseUI
+
+logger = get_logger()
+
+
+class Eval(BaseUI):
+
+    group = 'llm_eval'
+
+    locale_dict = {
+        'eval_backend': {
+            'label': {
+                'zh': '评测后端',
+                'en': 'Eval backend'
+            },
+            'info': {
+                'zh': '选择评测后端',
+                'en': 'Select eval backend'
+            }
+        },
+        'eval_dataset': {
+            'label': {
+                'zh': '评测数据集',
+                'en': 'Evaluation dataset'
+            },
+            'info': {
+                'zh': '选择评测数据集，支持多选 (先选择评测后端)',
+                'en': 'Select eval dataset, multiple datasets supported (select eval backend first)'
+            }
+        },
+        'eval_limit': {
+            'label': {
+                'zh': '评测数据个数',
+                'en': 'Eval numbers for each dataset'
+            },
+            'info': {
+                'zh': '每个评测集的取样数',
+                'en': 'Number of rows sampled from each dataset'
+            }
+        },
+        'eval_output_dir': {
+            'label': {
+                'zh': '评测输出目录',
+                'en': 'Eval output dir'
+            },
+            'info': {
+                'zh': '评测结果的输出目录',
+                'en': 'The dir to save the eval results'
+            }
+        },
+        'custom_eval_config': {
+            'label': {
+                'zh': '自定义数据集评测配置',
+                'en': 'Custom eval config'
+            },
+            'info': {
+                'zh': '可以使用该配置评测自己的数据集，详见github文档的评测部分',
+                'en': 'Use this config to eval your own datasets, check the docs in github for details'
+            }
+        },
+        'eval_url': {
+            'label': {
+                'zh': '评测链接',
+                'en': 'The eval url'
+            },
+            'info': {
+                'zh':
+                'OpenAI样式的评测链接(如：http://localhost:8080/v1/chat/completions)，用于评测接口（模型类型输入为实际模型类型）',
+                'en':
+                'The OpenAI style link(like: http://localhost:8080/v1/chat/completions) for '
+                'evaluation(Input actual model type into model_type)'
+            }
+        },
+        'api_key': {
+            'label': {
+                'zh': '接口token',
+                'en': 'The url token'
+            },
+            'info': {
+                'zh': 'eval_url的token',
+                'en': 'The token used with eval_url'
+            }
+        },
+        'infer_backend': {
+            'label': {
+                'zh': '推理框架',
+                'en': 'Infer backend'
+            },
+        }
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        try:
+            eval_dataset_dict = EvalArguments.list_eval_dataset()
+            default_backend = EvalArguments.eval_backend
+        except Exception as e:
+            logger.warn(e)
+            eval_dataset_dict = {}
+            default_backend = None
+
+        with gr.Row():
+            gr.Dropdown(elem_id='eval_backend', choices=list(eval_dataset_dict.keys()), value=default_backend, scale=20)
+            gr.Dropdown(
+                elem_id='eval_dataset',
+                is_list=True,
+                choices=eval_dataset_dict.get(default_backend, []),
+                multiselect=True,
+                allow_custom_value=True,
+                scale=20)
+            gr.Textbox(elem_id='eval_limit', scale=20)
+            gr.Dropdown(elem_id='infer_backend', scale=20)
+        with gr.Row():
+            gr.Textbox(elem_id='custom_eval_config', scale=20)
+            gr.Textbox(elem_id='eval_output_dir', scale=20)
+            gr.Textbox(elem_id='eval_url', scale=20)
+            gr.Textbox(elem_id='api_key', scale=20)
+
+        def update_eval_dataset(backend):
+            return gr.update(choices=eval_dataset_dict[backend])
+
+        cls.element('eval_backend').change(update_eval_dataset, [cls.element('eval_backend')],
+                                           [cls.element('eval_dataset')])

swift/ui/llm_eval/llm_eval.py

@@ -0,0 +1,202 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import gradio as gr
+import json
+import os
+import re
+import sys
+import time
+from datetime import datetime
+from functools import partial
+from json import JSONDecodeError
+from transformers.utils import is_torch_cuda_available, is_torch_npu_available
+from typing import Type
+
+from swift.arguments import EvalArguments
+from swift.utils import get_device_count
+from ..base import BaseUI
+from ..llm_train import run_command_in_background_with_popen
+from .eval import Eval
+from .model import Model
+from .runtime import EvalRuntime
+
+
+class LLMEval(BaseUI):
+    group = 'llm_eval'
+
+    sub_ui = [Model, Eval, EvalRuntime]
+
+    cmd = 'eval'
+
+    locale_dict = {
+        'llm_eval': {
+            'label': {
+                'zh': 'LLM评测',
+                'en': 'LLM Evaluation',
+            }
+        },
+        'more_params': {
+            'label': {
+                'zh': '更多参数',
+                'en': 'More params'
+            },
+            'info': {
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
+            }
+        },
+        'evaluate': {
+            'value': {
+                'zh': '开始评测',
+                'en': 'Begin Evaluation'
+            },
+        },
+        'gpu_id': {
+            'label': {
+                'zh': '选择可用GPU',
+                'en': 'Choose GPU'
+            },
+            'info': {
+                'zh': '选择训练使用的GPU号，如CUDA不可用只能选择CPU',
+                'en': 'Select GPU to train'
+            }
+        },
+    }
+
+    choice_dict = BaseUI.get_choices_from_dataclass(EvalArguments)
+    default_dict = BaseUI.get_default_value_from_dataclass(EvalArguments)
+    arguments = BaseUI.get_argument_names(EvalArguments)
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.TabItem(elem_id='llm_eval', label=''):
+            default_device = 'cpu'
+            device_count = get_device_count()
+            if device_count > 0:
+                default_device = '0'
+            with gr.Blocks():
+                Model.build_ui(base_tab)
+                Eval.build_ui(base_tab)
+                EvalRuntime.build_ui(base_tab)
+                with gr.Row(equal_height=True):
+                    gr.Textbox(elem_id='more_params', lines=4, scale=20)
+                    gr.Button(elem_id='evaluate', scale=2, variant='primary')
+                gr.Dropdown(
+                    elem_id='gpu_id',
+                    multiselect=True,
+                    choices=[str(i) for i in range(device_count)] + ['cpu'],
+                    value=default_device,
+                    scale=8)
+
+                cls.element('evaluate').click(
+                    cls.eval_model, list(base_tab.valid_elements().values()),
+                    [cls.element('runtime_tab'), cls.element('running_tasks')])
+
+                base_tab.element('running_tasks').change(
+                    partial(EvalRuntime.task_changed, base_tab=base_tab), [base_tab.element('running_tasks')],
+                    list(base_tab.valid_elements().values()) + [cls.element('log')])
+                EvalRuntime.element('kill_task').click(
+                    EvalRuntime.kill_task,
+                    [EvalRuntime.element('running_tasks')],
+                    [EvalRuntime.element('running_tasks')] + [EvalRuntime.element('log')],
+                )
+
+    @classmethod
+    def eval(cls, *args):
+        eval_args = cls.get_default_value_from_dataclass(EvalArguments)
+        kwargs = {}
+        kwargs_is_list = {}
+        other_kwargs = {}
+        more_params = {}
+        more_params_cmd = ''
+        keys = cls.valid_element_keys()
+        for key, value in zip(keys, args):
+            compare_value = eval_args.get(key)
+            compare_value_arg = str(compare_value) if not isinstance(compare_value, (list, dict)) else compare_value
+            compare_value_ui = str(value) if not isinstance(value, (list, dict)) else value
+            if key in eval_args and compare_value_ui != compare_value_arg and value:
+                if isinstance(value, str) and re.fullmatch(cls.int_regex, value):
+                    value = int(value)
+                elif isinstance(value, str) and re.fullmatch(cls.float_regex, value):
+                    value = float(value)
+                elif isinstance(value, str) and re.fullmatch(cls.bool_regex, value):
+                    value = True if value.lower() == 'true' else False
+                kwargs[key] = value if not isinstance(value, list) else ' '.join(value)
+                kwargs_is_list[key] = isinstance(value, list) or getattr(cls.element(key), 'is_list', False)
+            else:
+                other_kwargs[key] = value
+            if key == 'more_params' and value:
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
+
+        kwargs.update(more_params)
+        model = kwargs.get('model')
+        if model and os.path.exists(model) and os.path.exists(os.path.join(model, 'args.json')):
+            if os.path.exists(os.path.join(model, 'adapter_config.json')):
+                kwargs['adapters'] = kwargs.pop('model')
+
+        eval_args = EvalArguments(
+            **{
+                key: value.split(' ') if key in kwargs_is_list and kwargs_is_list[key] else value
+                for key, value in kwargs.items()
+            })
+        params = ''
+        command = ['swift', 'eval']
+        sep = f'{cls.quote} {cls.quote}'
+        for e in kwargs:
+            if isinstance(kwargs[e], list):
+                params += f'--{e} {cls.quote}{sep.join(kwargs[e])}{cls.quote} '
+                command.extend([f'--{e}'] + kwargs[e])
+            elif e in kwargs_is_list and kwargs_is_list[e]:
+                all_args = [arg for arg in kwargs[e].split(' ') if arg.strip()]
+                params += f'--{e} {cls.quote}{sep.join(all_args)}{cls.quote} '
+                command.extend([f'--{e}'] + all_args)
+            else:
+                params += f'--{e} {cls.quote}{kwargs[e]}{cls.quote} '
+                command.extend([f'--{e}', f'{kwargs[e]}'])
+        if more_params_cmd != '':
+            params += f'{more_params_cmd.strip()} '
+            more_params_cmd = [param.strip() for param in more_params_cmd.split('--')]
+            more_params_cmd = [param.split(' ') for param in more_params_cmd if param]
+            for param in more_params_cmd:
+                command.extend([f'--{param[0]}'] + param[1:])
+        all_envs = {}
+        devices = other_kwargs['gpu_id']
+        devices = [d for d in devices if d]
+        assert (len(devices) == 1 or 'cpu' not in devices)
+        gpus = ','.join(devices)
+        cuda_param = ''
+        if gpus != 'cpu':
+            if is_torch_npu_available():
+                cuda_param = f'ASCEND_RT_VISIBLE_DEVICES={gpus}'
+                all_envs['ASCEND_RT_VISIBLE_DEVICES'] = gpus
+            elif is_torch_cuda_available():
+                cuda_param = f'CUDA_VISIBLE_DEVICES={gpus}'
+                all_envs['CUDA_VISIBLE_DEVICES'] = gpus
+            else:
+                cuda_param = ''
+        now = datetime.now()
+        time_str = f'{now.year}{now.month}{now.day}{now.hour}{now.minute}{now.second}'
+        file_path = f'output/{eval_args.model_type}-{time_str}'
+        if not os.path.exists(file_path):
+            os.makedirs(file_path, exist_ok=True)
+        log_file = os.path.join(os.getcwd(), f'{file_path}/run_eval.log')
+        eval_args.log_file = log_file
+        params += f'--log_file "{log_file}" '
+        command.extend(['--log_file', f'{log_file}'])
+        params += '--ignore_args_error true '
+        command.extend(['--ignore_args_error', 'true'])
+        if sys.platform == 'win32':
+            if cuda_param:
+                cuda_param = f'set {cuda_param} && '
+            run_command = f'{cuda_param}start /b swift eval {params} > {log_file} 2>&1'
+        else:
+            run_command = f'{cuda_param} nohup swift eval {params} > {log_file} 2>&1 &'
+        return command, all_envs, run_command, eval_args, log_file
+
+    @classmethod
+    def eval_model(cls, *args):
+        command, all_envs, run_command, eval_args, log_file = cls.eval(*args)
+        run_command_in_background_with_popen(command, all_envs, log_file)
+        return gr.update(open=True), EvalRuntime.refresh_tasks(log_file)

swift/ui/llm_eval/model.py

@@ -0,0 +1,78 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import gradio as gr
+from functools import partial
+from typing import Type
+
+from swift.arguments import EvalArguments
+from swift.model import ModelType, get_model_list
+from swift.template import TEMPLATE_MAPPING
+from ..base import BaseUI
+
+
+class Model(BaseUI):
+
+    group = 'llm_eval'
+
+    locale_dict = {
+        'checkpoint': {
+            'value': {
+                'zh': '训练后的模型',
+                'en': 'Trained model'
+            }
+        },
+        'model_type': {
+            'label': {
+                'zh': '选择模型类型',
+                'en': 'Select Model Type'
+            },
+            'info': {
+                'zh': 'SWIFT已支持的模型类型',
+                'en': 'Base model type supported by SWIFT'
+            }
+        },
+        'model': {
+            'label': {
+                'zh': '模型id或路径',
+                'en': 'Model id or path'
+            },
+            'info': {
+                'zh': '实际的模型id，如果是训练后的模型请填入checkpoint-xxx的目录',
+                'en': 'The actual model id or path, if is a trained model, please fill in the checkpoint-xxx dir'
+            }
+        },
+        'reset': {
+            'value': {
+                'zh': '恢复初始值',
+                'en': 'Reset to default'
+            },
+        },
+        'template': {
+            'label': {
+                'zh': '模型Prompt模板类型',
+                'en': 'Prompt template type'
+            },
+            'info': {
+                'zh': '选择匹配模型的Prompt模板',
+                'en': 'Choose the template type of the model'
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Row():
+            gr.Dropdown(
+                elem_id='model',
+                scale=20,
+                choices=get_model_list(),
+                value='Qwen/Qwen2.5-7B-Instruct',
+                allow_custom_value=True)
+            gr.Dropdown(elem_id='model_type', choices=ModelType.get_model_name_list(), scale=20)
+            gr.Dropdown(elem_id='template', choices=list(TEMPLATE_MAPPING.keys()), scale=20)
+
+    @classmethod
+    def after_build_ui(cls, base_tab: Type['BaseUI']):
+        cls.element('model').change(
+            partial(cls.update_input_model, arg_cls=EvalArguments, has_record=False),
+            inputs=[cls.element('model')],
+            outputs=list(cls.valid_elements().values()))

swift/ui/llm_eval/runtime.py

@@ -0,0 +1,107 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import gradio as gr
+from packaging import version
+from typing import Type
+
+from swift.utils import get_logger
+from ..base import BaseUI
+from ..llm_infer import Runtime
+
+logger = get_logger()
+
+
+class EvalRuntime(Runtime):
+
+    group = 'llm_eval'
+
+    cmd = 'eval'
+
+    locale_dict = {
+        'runtime_tab': {
+            'label': {
+                'zh': '运行时',
+                'en': 'Runtime'
+            },
+        },
+        'running_cmd': {
+            'label': {
+                'zh': '运行命令',
+                'en': 'Command line'
+            },
+            'info': {
+                'zh': '执行的实际命令',
+                'en': 'The actual command'
+            }
+        },
+        'show_log': {
+            'value': {
+                'zh': '展示评测状态',
+                'en': 'Show eval status'
+            },
+        },
+        'stop_show_log': {
+            'value': {
+                'zh': '停止展示',
+                'en': 'Stop showing running status'
+            },
+        },
+        'log': {
+            'label': {
+                'zh': '日志输出',
+                'en': 'Logging content'
+            },
+            'info': {
+                'zh': '如果日志无更新请再次点击"展示评测状态"',
+                'en': 'Please press "Show eval status" if the log content is not updating'
+            }
+        },
+        'running_tasks': {
+            'label': {
+                'zh': '运行中评测',
+                'en': 'Running evaluation'
+            },
+            'info': {
+                'zh': '所有的swift eval命令启动的任务',
+                'en': 'All tasks started by swift eval'
+            }
+        },
+        'refresh_tasks': {
+            'value': {
+                'zh': '找回评测',
+                'en': 'Find evaluation'
+            },
+        },
+        'kill_task': {
+            'value': {
+                'zh': '杀死评测',
+                'en': 'Kill evaluation'
+            },
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='runtime_tab', open=False, visible=True):
+            with gr.Blocks():
+                with gr.Row(equal_height=True):
+                    gr.Dropdown(elem_id='running_tasks', scale=10)
+                    gr.Button(elem_id='refresh_tasks', scale=1, variant='primary')
+                    gr.Button(elem_id='show_log', scale=1, variant='primary')
+                    gr.Button(elem_id='stop_show_log', scale=1)
+                    gr.Button(elem_id='kill_task', scale=1, size='lg')
+                with gr.Row():
+                    gr.Textbox(elem_id='log', lines=6, visible=False)
+
+                concurrency_limit = {}
+                if version.parse(gr.__version__) >= version.parse('4.0.0'):
+                    concurrency_limit = {'concurrency_limit': 5}
+                cls.log_event = base_tab.element('show_log').click(cls.update_log, [], [cls.element('log')]).then(
+                    cls.wait, [base_tab.element('running_tasks')], [cls.element('log')], **concurrency_limit)
+
+                base_tab.element('stop_show_log').click(cls.break_log_event, [cls.element('running_tasks')], [])
+
+                base_tab.element('refresh_tasks').click(
+                    cls.refresh_tasks,
+                    [base_tab.element('running_tasks')],
+                    [base_tab.element('running_tasks')],
+                )