chore: import upstream snapshot with attribution

swift/ui/llm_sample/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from .llm_sample import LLMSample

swift/ui/llm_sample/llm_sample.py

@@ -0,0 +1,273 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import gradio as gr
+import json
+import os
+import re
+import sys
+import time
+from datetime import datetime
+from functools import partial
+from json import JSONDecodeError
+from transformers.utils import is_torch_cuda_available, is_torch_npu_available
+from typing import Type
+
+from swift.arguments import SamplingArguments
+from swift.dataset import get_dataset_list
+from swift.utils import get_device_count, get_logger
+from ..base import BaseUI
+from ..llm_train import run_command_in_background_with_popen
+from .model import Model
+from .runtime import SampleRuntime
+from .sample import Sample
+
+logger = get_logger()
+
+
+class LLMSample(BaseUI):
+
+    group = 'llm_sample'
+
+    is_multimodal = True
+
+    sub_ui = [Model, Sample, SampleRuntime]
+
+    locale_dict = {
+        'llm_sample': {
+            'label': {
+                'zh': 'LLM采样',
+                'en': 'LLM Sampling',
+            }
+        },
+        'sample': {
+            'value': {
+                'zh': '开始采样',
+                'en': 'Start sampling',
+            }
+        },
+        'load_alert': {
+            'value': {
+                'zh': '采样中，请点击"展示采样状态"查看',
+                'en': 'Start to sample, '
+                'please Click "Show running '
+                'status" to view details',
+            }
+        },
+        'gpu_id': {
+            'label': {
+                'zh': '选择可用GPU',
+                'en': 'Choose GPU'
+            },
+            'info': {
+                'zh': '选择采样使用的GPU号，如CUDA不可用只能选择CPU',
+                'en': 'Select GPU to sample'
+            }
+        },
+        'dataset': {
+            'label': {
+                'zh': '数据集名称',
+                'en': 'Dataset id/path'
+            },
+            'info': {
+                'zh': '选择采样的数据集，支持复选/本地路径',
+                'en': 'The dataset(s) to train the models, support multi select and local folder/files'
+            }
+        },
+        'num_sampling_batch_size': {
+            'label': {
+                'zh': '每次采样的批次大小',
+                'en': 'The batch size of sampling'
+            }
+        },
+        'num_sampling_batches': {
+            'label': {
+                'zh': '采样批次数量',
+                'en': 'Num of Sampling batches'
+            }
+        },
+        'output_dir': {
+            'label': {
+                'zh': '存储目录',
+                'en': 'The output dir',
+            },
+            'info': {
+                'zh': '设置采样结果存储在哪个文件夹下',
+                'en': 'Set the output folder',
+            }
+        },
+        'envs': {
+            'label': {
+                'zh': '环境变量',
+                'en': 'Extra env vars'
+            },
+        },
+        'more_params': {
+            'label': {
+                'zh': '更多参数',
+                'en': 'More params'
+            },
+            'info': {
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
+            }
+        },
+    }
+
+    choice_dict = BaseUI.get_choices_from_dataclass(SamplingArguments)
+    default_dict = BaseUI.get_default_value_from_dataclass(SamplingArguments)
+    arguments = BaseUI.get_argument_names(SamplingArguments)
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.TabItem(elem_id='llm_sample', label=''):
+            default_device = 'cpu'
+            device_count = get_device_count()
+            if device_count > 0:
+                default_device = '0'
+            with gr.Blocks():
+                Model.build_ui(base_tab)
+                Sample.build_ui(base_tab)
+                with gr.Row():
+                    gr.Dropdown(
+                        elem_id='dataset',
+                        multiselect=True,
+                        choices=get_dataset_list(),
+                        scale=20,
+                        allow_custom_value=True)
+                    gr.Slider(elem_id='num_sampling_batch_size', minimum=1, maximum=128, step=1, value=1, scale=10)
+                    gr.Slider(elem_id='num_sampling_batches', minimum=1, maximum=128, step=1, value=1, scale=10)
+                SampleRuntime.build_ui(base_tab)
+                with gr.Row(equal_height=True):
+                    gr.Dropdown(
+                        elem_id='gpu_id',
+                        multiselect=True,
+                        choices=[str(i) for i in range(device_count)] + ['cpu'],
+                        value=default_device,
+                        scale=20)
+                    gr.Textbox(elem_id='output_dir', value='sample_output', scale=20)
+                    gr.Textbox(elem_id='envs', scale=20)
+                    gr.Button(elem_id='sample', scale=2, variant='primary')
+                with gr.Row():
+                    gr.Textbox(elem_id='more_params', lines=4)
+
+                cls.element('sample').click(
+                    cls.sample_model, list(base_tab.valid_elements().values()),
+                    [cls.element('runtime_tab'), cls.element('running_tasks')])
+
+                base_tab.element('running_tasks').change(
+                    partial(SampleRuntime.task_changed, base_tab=base_tab), [base_tab.element('running_tasks')],
+                    list(cls.valid_elements().values()) + [cls.element('log')])
+                SampleRuntime.element('kill_task').click(
+                    SampleRuntime.kill_task,
+                    [SampleRuntime.element('running_tasks')],
+                    [SampleRuntime.element('running_tasks')] + [SampleRuntime.element('log')],
+                )
+
+    @classmethod
+    def sample(cls, *args):
+        sample_args = cls.get_default_value_from_dataclass(SamplingArguments)
+        kwargs = {}
+        kwargs_is_list = {}
+        other_kwargs = {}
+        more_params = {}
+        more_params_cmd = ''
+        keys = cls.valid_element_keys()
+        for key, value in zip(keys, args):
+            compare_value = sample_args.get(key)
+            compare_value_arg = str(compare_value) if not isinstance(compare_value, (list, dict)) else compare_value
+            compare_value_ui = str(value) if not isinstance(value, (list, dict)) else value
+            if key in sample_args and compare_value_ui != compare_value_arg and value:
+                if isinstance(value, str) and re.fullmatch(cls.int_regex, value):
+                    value = int(value)
+                elif isinstance(value, str) and re.fullmatch(cls.float_regex, value):
+                    value = float(value)
+                elif isinstance(value, str) and re.fullmatch(cls.bool_regex, value):
+                    value = True if value.lower() == 'true' else False
+                kwargs[key] = value if not isinstance(value, list) else ' '.join(value)
+                kwargs_is_list[key] = isinstance(value, list) or getattr(cls.element(key), 'is_list', False)
+            else:
+                other_kwargs[key] = value
+            if key == 'more_params' and value:
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
+
+        kwargs.update(more_params)
+        model = kwargs.get('model')
+        if os.path.exists(model) and os.path.exists(os.path.join(model, 'args.json')):
+            args_path = os.path.join(model, 'args.json')
+            if os.path.exists(os.path.join(model, 'adapter_config.json')):
+                kwargs['adapters'] = kwargs.pop('model')
+            with open(args_path, 'r', encoding='utf-8') as f:
+                _json = json.load(f)
+                kwargs['model_type'] = _json['model_type']
+                kwargs['tuner_type'] = _json['tuner_type']
+        sample_args = SamplingArguments(
+            **{
+                key: value.split(' ') if key in kwargs_is_list and kwargs_is_list[key] else value
+                for key, value in kwargs.items()
+            })
+
+        params = ''
+        command = ['swift', 'sample']
+        sep = f'{cls.quote} {cls.quote}'
+        for e in kwargs:
+            if isinstance(kwargs[e], list):
+                params += f'--{e} {cls.quote}{sep.join(kwargs[e])}{cls.quote} '
+                command.extend([f'--{e}'] + kwargs[e])
+            elif e in kwargs_is_list and kwargs_is_list[e]:
+                all_args = [arg for arg in kwargs[e].split(' ') if arg.strip()]
+                params += f'--{e} {cls.quote}{sep.join(all_args)}{cls.quote} '
+                command.extend([f'--{e}'] + all_args)
+            else:
+                params += f'--{e} {cls.quote}{kwargs[e]}{cls.quote} '
+                command.extend([f'--{e}', f'{kwargs[e]}'])
+        if more_params_cmd != '':
+            params += more_params_cmd + ' '
+            more_params_cmd = [param.strip() for param in more_params_cmd.split('--')]
+            more_params_cmd = [param.split(' ') for param in more_params_cmd if param]
+            for param in more_params_cmd:
+                command.extend([f'--{param[0]}'] + param[1:])
+        all_envs = {}
+        devices = other_kwargs['gpu_id']
+        devices = [d for d in devices if d]
+        assert (len(devices) == 1 or 'cpu' not in devices)
+        gpus = ','.join(devices)
+        cuda_param = ''
+        if gpus != 'cpu':
+            if is_torch_npu_available():
+                cuda_param = f'ASCEND_RT_VISIBLE_DEVICES={gpus}'
+                all_envs['ASCEND_RT_VISIBLE_DEVICES'] = gpus
+            elif is_torch_cuda_available():
+                cuda_param = f'CUDA_VISIBLE_DEVICES={gpus}'
+                all_envs['CUDA_VISIBLE_DEVICES'] = gpus
+            else:
+                cuda_param = ''
+        now = datetime.now()
+        time_str = f'{now.year}{now.month}{now.day}{now.hour}{now.minute}{now.second}'
+        file_path = f'output/{sample_args.model_type}-{time_str}'
+        if not os.path.exists(file_path):
+            os.makedirs(file_path, exist_ok=True)
+        log_file = os.path.join(os.getcwd(), f'{file_path}/run_sample.log')
+        sample_args.log_file = log_file
+        params += f'--log_file "{log_file}" '
+        command.extend(['--log_file', f'{log_file}'])
+        params += '--ignore_args_error true '
+        command.extend(['--ignore_args_error', 'true'])
+        if sys.platform == 'win32':
+            if cuda_param:
+                cuda_param = f'set {cuda_param} && '
+            run_command = f'{cuda_param}start /b swift sample {params} > {log_file} 2>&1'
+        else:
+            run_command = f'{cuda_param} nohup swift sample {params} > {log_file} 2>&1 &'
+        return command, all_envs, run_command, sample_args, log_file
+
+    @classmethod
+    def sample_model(cls, *args):
+        command, all_envs, run_command, sample_args, log_file = cls.sample(*args)
+        logger.info(f'Running sample command: {run_command}')
+        run_command_in_background_with_popen(command, all_envs, log_file)
+        gr.Info(cls.locale('load_alert', cls.lang)['value'])
+        time.sleep(2)
+        running_task = SampleRuntime.refresh_tasks(log_file)
+        return gr.update(open=True), running_task

swift/ui/llm_sample/model.py

@@ -0,0 +1,103 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import gradio as gr
+from functools import partial
+from typing import Type
+
+from swift.arguments import SamplingArguments
+from swift.model import ModelType, get_model_list
+from swift.template import TEMPLATE_MAPPING
+from ..base import BaseUI
+
+
+class Model(BaseUI):
+
+    group = 'llm_sample'
+
+    locale_dict = {
+        'model_type': {
+            'label': {
+                'zh': '选择模型类型',
+                'en': 'Select Model Type'
+            },
+            'info': {
+                'zh': 'SWIFT已支持的模型类型，model是服务名称时请置空',
+                'en': 'Base model type supported by SWIFT, Please leave it blank if model is the service name'
+            }
+        },
+        'model': {
+            'label': {
+                'zh': '模型id、路径或模型服务名称',
+                'en': 'Model id, path or server name'
+            },
+            'info': {
+                'zh':
+                '实际的模型id，如果是训练后的模型请填入checkpoint-xxx的目录，如果是模型服务请填入模型服务名称',
+                'en': ('The actual model id or path, if is a trained model, please fill in the checkpoint-xxx dir'
+                       'if is a model service, please fill in the server name')
+            }
+        },
+        'template': {
+            'label': {
+                'zh': '模型Prompt模板类型',
+                'en': 'Prompt template type'
+            },
+            'info': {
+                'zh': '选择匹配模型的Prompt模板，model是服务名称时请置空',
+                'en': 'Choose the template type of the model, Please leave it blank if model is the service name'
+            }
+        },
+        'system': {
+            'label': {
+                'zh': 'System字段',
+                'en': 'System'
+            },
+            'info': {
+                'zh': 'System字段支持在加载模型后修改',
+                'en': 'System can be modified after the model weights loaded'
+            }
+        },
+        'prm_model': {
+            'label': {
+                'zh': '过程奖励模型',
+                'en': 'Process Reward Model'
+            },
+            'info': {
+                'zh': '可以是模型id，或者plugin中定义的prm key',
+                'en': 'It can be a model id, or a prm key defined in the plugin'
+            }
+        },
+        'orm_model': {
+            'label': {
+                'zh': '结果奖励模型',
+                'en': 'Outcome Reward Model'
+            },
+            'info': {
+                'zh': '通常是通配符或测试用例等，定义在plugin中',
+                'en': 'Usually a wildcard or test case, etc., defined in the plugin'
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Row(equal_height=True):
+            gr.Dropdown(
+                elem_id='model',
+                scale=20,
+                choices=get_model_list(),
+                value='Qwen/Qwen2.5-7B-Instruct',
+                allow_custom_value=True)
+            gr.Dropdown(elem_id='model_type', choices=ModelType.get_model_name_list(), scale=20)
+            gr.Dropdown(elem_id='template', choices=list(TEMPLATE_MAPPING.keys()), scale=20)
+        with gr.Row():
+            gr.Textbox(elem_id='system', lines=1)
+        with gr.Row():
+            gr.Textbox(elem_id='prm_model', scale=20)
+            gr.Textbox(elem_id='orm_model', scale=20)
+
+    @classmethod
+    def after_build_ui(cls, base_tab: Type['BaseUI']):
+        cls.element('model').change(
+            partial(cls.update_input_model, arg_cls=SamplingArguments, has_record=False),
+            inputs=[cls.element('model')],
+            outputs=list(cls.valid_elements().values()))

swift/ui/llm_sample/runtime.py

@@ -0,0 +1,75 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from swift.utils import get_logger
+from ..llm_infer import Runtime
+
+logger = get_logger()
+
+
+class SampleRuntime(Runtime):
+
+    group = 'llm_sample'
+
+    cmd = 'sample'
+
+    locale_dict = {
+        'runtime_tab': {
+            'label': {
+                'zh': '运行时',
+                'en': 'Runtime'
+            },
+        },
+        'running_cmd': {
+            'label': {
+                'zh': '运行命令',
+                'en': 'Command line'
+            },
+            'info': {
+                'zh': '执行的实际命令',
+                'en': 'The actual command'
+            }
+        },
+        'show_log': {
+            'value': {
+                'zh': '展示采样状态',
+                'en': 'Show running status'
+            },
+        },
+        'stop_show_log': {
+            'value': {
+                'zh': '停止展示',
+                'en': 'Stop showing running status'
+            },
+        },
+        'log': {
+            'label': {
+                'zh': '日志输出',
+                'en': 'Logging content'
+            },
+            'info': {
+                'zh': '如果日志无更新请再次点击"展示采样状态"',
+                'en': 'Please press "Show running status" if the log content is not updating'
+            }
+        },
+        'running_tasks': {
+            'label': {
+                'zh': '运行中采样',
+                'en': 'Running sampling'
+            },
+            'info': {
+                'zh': '所有的swift sample命令启动的任务',
+                'en': 'Started by swift sample'
+            }
+        },
+        'refresh_tasks': {
+            'value': {
+                'zh': '找回采样',
+                'en': 'Find sampling'
+            },
+        },
+        'kill_task': {
+            'value': {
+                'zh': '杀死采样',
+                'en': 'Kill running task'
+            },
+        },
+    }

swift/ui/llm_sample/sample.py

@@ -0,0 +1,85 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import gradio as gr
+from typing import Type
+
+from ..base import BaseUI
+
+
+class Sample(BaseUI):
+
+    group = 'llm_sample'
+
+    locale_dict = {
+        'sampler_type': {
+            'label': {
+                'zh': '采样类型',
+                'en': 'Sampler type'
+            },
+        },
+        'sampler_engine': {
+            'label': {
+                'zh': '推理引擎',
+                'en': 'Infer engine'
+            },
+        },
+        'num_return_sequences': {
+            'label': {
+                'zh': '采样返回的原始序列数量',
+                'en': 'Num of original sequences returned by sampling'
+            },
+        },
+        'n_best_to_keep': {
+            'label': {
+                'zh': '最佳序列数量',
+                'en': 'Num of best sequences'
+            },
+        },
+        'max_new_tokens': {
+            'label': {
+                'zh': '生成序列最大长度',
+                'en': 'Max new tokens'
+            },
+        },
+        'temperature': {
+            'label': {
+                'zh': '采样温度',
+                'en': 'Temperature'
+            },
+        },
+        'top_k': {
+            'label': {
+                'zh': 'Top-k',
+                'en': 'Top-k'
+            },
+        },
+        'top_p': {
+            'label': {
+                'zh': 'Top-p',
+                'en': 'Top-p'
+            },
+        },
+        'repetition_penalty': {
+            'label': {
+                'zh': '重复惩罚',
+                'en': 'Repetition Penalty'
+            },
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Row():
+            gr.Dropdown(elem_id='sampler_type', choices=['sample', 'distill'], value='sample', scale=5)
+            gr.Dropdown(
+                elem_id='sampler_engine',
+                choices=['transformers', 'lmdeploy', 'vllm', 'no', 'client'],
+                value='transformers',
+                scale=5)
+            gr.Slider(elem_id='num_return_sequences', minimum=1, maximum=128, step=1, value=64, scale=5)
+            gr.Slider(elem_id='n_best_to_keep', minimum=1, maximum=64, step=1, value=5, scale=5)
+        with gr.Row():
+            gr.Textbox(elem_id='max_new_tokens', lines=1, value='2048')
+            gr.Slider(elem_id='temperature', minimum=0.0, maximum=10, step=0.1, value=1.0)
+            gr.Slider(elem_id='top_k', minimum=1, maximum=100, step=5, value=20)
+            gr.Slider(elem_id='top_p', minimum=0.0, maximum=1.0, step=0.05, value=0.7)
+            gr.Slider(elem_id='repetition_penalty', minimum=0.0, maximum=10, step=0.05, value=1.05)