wehub-resource-sync
ec436095dd
Book-CI / test (macos-latest) (push) Has been cancelled
Book-CI / test (ubuntu-latest) (push) Has been cancelled
Book-CI / test (windows-latest) (push) Has been cancelled
Release Fake Tag / publish (push) Has been cancelled
Deploy / deploy (macos-latest) (push) Has been cancelled
Deploy / deploy (ubuntu-latest) (push) Has been cancelled
Deploy / deploy (windows-latest) (push) Has been cancelled
Release to PyPI / Build & publish sglang-kt (push) Has been cancelled
Release to PyPI / Build kt-kernel (Python 3.11) (push) Has been cancelled
Release to PyPI / Build kt-kernel (Python 3.12) (push) Has been cancelled
Release to PyPI / Publish kt-kernel to PyPI (push) Has been cancelled
46 lines
2.3 KiB
Python
46 lines
2.3 KiB
Python
#!/usr/bin/env python
|
|
# coding=utf-8
|
|
'''
|
|
Description :
|
|
Author : qiyuxinlin
|
|
Date : 2024-07-25 11:50:16
|
|
Version : 1.0.0
|
|
LastEditors : qiyuxinlin
|
|
LastEditTime : 2024-07-25 12:54:48
|
|
Copyright (c) 2024 by KVCache.AI, All Rights Reserved.
|
|
'''
|
|
from ktransformers.server.config.config import Config
|
|
from ktransformers.server.backend.args import ConfigArgs
|
|
from ktransformers.server.backend.context_manager import ThreadContextManager
|
|
from ktransformers.server.backend.interfaces.exllamav2 import ExllamaInterface
|
|
from ktransformers.server.backend.interfaces.transformers import TransformersInterface
|
|
from ktransformers.server.backend.interfaces.ktransformers import KTransformersInterface
|
|
|
|
def create_interface(config: Config, default_args: ConfigArgs, input_args=None):
|
|
if config.backend_type=='transformers':
|
|
from ktransformers.server.backend.interfaces.transformers import TransformersInterface as BackendInterface
|
|
elif config.backend_type == 'exllamav2':
|
|
from ktransformers.server.backend.interfaces.exllamav2 import ExllamaInterface as BackendInterface
|
|
elif config.backend_type == 'ktransformers':
|
|
from ktransformers.server.backend.interfaces.ktransformers import KTransformersInterface as BackendInterface
|
|
elif config.backend_type == 'balance_serve':
|
|
from ktransformers.server.backend.interfaces.balance_serve import BalanceServeInterface as BackendInterface
|
|
else:
|
|
raise NotImplementedError(f'{config.backend_type} not implemented')
|
|
if config.backend_type == 'ktransformers':
|
|
GlobalInterface.interface = BackendInterface(default_args, input_args)
|
|
elif config.backend_type == 'balance_serve':
|
|
GlobalInterface.interface = BackendInterface(default_args, input_args)
|
|
else:
|
|
GlobalInterface.interface = BackendInterface(default_args)
|
|
GlobalContextManager.context_manager = ThreadContextManager(GlobalInterface.interface)
|
|
|
|
class GlobalContextManager:
|
|
context_manager: ThreadContextManager
|
|
class GlobalInterface:
|
|
interface: TransformersInterface | KTransformersInterface | ExllamaInterface
|
|
|
|
def get_thread_context_manager() -> GlobalContextManager:
|
|
return GlobalContextManager.context_manager
|
|
def get_interface() -> GlobalInterface:
|
|
return GlobalInterface.interface |