chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,159 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
|
||||
project(FunASRWebscoket)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 14 CACHE STRING "The C++ version to be used.")
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
|
||||
|
||||
option(ENABLE_WEBSOCKET "Whether to build websocket server" ON)
|
||||
option(ENABLE_PORTAUDIO "Whether to build portaudio" ON)
|
||||
option(ENABLE_GLOG "Whether to build glog" ON)
|
||||
option(ENABLE_FST "Whether to build openfst" ON) # ITN need openfst compiled
|
||||
option(BUILD_SHARED_LIBS "Build shared libraries" ON)
|
||||
option(GPU "Whether to build with GPU" OFF)
|
||||
|
||||
if(WIN32)
|
||||
file(REMOVE ${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/glog/src/config.h
|
||||
${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/glog/src/glog/export.h
|
||||
${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/glog/src/glog/logging.h
|
||||
${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/glog/src/glog/raw_logging.h
|
||||
${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/glog/src/glog/stl_logging.h
|
||||
${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/glog/src/glog/vlog_is_on.h)
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -fPIC")
|
||||
endif()
|
||||
|
||||
if(GPU)
|
||||
add_definitions(-DUSE_GPU)
|
||||
set(TORCH_DIR "/usr/local/lib/python3.8/dist-packages/torch")
|
||||
set(TORCH_BLADE_DIR "/usr/local/lib/python3.8/dist-packages/torch_blade")
|
||||
include_directories(${TORCH_DIR}/include)
|
||||
include_directories(${TORCH_DIR}/include/torch/csrc/api/include)
|
||||
link_directories(${TORCH_DIR}/lib)
|
||||
link_directories(${TORCH_BLADE_DIR})
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -D_GLIBCXX_USE_CXX11_ABI=0")
|
||||
endif()
|
||||
|
||||
if(ENABLE_WEBSOCKET)
|
||||
# cmake_policy(SET CMP0135 NEW)
|
||||
include(FetchContent)
|
||||
|
||||
if(NOT EXISTS ${PROJECT_SOURCE_DIR}/third_party/websocket/websocketpp )
|
||||
FetchContent_Declare(websocketpp
|
||||
GIT_REPOSITORY https://github.com/zaphoyd/websocketpp.git
|
||||
GIT_TAG 0.8.2
|
||||
SOURCE_DIR ${PROJECT_SOURCE_DIR}/third_party/websocket
|
||||
)
|
||||
|
||||
FetchContent_MakeAvailable(websocketpp)
|
||||
endif()
|
||||
include_directories(${PROJECT_SOURCE_DIR}/third_party/websocket)
|
||||
|
||||
if(NOT EXISTS ${PROJECT_SOURCE_DIR}/third_party/asio/asio )
|
||||
FetchContent_Declare(asio
|
||||
URL https://github.com/chriskohlhoff/asio/archive/refs/tags/asio-1-24-0.tar.gz
|
||||
SOURCE_DIR ${PROJECT_SOURCE_DIR}/third_party/asio
|
||||
)
|
||||
|
||||
FetchContent_MakeAvailable(asio)
|
||||
endif()
|
||||
include_directories(${PROJECT_SOURCE_DIR}/third_party/asio/asio/include)
|
||||
|
||||
if(NOT EXISTS ${PROJECT_SOURCE_DIR}/third_party/json/ChangeLog.md )
|
||||
FetchContent_Declare(json
|
||||
URL https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.tar.gz
|
||||
SOURCE_DIR ${PROJECT_SOURCE_DIR}/third_party/json
|
||||
)
|
||||
|
||||
FetchContent_MakeAvailable(json)
|
||||
endif()
|
||||
include_directories(${PROJECT_SOURCE_DIR}/third_party/json/include)
|
||||
|
||||
endif()
|
||||
|
||||
if(ENABLE_PORTAUDIO)
|
||||
include(FetchContent)
|
||||
|
||||
set(portaudio_URL "http://files.portaudio.com/archives/pa_stable_v190700_20210406.tgz")
|
||||
set(portaudio_URL2 "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/pa_stable_v190700_20210406.tgz")
|
||||
set(portaudio_HASH "SHA256=47efbf42c77c19a05d22e627d42873e991ec0c1357219c0d74ce6a2948cb2def")
|
||||
|
||||
FetchContent_Declare(portaudio
|
||||
URL
|
||||
${portaudio_URL}
|
||||
${portaudio_URL2}
|
||||
URL_HASH ${portaudio_HASH}
|
||||
)
|
||||
|
||||
FetchContent_GetProperties(portaudio)
|
||||
if(NOT portaudio_POPULATED)
|
||||
message(STATUS "Downloading portaudio from ${portaudio_URL}")
|
||||
FetchContent_Populate(portaudio)
|
||||
endif()
|
||||
message(STATUS "portaudio is downloaded to ${portaudio_SOURCE_DIR}")
|
||||
message(STATUS "portaudio's binary dir is ${portaudio_BINARY_DIR}")
|
||||
|
||||
add_subdirectory(${portaudio_SOURCE_DIR} ${portaudio_BINARY_DIR} EXCLUDE_FROM_ALL)
|
||||
if(NOT WIN32)
|
||||
target_compile_options(portaudio PRIVATE "-Wno-deprecated-declarations")
|
||||
else()
|
||||
install(TARGETS portaudio DESTINATION ..)
|
||||
endif()
|
||||
|
||||
endif()
|
||||
|
||||
# Include generated *.pb.h files
|
||||
link_directories(${ONNXRUNTIME_DIR}/lib)
|
||||
link_directories(${FFMPEG_DIR}/lib)
|
||||
|
||||
if(ENABLE_GLOG)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/glog/src)
|
||||
set(BUILD_TESTING OFF)
|
||||
add_subdirectory(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/glog glog)
|
||||
include_directories(${glog_BINARY_DIR})
|
||||
|
||||
endif()
|
||||
|
||||
if(ENABLE_FST)
|
||||
# fst depend on glog and gflags
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/gflags)
|
||||
add_subdirectory(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/gflags gflags)
|
||||
include_directories(${gflags_BINARY_DIR}/include)
|
||||
|
||||
# the following openfst if cloned from https://github.com/kkm000/openfst.git
|
||||
# with some patch to fix the make errors.
|
||||
add_subdirectory(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/openfst openfst)
|
||||
include_directories(${openfst_SOURCE_DIR}/src/include)
|
||||
if(WIN32)
|
||||
include_directories(${openfst_SOURCE_DIR}/src/lib)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/include/)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/src)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/yaml-cpp/include/)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/kaldi-native-fbank)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/jieba/include)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/jieba/include/limonp/include)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/kaldi)
|
||||
|
||||
add_subdirectory(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/yaml-cpp yaml-cpp)
|
||||
add_subdirectory(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc csrc)
|
||||
add_subdirectory(${PROJECT_SOURCE_DIR}/../onnxruntime/src src)
|
||||
add_subdirectory(${PROJECT_SOURCE_DIR}/../onnxruntime/third_party/kaldi kaldi)
|
||||
|
||||
# install openssl first apt-get install libssl-dev
|
||||
find_package(OpenSSL REQUIRED)
|
||||
|
||||
message("CXX_FLAGS "${CMAKE_CXX_FLAGS})
|
||||
# 获取项目中所有包含文件夹的路径
|
||||
get_property(includes DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
|
||||
# 遍历并输出每个包含文件夹的路径
|
||||
foreach(include ${includes})
|
||||
message("Include directory: ${include}")
|
||||
endforeach()
|
||||
|
||||
add_subdirectory(bin)
|
||||
@@ -0,0 +1,25 @@
|
||||
|
||||
if(WIN32)
|
||||
include_directories(${ONNXRUNTIME_DIR}/include)
|
||||
include_directories(${FFMPEG_DIR}/include)
|
||||
include_directories(${OPENSSL_ROOT_DIR}//include)
|
||||
link_directories(${OPENSSL_ROOT_DIR}/lib)
|
||||
add_definitions(-D_WEBSOCKETPP_CPP11_RANDOM_DEVICE_)
|
||||
add_definitions(-D_WEBSOCKETPP_CPP11_TYPE_TRAITS_)
|
||||
add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/bigobj>")
|
||||
add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/utf-8>")
|
||||
SET(RELATION_SOURCE "../../onnxruntime/src/resample.cpp" "../../onnxruntime/src/util.cpp" "../../onnxruntime/src/alignedmem.cpp" "../../onnxruntime/src/encode_converter.cpp")
|
||||
endif()
|
||||
|
||||
add_executable(funasr-wss-server "funasr-wss-server.cpp" "websocket-server.cpp" ${RELATION_SOURCE})
|
||||
add_executable(funasr-wss-server-2pass "funasr-wss-server-2pass.cpp" "websocket-server-2pass.cpp" ${RELATION_SOURCE})
|
||||
add_executable(funasr-wss-client "funasr-wss-client.cpp" ${RELATION_SOURCE})
|
||||
add_executable(funasr-wss-client-2pass "funasr-wss-client-2pass.cpp" "microphone.cpp" ${RELATION_SOURCE})
|
||||
|
||||
target_link_options(funasr-wss-server PRIVATE "-Wl,--no-as-needed")
|
||||
target_link_options(funasr-wss-server-2pass PRIVATE "-Wl,--no-as-needed")
|
||||
|
||||
target_link_libraries(funasr-wss-client PUBLIC funasr ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY})
|
||||
target_link_libraries(funasr-wss-client-2pass PUBLIC funasr ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY} portaudio)
|
||||
target_link_libraries(funasr-wss-server PUBLIC funasr ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY})
|
||||
target_link_libraries(funasr-wss-server-2pass PUBLIC funasr ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY})
|
||||
@@ -0,0 +1,658 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
/* 2022-2023 by zhaomingwork */
|
||||
|
||||
// client for websocket, support multiple threads
|
||||
// ./funasr-wss-client --server-ip <string>
|
||||
// --port <string>
|
||||
// --wav-path <string>
|
||||
// [--thread-num <int>]
|
||||
// [--is-ssl <int>] [--]
|
||||
// [--version] [-h]
|
||||
// example:
|
||||
// ./funasr-wss-client --server-ip 127.0.0.1 --port 10095 --wav-path test.wav
|
||||
// --thread-num 1 --is-ssl 1
|
||||
|
||||
#define ASIO_STANDALONE 1
|
||||
#include <glog/logging.h>
|
||||
#include "portaudio.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
#include <websocketpp/client.hpp>
|
||||
#include <websocketpp/common/thread.hpp>
|
||||
#include <websocketpp/config/asio_client.hpp>
|
||||
#include "util.h"
|
||||
#include "audio.h"
|
||||
#include "nlohmann/json.hpp"
|
||||
#include "tclap/CmdLine.h"
|
||||
#include "microphone.h"
|
||||
|
||||
/**
|
||||
* Define a semi-cross platform helper method that waits/sleeps for a bit.
|
||||
*/
|
||||
void WaitABit() {
|
||||
#ifdef WIN32
|
||||
Sleep(500);
|
||||
#else
|
||||
usleep(500);
|
||||
#endif
|
||||
}
|
||||
std::atomic<int> wav_index(0);
|
||||
|
||||
typedef websocketpp::config::asio_client::message_type::ptr message_ptr;
|
||||
typedef websocketpp::lib::shared_ptr<websocketpp::lib::asio::ssl::context>
|
||||
context_ptr;
|
||||
using websocketpp::lib::bind;
|
||||
using websocketpp::lib::placeholders::_1;
|
||||
using websocketpp::lib::placeholders::_2;
|
||||
context_ptr OnTlsInit(websocketpp::connection_hdl) {
|
||||
context_ptr ctx = websocketpp::lib::make_shared<asio::ssl::context>(
|
||||
asio::ssl::context::sslv23);
|
||||
|
||||
try {
|
||||
ctx->set_options(
|
||||
asio::ssl::context::default_workarounds | asio::ssl::context::no_sslv2 |
|
||||
asio::ssl::context::no_sslv3 | asio::ssl::context::single_dh_use);
|
||||
|
||||
} catch (std::exception& e) {
|
||||
LOG(ERROR) << e.what();
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
// template for tls or not config
|
||||
template <typename T>
|
||||
class WebsocketClient {
|
||||
public:
|
||||
// typedef websocketpp::client<T> client;
|
||||
// typedef websocketpp::client<websocketpp::config::asio_tls_client>
|
||||
// wss_client;
|
||||
typedef websocketpp::lib::lock_guard<websocketpp::lib::mutex> scoped_lock;
|
||||
|
||||
WebsocketClient(int is_ssl) : m_open(false), m_done(false) {
|
||||
// set up access channels to only log interesting things
|
||||
m_client.clear_access_channels(websocketpp::log::alevel::all);
|
||||
m_client.set_access_channels(websocketpp::log::alevel::connect);
|
||||
m_client.set_access_channels(websocketpp::log::alevel::disconnect);
|
||||
m_client.set_access_channels(websocketpp::log::alevel::app);
|
||||
|
||||
// Initialize the Asio transport policy
|
||||
m_client.init_asio();
|
||||
|
||||
// Bind the handlers we are using
|
||||
using websocketpp::lib::bind;
|
||||
using websocketpp::lib::placeholders::_1;
|
||||
m_client.set_open_handler(bind(&WebsocketClient::on_open, this, _1));
|
||||
m_client.set_close_handler(bind(&WebsocketClient::on_close, this, _1));
|
||||
|
||||
m_client.set_message_handler(
|
||||
[this](websocketpp::connection_hdl hdl, message_ptr msg) {
|
||||
on_message(hdl, msg);
|
||||
});
|
||||
|
||||
m_client.set_fail_handler(bind(&WebsocketClient::on_fail, this, _1));
|
||||
m_client.clear_access_channels(websocketpp::log::alevel::all);
|
||||
}
|
||||
|
||||
void on_message(websocketpp::connection_hdl hdl, message_ptr msg) {
|
||||
const std::string& payload = msg->get_payload();
|
||||
switch (msg->get_opcode()) {
|
||||
case websocketpp::frame::opcode::text:
|
||||
nlohmann::json jsonresult = nlohmann::json::parse(payload);
|
||||
LOG(INFO) << "Thread: " << this_thread::get_id()
|
||||
<< ",on_message = " << payload;
|
||||
|
||||
if (jsonresult["is_final"] == true) {
|
||||
websocketpp::lib::error_code ec;
|
||||
|
||||
m_client.close(hdl, websocketpp::close::status::going_away, "", ec);
|
||||
|
||||
if (ec) {
|
||||
LOG(ERROR) << "Error closing connection " << ec.message();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This method will block until the connection is complete
|
||||
void run(const std::string& uri, const std::vector<string>& wav_list,
|
||||
const std::vector<string>& wav_ids, int audio_fs, std::string asr_mode,
|
||||
std::vector<int> chunk_size, const std::unordered_map<std::string, int>& hws_map,
|
||||
bool is_record=false, int use_itn=1, int svs_itn=1) {
|
||||
// Create a new connection to the given URI
|
||||
websocketpp::lib::error_code ec;
|
||||
typename websocketpp::client<T>::connection_ptr con =
|
||||
m_client.get_connection(uri, ec);
|
||||
if (ec) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Get Connection Error: " + ec.message());
|
||||
return;
|
||||
}
|
||||
// Grab a handle for this connection so we can talk to it in a thread
|
||||
// safe manor after the event loop starts.
|
||||
m_hdl = con->get_handle();
|
||||
|
||||
// Queue the connection. No DNS queries or network connections will be
|
||||
// made until the io_service event loop is run.
|
||||
m_client.connect(con);
|
||||
|
||||
// Create a thread to run the ASIO io_service event loop
|
||||
websocketpp::lib::thread asio_thread(&websocketpp::client<T>::run,
|
||||
&m_client);
|
||||
if(is_record){
|
||||
send_rec_data(asr_mode, chunk_size, hws_map, use_itn, svs_itn);
|
||||
}else{
|
||||
send_wav_data(wav_list[0], wav_ids[0], audio_fs, asr_mode, chunk_size, hws_map, use_itn, svs_itn);
|
||||
}
|
||||
|
||||
WaitABit();
|
||||
|
||||
asio_thread.join();
|
||||
}
|
||||
|
||||
// The open handler will signal that we are ready to start sending data
|
||||
void on_open(websocketpp::connection_hdl) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Connection opened, starting data!");
|
||||
|
||||
scoped_lock guard(m_lock);
|
||||
m_open = true;
|
||||
}
|
||||
|
||||
// The close handler will signal that we should stop sending data
|
||||
void on_close(websocketpp::connection_hdl) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Connection closed, stopping data!");
|
||||
|
||||
scoped_lock guard(m_lock);
|
||||
m_done = true;
|
||||
}
|
||||
|
||||
// The fail handler will signal that we should stop sending data
|
||||
void on_fail(websocketpp::connection_hdl) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Connection failed, stopping data!");
|
||||
|
||||
scoped_lock guard(m_lock);
|
||||
m_done = true;
|
||||
}
|
||||
// send wav to server
|
||||
void send_wav_data(string wav_path, string wav_id, int audio_fs, std::string asr_mode,
|
||||
std::vector<int> chunk_vector, const std::unordered_map<std::string, int>& hws_map,
|
||||
int use_itn, int svs_itn) {
|
||||
uint64_t count = 0;
|
||||
std::stringstream val;
|
||||
|
||||
funasr::Audio audio(1);
|
||||
int32_t sampling_rate = audio_fs;
|
||||
std::string wav_format = "pcm";
|
||||
if (funasr::IsTargetFile(wav_path.c_str(), "wav")) {
|
||||
if (!audio.LoadWav(wav_path.c_str(), &sampling_rate, false))
|
||||
return;
|
||||
} else if (funasr::IsTargetFile(wav_path.c_str(), "pcm")) {
|
||||
if (!audio.LoadPcmwav(wav_path.c_str(), &sampling_rate, false)) return;
|
||||
} else {
|
||||
wav_format = "others";
|
||||
if (!audio.LoadOthers2Char(wav_path.c_str())) return;
|
||||
}
|
||||
|
||||
float* buff;
|
||||
int len;
|
||||
int flag = 0;
|
||||
bool wait = false;
|
||||
while (1) {
|
||||
{
|
||||
scoped_lock guard(m_lock);
|
||||
// If the connection has been closed, stop generating data
|
||||
if (m_done) {
|
||||
break;
|
||||
}
|
||||
// If the connection hasn't been opened yet wait a bit and retry
|
||||
if (!m_open) {
|
||||
wait = true;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (wait) {
|
||||
// LOG(INFO) << "wait.." << m_open;
|
||||
WaitABit();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
websocketpp::lib::error_code ec;
|
||||
|
||||
nlohmann::json jsonbegin;
|
||||
nlohmann::json chunk_size = nlohmann::json::array();
|
||||
chunk_size.push_back(chunk_vector[0]);
|
||||
chunk_size.push_back(chunk_vector[1]);
|
||||
chunk_size.push_back(chunk_vector[2]);
|
||||
jsonbegin["mode"] = asr_mode;
|
||||
jsonbegin["chunk_size"] = chunk_size;
|
||||
jsonbegin["wav_name"] = wav_id;
|
||||
jsonbegin["wav_format"] = wav_format;
|
||||
jsonbegin["audio_fs"] = sampling_rate;
|
||||
jsonbegin["is_speaking"] = true;
|
||||
jsonbegin["itn"] = true;
|
||||
jsonbegin["svs_itn"] = true;
|
||||
if(use_itn == 0){
|
||||
jsonbegin["itn"] = false;
|
||||
}
|
||||
if(svs_itn == 0){
|
||||
jsonbegin["svs_itn"] = false;
|
||||
}
|
||||
if(!hws_map.empty()){
|
||||
LOG(INFO) << "hotwords: ";
|
||||
for (const auto& pair : hws_map) {
|
||||
LOG(INFO) << pair.first << " : " << pair.second;
|
||||
}
|
||||
nlohmann::json json_map(hws_map);
|
||||
std::string json_map_str = json_map.dump();
|
||||
jsonbegin["hotwords"] = json_map_str;
|
||||
}
|
||||
m_client.send(m_hdl, jsonbegin.dump(), websocketpp::frame::opcode::text,
|
||||
ec);
|
||||
|
||||
// fetch wav data use asr engine api
|
||||
if (wav_format == "pcm") {
|
||||
while (audio.Fetch(buff, len, flag) > 0) {
|
||||
short* iArray = new short[len];
|
||||
for (size_t i = 0; i < len; ++i) {
|
||||
iArray[i] = (short)(buff[i] * 32768);
|
||||
}
|
||||
|
||||
// send data to server
|
||||
int offset = 0;
|
||||
int block_size = 102400;
|
||||
while (offset < len) {
|
||||
int send_block = 0;
|
||||
if (offset + block_size <= len) {
|
||||
send_block = block_size;
|
||||
} else {
|
||||
send_block = len - offset;
|
||||
}
|
||||
m_client.send(m_hdl, iArray + offset, send_block * sizeof(short),
|
||||
websocketpp::frame::opcode::binary, ec);
|
||||
offset += send_block;
|
||||
}
|
||||
|
||||
LOG(INFO) << "sended data len=" << len * sizeof(short);
|
||||
if (ec) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Send Error: " + ec.message());
|
||||
break;
|
||||
}
|
||||
delete[] iArray;
|
||||
}
|
||||
} else {
|
||||
int offset = 0;
|
||||
int block_size = 204800;
|
||||
len = audio.GetSpeechLen();
|
||||
char* others_buff = audio.GetSpeechChar();
|
||||
|
||||
while (offset < len) {
|
||||
int send_block = 0;
|
||||
if (offset + block_size <= len) {
|
||||
send_block = block_size;
|
||||
} else {
|
||||
send_block = len - offset;
|
||||
}
|
||||
m_client.send(m_hdl, others_buff + offset, send_block,
|
||||
websocketpp::frame::opcode::binary, ec);
|
||||
offset += send_block;
|
||||
}
|
||||
|
||||
LOG(INFO) << "sended data len=" << len;
|
||||
if (ec) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Send Error: " + ec.message());
|
||||
}
|
||||
}
|
||||
|
||||
nlohmann::json jsonresult;
|
||||
jsonresult["is_speaking"] = false;
|
||||
m_client.send(m_hdl, jsonresult.dump(), websocketpp::frame::opcode::text,
|
||||
ec);
|
||||
WaitABit();
|
||||
}
|
||||
|
||||
static int RecordCallback(const void* inputBuffer, void* outputBuffer,
|
||||
unsigned long framesPerBuffer, const PaStreamCallbackTimeInfo* timeInfo,
|
||||
PaStreamCallbackFlags statusFlags, void* userData)
|
||||
{
|
||||
std::vector<float>* buffer = static_cast<std::vector<float>*>(userData);
|
||||
const float* input = static_cast<const float*>(inputBuffer);
|
||||
|
||||
for (unsigned int i = 0; i < framesPerBuffer; i++)
|
||||
{
|
||||
buffer->push_back(input[i]);
|
||||
}
|
||||
|
||||
return paContinue;
|
||||
}
|
||||
|
||||
void send_rec_data(std::string asr_mode, std::vector<int> chunk_vector,
|
||||
const std::unordered_map<std::string, int>& hws_map, int use_itn, int svs_itn) {
|
||||
// first message
|
||||
bool wait = false;
|
||||
while (1) {
|
||||
{
|
||||
scoped_lock guard(m_lock);
|
||||
// If the connection has been closed, stop generating data
|
||||
if (m_done) {
|
||||
break;
|
||||
}
|
||||
// If the connection hasn't been opened yet wait a bit and retry
|
||||
if (!m_open) {
|
||||
wait = true;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (wait) {
|
||||
// LOG(INFO) << "wait.." << m_open;
|
||||
WaitABit();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
websocketpp::lib::error_code ec;
|
||||
|
||||
float sample_rate = 16000;
|
||||
nlohmann::json jsonbegin;
|
||||
nlohmann::json chunk_size = nlohmann::json::array();
|
||||
chunk_size.push_back(chunk_vector[0]);
|
||||
chunk_size.push_back(chunk_vector[1]);
|
||||
chunk_size.push_back(chunk_vector[2]);
|
||||
jsonbegin["mode"] = asr_mode;
|
||||
jsonbegin["chunk_size"] = chunk_size;
|
||||
jsonbegin["wav_name"] = "record";
|
||||
jsonbegin["wav_format"] = "pcm";
|
||||
jsonbegin["audio_fs"] = sample_rate;
|
||||
jsonbegin["is_speaking"] = true;
|
||||
jsonbegin["itn"] = true;
|
||||
jsonbegin["svs_itn"] = true;
|
||||
if(use_itn == 0){
|
||||
jsonbegin["itn"] = false;
|
||||
}
|
||||
if(svs_itn == 0){
|
||||
jsonbegin["svs_itn"] = false;
|
||||
}
|
||||
if(!hws_map.empty()){
|
||||
LOG(INFO) << "hotwords: ";
|
||||
for (const auto& pair : hws_map) {
|
||||
LOG(INFO) << pair.first << " : " << pair.second;
|
||||
}
|
||||
nlohmann::json json_map(hws_map);
|
||||
std::string json_map_str = json_map.dump();
|
||||
jsonbegin["hotwords"] = json_map_str;
|
||||
}
|
||||
m_client.send(m_hdl, jsonbegin.dump(), websocketpp::frame::opcode::text,
|
||||
ec);
|
||||
// mic
|
||||
Microphone mic;
|
||||
PaDeviceIndex num_devices = Pa_GetDeviceCount();
|
||||
LOG(INFO) << "Num devices: " << num_devices;
|
||||
|
||||
PaStreamParameters param;
|
||||
|
||||
param.device = Pa_GetDefaultInputDevice();
|
||||
if (param.device == paNoDevice) {
|
||||
LOG(INFO) << "No default input device found";
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
LOG(INFO) << "Use default device: " << param.device;
|
||||
|
||||
const PaDeviceInfo *info = Pa_GetDeviceInfo(param.device);
|
||||
LOG(INFO) << " Name: " << info->name;
|
||||
LOG(INFO) << " Max input channels: " << info->maxInputChannels;
|
||||
|
||||
param.channelCount = 1;
|
||||
param.sampleFormat = paFloat32;
|
||||
|
||||
param.suggestedLatency = info->defaultLowInputLatency;
|
||||
param.hostApiSpecificStreamInfo = nullptr;
|
||||
|
||||
PaStream *stream;
|
||||
std::vector<float> buffer;
|
||||
PaError err =
|
||||
Pa_OpenStream(&stream, ¶m, nullptr, /* &outputParameters, */
|
||||
sample_rate,
|
||||
0, // frames per buffer
|
||||
paClipOff, // we won't output out of range samples
|
||||
// so don't bother clipping them
|
||||
RecordCallback, &buffer);
|
||||
if (err != paNoError) {
|
||||
LOG(ERROR) << "portaudio error: " << Pa_GetErrorText(err);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
err = Pa_StartStream(stream);
|
||||
LOG(INFO) << "Started: ";
|
||||
|
||||
if (err != paNoError) {
|
||||
LOG(ERROR) << "portaudio error: " << Pa_GetErrorText(err);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
while(true){
|
||||
int len = buffer.size();
|
||||
short* iArray = new short[len];
|
||||
for (size_t i = 0; i < len; ++i) {
|
||||
iArray[i] = (short)(buffer[i] * 32768);
|
||||
}
|
||||
|
||||
m_client.send(m_hdl, iArray, len * sizeof(short),
|
||||
websocketpp::frame::opcode::binary, ec);
|
||||
buffer.clear();
|
||||
|
||||
if (ec) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Send Error: " + ec.message());
|
||||
}
|
||||
delete[] iArray;
|
||||
Pa_Sleep(20); // sleep for 20ms
|
||||
}
|
||||
|
||||
nlohmann::json jsonresult;
|
||||
jsonresult["is_speaking"] = false;
|
||||
m_client.send(m_hdl, jsonresult.dump(), websocketpp::frame::opcode::text,
|
||||
ec);
|
||||
|
||||
err = Pa_CloseStream(stream);
|
||||
if (err != paNoError) {
|
||||
LOG(INFO) << "portaudio error: " << Pa_GetErrorText(err);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
websocketpp::client<T> m_client;
|
||||
|
||||
private:
|
||||
websocketpp::connection_hdl m_hdl;
|
||||
websocketpp::lib::mutex m_lock;
|
||||
bool m_open;
|
||||
bool m_done;
|
||||
int total_num = 0;
|
||||
};
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
SetConsoleOutputCP(65001);
|
||||
#endif
|
||||
google::InitGoogleLogging(argv[0]);
|
||||
FLAGS_logtostderr = true;
|
||||
|
||||
TCLAP::CmdLine cmd("funasr-wss-client-2pass", ' ', "1.0");
|
||||
TCLAP::ValueArg<std::string> server_ip_("", "server-ip", "server-ip", true,
|
||||
"127.0.0.1", "string");
|
||||
TCLAP::ValueArg<std::string> port_("", "port", "port", true, "10095",
|
||||
"string");
|
||||
TCLAP::ValueArg<std::string> wav_path_(
|
||||
"", "wav-path",
|
||||
"the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: "
|
||||
"asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)",
|
||||
false, "", "string");
|
||||
TCLAP::ValueArg<std::int32_t> audio_fs_("", "audio-fs", "the sample rate of audio", false, 16000, "int32_t");
|
||||
TCLAP::ValueArg<int> record_(
|
||||
"", "record",
|
||||
"record is 1 means use record", false, 0,
|
||||
"int");
|
||||
TCLAP::ValueArg<std::string> asr_mode_("", ASR_MODE, "offline, online, 2pass",
|
||||
false, "2pass", "string");
|
||||
TCLAP::ValueArg<std::string> chunk_size_("", "chunk-size",
|
||||
"chunk_size: 5-10-5 or 5-12-5",
|
||||
false, "5-10-5", "string");
|
||||
TCLAP::ValueArg<int> thread_num_("", "thread-num", "thread-num", false, 1,
|
||||
"int");
|
||||
TCLAP::ValueArg<int> is_ssl_(
|
||||
"", "is-ssl",
|
||||
"is-ssl is 1 means use wss connection, or use ws connection", false, 1,
|
||||
"int");
|
||||
TCLAP::ValueArg<int> use_itn_(
|
||||
"", "use-itn",
|
||||
"use-itn is 1 means use itn, 0 means not use itn", false, 1,
|
||||
"int");
|
||||
TCLAP::ValueArg<int> svs_itn_(
|
||||
"", "svs-itn",
|
||||
"svs-itn is 1 means use itn and punc, 0 means not use", false, 1, "int");
|
||||
TCLAP::ValueArg<std::string> hotword_("", HOTWORD,
|
||||
"the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string");
|
||||
|
||||
cmd.add(server_ip_);
|
||||
cmd.add(port_);
|
||||
cmd.add(wav_path_);
|
||||
cmd.add(audio_fs_);
|
||||
cmd.add(asr_mode_);
|
||||
cmd.add(record_);
|
||||
cmd.add(chunk_size_);
|
||||
cmd.add(thread_num_);
|
||||
cmd.add(is_ssl_);
|
||||
cmd.add(use_itn_);
|
||||
cmd.add(svs_itn_);
|
||||
cmd.add(hotword_);
|
||||
cmd.parse(argc, argv);
|
||||
|
||||
std::string server_ip = server_ip_.getValue();
|
||||
std::string port = port_.getValue();
|
||||
std::string wav_path = wav_path_.getValue();
|
||||
std::string asr_mode = asr_mode_.getValue();
|
||||
std::string chunk_size_str = chunk_size_.getValue();
|
||||
int use_itn = use_itn_.getValue();
|
||||
int svs_itn = svs_itn_.getValue();
|
||||
// get chunk_size
|
||||
std::vector<int> chunk_size;
|
||||
std::stringstream ss(chunk_size_str);
|
||||
std::string item;
|
||||
while (std::getline(ss, item, '-')) {
|
||||
try {
|
||||
chunk_size.push_back(stoi(item));
|
||||
} catch (const invalid_argument&) {
|
||||
LOG(ERROR) << "Invalid argument: " << item;
|
||||
exit(-1);
|
||||
}
|
||||
}
|
||||
|
||||
int threads_num = thread_num_.getValue();
|
||||
int is_ssl = is_ssl_.getValue();
|
||||
int is_record = record_.getValue();
|
||||
|
||||
std::string uri = "";
|
||||
if (is_ssl == 1) {
|
||||
uri = "wss://" + server_ip + ":" + port;
|
||||
} else {
|
||||
uri = "ws://" + server_ip + ":" + port;
|
||||
}
|
||||
|
||||
// hotwords
|
||||
std::string hotword_path = hotword_.getValue();
|
||||
unordered_map<string, int> hws_map;
|
||||
if(!hotword_path.empty()){
|
||||
LOG(INFO) << "hotword path: " << hotword_path;
|
||||
funasr::ExtractHws(hotword_path, hws_map);
|
||||
}
|
||||
|
||||
int audio_fs = audio_fs_.getValue();
|
||||
if(is_record == 1){
|
||||
std::vector<string> tmp_wav_list;
|
||||
std::vector<string> tmp_wav_ids;
|
||||
|
||||
if (is_ssl == 1) {
|
||||
WebsocketClient<websocketpp::config::asio_tls_client> c(is_ssl);
|
||||
|
||||
c.m_client.set_tls_init_handler(bind(&OnTlsInit, ::_1));
|
||||
|
||||
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, true, use_itn, svs_itn);
|
||||
} else {
|
||||
WebsocketClient<websocketpp::config::asio_client> c(is_ssl);
|
||||
|
||||
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, true, use_itn, svs_itn);
|
||||
}
|
||||
|
||||
}else{
|
||||
// read wav_path
|
||||
std::vector<string> wav_list;
|
||||
std::vector<string> wav_ids;
|
||||
string default_id = "wav_default_id";
|
||||
if (funasr::IsTargetFile(wav_path, "scp")) {
|
||||
ifstream in(wav_path);
|
||||
if (!in.is_open()) {
|
||||
printf("Failed to open scp file");
|
||||
return 0;
|
||||
}
|
||||
string line;
|
||||
while (getline(in, line)) {
|
||||
istringstream iss(line);
|
||||
string column1, column2;
|
||||
iss >> column1 >> column2;
|
||||
wav_list.emplace_back(column2);
|
||||
wav_ids.emplace_back(column1);
|
||||
}
|
||||
in.close();
|
||||
} else {
|
||||
wav_list.emplace_back(wav_path);
|
||||
wav_ids.emplace_back(default_id);
|
||||
}
|
||||
|
||||
for (size_t wav_i = 0; wav_i < wav_list.size(); wav_i = wav_i + threads_num) {
|
||||
std::vector<websocketpp::lib::thread> client_threads;
|
||||
for (size_t i = 0; i < threads_num; i++) {
|
||||
if (wav_i + i >= wav_list.size()) {
|
||||
break;
|
||||
}
|
||||
std::vector<string> tmp_wav_list;
|
||||
std::vector<string> tmp_wav_ids;
|
||||
|
||||
tmp_wav_list.emplace_back(wav_list[wav_i + i]);
|
||||
tmp_wav_ids.emplace_back(wav_ids[wav_i + i]);
|
||||
|
||||
client_threads.emplace_back(
|
||||
[uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, is_ssl, hws_map, use_itn, svs_itn]() {
|
||||
if (is_ssl == 1) {
|
||||
WebsocketClient<websocketpp::config::asio_tls_client> c(is_ssl);
|
||||
|
||||
c.m_client.set_tls_init_handler(bind(&OnTlsInit, ::_1));
|
||||
|
||||
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, false, use_itn, svs_itn);
|
||||
} else {
|
||||
WebsocketClient<websocketpp::config::asio_client> c(is_ssl);
|
||||
|
||||
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, false, use_itn, svs_itn);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (auto& t : client_threads) {
|
||||
t.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
/* 2022-2023 by zhaomingwork */
|
||||
|
||||
// client for websocket, support multiple threads
|
||||
// ./funasr-wss-client --server-ip <string>
|
||||
// --port <string>
|
||||
// --wav-path <string>
|
||||
// [--thread-num <int>]
|
||||
// [--is-ssl <int>] [--]
|
||||
// [--version] [-h]
|
||||
// example:
|
||||
// ./funasr-wss-client --server-ip 127.0.0.1 --port 10095 --wav-path test.wav --thread-num 1 --is-ssl 1
|
||||
|
||||
#define ASIO_STANDALONE 1
|
||||
#include <websocketpp/client.hpp>
|
||||
#include <websocketpp/common/thread.hpp>
|
||||
#include <websocketpp/config/asio_client.hpp>
|
||||
#include <fstream>
|
||||
#include <atomic>
|
||||
#include <thread>
|
||||
#include <glog/logging.h>
|
||||
#include "util.h"
|
||||
#include "audio.h"
|
||||
#include "nlohmann/json.hpp"
|
||||
#include "tclap/CmdLine.h"
|
||||
|
||||
/**
|
||||
* Define a semi-cross platform helper method that waits/sleeps for a bit.
|
||||
*/
|
||||
void WaitABit() {
|
||||
#ifdef WIN32
|
||||
Sleep(200);
|
||||
#else
|
||||
usleep(200);
|
||||
#endif
|
||||
}
|
||||
std::atomic<int> wav_index(0);
|
||||
|
||||
typedef websocketpp::config::asio_client::message_type::ptr message_ptr;
|
||||
typedef websocketpp::lib::shared_ptr<websocketpp::lib::asio::ssl::context> context_ptr;
|
||||
using websocketpp::lib::bind;
|
||||
using websocketpp::lib::placeholders::_1;
|
||||
using websocketpp::lib::placeholders::_2;
|
||||
context_ptr OnTlsInit(websocketpp::connection_hdl) {
|
||||
context_ptr ctx = websocketpp::lib::make_shared<asio::ssl::context>(
|
||||
asio::ssl::context::sslv23);
|
||||
|
||||
try {
|
||||
ctx->set_options(
|
||||
asio::ssl::context::default_workarounds | asio::ssl::context::no_sslv2 |
|
||||
asio::ssl::context::no_sslv3 | asio::ssl::context::single_dh_use);
|
||||
|
||||
} catch (std::exception& e) {
|
||||
LOG(ERROR) << e.what();
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
// template for tls or not config
|
||||
template <typename T>
|
||||
class WebsocketClient {
|
||||
public:
|
||||
// typedef websocketpp::client<T> client;
|
||||
// typedef websocketpp::client<websocketpp::config::asio_tls_client>
|
||||
// wss_client;
|
||||
typedef websocketpp::lib::lock_guard<websocketpp::lib::mutex> scoped_lock;
|
||||
|
||||
WebsocketClient(int is_ssl) : m_open(false), m_done(false) {
|
||||
// set up access channels to only log interesting things
|
||||
m_client.clear_access_channels(websocketpp::log::alevel::all);
|
||||
m_client.set_access_channels(websocketpp::log::alevel::connect);
|
||||
m_client.set_access_channels(websocketpp::log::alevel::disconnect);
|
||||
m_client.set_access_channels(websocketpp::log::alevel::app);
|
||||
|
||||
// Initialize the Asio transport policy
|
||||
m_client.init_asio();
|
||||
|
||||
// Bind the handlers we are using
|
||||
using websocketpp::lib::bind;
|
||||
using websocketpp::lib::placeholders::_1;
|
||||
m_client.set_open_handler(bind(&WebsocketClient::on_open, this, _1));
|
||||
m_client.set_close_handler(bind(&WebsocketClient::on_close, this, _1));
|
||||
|
||||
m_client.set_message_handler(
|
||||
[this](websocketpp::connection_hdl hdl, message_ptr msg) {
|
||||
on_message(hdl, msg);
|
||||
});
|
||||
|
||||
m_client.set_fail_handler(bind(&WebsocketClient::on_fail, this, _1));
|
||||
m_client.clear_access_channels(websocketpp::log::alevel::all);
|
||||
}
|
||||
|
||||
void on_message(websocketpp::connection_hdl hdl, message_ptr msg) {
|
||||
const std::string& payload = msg->get_payload();
|
||||
switch (msg->get_opcode()) {
|
||||
case websocketpp::frame::opcode::text:
|
||||
total_recv=total_recv+1;
|
||||
LOG(INFO)<< "Thread: " << this_thread::get_id() << ", total_recv=" << total_recv <<", on_message = " << payload;
|
||||
std::unique_lock<std::mutex> lock(msg_lock);
|
||||
cv.notify_one();
|
||||
if(close_client)
|
||||
{
|
||||
LOG(INFO)<< "Thread: " << this_thread::get_id() << ", close client thread";
|
||||
websocketpp::lib::error_code ec;
|
||||
m_client.close(m_hdl, websocketpp::close::status::going_away, "", ec);
|
||||
if (ec){
|
||||
LOG(ERROR)<< "Error closing connection " << ec.message();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This method will block until the connection is complete
|
||||
void run(const std::string& uri, const std::vector<string>& wav_list, const std::vector<string>& wav_ids,
|
||||
int audio_fs, const std::unordered_map<std::string, int>& hws_map, int use_itn=1, int svs_itn=1) {
|
||||
// Create a new connection to the given URI
|
||||
websocketpp::lib::error_code ec;
|
||||
typename websocketpp::client<T>::connection_ptr con =
|
||||
m_client.get_connection(uri, ec);
|
||||
if (ec) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Get Connection Error: " + ec.message());
|
||||
return;
|
||||
}
|
||||
// Grab a handle for this connection so we can talk to it in a thread
|
||||
// safe manor after the event loop starts.
|
||||
m_hdl = con->get_handle();
|
||||
|
||||
// Queue the connection. No DNS queries or network connections will be
|
||||
// made until the io_service event loop is run.
|
||||
m_client.connect(con);
|
||||
|
||||
// Create a thread to run the ASIO io_service event loop
|
||||
websocketpp::lib::thread asio_thread(&websocketpp::client<T>::run,
|
||||
&m_client);
|
||||
bool send_hotword = true;
|
||||
while(true){
|
||||
int i = wav_index.fetch_add(1);
|
||||
if (i >= wav_list.size()) {
|
||||
break;
|
||||
}
|
||||
if (total_send !=0){
|
||||
std::unique_lock<std::mutex> lock(msg_lock);
|
||||
cv.wait(lock);
|
||||
}
|
||||
total_send += 1;
|
||||
send_wav_data(wav_list[i], wav_ids[i], audio_fs, hws_map, send_hotword, use_itn, svs_itn);
|
||||
if(send_hotword){
|
||||
send_hotword = false;
|
||||
}
|
||||
}
|
||||
close_client = true;
|
||||
asio_thread.join();
|
||||
|
||||
}
|
||||
|
||||
// The open handler will signal that we are ready to start sending data
|
||||
void on_open(websocketpp::connection_hdl) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Connection opened, starting data!");
|
||||
|
||||
scoped_lock guard(m_lock);
|
||||
m_open = true;
|
||||
}
|
||||
|
||||
// The close handler will signal that we should stop sending data
|
||||
void on_close(websocketpp::connection_hdl) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Connection closed, stopping data!");
|
||||
|
||||
scoped_lock guard(m_lock);
|
||||
m_done = true;
|
||||
}
|
||||
|
||||
// The fail handler will signal that we should stop sending data
|
||||
void on_fail(websocketpp::connection_hdl) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Connection failed, stopping data!");
|
||||
|
||||
scoped_lock guard(m_lock);
|
||||
m_done = true;
|
||||
}
|
||||
// send wav to server
|
||||
void send_wav_data(string wav_path, string wav_id, int audio_fs,
|
||||
const std::unordered_map<std::string, int>& hws_map,
|
||||
bool send_hotword, bool use_itn, bool svs_itn) {
|
||||
uint64_t count = 0;
|
||||
std::stringstream val;
|
||||
|
||||
funasr::Audio audio(1);
|
||||
int32_t sampling_rate = audio_fs;
|
||||
std::string wav_format = "pcm";
|
||||
if(funasr::IsTargetFile(wav_path.c_str(), "pcm")){
|
||||
if (!audio.LoadPcmwav(wav_path.c_str(), &sampling_rate, false))
|
||||
return ;
|
||||
}else{
|
||||
wav_format = "others";
|
||||
if (!audio.LoadOthers2Char(wav_path.c_str()))
|
||||
return ;
|
||||
}
|
||||
|
||||
float* buff;
|
||||
int len;
|
||||
int flag = 0;
|
||||
bool wait = false;
|
||||
while (1) {
|
||||
{
|
||||
scoped_lock guard(m_lock);
|
||||
// If the connection has been closed, stop generating data
|
||||
if (m_done) {
|
||||
break;
|
||||
}
|
||||
// If the connection hasn't been opened yet wait a bit and retry
|
||||
if (!m_open) {
|
||||
wait = true;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (wait) {
|
||||
// LOG(INFO) << "wait.." << m_open;
|
||||
WaitABit();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
websocketpp::lib::error_code ec;
|
||||
|
||||
nlohmann::json jsonbegin;
|
||||
nlohmann::json chunk_size = nlohmann::json::array();
|
||||
chunk_size.push_back(5);
|
||||
chunk_size.push_back(10);
|
||||
chunk_size.push_back(5);
|
||||
jsonbegin["chunk_size"] = chunk_size;
|
||||
jsonbegin["chunk_interval"] = 10;
|
||||
jsonbegin["wav_name"] = wav_id;
|
||||
jsonbegin["wav_format"] = wav_format;
|
||||
jsonbegin["audio_fs"] = sampling_rate;
|
||||
jsonbegin["itn"] = true;
|
||||
jsonbegin["svs_itn"] = true;
|
||||
if(use_itn == 0){
|
||||
jsonbegin["itn"] = false;
|
||||
}
|
||||
if(svs_itn == 0){
|
||||
jsonbegin["svs_itn"] = false;
|
||||
}
|
||||
jsonbegin["is_speaking"] = true;
|
||||
if(send_hotword){
|
||||
if(!hws_map.empty()){
|
||||
LOG(INFO) << "hotwords: ";
|
||||
for (const auto& pair : hws_map) {
|
||||
LOG(INFO) << pair.first << " : " << pair.second;
|
||||
}
|
||||
nlohmann::json json_map(hws_map);
|
||||
std::string json_map_str = json_map.dump();
|
||||
jsonbegin["hotwords"] = json_map_str;
|
||||
}
|
||||
}
|
||||
m_client.send(m_hdl, jsonbegin.dump(), websocketpp::frame::opcode::text,
|
||||
ec);
|
||||
|
||||
// fetch wav data use asr engine api
|
||||
if(wav_format == "pcm"){
|
||||
while (audio.Fetch(buff, len, flag) > 0) {
|
||||
short* iArray = new short[len];
|
||||
for (size_t i = 0; i < len; ++i) {
|
||||
iArray[i] = (short)(buff[i]*32768);
|
||||
}
|
||||
|
||||
// send data to server
|
||||
int offset = 0;
|
||||
int block_size = 102400;
|
||||
while(offset < len){
|
||||
int send_block = 0;
|
||||
if (offset + block_size <= len){
|
||||
send_block = block_size;
|
||||
}else{
|
||||
send_block = len - offset;
|
||||
}
|
||||
m_client.send(m_hdl, iArray+offset, send_block * sizeof(short),
|
||||
websocketpp::frame::opcode::binary, ec);
|
||||
offset += send_block;
|
||||
}
|
||||
|
||||
LOG(INFO)<< "Thread: " << this_thread::get_id() << ", sended data len=" << len * sizeof(short);
|
||||
// The most likely error that we will get is that the connection is
|
||||
// not in the right state. Usually this means we tried to send a
|
||||
// message to a connection that was closed or in the process of
|
||||
// closing. While many errors here can be easily recovered from,
|
||||
// in this simple example, we'll stop the data loop.
|
||||
if (ec) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Send Error: " + ec.message());
|
||||
break;
|
||||
}
|
||||
delete[] iArray;
|
||||
// WaitABit();
|
||||
}
|
||||
}else{
|
||||
int offset = 0;
|
||||
int block_size = 204800;
|
||||
len = audio.GetSpeechLen();
|
||||
char* others_buff = audio.GetSpeechChar();
|
||||
|
||||
while(offset < len){
|
||||
int send_block = 0;
|
||||
if (offset + block_size <= len){
|
||||
send_block = block_size;
|
||||
}else{
|
||||
send_block = len - offset;
|
||||
}
|
||||
m_client.send(m_hdl, others_buff+offset, send_block,
|
||||
websocketpp::frame::opcode::binary, ec);
|
||||
offset += send_block;
|
||||
}
|
||||
|
||||
LOG(INFO)<< "Thread: " << this_thread::get_id() << ", sended data len=" << len;
|
||||
// The most likely error that we will get is that the connection is
|
||||
// not in the right state. Usually this means we tried to send a
|
||||
// message to a connection that was closed or in the process of
|
||||
// closing. While many errors here can be easily recovered from,
|
||||
// in this simple example, we'll stop the data loop.
|
||||
if (ec) {
|
||||
m_client.get_alog().write(websocketpp::log::alevel::app,
|
||||
"Send Error: " + ec.message());
|
||||
}
|
||||
}
|
||||
|
||||
nlohmann::json jsonresult;
|
||||
jsonresult["is_speaking"] = false;
|
||||
m_client.send(m_hdl, jsonresult.dump(), websocketpp::frame::opcode::text,
|
||||
ec);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(20));
|
||||
}
|
||||
websocketpp::client<T> m_client;
|
||||
|
||||
private:
|
||||
websocketpp::connection_hdl m_hdl;
|
||||
websocketpp::lib::mutex m_lock;
|
||||
websocketpp::lib::mutex msg_lock;
|
||||
websocketpp::lib::condition_variable cv;
|
||||
bool m_open;
|
||||
bool m_done;
|
||||
bool close_client=false;
|
||||
int total_send=0;
|
||||
int total_recv=0;
|
||||
};
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
SetConsoleOutputCP(65001);
|
||||
#endif
|
||||
google::InitGoogleLogging(argv[0]);
|
||||
FLAGS_logtostderr = true;
|
||||
|
||||
TCLAP::CmdLine cmd("funasr-wss-client", ' ', "1.0");
|
||||
TCLAP::ValueArg<std::string> server_ip_("", "server-ip", "server-ip", true,
|
||||
"127.0.0.1", "string");
|
||||
TCLAP::ValueArg<std::string> port_("", "port", "port", true, "10095", "string");
|
||||
TCLAP::ValueArg<std::string> wav_path_("", "wav-path",
|
||||
"the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)",
|
||||
true, "", "string");
|
||||
TCLAP::ValueArg<std::int32_t> audio_fs_("", "audio-fs", "the sample rate of audio", false, 16000, "int32_t");
|
||||
TCLAP::ValueArg<int> thread_num_("", "thread-num", "thread-num",
|
||||
false, 1, "int");
|
||||
TCLAP::ValueArg<int> is_ssl_(
|
||||
"", "is-ssl", "is-ssl is 1 means use wss connection, or use ws connection",
|
||||
false, 1, "int");
|
||||
TCLAP::ValueArg<int> use_itn_(
|
||||
"", "use-itn",
|
||||
"use-itn is 1 means use itn, 0 means not use itn", false, 1, "int");
|
||||
TCLAP::ValueArg<int> svs_itn_(
|
||||
"", "svs-itn",
|
||||
"svs-itn is 1 means use itn and punc, 0 means not use", false, 1, "int");
|
||||
TCLAP::ValueArg<std::string> hotword_("", HOTWORD,
|
||||
"the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string");
|
||||
|
||||
cmd.add(server_ip_);
|
||||
cmd.add(port_);
|
||||
cmd.add(wav_path_);
|
||||
cmd.add(audio_fs_);
|
||||
cmd.add(thread_num_);
|
||||
cmd.add(is_ssl_);
|
||||
cmd.add(use_itn_);
|
||||
cmd.add(svs_itn_);
|
||||
cmd.add(hotword_);
|
||||
cmd.parse(argc, argv);
|
||||
|
||||
std::string server_ip = server_ip_.getValue();
|
||||
std::string port = port_.getValue();
|
||||
std::string wav_path = wav_path_.getValue();
|
||||
int threads_num = thread_num_.getValue();
|
||||
int is_ssl = is_ssl_.getValue();
|
||||
int use_itn = use_itn_.getValue();
|
||||
int svs_itn = svs_itn_.getValue();
|
||||
|
||||
std::vector<websocketpp::lib::thread> client_threads;
|
||||
std::string uri = "";
|
||||
if (is_ssl == 1) {
|
||||
uri = "wss://" + server_ip + ":" + port;
|
||||
} else {
|
||||
uri = "ws://" + server_ip + ":" + port;
|
||||
}
|
||||
|
||||
// hotwords
|
||||
std::string hotword_path = hotword_.getValue();
|
||||
unordered_map<string, int> hws_map;
|
||||
if(!hotword_path.empty()){
|
||||
LOG(INFO) << "hotword path: " << hotword_path;
|
||||
funasr::ExtractHws(hotword_path, hws_map);
|
||||
}
|
||||
|
||||
// read wav_path
|
||||
std::vector<string> wav_list;
|
||||
std::vector<string> wav_ids;
|
||||
string default_id = "wav_default_id";
|
||||
if(funasr::IsTargetFile(wav_path, "scp")){
|
||||
ifstream in(wav_path);
|
||||
if (!in.is_open()) {
|
||||
printf("Failed to open scp file");
|
||||
return 0;
|
||||
}
|
||||
string line;
|
||||
while(getline(in, line))
|
||||
{
|
||||
istringstream iss(line);
|
||||
string column1, column2;
|
||||
iss >> column1 >> column2;
|
||||
wav_list.emplace_back(column2);
|
||||
wav_ids.emplace_back(column1);
|
||||
}
|
||||
in.close();
|
||||
}else{
|
||||
wav_list.emplace_back(wav_path);
|
||||
wav_ids.emplace_back(default_id);
|
||||
}
|
||||
|
||||
int audio_fs = audio_fs_.getValue();
|
||||
for (size_t i = 0; i < threads_num; i++) {
|
||||
client_threads.emplace_back([uri, wav_list, wav_ids, audio_fs, is_ssl, hws_map, use_itn, svs_itn]() {
|
||||
if (is_ssl == 1) {
|
||||
WebsocketClient<websocketpp::config::asio_tls_client> c(is_ssl);
|
||||
|
||||
c.m_client.set_tls_init_handler(bind(&OnTlsInit, ::_1));
|
||||
|
||||
c.run(uri, wav_list, wav_ids, audio_fs, hws_map, use_itn, svs_itn);
|
||||
} else {
|
||||
WebsocketClient<websocketpp::config::asio_client> c(is_ssl);
|
||||
|
||||
c.run(uri, wav_list, wav_ids, audio_fs, hws_map, use_itn, svs_itn);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (auto& t : client_threads) {
|
||||
t.join();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,597 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
/* 2022-2023 by zhaomingwork */
|
||||
|
||||
#include "websocket-server-2pass.h"
|
||||
#ifdef _WIN32
|
||||
#include "win_func.h"
|
||||
#else
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include <fstream>
|
||||
#include "util.h"
|
||||
|
||||
// hotwords
|
||||
std::unordered_map<std::string, int> hws_map_;
|
||||
int fst_inc_wts_=20;
|
||||
float global_beam_, lattice_beam_, am_scale_;
|
||||
|
||||
using namespace std;
|
||||
void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key,
|
||||
std::map<std::string, std::string>& model_path) {
|
||||
model_path.insert({key, value_arg.getValue()});
|
||||
LOG(INFO) << key << " : " << value_arg.getValue();
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
SetConsoleOutputCP(65001);
|
||||
#endif
|
||||
try {
|
||||
google::InitGoogleLogging(argv[0]);
|
||||
FLAGS_logtostderr = true;
|
||||
std::string tpass_version = "";
|
||||
#ifdef _WIN32
|
||||
tpass_version = "0.1.0";
|
||||
#endif
|
||||
TCLAP::CmdLine cmd("funasr-wss-server", ' ', tpass_version);
|
||||
TCLAP::ValueArg<std::string> download_model_dir(
|
||||
"", "download-model-dir",
|
||||
"Download model from Modelscope to download_model_dir", false,
|
||||
"/workspace/models", "string");
|
||||
TCLAP::ValueArg<std::string> offline_model_dir(
|
||||
"", OFFLINE_MODEL_DIR,
|
||||
"default: damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx, the asr model path, which "
|
||||
"contains model_quant.onnx, config.yaml, am.mvn",
|
||||
false, "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx", "string");
|
||||
TCLAP::ValueArg<std::string> online_model_dir(
|
||||
"", ONLINE_MODEL_DIR,
|
||||
"default: damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx, the asr model path, which "
|
||||
"contains model_quant.onnx, config.yaml, am.mvn",
|
||||
false, "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx", "string");
|
||||
|
||||
TCLAP::ValueArg<std::string> offline_model_revision(
|
||||
"", "offline-model-revision", "ASR offline model revision", false,
|
||||
"v2.0.5", "string");
|
||||
|
||||
TCLAP::ValueArg<std::string> online_model_revision(
|
||||
"", "online-model-revision", "ASR online model revision", false,
|
||||
"v2.0.5", "string");
|
||||
|
||||
TCLAP::ValueArg<std::string> quantize(
|
||||
"", QUANTIZE,
|
||||
"true (Default), load the model of model_quant.onnx in model_dir. If "
|
||||
"set "
|
||||
"false, load the model of model.onnx in model_dir",
|
||||
false, "true", "string");
|
||||
TCLAP::ValueArg<std::string> vad_dir(
|
||||
"", VAD_DIR,
|
||||
"default: damo/speech_fsmn_vad_zh-cn-16k-common-onnx, the vad model path, which contains "
|
||||
"model_quant.onnx, vad.yaml, vad.mvn",
|
||||
false, "damo/speech_fsmn_vad_zh-cn-16k-common-onnx", "string");
|
||||
TCLAP::ValueArg<std::string> vad_revision(
|
||||
"", "vad-revision", "VAD model revision", false, "v2.0.4", "string");
|
||||
TCLAP::ValueArg<std::string> vad_quant(
|
||||
"", VAD_QUANT,
|
||||
"true (Default), load the model of model_quant.onnx in vad_dir. If set "
|
||||
"false, load the model of model.onnx in vad_dir",
|
||||
false, "true", "string");
|
||||
TCLAP::ValueArg<std::string> punc_dir(
|
||||
"", PUNC_DIR,
|
||||
"default: damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx, the punc model path, which contains "
|
||||
"model_quant.onnx, punc.yaml",
|
||||
false, "damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727-onnx", "string");
|
||||
TCLAP::ValueArg<std::string> punc_revision(
|
||||
"", "punc-revision", "PUNC model revision", false, "v2.0.5", "string");
|
||||
TCLAP::ValueArg<std::string> punc_quant(
|
||||
"", PUNC_QUANT,
|
||||
"true (Default), load the model of model_quant.onnx in punc_dir. If "
|
||||
"set "
|
||||
"false, load the model of model.onnx in punc_dir",
|
||||
false, "true", "string");
|
||||
TCLAP::ValueArg<std::string> itn_dir(
|
||||
"", ITN_DIR,
|
||||
"default: thuduj12/fst_itn_zh, the itn model path, which contains "
|
||||
"zh_itn_tagger.fst, zh_itn_verbalizer.fst",
|
||||
false, "thuduj12/fst_itn_zh", "string");
|
||||
TCLAP::ValueArg<std::string> itn_revision(
|
||||
"", "itn-revision", "ITN model revision", false, "v1.0.1", "string");
|
||||
|
||||
TCLAP::ValueArg<std::string> listen_ip("", "listen-ip", "listen ip", false,
|
||||
"0.0.0.0", "string");
|
||||
TCLAP::ValueArg<int> port("", "port", "port", false, 10095, "int");
|
||||
TCLAP::ValueArg<int> io_thread_num("", "io-thread-num", "io thread num",
|
||||
false, 2, "int");
|
||||
TCLAP::ValueArg<int> decoder_thread_num(
|
||||
"", "decoder-thread-num", "decoder thread num", false, 8, "int");
|
||||
TCLAP::ValueArg<int> model_thread_num("", "model-thread-num",
|
||||
"model thread num", false, 2, "int");
|
||||
|
||||
TCLAP::ValueArg<std::string> certfile(
|
||||
"", "certfile",
|
||||
"default: ../../../ssl_key/server.crt, path of certficate for WSS "
|
||||
"connection. if it is empty, it will be in WS mode.",
|
||||
false, "../../../ssl_key/server.crt", "string");
|
||||
TCLAP::ValueArg<std::string> keyfile(
|
||||
"", "keyfile",
|
||||
"default: ../../../ssl_key/server.key, path of keyfile for WSS "
|
||||
"connection",
|
||||
false, "../../../ssl_key/server.key", "string");
|
||||
|
||||
TCLAP::ValueArg<float> global_beam("", GLOB_BEAM, "the decoding beam for beam searching ", false, 3.0, "float");
|
||||
TCLAP::ValueArg<float> lattice_beam("", LAT_BEAM, "the lattice generation beam for beam searching ", false, 3.0, "float");
|
||||
TCLAP::ValueArg<float> am_scale("", AM_SCALE, "the acoustic scale for beam searching ", false, 10.0, "float");
|
||||
|
||||
TCLAP::ValueArg<std::string> lm_dir("", LM_DIR,
|
||||
"the LM model path, which contains compiled models: TLG.fst, config.yaml ", false, "damo/speech_ngram_lm_zh-cn-ai-wesp-fst", "string");
|
||||
TCLAP::ValueArg<std::string> lm_revision(
|
||||
"", "lm-revision", "LM model revision", false, "v1.0.2", "string");
|
||||
TCLAP::ValueArg<std::string> hotword("", HOTWORD,
|
||||
"the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)",
|
||||
false, "/workspace/resources/hotwords.txt", "string");
|
||||
TCLAP::ValueArg<std::int32_t> fst_inc_wts("", FST_INC_WTS,
|
||||
"the fst hotwords incremental bias", false, 20, "int32_t");
|
||||
|
||||
// add file
|
||||
cmd.add(hotword);
|
||||
cmd.add(fst_inc_wts);
|
||||
cmd.add(global_beam);
|
||||
cmd.add(lattice_beam);
|
||||
cmd.add(am_scale);
|
||||
|
||||
cmd.add(certfile);
|
||||
cmd.add(keyfile);
|
||||
|
||||
cmd.add(download_model_dir);
|
||||
cmd.add(offline_model_dir);
|
||||
cmd.add(online_model_dir);
|
||||
cmd.add(offline_model_revision);
|
||||
cmd.add(online_model_revision);
|
||||
cmd.add(quantize);
|
||||
cmd.add(vad_dir);
|
||||
cmd.add(vad_revision);
|
||||
cmd.add(vad_quant);
|
||||
cmd.add(punc_dir);
|
||||
cmd.add(punc_revision);
|
||||
cmd.add(punc_quant);
|
||||
cmd.add(itn_dir);
|
||||
cmd.add(itn_revision);
|
||||
cmd.add(lm_dir);
|
||||
cmd.add(lm_revision);
|
||||
|
||||
cmd.add(listen_ip);
|
||||
cmd.add(port);
|
||||
cmd.add(io_thread_num);
|
||||
cmd.add(decoder_thread_num);
|
||||
cmd.add(model_thread_num);
|
||||
cmd.parse(argc, argv);
|
||||
|
||||
std::map<std::string, std::string> model_path;
|
||||
GetValue(offline_model_dir, OFFLINE_MODEL_DIR, model_path);
|
||||
GetValue(online_model_dir, ONLINE_MODEL_DIR, model_path);
|
||||
GetValue(quantize, QUANTIZE, model_path);
|
||||
GetValue(vad_dir, VAD_DIR, model_path);
|
||||
GetValue(vad_quant, VAD_QUANT, model_path);
|
||||
GetValue(punc_dir, PUNC_DIR, model_path);
|
||||
GetValue(punc_quant, PUNC_QUANT, model_path);
|
||||
GetValue(itn_dir, ITN_DIR, model_path);
|
||||
GetValue(lm_dir, LM_DIR, model_path);
|
||||
GetValue(hotword, HOTWORD, model_path);
|
||||
|
||||
GetValue(offline_model_revision, "offline-model-revision", model_path);
|
||||
GetValue(online_model_revision, "online-model-revision", model_path);
|
||||
GetValue(vad_revision, "vad-revision", model_path);
|
||||
GetValue(punc_revision, "punc-revision", model_path);
|
||||
GetValue(itn_revision, "itn-revision", model_path);
|
||||
GetValue(lm_revision, "lm-revision", model_path);
|
||||
|
||||
global_beam_ = global_beam.getValue();
|
||||
lattice_beam_ = lattice_beam.getValue();
|
||||
am_scale_ = am_scale.getValue();
|
||||
|
||||
// Download model form Modelscope
|
||||
try {
|
||||
std::string s_download_model_dir = download_model_dir.getValue();
|
||||
|
||||
std::string s_vad_path = model_path[VAD_DIR];
|
||||
std::string s_vad_quant = model_path[VAD_QUANT];
|
||||
std::string s_offline_asr_path = model_path[OFFLINE_MODEL_DIR];
|
||||
std::string s_online_asr_path = model_path[ONLINE_MODEL_DIR];
|
||||
std::string s_asr_quant = model_path[QUANTIZE];
|
||||
std::string s_punc_path = model_path[PUNC_DIR];
|
||||
std::string s_punc_quant = model_path[PUNC_QUANT];
|
||||
std::string s_itn_path = model_path[ITN_DIR];
|
||||
std::string s_lm_path = model_path[LM_DIR];
|
||||
|
||||
std::string python_cmd =
|
||||
"python -m funasr.download.runtime_sdk_download_tool --type onnx --quantize True ";
|
||||
|
||||
if (!s_vad_path.empty()) {
|
||||
std::string python_cmd_vad;
|
||||
std::string down_vad_path;
|
||||
std::string down_vad_model;
|
||||
|
||||
if (access(s_vad_path.c_str(), F_OK) == 0) {
|
||||
// local
|
||||
python_cmd_vad = python_cmd + " --model-name " + s_vad_path +
|
||||
" --export-dir ./ " + " --model_revision " +
|
||||
model_path["vad-revision"];
|
||||
down_vad_path = s_vad_path;
|
||||
} else {
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_vad_path
|
||||
<< " from modelscope: ";
|
||||
python_cmd_vad = python_cmd + " --model-name " +
|
||||
s_vad_path +
|
||||
" --export-dir " + s_download_model_dir +
|
||||
" --model_revision " + model_path["vad-revision"];
|
||||
down_vad_path =
|
||||
s_download_model_dir +
|
||||
"/" + s_vad_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_vad.c_str());
|
||||
if (ret != 0) {
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local vad model path, you can ignore the errors.";
|
||||
}
|
||||
down_vad_model = down_vad_path + "/model_quant.onnx";
|
||||
if (s_vad_quant == "false" || s_vad_quant == "False" ||
|
||||
s_vad_quant == "FALSE") {
|
||||
down_vad_model = down_vad_path + "/model.onnx";
|
||||
}
|
||||
|
||||
if (access(down_vad_model.c_str(), F_OK) != 0) {
|
||||
LOG(ERROR) << down_vad_model << " do not exists.";
|
||||
exit(-1);
|
||||
} else {
|
||||
model_path[VAD_DIR] = down_vad_path;
|
||||
LOG(INFO) << "Set " << VAD_DIR << " : " << model_path[VAD_DIR];
|
||||
}
|
||||
}
|
||||
else {
|
||||
LOG(INFO) << "VAD model is not set, use default.";
|
||||
}
|
||||
|
||||
if (!s_offline_asr_path.empty()) {
|
||||
std::string python_cmd_asr;
|
||||
std::string down_asr_path;
|
||||
std::string down_asr_model;
|
||||
|
||||
size_t found = s_offline_asr_path.find("speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404");
|
||||
if (found != std::string::npos) {
|
||||
model_path["offline-model-revision"]="v2.0.5";
|
||||
}
|
||||
|
||||
found = s_offline_asr_path.find("speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404");
|
||||
if (found != std::string::npos) {
|
||||
model_path["offline-model-revision"]="v2.0.5";
|
||||
}
|
||||
|
||||
found = s_offline_asr_path.find("speech_paraformer-large_asr_nat-en-16k-common-vocab10020");
|
||||
if (found != std::string::npos) {
|
||||
model_path["model-revision"]="v2.0.5";
|
||||
s_itn_path="";
|
||||
s_lm_path="";
|
||||
}
|
||||
found = s_offline_asr_path.find(MODEL_SVS);
|
||||
if (found != std::string::npos) {
|
||||
model_path["model-revision"]="v2.0.5";
|
||||
s_lm_path="";
|
||||
model_path[LM_DIR]="";
|
||||
}
|
||||
|
||||
if (access(s_offline_asr_path.c_str(), F_OK) == 0) {
|
||||
// local
|
||||
python_cmd_asr = python_cmd + " --model-name " + s_offline_asr_path +
|
||||
" --export-dir ./ " + " --model_revision " +
|
||||
model_path["offline-model-revision"];
|
||||
down_asr_path = s_offline_asr_path;
|
||||
}
|
||||
else {
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_offline_asr_path
|
||||
<< " from modelscope : ";
|
||||
python_cmd_asr = python_cmd + " --model-name " +
|
||||
s_offline_asr_path +
|
||||
" --export-dir " + s_download_model_dir +
|
||||
" --model_revision " + model_path["offline-model-revision"];
|
||||
down_asr_path
|
||||
= s_download_model_dir + "/" + s_offline_asr_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_asr.c_str());
|
||||
if (ret != 0) {
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local asr model path, you can ignore the errors.";
|
||||
}
|
||||
down_asr_model = down_asr_path + "/model_quant.onnx";
|
||||
if (s_asr_quant == "false" || s_asr_quant == "False" ||
|
||||
s_asr_quant == "FALSE") {
|
||||
down_asr_model = down_asr_path + "/model.onnx";
|
||||
}
|
||||
|
||||
if (access(down_asr_model.c_str(), F_OK) != 0) {
|
||||
LOG(ERROR) << down_asr_model << " do not exists.";
|
||||
exit(-1);
|
||||
} else {
|
||||
model_path[OFFLINE_MODEL_DIR] = down_asr_path;
|
||||
LOG(INFO) << "Set " << OFFLINE_MODEL_DIR << " : " << model_path[OFFLINE_MODEL_DIR];
|
||||
}
|
||||
} else {
|
||||
LOG(INFO) << "ASR Offline model is not set, use default.";
|
||||
}
|
||||
|
||||
if (!s_online_asr_path.empty()) {
|
||||
std::string python_cmd_asr;
|
||||
std::string down_asr_path;
|
||||
std::string down_asr_model;
|
||||
|
||||
if (access(s_online_asr_path.c_str(), F_OK) == 0) {
|
||||
// local
|
||||
python_cmd_asr = python_cmd + " --model-name " + s_online_asr_path +
|
||||
" --export-dir ./ " + " --model_revision " +
|
||||
model_path["online-model-revision"];
|
||||
down_asr_path = s_online_asr_path;
|
||||
} else {
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_online_asr_path
|
||||
<< " from modelscope : ";
|
||||
python_cmd_asr = python_cmd + " --model-name " +
|
||||
s_online_asr_path +
|
||||
" --export-dir " + s_download_model_dir +
|
||||
" --model_revision " + model_path["online-model-revision"];
|
||||
down_asr_path
|
||||
= s_download_model_dir + "/" + s_online_asr_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_asr.c_str());
|
||||
if (ret != 0) {
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local asr model path, you can ignore the errors.";
|
||||
}
|
||||
down_asr_model = down_asr_path + "/model_quant.onnx";
|
||||
if (s_asr_quant == "false" || s_asr_quant == "False" ||
|
||||
s_asr_quant == "FALSE") {
|
||||
down_asr_model = down_asr_path + "/model.onnx";
|
||||
}
|
||||
|
||||
if (access(down_asr_model.c_str(), F_OK) != 0) {
|
||||
LOG(ERROR) << down_asr_model << " do not exists.";
|
||||
exit(-1);
|
||||
} else {
|
||||
model_path[ONLINE_MODEL_DIR] = down_asr_path;
|
||||
LOG(INFO) << "Set " << ONLINE_MODEL_DIR << " : " << model_path[ONLINE_MODEL_DIR];
|
||||
}
|
||||
} else {
|
||||
LOG(INFO) << "ASR online model is not set, use default.";
|
||||
}
|
||||
|
||||
if (!s_lm_path.empty() && s_lm_path != "NONE" && s_lm_path != "none") {
|
||||
std::string python_cmd_lm;
|
||||
std::string down_lm_path;
|
||||
std::string down_lm_model;
|
||||
|
||||
if (access(s_lm_path.c_str(), F_OK) == 0) {
|
||||
// local
|
||||
python_cmd_lm = python_cmd + " --model-name " + s_lm_path +
|
||||
" --export-dir ./ " + " --model_revision " +
|
||||
model_path["lm-revision"] + " --export False ";
|
||||
down_lm_path = s_lm_path;
|
||||
} else {
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_lm_path
|
||||
<< " from modelscope : ";
|
||||
python_cmd_lm = python_cmd + " --model-name " +
|
||||
s_lm_path +
|
||||
" --export-dir " + s_download_model_dir +
|
||||
" --model_revision " + model_path["lm-revision"]
|
||||
+ " --export False ";
|
||||
down_lm_path =
|
||||
s_download_model_dir +
|
||||
"/" + s_lm_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_lm.c_str());
|
||||
if (ret != 0) {
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local lm model path, you can ignore the errors.";
|
||||
}
|
||||
down_lm_model = down_lm_path + "/TLG.fst";
|
||||
|
||||
if (access(down_lm_model.c_str(), F_OK) != 0) {
|
||||
LOG(ERROR) << down_lm_model << " do not exists.";
|
||||
exit(-1);
|
||||
} else {
|
||||
model_path[LM_DIR] = down_lm_path;
|
||||
LOG(INFO) << "Set " << LM_DIR << " : " << model_path[LM_DIR];
|
||||
}
|
||||
} else {
|
||||
LOG(INFO) << "LM model is not set, not executed.";
|
||||
model_path[LM_DIR] = "";
|
||||
}
|
||||
|
||||
if (!s_punc_path.empty()) {
|
||||
std::string python_cmd_punc;
|
||||
std::string down_punc_path;
|
||||
std::string down_punc_model;
|
||||
|
||||
if (access(s_punc_path.c_str(), F_OK) == 0) {
|
||||
// local
|
||||
python_cmd_punc = python_cmd + " --model-name " + s_punc_path +
|
||||
" --export-dir ./ " + " --model_revision " +
|
||||
model_path["punc-revision"];
|
||||
down_punc_path = s_punc_path;
|
||||
} else {
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_punc_path
|
||||
<< " from modelscope : ";
|
||||
python_cmd_punc = python_cmd + " --model-name " +
|
||||
s_punc_path +
|
||||
" --export-dir " + s_download_model_dir +
|
||||
" --model_revision " + model_path["punc-revision"];
|
||||
down_punc_path =
|
||||
s_download_model_dir +
|
||||
"/" + s_punc_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_punc.c_str());
|
||||
if (ret != 0) {
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local punc model path, you can ignore the errors.";
|
||||
}
|
||||
down_punc_model = down_punc_path + "/model_quant.onnx";
|
||||
if (s_punc_quant == "false" || s_punc_quant == "False" ||
|
||||
s_punc_quant == "FALSE") {
|
||||
down_punc_model = down_punc_path + "/model.onnx";
|
||||
}
|
||||
|
||||
if (access(down_punc_model.c_str(), F_OK) != 0) {
|
||||
LOG(ERROR) << down_punc_model << " do not exists.";
|
||||
exit(-1);
|
||||
} else {
|
||||
model_path[PUNC_DIR] = down_punc_path;
|
||||
LOG(INFO) << "Set " << PUNC_DIR << " : " << model_path[PUNC_DIR];
|
||||
}
|
||||
} else {
|
||||
LOG(INFO) << "PUNC model is not set, use default.";
|
||||
}
|
||||
|
||||
if (!s_itn_path.empty()) {
|
||||
std::string python_cmd_itn;
|
||||
std::string down_itn_path;
|
||||
std::string down_itn_model;
|
||||
|
||||
if (access(s_itn_path.c_str(), F_OK) == 0) {
|
||||
// local
|
||||
python_cmd_itn = python_cmd + " --model-name " + s_itn_path +
|
||||
" --export-dir ./ " + " --model_revision " +
|
||||
model_path["itn-revision"] + " --export False ";
|
||||
down_itn_path = s_itn_path;
|
||||
} else {
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_itn_path
|
||||
<< " from modelscope : ";
|
||||
python_cmd_itn = python_cmd + " --model-name " +
|
||||
s_itn_path +
|
||||
" --export-dir " + s_download_model_dir +
|
||||
" --model_revision " + model_path["itn-revision"]
|
||||
+ " --export False ";
|
||||
down_itn_path =
|
||||
s_download_model_dir +
|
||||
"/" + s_itn_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_itn.c_str());
|
||||
if (ret != 0) {
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local itn model path, you can ignore the errors.";
|
||||
}
|
||||
down_itn_model = down_itn_path + "/zh_itn_tagger.fst";
|
||||
|
||||
if (access(down_itn_model.c_str(), F_OK) != 0) {
|
||||
LOG(ERROR) << down_itn_model << " do not exists.";
|
||||
exit(-1);
|
||||
} else {
|
||||
model_path[ITN_DIR] = down_itn_path;
|
||||
LOG(INFO) << "Set " << ITN_DIR << " : " << model_path[ITN_DIR];
|
||||
}
|
||||
} else {
|
||||
LOG(INFO) << "ITN model is not set, use default.";
|
||||
}
|
||||
|
||||
} catch (std::exception const& e) {
|
||||
LOG(ERROR) << "Error: " << e.what();
|
||||
}
|
||||
|
||||
std::string s_listen_ip = listen_ip.getValue();
|
||||
int s_port = port.getValue();
|
||||
int s_io_thread_num = io_thread_num.getValue();
|
||||
int s_decoder_thread_num = decoder_thread_num.getValue();
|
||||
|
||||
int s_model_thread_num = model_thread_num.getValue();
|
||||
|
||||
asio::io_context io_decoder; // context for decoding
|
||||
asio::io_context io_server; // context for server
|
||||
|
||||
std::vector<std::thread> decoder_threads;
|
||||
|
||||
std::string s_certfile = certfile.getValue();
|
||||
std::string s_keyfile = keyfile.getValue();
|
||||
|
||||
// hotword file
|
||||
std::string hotword_path;
|
||||
hotword_path = model_path.at(HOTWORD);
|
||||
fst_inc_wts_ = fst_inc_wts.getValue();
|
||||
LOG(INFO) << "hotword path: " << hotword_path;
|
||||
funasr::ExtractHws(hotword_path, hws_map_);
|
||||
|
||||
bool is_ssl = false;
|
||||
if (!s_certfile.empty() && access(s_certfile.c_str(), F_OK) == 0) {
|
||||
is_ssl = true;
|
||||
}
|
||||
|
||||
auto conn_guard = asio::make_work_guard(
|
||||
io_decoder); // make sure threads can wait in the queue
|
||||
auto server_guard = asio::make_work_guard(
|
||||
io_server); // make sure threads can wait in the queue
|
||||
// create threads pool
|
||||
for (int32_t i = 0; i < s_decoder_thread_num; ++i) {
|
||||
decoder_threads.emplace_back([&io_decoder]() { io_decoder.run(); });
|
||||
}
|
||||
|
||||
server server_; // server for websocket
|
||||
wss_server wss_server_;
|
||||
server* server = nullptr;
|
||||
wss_server* wss_server = nullptr;
|
||||
if (is_ssl) {
|
||||
LOG(INFO)<< "SSL is opened!";
|
||||
wss_server_.init_asio(&io_server); // init asio
|
||||
wss_server_.set_reuse_addr(
|
||||
true); // reuse address as we create multiple threads
|
||||
|
||||
// list on port for accept
|
||||
wss_server_.listen(asio::ip::address::from_string(s_listen_ip), s_port);
|
||||
wss_server = &wss_server_;
|
||||
|
||||
} else {
|
||||
LOG(INFO)<< "SSL is closed!";
|
||||
server_.init_asio(&io_server); // init asio
|
||||
server_.set_reuse_addr(
|
||||
true); // reuse address as we create multiple threads
|
||||
|
||||
// list on port for accept
|
||||
server_.listen(asio::ip::address::from_string(s_listen_ip), s_port);
|
||||
server = &server_;
|
||||
|
||||
}
|
||||
|
||||
WebSocketServer websocket_srv(
|
||||
io_decoder, is_ssl, server, wss_server, s_certfile,
|
||||
s_keyfile); // websocket server for asr engine
|
||||
websocket_srv.initAsr(model_path, s_model_thread_num); // init asr model
|
||||
|
||||
LOG(INFO) << "decoder-thread-num: " << s_decoder_thread_num;
|
||||
LOG(INFO) << "io-thread-num: " << s_io_thread_num;
|
||||
LOG(INFO) << "model-thread-num: " << s_model_thread_num;
|
||||
LOG(INFO) << "asr model init finished. listen on port:" << s_port;
|
||||
|
||||
// Start the ASIO network io_service run loop
|
||||
std::vector<std::thread> ts;
|
||||
// create threads for io network
|
||||
for (size_t i = 0; i < s_io_thread_num; i++) {
|
||||
ts.emplace_back([&io_server]() { io_server.run(); });
|
||||
}
|
||||
// wait for theads
|
||||
for (size_t i = 0; i < s_io_thread_num; i++) {
|
||||
ts[i].join();
|
||||
}
|
||||
|
||||
// wait for theads
|
||||
for (auto& t : decoder_threads) {
|
||||
t.join();
|
||||
}
|
||||
|
||||
} catch (std::exception const& e) {
|
||||
LOG(ERROR) << "Error: " << e.what();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
/* 2022-2023 by zhaomingwork */
|
||||
|
||||
#include "websocket-server.h"
|
||||
#ifdef _WIN32
|
||||
#include "win_func.h"
|
||||
#else
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include <fstream>
|
||||
#include "util.h"
|
||||
|
||||
// hotwords
|
||||
std::unordered_map<std::string, int> hws_map_;
|
||||
int fst_inc_wts_=20;
|
||||
float global_beam_, lattice_beam_, am_scale_;
|
||||
|
||||
using namespace std;
|
||||
void GetValue(TCLAP::ValueArg<std::string>& value_arg, string key,
|
||||
std::map<std::string, std::string>& model_path) {
|
||||
model_path.insert({key, value_arg.getValue()});
|
||||
LOG(INFO) << key << " : " << value_arg.getValue();
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
SetConsoleOutputCP(65001);
|
||||
#endif
|
||||
try {
|
||||
|
||||
google::InitGoogleLogging(argv[0]);
|
||||
FLAGS_logtostderr = true;
|
||||
std::string offline_version = "";
|
||||
#ifdef _WIN32
|
||||
offline_version = "0.1.0";
|
||||
#endif
|
||||
TCLAP::CmdLine cmd("funasr-wss-server", ' ', offline_version);
|
||||
TCLAP::ValueArg<std::string> download_model_dir(
|
||||
"", "download-model-dir",
|
||||
"Download model from Modelscope to download_model_dir",
|
||||
false, "/workspace/models", "string");
|
||||
TCLAP::ValueArg<std::string> model_dir(
|
||||
"", MODEL_DIR,
|
||||
"default: /workspace/models/asr, the asr model path, which contains *.onnx/*.torchscript, config.yaml, am.mvn",
|
||||
false, "/workspace/models/asr", "string");
|
||||
TCLAP::ValueArg<std::string> model_revision(
|
||||
"", "model-revision",
|
||||
"ASR model revision",
|
||||
false, "v2.0.5", "string");
|
||||
TCLAP::ValueArg<std::string> quantize(
|
||||
"", QUANTIZE,
|
||||
"true (Default), load the model of model_quant.onnx in model_dir. If set "
|
||||
"false, load the model of model.onnx in model_dir",
|
||||
false, "true", "string");
|
||||
TCLAP::ValueArg<std::string> bladedisc(
|
||||
"", BLADEDISC,
|
||||
"true (Default), load the model of bladedisc in model_dir.",
|
||||
false, "true", "string");
|
||||
TCLAP::ValueArg<std::string> vad_dir(
|
||||
"", VAD_DIR,
|
||||
"default: /workspace/models/vad, the vad model path, which contains model_quant.onnx, vad.yaml, vad.mvn",
|
||||
false, "/workspace/models/vad", "string");
|
||||
TCLAP::ValueArg<std::string> vad_revision(
|
||||
"", "vad-revision",
|
||||
"VAD model revision",
|
||||
false, "v2.0.6", "string");
|
||||
TCLAP::ValueArg<std::string> vad_quant(
|
||||
"", VAD_QUANT,
|
||||
"true (Default), load the model of model_quant.onnx in vad_dir. If set "
|
||||
"false, load the model of model.onnx in vad_dir",
|
||||
false, "true", "string");
|
||||
TCLAP::ValueArg<std::string> punc_dir(
|
||||
"", PUNC_DIR,
|
||||
"default: /workspace/models/punc, the punc model path, which contains model_quant.onnx, punc.yaml",
|
||||
false, "/workspace/models/punc",
|
||||
"string");
|
||||
TCLAP::ValueArg<std::string> punc_revision(
|
||||
"", "punc-revision",
|
||||
"PUNC model revision",
|
||||
false, "v2.0.5", "string");
|
||||
TCLAP::ValueArg<std::string> punc_quant(
|
||||
"", PUNC_QUANT,
|
||||
"true (Default), load the model of model_quant.onnx in punc_dir. If set "
|
||||
"false, load the model of model.onnx in punc_dir",
|
||||
false, "true", "string");
|
||||
TCLAP::ValueArg<std::string> itn_dir(
|
||||
"", ITN_DIR,
|
||||
"default: thuduj12/fst_itn_zh, the itn model path, which contains "
|
||||
"zh_itn_tagger.fst, zh_itn_verbalizer.fst",
|
||||
false, "thuduj12/fst_itn_zh", "string");
|
||||
TCLAP::ValueArg<std::string> itn_revision(
|
||||
"", "itn-revision", "ITN model revision", false, "v1.0.1", "string");
|
||||
|
||||
TCLAP::ValueArg<std::string> listen_ip("", "listen-ip", "listen ip", false,
|
||||
"0.0.0.0", "string");
|
||||
TCLAP::ValueArg<int> port("", "port", "port", false, 10095, "int");
|
||||
TCLAP::ValueArg<int> io_thread_num("", "io-thread-num", "io thread num",
|
||||
false, 2, "int");
|
||||
TCLAP::ValueArg<int> decoder_thread_num(
|
||||
"", "decoder-thread-num", "decoder thread num", false, 8, "int");
|
||||
TCLAP::ValueArg<int> model_thread_num("", "model-thread-num",
|
||||
"model thread num", false, 1, "int");
|
||||
|
||||
TCLAP::ValueArg<std::string> certfile("", "certfile",
|
||||
"default: ../../../ssl_key/server.crt, path of certficate for WSS connection. if it is empty, it will be in WS mode.",
|
||||
false, "../../../ssl_key/server.crt", "string");
|
||||
TCLAP::ValueArg<std::string> keyfile("", "keyfile",
|
||||
"default: ../../../ssl_key/server.key, path of keyfile for WSS connection",
|
||||
false, "../../../ssl_key/server.key", "string");
|
||||
|
||||
TCLAP::ValueArg<float> global_beam("", GLOB_BEAM, "the decoding beam for beam searching ", false, 3.0, "float");
|
||||
TCLAP::ValueArg<float> lattice_beam("", LAT_BEAM, "the lattice generation beam for beam searching ", false, 3.0, "float");
|
||||
TCLAP::ValueArg<float> am_scale("", AM_SCALE, "the acoustic scale for beam searching ", false, 10.0, "float");
|
||||
|
||||
TCLAP::ValueArg<std::string> lm_dir("", LM_DIR,
|
||||
"the LM model path, which contains compiled models: TLG.fst, config.yaml ", false, "damo/speech_ngram_lm_zh-cn-ai-wesp-fst", "string");
|
||||
TCLAP::ValueArg<std::string> lm_revision(
|
||||
"", "lm-revision", "LM model revision", false, "v1.0.2", "string");
|
||||
TCLAP::ValueArg<std::string> hotword("", HOTWORD,
|
||||
"the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)",
|
||||
false, "/workspace/resources/hotwords.txt", "string");
|
||||
TCLAP::ValueArg<std::int32_t> fst_inc_wts("", FST_INC_WTS,
|
||||
"the fst hotwords incremental bias", false, 20, "int32_t");
|
||||
TCLAP::SwitchArg use_gpu("", INFER_GPU, "Whether to use GPU, default is false", false);
|
||||
TCLAP::ValueArg<std::int32_t> batch_size("", BATCHSIZE, "batch_size for ASR model when using GPU", false, 4, "int32_t");
|
||||
|
||||
// add file
|
||||
cmd.add(hotword);
|
||||
cmd.add(fst_inc_wts);
|
||||
cmd.add(global_beam);
|
||||
cmd.add(lattice_beam);
|
||||
cmd.add(am_scale);
|
||||
|
||||
cmd.add(certfile);
|
||||
cmd.add(keyfile);
|
||||
cmd.add(download_model_dir);
|
||||
cmd.add(model_dir);
|
||||
cmd.add(model_revision);
|
||||
cmd.add(quantize);
|
||||
cmd.add(bladedisc);
|
||||
cmd.add(vad_dir);
|
||||
cmd.add(vad_revision);
|
||||
cmd.add(vad_quant);
|
||||
cmd.add(punc_dir);
|
||||
cmd.add(punc_revision);
|
||||
cmd.add(punc_quant);
|
||||
cmd.add(itn_dir);
|
||||
cmd.add(itn_revision);
|
||||
cmd.add(lm_dir);
|
||||
cmd.add(lm_revision);
|
||||
|
||||
cmd.add(listen_ip);
|
||||
cmd.add(port);
|
||||
cmd.add(io_thread_num);
|
||||
cmd.add(decoder_thread_num);
|
||||
cmd.add(model_thread_num);
|
||||
cmd.add(use_gpu);
|
||||
cmd.add(batch_size);
|
||||
cmd.parse(argc, argv);
|
||||
|
||||
std::map<std::string, std::string> model_path;
|
||||
GetValue(model_dir, MODEL_DIR, model_path);
|
||||
GetValue(quantize, QUANTIZE, model_path);
|
||||
GetValue(bladedisc, BLADEDISC, model_path);
|
||||
GetValue(vad_dir, VAD_DIR, model_path);
|
||||
GetValue(vad_quant, VAD_QUANT, model_path);
|
||||
GetValue(punc_dir, PUNC_DIR, model_path);
|
||||
GetValue(punc_quant, PUNC_QUANT, model_path);
|
||||
GetValue(itn_dir, ITN_DIR, model_path);
|
||||
GetValue(lm_dir, LM_DIR, model_path);
|
||||
GetValue(hotword, HOTWORD, model_path);
|
||||
|
||||
GetValue(model_revision, "model-revision", model_path);
|
||||
GetValue(vad_revision, "vad-revision", model_path);
|
||||
GetValue(punc_revision, "punc-revision", model_path);
|
||||
GetValue(itn_revision, "itn-revision", model_path);
|
||||
GetValue(lm_revision, "lm-revision", model_path);
|
||||
|
||||
global_beam_ = global_beam.getValue();
|
||||
lattice_beam_ = lattice_beam.getValue();
|
||||
am_scale_ = am_scale.getValue();
|
||||
bool use_gpu_ = use_gpu.getValue();
|
||||
int batch_size_ = batch_size.getValue();
|
||||
|
||||
// Download model form Modelscope
|
||||
try{
|
||||
std::string s_download_model_dir = download_model_dir.getValue();
|
||||
|
||||
std::string s_vad_path = model_path[VAD_DIR];
|
||||
std::string s_vad_quant = model_path[VAD_QUANT];
|
||||
std::string s_asr_path = model_path[MODEL_DIR];
|
||||
std::string s_asr_quant = model_path[QUANTIZE];
|
||||
std::string s_punc_path = model_path[PUNC_DIR];
|
||||
std::string s_punc_quant = model_path[PUNC_QUANT];
|
||||
std::string s_itn_path = model_path[ITN_DIR];
|
||||
std::string s_lm_path = model_path[LM_DIR];
|
||||
std::string s_blade = model_path[BLADEDISC];
|
||||
|
||||
std::string python_cmd = "python -m funasr.download.runtime_sdk_download_tool ";
|
||||
|
||||
if(vad_dir.isSet() && !s_vad_path.empty()){
|
||||
std::string python_cmd_vad;
|
||||
std::string down_vad_path;
|
||||
std::string down_vad_model;
|
||||
|
||||
if (access(s_vad_path.c_str(), F_OK) == 0){
|
||||
// local
|
||||
python_cmd_vad = python_cmd + " --type onnx " + " --quantize " + s_vad_quant + " --model-name " + s_vad_path + " --export-dir ./ " + " --model_revision " + model_path["vad-revision"];
|
||||
down_vad_path = s_vad_path;
|
||||
}else{
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_vad_path << " from modelscope: ";
|
||||
python_cmd_vad = python_cmd + " --type onnx " + " --quantize " + s_vad_quant + " --model-name " + s_vad_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["vad-revision"];
|
||||
down_vad_path = s_download_model_dir+"/"+s_vad_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_vad.c_str());
|
||||
if(ret !=0){
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local vad model path, you can ignore the errors.";
|
||||
}
|
||||
down_vad_model = down_vad_path+"/model_quant.onnx";
|
||||
if(s_vad_quant=="false" || s_vad_quant=="False" || s_vad_quant=="FALSE"){
|
||||
down_vad_model = down_vad_path+"/model.onnx";
|
||||
}
|
||||
|
||||
if (access(down_vad_model.c_str(), F_OK) != 0){
|
||||
LOG(ERROR) << down_vad_model << " do not exists.";
|
||||
exit(-1);
|
||||
}else{
|
||||
model_path[VAD_DIR]=down_vad_path;
|
||||
LOG(INFO) << "Set " << VAD_DIR << " : " << model_path[VAD_DIR];
|
||||
}
|
||||
}else{
|
||||
LOG(INFO) << "VAD model is not set, use default.";
|
||||
}
|
||||
|
||||
if(model_dir.isSet() && !s_asr_path.empty()){
|
||||
std::string python_cmd_asr;
|
||||
std::string down_asr_path;
|
||||
std::string down_asr_model;
|
||||
std::string model_type = "onnx";
|
||||
|
||||
// modify model-revision by model name
|
||||
size_t found = s_asr_path.find("speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404");
|
||||
if (found != std::string::npos) {
|
||||
model_path["model-revision"]="v2.0.5";
|
||||
}
|
||||
|
||||
found = s_asr_path.find("speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404");
|
||||
if (found != std::string::npos) {
|
||||
model_path["model-revision"]="v2.0.5";
|
||||
}
|
||||
|
||||
found = s_asr_path.find("speech_paraformer-large_asr_nat-en-16k-common-vocab10020");
|
||||
if (found != std::string::npos) {
|
||||
model_path["model-revision"]="v2.0.5";
|
||||
s_itn_path="";
|
||||
s_lm_path="";
|
||||
}
|
||||
|
||||
found = s_asr_path.find(MODEL_SVS);
|
||||
if (found != std::string::npos) {
|
||||
model_path["model-revision"]="v2.0.5";
|
||||
s_itn_path="";
|
||||
model_path[ITN_DIR]="";
|
||||
s_lm_path="";
|
||||
model_path[LM_DIR]="";
|
||||
s_punc_path="";
|
||||
model_path[PUNC_DIR]="";
|
||||
}
|
||||
|
||||
if (use_gpu_){
|
||||
model_type = "torchscript";
|
||||
if (s_blade=="true" || s_blade=="True" || s_blade=="TRUE"){
|
||||
model_type = "bladedisc";
|
||||
}
|
||||
}
|
||||
|
||||
if (access(s_asr_path.c_str(), F_OK) == 0){
|
||||
// local
|
||||
python_cmd_asr = python_cmd + " --type " + model_type + " --quantize " + s_asr_quant + " --model-name " + s_asr_path + " --export-dir ./ " + " --model_revision " + model_path["model-revision"];
|
||||
down_asr_path = s_asr_path;
|
||||
}else{
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_asr_path << " from modelscope: ";
|
||||
python_cmd_asr = python_cmd + " --type " + model_type + " --quantize " + s_asr_quant + " --model-name " + s_asr_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["model-revision"];
|
||||
down_asr_path = s_download_model_dir+"/"+s_asr_path;
|
||||
}
|
||||
|
||||
down_asr_model = down_asr_path+"/model_quant.onnx";
|
||||
if(s_asr_quant=="false" || s_asr_quant=="False" || s_asr_quant=="FALSE"){
|
||||
down_asr_model = down_asr_path+"/model.onnx";
|
||||
}
|
||||
|
||||
if (use_gpu_){
|
||||
down_asr_model = down_asr_path+"/model.torchscript";
|
||||
if (s_blade=="true" || s_blade=="True" || s_blade=="TRUE"){
|
||||
down_asr_model = down_asr_path+"/model_blade.torchscript";
|
||||
}
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_asr.c_str());
|
||||
if(ret !=0){
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local asr model path, you can ignore the errors.";
|
||||
}
|
||||
|
||||
if (access(down_asr_model.c_str(), F_OK) != 0){
|
||||
LOG(ERROR) << down_asr_model << " do not exists.";
|
||||
exit(-1);
|
||||
}else{
|
||||
model_path[MODEL_DIR]=down_asr_path;
|
||||
LOG(INFO) << "Set " << MODEL_DIR << " : " << model_path[MODEL_DIR];
|
||||
}
|
||||
}else{
|
||||
LOG(INFO) << "ASR model is not set, use default.";
|
||||
}
|
||||
|
||||
if (!s_itn_path.empty()) {
|
||||
std::string python_cmd_itn;
|
||||
std::string down_itn_path;
|
||||
std::string down_itn_model;
|
||||
|
||||
if (access(s_itn_path.c_str(), F_OK) == 0) {
|
||||
// local
|
||||
python_cmd_itn = python_cmd + " --type onnx " + " --model-name " + s_itn_path +
|
||||
" --export-dir ./ " + " --model_revision " +
|
||||
model_path["itn-revision"] + " --export False ";
|
||||
down_itn_path = s_itn_path;
|
||||
} else {
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_itn_path
|
||||
<< " from modelscope : ";
|
||||
python_cmd_itn = python_cmd + " --type onnx " + " --model-name " +
|
||||
s_itn_path +
|
||||
" --export-dir " + s_download_model_dir +
|
||||
" --model_revision " + model_path["itn-revision"]
|
||||
+ " --export False ";
|
||||
down_itn_path =
|
||||
s_download_model_dir +
|
||||
"/" + s_itn_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_itn.c_str());
|
||||
if (ret != 0) {
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local itn model path, you can ignore the errors.";
|
||||
}
|
||||
down_itn_model = down_itn_path + "/zh_itn_tagger.fst";
|
||||
|
||||
if (access(down_itn_model.c_str(), F_OK) != 0) {
|
||||
LOG(ERROR) << down_itn_model << " do not exists.";
|
||||
exit(-1);
|
||||
} else {
|
||||
model_path[ITN_DIR] = down_itn_path;
|
||||
LOG(INFO) << "Set " << ITN_DIR << " : " << model_path[ITN_DIR];
|
||||
}
|
||||
} else {
|
||||
LOG(INFO) << "ITN model is not set, not executed.";
|
||||
}
|
||||
|
||||
if (!s_lm_path.empty() && s_lm_path != "NONE" && s_lm_path != "none") {
|
||||
std::string python_cmd_lm;
|
||||
std::string down_lm_path;
|
||||
std::string down_lm_model;
|
||||
|
||||
if (access(s_lm_path.c_str(), F_OK) == 0) {
|
||||
// local
|
||||
python_cmd_lm = python_cmd + " --type onnx " + " --model-name " + s_lm_path +
|
||||
" --export-dir ./ " + " --model_revision " +
|
||||
model_path["lm-revision"] + " --export False ";
|
||||
down_lm_path = s_lm_path;
|
||||
} else {
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_lm_path
|
||||
<< " from modelscope : ";
|
||||
python_cmd_lm = python_cmd + " --type onnx " + " --model-name " +
|
||||
s_lm_path +
|
||||
" --export-dir " + s_download_model_dir +
|
||||
" --model_revision " + model_path["lm-revision"]
|
||||
+ " --export False ";
|
||||
down_lm_path =
|
||||
s_download_model_dir +
|
||||
"/" + s_lm_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_lm.c_str());
|
||||
if (ret != 0) {
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local lm model path, you can ignore the errors.";
|
||||
}
|
||||
down_lm_model = down_lm_path + "/TLG.fst";
|
||||
|
||||
if (access(down_lm_model.c_str(), F_OK) != 0) {
|
||||
LOG(ERROR) << down_lm_model << " do not exists.";
|
||||
exit(-1);
|
||||
} else {
|
||||
model_path[LM_DIR] = down_lm_path;
|
||||
LOG(INFO) << "Set " << LM_DIR << " : " << model_path[LM_DIR];
|
||||
}
|
||||
} else {
|
||||
LOG(INFO) << "LM model is not set, not executed.";
|
||||
model_path[LM_DIR] = "";
|
||||
}
|
||||
|
||||
if(punc_dir.isSet() && !s_punc_path.empty()){
|
||||
std::string python_cmd_punc;
|
||||
std::string down_punc_path;
|
||||
std::string down_punc_model;
|
||||
|
||||
if (access(s_punc_path.c_str(), F_OK) == 0){
|
||||
// local
|
||||
python_cmd_punc = python_cmd + " --type onnx " + "--quantize " + s_punc_quant + " --model-name " + s_punc_path + " --export-dir ./ " + " --model_revision " + model_path["punc-revision"];
|
||||
down_punc_path = s_punc_path;
|
||||
}else{
|
||||
// modelscope
|
||||
LOG(INFO) << "Download model: " << s_punc_path << " from modelscope: ";
|
||||
python_cmd_punc = python_cmd + " --type onnx " + "--quantize " + s_punc_quant + " --model-name " + s_punc_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["punc-revision"];
|
||||
down_punc_path = s_download_model_dir+"/"+s_punc_path;
|
||||
}
|
||||
|
||||
int ret = system(python_cmd_punc.c_str());
|
||||
if(ret !=0){
|
||||
LOG(INFO) << "Failed to download model from modelscope. If you set local punc model path, you can ignore the errors.";
|
||||
}
|
||||
down_punc_model = down_punc_path+"/model_quant.onnx";
|
||||
if(s_punc_quant=="false" || s_punc_quant=="False" || s_punc_quant=="FALSE"){
|
||||
down_punc_model = down_punc_path+"/model.onnx";
|
||||
}
|
||||
|
||||
if (access(down_punc_model.c_str(), F_OK) != 0){
|
||||
LOG(ERROR) << down_punc_model << " do not exists.";
|
||||
exit(-1);
|
||||
}else{
|
||||
model_path[PUNC_DIR]=down_punc_path;
|
||||
LOG(INFO) << "Set " << PUNC_DIR << " : " << model_path[PUNC_DIR];
|
||||
}
|
||||
}else{
|
||||
LOG(INFO) << "PUNC model is not set, use default.";
|
||||
}
|
||||
|
||||
} catch (std::exception const& e) {
|
||||
LOG(ERROR) << "Error: " << e.what();
|
||||
}
|
||||
|
||||
std::string s_listen_ip = listen_ip.getValue();
|
||||
int s_port = port.getValue();
|
||||
int s_io_thread_num = io_thread_num.getValue();
|
||||
int s_decoder_thread_num = decoder_thread_num.getValue();
|
||||
|
||||
int s_model_thread_num = model_thread_num.getValue();
|
||||
|
||||
asio::io_context io_decoder; // context for decoding
|
||||
asio::io_context io_server; // context for server
|
||||
|
||||
std::vector<std::thread> decoder_threads;
|
||||
|
||||
std::string s_certfile = certfile.getValue();
|
||||
std::string s_keyfile = keyfile.getValue();
|
||||
|
||||
// hotword file
|
||||
std::string hotword_path;
|
||||
hotword_path = model_path.at(HOTWORD);
|
||||
fst_inc_wts_ = fst_inc_wts.getValue();
|
||||
LOG(INFO) << "hotword path: " << hotword_path;
|
||||
funasr::ExtractHws(hotword_path, hws_map_);
|
||||
|
||||
bool is_ssl = false;
|
||||
if (!s_certfile.empty() && access(s_certfile.c_str(), F_OK) == 0) {
|
||||
is_ssl = true;
|
||||
}
|
||||
|
||||
auto conn_guard = asio::make_work_guard(
|
||||
io_decoder); // make sure threads can wait in the queue
|
||||
auto server_guard = asio::make_work_guard(
|
||||
io_server); // make sure threads can wait in the queue
|
||||
// create threads pool
|
||||
for (int32_t i = 0; i < s_decoder_thread_num; ++i) {
|
||||
decoder_threads.emplace_back([&io_decoder]() { io_decoder.run(); });
|
||||
}
|
||||
|
||||
server server_; // server for websocket
|
||||
wss_server wss_server_;
|
||||
server* server = nullptr;
|
||||
wss_server* wss_server = nullptr;
|
||||
if (is_ssl) {
|
||||
LOG(INFO)<< "SSL is opened!";
|
||||
wss_server_.init_asio(&io_server); // init asio
|
||||
wss_server_.set_reuse_addr(
|
||||
true); // reuse address as we create multiple threads
|
||||
|
||||
// list on port for accept
|
||||
wss_server_.listen(asio::ip::address::from_string(s_listen_ip), s_port);
|
||||
wss_server = &wss_server_;
|
||||
} else {
|
||||
LOG(INFO)<< "SSL is closed!";
|
||||
server_.init_asio(&io_server); // init asio
|
||||
server_.set_reuse_addr(
|
||||
true); // reuse address as we create multiple threads
|
||||
|
||||
// list on port for accept
|
||||
server_.listen(asio::ip::address::from_string(s_listen_ip), s_port);
|
||||
server = &server_;
|
||||
}
|
||||
|
||||
|
||||
WebSocketServer websocket_srv(
|
||||
io_decoder, is_ssl, server, wss_server, s_certfile,
|
||||
s_keyfile); // websocket server for asr engine
|
||||
websocket_srv.initAsr(model_path, s_model_thread_num, use_gpu_, batch_size_); // init asr model
|
||||
|
||||
LOG(INFO) << "decoder-thread-num: " << s_decoder_thread_num;
|
||||
LOG(INFO) << "io-thread-num: " << s_io_thread_num;
|
||||
LOG(INFO) << "model-thread-num: " << s_model_thread_num;
|
||||
LOG(INFO) << "asr model init finished. listen on port:" << s_port;
|
||||
|
||||
// Start the ASIO network io_service run loop
|
||||
std::vector<std::thread> ts;
|
||||
// create threads for io network
|
||||
for (size_t i = 0; i < s_io_thread_num; i++) {
|
||||
ts.emplace_back([&io_server]() { io_server.run(); });
|
||||
}
|
||||
// wait for theads
|
||||
for (size_t i = 0; i < s_io_thread_num; i++) {
|
||||
ts[i].join();
|
||||
}
|
||||
|
||||
// wait for theads
|
||||
for (auto& t : decoder_threads) {
|
||||
t.join();
|
||||
}
|
||||
|
||||
} catch (std::exception const& e) {
|
||||
LOG(ERROR) << "Error: " << e.what();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
|
||||
#include "microphone.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "portaudio.h" // NOLINT
|
||||
|
||||
Microphone::Microphone() {
|
||||
PaError err = Pa_Initialize();
|
||||
if (err != paNoError) {
|
||||
LOG(ERROR)<<"portaudio error: " << Pa_GetErrorText(err);
|
||||
exit(-1);
|
||||
}
|
||||
}
|
||||
|
||||
Microphone::~Microphone() {
|
||||
PaError err = Pa_Terminate();
|
||||
if (err != paNoError) {
|
||||
LOG(ERROR)<<"portaudio error: " << Pa_GetErrorText(err);
|
||||
exit(-1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
|
||||
#ifndef WEBSOCKET_MICROPHONE_H_
|
||||
#define WEBSOCKET_MICROPHONE_H_
|
||||
#include <glog/logging.h>
|
||||
|
||||
class Microphone {
|
||||
public:
|
||||
Microphone();
|
||||
~Microphone();
|
||||
};
|
||||
|
||||
#endif // WEBSOCKET_MICROPHONE_H_
|
||||
@@ -0,0 +1,649 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
/* 2022-2023 by zhaomingwork */
|
||||
|
||||
// websocket server for asr engine
|
||||
// take some ideas from https://github.com/k2-fsa/sherpa-onnx
|
||||
// online-websocket-server-impl.cc, thanks. The websocket server has two threads
|
||||
// pools, one for handle network data and one for asr decoder.
|
||||
// now only support offline engine.
|
||||
|
||||
#include "websocket-server-2pass.h"
|
||||
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <iostream>
|
||||
#include <chrono>
|
||||
|
||||
extern std::unordered_map<std::string, int> hws_map_;
|
||||
extern int fst_inc_wts_;
|
||||
extern float global_beam_, lattice_beam_, am_scale_;
|
||||
|
||||
int64_t getCurrentTimeMillis() {
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
|
||||
return millis;
|
||||
}
|
||||
|
||||
context_ptr WebSocketServer::on_tls_init(tls_mode mode,
|
||||
websocketpp::connection_hdl hdl,
|
||||
std::string& s_certfile,
|
||||
std::string& s_keyfile) {
|
||||
namespace asio = websocketpp::lib::asio;
|
||||
|
||||
LOG(INFO) << "on_tls_init called with hdl: " << hdl.lock().get();
|
||||
LOG(INFO) << "using TLS mode: "
|
||||
<< (mode == MOZILLA_MODERN ? "Mozilla Modern"
|
||||
: "Mozilla Intermediate");
|
||||
|
||||
context_ptr ctx = websocketpp::lib::make_shared<asio::ssl::context>(
|
||||
asio::ssl::context::sslv23);
|
||||
|
||||
try {
|
||||
if (mode == MOZILLA_MODERN) {
|
||||
// Modern disables TLSv1
|
||||
ctx->set_options(
|
||||
asio::ssl::context::default_workarounds |
|
||||
asio::ssl::context::no_sslv2 | asio::ssl::context::no_sslv3 |
|
||||
asio::ssl::context::no_tlsv1 | asio::ssl::context::single_dh_use);
|
||||
} else {
|
||||
ctx->set_options(asio::ssl::context::default_workarounds |
|
||||
asio::ssl::context::no_sslv2 |
|
||||
asio::ssl::context::no_sslv3 |
|
||||
asio::ssl::context::single_dh_use);
|
||||
}
|
||||
|
||||
ctx->use_certificate_chain_file(s_certfile);
|
||||
ctx->use_private_key_file(s_keyfile, asio::ssl::context::pem);
|
||||
|
||||
} catch (std::exception& e) {
|
||||
LOG(INFO) << "Exception: " << e.what();
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
nlohmann::json handle_result(FUNASR_RESULT result, websocketpp::connection_hdl& hdl, std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,std::owner_less<websocketpp::connection_hdl>>& data_map) {
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg = nullptr;
|
||||
auto it = data_map.find(hdl);
|
||||
if (it != data_map.end()) {
|
||||
data_msg = it->second;
|
||||
}
|
||||
|
||||
websocketpp::lib::error_code ec;
|
||||
nlohmann::json jsonresult;
|
||||
jsonresult["text"] = "";
|
||||
|
||||
std::string tmp_online_msg = FunASRGetResult(result, 0);
|
||||
if (tmp_online_msg != "") {
|
||||
LOG(INFO) << "wav_name: " << data_msg->msg["wav_name"].get<std::string>() << " | online_res :" << tmp_online_msg;
|
||||
jsonresult["text"] = tmp_online_msg;
|
||||
jsonresult["mode"] = "2pass-online";
|
||||
jsonresult["slice_type"] = 1;
|
||||
jsonresult["index"] = data_msg->index;
|
||||
|
||||
// 如果是第一句话的第一个实时结果或新的句子开始
|
||||
if (!data_msg->is_sentence_started) {
|
||||
data_msg->start_time = FunASRGetTpassStart(result); // 记录句子的开始时间
|
||||
jsonresult["slice_type"] = 0; //0：一段话开始识别; 1：一段话识别中; 2：一段话识别结束
|
||||
data_msg->is_sentence_started = true;
|
||||
}
|
||||
}
|
||||
|
||||
data_msg->end_time = FunASRGetTpassEnd(result); // 记录句子的结束时间
|
||||
jsonresult["timestamp"] = data_msg->timestamp;
|
||||
|
||||
std::string tmp_tpass_msg = FunASRGetTpassResult(result, 0);
|
||||
if (tmp_tpass_msg != "") {
|
||||
LOG(INFO) << "wav_name: " << data_msg->msg["wav_name"].get<std::string>() << " | offline results : " << tmp_tpass_msg;
|
||||
jsonresult["text"] = tmp_tpass_msg;
|
||||
jsonresult["mode"] = "2pass-offline";
|
||||
|
||||
// 句子结束，记录结束时间
|
||||
jsonresult["start_time"] = data_msg->start_time;
|
||||
jsonresult["end_time"] = data_msg->end_time;
|
||||
jsonresult["slice_type"] = 2;
|
||||
jsonresult["index"] = data_msg->index;
|
||||
|
||||
data_msg->index++; //句子序号
|
||||
data_msg->is_sentence_started = false; // 重置句子状态
|
||||
}
|
||||
|
||||
std::string tmp_stamp_msg = FunASRGetStamp(result);
|
||||
if (tmp_stamp_msg != "") {
|
||||
LOG(INFO) << "offline stamps : " << tmp_stamp_msg;
|
||||
jsonresult["timestamp"] = tmp_stamp_msg;
|
||||
}
|
||||
|
||||
std::string tmp_stamp_sents = FunASRGetStampSents(result);
|
||||
if (tmp_stamp_sents != "") {
|
||||
try{
|
||||
nlohmann::json json_stamp = nlohmann::json::parse(tmp_stamp_sents);
|
||||
LOG(INFO) << "offline stamp_sents : " << json_stamp;
|
||||
jsonresult["stamp_sents"] = json_stamp;
|
||||
}catch (std::exception const &e)
|
||||
{
|
||||
LOG(ERROR)<< tmp_stamp_sents << e.what();
|
||||
jsonresult["stamp_sents"] = "";
|
||||
}
|
||||
}
|
||||
|
||||
return jsonresult;
|
||||
}
|
||||
// feed buffer to asr engine for decoder
|
||||
void WebSocketServer::do_decoder(
|
||||
std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,std::owner_less<websocketpp::connection_hdl>>& data_map,
|
||||
std::vector<char>& buffer,
|
||||
websocketpp::connection_hdl& hdl,
|
||||
nlohmann::json& msg,
|
||||
std::vector<std::vector<std::string>>& punc_cache,
|
||||
std::vector<std::vector<float>> &hotwords_embedding,
|
||||
websocketpp::lib::mutex& thread_lock,
|
||||
bool& is_final,
|
||||
std::string wav_name,
|
||||
std::string modetype,
|
||||
bool itn,
|
||||
int audio_fs,
|
||||
std::string wav_format,
|
||||
FUNASR_HANDLE& tpass_online_handle,
|
||||
FUNASR_DEC_HANDLE& decoder_handle,
|
||||
std::string svs_lang,
|
||||
bool sys_itn) {
|
||||
// lock for each connection
|
||||
if(!tpass_online_handle){
|
||||
scoped_lock guard(thread_lock);
|
||||
LOG(INFO) << "tpass_online_handle is free, return";
|
||||
msg["access_num"]=(int)msg["access_num"]-1;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
FUNASR_RESULT Result = nullptr;
|
||||
int asr_mode_ = 2;
|
||||
if (modetype == "offline") {
|
||||
asr_mode_ = 0;
|
||||
} else if (modetype == "online") {
|
||||
asr_mode_ = 1;
|
||||
} else if (modetype == "2pass") {
|
||||
asr_mode_ = 2;
|
||||
}
|
||||
|
||||
while (buffer.size() >= 800 * 2 && !msg["is_eof"]) {
|
||||
std::vector<char> subvector = {buffer.begin(), buffer.begin() + 800 * 2};
|
||||
buffer.erase(buffer.begin(), buffer.begin() + 800 * 2);
|
||||
|
||||
try {
|
||||
if (tpass_online_handle) {
|
||||
Result = FunTpassInferBuffer(tpass_handle, tpass_online_handle,
|
||||
subvector.data(), subvector.size(),
|
||||
punc_cache, false, audio_fs,
|
||||
wav_format, (ASR_TYPE)asr_mode_,
|
||||
hotwords_embedding, itn, decoder_handle,
|
||||
svs_lang, sys_itn);
|
||||
|
||||
} else {
|
||||
scoped_lock guard(thread_lock);
|
||||
msg["access_num"]=(int)msg["access_num"]-1;
|
||||
return;
|
||||
}
|
||||
} catch (std::exception const& e) {
|
||||
scoped_lock guard(thread_lock);
|
||||
LOG(ERROR) << e.what();
|
||||
msg["access_num"]=(int)msg["access_num"]-1;
|
||||
return;
|
||||
}
|
||||
if (Result) {
|
||||
websocketpp::lib::error_code ec;
|
||||
nlohmann::json jsonresult = handle_result(Result, hdl, data_map);
|
||||
jsonresult["wav_name"] = wav_name;
|
||||
jsonresult["is_final"] = false;
|
||||
if (jsonresult["text"] != "") {
|
||||
//LOG(INFO) << "jsonresult: " << jsonresult.dump(4);
|
||||
if (is_ssl) {
|
||||
wss_server_->send(hdl, jsonresult.dump(),
|
||||
websocketpp::frame::opcode::text, ec);
|
||||
} else {
|
||||
server_->send(hdl, jsonresult.dump(),
|
||||
websocketpp::frame::opcode::text, ec);
|
||||
}
|
||||
}
|
||||
FunASRFreeResult(Result);
|
||||
}
|
||||
}
|
||||
if (is_final && !msg["is_eof"]) {
|
||||
try {
|
||||
if (tpass_online_handle) {
|
||||
Result = FunTpassInferBuffer(tpass_handle, tpass_online_handle,
|
||||
buffer.data(), buffer.size(), punc_cache,
|
||||
is_final, audio_fs,
|
||||
wav_format, (ASR_TYPE)asr_mode_,
|
||||
hotwords_embedding, itn, decoder_handle,
|
||||
svs_lang, sys_itn);
|
||||
} else {
|
||||
scoped_lock guard(thread_lock);
|
||||
msg["access_num"]=(int)msg["access_num"]-1;
|
||||
return;
|
||||
}
|
||||
} catch (std::exception const& e) {
|
||||
scoped_lock guard(thread_lock);
|
||||
LOG(ERROR) << e.what();
|
||||
msg["access_num"]=(int)msg["access_num"]-1;
|
||||
return;
|
||||
}
|
||||
if(punc_cache.size()>0){
|
||||
for (auto& vec : punc_cache) {
|
||||
vec.clear();
|
||||
}
|
||||
}
|
||||
if (Result) {
|
||||
websocketpp::lib::error_code ec;
|
||||
nlohmann::json jsonresult = handle_result(Result, hdl, data_map);
|
||||
jsonresult["wav_name"] = wav_name;
|
||||
jsonresult["is_final"] = true;
|
||||
//LOG(INFO) << "jsonresult: " << jsonresult.dump(4);
|
||||
if (is_ssl) {
|
||||
wss_server_->send(hdl, jsonresult.dump(),
|
||||
websocketpp::frame::opcode::text, ec);
|
||||
} else {
|
||||
server_->send(hdl, jsonresult.dump(),
|
||||
websocketpp::frame::opcode::text, ec);
|
||||
}
|
||||
FunASRFreeResult(Result);
|
||||
}else{
|
||||
if(wav_format != "pcm" && wav_format != "PCM"){
|
||||
websocketpp::lib::error_code ec;
|
||||
nlohmann::json jsonresult;
|
||||
jsonresult["text"] = "ERROR. Real-time transcription service ONLY SUPPORT PCM stream.";
|
||||
jsonresult["wav_name"] = wav_name;
|
||||
jsonresult["is_final"] = true;
|
||||
if (is_ssl) {
|
||||
wss_server_->send(hdl, jsonresult.dump(),
|
||||
websocketpp::frame::opcode::text, ec);
|
||||
} else {
|
||||
server_->send(hdl, jsonresult.dump(),
|
||||
websocketpp::frame::opcode::text, ec);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (std::exception const& e) {
|
||||
std::cerr << "Error: " << e.what() << std::endl;
|
||||
}
|
||||
scoped_lock guard(thread_lock);
|
||||
msg["access_num"]=(int)msg["access_num"]-1;
|
||||
|
||||
}
|
||||
|
||||
void WebSocketServer::on_open(websocketpp::connection_hdl hdl) {
|
||||
scoped_lock guard(m_lock); // for threads safty
|
||||
try{
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg =
|
||||
std::make_shared<FUNASR_MESSAGE>(); // put a new data vector for new
|
||||
// connection
|
||||
data_msg->samples = std::make_shared<std::vector<char>>();
|
||||
data_msg->thread_lock = std::make_shared<websocketpp::lib::mutex>();
|
||||
|
||||
data_msg->msg = nlohmann::json::parse("{}");
|
||||
data_msg->msg["wav_format"] = "pcm";
|
||||
data_msg->msg["wav_name"] = "wav-default-id";
|
||||
data_msg->msg["mode"] = "2pass";
|
||||
data_msg->msg["itn"] = true;
|
||||
data_msg->msg["audio_fs"] = 16000; // default is 16k
|
||||
data_msg->msg["access_num"] = 0; // the number of access for this object, when it is 0, we can free it saftly
|
||||
data_msg->msg["is_eof"]=false; // if this connection is closed
|
||||
data_msg->msg["svs_lang"]="auto";
|
||||
data_msg->msg["svs_itn"]=true;
|
||||
FUNASR_DEC_HANDLE decoder_handle =
|
||||
FunASRWfstDecoderInit(tpass_handle, ASR_TWO_PASS, global_beam_, lattice_beam_, am_scale_);
|
||||
data_msg->decoder_handle = decoder_handle;
|
||||
data_msg->punc_cache =
|
||||
std::make_shared<std::vector<std::vector<std::string>>>(2);
|
||||
data_msg->strand_ = std::make_shared<asio::io_context::strand>(io_decoder_);
|
||||
|
||||
data_msg->is_sentence_started = false;
|
||||
|
||||
data_msg->timestamp = getCurrentTimeMillis();
|
||||
|
||||
data_map.emplace(hdl, data_msg);
|
||||
}catch (std::exception const& e) {
|
||||
std::cerr << "Error: " << e.what() << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void remove_hdl(
|
||||
websocketpp::connection_hdl hdl,
|
||||
std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,
|
||||
std::owner_less<websocketpp::connection_hdl>>& data_map) {
|
||||
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg = nullptr;
|
||||
auto it_data = data_map.find(hdl);
|
||||
if (it_data != data_map.end()) {
|
||||
data_msg = it_data->second;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
// scoped_lock guard_decoder(*(data_msg->thread_lock)); //wait for do_decoder
|
||||
// finished and avoid access freed tpass_online_handle
|
||||
unique_lock guard_decoder(*(data_msg->thread_lock));
|
||||
if (data_msg->msg["access_num"]==0 && data_msg->msg["is_eof"]==true) {
|
||||
FunWfstDecoderUnloadHwsRes(data_msg->decoder_handle);
|
||||
FunASRWfstDecoderUninit(data_msg->decoder_handle);
|
||||
data_msg->decoder_handle = nullptr;
|
||||
FunTpassOnlineUninit(data_msg->tpass_online_handle);
|
||||
data_msg->tpass_online_handle = nullptr;
|
||||
data_map.erase(hdl);
|
||||
}
|
||||
|
||||
guard_decoder.unlock();
|
||||
}
|
||||
|
||||
void WebSocketServer::on_close(websocketpp::connection_hdl hdl) {
|
||||
scoped_lock guard(m_lock);
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg = nullptr;
|
||||
auto it_data = data_map.find(hdl);
|
||||
if (it_data != data_map.end()) {
|
||||
data_msg = it_data->second;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
unique_lock guard_decoder(*(data_msg->thread_lock));
|
||||
data_msg->msg["is_eof"]=true;
|
||||
guard_decoder.unlock();
|
||||
}
|
||||
|
||||
// remove closed connection
|
||||
void WebSocketServer::check_and_clean_connection() {
|
||||
while(true){
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5000));
|
||||
std::vector<websocketpp::connection_hdl> to_remove; // remove list
|
||||
auto iter = data_map.begin();
|
||||
while (iter != data_map.end()) { // loop to find closed connection
|
||||
websocketpp::connection_hdl hdl = iter->first;
|
||||
try{
|
||||
if (is_ssl) {
|
||||
wss_server::connection_ptr con = wss_server_->get_con_from_hdl(hdl);
|
||||
if (con->get_state() != 1) { // session::state::open ==1
|
||||
to_remove.push_back(hdl);
|
||||
}
|
||||
} else {
|
||||
server::connection_ptr con = server_->get_con_from_hdl(hdl);
|
||||
if (con->get_state() != 1) { // session::state::open ==1
|
||||
to_remove.push_back(hdl);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (std::exception const &e)
|
||||
{
|
||||
// if connection is close, we set is_eof = true
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg = nullptr;
|
||||
auto it_data = data_map.find(hdl);
|
||||
if (it_data != data_map.end()) {
|
||||
data_msg = it_data->second;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
unique_lock guard_decoder(*(data_msg->thread_lock));
|
||||
data_msg->msg["is_eof"]=true;
|
||||
guard_decoder.unlock();
|
||||
to_remove.push_back(hdl);
|
||||
LOG(INFO)<<"connection is closed.";
|
||||
|
||||
}
|
||||
iter++;
|
||||
}
|
||||
for (auto hdl : to_remove) {
|
||||
{
|
||||
unique_lock lock(m_lock);
|
||||
remove_hdl(hdl, data_map);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
void WebSocketServer::on_message(websocketpp::connection_hdl hdl,
|
||||
message_ptr msg) {
|
||||
unique_lock lock(m_lock);
|
||||
// find the sample data vector according to one connection
|
||||
|
||||
std::shared_ptr<FUNASR_MESSAGE> msg_data = nullptr;
|
||||
|
||||
auto it_data = data_map.find(hdl);
|
||||
if (it_data != data_map.end()) {
|
||||
msg_data = it_data->second;
|
||||
if(msg_data->msg["is_eof"]){
|
||||
lock.unlock();
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
lock.unlock();
|
||||
return;
|
||||
}
|
||||
|
||||
std::shared_ptr<std::vector<char>> sample_data_p = msg_data->samples;
|
||||
std::shared_ptr<std::vector<std::vector<std::string>>> punc_cache_p =
|
||||
msg_data->punc_cache;
|
||||
std::shared_ptr<websocketpp::lib::mutex> thread_lock_p = msg_data->thread_lock;
|
||||
|
||||
lock.unlock();
|
||||
|
||||
if (sample_data_p == nullptr) {
|
||||
LOG(INFO) << "error when fetch sample data vector";
|
||||
return;
|
||||
}
|
||||
|
||||
const std::string& payload = msg->get_payload(); // get msg type
|
||||
unique_lock guard_decoder(*(thread_lock_p)); // mutex for one connection
|
||||
switch (msg->get_opcode()) {
|
||||
case websocketpp::frame::opcode::text: {
|
||||
nlohmann::json jsonresult;
|
||||
try{
|
||||
jsonresult = nlohmann::json::parse(payload);
|
||||
}catch (std::exception const &e)
|
||||
{
|
||||
LOG(ERROR)<<e.what();
|
||||
msg_data->msg["is_eof"]=true;
|
||||
guard_decoder.unlock();
|
||||
return;
|
||||
}
|
||||
|
||||
if (jsonresult.contains("wav_name")) {
|
||||
msg_data->msg["wav_name"] = jsonresult["wav_name"];
|
||||
}
|
||||
if (jsonresult.contains("mode")) {
|
||||
msg_data->msg["mode"] = jsonresult["mode"];
|
||||
}
|
||||
if (jsonresult.contains("wav_format")) {
|
||||
msg_data->msg["wav_format"] = jsonresult["wav_format"];
|
||||
}
|
||||
|
||||
// hotwords: fst/nn
|
||||
if(msg_data->hotwords_embedding == nullptr){
|
||||
std::unordered_map<std::string, int> merged_hws_map;
|
||||
std::string nn_hotwords = "";
|
||||
|
||||
if (jsonresult["hotwords"] != nullptr) {
|
||||
std::string json_string = jsonresult["hotwords"];
|
||||
if (!json_string.empty()){
|
||||
nlohmann::json json_fst_hws;
|
||||
try{
|
||||
json_fst_hws = nlohmann::json::parse(json_string);
|
||||
if(json_fst_hws.type() == nlohmann::json::value_t::object){
|
||||
// fst
|
||||
try{
|
||||
std::unordered_map<std::string, int> client_hws_map = json_fst_hws;
|
||||
merged_hws_map.insert(client_hws_map.begin(), client_hws_map.end());
|
||||
} catch (const std::exception& e) {
|
||||
LOG(INFO) << e.what();
|
||||
}
|
||||
}
|
||||
} catch (std::exception const &e)
|
||||
{
|
||||
LOG(ERROR)<<e.what();
|
||||
// nn
|
||||
std::string client_nn_hws = jsonresult["hotwords"];
|
||||
nn_hotwords += " " + client_nn_hws;
|
||||
// LOG(INFO) << "nn hotwords: " << client_nn_hws;
|
||||
}
|
||||
}
|
||||
}
|
||||
merged_hws_map.insert(hws_map_.begin(), hws_map_.end());
|
||||
|
||||
// fst
|
||||
LOG(INFO) << "hotwords: ";
|
||||
for (const auto& pair : merged_hws_map) {
|
||||
nn_hotwords += " " + pair.first;
|
||||
LOG(INFO) << pair.first << " : " << pair.second;
|
||||
}
|
||||
FunWfstDecoderLoadHwsRes(msg_data->decoder_handle, fst_inc_wts_, merged_hws_map);
|
||||
|
||||
// nn
|
||||
std::vector<std::vector<float>> new_hotwords_embedding = CompileHotwordEmbedding(tpass_handle, nn_hotwords, ASR_TWO_PASS);
|
||||
msg_data->hotwords_embedding =
|
||||
std::make_shared<std::vector<std::vector<float>>>(new_hotwords_embedding);
|
||||
}
|
||||
|
||||
if (jsonresult.contains("audio_fs")) {
|
||||
msg_data->msg["audio_fs"] = jsonresult["audio_fs"];
|
||||
}
|
||||
if (jsonresult.contains("chunk_size")) {
|
||||
if (msg_data->tpass_online_handle == nullptr) {
|
||||
std::vector<int> chunk_size_vec =
|
||||
jsonresult["chunk_size"].get<std::vector<int>>();
|
||||
// check chunk_size_vec
|
||||
if(chunk_size_vec.size() == 3 && chunk_size_vec[1] != 0){
|
||||
FUNASR_HANDLE tpass_online_handle =
|
||||
FunTpassOnlineInit(tpass_handle, chunk_size_vec);
|
||||
msg_data->tpass_online_handle = tpass_online_handle;
|
||||
}else{
|
||||
LOG(ERROR) << "Wrong chunk_size!";
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (jsonresult.contains("itn")) {
|
||||
msg_data->msg["itn"] = jsonresult["itn"];
|
||||
}
|
||||
if (jsonresult.contains("svs_lang")) {
|
||||
msg_data->msg["svs_lang"] = jsonresult["svs_lang"];
|
||||
}
|
||||
if (jsonresult.contains("svs_itn")) {
|
||||
msg_data->msg["svs_itn"] = jsonresult["svs_itn"];
|
||||
}
|
||||
LOG(INFO) << "jsonresult=" << jsonresult
|
||||
<< ", msg_data->msg=" << msg_data->msg;
|
||||
if ((jsonresult["is_speaking"] == false ||
|
||||
jsonresult["is_finished"] == true) &&
|
||||
msg_data->msg["is_eof"] != true &&
|
||||
msg_data->hotwords_embedding != nullptr) {
|
||||
LOG(INFO) << "client done";
|
||||
|
||||
// if it is in final message, post the sample_data to decode
|
||||
try{
|
||||
|
||||
std::vector<std::vector<float>> hotwords_embedding_(*(msg_data->hotwords_embedding));
|
||||
msg_data->strand_->post(
|
||||
std::bind(&WebSocketServer::do_decoder, this,
|
||||
data_map,
|
||||
std::move(*(sample_data_p.get())), std::move(hdl),
|
||||
std::ref(msg_data->msg), std::ref(*(punc_cache_p.get())),
|
||||
std::move(hotwords_embedding_),
|
||||
std::ref(*thread_lock_p), std::move(true),
|
||||
msg_data->msg["wav_name"],
|
||||
msg_data->msg["mode"],
|
||||
msg_data->msg["itn"],
|
||||
msg_data->msg["audio_fs"],
|
||||
msg_data->msg["wav_format"],
|
||||
std::ref(msg_data->tpass_online_handle),
|
||||
std::ref(msg_data->decoder_handle),
|
||||
msg_data->msg["svs_lang"],
|
||||
msg_data->msg["svs_itn"]));
|
||||
msg_data->msg["access_num"]=(int)(msg_data->msg["access_num"])+1;
|
||||
}
|
||||
catch (std::exception const &e)
|
||||
{
|
||||
LOG(ERROR)<<e.what();
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case websocketpp::frame::opcode::binary: {
|
||||
// recived binary data
|
||||
const auto* pcm_data = static_cast<const char*>(payload.data());
|
||||
int32_t num_samples = payload.size();
|
||||
|
||||
if (isonline) {
|
||||
sample_data_p->insert(sample_data_p->end(), pcm_data,
|
||||
pcm_data + num_samples);
|
||||
int setpsize =
|
||||
800 * 2; // TODO, need get from client
|
||||
// if sample_data size > setpsize, we post data to decode
|
||||
if (sample_data_p->size() > setpsize) {
|
||||
int chunksize = floor(sample_data_p->size() / setpsize);
|
||||
// make sure the subvector size is an integer multiple of setpsize
|
||||
std::vector<char> subvector = {
|
||||
sample_data_p->begin(),
|
||||
sample_data_p->begin() + chunksize * setpsize};
|
||||
// keep remain in sample_data
|
||||
sample_data_p->erase(sample_data_p->begin(),
|
||||
sample_data_p->begin() + chunksize * setpsize);
|
||||
|
||||
try{
|
||||
// post to decode
|
||||
if (msg_data->msg["is_eof"] != true && msg_data->hotwords_embedding != nullptr) {
|
||||
std::vector<std::vector<float>> hotwords_embedding_(*(msg_data->hotwords_embedding));
|
||||
msg_data->strand_->post(
|
||||
std::bind(&WebSocketServer::do_decoder, this,
|
||||
data_map,
|
||||
std::move(subvector), std::move(hdl),
|
||||
std::ref(msg_data->msg),
|
||||
std::ref(*(punc_cache_p.get())),
|
||||
std::move(hotwords_embedding_),
|
||||
std::ref(*thread_lock_p), std::move(false),
|
||||
msg_data->msg["wav_name"],
|
||||
msg_data->msg["mode"],
|
||||
msg_data->msg["itn"],
|
||||
msg_data->msg["audio_fs"],
|
||||
msg_data->msg["wav_format"],
|
||||
std::ref(msg_data->tpass_online_handle),
|
||||
std::ref(msg_data->decoder_handle),
|
||||
msg_data->msg["svs_lang"],
|
||||
msg_data->msg["svs_itn"]));
|
||||
msg_data->msg["access_num"]=(int)(msg_data->msg["access_num"])+1;
|
||||
}
|
||||
}
|
||||
catch (std::exception const &e)
|
||||
{
|
||||
LOG(ERROR)<<e.what();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sample_data_p->insert(sample_data_p->end(), pcm_data,
|
||||
pcm_data + num_samples);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
guard_decoder.unlock();
|
||||
}
|
||||
|
||||
// init asr model
|
||||
void WebSocketServer::initAsr(std::map<std::string, std::string>& model_path,
|
||||
int thread_num) {
|
||||
try {
|
||||
tpass_handle = FunTpassInit(model_path, thread_num);
|
||||
if (!tpass_handle) {
|
||||
LOG(ERROR) << "FunTpassInit init failed";
|
||||
exit(-1);
|
||||
}
|
||||
LOG(INFO) << "initAsr run check_and_clean_connection";
|
||||
std::thread clean_thread(&WebSocketServer::check_and_clean_connection,this);
|
||||
clean_thread.detach();
|
||||
LOG(INFO) << "initAsr run check_and_clean_connection finished";
|
||||
|
||||
} catch (const std::exception& e) {
|
||||
LOG(INFO) << e.what();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
/* 2022-2023 by zhaomingwork */
|
||||
|
||||
// websocket server for asr engine
|
||||
// take some ideas from https://github.com/k2-fsa/sherpa-onnx
|
||||
// online-websocket-server-impl.cc, thanks. The websocket server has two threads
|
||||
// pools, one for handle network data and one for asr decoder.
|
||||
// now only support offline engine.
|
||||
|
||||
#ifndef WEBSOCKET_SERVER_H_
|
||||
#define WEBSOCKET_SERVER_H_
|
||||
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#define ASIO_STANDALONE 1 // not boost
|
||||
#include <glog/logging.h>
|
||||
#include "util/text-utils.h"
|
||||
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <websocketpp/common/thread.hpp>
|
||||
#include <websocketpp/config/asio.hpp>
|
||||
#include <websocketpp/server.hpp>
|
||||
|
||||
#include "asio.hpp"
|
||||
#include "com-define.h"
|
||||
#include "funasrruntime.h"
|
||||
#include "nlohmann/json.hpp"
|
||||
#include "tclap/CmdLine.h"
|
||||
typedef websocketpp::server<websocketpp::config::asio> server;
|
||||
typedef websocketpp::server<websocketpp::config::asio_tls> wss_server;
|
||||
typedef server::message_ptr message_ptr;
|
||||
using websocketpp::lib::bind;
|
||||
using websocketpp::lib::placeholders::_1;
|
||||
using websocketpp::lib::placeholders::_2;
|
||||
|
||||
typedef websocketpp::lib::lock_guard<websocketpp::lib::mutex> scoped_lock;
|
||||
typedef websocketpp::lib::unique_lock<websocketpp::lib::mutex> unique_lock;
|
||||
typedef websocketpp::lib::shared_ptr<websocketpp::lib::asio::ssl::context>
|
||||
context_ptr;
|
||||
|
||||
typedef struct {
|
||||
std::string msg;
|
||||
float snippet_time;
|
||||
} FUNASR_RECOG_RESULT;
|
||||
|
||||
typedef struct {
|
||||
nlohmann::json msg;
|
||||
std::shared_ptr<std::vector<char>> samples;
|
||||
std::shared_ptr<std::vector<std::vector<std::string>>> punc_cache;
|
||||
std::shared_ptr<std::vector<std::vector<float>>> hotwords_embedding=nullptr;
|
||||
std::shared_ptr<websocketpp::lib::mutex> thread_lock; // lock for each connection
|
||||
FUNASR_HANDLE tpass_online_handle=nullptr;
|
||||
std::string online_res = "";
|
||||
std::string tpass_res = "";
|
||||
std::shared_ptr<asio::io_context::strand> strand_; // for data execute in order
|
||||
FUNASR_DEC_HANDLE decoder_handle=nullptr;
|
||||
|
||||
bool is_sentence_started = false;
|
||||
int64_t start_time = 0;
|
||||
int64_t end_time = 0;
|
||||
int64_t index = 0;
|
||||
int64_t timestamp = 0;
|
||||
} FUNASR_MESSAGE;
|
||||
|
||||
// See https://wiki.mozilla.org/Security/Server_Side_TLS for more details about
|
||||
// the TLS modes. The code below demonstrates how to implement both the modern
|
||||
enum tls_mode { MOZILLA_INTERMEDIATE = 1, MOZILLA_MODERN = 2 };
|
||||
class WebSocketServer {
|
||||
public:
|
||||
WebSocketServer(asio::io_context& io_decoder, bool is_ssl, server* server,
|
||||
wss_server* wss_server, std::string& s_certfile,
|
||||
std::string& s_keyfile)
|
||||
: io_decoder_(io_decoder),
|
||||
is_ssl(is_ssl),
|
||||
server_(server),
|
||||
wss_server_(wss_server) {
|
||||
if (is_ssl) {
|
||||
std::cout << "certfile path is " << s_certfile << std::endl;
|
||||
wss_server->set_tls_init_handler(
|
||||
bind<context_ptr>(&WebSocketServer::on_tls_init, this,
|
||||
MOZILLA_INTERMEDIATE, ::_1, s_certfile, s_keyfile));
|
||||
wss_server_->set_message_handler(
|
||||
[this](websocketpp::connection_hdl hdl, message_ptr msg) {
|
||||
on_message(hdl, msg);
|
||||
});
|
||||
// set open handle
|
||||
wss_server_->set_open_handler(
|
||||
[this](websocketpp::connection_hdl hdl) { on_open(hdl); });
|
||||
// set close handle
|
||||
wss_server_->set_close_handler(
|
||||
[this](websocketpp::connection_hdl hdl) { on_close(hdl); });
|
||||
// begin accept
|
||||
wss_server_->start_accept();
|
||||
// not print log
|
||||
wss_server_->clear_access_channels(websocketpp::log::alevel::all);
|
||||
|
||||
} else {
|
||||
// set message handle
|
||||
server_->set_message_handler(
|
||||
[this](websocketpp::connection_hdl hdl, message_ptr msg) {
|
||||
on_message(hdl, msg);
|
||||
});
|
||||
// set open handle
|
||||
server_->set_open_handler(
|
||||
[this](websocketpp::connection_hdl hdl) { on_open(hdl); });
|
||||
// set close handle
|
||||
server_->set_close_handler(
|
||||
[this](websocketpp::connection_hdl hdl) { on_close(hdl); });
|
||||
// begin accept
|
||||
server_->start_accept();
|
||||
// not print log
|
||||
server_->clear_access_channels(websocketpp::log::alevel::all);
|
||||
}
|
||||
}
|
||||
void do_decoder(std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,std::owner_less<websocketpp::connection_hdl>>& data_map,
|
||||
std::vector<char>& buffer,
|
||||
websocketpp::connection_hdl& hdl,
|
||||
nlohmann::json& msg,
|
||||
std::vector<std::vector<std::string>>& punc_cache,
|
||||
std::vector<std::vector<float>> &hotwords_embedding,
|
||||
websocketpp::lib::mutex& thread_lock, bool& is_final,
|
||||
std::string wav_name,
|
||||
std::string modetype,
|
||||
bool itn,
|
||||
int audio_fs,
|
||||
std::string wav_format,
|
||||
FUNASR_HANDLE& tpass_online_handle,
|
||||
FUNASR_DEC_HANDLE& decoder_handle,
|
||||
std::string svs_lang,
|
||||
bool sys_itn);
|
||||
|
||||
void initAsr(std::map<std::string, std::string>& model_path, int thread_num);
|
||||
void on_message(websocketpp::connection_hdl hdl, message_ptr msg);
|
||||
void on_open(websocketpp::connection_hdl hdl);
|
||||
void on_close(websocketpp::connection_hdl hdl);
|
||||
context_ptr on_tls_init(tls_mode mode, websocketpp::connection_hdl hdl,
|
||||
std::string& s_certfile, std::string& s_keyfile);
|
||||
|
||||
private:
|
||||
void check_and_clean_connection();
|
||||
asio::io_context& io_decoder_; // threads for asr decoder
|
||||
// std::ofstream fout;
|
||||
// FUNASR_HANDLE asr_handle; // asr engine handle
|
||||
FUNASR_HANDLE tpass_handle=nullptr;
|
||||
bool isonline = true; // online or offline engine, now only support offline
|
||||
bool is_ssl = true;
|
||||
server* server_; // websocket server
|
||||
wss_server* wss_server_; // websocket server
|
||||
|
||||
// use map to keep the received samples data from one connection in offline
|
||||
// engine. if for online engline, a data struct is needed(TODO)
|
||||
|
||||
std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,
|
||||
std::owner_less<websocketpp::connection_hdl>>
|
||||
data_map;
|
||||
websocketpp::lib::mutex m_lock; // mutex for sample_map
|
||||
};
|
||||
|
||||
#endif // WEBSOCKET_SERVER_H_
|
||||
@@ -0,0 +1,433 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
/* 2022-2023 by zhaomingwork */
|
||||
|
||||
// websocket server for asr engine
|
||||
// take some ideas from https://github.com/k2-fsa/sherpa-onnx
|
||||
// online-websocket-server-impl.cc, thanks. The websocket server has two threads
|
||||
// pools, one for handle network data and one for asr decoder.
|
||||
// now only support offline engine.
|
||||
|
||||
#include "websocket-server.h"
|
||||
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
extern std::unordered_map<std::string, int> hws_map_;
|
||||
extern int fst_inc_wts_;
|
||||
extern float global_beam_, lattice_beam_, am_scale_;
|
||||
|
||||
context_ptr WebSocketServer::on_tls_init(tls_mode mode,
|
||||
websocketpp::connection_hdl hdl,
|
||||
std::string& s_certfile,
|
||||
std::string& s_keyfile) {
|
||||
namespace asio = websocketpp::lib::asio;
|
||||
|
||||
LOG(INFO) << "on_tls_init called with hdl: " << hdl.lock().get();
|
||||
LOG(INFO) << "using TLS mode: "
|
||||
<< (mode == MOZILLA_MODERN ? "Mozilla Modern"
|
||||
: "Mozilla Intermediate");
|
||||
|
||||
context_ptr ctx = websocketpp::lib::make_shared<asio::ssl::context>(
|
||||
asio::ssl::context::sslv23);
|
||||
|
||||
try {
|
||||
if (mode == MOZILLA_MODERN) {
|
||||
// Modern disables TLSv1
|
||||
ctx->set_options(
|
||||
asio::ssl::context::default_workarounds |
|
||||
asio::ssl::context::no_sslv2 | asio::ssl::context::no_sslv3 |
|
||||
asio::ssl::context::no_tlsv1 | asio::ssl::context::single_dh_use);
|
||||
} else {
|
||||
ctx->set_options(asio::ssl::context::default_workarounds |
|
||||
asio::ssl::context::no_sslv2 |
|
||||
asio::ssl::context::no_sslv3 |
|
||||
asio::ssl::context::single_dh_use);
|
||||
}
|
||||
|
||||
ctx->use_certificate_chain_file(s_certfile);
|
||||
ctx->use_private_key_file(s_keyfile, asio::ssl::context::pem);
|
||||
|
||||
} catch (std::exception& e) {
|
||||
LOG(INFO) << "Exception: " << e.what();
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
// feed buffer to asr engine for decoder
|
||||
void WebSocketServer::do_decoder(const std::vector<char>& buffer,
|
||||
websocketpp::connection_hdl& hdl,
|
||||
nlohmann::json& msg,
|
||||
websocketpp::lib::mutex& thread_lock,
|
||||
std::vector<std::vector<float>> &hotwords_embedding,
|
||||
std::string wav_name,
|
||||
bool itn,
|
||||
int audio_fs,
|
||||
std::string wav_format,
|
||||
FUNASR_DEC_HANDLE& decoder_handle,
|
||||
std::string svs_lang,
|
||||
bool sys_itn) {
|
||||
try {
|
||||
int num_samples = buffer.size(); // the size of the buf
|
||||
|
||||
if (!buffer.empty() && hotwords_embedding.size() > 0) {
|
||||
std::string asr_result="";
|
||||
std::string stamp_res="";
|
||||
std::string stamp_sents="";
|
||||
try{
|
||||
FUNASR_RESULT Result = FunOfflineInferBuffer(
|
||||
asr_handle, buffer.data(), buffer.size(), RASR_NONE, nullptr,
|
||||
hotwords_embedding, audio_fs, wav_format, itn, decoder_handle,
|
||||
svs_lang, sys_itn);
|
||||
if (Result != nullptr){
|
||||
asr_result = FunASRGetResult(Result, 0); // get decode result
|
||||
stamp_res = FunASRGetStamp(Result);
|
||||
stamp_sents = FunASRGetStampSents(Result);
|
||||
FunASRFreeResult(Result);
|
||||
} else{
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(20));
|
||||
LOG(ERROR) << "FUNASR_RESULT is nullptr.";
|
||||
}
|
||||
}catch (std::exception const& e) {
|
||||
LOG(ERROR) << e.what();
|
||||
}
|
||||
|
||||
websocketpp::lib::error_code ec;
|
||||
nlohmann::json jsonresult; // result json
|
||||
jsonresult["text"] = asr_result; // put result in 'text'
|
||||
jsonresult["mode"] = "offline";
|
||||
jsonresult["is_final"] = false;
|
||||
if(stamp_res != ""){
|
||||
jsonresult["timestamp"] = stamp_res;
|
||||
}
|
||||
if(stamp_sents != ""){
|
||||
try{
|
||||
nlohmann::json json_stamp = nlohmann::json::parse(stamp_sents);
|
||||
jsonresult["stamp_sents"] = json_stamp;
|
||||
}catch (std::exception const &e)
|
||||
{
|
||||
LOG(ERROR)<<e.what();
|
||||
jsonresult["stamp_sents"] = "";
|
||||
}
|
||||
}
|
||||
jsonresult["wav_name"] = wav_name;
|
||||
|
||||
// send the json to client
|
||||
if (is_ssl) {
|
||||
wss_server_->send(hdl, jsonresult.dump(),
|
||||
websocketpp::frame::opcode::text, ec);
|
||||
} else {
|
||||
server_->send(hdl, jsonresult.dump(), websocketpp::frame::opcode::text,
|
||||
ec);
|
||||
}
|
||||
|
||||
LOG(INFO) << "buffer.size=" << buffer.size() << ",result json=" << jsonresult.dump();
|
||||
}else{
|
||||
LOG(INFO) << "Sent empty msg";
|
||||
websocketpp::lib::error_code ec;
|
||||
nlohmann::json jsonresult; // result json
|
||||
jsonresult["text"] = ""; // put result in 'text'
|
||||
jsonresult["mode"] = "offline";
|
||||
jsonresult["is_final"] = false;
|
||||
jsonresult["wav_name"] = wav_name;
|
||||
|
||||
// send the json to client
|
||||
if (is_ssl) {
|
||||
wss_server_->send(hdl, jsonresult.dump(),
|
||||
websocketpp::frame::opcode::text, ec);
|
||||
} else {
|
||||
server_->send(hdl, jsonresult.dump(), websocketpp::frame::opcode::text,
|
||||
ec);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (std::exception const& e) {
|
||||
std::cerr << "Error: " << e.what() << std::endl;
|
||||
}
|
||||
scoped_lock guard(thread_lock);
|
||||
msg["access_num"]=(int)msg["access_num"]-1;
|
||||
}
|
||||
|
||||
void WebSocketServer::on_open(websocketpp::connection_hdl hdl) {
|
||||
scoped_lock guard(m_lock); // for threads safty
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg =
|
||||
std::make_shared<FUNASR_MESSAGE>(); // put a new data vector for new
|
||||
// connection
|
||||
data_msg->samples = std::make_shared<std::vector<char>>();
|
||||
data_msg->thread_lock = std::make_shared<websocketpp::lib::mutex>();
|
||||
data_msg->msg = nlohmann::json::parse("{}");
|
||||
data_msg->msg["wav_format"] = "pcm";
|
||||
data_msg->msg["wav_name"] = "wav-default-id";
|
||||
data_msg->msg["itn"] = true;
|
||||
data_msg->msg["audio_fs"] = 16000; // default is 16k
|
||||
data_msg->msg["access_num"] = 0; // the number of access for this object, when it is 0, we can free it saftly
|
||||
data_msg->msg["is_eof"]=false;
|
||||
data_msg->msg["svs_lang"]="auto";
|
||||
data_msg->msg["svs_itn"]=true;
|
||||
FUNASR_DEC_HANDLE decoder_handle =
|
||||
FunASRWfstDecoderInit(asr_handle, ASR_OFFLINE, global_beam_, lattice_beam_, am_scale_);
|
||||
data_msg->decoder_handle = decoder_handle;
|
||||
data_map.emplace(hdl, data_msg);
|
||||
LOG(INFO) << "on_open, active connections: " << data_map.size();
|
||||
}
|
||||
|
||||
void WebSocketServer::on_close(websocketpp::connection_hdl hdl) {
|
||||
scoped_lock guard(m_lock);
|
||||
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg = nullptr;
|
||||
auto it_data = data_map.find(hdl);
|
||||
if (it_data != data_map.end()) {
|
||||
data_msg = it_data->second;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
unique_lock guard_decoder(*(data_msg->thread_lock));
|
||||
data_msg->msg["is_eof"]=true;
|
||||
guard_decoder.unlock();
|
||||
|
||||
LOG(INFO) << "on_close, active connections: " << data_map.size();
|
||||
}
|
||||
|
||||
void remove_hdl(
|
||||
websocketpp::connection_hdl hdl,
|
||||
std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,
|
||||
std::owner_less<websocketpp::connection_hdl>>& data_map) {
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg = nullptr;
|
||||
auto it_data = data_map.find(hdl);
|
||||
if (it_data != data_map.end()) {
|
||||
data_msg = it_data->second;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
unique_lock guard_decoder(*(data_msg->thread_lock));
|
||||
if (data_msg->msg["access_num"]==0 && data_msg->msg["is_eof"]==true) {
|
||||
FunWfstDecoderUnloadHwsRes(data_msg->decoder_handle);
|
||||
FunASRWfstDecoderUninit(data_msg->decoder_handle);
|
||||
data_msg->decoder_handle = nullptr;
|
||||
data_map.erase(hdl);
|
||||
LOG(INFO) << "remove one connection";
|
||||
}
|
||||
guard_decoder.unlock();
|
||||
}
|
||||
|
||||
void WebSocketServer::check_and_clean_connection() {
|
||||
while(true){
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5000));
|
||||
std::vector<websocketpp::connection_hdl> to_remove; // remove list
|
||||
auto iter = data_map.begin();
|
||||
while (iter != data_map.end()) { // loop to find closed connection
|
||||
websocketpp::connection_hdl hdl = iter->first;
|
||||
try{
|
||||
if (is_ssl) {
|
||||
wss_server::connection_ptr con = wss_server_->get_con_from_hdl(hdl);
|
||||
if (con->get_state() != 1) { // session::state::open ==1
|
||||
to_remove.push_back(hdl);
|
||||
}
|
||||
} else {
|
||||
server::connection_ptr con = server_->get_con_from_hdl(hdl);
|
||||
if (con->get_state() != 1) { // session::state::open ==1
|
||||
to_remove.push_back(hdl);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (std::exception const &e)
|
||||
{
|
||||
// if connection is close, we set is_eof = true
|
||||
std::shared_ptr<FUNASR_MESSAGE> data_msg = nullptr;
|
||||
auto it_data = data_map.find(hdl);
|
||||
if (it_data != data_map.end()) {
|
||||
data_msg = it_data->second;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
unique_lock guard_decoder(*(data_msg->thread_lock));
|
||||
data_msg->msg["is_eof"]=true;
|
||||
guard_decoder.unlock();
|
||||
to_remove.push_back(hdl);
|
||||
LOG(INFO)<<"connection is closed.";
|
||||
|
||||
}
|
||||
iter++;
|
||||
}
|
||||
for (auto hdl : to_remove) {
|
||||
{
|
||||
unique_lock lock(m_lock);
|
||||
remove_hdl(hdl, data_map);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void WebSocketServer::on_message(websocketpp::connection_hdl hdl,
|
||||
message_ptr msg) {
|
||||
unique_lock lock(m_lock);
|
||||
// find the sample data vector according to one connection
|
||||
|
||||
std::shared_ptr<FUNASR_MESSAGE> msg_data = nullptr;
|
||||
|
||||
auto it_data = data_map.find(hdl);
|
||||
if (it_data != data_map.end()) {
|
||||
msg_data = it_data->second;
|
||||
if(msg_data->msg["is_eof"]){
|
||||
lock.unlock();
|
||||
return;
|
||||
}
|
||||
} else{
|
||||
lock.unlock();
|
||||
return;
|
||||
}
|
||||
|
||||
std::shared_ptr<std::vector<char>> sample_data_p = msg_data->samples;
|
||||
std::shared_ptr<websocketpp::lib::mutex> thread_lock_p = msg_data->thread_lock;
|
||||
|
||||
lock.unlock();
|
||||
if (sample_data_p == nullptr) {
|
||||
LOG(INFO) << "error when fetch sample data vector";
|
||||
return;
|
||||
}
|
||||
|
||||
const std::string& payload = msg->get_payload(); // get msg type
|
||||
unique_lock guard_decoder(*(thread_lock_p)); // mutex for one connection
|
||||
switch (msg->get_opcode()) {
|
||||
case websocketpp::frame::opcode::text: {
|
||||
nlohmann::json jsonresult;
|
||||
try{
|
||||
jsonresult = nlohmann::json::parse(payload);
|
||||
}catch (std::exception const &e)
|
||||
{
|
||||
LOG(ERROR)<<e.what();
|
||||
msg_data->msg["is_eof"]=true;
|
||||
guard_decoder.unlock();
|
||||
return;
|
||||
}
|
||||
|
||||
if (jsonresult["wav_name"] != nullptr) {
|
||||
msg_data->msg["wav_name"] = jsonresult["wav_name"];
|
||||
}
|
||||
if (jsonresult["wav_format"] != nullptr) {
|
||||
msg_data->msg["wav_format"] = jsonresult["wav_format"];
|
||||
}
|
||||
|
||||
// hotwords: fst/nn
|
||||
if(msg_data->hotwords_embedding == nullptr){
|
||||
std::unordered_map<std::string, int> merged_hws_map;
|
||||
std::string nn_hotwords = "";
|
||||
|
||||
if (jsonresult["hotwords"] != nullptr) {
|
||||
std::string json_string = jsonresult["hotwords"];
|
||||
if (!json_string.empty()){
|
||||
nlohmann::json json_fst_hws;
|
||||
try{
|
||||
json_fst_hws = nlohmann::json::parse(json_string);
|
||||
if(json_fst_hws.type() == nlohmann::json::value_t::object){
|
||||
// fst
|
||||
try{
|
||||
std::unordered_map<std::string, int> client_hws_map = json_fst_hws;
|
||||
merged_hws_map.insert(client_hws_map.begin(), client_hws_map.end());
|
||||
} catch (const std::exception& e) {
|
||||
LOG(INFO) << e.what();
|
||||
}
|
||||
}
|
||||
} catch (std::exception const &e)
|
||||
{
|
||||
LOG(ERROR)<<e.what();
|
||||
// nn
|
||||
std::string client_nn_hws = jsonresult["hotwords"];
|
||||
nn_hotwords += " " + client_nn_hws;
|
||||
// LOG(INFO) << "nn hotwords: " << client_nn_hws;
|
||||
}
|
||||
}
|
||||
}
|
||||
merged_hws_map.insert(hws_map_.begin(), hws_map_.end());
|
||||
|
||||
// fst
|
||||
LOG(INFO) << "hotwords: ";
|
||||
for (const auto& pair : merged_hws_map) {
|
||||
nn_hotwords += " " + pair.first;
|
||||
LOG(INFO) << pair.first << " : " << pair.second;
|
||||
}
|
||||
FunWfstDecoderLoadHwsRes(msg_data->decoder_handle, fst_inc_wts_, merged_hws_map);
|
||||
|
||||
// nn
|
||||
std::vector<std::vector<float>> new_hotwords_embedding= CompileHotwordEmbedding(asr_handle, nn_hotwords);
|
||||
msg_data->hotwords_embedding =
|
||||
std::make_shared<std::vector<std::vector<float>>>(new_hotwords_embedding);
|
||||
}
|
||||
if (jsonresult.contains("audio_fs")) {
|
||||
msg_data->msg["audio_fs"] = jsonresult["audio_fs"];
|
||||
}
|
||||
if (jsonresult.contains("itn")) {
|
||||
msg_data->msg["itn"] = jsonresult["itn"];
|
||||
}
|
||||
if (jsonresult.contains("svs_lang")) {
|
||||
msg_data->msg["svs_lang"] = jsonresult["svs_lang"];
|
||||
}
|
||||
if (jsonresult.contains("svs_itn")) {
|
||||
msg_data->msg["svs_itn"] = jsonresult["svs_itn"];
|
||||
}
|
||||
if ((jsonresult["is_speaking"] == false ||
|
||||
jsonresult["is_finished"] == true) &&
|
||||
msg_data->msg["is_eof"] != true &&
|
||||
msg_data->hotwords_embedding != nullptr) {
|
||||
LOG(INFO) << "client done";
|
||||
// for offline, send all receive data to decoder engine
|
||||
std::vector<std::vector<float>> hotwords_embedding_(*(msg_data->hotwords_embedding));
|
||||
asio::post(io_decoder_,
|
||||
std::bind(&WebSocketServer::do_decoder, this,
|
||||
std::move(*(sample_data_p.get())),
|
||||
std::move(hdl),
|
||||
std::ref(msg_data->msg),
|
||||
std::ref(*thread_lock_p),
|
||||
std::move(hotwords_embedding_),
|
||||
msg_data->msg["wav_name"],
|
||||
msg_data->msg["itn"],
|
||||
msg_data->msg["audio_fs"],
|
||||
msg_data->msg["wav_format"],
|
||||
std::ref(msg_data->decoder_handle),
|
||||
msg_data->msg["svs_lang"],
|
||||
msg_data->msg["svs_itn"]));
|
||||
msg_data->msg["access_num"]=(int)(msg_data->msg["access_num"])+1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case websocketpp::frame::opcode::binary: {
|
||||
// recived binary data
|
||||
const auto* pcm_data = static_cast<const char*>(payload.data());
|
||||
int32_t num_samples = payload.size();
|
||||
|
||||
if (isonline) {
|
||||
// TODO
|
||||
} else {
|
||||
// for offline, we add receive data to end of the sample data vector
|
||||
sample_data_p->insert(sample_data_p->end(), pcm_data,
|
||||
pcm_data + num_samples);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
guard_decoder.unlock();
|
||||
}
|
||||
|
||||
// init asr model
|
||||
void WebSocketServer::initAsr(std::map<std::string, std::string>& model_path,
|
||||
int thread_num, bool use_gpu, int batch_size) {
|
||||
try {
|
||||
// init model with api
|
||||
|
||||
asr_handle = FunOfflineInit(model_path, thread_num, use_gpu, batch_size);
|
||||
LOG(INFO) << "model successfully inited";
|
||||
|
||||
LOG(INFO) << "initAsr run check_and_clean_connection";
|
||||
std::thread clean_thread(&WebSocketServer::check_and_clean_connection,this);
|
||||
clean_thread.detach();
|
||||
LOG(INFO) << "initAsr run check_and_clean_connection finished";
|
||||
|
||||
} catch (const std::exception& e) {
|
||||
LOG(INFO) << e.what();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
/* 2022-2023 by zhaomingwork */
|
||||
|
||||
// websocket server for asr engine
|
||||
// take some ideas from https://github.com/k2-fsa/sherpa-onnx
|
||||
// online-websocket-server-impl.cc, thanks. The websocket server has two threads
|
||||
// pools, one for handle network data and one for asr decoder.
|
||||
// now only support offline engine.
|
||||
|
||||
#ifndef WEBSOCKET_SERVER_H_
|
||||
#define WEBSOCKET_SERVER_H_
|
||||
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <unordered_map>
|
||||
#define ASIO_STANDALONE 1 // not boost
|
||||
#include <glog/logging.h>
|
||||
#include "util/text-utils.h"
|
||||
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <websocketpp/common/thread.hpp>
|
||||
#include <websocketpp/config/asio.hpp>
|
||||
#include <websocketpp/server.hpp>
|
||||
|
||||
#include "asio.hpp"
|
||||
#include "com-define.h"
|
||||
#include "funasrruntime.h"
|
||||
#include "nlohmann/json.hpp"
|
||||
#include "tclap/CmdLine.h"
|
||||
typedef websocketpp::server<websocketpp::config::asio> server;
|
||||
typedef websocketpp::server<websocketpp::config::asio_tls> wss_server;
|
||||
typedef server::message_ptr message_ptr;
|
||||
using websocketpp::lib::bind;
|
||||
using websocketpp::lib::placeholders::_1;
|
||||
using websocketpp::lib::placeholders::_2;
|
||||
|
||||
typedef websocketpp::lib::lock_guard<websocketpp::lib::mutex> scoped_lock;
|
||||
typedef websocketpp::lib::unique_lock<websocketpp::lib::mutex> unique_lock;
|
||||
typedef websocketpp::lib::shared_ptr<websocketpp::lib::asio::ssl::context>
|
||||
context_ptr;
|
||||
|
||||
typedef struct {
|
||||
std::string msg="";
|
||||
std::string stamp="";
|
||||
std::string stamp_sents;
|
||||
std::string tpass_msg="";
|
||||
float snippet_time=0;
|
||||
} FUNASR_RECOG_RESULT;
|
||||
|
||||
typedef struct {
|
||||
nlohmann::json msg;
|
||||
std::shared_ptr<std::vector<char>> samples;
|
||||
std::shared_ptr<std::vector<std::vector<float>>> hotwords_embedding=nullptr;
|
||||
std::shared_ptr<websocketpp::lib::mutex> thread_lock; // lock for each connection
|
||||
FUNASR_DEC_HANDLE decoder_handle=nullptr;
|
||||
} FUNASR_MESSAGE;
|
||||
|
||||
// See https://wiki.mozilla.org/Security/Server_Side_TLS for more details about
|
||||
// the TLS modes. The code below demonstrates how to implement both the modern
|
||||
enum tls_mode { MOZILLA_INTERMEDIATE = 1, MOZILLA_MODERN = 2 };
|
||||
class WebSocketServer {
|
||||
public:
|
||||
WebSocketServer(asio::io_context& io_decoder, bool is_ssl, server* server,
|
||||
wss_server* wss_server, std::string& s_certfile,
|
||||
std::string& s_keyfile)
|
||||
: io_decoder_(io_decoder),
|
||||
is_ssl(is_ssl),
|
||||
server_(server),
|
||||
wss_server_(wss_server){
|
||||
if (is_ssl) {
|
||||
std::cout << "certfile path is " << s_certfile << std::endl;
|
||||
wss_server->set_tls_init_handler(
|
||||
bind<context_ptr>(&WebSocketServer::on_tls_init, this,
|
||||
MOZILLA_INTERMEDIATE, ::_1, s_certfile, s_keyfile));
|
||||
wss_server_->set_message_handler(
|
||||
[this](websocketpp::connection_hdl hdl, message_ptr msg) {
|
||||
on_message(hdl, msg);
|
||||
});
|
||||
// set open handle
|
||||
wss_server_->set_open_handler(
|
||||
[this](websocketpp::connection_hdl hdl) { on_open(hdl); });
|
||||
// set close handle
|
||||
wss_server_->set_close_handler(
|
||||
[this](websocketpp::connection_hdl hdl) { on_close(hdl); });
|
||||
// begin accept
|
||||
wss_server_->start_accept();
|
||||
// not print log
|
||||
wss_server_->clear_access_channels(websocketpp::log::alevel::all);
|
||||
|
||||
} else {
|
||||
// set message handle
|
||||
server_->set_message_handler(
|
||||
[this](websocketpp::connection_hdl hdl, message_ptr msg) {
|
||||
on_message(hdl, msg);
|
||||
});
|
||||
// set open handle
|
||||
server_->set_open_handler(
|
||||
[this](websocketpp::connection_hdl hdl) { on_open(hdl); });
|
||||
// set close handle
|
||||
server_->set_close_handler(
|
||||
[this](websocketpp::connection_hdl hdl) { on_close(hdl); });
|
||||
// begin accept
|
||||
server_->start_accept();
|
||||
// not print log
|
||||
server_->clear_access_channels(websocketpp::log::alevel::all);
|
||||
}
|
||||
}
|
||||
void do_decoder(const std::vector<char>& buffer,
|
||||
websocketpp::connection_hdl& hdl,
|
||||
nlohmann::json& msg,
|
||||
websocketpp::lib::mutex& thread_lock,
|
||||
std::vector<std::vector<float>> &hotwords_embedding,
|
||||
std::string wav_name,
|
||||
bool itn,
|
||||
int audio_fs,
|
||||
std::string wav_format,
|
||||
FUNASR_DEC_HANDLE& decoder_handle,
|
||||
std::string svs_lang,
|
||||
bool sys_itn);
|
||||
|
||||
void initAsr(std::map<std::string, std::string>& model_path, int thread_num, bool use_gpu=false, int batch_size=1);
|
||||
void on_message(websocketpp::connection_hdl hdl, message_ptr msg);
|
||||
void on_open(websocketpp::connection_hdl hdl);
|
||||
void on_close(websocketpp::connection_hdl hdl);
|
||||
context_ptr on_tls_init(tls_mode mode, websocketpp::connection_hdl hdl,
|
||||
std::string& s_certfile, std::string& s_keyfile);
|
||||
|
||||
private:
|
||||
void check_and_clean_connection();
|
||||
asio::io_context& io_decoder_; // threads for asr decoder
|
||||
// std::ofstream fout;
|
||||
FUNASR_HANDLE asr_handle; // asr engine handle
|
||||
bool isonline = false; // online or offline engine, now only support offline
|
||||
bool is_ssl = true;
|
||||
server* server_; // websocket server
|
||||
wss_server* wss_server_; // websocket server
|
||||
|
||||
// use map to keep the received samples data from one connection in offline
|
||||
// engine. if for online engline, a data struct is needed(TODO)
|
||||
|
||||
std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,
|
||||
std::owner_less<websocketpp::connection_hdl>>
|
||||
data_map;
|
||||
websocketpp::lib::mutex m_lock; // mutex for sample_map
|
||||
};
|
||||
|
||||
// std::unordered_map<std::string, int>& hws_map, int fst_inc_wts, std::string& nn_hotwords
|
||||
#endif // WEBSOCKET_SERVER_H_
|
||||
@@ -0,0 +1,2 @@
|
||||
阿里巴巴 20
|
||||
通义实验室 30
|
||||
@@ -0,0 +1,64 @@
|
||||
# Advanced Development Guide (File transcription service) ([click](../docs/SDK_advanced_guide_offline.md))
|
||||
# Real-time Speech Transcription Service Development Guide ([click](../docs/SDK_advanced_guide_online.md))
|
||||
|
||||
|
||||
# If you want to compile the file yourself, you can follow the steps below.
|
||||
## Building for Linux/Unix
|
||||
### Download onnxruntime
|
||||
```shell
|
||||
wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/onnxruntime-linux-x64-1.14.0.tgz
|
||||
tar -zxvf onnxruntime-linux-x64-1.14.0.tgz
|
||||
```
|
||||
|
||||
### Download ffmpeg
|
||||
```shell
|
||||
wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/ffmpeg-master-latest-linux64-gpl-shared.tar.xz
|
||||
tar -xvf ffmpeg-master-latest-linux64-gpl-shared.tar.xz
|
||||
```
|
||||
|
||||
### Install deps
|
||||
```shell
|
||||
# openblas
|
||||
sudo apt-get install libopenblas-dev #ubuntu
|
||||
# sudo yum -y install openblas-devel #centos
|
||||
|
||||
# openssl
|
||||
apt-get install libssl-dev #ubuntu
|
||||
# yum install openssl-devel #centos
|
||||
```
|
||||
|
||||
### Build runtime
|
||||
```shell
|
||||
git clone https://github.com/modelscope/FunASR.git && cd FunASR/runtime/websocket
|
||||
mkdir build && cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release .. -DONNXRUNTIME_DIR=/path/to/onnxruntime-linux-x64-1.14.0 -DFFMPEG_DIR=/path/to/ffmpeg-master-latest-linux64-gpl-shared
|
||||
make -j 4
|
||||
```
|
||||
|
||||
|
||||
## Building for Windows
|
||||
### Download onnxruntime
|
||||
https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/onnxruntime-win-x64-1.16.1.zip
|
||||
|
||||
Download to d:\onnxruntime-win-x64-1.16.1
|
||||
|
||||
### Download ffmpeg
|
||||
https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/ffmpeg-master-latest-win64-gpl-shared.zip
|
||||
|
||||
Download to d:\ffmpeg-master-latest-win64-gpl-shared
|
||||
|
||||
### Download openssl
|
||||
https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/openssl-1.1.1w.zip
|
||||
|
||||
Download to d:\openssl-1.1.1w
|
||||
|
||||
### Build runtime
|
||||
```
|
||||
git clone https://github.com/modelscope/FunASR.git
|
||||
cd FunASR/runtime/websocket
|
||||
mkdir build
|
||||
cd build
|
||||
cmake ../ -D OPENSSL_ROOT_DIR=d:/openssl-1.1.1w -D FFMPEG_DIR=d:/ffmpeg-master-latest-win64-gpl-shared -D ONNXRUNTIME_DIR=d:/onnxruntime-win-x64-1.16.1
|
||||
```
|
||||
Open FunASR/runtime/websocket/build/FunASRWebscoket.sln in Visual Studio and complete the compilation.
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# FunASR离线文件转写服务开发指南([点击此处](../docs/SDK_advanced_guide_offline_zh.md))
|
||||
|
||||
# FunASR实时语音听写服务开发指南([点击此处](../docs/SDK_advanced_guide_online_zh.md))
|
||||
|
||||
# 如果您想自己编译文件，可以参考下述步骤
|
||||
## Linux/Unix 平台编译
|
||||
### 下载 onnxruntime
|
||||
```shell
|
||||
wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/onnxruntime-linux-x64-1.14.0.tgz
|
||||
tar -zxvf onnxruntime-linux-x64-1.14.0.tgz
|
||||
```
|
||||
|
||||
### 下载 ffmpeg
|
||||
```shell
|
||||
wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/ffmpeg-master-latest-linux64-gpl-shared.tar.xz
|
||||
tar -xvf ffmpeg-master-latest-linux64-gpl-shared.tar.xz
|
||||
```
|
||||
|
||||
### 安装依赖
|
||||
```shell
|
||||
# openblas
|
||||
sudo apt-get install libopenblas-dev #ubuntu
|
||||
# sudo yum -y install openblas-devel #centos
|
||||
|
||||
# openssl
|
||||
apt-get install libssl-dev #ubuntu
|
||||
# yum install openssl-devel #centos
|
||||
```
|
||||
|
||||
### 编译 runtime
|
||||
|
||||
```shell
|
||||
git clone https://github.com/modelscope/FunASR.git && cd FunASR/runtime/websocket
|
||||
mkdir build && cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release .. -DONNXRUNTIME_DIR=/path/to/onnxruntime-linux-x64-1.14.0 -DFFMPEG_DIR=/path/to/ffmpeg-master-latest-linux64-gpl-shared
|
||||
make -j 4
|
||||
```
|
||||
|
||||
|
||||
## Windows 平台编译
|
||||
### 下载 onnxruntime
|
||||
https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/onnxruntime-win-x64-1.16.1.zip
|
||||
|
||||
下载并解压到 d:/onnxruntime-win-x64-1.16.1
|
||||
|
||||
### 下载 ffmpeg
|
||||
https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/ffmpeg-master-latest-win64-gpl-shared.zip
|
||||
|
||||
下载并解压到 d:/ffmpeg-master-latest-win64-gpl-shared
|
||||
|
||||
### 编译 openssl
|
||||
https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/dep_libs/openssl-1.1.1w.zip
|
||||
|
||||
下载解压到 d:/openssl-1.1.1w
|
||||
|
||||
### 编译 runtime
|
||||
```
|
||||
git clone https://github.com/modelscope/FunASR.git
|
||||
cd FunASR/runtime/websocket
|
||||
mkdir build
|
||||
cd build
|
||||
cmake ../ -D OPENSSL_ROOT_DIR=d:/openssl-1.1.1w -D FFMPEG_DIR=d:/ffmpeg-master-latest-win64-gpl-shared -D ONNXRUNTIME_DIR=d:/onnxruntime-win-x64-1.16.1
|
||||
```
|
||||
Visual Studio 打开 FunASR/runtime/websocket/build/FunASRWebscoket.sln 完成编译；
|
||||
编译后的可执行文件位于：FunASR/runtime/websocket/build/bin/Debug;
|
||||
从 onnxruntime-win-x64-1.16.1/lib, ffmpeg-master-latest-win64-gpl-shared/bin, openssl-1.1.1w/bin copy相关的DLL库至: FunASR/runtime/websocket/build/bin/Debug
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
#### Download onnxruntime
|
||||
```shell
|
||||
bash third_party/download_onnxruntime.sh
|
||||
```
|
||||
|
||||
#### Download ffmpeg
|
||||
```shell
|
||||
bash third_party/download_ffmpeg.sh
|
||||
```
|
||||
|
||||
#### Install openblas and openssl
|
||||
```shell
|
||||
sudo apt-get install libopenblas-dev libssl-dev #ubuntu
|
||||
# sudo yum -y install openblas-devel openssl-devel #centos
|
||||
```
|
||||
Reference in New Issue
Block a user