chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+193
@@ -0,0 +1,193 @@
|
||||
import os
|
||||
import numpy as np
|
||||
import sys
|
||||
|
||||
|
||||
def compute_wer(ref_file, hyp_file, cer_detail_file):
|
||||
rst = {
|
||||
"Wrd": 0,
|
||||
"Corr": 0,
|
||||
"Ins": 0,
|
||||
"Del": 0,
|
||||
"Sub": 0,
|
||||
"Snt": 0,
|
||||
"Err": 0.0,
|
||||
"S.Err": 0.0,
|
||||
"wrong_words": 0,
|
||||
"wrong_sentences": 0,
|
||||
}
|
||||
|
||||
hyp_dict = {}
|
||||
ref_dict = {}
|
||||
with open(hyp_file, "r") as hyp_reader:
|
||||
for line in hyp_reader:
|
||||
key = line.strip().split()[0]
|
||||
value = line.strip().split()[1:]
|
||||
hyp_dict[key] = value
|
||||
with open(ref_file, "r") as ref_reader:
|
||||
for line in ref_reader:
|
||||
key = line.strip().split()[0]
|
||||
value = line.strip().split()[1:]
|
||||
ref_dict[key] = value
|
||||
|
||||
cer_detail_writer = open(cer_detail_file, "w")
|
||||
for hyp_key in hyp_dict:
|
||||
if hyp_key in ref_dict:
|
||||
out_item = compute_wer_by_line(hyp_dict[hyp_key], ref_dict[hyp_key])
|
||||
rst["Wrd"] += out_item["nwords"]
|
||||
rst["Corr"] += out_item["cor"]
|
||||
rst["wrong_words"] += out_item["wrong"]
|
||||
rst["Ins"] += out_item["ins"]
|
||||
rst["Del"] += out_item["del"]
|
||||
rst["Sub"] += out_item["sub"]
|
||||
rst["Snt"] += 1
|
||||
if out_item["wrong"] > 0:
|
||||
rst["wrong_sentences"] += 1
|
||||
cer_detail_writer.write(hyp_key + print_cer_detail(out_item) + "\n")
|
||||
cer_detail_writer.write("ref:" + "\t" + "".join(ref_dict[hyp_key]) + "\n")
|
||||
cer_detail_writer.write("hyp:" + "\t" + "".join(hyp_dict[hyp_key]) + "\n")
|
||||
|
||||
if rst["Wrd"] > 0:
|
||||
rst["Err"] = round(rst["wrong_words"] * 100 / rst["Wrd"], 2)
|
||||
if rst["Snt"] > 0:
|
||||
rst["S.Err"] = round(rst["wrong_sentences"] * 100 / rst["Snt"], 2)
|
||||
|
||||
cer_detail_writer.write("\n")
|
||||
cer_detail_writer.write(
|
||||
"%WER "
|
||||
+ str(rst["Err"])
|
||||
+ " [ "
|
||||
+ str(rst["wrong_words"])
|
||||
+ " / "
|
||||
+ str(rst["Wrd"])
|
||||
+ ", "
|
||||
+ str(rst["Ins"])
|
||||
+ " ins, "
|
||||
+ str(rst["Del"])
|
||||
+ " del, "
|
||||
+ str(rst["Sub"])
|
||||
+ " sub ]"
|
||||
+ "\n"
|
||||
)
|
||||
cer_detail_writer.write(
|
||||
"%SER "
|
||||
+ str(rst["S.Err"])
|
||||
+ " [ "
|
||||
+ str(rst["wrong_sentences"])
|
||||
+ " / "
|
||||
+ str(rst["Snt"])
|
||||
+ " ]"
|
||||
+ "\n"
|
||||
)
|
||||
cer_detail_writer.write(
|
||||
"Scored "
|
||||
+ str(len(hyp_dict))
|
||||
+ " sentences, "
|
||||
+ str(len(hyp_dict) - rst["Snt"])
|
||||
+ " not present in hyp."
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def compute_wer_by_line(hyp, ref):
|
||||
hyp = list(map(lambda x: x.lower(), hyp))
|
||||
ref = list(map(lambda x: x.lower(), ref))
|
||||
|
||||
len_hyp = len(hyp)
|
||||
len_ref = len(ref)
|
||||
|
||||
cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)
|
||||
|
||||
ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)
|
||||
|
||||
for i in range(len_hyp + 1):
|
||||
cost_matrix[i][0] = i
|
||||
for j in range(len_ref + 1):
|
||||
cost_matrix[0][j] = j
|
||||
|
||||
for i in range(1, len_hyp + 1):
|
||||
for j in range(1, len_ref + 1):
|
||||
if hyp[i - 1] == ref[j - 1]:
|
||||
cost_matrix[i][j] = cost_matrix[i - 1][j - 1]
|
||||
else:
|
||||
substitution = cost_matrix[i - 1][j - 1] + 1
|
||||
insertion = cost_matrix[i - 1][j] + 1
|
||||
deletion = cost_matrix[i][j - 1] + 1
|
||||
|
||||
compare_val = [substitution, insertion, deletion]
|
||||
|
||||
min_val = min(compare_val)
|
||||
operation_idx = compare_val.index(min_val) + 1
|
||||
cost_matrix[i][j] = min_val
|
||||
ops_matrix[i][j] = operation_idx
|
||||
|
||||
match_idx = []
|
||||
i = len_hyp
|
||||
j = len_ref
|
||||
rst = {"nwords": len_ref, "cor": 0, "wrong": 0, "ins": 0, "del": 0, "sub": 0}
|
||||
while i >= 0 or j >= 0:
|
||||
i_idx = max(0, i)
|
||||
j_idx = max(0, j)
|
||||
|
||||
if ops_matrix[i_idx][j_idx] == 0: # correct
|
||||
if i - 1 >= 0 and j - 1 >= 0:
|
||||
match_idx.append((j - 1, i - 1))
|
||||
rst["cor"] += 1
|
||||
|
||||
i -= 1
|
||||
j -= 1
|
||||
|
||||
elif ops_matrix[i_idx][j_idx] == 2: # insert
|
||||
i -= 1
|
||||
rst["ins"] += 1
|
||||
|
||||
elif ops_matrix[i_idx][j_idx] == 3: # delete
|
||||
j -= 1
|
||||
rst["del"] += 1
|
||||
|
||||
elif ops_matrix[i_idx][j_idx] == 1: # substitute
|
||||
i -= 1
|
||||
j -= 1
|
||||
rst["sub"] += 1
|
||||
|
||||
if i < 0 and j >= 0:
|
||||
rst["del"] += 1
|
||||
elif j < 0 and i >= 0:
|
||||
rst["ins"] += 1
|
||||
|
||||
match_idx.reverse()
|
||||
wrong_cnt = cost_matrix[len_hyp][len_ref]
|
||||
rst["wrong"] = wrong_cnt
|
||||
|
||||
return rst
|
||||
|
||||
|
||||
def print_cer_detail(rst):
|
||||
return (
|
||||
"("
|
||||
+ "nwords="
|
||||
+ str(rst["nwords"])
|
||||
+ ",cor="
|
||||
+ str(rst["cor"])
|
||||
+ ",ins="
|
||||
+ str(rst["ins"])
|
||||
+ ",del="
|
||||
+ str(rst["del"])
|
||||
+ ",sub="
|
||||
+ str(rst["sub"])
|
||||
+ ") corr:"
|
||||
+ "{:.2%}".format(rst["cor"] / rst["nwords"])
|
||||
+ ",cer:"
|
||||
+ "{:.2%}".format(rst["wrong"] / rst["nwords"])
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 4:
|
||||
print("usage : python compute-wer.py test.ref test.hyp test.wer")
|
||||
sys.exit(0)
|
||||
|
||||
ref_file = sys.argv[1]
|
||||
hyp_file = sys.argv[2]
|
||||
cer_detail_file = sys.argv[3]
|
||||
compute_wer(ref_file, hyp_file, cer_detail_file)
|
||||
@@ -0,0 +1,193 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import kaldi_native_fbank as knf
|
||||
|
||||
root_dir = Path(__file__).resolve().parent
|
||||
|
||||
logger_initialized = {}
|
||||
|
||||
|
||||
class WavFrontend:
|
||||
"""Conventional frontend structure for ASR."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cmvn_file: str = None,
|
||||
fs: int = 16000,
|
||||
window: str = "hamming",
|
||||
n_mels: int = 80,
|
||||
frame_length: int = 25,
|
||||
frame_shift: int = 10,
|
||||
lfr_m: int = 1,
|
||||
lfr_n: int = 1,
|
||||
dither: float = 1.0,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
|
||||
opts = knf.FbankOptions()
|
||||
opts.frame_opts.samp_freq = fs
|
||||
opts.frame_opts.dither = dither
|
||||
opts.frame_opts.window_type = window
|
||||
opts.frame_opts.frame_shift_ms = float(frame_shift)
|
||||
opts.frame_opts.frame_length_ms = float(frame_length)
|
||||
opts.mel_opts.num_bins = n_mels
|
||||
opts.energy_floor = 0
|
||||
opts.frame_opts.snip_edges = True
|
||||
opts.mel_opts.debug_mel = False
|
||||
self.opts = opts
|
||||
|
||||
self.lfr_m = lfr_m
|
||||
self.lfr_n = lfr_n
|
||||
self.cmvn_file = cmvn_file
|
||||
|
||||
if self.cmvn_file:
|
||||
self.cmvn = self.load_cmvn()
|
||||
self.fbank_fn = None
|
||||
self.fbank_beg_idx = 0
|
||||
self.reset_status()
|
||||
|
||||
def fbank(self, waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
||||
waveform = waveform * (1 << 15)
|
||||
self.fbank_fn = knf.OnlineFbank(self.opts)
|
||||
self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
|
||||
frames = self.fbank_fn.num_frames_ready
|
||||
mat = np.empty([frames, self.opts.mel_opts.num_bins])
|
||||
for i in range(frames):
|
||||
mat[i, :] = self.fbank_fn.get_frame(i)
|
||||
feat = mat.astype(np.float32)
|
||||
feat_len = np.array(mat.shape[0]).astype(np.int32)
|
||||
return feat, feat_len
|
||||
|
||||
def fbank_online(self, waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
||||
waveform = waveform * (1 << 15)
|
||||
# self.fbank_fn = knf.OnlineFbank(self.opts)
|
||||
self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
|
||||
frames = self.fbank_fn.num_frames_ready
|
||||
mat = np.empty([frames, self.opts.mel_opts.num_bins])
|
||||
for i in range(self.fbank_beg_idx, frames):
|
||||
mat[i, :] = self.fbank_fn.get_frame(i)
|
||||
# self.fbank_beg_idx += (frames-self.fbank_beg_idx)
|
||||
feat = mat.astype(np.float32)
|
||||
feat_len = np.array(mat.shape[0]).astype(np.int32)
|
||||
return feat, feat_len
|
||||
|
||||
def reset_status(self):
|
||||
self.fbank_fn = knf.OnlineFbank(self.opts)
|
||||
self.fbank_beg_idx = 0
|
||||
|
||||
def lfr_cmvn(self, feat: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
||||
if self.lfr_m != 1 or self.lfr_n != 1:
|
||||
feat = self.apply_lfr(feat, self.lfr_m, self.lfr_n)
|
||||
|
||||
if self.cmvn_file:
|
||||
feat = self.apply_cmvn(feat)
|
||||
|
||||
feat_len = np.array(feat.shape[0]).astype(np.int32)
|
||||
return feat, feat_len
|
||||
|
||||
@staticmethod
|
||||
def apply_lfr(inputs: np.ndarray, lfr_m: int, lfr_n: int) -> np.ndarray:
|
||||
LFR_inputs = []
|
||||
|
||||
T = inputs.shape[0]
|
||||
T_lfr = int(np.ceil(T / lfr_n))
|
||||
left_padding = np.tile(inputs[0], ((lfr_m - 1) // 2, 1))
|
||||
inputs = np.vstack((left_padding, inputs))
|
||||
T = T + (lfr_m - 1) // 2
|
||||
for i in range(T_lfr):
|
||||
if lfr_m <= T - i * lfr_n:
|
||||
LFR_inputs.append((inputs[i * lfr_n : i * lfr_n + lfr_m]).reshape(1, -1))
|
||||
else:
|
||||
# process last LFR frame
|
||||
num_padding = lfr_m - (T - i * lfr_n)
|
||||
frame = inputs[i * lfr_n :].reshape(-1)
|
||||
for _ in range(num_padding):
|
||||
frame = np.hstack((frame, inputs[-1]))
|
||||
|
||||
LFR_inputs.append(frame)
|
||||
LFR_outputs = np.vstack(LFR_inputs).astype(np.float32)
|
||||
return LFR_outputs
|
||||
|
||||
def apply_cmvn(self, inputs: np.ndarray) -> np.ndarray:
|
||||
"""
|
||||
Apply CMVN with mvn data
|
||||
"""
|
||||
frame, dim = inputs.shape
|
||||
means = np.tile(self.cmvn[0:1, :dim], (frame, 1))
|
||||
vars = np.tile(self.cmvn[1:2, :dim], (frame, 1))
|
||||
inputs = (inputs + means) * vars
|
||||
return inputs
|
||||
|
||||
def load_cmvn(
|
||||
self,
|
||||
) -> np.ndarray:
|
||||
with open(self.cmvn_file, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
means_list = []
|
||||
vars_list = []
|
||||
for i in range(len(lines)):
|
||||
line_item = lines[i].split()
|
||||
if line_item[0] == "<AddShift>":
|
||||
line_item = lines[i + 1].split()
|
||||
if line_item[0] == "<LearnRateCoef>":
|
||||
add_shift_line = line_item[3 : (len(line_item) - 1)]
|
||||
means_list = list(add_shift_line)
|
||||
continue
|
||||
elif line_item[0] == "<Rescale>":
|
||||
line_item = lines[i + 1].split()
|
||||
if line_item[0] == "<LearnRateCoef>":
|
||||
rescale_line = line_item[3 : (len(line_item) - 1)]
|
||||
vars_list = list(rescale_line)
|
||||
continue
|
||||
|
||||
means = np.array(means_list).astype(np.float64)
|
||||
vars = np.array(vars_list).astype(np.float64)
|
||||
cmvn = np.array([means, vars])
|
||||
return cmvn
|
||||
|
||||
|
||||
def load_bytes(input):
|
||||
middle_data = np.frombuffer(input, dtype=np.int16)
|
||||
middle_data = np.asarray(middle_data)
|
||||
if middle_data.dtype.kind not in "iu":
|
||||
raise TypeError("'middle_data' must be an array of integers")
|
||||
dtype = np.dtype("float32")
|
||||
if dtype.kind != "f":
|
||||
raise TypeError("'dtype' must be a floating point type")
|
||||
|
||||
i = np.iinfo(middle_data.dtype)
|
||||
abs_max = 2 ** (i.bits - 1)
|
||||
offset = i.min + abs_max
|
||||
array = np.frombuffer((middle_data.astype(dtype) - offset) / abs_max, dtype=np.float32)
|
||||
return array
|
||||
|
||||
|
||||
def test():
|
||||
path = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav"
|
||||
import librosa
|
||||
|
||||
cmvn_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/am.mvn"
|
||||
config_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/config.yaml"
|
||||
from funasr.runtime.python.onnxruntime.rapid_paraformer.utils.utils import read_yaml
|
||||
|
||||
config = read_yaml(config_file)
|
||||
waveform, _ = librosa.load(path, sr=None)
|
||||
frontend = WavFrontend(
|
||||
cmvn_file=cmvn_file,
|
||||
**config["frontend_conf"],
|
||||
)
|
||||
speech, _ = frontend.fbank_online(waveform) # 1d, (sample,), numpy
|
||||
feat, feat_len = frontend.lfr_cmvn(
|
||||
speech
|
||||
) # 2d, (frame, 450), np.float32 -> torch, torch.from_numpy(), dtype, (1, frame, 450)
|
||||
|
||||
frontend.reset_status() # clear cache
|
||||
return feat, feat_len
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test()
|
||||
@@ -0,0 +1,364 @@
|
||||
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||
|
||||
import string
|
||||
import logging
|
||||
from typing import Any, List, Union
|
||||
|
||||
|
||||
def isChinese(ch: str):
|
||||
if "\u4e00" <= ch <= "\u9fff" or "\u0030" <= ch <= "\u0039":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def isAllChinese(word: Union[List[Any], str]):
|
||||
word_lists = []
|
||||
for i in word:
|
||||
cur = i.replace(" ", "")
|
||||
cur = cur.replace("</s>", "")
|
||||
cur = cur.replace("<s>", "")
|
||||
word_lists.append(cur)
|
||||
|
||||
if len(word_lists) == 0:
|
||||
return False
|
||||
|
||||
for ch in word_lists:
|
||||
if isChinese(ch) is False:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def isAllAlpha(word: Union[List[Any], str]):
|
||||
word_lists = []
|
||||
for i in word:
|
||||
cur = i.replace(" ", "")
|
||||
cur = cur.replace("</s>", "")
|
||||
cur = cur.replace("<s>", "")
|
||||
word_lists.append(cur)
|
||||
|
||||
if len(word_lists) == 0:
|
||||
return False
|
||||
|
||||
for ch in word_lists:
|
||||
if ch.isalpha() is False and ch != "'":
|
||||
return False
|
||||
elif ch.isalpha() is True and isChinese(ch) is True:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# def abbr_dispose(words: List[Any]) -> List[Any]:
|
||||
def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]:
|
||||
words_size = len(words)
|
||||
word_lists = []
|
||||
abbr_begin = []
|
||||
abbr_end = []
|
||||
last_num = -1
|
||||
ts_lists = []
|
||||
ts_nums = []
|
||||
ts_index = 0
|
||||
for num in range(words_size):
|
||||
if num <= last_num:
|
||||
continue
|
||||
|
||||
if len(words[num]) == 1 and words[num].encode("utf-8").isalpha():
|
||||
if (
|
||||
num + 1 < words_size
|
||||
and words[num + 1] == " "
|
||||
and num + 2 < words_size
|
||||
and len(words[num + 2]) == 1
|
||||
and words[num + 2].encode("utf-8").isalpha()
|
||||
):
|
||||
# found the begin of abbr
|
||||
abbr_begin.append(num)
|
||||
num += 2
|
||||
abbr_end.append(num)
|
||||
# to find the end of abbr
|
||||
while True:
|
||||
num += 1
|
||||
if num < words_size and words[num] == " ":
|
||||
num += 1
|
||||
if (
|
||||
num < words_size
|
||||
and len(words[num]) == 1
|
||||
and words[num].encode("utf-8").isalpha()
|
||||
):
|
||||
abbr_end.pop()
|
||||
abbr_end.append(num)
|
||||
last_num = num
|
||||
else:
|
||||
break
|
||||
else:
|
||||
break
|
||||
|
||||
for num in range(words_size):
|
||||
if words[num] == " ":
|
||||
ts_nums.append(ts_index)
|
||||
else:
|
||||
ts_nums.append(ts_index)
|
||||
ts_index += 1
|
||||
last_num = -1
|
||||
for num in range(words_size):
|
||||
if num <= last_num:
|
||||
continue
|
||||
|
||||
if num in abbr_begin:
|
||||
if time_stamp is not None:
|
||||
begin = time_stamp[ts_nums[num]][0]
|
||||
word_lists.append(words[num].upper())
|
||||
num += 1
|
||||
while num < words_size:
|
||||
if num in abbr_end:
|
||||
word_lists.append(words[num].upper())
|
||||
last_num = num
|
||||
break
|
||||
else:
|
||||
if words[num].encode("utf-8").isalpha():
|
||||
word_lists.append(words[num].upper())
|
||||
num += 1
|
||||
if time_stamp is not None:
|
||||
end = time_stamp[ts_nums[num]][1]
|
||||
ts_lists.append([begin, end])
|
||||
else:
|
||||
word_lists.append(words[num])
|
||||
if time_stamp is not None and words[num] != " ":
|
||||
begin = time_stamp[ts_nums[num]][0]
|
||||
end = time_stamp[ts_nums[num]][1]
|
||||
ts_lists.append([begin, end])
|
||||
begin = end
|
||||
|
||||
if time_stamp is not None:
|
||||
return word_lists, ts_lists
|
||||
else:
|
||||
return word_lists
|
||||
|
||||
|
||||
def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):
|
||||
middle_lists = []
|
||||
word_lists = []
|
||||
word_item = ""
|
||||
ts_lists = []
|
||||
|
||||
# wash words lists
|
||||
for i in words:
|
||||
word = ""
|
||||
if isinstance(i, str):
|
||||
word = i
|
||||
else:
|
||||
word = i.decode("utf-8")
|
||||
|
||||
if word in ["<s>", "</s>", "<unk>"]:
|
||||
continue
|
||||
else:
|
||||
middle_lists.append(word)
|
||||
|
||||
# all chinese characters
|
||||
if isAllChinese(middle_lists):
|
||||
for i, ch in enumerate(middle_lists):
|
||||
word_lists.append(ch.replace(" ", ""))
|
||||
if time_stamp is not None:
|
||||
ts_lists = time_stamp
|
||||
|
||||
# all alpha characters
|
||||
elif isAllAlpha(middle_lists):
|
||||
ts_flag = True
|
||||
for i, ch in enumerate(middle_lists):
|
||||
if ts_flag and time_stamp is not None:
|
||||
begin = time_stamp[i][0]
|
||||
end = time_stamp[i][1]
|
||||
word = ""
|
||||
if "@@" in ch:
|
||||
word = ch.replace("@@", "")
|
||||
word_item += word
|
||||
if time_stamp is not None:
|
||||
ts_flag = False
|
||||
end = time_stamp[i][1]
|
||||
else:
|
||||
word_item += ch
|
||||
word_lists.append(word_item)
|
||||
word_lists.append(" ")
|
||||
word_item = ""
|
||||
if time_stamp is not None:
|
||||
ts_flag = True
|
||||
end = time_stamp[i][1]
|
||||
ts_lists.append([begin, end])
|
||||
begin = end
|
||||
|
||||
# mix characters
|
||||
else:
|
||||
alpha_blank = False
|
||||
ts_flag = True
|
||||
begin = -1
|
||||
end = -1
|
||||
for i, ch in enumerate(middle_lists):
|
||||
if ts_flag and time_stamp is not None:
|
||||
begin = time_stamp[i][0]
|
||||
end = time_stamp[i][1]
|
||||
word = ""
|
||||
if isAllChinese(ch):
|
||||
if alpha_blank is True:
|
||||
word_lists.pop()
|
||||
word_lists.append(ch)
|
||||
alpha_blank = False
|
||||
if time_stamp is not None:
|
||||
ts_flag = True
|
||||
ts_lists.append([begin, end])
|
||||
begin = end
|
||||
elif "@@" in ch:
|
||||
word = ch.replace("@@", "")
|
||||
word_item += word
|
||||
alpha_blank = False
|
||||
if time_stamp is not None:
|
||||
ts_flag = False
|
||||
end = time_stamp[i][1]
|
||||
elif isAllAlpha(ch):
|
||||
word_item += ch
|
||||
word_lists.append(word_item)
|
||||
word_lists.append(" ")
|
||||
word_item = ""
|
||||
alpha_blank = True
|
||||
if time_stamp is not None:
|
||||
ts_flag = True
|
||||
end = time_stamp[i][1]
|
||||
ts_lists.append([begin, end])
|
||||
begin = end
|
||||
else:
|
||||
raise ValueError("invalid character: {}".format(ch))
|
||||
|
||||
if time_stamp is not None:
|
||||
word_lists, ts_lists = abbr_dispose(word_lists, ts_lists)
|
||||
real_word_lists = []
|
||||
for ch in word_lists:
|
||||
if ch != " ":
|
||||
real_word_lists.append(ch)
|
||||
sentence = " ".join(real_word_lists).strip()
|
||||
return sentence, ts_lists, real_word_lists
|
||||
else:
|
||||
word_lists = abbr_dispose(word_lists)
|
||||
real_word_lists = []
|
||||
for ch in word_lists:
|
||||
if ch != " ":
|
||||
real_word_lists.append(ch)
|
||||
sentence = "".join(word_lists).strip()
|
||||
return sentence, real_word_lists
|
||||
|
||||
|
||||
emo_dict = {
|
||||
"<|HAPPY|>": "😊",
|
||||
"<|SAD|>": "😔",
|
||||
"<|ANGRY|>": "😡",
|
||||
"<|NEUTRAL|>": "",
|
||||
"<|FEARFUL|>": "😰",
|
||||
"<|DISGUSTED|>": "🤢",
|
||||
"<|SURPRISED|>": "😮",
|
||||
}
|
||||
|
||||
event_dict = {
|
||||
"<|BGM|>": "🎼",
|
||||
"<|Speech|>": "",
|
||||
"<|Applause|>": "👏",
|
||||
"<|Laughter|>": "😀",
|
||||
"<|Cry|>": "😭",
|
||||
"<|Sneeze|>": "🤧",
|
||||
"<|Breath|>": "",
|
||||
"<|Cough|>": "🤧",
|
||||
}
|
||||
|
||||
lang_dict = {
|
||||
"<|zh|>": "<|lang|>",
|
||||
"<|en|>": "<|lang|>",
|
||||
"<|yue|>": "<|lang|>",
|
||||
"<|ja|>": "<|lang|>",
|
||||
"<|ko|>": "<|lang|>",
|
||||
"<|nospeech|>": "<|lang|>",
|
||||
}
|
||||
|
||||
emoji_dict = {
|
||||
"<|nospeech|><|Event_UNK|>": "❓",
|
||||
"<|zh|>": "",
|
||||
"<|en|>": "",
|
||||
"<|yue|>": "",
|
||||
"<|ja|>": "",
|
||||
"<|ko|>": "",
|
||||
"<|nospeech|>": "",
|
||||
"<|HAPPY|>": "😊",
|
||||
"<|SAD|>": "😔",
|
||||
"<|ANGRY|>": "😡",
|
||||
"<|NEUTRAL|>": "",
|
||||
"<|BGM|>": "🎼",
|
||||
"<|Speech|>": "",
|
||||
"<|Applause|>": "👏",
|
||||
"<|Laughter|>": "😀",
|
||||
"<|FEARFUL|>": "😰",
|
||||
"<|DISGUSTED|>": "🤢",
|
||||
"<|SURPRISED|>": "😮",
|
||||
"<|Cry|>": "😭",
|
||||
"<|EMO_UNKNOWN|>": "",
|
||||
"<|Sneeze|>": "🤧",
|
||||
"<|Breath|>": "",
|
||||
"<|Cough|>": "😷",
|
||||
"<|Sing|>": "",
|
||||
"<|Speech_Noise|>": "",
|
||||
"<|withitn|>": "",
|
||||
"<|woitn|>": "",
|
||||
"<|GBG|>": "",
|
||||
"<|Event_UNK|>": "",
|
||||
}
|
||||
|
||||
emo_set = {"😊", "😔", "😡", "😰", "🤢", "😮"}
|
||||
event_set = {
|
||||
"🎼",
|
||||
"👏",
|
||||
"😀",
|
||||
"😭",
|
||||
"🤧",
|
||||
"😷",
|
||||
}
|
||||
|
||||
|
||||
def format_str_v2(s):
|
||||
sptk_dict = {}
|
||||
for sptk in emoji_dict:
|
||||
sptk_dict[sptk] = s.count(sptk)
|
||||
s = s.replace(sptk, "")
|
||||
emo = "<|NEUTRAL|>"
|
||||
for e in emo_dict:
|
||||
if sptk_dict[e] > sptk_dict[emo]:
|
||||
emo = e
|
||||
for e in event_dict:
|
||||
if sptk_dict[e] > 0:
|
||||
s = event_dict[e] + s
|
||||
s = s + emo_dict[emo]
|
||||
|
||||
for emoji in emo_set.union(event_set):
|
||||
s = s.replace(" " + emoji, emoji)
|
||||
s = s.replace(emoji + " ", emoji)
|
||||
return s.strip()
|
||||
|
||||
|
||||
def rich_transcription_postprocess(s):
|
||||
def get_emo(s):
|
||||
return s[-1] if s[-1] in emo_set else None
|
||||
|
||||
def get_event(s):
|
||||
return s[0] if s[0] in event_set else None
|
||||
|
||||
s = s.replace("<|nospeech|><|Event_UNK|>", "❓")
|
||||
for lang in lang_dict:
|
||||
s = s.replace(lang, "<|lang|>")
|
||||
s_list = [format_str_v2(s_i).strip(" ") for s_i in s.split("<|lang|>")]
|
||||
new_s = " " + s_list[0]
|
||||
cur_ent_event = get_event(new_s)
|
||||
for i in range(1, len(s_list)):
|
||||
if len(s_list[i]) == 0:
|
||||
continue
|
||||
if get_event(s_list[i]) == cur_ent_event and get_event(s_list[i]) != None:
|
||||
s_list[i] = s_list[i][1:]
|
||||
# else:
|
||||
cur_ent_event = get_event(s_list[i])
|
||||
if get_emo(s_list[i]) != None and get_emo(s_list[i]) == get_emo(new_s):
|
||||
new_s = new_s[:-1]
|
||||
new_s += s_list[i].strip().lstrip()
|
||||
new_s = new_s.replace("The.", " ")
|
||||
return new_s.strip()
|
||||
@@ -0,0 +1,53 @@
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
from typing import List
|
||||
from typing import Union
|
||||
|
||||
import sentencepiece as spm
|
||||
|
||||
|
||||
class SentencepiecesTokenizer:
|
||||
def __init__(self, bpemodel: Union[Path, str], **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.bpemodel = str(bpemodel)
|
||||
# NOTE(kamo):
|
||||
# Don't build SentencePieceProcessor in __init__()
|
||||
# because it's not picklable and it may cause following error,
|
||||
# "TypeError: can't pickle SwigPyObject objects",
|
||||
# when giving it as argument of "multiprocessing.Process()".
|
||||
self.sp = None
|
||||
self._build_sentence_piece_processor()
|
||||
|
||||
def __repr__(self):
|
||||
return f'{self.__class__.__name__}(model="{self.bpemodel}")'
|
||||
|
||||
def _build_sentence_piece_processor(self):
|
||||
# Build SentencePieceProcessor lazily.
|
||||
if self.sp is None:
|
||||
self.sp = spm.SentencePieceProcessor()
|
||||
self.sp.load(self.bpemodel)
|
||||
|
||||
def text2tokens(self, line: str) -> List[str]:
|
||||
self._build_sentence_piece_processor()
|
||||
return self.sp.EncodeAsPieces(line)
|
||||
|
||||
def tokens2text(self, tokens: Iterable[str]) -> str:
|
||||
self._build_sentence_piece_processor()
|
||||
return self.sp.DecodePieces(list(tokens))
|
||||
|
||||
def encode(self, line: str, **kwargs) -> List[int]:
|
||||
self._build_sentence_piece_processor()
|
||||
return self.sp.EncodeAsIds(line)
|
||||
|
||||
def decode(self, line: List[int], **kwargs):
|
||||
self._build_sentence_piece_processor()
|
||||
return self.sp.DecodeIds(line)
|
||||
|
||||
def get_vocab_size(self):
|
||||
return self.sp.GetPieceSize()
|
||||
|
||||
def ids2tokens(self, *args, **kwargs):
|
||||
return self.decode(*args, **kwargs)
|
||||
|
||||
def tokens2ids(self, *args, **kwargs):
|
||||
return self.encode(*args, **kwargs)
|
||||
@@ -0,0 +1,62 @@
|
||||
import numpy as np
|
||||
|
||||
|
||||
def time_stamp_lfr6_onnx(us_cif_peak, char_list, begin_time=0.0, total_offset=-1.5):
|
||||
if not len(char_list):
|
||||
return "", []
|
||||
START_END_THRESHOLD = 5
|
||||
MAX_TOKEN_DURATION = 30
|
||||
TIME_RATE = 10.0 * 6 / 1000 / 3 # 3 times upsampled
|
||||
cif_peak = us_cif_peak.reshape(-1).cpu()
|
||||
num_frames = cif_peak.shape[-1]
|
||||
if char_list[-1] == "</s>":
|
||||
char_list = char_list[:-1]
|
||||
# char_list = [i for i in text]
|
||||
timestamp_list = []
|
||||
new_char_list = []
|
||||
# for bicif model trained with large data, cif2 actually fires when a character starts
|
||||
# so treat the frames between two peaks as the duration of the former token
|
||||
fire_place = np.where(cif_peak > 1.0 - 1e-4)[0] + total_offset # np format
|
||||
num_peak = len(fire_place)
|
||||
assert num_peak == len(char_list) + 1 # number of peaks is supposed to be number of tokens + 1
|
||||
# begin silence
|
||||
if fire_place[0] > START_END_THRESHOLD:
|
||||
# char_list.insert(0, '<sil>')
|
||||
timestamp_list.append([0.0, fire_place[0] * TIME_RATE])
|
||||
new_char_list.append("<sil>")
|
||||
# tokens timestamp
|
||||
for i in range(len(fire_place) - 1):
|
||||
new_char_list.append(char_list[i])
|
||||
if (
|
||||
i == len(fire_place) - 2
|
||||
or MAX_TOKEN_DURATION < 0
|
||||
or fire_place[i + 1] - fire_place[i] < MAX_TOKEN_DURATION
|
||||
):
|
||||
timestamp_list.append([fire_place[i] * TIME_RATE, fire_place[i + 1] * TIME_RATE])
|
||||
else:
|
||||
# cut the duration to token and sil of the 0-weight frames last long
|
||||
_split = fire_place[i] + MAX_TOKEN_DURATION
|
||||
timestamp_list.append([fire_place[i] * TIME_RATE, _split * TIME_RATE])
|
||||
timestamp_list.append([_split * TIME_RATE, fire_place[i + 1] * TIME_RATE])
|
||||
new_char_list.append("<sil>")
|
||||
# tail token and end silence
|
||||
if num_frames - fire_place[-1] > START_END_THRESHOLD:
|
||||
_end = (num_frames + fire_place[-1]) / 2
|
||||
timestamp_list[-1][1] = _end * TIME_RATE
|
||||
timestamp_list.append([_end * TIME_RATE, num_frames * TIME_RATE])
|
||||
new_char_list.append("<sil>")
|
||||
else:
|
||||
timestamp_list[-1][1] = num_frames * TIME_RATE
|
||||
if begin_time: # add offset time in model with vad
|
||||
for i in range(len(timestamp_list)):
|
||||
timestamp_list[i][0] = timestamp_list[i][0] + begin_time / 1000.0
|
||||
timestamp_list[i][1] = timestamp_list[i][1] + begin_time / 1000.0
|
||||
assert len(new_char_list) == len(timestamp_list)
|
||||
res_str = ""
|
||||
for char, timestamp in zip(new_char_list, timestamp_list):
|
||||
res_str += "{} {} {};".format(char, timestamp[0], timestamp[1])
|
||||
res = []
|
||||
for char, timestamp in zip(new_char_list, timestamp_list):
|
||||
if char != "<sil>":
|
||||
res.append([int(timestamp[0] * 1000), int(timestamp[1] * 1000)])
|
||||
return res_str, res
|
||||
@@ -0,0 +1,161 @@
|
||||
# -*- encoding: utf-8 -*-
|
||||
import yaml
|
||||
import logging
|
||||
import functools
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union
|
||||
|
||||
root_dir = Path(__file__).resolve().parent
|
||||
logger_initialized = {}
|
||||
|
||||
def pad_list(xs, pad_value, max_len=None):
|
||||
n_batch = len(xs)
|
||||
if max_len is None:
|
||||
max_len = max(x.size(0) for x in xs)
|
||||
# pad = xs[0].new(n_batch, max_len, *xs[0].size()[1:]).fill_(pad_value)
|
||||
# numpy format
|
||||
pad = (np.zeros((n_batch, max_len)) + pad_value).astype(np.int32)
|
||||
for i in range(n_batch):
|
||||
pad[i, : xs[i].shape[0]] = xs[i]
|
||||
|
||||
return pad
|
||||
|
||||
class TokenIDConverter:
|
||||
def __init__(
|
||||
self,
|
||||
token_list: Union[List, str],
|
||||
):
|
||||
|
||||
self.token_list = token_list
|
||||
self.unk_symbol = token_list[-1]
|
||||
self.token2id = {v: i for i, v in enumerate(self.token_list)}
|
||||
self.unk_id = self.token2id[self.unk_symbol]
|
||||
|
||||
def get_num_vocabulary_size(self) -> int:
|
||||
return len(self.token_list)
|
||||
|
||||
def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]:
|
||||
if isinstance(integers, np.ndarray) and integers.ndim != 1:
|
||||
raise TokenIDConverterError(f"Must be 1 dim ndarray, but got {integers.ndim}")
|
||||
return [self.token_list[i] for i in integers]
|
||||
|
||||
def tokens2ids(self, tokens: Iterable[str]) -> List[int]:
|
||||
|
||||
return [self.token2id.get(i, self.unk_id) for i in tokens]
|
||||
|
||||
|
||||
class CharTokenizer:
|
||||
def __init__(
|
||||
self,
|
||||
symbol_value: Union[Path, str, Iterable[str]] = None,
|
||||
space_symbol: str = "<space>",
|
||||
remove_non_linguistic_symbols: bool = False,
|
||||
):
|
||||
|
||||
self.space_symbol = space_symbol
|
||||
self.non_linguistic_symbols = self.load_symbols(symbol_value)
|
||||
self.remove_non_linguistic_symbols = remove_non_linguistic_symbols
|
||||
|
||||
@staticmethod
|
||||
def load_symbols(value: Union[Path, str, Iterable[str]] = None) -> Set:
|
||||
if value is None:
|
||||
return set()
|
||||
|
||||
if isinstance(value, Iterable[str]):
|
||||
return set(value)
|
||||
|
||||
file_path = Path(value)
|
||||
if not file_path.exists():
|
||||
logging.warning("%s doesn't exist.", file_path)
|
||||
return set()
|
||||
|
||||
with file_path.open("r", encoding="utf-8") as f:
|
||||
return set(line.rstrip() for line in f)
|
||||
|
||||
def text2tokens(self, line: Union[str, list]) -> List[str]:
|
||||
tokens = []
|
||||
while len(line) != 0:
|
||||
for w in self.non_linguistic_symbols:
|
||||
if line.startswith(w):
|
||||
if not self.remove_non_linguistic_symbols:
|
||||
tokens.append(line[: len(w)])
|
||||
line = line[len(w) :]
|
||||
break
|
||||
else:
|
||||
t = line[0]
|
||||
if t == " ":
|
||||
t = "<space>"
|
||||
tokens.append(t)
|
||||
line = line[1:]
|
||||
return tokens
|
||||
|
||||
def tokens2text(self, tokens: Iterable[str]) -> str:
|
||||
tokens = [t if t != self.space_symbol else " " for t in tokens]
|
||||
return "".join(tokens)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"{self.__class__.__name__}("
|
||||
f'space_symbol="{self.space_symbol}"'
|
||||
f'non_linguistic_symbols="{self.non_linguistic_symbols}"'
|
||||
f")"
|
||||
)
|
||||
|
||||
|
||||
class Hypothesis(NamedTuple):
|
||||
"""Hypothesis data type."""
|
||||
|
||||
yseq: np.ndarray
|
||||
score: Union[float, np.ndarray] = 0
|
||||
scores: Dict[str, Union[float, np.ndarray]] = dict()
|
||||
states: Dict[str, Any] = dict()
|
||||
|
||||
def asdict(self) -> dict:
|
||||
"""Convert data to JSON-friendly dict."""
|
||||
return self._replace(
|
||||
yseq=self.yseq.tolist(),
|
||||
score=float(self.score),
|
||||
scores={k: float(v) for k, v in self.scores.items()},
|
||||
)._asdict()
|
||||
|
||||
|
||||
def read_yaml(yaml_path: Union[str, Path]) -> Dict:
|
||||
if not Path(yaml_path).exists():
|
||||
raise FileExistsError(f"The {yaml_path} does not exist.")
|
||||
|
||||
with open(str(yaml_path), "rb") as f:
|
||||
data = yaml.load(f, Loader=yaml.Loader)
|
||||
return data
|
||||
|
||||
|
||||
@functools.lru_cache()
|
||||
def get_logger(name="funasr_torch"):
|
||||
"""Initialize and get a logger by name.
|
||||
If the logger has not been initialized, this method will initialize the
|
||||
logger by adding one or two handlers, otherwise the initialized logger will
|
||||
be directly returned. During initialization, a StreamHandler will always be
|
||||
added.
|
||||
Args:
|
||||
name (str): Logger name.
|
||||
Returns:
|
||||
logging.Logger: The expected logger.
|
||||
"""
|
||||
logger = logging.getLogger(name)
|
||||
if name in logger_initialized:
|
||||
return logger
|
||||
|
||||
for logger_name in logger_initialized:
|
||||
if name.startswith(logger_name):
|
||||
return logger
|
||||
|
||||
formatter = logging.Formatter(
|
||||
"[%(asctime)s] %(name)s %(levelname)s: %(message)s", datefmt="%Y/%m/%d %H:%M:%S"
|
||||
)
|
||||
|
||||
sh = logging.StreamHandler()
|
||||
sh.setFormatter(formatter)
|
||||
logger.addHandler(sh)
|
||||
logger_initialized[name] = True
|
||||
logger.propagate = False
|
||||
return logger
|
||||
Reference in New Issue
Block a user