chore: import upstream snapshot with attribution

examples/industrial_data_pretraining/bicif_paraformer/demo.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(
+    model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+    vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+    punc_model="iic/punc_ct-transformer_cn-en-common-vocab471067-large",
+    # spk_model="iic/speech_campplus_sv_zh-cn_16k-common",
+)
+
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_vad_punc_example.wav",
+    batch_size_s=300,
+    batch_size_threshold_s=60,
+)
+print(res)

examples/industrial_data_pretraining/bicif_paraformer/demo.sh

@@ -0,0 +1,18 @@
+
+model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
+#punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
+punc_model="iic/punc_ct-transformer_cn-en-common-vocab471067-large"
+spk_model="iic/speech_campplus_sv_zh-cn_16k-common"
+
+python funasr/bin/inference.py \
++model=${model} \
++vad_model=${vad_model} \
++punc_model=${punc_model} \
++spk_model=${spk_model} \
++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_vad_punc_example.wav" \
++output_dir="./outputs/debug" \
++device="cpu" \
++batch_size_s=300 \
++batch_size_threshold_s=60
+

examples/industrial_data_pretraining/bicif_paraformer/export.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method1, inference from model hub
+
+from funasr import AutoModel
+
+model = AutoModel(
+    model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+    device="cpu",
+)
+
+res = model.export(type="torchscript", quantize=False)
+print(res)
+
+
+# # method2, inference from local path
+# from funasr import AutoModel
+
+# model = AutoModel(
+#     model="/Users/zhifu/.cache/modelscope/hub/iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+#     device="cpu",
+# )
+
+# res = model.export(type="onnx", quantize=False)
+# print(res)

examples/industrial_data_pretraining/bicif_paraformer/export.sh

@@ -0,0 +1,23 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method1, inference from model hub
+export HYDRA_FULL_ERROR=1
+
+
+model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+
+python -m funasr.bin.export \
+++model=${model} \
+++type="onnx" \
+++quantize=false \
+++device="cpu"
+
+# method2, inference from local path
+model="/Users/zhifu/.cache/modelscope/hub/iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+
+python -m funasr.bin.export \
+++model=${model} \
+++type="onnx" \
+++quantize=false \
+++device="cpu"

examples/industrial_data_pretraining/bicif_paraformer/finetune.sh

@@ -0,0 +1,84 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+workspace=`pwd`
+
+# method1, finetune from model hub
+
+# which gpu to train or finetune
+export CUDA_VISIBLE_DEVICES="0,1"
+gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+
+# model_name from model_hub, or model_dir in local path
+
+## option 1, download model automatically
+model_name_or_model_dir="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+
+## option 2, download model by git
+#local_path_root=${workspace}/modelscope_models
+#mkdir -p ${local_path_root}/${model_name_or_model_dir}
+#git clone https://www.modelscope.cn/${model_name_or_model_dir}.git ${local_path_root}/${model_name_or_model_dir}
+#model_name_or_model_dir=${local_path_root}/${model_name_or_model_dir}
+
+
+# data dir, which contains: train.json, val.json
+data_dir="../../../data/list"
+
+train_data="${data_dir}/train.jsonl"
+val_data="${data_dir}/val.jsonl"
+
+# generate train.jsonl and val.jsonl from wav.scp and text.txt
+scp2jsonl \
+++scp_file_list='["../../../data/list/train_wav.scp", "../../../data/list/train_text.txt"]' \
+++data_type_list='["source", "target"]' \
+++jsonl_file_out="${train_data}"
+
+scp2jsonl \
+++scp_file_list='["../../../data/list/val_wav.scp", "../../../data/list/val_text.txt"]' \
+++data_type_list='["source", "target"]' \
+++jsonl_file_out="${val_data}"
+
+
+# exp output dir
+output_dir="./outputs"
+log_file="${output_dir}/log.txt"
+
+
+mkdir -p ${output_dir}
+echo "log_file: ${log_file}"
+
+deepspeed_config=${workspace}/../../ds_stage1.json
+
+DISTRIBUTED_ARGS="
+    --nnodes ${WORLD_SIZE:-1} \
+    --nproc_per_node $gpu_num \
+    --node_rank ${RANK:-0} \
+    --master_addr ${MASTER_ADDR:-127.0.0.1} \
+    --master_port ${MASTER_PORT:-26669}
+"
+
+echo $DISTRIBUTED_ARGS
+
+torchrun $DISTRIBUTED_ARGS \
+../../../funasr/bin/train_ds.py \
+++model="${model_name_or_model_dir}" \
+++train_data_set_list="${train_data}" \
+++valid_data_set_list="${val_data}" \
+++dataset="AudioDataset" \
+++dataset_conf.index_ds="IndexDSJsonl" \
+++dataset_conf.data_split_num=1 \
+++dataset_conf.batch_sampler="BatchSampler" \
+++dataset_conf.batch_size=6000  \
+++dataset_conf.sort_size=1024 \
+++dataset_conf.batch_type="token" \
+++dataset_conf.num_workers=4 \
+++train_conf.max_epoch=50 \
+++train_conf.log_interval=1 \
+++train_conf.resume=true \
+++train_conf.validate_interval=2000 \
+++train_conf.save_checkpoint_interval=2000 \
+++train_conf.keep_nbest_models=20 \
+++train_conf.use_deepspeed=false \
+++train_conf.deepspeed_config=${deepspeed_config} \
+++optim_conf.lr=0.0002 \
+++output_dir="${output_dir}" &> ${log_file}

examples/industrial_data_pretraining/campplus_sv/demo.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(model="iic/speech_campplus_sv_zh-cn_16k-common")
+
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav"
+)
+print(res)

examples/industrial_data_pretraining/conformer/demo.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(model="iic/speech_conformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch")
+
+res = model.generate(
+    input="https://modelscope.oss-cn-beijing.aliyuncs.com/test/audios/asr_example.wav",
+    decoding_ctc_weight=0.0,
+)
+print(res)

examples/industrial_data_pretraining/conformer/demo.sh

@@ -0,0 +1,9 @@
+
+model="iic/speech_conformer_asr_nat-zh-cn-16k-aishell1-vocab4234-pytorch"
+
+python funasr/bin/inference.py \
++model=${model} \
++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav" \
++output_dir="./outputs/debug" \
++device="cpu" \
+

examples/industrial_data_pretraining/contextual_paraformer/demo.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(model="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404")
+
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav",
+    hotword="达摩院 魔搭",
+)
+print(res)

examples/industrial_data_pretraining/contextual_paraformer/demo.sh

@@ -0,0 +1,11 @@
+
+model="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404"
+
+
+python ../../../funasr/bin/inference.py \
++model=${model} \
++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav" \
++output_dir="./outputs/debug" \
++device="cpu" \
++"hotword='达摩院 魔搭'"
+

examples/industrial_data_pretraining/contextual_paraformer/demo2.sh

@@ -0,0 +1,9 @@
+python -m funasr.bin.inference \
+--config-path="/nfs/yufan.yf/workspace/model_download/modelscope/hub/damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404" \
+--config-name="config.yaml" \
+++init_param="/nfs/yufan.yf/workspace/model_download/modelscope/hub/damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404/model.pb" \
+++tokenizer_conf.token_list="/nfs/yufan.yf/workspace/model_download/modelscope/hub/damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404/tokens.txt" \
+++frontend_conf.cmvn_file="/nfs/yufan.yf/workspace/model_download/modelscope/hub/damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404/am.mvn" \
+++input="/nfs/yufan.yf/workspace/model_download/modelscope/hub/damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404/asr_example_zh.wav" \
+++output_dir="./outputs/debug2" \
+++device="" \

examples/industrial_data_pretraining/contextual_paraformer/finetune.sh

@@ -0,0 +1,85 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+workspace=`pwd`
+
+# method1, finetune from model hub
+
+# which gpu to train or finetune
+export CUDA_VISIBLE_DEVICES="0,1"
+gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+
+# model_name from model_hub, or model_dir in local path
+
+## option 1, download model automatically
+model_name_or_model_dir="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404"
+
+
+## option 2, download model by git
+#local_path_root=${workspace}/modelscope_models
+#mkdir -p ${local_path_root}/${model_name_or_model_dir}
+#git clone https://www.modelscope.cn/${model_name_or_model_dir}.git ${local_path_root}/${model_name_or_model_dir}
+#model_name_or_model_dir=${local_path_root}/${model_name_or_model_dir}
+
+
+# data dir, which contains: train.json, val.json
+data_dir="../../../data/list"
+
+train_data="${data_dir}/train.jsonl"
+val_data="${data_dir}/val.jsonl"
+
+# generate train.jsonl and val.jsonl from wav.scp and text.txt
+scp2jsonl \
+++scp_file_list='["../../../data/list/train_wav.scp", "../../../data/list/train_text.txt"]' \
+++data_type_list='["source", "target"]' \
+++jsonl_file_out="${train_data}"
+
+scp2jsonl \
+++scp_file_list='["../../../data/list/val_wav.scp", "../../../data/list/val_text.txt"]' \
+++data_type_list='["source", "target"]' \
+++jsonl_file_out="${val_data}"
+
+
+# exp output dir
+output_dir="./outputs"
+log_file="${output_dir}/log.txt"
+
+
+mkdir -p ${output_dir}
+echo "log_file: ${log_file}"
+
+deepspeed_config=${workspace}/../../ds_stage1.json
+
+DISTRIBUTED_ARGS="
+    --nnodes ${WORLD_SIZE:-1} \
+    --nproc_per_node $gpu_num \
+    --node_rank ${RANK:-0} \
+    --master_addr ${MASTER_ADDR:-127.0.0.1} \
+    --master_port ${MASTER_PORT:-26669}
+"
+
+echo $DISTRIBUTED_ARGS
+
+torchrun $DISTRIBUTED_ARGS \
+../../../funasr/bin/train_ds.py \
+++model="${model_name_or_model_dir}" \
+++train_data_set_list="${train_data}" \
+++valid_data_set_list="${val_data}" \
+++dataset="AudioDatasetHotword" \
+++dataset_conf.index_ds="IndexDSJsonl" \
+++dataset_conf.data_split_num=1 \
+++dataset_conf.batch_sampler="BatchSampler" \
+++dataset_conf.batch_size=6000  \
+++dataset_conf.sort_size=1024 \
+++dataset_conf.batch_type="token" \
+++dataset_conf.num_workers=4 \
+++train_conf.max_epoch=50 \
+++train_conf.log_interval=1 \
+++train_conf.resume=true \
+++train_conf.validate_interval=2000 \
+++train_conf.save_checkpoint_interval=2000 \
+++train_conf.keep_nbest_models=20 \
+++train_conf.use_deepspeed=false \
+++train_conf.deepspeed_config=${deepspeed_config} \
+++optim_conf.lr=0.0002 \
+++output_dir="${output_dir}" &> ${log_file}

examples/industrial_data_pretraining/contextual_paraformer/path.sh

@@ -0,0 +1,6 @@
+export FUNASR_DIR=$PWD/../../../
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PATH=$FUNASR_DIR/funasr/bin:$PATH
+export PYTHONPATH=$FUNASR_DIR/funasr/bin:$FUNASR_DIR/funasr:$FUNASR_DIR:$PYTHONPATH

examples/industrial_data_pretraining/ct_transformer/demo.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch")
+
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt"
+)
+print(res)
+
+
+from funasr import AutoModel
+
+model = AutoModel(model="iic/punc_ct-transformer_cn-en-common-vocab471067-large")
+
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt"
+)
+print(res)

examples/industrial_data_pretraining/ct_transformer/demo.sh

@@ -0,0 +1,12 @@
+
+#model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
+#
+
+model="iic/punc_ct-transformer_cn-en-common-vocab471067-large"
+
+
+python funasr/bin/inference.py \
++model=${model} \
++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt" \
++output_dir="./outputs/debug" \
++device="cpu"

examples/industrial_data_pretraining/ct_transformer/export.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method1, inference from model hub
+
+from funasr import AutoModel
+
+model = AutoModel(
+    model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+)
+
+res = model.export(type="onnx", quantize=False)
+print(res)
+
+
+# method2, inference from local path
+from funasr import AutoModel
+
+model = AutoModel(
+    model="/Users/zhifu/.cache/modelscope/hub/iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
+)
+
+res = model.export(type="onnx", quantize=False)
+print(res)

examples/industrial_data_pretraining/ct_transformer/export.sh

@@ -0,0 +1,26 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method1, inference from model hub
+export HYDRA_FULL_ERROR=1
+
+
+model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
+
+
+python -m funasr.bin.export \
+++model=${model} \
+++model_revision=${model_revision} \
+++type="onnx" \
+++quantize=false \
+++device="cpu"
+
+
+# method2, inference from local path
+model="/Users/zhifu/.cache/modelscope/hub/iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"
+
+python -m funasr.bin.export \
+++model=${model} \
+++type="onnx" \
+++quantize=false \
+++device="cpu"

examples/industrial_data_pretraining/ct_transformer_streaming/demo.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(model="iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727")
+
+inputs = "跨境河流是养育沿岸|人民的生命之源长期以来为帮助下游地区防灾减灾中方技术人员|在上游地区极为恶劣的自然条件下克服巨大困难甚至冒着生命危险|向印方提供汛期水文资料处理紧急事件中方重视印方在跨境河流问题上的关切|愿意进一步完善双方联合工作机制|凡是|中方能做的我们|都会去做而且会做得更好我请印度朋友们放心中国在上游的|任何开发利用都会经过科学|规划和论证兼顾上下游的利益"
+vads = inputs.split("|")
+rec_result_all = "outputs: "
+cache = {}
+for vad in vads:
+    rec_result = model.generate(input=vad, cache=cache)
+    rec_result_all += rec_result[0]["text"]
+
+print(rec_result_all)

examples/industrial_data_pretraining/ct_transformer_streaming/demo.sh

@@ -0,0 +1,9 @@
+
+model="iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727"
+
+
+python funasr/bin/inference.py \
++model=${model} \
++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt" \
++output_dir="./outputs/debug" \
++device="cpu"

examples/industrial_data_pretraining/ct_transformer_streaming/export.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method1, inference from model hub
+
+from funasr import AutoModel
+
+model = AutoModel(
+    model="iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727",
+)
+
+res = model.export(type="onnx", quantize=False)
+print(res)
+
+
+# method2, inference from local path
+from funasr import AutoModel
+
+model = AutoModel(
+    model="/Users/zhifu/.cache/modelscope/hub/iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727"
+)
+
+res = model.export(type="onnx", quantize=False)
+print(res)

examples/industrial_data_pretraining/ct_transformer_streaming/export.sh

@@ -0,0 +1,28 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method1, inference from model hub
+export HYDRA_FULL_ERROR=1
+
+
+model="iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727"
+
+
+python -m funasr.bin.export \
+++model=${model} \
+++model_revision=${model_revision} \
+++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav" \
+++type="onnx" \
+++quantize=false \
+++device="cpu"
+
+
+# method2, inference from local path
+model="/Users/zhifu/.cache/modelscope/hub/iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727"
+
+python -m funasr.bin.export \
+++model=${model} \
+++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav" \
+++type="onnx" \
+++quantize=false \
+++device="cpu"

examples/industrial_data_pretraining/ctc/demo.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+import sys
+from funasr import AutoModel
+
+
+model_dir = "/Users/zhifu/Downloads/modelscope_models/ctc_model"
+input_file = (
+    "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav"
+)
+
+model = AutoModel(
+    model=model_dir,
+)
+
+res = model.generate(
+    input=input_file,
+    cache={},
+)
+
+print(res)

examples/industrial_data_pretraining/ctc/infer_from_local.sh

@@ -0,0 +1,31 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method2, inference from local model
+
+# for more input type, please ref to readme.md
+model_dir=$1
+input_file=$2
+output_dir=$3
+
+# download model
+device="cuda:0" # "cuda:0" for gpu0, "cuda:1" for gpu1, "cpu"
+
+tokens="${model_dir}/tokens.json"
+cmvn_file="${model_dir}/am.mvn"
+
+config="config.yaml"
+init_param="${model_dir}/model.pt"
+
+mkdir -p ${output_dir}
+
+python -m funasr.bin.inference \
+--config-path "${model_dir}" \
+--config-name "${config}" \
+++init_param="${init_param}" \
+++tokenizer_conf.token_list="${tokens}" \
+++frontend_conf.cmvn_file="${cmvn_file}" \
+++input="${input_file}" \
+++output_dir="${output_dir}" \
+++device="${device}" \
+

examples/industrial_data_pretraining/eres2netv2_sv/demo.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+# ASR with speaker diarization using ERes2NetV2
+model = AutoModel(
+    model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+    vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+    vad_kwargs={"max_single_segment_time": 60000},
+    punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+    spk_model="iic/speech_eres2netv2_sv_zh-cn_16k-common",
+    device="cuda:0",
+)
+
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav",
+    batch_size_s=300,
+)
+
+for sentence in res:
+    print(f"[Speaker {sentence.get('spk', '?')}] {sentence['text']}")

examples/industrial_data_pretraining/eres2netv2_sv/demo_sv.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+# Standalone speaker verification / embedding extraction
+model = AutoModel(
+    model="iic/speech_eres2netv2_sv_zh-cn_16k-common",
+    device="cuda:0",
+)
+
+# Extract speaker embedding
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav",
+)
+
+embedding = res[0]["spk_embedding"]
+print(f"Speaker embedding shape: {embedding.shape}")

examples/industrial_data_pretraining/fsmn_kws/conf/fsmn_4e_l10r2_250_128_fdim80_t2599.yaml

@@ -0,0 +1,95 @@
+
+# network architecture
+model: FsmnKWS
+model_conf:
+    ctc_weight: 1.0
+
+# encoder related
+encoder: FSMN
+encoder_conf:
+    input_dim: 400
+    input_affine_dim: 140
+    fsmn_layers: 4
+    linear_dim: 250
+    proj_dim: 128
+    lorder: 10
+    rorder: 2
+    lstride: 1
+    rstride: 1
+    output_affine_dim: 140
+    output_dim: 2599
+    use_softmax: false
+
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 5
+    lfr_n: 3
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 3
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 100
+  keep_nbest_models: 10
+  avg_nbest_model: 10
+  avg_keep_nbest_models_type: loss
+  validate_interval: 50000
+  save_checkpoint_interval: 50000
+  avg_checkpoint_interval: 1000
+  log_interval: 50
+
+optim: adam
+optim_conf:
+   lr: 0.0005
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 10000
+
+dataset: AudioDataset
+dataset_conf:
+    index_ds: IndexDSJsonl
+    batch_sampler: EspnetStyleBatchSampler
+    batch_type: length # example or length
+    batch_size: 32000 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
+    max_token_length: 1600 # filter samples if source_token_len+target_token_len > max_token_length,
+    buffer_size: 2048
+    shuffle: true
+    num_workers: 8
+    preprocessor_speech: SpeechPreprocessSpeedPerturb
+    preprocessor_speech_conf:
+      speed_perturb: [0.9, 1.0, 1.1]
+
+tokenizer: CharTokenizer
+tokenizer_conf:
+    unk_symbol: <unk>
+    split_with_space: true
+
+ctc_conf:
+    dropout_rate: 0.0
+    ctc_type: builtin
+    reduce: true
+    ignore_nan_grad: true
+    extra_linear: false
+
+normalize: null

examples/industrial_data_pretraining/fsmn_kws/conf/fsmn_4e_l10r2_280_200_fdim40_t2602.yaml

@@ -0,0 +1,95 @@
+
+# network architecture
+model: FsmnKWS
+model_conf:
+    ctc_weight: 1.0
+
+# encoder related
+encoder: FSMN
+encoder_conf:
+    input_dim: 360
+    input_affine_dim: 280
+    fsmn_layers: 4
+    linear_dim: 280
+    proj_dim: 200
+    lorder: 10
+    rorder: 2
+    lstride: 1
+    rstride: 1
+    output_affine_dim: 400
+    output_dim: 2602
+    use_softmax: false
+
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 40
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 9
+    lfr_n: 3
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 3
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 100
+  keep_nbest_models: 10
+  avg_nbest_model: 10
+  avg_keep_nbest_models_type: loss
+  validate_interval: 50000
+  save_checkpoint_interval: 50000
+  avg_checkpoint_interval: 1000
+  log_interval: 50
+
+optim: adam
+optim_conf:
+   lr: 0.0005
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 10000
+
+dataset: AudioDataset
+dataset_conf:
+    index_ds: IndexDSJsonl
+    batch_sampler: EspnetStyleBatchSampler
+    batch_type: length # example or length
+    batch_size: 32000 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
+    max_token_length: 1600 # filter samples if source_token_len+target_token_len > max_token_length,
+    buffer_size: 2048
+    shuffle: true
+    num_workers: 8
+    preprocessor_speech: SpeechPreprocessSpeedPerturb
+    preprocessor_speech_conf:
+      speed_perturb: [0.9, 1.0, 1.1]
+
+tokenizer: CharTokenizer
+tokenizer_conf:
+    unk_symbol: <unk>
+    split_with_space: true
+
+ctc_conf:
+    dropout_rate: 0.0
+    ctc_type: builtin
+    reduce: true
+    ignore_nan_grad: true
+    extra_linear: false
+
+normalize: null

examples/industrial_data_pretraining/fsmn_kws/convert.py

@@ -0,0 +1,134 @@
+from __future__ import print_function
+
+import argparse
+import copy
+import logging
+import os
+from shutil import copyfile
+
+import torch
+import yaml
+from typing import Union
+
+
+from funasr.models.fsmn_kws.model import FsmnKWSConvert
+
+
+def count_parameters(model):
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description=
+        'load and convert network to each other between kaldi/pytorch format')
+    parser.add_argument('--config', required=True, help='config file')
+    parser.add_argument(
+        '--network_file',
+        default='',
+        required=True,
+        help='input network, support kaldi.txt/pytorch.pt')
+    parser.add_argument('--model_dir', required=True, help='save model dir')
+    parser.add_argument('--model_name', required=True, help='save model name')
+    parser.add_argument('--convert_to',
+                        default='kaldi',
+                        required=True,
+                        help='target network type, kaldi/pytorch')
+
+    args = parser.parse_args()
+    return args
+
+
+def convert_to_kaldi(
+    configs,
+    network_file,
+    model_dir,
+    model_name="convert.kaldi.txt"
+):
+    copyfile(network_file, os.path.join(model_dir, 'origin.torch.pt'))
+
+    model = FsmnKWSConvert(
+            vocab_size=configs['encoder_conf']['output_dim'],
+            encoder='FSMNConvert',
+            encoder_conf=configs['encoder_conf'],
+            ctc_conf=configs['ctc_conf'],
+    )
+    print(model)
+    num_params = count_parameters(model)
+    print('the number of model params: {}'.format(num_params))
+
+    states= torch.load(network_file, map_location='cpu')
+    model.load_state_dict(states["state_dict"])
+
+    kaldi_text = os.path.join(model_dir, model_name)
+    with open(kaldi_text, 'w', encoding='utf8') as fout:
+        nnet_desp = model.to_kaldi_net()
+        fout.write(nnet_desp)
+    fout.close()
+
+
+def convert_to_pytorch(
+    configs,
+    network_file,
+    model_dir,
+    model_name="convert.torch.pt"
+):
+    model = FsmnKWSConvert(
+            vocab_size=configs['encoder_conf']['output_dim'],
+            frontend=None,
+            specaug=None,
+            normalize=None,
+            encoder='FSMNConvert',
+            encoder_conf=configs['encoder_conf'],
+            ctc_conf=configs['ctc_conf'],
+    )
+
+    num_params = count_parameters(model)
+    print('the number of model params: {}'.format(num_params))
+
+    copyfile(network_file, os.path.join(model_dir, 'origin.kaldi.txt'))
+    model.to_pytorch_net(network_file)
+
+    save_model_path = os.path.join(model_dir, model_name)
+    torch.save({"model": model.state_dict()}, save_model_path)
+
+    print('convert torch format back to kaldi')
+    kaldi_text = os.path.join(model_dir, 'convert.kaldi.txt')
+    with open(kaldi_text, 'w', encoding='utf8') as fout:
+        nnet_desp = model.to_kaldi_net()
+        fout.write(nnet_desp)
+    fout.close()
+
+    print('Done!')
+
+
+def main():
+    args = get_args()
+    logging.basicConfig(level=logging.DEBUG,
+                        format='%(asctime)s %(levelname)s %(message)s')
+    print(args)
+    with open(args.config, 'r') as fin:
+        configs = yaml.load(fin, Loader=yaml.FullLoader)
+
+    if args.convert_to == 'pytorch':
+        print('convert kaldi net to pytorch...')
+        convert_to_pytorch(
+            configs,
+            args.network_file,
+            args.model_dir,
+            args.model_name
+        )
+    elif args.convert_to == 'kaldi':
+        print('convert pytorch net to kaldi...')
+        convert_to_kaldi(
+            configs,
+            args.network_file,
+            args.model_dir,
+            args.model_name
+        )
+    else:
+        print('unsupported target network type: {}'.format(args.convert_to))
+
+
+if __name__ == '__main__':
+    main()

examples/industrial_data_pretraining/fsmn_kws/convert.sh

@@ -0,0 +1,26 @@
+
+workspace=`pwd`
+
+# download model
+local_path_root=${workspace}/modelscope_models_kws
+mkdir -p ${local_path_root}
+
+local_path=${local_path_root}/speech_charctc_kws_phone-xiaoyun
+if [ ! -d "$local_path" ]; then
+    git clone https://www.modelscope.cn/iic/speech_charctc_kws_phone-xiaoyun.git ${local_path}
+fi
+
+export PATH=${local_path}/runtime:$PATH
+export LD_LIBRARY_PATH=${local_path}/runtime:$LD_LIBRARY_PATH
+
+config=./conf/fsmn_4e_l10r2_250_128_fdim80_t2599.yaml
+torch_nnet=exp/finetune_outputs/model.pt.avg10
+out_dir=exp/finetune_outputs
+
+if [ ! -d "$out_dir" ]; then
+    mkdir -p $out_dir
+fi
+
+python convert.py --config $config --network_file $torch_nnet --model_dir $out_dir --model_name "convert.kaldi.txt" --convert_to kaldi
+
+nnet-copy --binary=true ${out_dir}/convert.kaldi.txt ${out_dir}/convert.kaldi.net

examples/industrial_data_pretraining/fsmn_kws/demo.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(
+    model="iic/speech_charctc_kws_phone-xiaoyun",
+    keywords="小云小云",
+    output_dir="./outputs/debug",
+    device='cpu'
+)
+
+test_wav = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/KWS/pos_testset/kws_xiaoyunxiaoyun.wav"
+
+res = model.generate(input=test_wav, cache={},)
+print(res)

examples/industrial_data_pretraining/fsmn_kws/finetune.sh

@@ -0,0 +1,173 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+#!/usr/bin/env bash
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+. ./path.sh
+workspace=`pwd`
+
+CUDA_VISIBLE_DEVICES="0,1"
+
+stage=2
+stop_stage=3
+
+inference_device="cuda" #"cpu"
+inference_checkpoint="model.pt.avg10"
+inference_scp="wav.scp"
+inference_batch_size=32
+nj=32
+test_sets="test"
+
+# model_name from model_hub, or model_dir in local path
+
+## option 1, download model automatically, unsupported currently
+model_name_or_model_dir="iic/speech_charctc_kws_phone-xiaoyun"
+
+## option 2, download model by git
+local_path_root=${workspace}/modelscope_models
+model_name_or_model_dir=${local_path_root}/${model_name_or_model_dir}
+if [ ! -d $model_name_or_model_dir ]; then
+  mkdir -p ${model_name_or_model_dir}
+  git clone https://www.modelscope.cn/iic/speech_charctc_kws_phone-xiaoyun.git ${model_name_or_model_dir}
+fi
+
+config=fsmn_4e_l10r2_250_128_fdim80_t2599.yaml
+token_list=${model_name_or_model_dir}/funasr/tokens_2599.txt
+lexicon_list=${model_name_or_model_dir}/funasr/lexicon.txt
+cmvn_file=${model_name_or_model_dir}/funasr/am.mvn.dim80_l2r2
+init_param="${model_name_or_model_dir}/funasr/basetrain_fsmn_4e_l10r2_250_128_fdim80_t2599.pt"
+
+
+# data prepare
+# data dir, which contains: train.json, val.json
+data_dir=../../data
+
+train_data="${data_dir}/train.jsonl"
+val_data="${data_dir}/val.jsonl"
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  echo "stage 1: Generate audio json list"
+  # generate train.jsonl and val.jsonl from wav.scp and text.txt
+  python $FUNASR_DIR/funasr/datasets/audio_datasets/scp2jsonl.py \
+  ++scp_file_list='['''${data_dir}/train_wav.scp''', '''${data_dir}/train_text.txt''']' \
+  ++data_type_list='["source", "target"]' \
+  ++jsonl_file_out="${train_data}"
+
+  python $FUNASR_DIR/funasr/datasets/audio_datasets/scp2jsonl.py \
+  ++scp_file_list='['''${data_dir}/val_wav.scp''', '''${data_dir}/val_text.txt''']' \
+  ++data_type_list='["source", "target"]' \
+  ++jsonl_file_out="${val_data}"
+fi
+
+# exp output dir
+output_dir="${workspace}/exp/finetune_outputs"
+
+# Training Stage
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  echo "stage 2: KWS Training"
+
+  mkdir -p ${output_dir}
+  current_time=$(date "+%Y-%m-%d_%H-%M")
+  log_file="${output_dir}/train.log.txt.${current_time}"
+  echo "log_file: ${log_file}"
+	echo "finetune use basetrain model: ${init_param}"
+
+  export CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES
+  gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+  torchrun --nnodes 1 --nproc_per_node ${gpu_num} \
+  ../../../funasr/bin/train.py \
+  --config-path "${workspace}/conf" \
+  --config-name "${config}" \
+  ++init_param="${init_param}" \
+  ++disable_update=true \
+  ++train_data_set_list="${train_data}" \
+  ++valid_data_set_list="${val_data}" \
+  ++tokenizer_conf.token_list="${token_list}" \
+  ++tokenizer_conf.seg_dict="${lexicon_list}" \
+  ++frontend_conf.cmvn_file="${cmvn_file}" \
+  ++output_dir="${output_dir}" &> ${log_file}
+fi
+
+
+# Testing Stage
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  echo "stage 3: Inference"
+  keywords=(小云小云)
+  keywords_string=$(IFS=,; echo "${keywords[*]}")
+  echo "keywords: $keywords_string"
+
+  if [ ${inference_device} == "cuda" ]; then
+      nj=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+  else
+      inference_batch_size=1
+      CUDA_VISIBLE_DEVICES=""
+      for JOB in $(seq ${nj}); do
+          CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES"-1,"
+      done
+  fi
+
+  for dset in ${test_sets}; do
+    inference_dir="${output_dir}/inference-${inference_checkpoint}/${dset}"
+    _logdir="${inference_dir}/logdir"
+    echo "inference_dir: ${inference_dir}"
+
+    mkdir -p "${_logdir}"
+    test_data_dir="${data_dir}/${dset}"
+    key_file=${test_data_dir}/${inference_scp}
+
+    split_scps=
+    for JOB in $(seq "${nj}"); do
+        split_scps+=" ${_logdir}/keys.${JOB}.scp"
+    done
+    $FUNASR_DIR/examples/aishell/paraformer/utils/split_scp.pl "${key_file}" ${split_scps}
+
+    gpuid_list_array=(${CUDA_VISIBLE_DEVICES//,/ })
+    for JOB in $(seq ${nj}); do
+        {
+          id=$((JOB-1))
+          gpuid=${gpuid_list_array[$id]}
+
+          echo "${output_dir}"
+
+          export CUDA_VISIBLE_DEVICES=${gpuid}
+          python ../../../funasr/bin/inference.py \
+          --config-path="${output_dir}" \
+          --config-name="config.yaml" \
+          ++init_param="${output_dir}/${inference_checkpoint}" \
+          ++tokenizer_conf.token_list="${token_list}" \
+          ++tokenizer_conf.seg_dict="${lexicon_list}" \
+          ++frontend_conf.cmvn_file="${cmvn_file}" \
+          ++keywords="\"$keywords_string"\" \
+          ++input="${_logdir}/keys.${JOB}.scp" \
+          ++output_dir="${inference_dir}/${JOB}" \
+          ++device="${inference_device}" \
+          ++ncpu=1 \
+          ++disable_log=true \
+          ++batch_size="${inference_batch_size}" &> ${_logdir}/log.${JOB}.txt
+        }&
+
+    done
+    wait
+
+    for f in detect; do
+        if [ -f "${inference_dir}/${JOB}/${f}" ]; then
+          for JOB in $(seq "${nj}"); do
+              cat "${inference_dir}/${JOB}/${f}"
+          done | sort -k1 >"${inference_dir}/${f}"
+        fi
+    done
+
+    python funasr/utils/compute_det_ctc.py \
+        --keywords ${keywords_string} \
+        --test_data ${test_data_dir}/wav.scp \
+        --trans_data ${test_data_dir}/text \
+        --score_file ${inference_dir}/detect \
+        --stats_dir ${inference_dir}
+  done
+
+fi

examples/industrial_data_pretraining/fsmn_kws/funasr

@@ -0,0 +1 @@
+../../../funasr

examples/industrial_data_pretraining/fsmn_kws/infer.sh

@@ -0,0 +1,20 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method1, inference from model hub
+
+model="iic/speech_charctc_kws_phone-xiaoyun"
+
+# for more input type, please ref to readme.md
+input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/KWS/pos_testset/kws_xiaoyunxiaoyun.wav"
+
+keywords=(小云小云)
+keywords_string=$(IFS=,; echo "${keywords[*]}")
+echo "keywords: $keywords_string"
+
+python funasr/bin/inference.py \
++model=${model} \
++input=${input} \
++output_dir="./outputs/debug" \
++device="cpu" \
+++keywords="\"$keywords_string"\"

examples/industrial_data_pretraining/fsmn_kws/infer_from_local.sh

@@ -0,0 +1,41 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method2, inference from local model
+
+# for more input type, please ref to readme.md
+input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/KWS/pos_testset/kws_xiaoyunxiaoyun.wav"
+
+output_dir="./outputs/debug"
+
+workspace=`pwd`
+
+# download model
+local_path_root=${workspace}/modelscope_models
+mkdir -p ${local_path_root}
+local_path=${local_path_root}/speech_charctc_kws_phone-xiaoyun
+git clone https://www.modelscope.cn/iic/speech_charctc_kws_phone-xiaoyun.git ${local_path}
+
+device="cuda:0" # "cuda:0" for gpu0, "cuda:1" for gpu1, "cpu"
+
+config="inference_fsmn_4e_l10r2_250_128_fdim80_t2599.yaml"
+tokens="${local_path}/funasr/tokens_2599.txt"
+seg_dict="${local_path}/funasr/lexicon.txt"
+init_param="${local_path}/funasr/finetune_fsmn_4e_l10r2_250_128_fdim80_t2599_xiaoyun_xiaoyun.pt"
+cmvn_file="${local_path}/funasr/am.mvn.dim80_l2r2"
+
+keywords=(小云小云)
+keywords_string=$(IFS=,; echo "${keywords[*]}")
+echo "keywords: $keywords_string"
+
+python -m funasr.bin.inference \
+--config-path "${local_path}/funasr" \
+--config-name "${config}" \
+++init_param="${init_param}" \
+++frontend_conf.cmvn_file="${cmvn_file}" \
+++tokenizer_conf.token_list="${tokens}" \
+++tokenizer_conf.seg_dict="${seg_dict}" \
+++input="${input}" \
+++output_dir="${output_dir}" \
+++device="${device}" \
+++keywords="\"$keywords_string"\"

examples/industrial_data_pretraining/fsmn_kws/path.sh

@@ -0,0 +1,5 @@
+export FUNASR_DIR=$PWD/../../..
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PATH=$FUNASR_DIR/funasr/bin:$PATH

examples/industrial_data_pretraining/fsmn_kws_mt/conf/fsmn_4e_l10r2_250_128_fdim80_t2599_t4.yaml

@@ -0,0 +1,103 @@
+
+# network architecture
+model: FsmnKWSMT
+model_conf:
+    ctc_weight: 1.0
+
+# encoder related
+encoder: FSMNMT
+encoder_conf:
+    input_dim: 400
+    input_affine_dim: 140
+    fsmn_layers: 4
+    linear_dim: 250
+    proj_dim: 128
+    lorder: 10
+    rorder: 2
+    lstride: 1
+    rstride: 1
+    output_affine_dim: 140
+    output_dim: 2599
+    output_dim2: 4
+    use_softmax: false
+
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 80
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 5
+    lfr_n: 3
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 3
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 100
+  keep_nbest_models: 100
+  avg_nbest_model: 10
+  avg_keep_nbest_models_type: loss
+  log_interval: 50
+
+optim: adam
+optim_conf:
+   lr: 0.001
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 10000
+
+dataset: KwsMTDataset
+dataset_conf:
+    index_ds: IndexDSJsonl
+    batch_sampler: EspnetStyleBatchSampler
+    batch_type: length # example or length
+    batch_size: 64000 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
+    max_token_length: 1600 # filter samples if source_token_len+target_token_len > max_token_length,
+    buffer_size: 2048
+    shuffle: true
+    num_workers: 8
+    preprocessor_speech: SpeechPreprocessSpeedPerturb
+    preprocessor_speech_conf:
+      speed_perturb: [0.9, 1.0, 1.1]
+    dataloader: DataloaderMapStyle
+
+tokenizer:
+    - CharTokenizer
+    - CharTokenizer
+
+tokenizer_conf:
+    - unk_symbol: <unk>
+      split_with_space: true
+      token_list: null
+      seg_dict: null
+    - unk_symbol: <unk>
+      split_with_space: true
+      token_list: null
+      seg_dict: null
+
+ctc_conf:
+    dropout_rate: 0.0
+    ctc_type: builtin  # ctc_type: focalctc, builtin
+    reduce: true
+    ignore_nan_grad: true
+    extra_linear: false
+
+normalize: null

examples/industrial_data_pretraining/fsmn_kws_mt/conf/fsmn_4e_l10r2_280_200_fdim40_t2602_t4.yaml

@@ -0,0 +1,103 @@
+
+# network architecture
+model: FsmnKWSMT
+model_conf:
+    ctc_weight: 1.0
+
+# encoder related
+encoder: FSMNMT
+encoder_conf:
+    input_dim: 360
+    input_affine_dim: 280
+    fsmn_layers: 4
+    linear_dim: 280
+    proj_dim: 200
+    lorder: 10
+    rorder: 2
+    lstride: 1
+    rstride: 1
+    output_affine_dim: 400
+    output_dim: 2602
+    output_dim2: 4
+    use_softmax: false
+
+frontend: WavFrontend
+frontend_conf:
+    fs: 16000
+    window: hamming
+    n_mels: 40
+    frame_length: 25
+    frame_shift: 10
+    lfr_m: 9
+    lfr_n: 3
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 3
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 100
+  keep_nbest_models: 100
+  avg_nbest_model: 10
+  avg_keep_nbest_models_type: loss
+  log_interval: 50
+
+optim: adam
+optim_conf:
+   lr: 0.001
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 10000
+
+dataset: KwsMTDataset
+dataset_conf:
+    index_ds: IndexDSJsonl
+    batch_sampler: EspnetStyleBatchSampler
+    batch_type: length # example or length
+    batch_size: 64000 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
+    max_token_length: 1600 # filter samples if source_token_len+target_token_len > max_token_length,
+    buffer_size: 2048
+    shuffle: true
+    num_workers: 8
+    preprocessor_speech: SpeechPreprocessSpeedPerturb
+    preprocessor_speech_conf:
+      speed_perturb: [0.9, 1.0, 1.1]
+    dataloader: DataloaderMapStyle
+
+tokenizer:
+    - CharTokenizer
+    - CharTokenizer
+
+tokenizer_conf:
+    - unk_symbol: <unk>
+      split_with_space: true
+      token_list: null
+      seg_dict: null
+    - unk_symbol: <unk>
+      split_with_space: true
+      token_list: null
+      seg_dict: null
+
+ctc_conf:
+    dropout_rate: 0.0
+    ctc_type: builtin  # ctc_type: focalctc, builtin
+    reduce: true
+    ignore_nan_grad: true
+    extra_linear: false
+
+normalize: null

examples/industrial_data_pretraining/fsmn_kws_mt/convert.py

@@ -0,0 +1,137 @@
+from __future__ import print_function
+
+import argparse
+import copy
+import logging
+import os
+from shutil import copyfile
+
+import torch
+import yaml
+from typing import Union
+from funasr.models.fsmn_kws_mt.encoder import FSMNMTConvert
+from funasr.models.fsmn_kws_mt.model import FsmnKWSMTConvert
+
+
+def count_parameters(model):
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description=
+        'load and convert network to each other between kaldi/pytorch format')
+    parser.add_argument('--config', required=True, help='config file')
+    parser.add_argument(
+        '--network_file',
+        default='',
+        required=True,
+        help='input network, support kaldi.txt/pytorch.pt')
+    parser.add_argument('--model_dir', required=True, help='save model dir')
+    parser.add_argument('--model_name', required=True, help='save model name')
+    parser.add_argument('--model_name2', required=True, help='save model name')
+    parser.add_argument('--convert_to',
+                        default='kaldi',
+                        required=True,
+                        help='target network type, kaldi/pytorch')
+
+    args = parser.parse_args()
+    return args
+
+
+def convert_to_kaldi(
+    configs,
+    network_file,
+    model_dir,
+    model_name="convert.kaldi.txt",
+    model_name2="convert.kaldi2.txt"
+):
+    copyfile(network_file, os.path.join(model_dir, 'origin.torch.pt'))
+
+    model = FsmnKWSMTConvert(
+        encoder='FSMNMTConvert',
+        encoder_conf=configs['encoder_conf'],
+        ctc_conf=configs['ctc_conf'],
+    )
+    print(model)
+    num_params = count_parameters(model)
+    print('the number of model params: {}'.format(num_params))
+
+    states= torch.load(network_file, map_location='cpu')
+    model.load_state_dict(states["state_dict"])
+
+    kaldi_text = os.path.join(model_dir, model_name)
+    with open(kaldi_text, 'w', encoding='utf8') as fout:
+        nnet_desp = model.to_kaldi_net()
+        fout.write(nnet_desp)
+    fout.close()
+
+    kaldi_text2 = os.path.join(model_dir, model_name2)
+    with open(kaldi_text2, 'w', encoding='utf8') as fout:
+        nnet_desp2 = model.to_kaldi_net2()
+        fout.write(nnet_desp2)
+    fout.close()
+
+
+def convert_to_pytorch(
+    configs,
+    network_file,
+    model_dir,
+    model_name="convert.torch.pt"
+):
+    model = FsmnKWSMTConvert(
+        encoder='FSMNMTConvert',
+        encoder_conf=configs['encoder_conf'],
+        ctc_conf=configs['ctc_conf'],
+    )
+
+    num_params = count_parameters(model)
+    print('the number of model params: {}'.format(num_params))
+
+    copyfile(network_file, os.path.join(model_dir, 'origin.kaldi.txt'))
+    model.to_pytorch_net(network_file)
+
+    save_model_path = os.path.join(model_dir, model_name)
+    torch.save({"model": model.state_dict()}, save_model_path)
+
+    print('convert torch format back to kaldi')
+    kaldi_text = os.path.join(model_dir, 'convert.kaldi.txt')
+    with open(kaldi_text, 'w', encoding='utf8') as fout:
+        nnet_desp = model.to_kaldi_net()
+        fout.write(nnet_desp)
+    fout.close()
+
+    print('Done!')
+
+
+def main():
+    args = get_args()
+    logging.basicConfig(level=logging.DEBUG,
+                        format='%(asctime)s %(levelname)s %(message)s')
+    print(args)
+    with open(args.config, 'r') as fin:
+        configs = yaml.load(fin, Loader=yaml.FullLoader)
+
+    if args.convert_to == 'pytorch':
+        print('convert kaldi net to pytorch...')
+        convert_to_pytorch(
+            configs,
+            args.network_file,
+            args.model_dir,
+            args.model_name,
+            args.model_name2,
+        )
+    elif args.convert_to == 'kaldi':
+        print('convert pytorch net to kaldi...')
+        convert_to_kaldi(
+            configs,
+            args.network_file,
+            args.model_dir,
+            args.model_name
+        )
+    else:
+        print('unsupported target network type: {}'.format(args.convert_to))
+
+
+if __name__ == '__main__':
+    main()

examples/industrial_data_pretraining/fsmn_kws_mt/convert.sh

@@ -0,0 +1,36 @@
+
+workspace=`pwd`
+
+# download model
+local_path_root=${workspace}/modelscope_models
+mkdir -p ${local_path_root}
+
+local_path=${local_path_root}/speech_charctc_kws_phone-xiaoyun_mt
+if [ ! -d "$local_path" ]; then
+    git clone https://www.modelscope.cn/iic/speech_charctc_kws_phone-xiaoyun_mt.git ${local_path}
+fi
+
+export PATH=${local_path}/runtime:$PATH
+export LD_LIBRARY_PATH=${local_path}/runtime:$LD_LIBRARY_PATH
+
+# finetune config file
+config=./conf/fsmn_4e_l10r2_250_128_fdim80_t2599_t4.yaml
+
+# finetune output checkpoint
+torch_nnet=exp/finetune_outputs/model.pt.avg10
+
+out_dir=exp/finetune_outputs
+
+if [ ! -d "$out_dir" ]; then
+    mkdir -p $out_dir
+fi
+
+python convert.py --config $config \
+	--network_file $torch_nnet \
+	--model_dir $out_dir \
+	--model_name "convert.kaldi.txt" \
+	--model_name2 "convert.kaldi2.txt" \
+	--convert_to kaldi
+
+nnet-copy --binary=true ${out_dir}/convert.kaldi.txt ${out_dir}/convert.kaldi.net
+nnet-copy --binary=true ${out_dir}/convert.kaldi2.txt ${out_dir}/convert.kaldi2.net

examples/industrial_data_pretraining/fsmn_kws_mt/demo.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(
+    model="iic/speech_charctc_kws_phone-xiaoyun_mt",
+    keywords="小云小云",
+    output_dir="./outputs/debug",
+    device='cpu'
+)
+
+test_wav = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/KWS/pos_testset/kws_xiaoyunxiaoyun.wav"
+
+res = model.generate(input=test_wav, cache={},)
+print(res)

examples/industrial_data_pretraining/fsmn_kws_mt/finetune.sh

@@ -0,0 +1,184 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+#!/usr/bin/env bash
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+. ./path.sh
+workspace=`pwd`
+
+CUDA_VISIBLE_DEVICES="0,1"
+
+stage=2
+stop_stage=3
+
+inference_device="cuda" #"cpu"
+inference_checkpoint="model.pt.avg10"
+inference_scp="wav.scp"
+inference_batch_size=32
+nj=32
+test_sets="test"
+
+# model_name from model_hub, or model_dir in local path
+
+## option 1, download model automatically, unsupported currently
+model_name_or_model_dir="iic/speech_charctc_kws_phone-xiaoyun_mt"
+
+## option 2, download model by git
+local_path_root=${workspace}/modelscope_models
+model_name_or_model_dir=${local_path_root}/${model_name_or_model_dir}
+if [ ! -d $model_name_or_model_dir ]; then
+  mkdir -p ${model_name_or_model_dir}
+  git clone https://www.modelscope.cn/iic/speech_charctc_kws_phone-xiaoyun_mt.git ${model_name_or_model_dir}
+fi
+
+config=fsmn_4e_l10r2_250_128_fdim80_t2599_t4.yaml
+token_list=${model_name_or_model_dir}/funasr/tokens_2599.txt
+token_list2=${model_name_or_model_dir}/funasr/tokens_xiaoyun.txt
+lexicon_list=${model_name_or_model_dir}/funasr/lexicon.txt
+cmvn_file=${model_name_or_model_dir}/funasr/am.mvn.dim80_l2r2
+init_param="${model_name_or_model_dir}/funasr/basetrain_fsmn_4e_l10r2_250_128_fdim80_t2599.pt"
+
+
+# data prepare
+# data dir, which contains: train.json, val.json
+data_dir=../../data
+
+train_data="${data_dir}/train.jsonl"
+val_data="${data_dir}/val.jsonl"
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  echo "stage 1: Generate audio json list"
+  # generate train.jsonl and val.jsonl from wav.scp and text.txt
+  python $FUNASR_DIR/funasr/datasets/audio_datasets/scp2jsonl.py \
+  ++scp_file_list='['''${data_dir}/train_wav.scp''', '''${data_dir}/train_text.txt''']' \
+  ++data_type_list='["source", "target"]' \
+  ++jsonl_file_out="${train_data}"
+
+  python $FUNASR_DIR/funasr/datasets/audio_datasets/scp2jsonl.py \
+  ++scp_file_list='['''${data_dir}/val_wav.scp''', '''${data_dir}/val_text.txt''']' \
+  ++data_type_list='["source", "target"]' \
+  ++jsonl_file_out="${val_data}"
+fi
+
+# exp output dir
+output_dir="${workspace}/exp/finetune_outputs"
+
+
+# Training Stage
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  echo "stage 2: KWS Training"
+
+  mkdir -p ${output_dir}
+  current_time=$(date "+%Y-%m-%d_%H-%M")
+  log_file="${output_dir}/train.log.txt.${current_time}"
+  echo "log_file: ${log_file}"
+  echo "finetune use basetrain model: ${init_param}"
+
+  export CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES
+  gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+  torchrun --nnodes 1 --nproc_per_node ${gpu_num} \
+  ../../../funasr/bin/train.py \
+  --config-path "${workspace}/conf" \
+  --config-name "${config}" \
+  ++init_param="${init_param}" \
+  ++token_lists='['''${token_list}''', '''${token_list2}''']' \
+  ++seg_dicts='['''${lexicon_list}''', '''${lexicon_list}''']' \
+  ++disable_update=true \
+  ++train_data_set_list="${train_data}" \
+  ++valid_data_set_list="${val_data}" \
+  ++frontend_conf.cmvn_file="${cmvn_file}" \
+  ++output_dir="${output_dir}" &> ${log_file}
+fi
+
+
+# Testing Stage
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+  echo "stage 3: Inference"
+  keywords=(小云小云)
+  keywords_string=$(IFS=,; echo "${keywords[*]}")
+  echo "keywords: $keywords_string"
+
+  if [ ${inference_device} == "cuda" ]; then
+      nj=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+  else
+      inference_batch_size=1
+      CUDA_VISIBLE_DEVICES=""
+      for JOB in $(seq ${nj}); do
+          CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES"-1,"
+      done
+  fi
+
+  for dset in ${test_sets}; do
+    inference_dir="${output_dir}/inference-${inference_checkpoint}/${dset}"
+    _logdir="${inference_dir}/logdir"
+    echo "inference_dir: ${inference_dir}"
+
+    mkdir -p "${_logdir}"
+    test_data_dir="${data_dir}/${dset}"
+    key_file=${test_data_dir}/${inference_scp}
+
+    split_scps=
+    for JOB in $(seq "${nj}"); do
+        split_scps+=" ${_logdir}/keys.${JOB}.scp"
+    done
+    $FUNASR_DIR/examples/aishell/paraformer/utils/split_scp.pl "${key_file}" ${split_scps}
+
+    gpuid_list_array=(${CUDA_VISIBLE_DEVICES//,/ })
+    for JOB in $(seq ${nj}); do
+        {
+          id=$((JOB-1))
+          gpuid=${gpuid_list_array[$id]}
+
+          echo "${output_dir}"
+
+          export CUDA_VISIBLE_DEVICES=${gpuid}
+          python ../../../funasr/bin/inference.py \
+          --config-path="${output_dir}" \
+          --config-name="config.yaml" \
+          ++init_param="${output_dir}/${inference_checkpoint}" \
+          ++token_lists='['''${token_list}''', '''${token_list2}''']' \
+          ++seg_dicts='['''${lexicon_list}''', '''${lexicon_list}''']' \
+          ++frontend_conf.cmvn_file="${cmvn_file}" \
+          ++keywords="\"$keywords_string"\" \
+          ++input="${_logdir}/keys.${JOB}.scp" \
+          ++output_dir="${inference_dir}/${JOB}" \
+          ++device="${inference_device}" \
+          ++ncpu=1 \
+          ++disable_log=true \
+          ++batch_size="${inference_batch_size}" &> ${_logdir}/log.${JOB}.txt
+        }&
+
+    done
+    wait
+
+    for f in detect detect2; do
+        if [ -f "${inference_dir}/${JOB}/${f}" ]; then
+          for JOB in $(seq "${nj}"); do
+              cat "${inference_dir}/${JOB}/${f}"
+          done | sort -k1 >"${inference_dir}/${f}"
+        fi
+    done
+
+    mkdir -p ${inference_dir}/task1
+    python funasr/utils/compute_det_ctc.py \
+        --keywords ${keywords_string} \
+        --test_data ${test_data_dir}/wav.scp \
+        --trans_data ${test_data_dir}/text \
+        --score_file ${inference_dir}/detect \
+        --stats_dir ${inference_dir}/task1
+
+    mkdir -p ${inference_dir}/task2
+    python funasr/utils/compute_det_ctc.py \
+        --keywords ${keywords_string} \
+        --test_data ${test_data_dir}/wav.scp \
+        --trans_data ${test_data_dir}/text \
+        --score_file ${inference_dir}/detect2 \
+        --stats_dir ${inference_dir}/task2
+  done
+
+fi

examples/industrial_data_pretraining/fsmn_kws_mt/funasr

@@ -0,0 +1 @@
+../../../funasr

examples/industrial_data_pretraining/fsmn_kws_mt/infer.sh

@@ -0,0 +1,20 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method1, inference from model hub
+
+model="iic/speech_charctc_kws_phone-xiaoyun_mt"
+
+# for more input type, please ref to readme.md
+input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/KWS/pos_testset/kws_xiaoyunxiaoyun.wav"
+
+keywords=(小云小云)
+keywords_string=$(IFS=,; echo "${keywords[*]}")
+echo "keywords: $keywords_string"
+
+python funasr/bin/inference.py \
++model=${model} \
++input=${input} \
++output_dir="./outputs/debug" \
++device="cpu" \
+++keywords="\"$keywords_string"\"

examples/industrial_data_pretraining/fsmn_kws_mt/infer_from_local.sh

@@ -0,0 +1,42 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# method2, inference from local model
+
+# for more input type, please ref to readme.md
+input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/KWS/pos_testset/kws_xiaoyunxiaoyun.wav"
+
+output_dir="./outputs/debug"
+
+workspace=`pwd`
+
+# download model
+local_path_root=${workspace}/modelscope_models
+mkdir -p ${local_path_root}
+local_path=${local_path_root}/speech_charctc_kws_phone-xiaoyun_mt
+git clone https://www.modelscope.cn/iic/speech_charctc_kws_phone-xiaoyun_mt.git ${local_path}
+
+device="cuda:0" # "cuda:0" for gpu0, "cuda:1" for gpu1, "cpu"
+
+config="inference_fsmn_4e_l10r2_280_200_fdim40_t2602_t4.yaml"
+tokens="${local_path}/funasr/tokens_2602.txt"
+tokens2="${local_path}/funasr/tokens_xiaoyun.txt"
+seg_dict="${local_path}/funasr/lexicon.txt"
+init_param="${local_path}/funasr/finetune_fsmn_4e_l10r2_280_200_fdim40_t2602_t4_xiaoyun_xiaoyun.pt"
+cmvn_file="${local_path}/funasr/am.mvn.dim40_l4r4"
+
+keywords=(小云小云)
+keywords_string=$(IFS=,; echo "${keywords[*]}")
+echo "keywords: $keywords_string"
+
+python -m funasr.bin.inference \
+--config-path "${local_path}/funasr" \
+--config-name "${config}" \
+++init_param="${init_param}" \
+++frontend_conf.cmvn_file="${cmvn_file}" \
+++token_lists='['''${tokens}''', '''${tokens2}''']' \
+++seg_dicts='['''${seg_dict}''', '''${seg_dict}''']' \
+++input="${input}" \
+++output_dir="${output_dir}" \
+++device="${device}" \
+++keywords="\"$keywords_string"\"

examples/industrial_data_pretraining/fsmn_kws_mt/path.sh

@@ -0,0 +1,5 @@
+export FUNASR_DIR=$PWD/../../..
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
+export PATH=$FUNASR_DIR/funasr/bin:$PATH

examples/industrial_data_pretraining/fsmn_vad_streaming/demo.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+wav_file = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav"
+
+model = AutoModel(model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch")
+
+res = model.generate(input=wav_file)
+print(res)
+
+# [[beg1, end1], [beg2, end2], .., [begN, endN]]
+# beg/end: ms
+
+
+import soundfile
+import os
+
+wav_file = os.path.join(model.model_path, "example/vad_example.wav")
+speech, sample_rate = soundfile.read(wav_file)
+
+chunk_size = 200  # ms
+chunk_stride = int(chunk_size * sample_rate / 1000)
+
+cache = {}
+
+total_chunk_num = int(len((speech) - 1) / chunk_stride + 1)
+for i in range(total_chunk_num):
+    speech_chunk = speech[i * chunk_stride : (i + 1) * chunk_stride]
+    is_final = i == total_chunk_num - 1
+    res = model.generate(
+        input=speech_chunk,
+        cache=cache,
+        is_final=is_final,
+        chunk_size=chunk_size,
+        disable_pbar=True,
+    )
+    # print(res)
+    if len(res[0]["value"]):
+        print(res)
+
+
+# 1. [[beg1, end1], [beg2, end2], .., [begN, endN]]; [[beg, end]]; [[beg1, end1], [beg2, end2]]
+# 2. [[beg, -1]]
+# 3. [[-1, end]]
+# beg/end: ms

examples/industrial_data_pretraining/fsmn_vad_streaming/demo.sh

@@ -0,0 +1,10 @@
+
+
+model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
+
+
+python funasr/bin/inference.py \
++model=${model} \
++input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav" \
++output_dir="./outputs/debug" \
++device="cpu" \

examples/industrial_data_pretraining/fsmn_vad_streaming/export.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+
+# method1, inference from model hub
+
+from funasr import AutoModel
+
+model = AutoModel(model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch")
+
+res = model.export(type="onnx", quantize=False)
+print(res)
+
+# method2, inference from local path
+
+from funasr import AutoModel
+
+model = AutoModel(
+    model="/Users/zhifu/.cache/modelscope/hub/iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
+)
+
+res = model.export(type="onnx", quantize=False)
+print(res)

examples/industrial_data_pretraining/fsmn_vad_streaming/export.sh

@@ -0,0 +1,24 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+
+# method1, inference from model hub
+export HYDRA_FULL_ERROR=1
+
+
+model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
+
+
+python -m funasr.bin.export \
+++model=${model} \
+++model_revision=${model_revision} \
+++type="onnx" \
+++quantize=false
+
+# method2, inference from local path
+model="/Users/zhifu/.cache/modelscope/hub/iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"
+
+python -m funasr.bin.export \
+++model=${model} \
+++type="onnx" \
+++quantize=false

examples/industrial_data_pretraining/fun_asr_nano/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

examples/industrial_data_pretraining/fun_asr_nano/README.md

@@ -0,0 +1,216 @@
+# Fun-ASR
+
+「[简体中文](README_zh.md)」|「English」
+
+Fun-ASR is an end-to-end speech recognition large model launched by Tongyi Lab. It is trained on tens of millions of hours of real speech data, possessing powerful contextual understanding capabilities and industry adaptability. It supports low-latency real-time transcription and covers 31 languages. It excels in vertical domains such as education and finance, accurately recognizing professional terminology and industry expressions, effectively addressing challenges like "hallucination" generation and language confusion, achieving "clear hearing, understanding meaning, and accurate writing."
+
+<div align="center">
+<img src="images/funasr-v2.png">
+</div>
+
+<div align="center">
+<h4>
+<a href="https://funaudiollm.github.io/funasr"> Homepage </a>
+｜<a href="#core-features"> Core Features </a>
+｜<a href="#performance-evaluation"> Performance Evaluation </a>
+｜<a href="#environment-setup"> Environment Setup </a>
+｜<a href="#usage-tutorial"> Usage Tutorial </a>
+
+</h4>
+
+Model Repository: [modelscope](https://www.modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512), [huggingface](https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512)
+
+Online Experience:
+[ModelScope Community Space](https://modelscope.cn/studios/FunAudioLLM/Fun-ASR-Nano), [huggingface space](https://huggingface.co/spaces/FunAudioLLM/Fun-ASR-Nano)
+
+</div>
+
+|                                                                           Model Name                                                                            |                                                                                                                                                                                                       Task Details                                                                                                                                                                                                       |         Training Data          | Parameters |
+| :-------------------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------: | :--------: |
+|       Fun-ASR-Nano <br> ([⭐](https://www.modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512) [🤗](https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512))       | Speech recognition supports Chinese, English, and Japanese. Chinese includes support for 7 dialects (Wu, Cantonese, Min, Hakka, Gan, Xiang, Jin) and 26 regional accents (Henan, Shanxi, Hubei, Sichuan, Chongqing, Yunnan, Guizhou, Guangdong, Guangxi and more than 20 other regions). English and Japanese cover multiple regional accents. Additional features include lyric recognition and rap speech recognition. |   Tens of millions of hours    |    800M    |
+| Fun-ASR-MLT-Nano <br> ([⭐](https://www.modelscope.cn/models/FunAudioLLM/Fun-ASR-MLT-Nano-2512) [🤗](https://huggingface.co/FunAudioLLM/Fun-ASR-MLT-Nano-2512)) |                                    Speech recognition supports Chinese, English, Cantonese, Japanese, Korean, Vietnamese, Indonesian, Thai, Malay, Filipino, Arabic, Hindi, Bulgarian, Croatian, Czech, Danish, Dutch, Estonian, Finnish, Greek, Hungarian, Irish, Latvian, Lithuanian, Maltese, Polish, Portuguese, Romanian, Slovak, Slovenian, Swedish, and 31 languages in total.                                    | Hundreds of thousands of hours |    800M    |
+
+<a name="What's News"></a>
+
+# What's New 🔥
+
+- 2025/12: [Fun-ASR-Nano-2512](https://modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512) is an end-to-end speech recognition large model trained on tens of millions of hours real speech data. It supports low-latency real-time transcription and covers 31 languages.
+- 2024/7: [FunASR](https://github.com/modelscope/FunASR) is a fundamental speech recognition toolkit that offers a variety of features, including speech recognition (ASR), Voice Activity Detection (VAD), Punctuation Restoration, Language Models, Speaker Verification, Speaker Diarization and multi-talker ASR.
+
+# Core Features 🎯
+
+**Fun-ASR** focuses on high-precision speech recognition, multi-language support, and industry customization capabilities
+
+- **Far-field High-noise Recognition:** Deeply optimized for far-distance sound pickup and high-noise scenarios (such as conference rooms, in-vehicle environments, industrial sites, etc.), improving recognition accuracy to **93%**.
+- **Chinese Dialects and Regional Accents:**
+  - Supports **7 major dialects**: Wu, Cantonese, Min, Hakka, Gan, Xiang, Jin
+  - Covers **26 regional accents**: including Henan, Shaanxi, Hubei, Sichuan, Chongqing, Yunnan, Guizhou, Guangdong, Guangxi and more than 20 other regions
+- **Multi-language Free Speech:** Supports recognition of **31 languages**, with focused optimization on East and Southeast Asian languages, supporting free language switching and mixed recognition.
+- **Music Background Lyric Recognition:** Enhanced speech recognition performance under music background interference, supporting accurate recognition of lyric content in songs.
+
+# Environment Setup 🐍
+
+```shell
+git clone https://github.com/FunAudioLLM/Fun-ASR.git
+cd Fun-ASR
+pip install -r requirements.txt
+```
+
+<a name="usage-tutorial"></a>
+
+# TODO
+
+- [x] Support returning timestamps
+- [ ] Support speaker diarization
+- [x] Support model training
+
+# Usage 🛠️
+
+## Inference
+
+### Using funasr for inference
+
+```python
+from funasr import AutoModel
+
+
+def main():
+    model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        remote_code="./model.py",
+        device="cuda:0",
+        # hub：download models from ms (for ModelScope) or hf (for Hugging Face).
+        hub="hf"
+    )
+
+    wav_path = f"{model.model_path}/example/zh.mp3"
+    res = model.generate(
+        input=[wav_path],
+        cache={},
+        batch_size=1,
+        hotwords=["开放时间"],
+        # 中文、英文、日文 for Fun-ASR-Nano-2512
+        # 中文、英文、粤语、日文、韩文、越南语、印尼语、泰语、马来语、菲律宾语、阿拉伯语、
+        # 印地语、保加利亚语、克罗地亚语、捷克语、丹麦语、荷兰语、爱沙尼亚语、芬兰语、希腊语、
+        # 匈牙利语、爱尔兰语、拉脱维亚语、立陶宛语、马耳他语、波兰语、葡萄牙语、罗马尼亚语、
+        # 斯洛伐克语、斯洛文尼亚语、瑞典语 for Fun-ASR-MLT-Nano-2512
+        language="中文",
+        itn=True, # or False
+    )
+    text = res[0]["text"]
+    print(text)
+
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        vad_model="fsmn-vad",
+        vad_kwargs={"max_single_segment_time": 30000},
+        remote_code="./model.py",
+        device="cuda:0",
+    )
+    res = model.generate(input=[wav_path], cache={}, batch_size=1)
+    text = res[0]["text"]
+    print(text)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+### Direct Inference
+
+```python
+from model import FunASRNano
+
+
+def main():
+    model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+    m, kwargs = FunASRNano.from_pretrained(model=model_dir, device="cuda:0")
+    m.eval()
+
+    wav_path = f"{kwargs['model_path']}/example/zh.mp3"
+    res = m.inference(data_in=[wav_path], **kwargs)
+    text = res[0][0]["text"]
+    print(text)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+<details><summary> Parameter Description (click to expand) </summary>
+
+- `model_dir`: Model name or local disk model path.
+- `trust_remote_code`: Whether to trust remote code for loading custom model implementations.
+- `remote_code`: Specify the location of specific model code (e.g., `model.py` in the current directory), supporting both absolute and relative paths.
+- `device`: Specify the device to use, such as "cuda:0" or "cpu".
+
+</details>
+
+# Finetune
+
+Please refer to [docs/finetune.md](docs/finetune.md)
+
+# Performance 📝
+
+We evaluated Fun-ASR against other state-of-the-art models on open-source benchmarks, Chinese dialect datasets, and industry-specific test sets. The results demonstrate that Fun-ASR achieves superior performance across various scenarios.
+
+### 1. Open-Source Dataset Performance (WER %)
+
+| Test set            | GLM-ASR-nano | GLM-ASR-nano\* | Whisper-large-v3 | Seed-ASR | Seed-ASR\* | Kimi-Audio | Step-Audio2 | FireRed-ASR | Fun-ASR-nano | Fun-ASR |
+| :------------------ | :----------: | :------------: | :--------------: | :------: | :--------: | :--------: | :---------: | :---------: | :----------: | :-----: |
+| **Model Size**      |     1.5B     |      1.5B      |       1.6B       |    -     |     -      |     -      |      -      |    1.1B     |     0.8B     |  7.7B   |
+| **OpenSource**      |      ✅      |       ✅       |        ✅        |    ❌    |     ❌     |     ✅     |     ✅      |     ✅      |      ✅      |   ❌    |
+| AIShell1            |     1.81     |      2.17      |       4.72       |   0.68   |    1.63    |    0.71    |    0.63     |    0.54     |     1.80     |  1.22   |
+| AIShell2            |      -       |      3.47      |       4.68       |   2.27   |    2.76    |    2.86    |    2.10     |    2.58     |     2.75     |  2.39   |
+| Fleurs-zh           |      -       |      3.65      |       5.18       |   3.43   |    3.23    |    3.11    |    2.68     |    4.81     |     2.56     |  2.53   |
+| Fleurs-en           |     5.78     |      6.95      |       6.23       |   9.39   |    9.39    |    6.99    |    3.03     |    10.79    |     5.96     |  4.74   |
+| Librispeech-clean   |     2.00     |      2.17      |       1.86       |   1.58   |    2.8     |    1.32    |    1.17     |    1.84     |     1.76     |  1.51   |
+| Librispeech-other   |     4.19     |      4.43      |       3.43       |   2.84   |    5.69    |    2.63    |    2.42     |    4.52     |     4.33     |  3.03   |
+| WenetSpeech Meeting |     6.73     |      8.21      |      18.39       |   5.69   |    7.07    |    6.24    |    4.75     |    4.95     |     6.60     |  6.17   |
+| WenetSpeech Net     |      -       |      6.33      |      11.89       |   4.66   |    4.84    |    6.45    |    4.67     |    4.94     |     6.01     |  5.46   |
+
+> _Note: Seed-ASR\* results are evaluated using the official API on volcengine; GLM-ASR-nano\* results are evaluated using the open-source checkpoint._
+
+### 2. Industry Dataset Performance (WER %)
+
+| Test set           | GLM-ASR-Nano | Whisper-large-v3 | Seed-ASR  | FireRed-ASR | Kimi-Audio | Paraformer v2 | Fun-ASR-nano |  Fun-ASR  |
+| :----------------- | :----------: | :--------------: | :-------: | :---------: | :--------: | :-----------: | :----------: | :-------: |
+| **Model Size**     |     1.5B     |       1.6B       |     -     |    1.1B     |     8B     |     0.2B      |     0.8B     |   7.7B    |
+| **OpenSource**     |      ✅      |        ✅        |    ❌     |     ✅      |     ✅     |      ✅       |      ✅      |    ❌     |
+| Nearfield          |    16.95     |      16.58       |   7.20    |    10.10    |    9.02    |     8.11      |     7.79     |   6.31    |
+| Farfield           |     9.44     |      22.21       |   4.59    |    7.49     |   10.95    |     9.55      |     5.79     |   4.34    |
+| Complex Background |    23.79     |      32.57       |   12.90   |    15.56    |   15.56    |     15.19     |    14.59     |   11.45   |
+| English General    |    16.47     |      18.56       |   15.65   |    21.62    |   18.12    |     19.48     |    15.28     |   13.73   |
+| Opensource         |     4.67     |       7.05       |   3.83    |    5.31     |    3.79    |     6.23      |     4.22     |   3.38    |
+| Dialect            |    54.21     |      66.14       |   29.45   |    52.82    |   71.94    |     41.16     |    28.18     |   15.21   |
+| Accent             |    19.78     |      36.03       |   10.23   |    14.05    |   27.20    |     17.80     |    12.90     |   10.31   |
+| Lyrics             |    46.56     |      54.82       |   30.26   |    42.87    |   65.18    |     50.14     |    30.85     |   21.00   |
+| Hiphop             |    43.32     |      46.56       |   29.46   |    33.88    |   57.25    |     43.79     |    30.87     |   28.58   |
+| **Average**        |  **26.13**   |    **33.39**     | **15.95** |  **22.63**  | **31.00**  |   **23.49**   |  **16.72**   | **12.70** |
+
+<div align="center">
+<img src="images/compare_en.png" width="800" />
+</div>
+
+## Remarkable Third-Party Work
+
+- **Built-in vLLM Inference Engine**: 2-3x faster decoding with AutoModelVLLM, streaming WebSocket service, tensor parallel. [Guide →](../../../docs/vllm_guide.md)
+- **Streaming WebSocket Service**: Real-time ASR with VAD + Speaker Diarization + Hotwords. [Quick Start →](docs/realtime_demo.md)
+- vLLM (GPU) Deployment Best Practices: An accelerated implementation of Fun-ASR using vLLM. [Repository](https://github.com/yuekaizhang/Fun-ASR-vllm)
+
+## Citations
+
+```bibtex
+@misc{an2025funasrtechnicalreport,
+      title={Fun-ASR Technical Report},
+      author={Keyu An and Yanni Chen and Zhigao Chen and Chong Deng and Zhihao Du and Changfeng Gao and Zhifu Gao and Bo Gong and Xiangang Li and Yabin Li and Ying Liu and Xiang Lv and Yunjie Ji and Yiheng Jiang and Bin Ma and Haoneng Luo and Chongjia Ni and Zexu Pan and Yiping Peng and Zhendong Peng and Peiyao Wang and Hao Wang and Haoxu Wang and Wen Wang and Wupeng Wang and Yuzhong Wu and Biao Tian and Zhentao Tan and Nan Yang and Bin Yuan and Jieping Ye and Jixing Yu and Qinglin Zhang and Kun Zou and Han Zhao and Shengkui Zhao and Jingren Zhou and Yanqiao Zhu},
+      year={2025},
+      eprint={2509.12508},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2509.12508},
+}
+```

examples/industrial_data_pretraining/fun_asr_nano/README_zh.md

@@ -0,0 +1,215 @@
+# Fun-ASR
+
+「简体中文」|「[English](README.md)」
+
+Fun-ASR 是通义实验室推出的端到端语音识别大模型，是基于数千万小时真实语音数据训练而成，具备强大的上下文理解能力与行业适应性，支持低延迟实时听写，并且覆盖 31 个语种。在教育、金融等垂直领域表现出色，能准确识别专业术语与行业表达，有效应对"幻觉"生成和语种混淆等挑战，实现"听得清、懂其意、写得准"。
+
+<div align="center">
+<img src="images/funasr-v2.png">
+</div>
+
+<div align="center">
+<h4>
+<a href="https://funaudiollm.github.io/funasr"> Homepage </a>
+｜<a href="#核心特性"> 核心特性 </a>
+｜<a href="#性能评测"> 性能评测 </a>
+｜<a href="#环境安装"> 环境安装 </a>
+｜<a href="#用法教程"> 用法教程 </a>
+
+</h4>
+
+模型仓库：[modelscope](https://www.modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512)，[huggingface](https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512)
+
+在线体验：
+[魔搭社区创空间](https://modelscope.cn/studios/FunAudioLLM/Fun-ASR-Nano)，[huggingface space](https://huggingface.co/spaces/FunAudioLLM/Fun-ASR-Nano)
+
+</div>
+
+|                                                                              模型                                                                               |                                                                                                                                                    介绍                                                                                                                                                    |  训练数据  | 参数 |
+| :-------------------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :--------: | :--: |
+|       Fun-ASR-Nano <br> ([⭐](https://www.modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512) [🤗](https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512))       |         支持中文、英文、日文。中文包含 7 种方言（吴语、粤语、闽语、客家话、赣语、湘语、晋语）及 26 种地域口音支持（河南、陕西、湖北、四川、重庆、云南、贵州、广东、广西、河北、天津、山东、安徽、南京、江苏、杭州、甘肃、宁夏）。英文、日文涵盖多种地域口音。额外功能包括歌词识别与说唱语音识别。          | 数千万小时 | 8 亿 |
+| Fun-ASR-MLT-Nano <br> ([⭐](https://www.modelscope.cn/models/FunAudioLLM/Fun-ASR-MLT-Nano-2512) [🤗](https://huggingface.co/FunAudioLLM/Fun-ASR-MLT-Nano-2512)) | 支持中文、英文、粤语、日文、韩文、越南语、印尼语、泰语、马来语、菲律宾语、阿拉伯语、印地语、保加利亚语、克罗地亚语、捷克语、丹麦语、荷兰语、爱沙尼亚语、芬兰语、希腊语、匈牙利语、爱尔兰语、拉脱维亚语、立陶宛语、马耳他语、波兰语、葡萄牙语、罗马尼亚语、斯洛伐克语、斯洛文尼亚语、瑞典语，共 31 种语言。 | 数十万小时 | 8 亿 |
+
+<a name="最新动态"></a>
+
+# 最新动态 🔥
+
+- 2025/12: [Fun-ASR-Nano-2512](https://modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512) 是一款基于数千万小时真实语音数据训练的端到端语音识别大模型。它支持低延迟实时转写，并涵盖 31 种语言识别功能。
+- 2024/7: [FunASR](https://github.com/modelscope/FunASR) 是一款功能全面的语音识别基础工具包，集成了多项核心功能，包括自动语音识别（ASR）、语音活动检测（VAD）、标点恢复、语言模型、说话人验证、说话人日志记录以及多说话人语音识别。
+
+# 核心特性 🎯
+
+**Fun-ASR** 专注于高精度语音识别、多语言支持和行业定制化能力
+
+- **远场高噪声识别：** 针对远距离拾音及高噪声场景（如会议室、车载环境、工业现场等）进行深度优化，识别准确率提升至 **93%**。
+- **中文方言与地方口音：**
+  - 支持 **7 大方言**：吴语、粤语、闽语、客家话、赣语、湘语、晋语
+  - 覆盖 **26 个地区口音**：包括河南、陕西、湖北、四川、重庆、云南、贵州、广东、广西等 20 多个地区
+- **多语言自由说：** 支持 **31 种语言**识别，重点优化东亚与东南亚语种，支持语种自由切换和混合识别。
+- **音乐背景歌词识别：** 强化在音乐背景干扰下的语音识别性能，支持对歌曲中歌词内容的精准识别。
+
+# 环境安装 🐍
+
+```shell
+git clone https://github.com/FunAudioLLM/Fun-ASR.git
+cd Fun-ASR
+pip install -r requirements.txt
+```
+
+<a name="用法教程"></a>
+
+# TODO
+
+- [x] 支持返回时间戳
+- [ ] 支持区分说话人识别
+- [x] 支持模型训练
+
+# 用法 🛠️
+
+## 推理
+
+### 使用 funasr 推理
+
+```python
+from funasr import AutoModel
+
+
+def main():
+    model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        remote_code="./model.py",
+        device="cuda:0",
+        hub="ms"
+    )
+
+    wav_path = f"{model.model_path}/example/zh.mp3"
+    res = model.generate(
+        input=[wav_path],
+        cache={},
+        batch_size=1,
+        hotwords=["开放时间"],
+        # 中文、英文、日文 for Fun-ASR-Nano-2512
+        # 中文、英文、粤语、日文、韩文、越南语、印尼语、泰语、马来语、菲律宾语、阿拉伯语、
+        # 印地语、保加利亚语、克罗地亚语、捷克语、丹麦语、荷兰语、爱沙尼亚语、芬兰语、希腊语、
+        # 匈牙利语、爱尔兰语、拉脱维亚语、立陶宛语、马耳他语、波兰语、葡萄牙语、罗马尼亚语、
+        # 斯洛伐克语、斯洛文尼亚语、瑞典语 for Fun-ASR-MLT-Nano-2512
+        language="中文",
+        itn=True, # or False
+    )
+    text = res[0]["text"]
+    print(text)
+
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        vad_model="fsmn-vad",
+        vad_kwargs={"max_single_segment_time": 30000},
+        remote_code="./model.py",
+        device="cuda:0",
+    )
+    res = model.generate(input=[wav_path], cache={}, batch_size=1)
+    text = res[0]["text"]
+    print(text)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+### 直接推理
+
+```python
+from model import FunASRNano
+
+
+def main():
+    model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+    m, kwargs = FunASRNano.from_pretrained(model=model_dir, device="cuda:0")
+    m.eval()
+
+    wav_path = f"{kwargs['model_path']}/example/zh.mp3"
+    res = m.inference(data_in=[wav_path], **kwargs)
+    text = res[0][0]["text"]
+    print(text)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+<details><summary> 参数说明（点击展开）</summary>
+
+- `model_dir`：模型名称，或本地磁盘中的模型路径。
+- `trust_remote_code`：是否信任远程代码，用于加载自定义模型实现。
+- `remote_code`：指定模型具体代码的位置（例如，当前目录下的 `model.py`），支持绝对路径与相对路径。
+- `device`：指定使用的设备，如 "cuda:0" 或 "cpu"。
+
+</details>
+
+# 微调
+
+详情请参考 [docs/finetune_zh.md](docs/finetune.md)
+
+# 性能评测 📝
+
+我们在开源基准数据集、中文方言测试集和工业测试集上，比较了 Fun-ASR 与其他模型的多语言语音识别性能。Fun-ASR 模型均具有明显的效果优势。
+
+### 1. 开源数据集性能 (WER %)
+
+| Test set            | GLM-ASR-nano | GLM-ASR-nano\* | Whisper-large-v3 | Seed-ASR | Seed-ASR\* | Kimi-Audio | Step-Audio2 | FireRed-ASR | Fun-ASR-nano | Fun-ASR |
+| :------------------ | :----------: | :------------: | :--------------: | :------: | :--------: | :--------: | :---------: | :---------: | :----------: | :-----: |
+| **Model Size**      |     1.5B     |      1.5B      |       1.6B       |    -     |     -      |     -      |      -      |    1.1B     |     0.8B     |  7.7B   |
+| **OpenSource**      |      ✅      |       ✅       |        ✅        |    ❌    |     ❌     |     ✅     |     ✅      |     ✅      |      ✅      |   ❌    |
+| AIShell1            |     1.81     |      2.17      |       4.72       |   0.68   |    1.63    |    0.71    |    0.63     |    0.54     |     1.80     |  1.22   |
+| AIShell2            |      -       |      3.47      |       4.68       |   2.27   |    2.76    |    2.86    |    2.10     |    2.58     |     2.75     |  2.39   |
+| Fleurs-zh           |      -       |      3.65      |       5.18       |   3.43   |    3.23    |    3.11    |    2.68     |    4.81     |     2.56     |  2.53   |
+| Fleurs-en           |     5.78     |      6.95      |       6.23       |   9.39   |    9.39    |    6.99    |    3.03     |    10.79    |     5.96     |  4.74   |
+| Librispeech-clean   |     2.00     |      2.17      |       1.86       |   1.58   |    2.8     |    1.32    |    1.17     |    1.84     |     1.76     |  1.51   |
+| Librispeech-other   |     4.19     |      4.43      |       3.43       |   2.84   |    5.69    |    2.63    |    2.42     |    4.52     |     4.33     |  3.03   |
+| WenetSpeech Meeting |     6.73     |      8.21      |      18.39       |   5.69   |    7.07    |    6.24    |    4.75     |    4.95     |     6.60     |  6.17   |
+| WenetSpeech Net     |      -       |      6.33      |      11.89       |   4.66   |    4.84    |    6.45    |    4.67     |    4.94     |     6.01     |  5.46   |
+
+> _注：Seed-ASR\* 结果使用 volcengine 上的官方 API 评估；GLM-ASR-nano\* 结果使用开源 checkpoint 评估。_
+
+### 2. 工业数据集性能 (WER %)
+
+| Test set           | GLM-ASR-Nano | Whisper-large-v3 | Seed-ASR  | FireRed-ASR | Kimi-Audio | Paraformer v2 | Fun-ASR-nano |  Fun-ASR  |
+| :----------------- | :----------: | :--------------: | :-------: | :---------: | :--------: | :-----------: | :----------: | :-------: |
+| **Model Size**     |     1.5B     |       1.6B       |     -     |    1.1B     |     8B     |     0.2B      |     0.8B     |   7.7B    |
+| **OpenSource**     |      ✅      |        ✅        |    ❌     |     ✅      |     ✅     |      ✅       |      ✅      |    ❌     |
+| Nearfield          |    16.95     |      16.58       |   7.20    |    10.10    |    9.02    |     8.11      |     7.79     |   6.31    |
+| Farfield           |     9.44     |      22.21       |   4.59    |    7.49     |   10.95    |     9.55      |     5.79     |   4.34    |
+| Complex Background |    23.79     |      32.57       |   12.90   |    15.56    |   15.56    |     15.19     |    14.59     |   11.45   |
+| English General    |    16.47     |      18.56       |   15.65   |    21.62    |   18.12    |     19.48     |    15.28     |   13.73   |
+| Opensource         |     4.67     |       7.05       |   3.83    |    5.31     |    3.79    |     6.23      |     4.22     |   3.38    |
+| Dialect            |    54.21     |      66.14       |   29.45   |    52.82    |   71.94    |     41.16     |    28.18     |   15.21   |
+| Accent             |    19.78     |      36.03       |   10.23   |    14.05    |   27.20    |     17.80     |    12.90     |   10.31   |
+| Lyrics             |    46.56     |      54.82       |   30.26   |    42.87    |   65.18    |     50.14     |    30.85     |   21.00   |
+| Hiphop             |    43.32     |      46.56       |   29.46   |    33.88    |   57.25    |     43.79     |    30.87     |   28.58   |
+| **Average**        |  **26.13**   |    **33.39**     | **15.95** |  **22.63**  | **31.00**  |   **23.49**   |  **16.72**   | **12.70** |
+
+<div align="center">
+<img src="images/compare_zh.png" width="800" />
+</div>
+
+## 优秀三方工作
+
+- **内置 vLLM 推理引擎**：AutoModelVLLM 解码加速 2-3 倍，流式 WebSocket 服务，tensor parallel 多卡并行。[文档 →](../../../docs/vllm_guide.md)
+- **流式 WebSocket 服务**：实时语音识别 + VAD 分句 + 说话人分离 + 热词定制。[快速上手 →](docs/realtime_demo.md)
+- vLLM (GPU) 最佳部署实践: 使用 vLLM 实现对 Fun-ASR 的加速. [Repository](https://github.com/yuekaizhang/Fun-ASR-vllm)
+
+## Citations
+
+```bibtex
+@misc{an2025funasrtechnicalreport,
+      title={Fun-ASR Technical Report},
+      author={Keyu An and Yanni Chen and Zhigao Chen and Chong Deng and Zhihao Du and Changfeng Gao and Zhifu Gao and Bo Gong and Xiangang Li and Yabin Li and Ying Liu and Xiang Lv and Yunjie Ji and Yiheng Jiang and Bin Ma and Haoneng Luo and Chongjia Ni and Zexu Pan and Yiping Peng and Zhendong Peng and Peiyao Wang and Hao Wang and Haoxu Wang and Wen Wang and Wupeng Wang and Yuzhong Wu and Biao Tian and Zhentao Tan and Nan Yang and Bin Yuan and Jieping Ye and Jixing Yu and Qinglin Zhang and Kun Zou and Han Zhao and Shengkui Zhao and Jingren Zhou and Yanqiao Zhu},
+      year={2025},
+      eprint={2509.12508},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2509.12508},
+}
+```

examples/industrial_data_pretraining/fun_asr_nano/__init__.py

@@ -0,0 +1 @@
+

examples/industrial_data_pretraining/fun_asr_nano/client_mic.html

@@ -0,0 +1,160 @@
+<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+<meta charset="UTF-8">
+<title>Fun-ASR-Nano · Streaming ASR with Speaker Diarization</title>
+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800;900&display=swap" rel="stylesheet">
+<style>
+* { margin:0; padding:0; box-sizing:border-box; }
+html, body { height:100%; }
+body { font-family:'Inter',system-ui,sans-serif; background:#080810; color:#e0e0e0; display:flex; justify-content:center; height:100vh; overflow:hidden; }
+
+/* === Hero Section === */
+.hero { padding:24px 28px 16px; flex-shrink:0; }
+.hero-top { display:flex; align-items:center; gap:16px; margin-bottom:14px; }
+.logo { font-size:30px; font-weight:900; letter-spacing:-1px; }
+.logo .g { background:linear-gradient(135deg,#64ffda,#00bfa5); -webkit-background-clip:text; -webkit-text-fill-color:transparent; }
+.logo .w { color:#fff; }
+.logo .s { color:rgba(255,255,255,0.3); font-weight:400; font-size:15px; margin-left:8px; }
+.hero-links { margin-left:auto; display:flex; gap:10px; }
+.hero-links a { font-size:12px; color:#aaa; text-decoration:none; padding:6px 14px; border-radius:8px; background:rgba(255,255,255,0.04); border:1px solid rgba(255,255,255,0.08); transition:all 0.2s; font-weight:500; }
+.hero-links a:hover { color:#64ffda; border-color:rgba(100,255,218,0.3); }
+
+.features { display:flex; gap:10px; flex-wrap:wrap; margin-bottom:14px; }
+.feat { font-size:11px; padding:5px 12px; border-radius:20px; font-weight:500; }
+.feat-green { color:#64ffda; background:rgba(100,255,218,0.08); border:1px solid rgba(100,255,218,0.2); }
+.feat-yellow { color:#fbbf24; background:rgba(251,191,36,0.08); border:1px solid rgba(251,191,36,0.2); }
+.feat-purple { color:#a78bfa; background:rgba(167,139,250,0.08); border:1px solid rgba(167,139,250,0.2); }
+.feat-blue { color:#38bdf8; background:rgba(56,189,248,0.08); border:1px solid rgba(56,189,248,0.2); }
+
+.hero-desc { font-size:13px; color:rgba(255,255,255,0.45); line-height:1.8; }
+.hero-desc b { color:rgba(255,255,255,0.75); font-weight:600; }
+.hero-desc a { color:#64ffda; text-decoration:none; font-weight:600; }
+.hero-desc a:hover { text-decoration:underline; }
+
+/* === Main Panel === */
+.main { flex:1; display:flex; flex-direction:column; padding:0 28px 20px; min-height:0; }
+.panel { flex:1; display:flex; flex-direction:column; background:rgba(16,16,24,0.95); border:1px solid rgba(255,255,255,0.05); border-radius:16px; box-shadow:0 10px 40px rgba(0,0,0,0.4); min-height:0; }
+
+.controls { padding:14px 20px; display:flex; align-items:center; gap:12px; border-bottom:1px solid rgba(255,255,255,0.04); flex-shrink:0; }
+.controls input[type=text] { background:rgba(255,255,255,0.05); border:1px solid rgba(255,255,255,0.1); border-radius:8px; padding:8px 12px; color:#ccc; font-size:12px; width:180px; outline:none; }
+.controls input[type=text]:focus { border-color:rgba(100,255,218,0.4); }
+.controls input[type=file] { display:none; }
+.controls label { font-size:11px; color:#888; display:flex; align-items:center; gap:5px; cursor:pointer; }
+.controls input[type=checkbox] { accent-color:#64ffda; }
+.btn { padding:8px 18px; border:none; border-radius:10px; font-size:12px; cursor:pointer; font-weight:600; transition:all 0.12s; }
+.btn:hover { transform:scale(1.04); }
+.btn:active { transform:scale(0.97); }
+.btn-mic { background:linear-gradient(135deg,#64ffda,#00bfa5); color:#080810; }
+.btn-file { background:linear-gradient(135deg,#fbbf24,#f59e0b); color:#1a1a2e; }
+.btn-hw { background:rgba(167,139,250,0.12); color:#a78bfa; border:1px solid rgba(167,139,250,0.3); }
+.btn-stop { background:linear-gradient(135deg,#ef4444,#dc2626); color:#fff; }
+.sta { margin-left:auto; display:flex; align-items:center; gap:6px; }
+.dot { width:7px; height:7px; border-radius:50%; background:#222; }
+.dot.on { background:#64ffda; box-shadow:0 0 10px rgba(100,255,218,0.6); animation:g 1.5s infinite; }
+@keyframes g { 0%,100%{opacity:1} 50%{opacity:0.3} }
+.sta span { font-size:10px; color:#555; }
+.hw-info { font-size:10px; color:#a78bfa; margin-left:2px; }
+
+.result { flex:1; overflow-y:auto; padding:18px 22px; min-height:0; }
+.result::-webkit-scrollbar { width:4px; }
+.result::-webkit-scrollbar-thumb { background:rgba(255,255,255,0.06); border-radius:2px; }
+.line { padding:6px 0; border-bottom:1px solid rgba(255,255,255,0.02); display:flex; align-items:baseline; gap:0; word-break:break-all; }
+.time { color:rgba(100,255,218,0.5); font-size:11px; font-family:'SF Mono','Menlo',monospace; width:100px; min-width:100px; flex-shrink:0; padding-top:2px; }
+.spk { font-size:10px; font-weight:600; padding:2px 8px; border-radius:5px; flex-shrink:0; margin-right:10px; }
+.text { color:rgba(255,255,255,0.88); font-size:15px; line-height:1.7; flex:1; }
+.partial .text { color:rgba(255,255,255,0.25); font-style:italic; }
+.ph { color:rgba(255,255,255,0.1); text-align:center; padding:50px 20px; font-size:14px; line-height:2; }
+</style>
+</head>
+<body><div style="width:920px;height:100%;display:flex;flex-direction:column">
+
+<div class="hero">
+    <div class="hero-top">
+        <div class="logo"><span class="g">Fun-ASR-Nano</span><span class="s">vLLM Engine</span></div>
+        <div class="hero-links">
+            <a href="https://github.com/modelscope/FunASR" target="_blank">GitHub</a>
+            <a href="https://modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512" target="_blank">ModelScope</a>
+            <a href="https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512" target="_blank">HuggingFace</a>
+        </div>
+    </div>
+    <div class="features">
+        <span class="feat feat-green">Streaming ASR</span>
+        <span class="feat feat-yellow">Speaker Diarization <span style="font-size:9px;opacity:0.7">(Beta)</span></span>
+        <span class="feat feat-purple">Hotword Customization</span>
+        <span class="feat feat-blue">31 Languages · 7 Dialects</span>
+    </div>
+    <p class="hero-desc">
+        基于 <b>FunASR</b> 的 <b>vLLM 推理引擎</b>，实现流式语音识别服务。支持实时 VAD 分句、说话人分离 <span style="color:#f59e0b;font-size:11px">(Beta)</span>、<b>热词定制化</b>（加载人名、地名等实体词列表，提升专有名词识别准确率）、31种语言及中文方言。所有代码与模型已全部开源。<br>
+        <span style="font-size:12px">Streaming ASR with vLLM engine, real-time VAD, speaker diarization <span style="color:#f59e0b">(Beta)</span>, <b>hotword customization</b> (names, places, entities), 31 languages &amp; Chinese dialects. Fully open-sourced.</span>
+        &nbsp;·&nbsp; <a href="https://www.funasr.com" target="_blank">www.funasr.com</a>
+    </p>
+</div>
+
+<div class="main">
+<div class="panel">
+    <div class="controls">
+        <input type="text" id="serverUrl" value="ws://localhost:10095">
+        <label><input type="checkbox" id="showSpk" checked> Speaker <span style="font-size:9px;color:#f59e0b;background:rgba(245,158,11,0.08);padding:1px 5px;border-radius:4px">Beta</span></label>
+        <button class="btn btn-mic" id="btnMic" onclick="startMic()">Mic</button>
+        <input type="file" id="fileInput" accept="audio/*,.wav,.mp3,.flac,.mp4,.m4a">
+        <button class="btn btn-file" id="btnFile" onclick="document.getElementById('fileInput').click()">Audio File</button>
+        <input type="file" id="hotwordFile" accept=".txt" style="display:none">
+        <button class="btn btn-hw" id="btnHw" onclick="document.getElementById('hotwordFile').click()">Hotwords</button>
+        <span class="hw-info" id="hwInfo"></span>
+        <select id="langSelect" style="background:rgba(255,255,255,0.05);border:1px solid rgba(255,255,255,0.1);border-radius:8px;padding:6px 10px;color:#ccc;font-size:11px;outline:none">
+            <option value="">Auto</option>
+            <option value="中文">中文</option>
+            <option value="English">English</option>
+            <option value="日本語">日本語</option>
+            <option value="한국어">한국어</option>
+            <option value="Deutsch">Deutsch</option>
+            <option value="Français">Français</option>
+            <option value="Español">Español</option>
+            <option value="Русский">Русский</option>
+            <option value="العربية">العربية</option>
+            <option value="Português">Português</option>
+            <option value="Italiano">Italiano</option>
+        </select>
+        <button class="btn btn-stop" id="btnStop" onclick="stopAll()" style="display:none">Stop</button>
+        <div class="sta"><div class="dot" id="dot"></div><span id="status">Ready</span></div>
+    </div>
+    <div class="result" id="resultBox">
+        <div class="ph">Click <b>Mic</b> for real-time recognition or <b>Audio File</b> to transcribe a file<br>Load a <b>Hotwords</b> file (.txt, one word per line) to boost recognition of names, places &amp; entities</div>
+    </div>
+</div>
+</div>
+
+<input type="text" id="hotwords" style="display:none">
+<script>
+let ws=null,mediaStream=null,audioContext=null,processor=null,isRecording=false;
+const C=['#64ffda','#f472b6','#fbbf24','#34d399','#a78bfa','#fb923c','#67e8f9','#f87171','#38bdf8','#c084fc'];
+
+document.getElementById('hotwordFile').addEventListener('change',function(e){
+    const f=e.target.files[0];if(!f)return;
+    const r=new FileReader();r.onload=ev=>{
+        const words=ev.target.result.split('\n').map(l=>l.trim()).filter(l=>l);
+        document.getElementById('hotwords').value=words.join(',');
+        document.getElementById('hwInfo').textContent=words.length+' words loaded';
+        setS('Hotwords: '+words.length+' loaded');
+    };r.readAsText(f);
+});
+document.getElementById('fileInput').addEventListener('change',function(){if(this.files.length)startFile();});
+
+function setS(m,on){document.getElementById('status').textContent=m;document.getElementById('dot').className=on?'dot on':'dot';}
+function sStop(){document.getElementById('btnMic').style.display='none';document.getElementById('btnFile').style.display='none';document.getElementById('btnHw').style.display='none';document.getElementById('btnStop').style.display='inline';}
+function sStart(){document.getElementById('btnMic').style.display='inline';document.getElementById('btnFile').style.display='inline';document.getElementById('btnHw').style.display='inline';document.getElementById('btnStop').style.display='none';}
+function render(ss,p,ps,d,f){
+    const b=document.getElementById('resultBox'),sk=document.getElementById('showSpk').checked;let h='';
+    ss.forEach(s=>{let st=s.start!==undefined?s.start:(s.start_ms||0),en=s.end!==undefined?s.end:(s.end_ms||0),sp=s.spk!==undefined?s.spk:-1,sh='';
+        if(sk&&sp>=0){let c=C[sp%C.length];sh='<span class="spk" style="color:'+c+';background:'+c+'12;border:1px solid '+c+'30">SPK'+sp+'</span>';}
+        h+='<div class="line"><span class="time">'+(st/1000).toFixed(1)+' - '+(en/1000).toFixed(1)+'s</span>'+sh+'<span class="text">'+s.text+'</span></div>';});
+    if(p)h+='<div class="line partial"><span class="time">'+(ps/1000).toFixed(1)+'s ...</span><span class="text">'+p+'</span></div>';
+    if(!h)h='<div class="ph">Listening...</div>';b.innerHTML=h;b.scrollTop=b.scrollHeight;}
+function con(cb){ws=new WebSocket(document.getElementById('serverUrl').value);ws.onopen=()=>{ws.send('START');var hw=document.getElementById('hotwords').value.trim();if(hw)ws.send('HOTWORDS:'+hw);var lang=document.getElementById('langSelect').value;if(lang)ws.send('LANGUAGE:'+lang);cb();};ws.onmessage=e=>{const d=JSON.parse(e.data);if(d.sentences!==undefined)render(d.sentences,d.partial,d.partial_start_ms,d.duration_ms,d.is_final);};ws.onerror=()=>setS('Error');ws.onclose=()=>{if(isRecording)stopAll();};}
+function startMic(){con(async()=>{setS('Recording',true);sStop();document.getElementById('resultBox').innerHTML='<div class="ph">Listening...</div>';try{mediaStream=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:16000,channelCount:1,echoCancellation:true}});}catch(e){setS('Mic denied');ws.close();sStart();return;}audioContext=new AudioContext({sampleRate:16000});const s=audioContext.createMediaStreamSource(mediaStream);processor=audioContext.createScriptProcessor(4096,1,1);processor.onaudioprocess=e=>{if(!isRecording)return;const f=e.inputBuffer.getChannelData(0),i=new Int16Array(f.length);for(let j=0;j<f.length;j++)i[j]=Math.max(-32768,Math.min(32767,Math.round(f[j]*32768)));if(ws&&ws.readyState===1)ws.send(i.buffer);};s.connect(processor);processor.connect(audioContext.destination);isRecording=true;});}
+function startFile(){const fi=document.getElementById('fileInput');if(!fi.files.length){setS('Pick a file');return;}const file=fi.files[0];setS('Decoding...');document.getElementById('resultBox').innerHTML='<div class="ph">Decoding '+file.name+'...</div>';const r=new FileReader();r.onload=async e=>{const a=new AudioContext({sampleRate:16000});let buf;try{buf=await a.decodeAudioData(e.target.result);}catch(err){setS('Decode failed');return;}const p=buf.getChannelData(0),i=new Int16Array(p.length);for(let j=0;j<p.length;j++)i[j]=Math.max(-32768,Math.min(32767,Math.round(p[j]*32768)));a.close();con(async()=>{setS('Streaming '+(p.length/16000|0)+'s',true);sStop();isRecording=true;for(let j=0;j<i.length&&isRecording;j+=4096){if(ws&&ws.readyState===1)ws.send(i.slice(j,j+4096).buffer);await new Promise(r=>setTimeout(r,50));}if(isRecording)stopAll();});};r.readAsArrayBuffer(file);}
+function stopAll(){isRecording=false;if(ws&&ws.readyState===1)ws.send('STOP');if(processor){processor.disconnect();processor=null;}if(audioContext){audioContext.close();audioContext=null;}if(mediaStream){mediaStream.getTracks().forEach(t=>t.stop());mediaStream=null;}setTimeout(()=>{if(ws){ws.close();ws=null;}},2000);sStart();setS('Done');}
+</script>
+</div></body>
+</html>

examples/industrial_data_pretraining/fun_asr_nano/client_python.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""Fun-ASR-Nano Python WebSocket Client.
+
+Supports real-time microphone recording and audio file streaming.
+
+Usage:
+    # Microphone mode
+    python client_python.py --server ws://localhost:10095 --mic
+
+    # File mode
+    python client_python.py --server ws://localhost:10095 --file audio.wav
+
+    # With hotwords
+    python client_python.py --server ws://localhost:10095 --file audio.wav --hotwords "张三,李四,北京"
+
+    # Disable speaker diarization display
+    python client_python.py --server ws://localhost:10095 --mic --no-spk
+"""
+
+import asyncio
+import argparse
+import json
+import sys
+import numpy as np
+
+try:
+    import websockets
+except ImportError:
+    print("Please install websockets: pip install websockets")
+    sys.exit(1)
+
+
+SAMPLE_RATE = 16000
+CHUNK_DURATION_MS = 100
+CHUNK_SAMPLES = int(SAMPLE_RATE * CHUNK_DURATION_MS / 1000)
+
+SPK_COLORS = [
+    "\033[36m", "\033[35m", "\033[33m", "\033[32m",
+    "\033[34m", "\033[91m", "\033[96m", "\033[95m",
+]
+RESET = "\033[0m"
+GRAY = "\033[90m"
+GREEN = "\033[92m"
+
+
+def format_time(ms):
+    s = ms / 1000
+    return f"{s:.1f}s"
+
+
+def print_result(data, show_spk=True):
+    """Print ASR result to terminal."""
+    sentences = data.get("sentences", [])
+    partial = data.get("partial", "")
+    partial_start = data.get("partial_start_ms", 0)
+    is_final = data.get("is_final", False)
+
+    sys.stdout.write("\033[2J\033[H")
+
+    print(f"{GREEN}Fun-ASR-Nano Streaming ASR{RESET}")
+    print(f"{GRAY}{'─' * 60}{RESET}")
+
+    for s in sentences:
+        start = s.get("start", s.get("start_ms", 0))
+        end = s.get("end", s.get("end_ms", 0))
+        spk = s.get("spk", -1)
+        text = s["text"]
+
+        time_str = f"{GRAY}[{format_time(start)}-{format_time(end)}]{RESET}"
+        spk_str = ""
+        if show_spk and spk >= 0:
+            color = SPK_COLORS[spk % len(SPK_COLORS)]
+            spk_str = f" {color}SPK{spk}{RESET}"
+
+        print(f"  {time_str}{spk_str} {text}")
+
+    if partial:
+        print(f"  {GRAY}[{format_time(partial_start)}-...] {partial}{RESET}")
+
+    if is_final:
+        print(f"\n{GRAY}{'─' * 60}{RESET}")
+        print(f"{GREEN}Done.{RESET} {len(sentences)} sentences")
+    else:
+        print(f"\n{GRAY}Recording... Press Ctrl+C to stop{RESET}")
+
+    sys.stdout.flush()
+
+
+async def run_mic(args):
+    """Stream from microphone."""
+    try:
+        import sounddevice as sd
+    except ImportError:
+        print("Please install sounddevice: pip install sounddevice")
+        sys.exit(1)
+
+    print(f"Connecting to {args.server}...")
+    async with websockets.connect(args.server, ping_interval=None) as ws:
+        await ws.send("START")
+        resp = await ws.recv()
+        event = json.loads(resp)
+        if event.get("event") != "started":
+            print(f"Unexpected response: {resp}")
+            return
+
+        if args.hotwords:
+            await ws.send(f"HOTWORDS:{args.hotwords}")
+            await ws.recv()
+
+        print("Recording... Press Ctrl+C to stop\n")
+
+        audio_queue = asyncio.Queue()
+
+        def audio_callback(indata, frames, time_info, status):
+            audio_queue.put_nowait(indata.copy())
+
+        stream = sd.InputStream(
+            samplerate=SAMPLE_RATE, channels=1, dtype='int16',
+            blocksize=CHUNK_SAMPLES, callback=audio_callback,
+        )
+
+        async def send_audio():
+            with stream:
+                while True:
+                    chunk = await audio_queue.get()
+                    await ws.send(chunk.tobytes())
+
+        async def recv_results():
+            async for msg in ws:
+                data = json.loads(msg)
+                if "sentences" in data:
+                    print_result(data, show_spk=args.spk)
+                if data.get("is_final") or data.get("event") == "stopped":
+                    break
+
+        send_task = asyncio.create_task(send_audio())
+        recv_task = asyncio.create_task(recv_results())
+
+        try:
+            await asyncio.gather(send_task, recv_task)
+        except (KeyboardInterrupt, asyncio.CancelledError):
+            pass
+        finally:
+            send_task.cancel()
+            if ws.open:
+                await ws.send("STOP")
+                async for msg in ws:
+                    data = json.loads(msg)
+                    if "sentences" in data:
+                        print_result(data, show_spk=args.spk)
+                    if data.get("is_final") or data.get("event") == "stopped":
+                        break
+
+
+async def run_file(args):
+    """Stream an audio file."""
+    try:
+        import soundfile as sf
+    except ImportError:
+        print("Please install soundfile: pip install soundfile")
+        sys.exit(1)
+
+    audio, sr = sf.read(args.file)
+    if sr != SAMPLE_RATE:
+        try:
+            import librosa
+            audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
+        except ImportError:
+            print(f"Audio is {sr}Hz, need 16kHz. Install librosa: pip install librosa")
+            sys.exit(1)
+    if audio.ndim > 1:
+        audio = audio[:, 0]
+    audio = audio.astype(np.float32)
+
+    duration = len(audio) / SAMPLE_RATE
+    print(f"File: {args.file} ({duration:.1f}s)")
+    print(f"Connecting to {args.server}...")
+
+    async with websockets.connect(args.server, ping_interval=None) as ws:
+        await ws.send("START")
+        await ws.recv()
+
+        if args.hotwords:
+            await ws.send(f"HOTWORDS:{args.hotwords}")
+            await ws.recv()
+
+        int16 = (audio * 32768).clip(-32768, 32767).astype(np.int16)
+
+        chunk_size = CHUNK_SAMPLES
+        total_chunks = (len(int16) + chunk_size - 1) // chunk_size
+
+        async def send_audio():
+            for i in range(0, len(int16), chunk_size):
+                chunk = int16[i:i+chunk_size]
+                await ws.send(chunk.tobytes())
+                await asyncio.sleep(CHUNK_DURATION_MS / 1000 * 0.5)
+            await ws.send("STOP")
+
+        async def recv_results():
+            async for msg in ws:
+                data = json.loads(msg)
+                if "sentences" in data:
+                    print_result(data, show_spk=args.spk)
+                if data.get("is_final") or data.get("event") == "stopped":
+                    break
+
+        await asyncio.gather(send_audio(), recv_results())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Fun-ASR-Nano Python Client")
+    parser.add_argument("--server", type=str, default="ws://localhost:10095")
+    parser.add_argument("--mic", action="store_true", help="Use microphone input")
+    parser.add_argument("--file", type=str, help="Audio file to transcribe")
+    parser.add_argument("--hotwords", type=str, default="", help="Hotwords (comma-separated)")
+    parser.add_argument("--spk", action="store_true", default=True, help="Show speaker IDs")
+    parser.add_argument("--no-spk", dest="spk", action="store_false")
+    args = parser.parse_args()
+
+    if not args.mic and not args.file:
+        parser.error("Specify --mic or --file")
+
+    try:
+        if args.mic:
+            asyncio.run(run_mic(args))
+        else:
+            asyncio.run(run_file(args))
+    except KeyboardInterrupt:
+        print(f"\n{RESET}Interrupted.")
+
+
+if __name__ == "__main__":
+    main()

examples/industrial_data_pretraining/fun_asr_nano/client_test.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+"""Fun-ASR-Nano Streaming ASR - Automated Test Script.
+
+Tests the WebSocket server with audio files and validates the response format.
+
+Usage:
+    python client_test.py --server ws://localhost:10095 --file test_audio.wav
+    python client_test.py --server ws://localhost:10095 --file test_audio.wav --hotwords "热词1,热词2"
+"""
+
+import asyncio
+import argparse
+import json
+import time
+import sys
+import numpy as np
+
+try:
+    import websockets
+except ImportError:
+    print("ERROR: pip install websockets")
+    sys.exit(1)
+
+try:
+    import soundfile as sf
+except ImportError:
+    print("ERROR: pip install soundfile")
+    sys.exit(1)
+
+
+SAMPLE_RATE = 16000
+
+
+def load_audio(file_path):
+    audio, sr = sf.read(file_path)
+    if sr != SAMPLE_RATE:
+        try:
+            import librosa
+            audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
+        except ImportError:
+            print(f"ERROR: Audio is {sr}Hz. Install librosa for resampling.")
+            sys.exit(1)
+    if audio.ndim > 1:
+        audio = audio[:, 0]
+    return audio.astype(np.float32)
+
+
+async def test_basic(server, audio, hotwords=None):
+    """Test basic streaming ASR flow."""
+    print("=" * 60)
+    print("TEST: Basic Streaming ASR")
+    print("=" * 60)
+
+    t_start = time.perf_counter()
+
+    async with websockets.connect(server, ping_interval=None) as ws:
+        await ws.send("START")
+        resp = json.loads(await ws.recv())
+        assert resp["event"] == "started", f"Expected 'started', got: {resp}"
+        print("  [PASS] START -> event:started")
+
+        if hotwords:
+            await ws.send(f"HOTWORDS:{hotwords}")
+            resp = json.loads(await ws.recv())
+            assert resp["event"] == "hotwords_set", f"Expected 'hotwords_set', got: {resp}"
+            print(f"  [PASS] HOTWORDS -> {len(resp['hotwords'])} words set")
+
+        int16 = (audio * 32768).clip(-32768, 32767).astype(np.int16)
+        chunk_size = 4096
+        partial_count = 0
+
+        for i in range(0, len(int16), chunk_size):
+            chunk = int16[i:i+chunk_size]
+            await ws.send(chunk.tobytes())
+            await asyncio.sleep(0.02)
+
+            try:
+                msg = await asyncio.wait_for(ws.recv(), timeout=0.01)
+                data = json.loads(msg)
+                if "sentences" in data:
+                    partial_count += 1
+                    assert "partial" in data
+                    assert "duration_ms" in data
+                    assert "is_final" in data
+                    assert isinstance(data["sentences"], list)
+            except asyncio.TimeoutError:
+                pass
+
+        await ws.send("STOP")
+
+        final_result = None
+        while True:
+            msg = await asyncio.wait_for(ws.recv(), timeout=30)
+            data = json.loads(msg)
+            if data.get("is_final"):
+                final_result = data
+            if data.get("event") == "stopped":
+                break
+
+        t_end = time.perf_counter()
+
+    assert final_result is not None, "No final result received"
+    assert final_result["is_final"] is True
+    sentences = final_result["sentences"]
+    assert len(sentences) > 0, "No sentences in final result"
+
+    print(f"  [PASS] Received final result: {len(sentences)} sentences")
+    print(f"  [PASS] Partial updates received: {partial_count}")
+
+    for s in sentences:
+        assert "text" in s, f"Missing 'text' in sentence: {s}"
+        assert "start" in s, f"Missing 'start' in sentence: {s}"
+        assert "end" in s, f"Missing 'end' in sentence: {s}"
+        assert isinstance(s["text"], str) and len(s["text"]) > 0
+        assert s["end"] > s["start"]
+
+    print("  [PASS] All sentences have valid format (text, start, end)")
+
+    has_spk = all("spk" in s for s in sentences)
+    if has_spk:
+        spk_ids = set(s["spk"] for s in sentences)
+        print(f"  [PASS] Speaker diarization: {len(spk_ids)} speakers detected")
+    else:
+        print("  [INFO] No speaker IDs in result")
+
+    elapsed = t_end - t_start
+    audio_duration = len(audio) / SAMPLE_RATE
+    rtf = elapsed / audio_duration
+    print(f"\n  Audio: {audio_duration:.1f}s | Time: {elapsed:.2f}s | RTF: {rtf:.3f}")
+
+    print("\n  --- Results ---")
+    for s in sentences:
+        spk = f" SPK{s['spk']}" if "spk" in s else ""
+        print(f"  [{s['start']/1000:.1f}-{s['end']/1000:.1f}s]{spk}: {s['text']}")
+
+    return True
+
+
+async def test_empty_audio(server):
+    """Test with very short/empty audio."""
+    print("\n" + "=" * 60)
+    print("TEST: Empty/Short Audio Handling")
+    print("=" * 60)
+
+    async with websockets.connect(server, ping_interval=None) as ws:
+        await ws.send("START")
+        await ws.recv()
+
+        short_audio = np.zeros(800, dtype=np.int16)
+        await ws.send(short_audio.tobytes())
+        await asyncio.sleep(0.1)
+
+        await ws.send("STOP")
+        final = None
+        while True:
+            msg = await asyncio.wait_for(ws.recv(), timeout=10)
+            data = json.loads(msg)
+            if data.get("is_final"):
+                final = data
+            if data.get("event") == "stopped":
+                break
+
+        assert final is not None
+        assert final["sentences"] == [] or all(s["text"].strip() for s in final["sentences"])
+        print("  [PASS] Short audio handled gracefully")
+
+    return True
+
+
+async def test_multiple_sessions(server, audio):
+    """Test multiple consecutive sessions on same connection."""
+    print("\n" + "=" * 60)
+    print("TEST: Multiple Sessions")
+    print("=" * 60)
+
+    audio_short = audio[:SAMPLE_RATE * 5]
+    int16 = (audio_short * 32768).clip(-32768, 32767).astype(np.int16)
+
+    async with websockets.connect(server, ping_interval=None) as ws:
+        for session_num in range(2):
+            await ws.send("START")
+            resp = json.loads(await ws.recv())
+            assert resp["event"] == "started"
+
+            for i in range(0, len(int16), 4096):
+                await ws.send(int16[i:i+4096].tobytes())
+                await asyncio.sleep(0.01)
+
+            await ws.send("STOP")
+            got_final = False
+            while True:
+                msg = await asyncio.wait_for(ws.recv(), timeout=15)
+                data = json.loads(msg)
+                if data.get("is_final"):
+                    got_final = True
+                if data.get("event") == "stopped":
+                    break
+            assert got_final, f"Session {session_num+1}: no final result"
+            print(f"  [PASS] Session {session_num+1} completed")
+
+    return True
+
+
+async def run_tests(args):
+    audio = load_audio(args.file)
+    print(f"Loaded: {args.file} ({len(audio)/SAMPLE_RATE:.1f}s)")
+    print(f"Server: {args.server}\n")
+
+    passed = 0
+    failed = 0
+
+    tests = [
+        ("Basic Streaming", lambda: test_basic(args.server, audio, args.hotwords)),
+        ("Empty Audio", lambda: test_empty_audio(args.server)),
+        ("Multiple Sessions", lambda: test_multiple_sessions(args.server, audio)),
+    ]
+
+    for name, test_fn in tests:
+        try:
+            await test_fn()
+            passed += 1
+        except Exception as e:
+            print(f"  [FAIL] {name}: {e}")
+            failed += 1
+
+    print("\n" + "=" * 60)
+    print(f"RESULTS: {passed} passed, {failed} failed")
+    print("=" * 60)
+    return failed == 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming Test")
+    parser.add_argument("--server", type=str, default="ws://localhost:10095")
+    parser.add_argument("--file", type=str, required=True, help="Test audio file")
+    parser.add_argument("--hotwords", type=str, default="", help="Hotwords (comma-separated)")
+    args = parser.parse_args()
+
+    success = asyncio.run(run_tests(args))
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

examples/industrial_data_pretraining/fun_asr_nano/ctc.py

@@ -0,0 +1,60 @@
+import torch
+import torch.nn.functional as F
+
+
+class CTC(torch.nn.Module):
+    """CTC module.
+
+    Args:
+        odim: dimension of outputs
+        encoder_output_size: number of encoder projection units
+        dropout_rate: dropout rate (0.0 ~ 1.0)
+        reduce: reduce the CTC loss into a scalar
+    """
+
+    def __init__(
+        self,
+        odim: int,
+        encoder_output_size: int,
+        dropout_rate: float = 0.0,
+        reduce: bool = True,
+        blank_id: int = 0,
+        **kwargs,
+    ):
+        super().__init__()
+        eprojs = encoder_output_size
+        self.dropout_rate = dropout_rate
+        self.ctc_lo = torch.nn.Linear(eprojs, odim)
+        self.blank_id = blank_id
+        self.ctc_loss = torch.nn.CTCLoss(reduction="none", blank=blank_id)
+        self.reduce = reduce
+
+    def softmax(self, hs_pad):
+        """softmax of frame activations
+
+        Args:
+            Tensor hs_pad: 3d tensor (B, Tmax, eprojs)
+        Returns:
+            torch.Tensor: softmax applied 3d tensor (B, Tmax, odim)
+        """
+        return F.softmax(self.ctc_lo(hs_pad), dim=2)
+
+    def log_softmax(self, hs_pad):
+        """log_softmax of frame activations
+
+        Args:
+            Tensor hs_pad: 3d tensor (B, Tmax, eprojs)
+        Returns:
+            torch.Tensor: log softmax applied 3d tensor (B, Tmax, odim)
+        """
+        return F.log_softmax(self.ctc_lo(hs_pad), dim=2)
+
+    def argmax(self, hs_pad):
+        """argmax of frame activations
+
+        Args:
+            torch.Tensor hs_pad: 3d tensor (B, Tmax, eprojs)
+        Returns:
+            torch.Tensor: argmax applied 2d tensor (B, Tmax)
+        """
+        return torch.argmax(self.ctc_lo(hs_pad), dim=2)

examples/industrial_data_pretraining/fun_asr_nano/data/train_example.jsonl

@@ -0,0 +1,20 @@
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0002.wav<|endofspeech|>"}, {"role": "assistant", "content": "几点了？"}], "speech_length": 145, "text_length": 3}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0001.wav<|endofspeech|>"}, {"role": "assistant", "content": "Margaret said Mister Hale, as he returned from showing his guest downstairs, I could not help watching your face with some anxiety when Mister Thornton made his confession of having been a shop boy."}], "speech_length": 1261, "text_length": 38}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写成英文：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0004.wav<|endofspeech|>"}, {"role": "assistant", "content": "Number ten. Fresh Nelly is waiting on you. Good night, husband."}], "speech_length": 520, "text_length": 16}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0002.wav<|endofspeech|>"}, {"role": "assistant", "content": "After early nightfall, the yellow lamps would light up here and there the squalid quarter of the brothels."}], "speech_length": 661, "text_length": 24}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0126.wav<|endofspeech|>"}, {"role": "assistant", "content": "因此，土地储备至关重要。"}], "speech_length": 348, "text_length": 6}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription without text normalization:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0002.wav<|endofspeech|>"}, {"role": "assistant", "content": "You don't mean that you thought me so silly"}], "speech_length": 282, "text_length": 11}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写成英文：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0001.wav<|endofspeech|>"}, {"role": "assistant", "content": "Stuff it into you, his belly counselled him."}], "speech_length": 326, "text_length": 11}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Transcribe speech into Chinese:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0127.wav<|endofspeech|>"}, {"role": "assistant", "content": "中原地产首席分析师张大伟说。"}], "speech_length": 443, "text_length": 9}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写，不进行文本规整：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0001.wav<|endofspeech|>"}, {"role": "assistant", "content": "换一首歌"}], "speech_length": 197, "text_length": 4}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Transcribe speech into Chinese:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0124.wav<|endofspeech|>"}, {"role": "assistant", "content": "为了规避三四线城市明显过剩的市场风险，"}], "speech_length": 522, "text_length": 11}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写成英文：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0003.wav<|endofspeech|>"}, {"role": "assistant", "content": "Hello, Bertie, any good in your mind?"}], "speech_length": 266, "text_length": 11}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Transcribe speech into Chinese:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0125.wav<|endofspeech|>"}, {"role": "assistant", "content": "标杆房企必然调整市场战略。"}], "speech_length": 429, "text_length": 7}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0000.wav<|endofspeech|>"}, {"role": "assistant", "content": "There's iron, they say, in all our blood, and a grain or two perhaps is good; but his, he makes me harshly feel, has got a little too much of steel. Anon."}], "speech_length": 1498, "text_length": 43}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0006.wav<|endofspeech|>"}, {"role": "assistant", "content": "The dull light fell more faintly upon the page whereon another equation began to unfold itself slowly and to spread abroad its widening tail."}], "speech_length": 1054, "text_length": 27}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写成英文：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0008.wav<|endofspeech|>"}, {"role": "assistant", "content": "The chaos in which his ardour extinguished itself was a cold indifferent knowledge of himself."}], "speech_length": 671, "text_length": 18}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0003.wav<|endofspeech|>"}, {"role": "assistant", "content": "早上好。"}], "speech_length": 147, "text_length": 3}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0005.wav<|endofspeech|>"}, {"role": "assistant", "content": "The music came nearer, and he recalled the words, the words of Shelley's fragment upon the moon, wandering companionless, pale for weariness."}], "speech_length": 962, "text_length": 30}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0004.wav<|endofspeech|>"}, {"role": "assistant", "content": "His statement of having been a shop boy was the thing I liked best of all."}], "speech_length": 446, "text_length": 17}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写成英文：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0007.wav<|endofspeech|>"}, {"role": "assistant", "content": "A cold lucid indifference reigned in his soul."}], "speech_length": 426, "text_length": 11}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0129.wav<|endofspeech|>"}, {"role": "assistant", "content": "也助推了土地市场的火爆。"}], "speech_length": 357, "text_length": 7}

examples/industrial_data_pretraining/fun_asr_nano/data/train_text.txt

@@ -0,0 +1,20 @@
+1089-134686-0004	Number ten. Fresh Nelly is waiting on you. Good night, husband.
+BAC009S0764W0127	中原地产首席分析师张大伟说。
+1089-134686-0002 After early nightfall, the yellow lamps would light up here and there the squalid quarter of the brothels.
+1688-142285-0001	Margaret said Mister Hale, as he returned from showing his guest downstairs, I could not help watching your face with some anxiety when Mister Thornton made his confession of having been a shop boy.
+1089-134686-0001	Stuff it into you, his belly counselled him.
+1688-142285-0002	You don't mean that you thought me so silly
+IT0011W0002	几点了？
+BAC009S0764W0126	因此，土地储备至关重要。
+BAC009S0764W0125	标杆房企必然调整市场战略。
+IT0011W0001	换一首歌。
+1688-142285-0000	There's iron, they say, in all our blood, and a grain or two perhaps is good; but his, he makes me harshly feel, has got a little too much of steel. Anon.
+BAC009S0764W0124	为了规避三四线城市明显过剩的市场风险，
+1089-134686-0003	Hello, Bertie, any good in your mind?
+1089-134686-0006	The dull light fell more faintly upon the page whereon another equation began to unfold itself slowly and to spread abroad its widening tail.
+IT0011W0003	早上好。
+1089-134686-0008	The chaos in which his ardour extinguished itself was a cold indifferent knowledge of himself.
+BAC009S0764W0129	也助推了土地市场的火爆。
+1089-134686-0005	The music came nearer, and he recalled the words, the words of Shelley's fragment upon the moon, wandering companionless, pale for weariness.
+1089-134686-0007	A cold lucid indifference reigned in his soul.
+1688-142285-0004	His statement of having been a shop boy was the thing I liked best of all.

examples/industrial_data_pretraining/fun_asr_nano/data/train_wav.scp

@@ -0,0 +1,20 @@
+1089-134686-0004	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0004.wav
+BAC009S0764W0127	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0127.wav
+1089-134686-0002	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0002.wav
+1688-142285-0001	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0001.wav
+1089-134686-0001	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0001.wav
+1688-142285-0002	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0002.wav
+IT0011W0002	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0002.wav
+BAC009S0764W0126	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0126.wav
+BAC009S0764W0125	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0125.wav
+IT0011W0001	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0001.wav
+1688-142285-0000	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0000.wav
+BAC009S0764W0124	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0124.wav
+1089-134686-0003	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0003.wav
+1089-134686-0006	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0006.wav
+IT0011W0003	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0003.wav
+1089-134686-0008	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0008.wav
+BAC009S0764W0129	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0129.wav
+1089-134686-0005	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0005.wav
+1089-134686-0007	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0007.wav
+1688-142285-0004	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0004.wav

examples/industrial_data_pretraining/fun_asr_nano/data/val_example.jsonl

@@ -0,0 +1,10 @@
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写成英文：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0000.wav<|endofspeech|>"}, {"role": "assistant", "content": "He hoped there would be stew for dinner, turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick peppered flour-fattened sauce."}], "speech_length": 1042, "text_length": 37}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0004.wav<|endofspeech|>"}, {"role": "assistant", "content": "放歌。"}], "speech_length": 131, "text_length": 3}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0130.wav<|endofspeech|>"}, {"role": "assistant", "content": "北京仅新增住宅土地供应10宗。"}], "speech_length": 523, "text_length": 9}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0005.wav<|endofspeech|>"}, {"role": "assistant", "content": "放首歌。"}], "speech_length": 155, "text_length": 4}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0121.wav<|endofspeech|>"}, {"role": "assistant", "content": "甚至出现交易几乎停滞的情况。"}], "speech_length": 418, "text_length": 7}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0009.wav<|endofspeech|>"}, {"role": "assistant", "content": "At most, by an alms given to a beggar whose blessing he fled from, he might hope wearily to win for himself some measure of actual grace."}], "speech_length": 1056, "text_length": 33}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写，不进行文本规整：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0123.wav<|endofspeech|>"}, {"role": "assistant", "content": "但因为聚集了过多公共资源"}], "speech_length": 398, "text_length": 7}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0003.wav<|endofspeech|>"}, {"role": "assistant", "content": "I really liked that account of himself better than anything else he said."}], "speech_length": 504, "text_length": 14}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0128.wav<|endofspeech|>"}, {"role": "assistant", "content": "一线城市土地供应量减少，"}], "speech_length": 355, "text_length": 6}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Speech transcription without text normalization:<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0122.wav<|endofspeech|>"}, {"role": "assistant", "content": "一二线城市虽然也处于调整中"}], "speech_length": 410, "text_length": 9}

examples/industrial_data_pretraining/fun_asr_nano/data/val_text.txt

@@ -0,0 +1,10 @@
+1688-142285-0003	I really liked that account of himself better than anything else he said.
+BAC009S0764W0130	北京仅新增住宅土地供应10宗。
+BAC009S0764W0123	但因为聚集了过多公共资源，
+IT0011W0004	放歌。
+IT0011W0005	放首歌。
+1089-134686-0009	At most, by an alms given to a beggar whose blessing he fled from, he might hope wearily to win for himself some measure of actual grace.
+1089-134686-0000	He hoped there would be stew for dinner, turnips and carrots and bruised potatoes and fat mutton pieces to be ladled out in thick peppered flour-fattened sauce.
+BAC009S0764W0121	甚至出现交易几乎停滞的情况。
+BAC009S0764W0128	一线城市土地供应量减少
+BAC009S0764W0122	一二线城市虽然也处于调整中，

examples/industrial_data_pretraining/fun_asr_nano/data/val_wav.scp

@@ -0,0 +1,10 @@
+1688-142285-0003	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1688-142285-0003.wav
+BAC009S0764W0130	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0130.wav
+BAC009S0764W0123	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0123.wav
+IT0011W0004	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0004.wav
+IT0011W0005	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0005.wav
+1089-134686-0009	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0009.wav
+1089-134686-0000	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/1089-134686-0000.wav
+BAC009S0764W0121	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0121.wav
+BAC009S0764W0128	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0128.wav
+BAC009S0764W0122	https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/BAC009S0764W0122.wav

examples/industrial_data_pretraining/fun_asr_nano/decode.py

@@ -0,0 +1,58 @@
+import os
+
+import hydra
+import torch
+from omegaconf import DictConfig, ListConfig, OmegaConf
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(cfg: DictConfig):
+    def to_plain_list(cfg_item):
+        if isinstance(cfg_item, ListConfig):
+            return OmegaConf.to_container(cfg_item, resolve=True)
+        elif isinstance(cfg_item, DictConfig):
+            return {k: to_plain_list(v) for k, v in cfg_item.items()}
+        else:
+            return cfg_item
+
+    kwargs = to_plain_list(cfg)
+
+    model_dir = kwargs.get("model_dir", "FunAudioLLM/Fun-ASR-Nano-2512")
+    scp_file = kwargs["scp_file"]
+    output_file = kwargs["output_file"]
+
+    device = (
+        "cuda:0"
+        if torch.cuda.is_available()
+        else "mps" if torch.backends.mps.is_available() else "cpu"
+    )
+
+    from funasr import AutoModel
+
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        vad_model="fsmn-vad",
+        vad_kwargs={"max_single_segment_time": 30000},
+        remote_code="./model.py",
+        device=device,
+    )
+
+    output_dir = os.path.dirname(output_file)
+    if output_dir and not os.path.exists(output_dir):
+        os.makedirs(output_dir, exist_ok=True)
+
+    with open(scp_file, "r", encoding="utf-8") as f1:
+        with open(output_file, "w", encoding="utf-8") as f2:
+            for line in f1:
+                line = line.strip()
+                if not line:
+                    continue
+                parts = line.split(maxsplit=1)
+                if len(parts) == 2:
+                    text = model.generate(input=[parts[1]], cache={}, batch_size=1)[0]["text"]
+                    f2.write(f"{parts[0]}\t{text}\n")
+
+
+if __name__ == "__main__":
+    main_hydra()

examples/industrial_data_pretraining/fun_asr_nano/deepspeed_conf/ds_stage1.json

@@ -0,0 +1,33 @@
+{
+  "train_micro_batch_size_per_gpu": 1,
+  "gradient_accumulation_steps": 1,
+  "steps_per_print": 100,
+  "gradient_clipping": 5,
+  "fp16": {
+    "enabled": false,
+    "auto_cast": false,
+    "loss_scale": 0,
+    "initial_scale_power": 16,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "consecutive_hysteresis": false,
+    "min_loss_scale": 1
+  },
+  "bf16": {
+   "enabled": true
+  },
+  "zero_force_ds_cpu_optimizer": false,
+  "zero_optimization": {
+    "stage": 1,
+    "offload_optimizer": {
+      "device": "none",
+      "pin_memory": true
+    },
+    "allgather_partitions": true,
+    "allgather_bucket_size": 5e8,
+    "overlap_comm": true,
+    "reduce_scatter": true,
+    "reduce_bucket_size": 5e8,
+    "contiguous_gradients" : true
+  }
+}

examples/industrial_data_pretraining/fun_asr_nano/demo1.py

@@ -0,0 +1,54 @@
+import torch
+
+
+def main():
+    model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+    device = (
+        "cuda:0"
+        if torch.cuda.is_available()
+        else "mps" if torch.backends.mps.is_available() else "cpu"
+    )
+
+    from funasr import AutoModel
+
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        remote_code="./model.py",
+        device=device,
+        # hub：download models from ms (for ModelScope) or hf (for Hugging Face).
+        hub="ms",
+    )
+
+    wav_path = f"{model.model_path}/example/zh.mp3"
+    res = model.generate(
+        input=[wav_path],
+        cache={},
+        batch_size=1,
+        hotwords=["开放时间"],
+        # 中文、英文、日文 for Fun-ASR-Nano-2512
+        # 中文、英文、粤语、日文、韩文、越南语、印尼语、泰语、马来语、菲律宾语、阿拉伯语、
+        # 印地语、保加利亚语、克罗地亚语、捷克语、丹麦语、荷兰语、爱沙尼亚语、芬兰语、希腊语、
+        # 匈牙利语、爱尔兰语、拉脱维亚语、立陶宛语、马耳他语、波兰语、葡萄牙语、罗马尼亚语、
+        # 斯洛伐克语、斯洛文尼亚语、瑞典语 for Fun-ASR-MLT-Nano-2512
+        language="中文",
+        itn=True,  # or False
+    )
+    text = res[0]["text"]
+    print(text)
+
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        vad_model="fsmn-vad",
+        vad_kwargs={"max_single_segment_time": 30000},
+        remote_code="./model.py",
+        device=device,
+    )
+    res = model.generate(input=[wav_path], cache={}, batch_size=1)
+    text = res[0]["text"]
+    print(text)
+
+
+if __name__ == "__main__":
+    main()

examples/industrial_data_pretraining/fun_asr_nano/demo2.py

@@ -0,0 +1,39 @@
+import numpy as np
+import soundfile as sf
+import torch
+
+from model import FunASRNano
+from tools.utils import load_audio
+
+
+def main():
+    model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+    device = (
+        "cuda:0"
+        if torch.cuda.is_available()
+        else "mps" if torch.backends.mps.is_available() else "cpu"
+    )
+    m, kwargs = FunASRNano.from_pretrained(model=model_dir, device=device)
+    tokenizer = kwargs.get("tokenizer", None)
+    m.eval()
+
+    wav_path = f"{kwargs['model_path']}/example/zh.mp3"
+    res = m.inference(data_in=[wav_path], **kwargs)
+    text = res[0][0]
+    print(text)
+
+    chunk_size = 0.72
+    duration = sf.info(wav_path).duration
+    cum_durations = np.arange(chunk_size, duration + chunk_size, chunk_size)
+    prev_text = ""
+    for idx, cum_duration in enumerate(cum_durations):
+        audio, rate = load_audio(wav_path, 16000, duration=round(cum_duration, 3))
+        prev_text = m.inference([torch.tensor(audio)], prev_text=prev_text, **kwargs)[0][0]["text"]
+        if idx != len(cum_durations) - 1:
+            prev_text = tokenizer.decode(tokenizer.encode(prev_text)[:-5]).replace("�", "")
+        if prev_text:
+            print(prev_text)
+
+
+if __name__ == "__main__":
+    main()

examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+"""
+Fun-ASR-Nano with Speaker Diarization
+
+This demo shows how to use Fun-ASR-Nano with VAD + Speaker Model + Punctuation Model
+to get per-sentence speaker labels.
+"""
+
+import torch
+from funasr import AutoModel
+
+
+def main():
+    model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+    device = (
+        "cuda:0"
+        if torch.cuda.is_available()
+        else "mps" if torch.backends.mps.is_available() else "cpu"
+    )
+
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        remote_code="./model.py",
+        vad_model="fsmn-vad",
+        vad_kwargs={"max_single_segment_time": 30000},
+        spk_model="cam++",
+        device=device,
+        hub="ms",
+    )
+
+    wav_path = f"{model.model_path}/example/zh.mp3"
+    res = model.generate(
+        input=[wav_path],
+        cache={},
+        batch_size=1,
+        language="中文",
+    )
+
+    # Print full text
+    print("Text:", res[0]["text"])
+    print()
+
+    # Print per-sentence results with speaker labels
+    print("Speaker Diarization Results:")
+    for sent in res[0]["sentence_info"]:
+        print(
+            f"  Speaker {sent['spk']}: [{sent['start']}ms - {sent['end']}ms] {sent['text']}"
+        )
+
+
+if __name__ == "__main__":
+    main()

examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""Demo: Fun-ASR-Nano with vLLM inference backend.
+
+Usage:
+    # Single GPU (greedy decoding)
+    python demo_vllm.py
+
+    # Multi-GPU tensor parallel
+    python demo_vllm.py --tensor-parallel-size 2
+
+    # Batch inference from wav.scp
+    python demo_vllm.py --input wav.scp --tensor-parallel-size 4 --batch-size 32
+
+    # With hotwords and language
+    python demo_vllm.py --input audio.wav --language 中文 --hotwords 开放时间 周一
+"""
+
+import argparse
+import os
+import time
+
+import torch
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Fun-ASR-Nano vLLM Inference Demo")
+    parser.add_argument(
+        "--model-dir",
+        type=str,
+        default="FunAudioLLM/Fun-ASR-Nano-2512",
+        help="Model name (from hub) or local directory path",
+    )
+    parser.add_argument("--input", type=str, default=None, help="Audio file, wav.scp, or jsonl")
+    parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])
+    parser.add_argument("--device", type=str, default="cuda:0", help="Device for audio encoder")
+    parser.add_argument("--dtype", type=str, default="bf16", choices=["bf16", "fp16", "fp32"])
+    parser.add_argument(
+        "--tensor-parallel-size", type=int, default=1, help="Number of GPUs for vLLM"
+    )
+    parser.add_argument("--gpu-memory-utilization", type=float, default=0.8)
+    parser.add_argument("--max-model-len", type=int, default=2048)
+    parser.add_argument("--max-new-tokens", type=int, default=512)
+    parser.add_argument("--language", type=str, default="中文", help="Language hint")
+    parser.add_argument("--hotwords", type=str, nargs="*", default=[], help="Hotwords list")
+    parser.add_argument("--no-itn", action="store_true", help="Disable inverse text normalization")
+    parser.add_argument("--batch-size", type=int, default=16, help="Batch size for inference")
+    parser.add_argument("--output", type=str, default=None, help="Output file for results")
+    args = parser.parse_args()
+
+    from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+    print(f"=" * 60)
+    print(f"Fun-ASR-Nano vLLM Inference")
+    print(f"=" * 60)
+    print(f"  Model: {args.model_dir}")
+    print(f"  Tensor Parallel: {args.tensor_parallel_size} GPU(s)")
+    print(f"  Dtype: {args.dtype}")
+    print(f"  Language: {args.language}")
+    print(f"  Hotwords: {args.hotwords or '(none)'}")
+    print()
+
+    t_load = time.perf_counter()
+    engine = FunASRNanoVLLM.from_pretrained(
+        model=args.model_dir,
+        hub=args.hub,
+        device=args.device,
+        dtype=args.dtype,
+        tensor_parallel_size=args.tensor_parallel_size,
+        gpu_memory_utilization=args.gpu_memory_utilization,
+        max_model_len=args.max_model_len,
+    )
+    print(f"Model loaded in {time.perf_counter() - t_load:.1f}s\n")
+
+    # Determine input files
+    if args.input is None:
+        # Use default example audio
+        example_dir = os.path.join(engine.model_dir, "example")
+        if os.path.isdir(example_dir):
+            wav_files = [
+                os.path.join(example_dir, f)
+                for f in sorted(os.listdir(example_dir))
+                if f.endswith((".wav", ".mp3", ".flac"))
+            ]
+        else:
+            print("No --input specified and no example/ directory found.")
+            print("Usage: python demo_vllm.py --input <audio_file_or_scp>")
+            return
+        if not wav_files:
+            print("No audio files found in example/ directory.")
+            return
+        audio_files = wav_files
+        print(f"Using example audio: {audio_files}")
+    elif args.input.endswith(".scp"):
+        audio_files = []
+        with open(args.input, "r") as f:
+            for line in f:
+                parts = line.strip().split(maxsplit=1)
+                if len(parts) == 2:
+                    audio_files.append(parts[1])
+                elif len(parts) == 1:
+                    audio_files.append(parts[0])
+        print(f"Loaded {len(audio_files)} files from {args.input}")
+    elif args.input.endswith(".jsonl"):
+        import json
+
+        audio_files = []
+        with open(args.input, "r") as f:
+            for line in f:
+                item = json.loads(line.strip())
+                audio_files.append(item["source"])
+        print(f"Loaded {len(audio_files)} files from {args.input}")
+    else:
+        audio_files = [args.input]
+
+    # Run inference in batches
+    all_results = []
+    total_audio_time = 0
+    total_infer_time = 0
+
+    print(f"\nProcessing {len(audio_files)} audio file(s)...")
+    for i in range(0, len(audio_files), args.batch_size):
+        batch = audio_files[i : i + args.batch_size]
+        t0 = time.perf_counter()
+        results = engine.generate(
+            inputs=batch,
+            hotwords=args.hotwords if args.hotwords else None,
+            language=args.language,
+            itn=not args.no_itn,
+            max_new_tokens=args.max_new_tokens,
+        )
+        t1 = time.perf_counter()
+        batch_time = t1 - t0
+        total_infer_time += batch_time
+        all_results.extend(results)
+
+        batch_num = i // args.batch_size + 1
+        total_batches = (len(audio_files) + args.batch_size - 1) // args.batch_size
+        print(f"  Batch {batch_num}/{total_batches}: {len(batch)} files in {batch_time:.2f}s")
+
+    # Print results
+    print(f"\n{'=' * 60}")
+    print(f"Results: {len(all_results)} samples, total inference time: {total_infer_time:.2f}s")
+    print(f"{'=' * 60}")
+    for r in all_results:
+        print(f"\n[{r['key']}]")
+        print(f"  Text: {r['text']}")
+        if "timestamps" in r and r["timestamps"]:
+            ts_preview = r["timestamps"][:5]
+            ts_str = " | ".join(
+                [f"{t['token']}({t['start_time']:.2f}-{t['end_time']:.2f}s)" for t in ts_preview]
+            )
+            if len(r["timestamps"]) > 5:
+                ts_str += f" ... ({len(r['timestamps'])} total)"
+            print(f"  Timestamps: {ts_str}")
+
+    # Save results to file
+    if args.output:
+        import json
+
+        with open(args.output, "w", encoding="utf-8") as f:
+            for r in all_results:
+                # Remove non-serializable fields
+                out = {k: v for k, v in r.items() if k != "timestamps"}
+                if "timestamps" in r:
+                    out["timestamps"] = r["timestamps"]
+                f.write(json.dumps(out, ensure_ascii=False) + "\n")
+        print(f"\nResults saved to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

examples/industrial_data_pretraining/fun_asr_nano/docs/finetune.md

@@ -0,0 +1,114 @@
+# Finetune
+
+「[简体中文](fintune_zh.md)」|「English」
+
+## Requirements
+
+```
+pip install funasr>=1.3.0
+```
+
+## Data Prepare
+
+Data examples
+
+```
+head -n1 data/train_example.jsonl | jq
+
+{
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    },
+    {
+      "role": "user",
+      "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0002.wav<|endofspeech|>"
+    },
+    {
+      "role": "assistant",
+      "content": "几点了？"
+    }
+  ],
+  "speech_length": 145,
+  "text_length": 3
+}
+```
+
+Full ref to `data/train_example.jsonl`
+
+Description：
+
+- The content of systemis fixed as `You are a helpful assistant.`
+- The content of userincludes the prompt and the path to the audio file (enclosed between `<|startofspeech|>!`and `<|endofspeech|>`).
+  - The default prompts are `语音转写：`and `Speech transcription: `.
+  - For corresponding languages, prompts can be combined, such as `语音转写成英文：`and `Transcribe speech into Chinese: `.
+  - When the text annotation corresponding to the audio file contains no Arabic numerals or punctuation marks, you can use `语音转写，不进行文本规整：`and `Speech transcription without text normalization: `.
+- The content of assistant corresponds to the text annotation of the audio file.
+- speech_length: The number of fbank frames of the audio file (10ms per frame).
+- text_length: The number of tokens in the annotation text of the audio file (encoded using `Qwen/Qwen3-0.6B`).
+
+- `messages[2]["content"]`: transcription
+- `speech_length`: number of fbank frames of the audio file
+- `text_length`: number of tokens of the transcription (tokenized by `Qwen3-0.6B`)
+
+We provide a data format conversion tool `scp2jsonl.py`, which can convert common speech recognition training data formats such as wav scp and transcription into the ChatML format.
+
+`train_wav.scp`
+
+```
+BAC009S0764W0121 https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/BAC009S0764W0121.wav
+BAC009S0916W0489 https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/BAC009S0916W0489.wav
+```
+
+`train_text.txt`
+
+```
+BAC009S0764W0121 甚至出现交易几乎停滞的情况
+BAC009S0916W0489 湖北一公司以员工名义贷款数十员工负债千万
+```
+
+```
+python tools/scp2jsonl.py \
+  ++scp_file=data/train_wav.scp \
+  ++transcript_file=data/train_text.txt \
+  ++jsonl_file=data/train_example.jsonl
+```
+
+## Finetune
+
+Modify the `audio_encoder_conf.freeze`, `audio_adaptor_conf.freeze`, and `llm_conf.freeze` in `finetune.sh`.
+
+Set the `freeze` parameter of the modules to be fine-tuned to false(by default, only the LLM is fine-tuned).
+
+For more detailed parameters, refer to: [SenseVoice Model Training and Testing](https://github.com/modelscope/FunASR/blob/main/docs/tutorial/README.md#Model%20Training%20and%20Testing)
+
+```
+bash finetune.sh
+```
+
+### Recommended Configuration
+
+- For training data less than 1000 hours, it is recommended to fine-tune the audio_adaptor.
+- For training data less than 5000 hours, it is recommended to fine-tune the audio_encoder and audio_adaptor.
+- For training data greater than 10000 hours, it is recommended to perform full-parameter fine-tuning.
+
+## Model Evaluation
+
+After model fine-tuning is completed, you can decode the model using the decode.py script:
+
+```
+python decode.py \
+  ++model_dir=/path/to/finetuned \
+  ++scp_file=data/val_wav.scp \
+  ++output_file=output.txt
+```
+
+After decoding is completed, text inverse normalization needs to be applied to the annotations and recognition results, and then the WER should be calculated:
+
+```
+python tools/whisper_mix_normalize.py data/val_text.txt data/val_norm.txt
+python tools/whisper_mix_normalize.py output.txt output_norm.txt
+compute-wer data/val_norm.txt output_norm.txt cer.txt
+tail -n8 cer.txt
+```

examples/industrial_data_pretraining/fun_asr_nano/docs/fintune_zh.md

@@ -0,0 +1,114 @@
+# 微调
+
+「简体中文」|「[English](finetune.md)」
+
+## 安装训练环境
+
+```
+pip install funasr>=1.3.0
+```
+
+## 数据准备
+
+数据格式需要包括如下几个字段：
+
+```
+head -n1 data/train_example.jsonl | jq
+
+{
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    },
+    {
+      "role": "user",
+      "content": "语音转写：<|startofspeech|>!https://modelscope.cn/datasets/FunAudioLLM/funasr-demo/resolve/master/audios/IT0011W0002.wav<|endofspeech|>"
+    },
+    {
+      "role": "assistant",
+      "content": "几点了？"
+    }
+  ],
+  "speech_length": 145,
+  "text_length": 3
+}
+```
+
+详细可以参考：`data/train_example.jsonl`
+
+数据准备细节介绍：
+
+- system 的 content 固定为 `You are a helpful assistant.`
+- user 的 content 包含了 prompt 和音频文件的路径（位于 `<|startofspeech|>!` 和 `<|endofspeech|>`之间）
+  - prompt 默认为`语音转写：`和`Speech transcription: `
+  - 可以结合对应的语种为`语音转写成英文：`和`Transcribe speech into Chinese: `
+  - 当音频文件对应的文本标注不含阿拉伯数字或者标点符号时，可以使用`语音转写，不进行文本规整：`和 `Speech transcription without text normalization: `
+- assistant 的 content 对应音频文件对应的文本标注
+- speech_length：音频文件的 fbank 帧数（一帧 10ms）
+- text_length：音频文件标注文本的 token 数 (用 `Qwen/Qwen3-0.6B` 编码)
+
+我们提供了数据格式转换工具 `scp2jsonl.py`，可以将常见的语音识别训练数据格式 wav scp 和 transcription 转成 ChatML 格式。
+
+`train_wav.scp`
+
+左边为数据唯一 ID，需与 `train_text.txt` 中的 ID 一一对应 右边为音频文件的路径，格式如下
+
+```
+BAC009S0764W0121 https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/BAC009S0764W0121.wav
+BAC009S0916W0489 https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/BAC009S0916W0489.wav
+```
+
+`train_text.txt`
+
+左边为数据唯一 ID，需与 `train_wav.scp` 中的 ID 一一对应 右边为音频文件标注文本，格式如下：
+
+```
+BAC009S0764W0121 甚至出现交易几乎停滞的情况
+BAC009S0916W0489 湖北一公司以员工名义贷款数十员工负债千万
+```
+
+```
+python tools/scp2jsonl.py \
+  ++scp_file=data/train_wav.scp \
+  ++transcript_file=data/train_text.txt \
+  ++jsonl_file=data/train_example.jsonl
+```
+
+## 启动训练
+
+修改 `finetune.sh` 中的 `audio_encoder_conf.freeze`, `audio_adaptor_conf.freeze` 和 `llm_conf.freeze`。
+
+将需要微调的模块 `freeze` 设置成 `false`（默认只微调 llm）。
+
+更多参数细节参考：[SenseVoice 模型训练与测试](https://github.com/modelscope/FunASR/blob/main/docs/tutorial/README_zh.md#%E6%A8%A1%E5%9E%8B%E8%AE%AD%E7%BB%83%E4%B8%8E%E6%B5%8B%E8%AF%95)
+
+```
+bash finetune.sh
+```
+
+### 推荐配置
+
+- 训练数据少于 1000 小时，建议微调 audio_adaptor
+- 训练数据少于 5000 小时，建议微调 audio_encoder 和 audio_adaptor
+- 训练数据大于 10000 小时，建议全量参数微调
+
+## 模型评测
+
+当模型微调结束后，可以使用 decode.py 脚本对模型进行解码：
+
+```
+python decode.py \
+  ++model_dir=/path/to/finetuned \
+  ++scp_file=data/val_wav.scp \
+  ++output_file=output.txt
+```
+
+解码结束后，需要对标注和识别结果做文本逆归一化，然后计算 WER：
+
+```
+python tools/whisper_mix_normalize.py data/val_text.txt data/val_norm.txt
+python tools/whisper_mix_normalize.py output.txt output_norm.txt
+compute-wer data/val_norm.txt output_norm.txt cer.txt
+tail -n8 cer.txt
+```

examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md

@@ -0,0 +1,63 @@
+# Fun-ASR-Nano 实时 WebSocket 服务 — 快速上手
+
+> 完整文档请参见：[FunASR vLLM 推理引擎指南](../../../../docs/vllm_guide.md)
+
+## 30 秒启动
+
+```bash
+cd examples/industrial_data_pretraining/fun_asr_nano
+
+# 安装依赖
+pip install -r requirements.txt
+# 按主文档选择并安装与当前 NVIDIA 驱动、CUDA runtime 和 PyTorch wheel 匹配的 vLLM 版本。
+
+# 启动服务
+CUDA_VISIBLE_DEVICES=0 python serve_realtime_ws.py --port 10095 --language 中文
+```
+
+## 客户端
+
+```bash
+# 浏览器
+open client_mic.html
+
+# Python 麦克风
+python client_python.py --server ws://localhost:10095 --mic
+
+# Python 文件
+python client_python.py --server ws://localhost:10095 --file audio.wav
+
+# 自动化测试
+python client_test.py --server ws://localhost:10095 --file audio.wav
+```
+
+## 远程访问
+
+```bash
+ssh -L 10095:localhost:10095 <server>
+# 然后本地打开 client_mic.html
+```
+
+## 功能
+
+- **vLLM 推理引擎**：RTF < 0.08，支持 tensor parallel 多卡加速
+- **流式 VAD 分句**：动态静音阈值，自然断句
+- **说话人分离 (Beta)**：流式 ID 分配 + 最终重聚类
+- **热词定制化**：加载人名、地名等实体词文件
+- **语种指定**：31 种语言 + 中文方言
+- **幻觉检测**：自动检测重复模式并截断
+
+## 文件列表
+
+| 文件 | 说明 |
+|------|------|
+| `serve_realtime_ws.py` | WebSocket 服务端 |
+| `client_mic.html` | 浏览器客户端 |
+| `client_python.py` | Python CLI 客户端 |
+| `client_test.py` | 自动化测试脚本 |
+| `热词列表` | 热词文件示例 |
+| `demo_vllm.py` | 离线 vLLM 推理 demo |
+
+## 详细文档
+
+- [FunASR vLLM 推理引擎指南](../../../../docs/vllm_guide.md) — 完整文档（离线/流式/WebSocket/API/FAQ）

examples/industrial_data_pretraining/fun_asr_nano/finetune.sh

@@ -0,0 +1,65 @@
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+workspace=`pwd`
+
+# which gpu to train or finetune
+export CUDA_VISIBLE_DEVICES="0"
+gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+
+# model_name from model_hub, or model_dir in local path
+model_name_or_model_dir="FunAudioLLM/Fun-ASR-Nano-2512"
+
+# data dir, which contains: train.json, val.json
+train_data=${workspace}/data/train_example.jsonl
+val_data=${workspace}/data/val_example.jsonl
+
+# exp output dir
+output_dir="./outputs"
+log_file="${output_dir}/log.txt"
+
+deepspeed_config=${workspace}/deepspeed_conf/ds_stage1.json
+
+mkdir -p ${output_dir}
+echo "log_file: ${log_file}"
+
+DISTRIBUTED_ARGS="
+    --nnodes ${WORLD_SIZE:-1} \
+    --nproc_per_node $gpu_num \
+    --node_rank ${RANK:-0} \
+    --master_addr ${MASTER_ADDR:-127.0.0.1} \
+    --master_port ${MASTER_PORT:-26669}
+"
+echo $DISTRIBUTED_ARGS
+
+# funasr trainer path
+train_tool=`which funasr-train-ds`
+echo "Using funasr trainer: ${train_tool}"
+
+torchrun $DISTRIBUTED_ARGS \
+${train_tool} \
+++model="${model_name_or_model_dir}" \
+++trust_remote_code=true \
+++train_data_set_list="${train_data}" \
+++valid_data_set_list="${val_data}" \
+++dataset_conf.data_split_num=1 \
+++dataset_conf.batch_sampler="BatchSampler" \
+++dataset_conf.batch_size=6000  \
+++dataset_conf.sort_size=1024 \
+++dataset_conf.batch_type="token" \
+++dataset_conf.num_workers=4 \
+++train_conf.max_epoch=50 \
+++train_conf.log_interval=1 \
+++train_conf.resume=true \
+++train_conf.validate_interval=2000 \
+++train_conf.save_checkpoint_interval=2000 \
+++train_conf.effective_save_name_excludes="None" \
+++train_conf.keep_nbest_models=20 \
+++train_conf.avg_nbest_model=10 \
+++train_conf.use_deepspeed=false \
+++train_conf.deepspeed_config=${deepspeed_config} \
+++optim_conf.lr=0.0002 \
+++audio_encoder_conf.freeze=true \
+++audio_adaptor_conf.freeze=true \
+++llm_conf.freeze=false \
+++output_dir="${output_dir}" &> ${log_file}

examples/industrial_data_pretraining/fun_asr_nano/model.py

@@ -0,0 +1,748 @@
+import logging
+import os
+import random
+import re
+import string
+import time
+import traceback
+from typing import Union
+
+import torch
+import torch.nn as nn
+
+from funasr.metrics.compute_acc import compute_accuracy
+from funasr.register import tables
+from funasr.train_utils.device_funcs import force_gatherable, to_device
+from funasr.utils.datadir_writer import DatadirWriter
+from funasr.utils.load_utils import extract_fbank, load_audio_text_image_video
+from funasr.models.fun_asr_nano.device_utils import resolve_autocast_device_type
+
+
+from ctc import CTC
+from tools.utils import forced_align
+
+dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
+
+
+@tables.register("model_classes", "FunASRNano")
+class FunASRNano(nn.Module):
+    def __init__(
+        self,
+        audio_encoder: str = None,
+        audio_encoder_conf: dict = None,
+        audio_adaptor: str = None,
+        audio_adaptor_conf: dict = None,
+        llm: str = None,
+        llm_conf: dict = None,
+        input_size: int = 80,
+        length_normalized_loss: bool = False,
+        **kwargs,
+    ):
+        super().__init__()
+
+        # audio encoder
+        hub = audio_encoder_conf.get("hub", None)
+        self.audio_encoder_activation_checkpoint = audio_encoder_conf.get(
+            "activation_checkpoint", False
+        )
+        if hub == "ms":
+            from funasr import AutoModel
+
+            model = AutoModel(model=audio_encoder, model_revision="master")
+            audio_encoder_output_size = (
+                model.model.encoder_output_size
+                if hasattr(model.model, "encoder_output_size")
+                else -1
+            )
+            audio_encoder = (
+                model.model.model.encoder if hasattr(model.model, "model") else model.model.encoder
+            )
+        else:
+            encoder_class = tables.encoder_classes.get(audio_encoder)
+            audio_encoder = encoder_class(input_size=input_size, **audio_encoder_conf)
+            audio_encoder_output_size = audio_encoder.output_size()
+        freeze = audio_encoder_conf.get("freeze", True)
+
+        if freeze:
+            for _, param in audio_encoder.named_parameters():
+                param.requires_grad = False
+            audio_encoder.eval()
+        self.audio_encoder = audio_encoder
+
+        # llm
+        self.llm = None
+        init_param_path = llm_conf.get("init_param_path", None)
+        llm_dim = None
+        from transformers import AutoConfig, AutoModelForCausalLM
+
+        llm_load_kwargs = llm_conf.get("load_kwargs", {})
+        config = AutoConfig.from_pretrained(init_param_path)
+        model = AutoModelForCausalLM.from_config(config, **llm_load_kwargs)
+
+        freeze = llm_conf.get("freeze", True)
+        if freeze:
+            for _, param in model.named_parameters():
+                param.requires_grad = False
+            model.eval()
+        if llm_conf.get("activation_checkpoint", False):
+            model.gradient_checkpointing_enable()
+
+        self.llm_dtype = llm_conf.get("llm_dtype", "fp32")
+        self.llm = model.to(dtype_map[self.llm_dtype])
+        llm_dim = model.get_input_embeddings().weight.shape[-1]
+
+        # adaptor
+        adaptor_class = tables.adaptor_classes.get(audio_adaptor)
+        if audio_encoder_output_size > 0:
+            audio_adaptor_conf["encoder_dim"] = audio_encoder_output_size
+        audio_adaptor_conf["llm_dim"] = (
+            llm_dim if llm_dim is not None else audio_adaptor_conf["llm_dim"]
+        )
+        audio_adaptor = adaptor_class(**audio_adaptor_conf)
+        freeze = audio_adaptor_conf.get("freeze", False)
+        if freeze:
+            for _, param in audio_adaptor.named_parameters():
+                param.requires_grad = False
+            audio_adaptor.eval()
+        self.audio_adaptor = audio_adaptor
+        self.use_low_frame_rate = audio_adaptor_conf.get("use_low_frame_rate", False)
+
+        # ctc decoder
+        self.ctc_decoder = None
+        # TODO: fix table name
+        ctc_decoder_class = tables.adaptor_classes.get(kwargs.get("ctc_decoder", None))
+        if ctc_decoder_class is not None:
+            ctc_tokenizer = (
+                kwargs.get("ctc_tokenizer", None)
+                if "ctc_tokenizer" in kwargs
+                else kwargs["dataset_conf"]["ctc_tokenizer"]
+            )
+            ctc_tokenizer_conf = (
+                kwargs.get("ctc_tokenizer_conf", None)
+                if "ctc_tokenizer_conf" in kwargs
+                else kwargs["dataset_conf"]["ctc_tokenizer_conf"]
+            )
+            if ctc_tokenizer is not None and ctc_tokenizer_conf is not None:
+                ctc_tokenizer_class = tables.tokenizer_classes.get(ctc_tokenizer)
+                ctc_tokenizer = ctc_tokenizer_class(**ctc_tokenizer_conf)
+                self.ctc_tokenizer = ctc_tokenizer
+            assert ctc_tokenizer is not None, f"ctc_tokenizer must be set"
+
+            ctc_vocab_size = kwargs.get("ctc_vocab_size", 60515)
+            ctc_decoder_conf = kwargs.get("ctc_decoder_conf", {})
+            if audio_encoder_output_size > 0:
+                ctc_decoder_conf["encoder_dim"] = audio_encoder_output_size
+            self.ctc_decoder = ctc_decoder_class(**ctc_decoder_conf)
+            init_param_path = ctc_decoder_conf.get("init_param_path", None)
+            if init_param_path is not None:
+                src_state = torch.load(init_param_path, map_location="cpu")
+                flag = self.ctc_decoder.load_state_dict(src_state, strict=False)
+                logging.info(f"Loading ctc_decoder ckpt: {init_param_path}, status: {flag}")
+            freeze = ctc_decoder_conf.get("freeze", False)
+            if freeze:
+                for _, param in self.ctc_decoder.named_parameters():
+                    param.requires_grad = False
+                self.ctc_decoder.eval()
+
+            ctc_conf = kwargs.get("ctc_conf", {})
+            self.blank_id = ctc_conf.get("blank_id", ctc_vocab_size - 1)
+            self.ctc_weight = kwargs.get("ctc_weight", 0.3)
+            self.ctc = CTC(
+                odim=ctc_vocab_size,
+                encoder_output_size=audio_encoder_output_size,
+                blank_id=self.blank_id,
+                **ctc_conf,
+            )
+            self.detach_ctc_decoder = kwargs.get("detach_ctc_decoder", True)
+            self.error_calculator = None
+
+        self.length_normalized_loss = length_normalized_loss
+        rank = int(os.environ.get("RANK", 0))
+        logging.info(f"rank: {rank}, model is builded.")
+
+    def forward(
+        self,
+        speech: torch.Tensor = None,
+        speech_lengths: torch.Tensor = None,
+        input_ids: torch.Tensor = None,
+        attention_mask: torch.Tensor = None,
+        labels_ids: torch.Tensor = None,
+        fbank_beg: torch.Tensor = None,
+        fbank_mask: torch.Tensor = None,
+        **kwargs,
+    ):
+        batch_size, token_num = input_ids.shape
+        stats = {}
+        input_ids[input_ids < 0] = 0
+        inputs_embeds = self.llm.model.get_input_embeddings()(input_ids)
+        if speech is not None:
+            if len(speech_lengths.size()) > 1:
+                speech_lengths = speech_lengths[:, 0]
+            batch_size_speech, frames, _ = speech.shape
+
+            # audio encoder
+            if self.audio_encoder_activation_checkpoint:
+                from torch.utils.checkpoint import checkpoint
+
+                encoder_out, encoder_out_lens = checkpoint(
+                    self.encode, speech, speech_lengths, use_reentrant=False
+                )
+            else:
+                encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
+
+            # audio_adaptor
+            encoder_out, encoder_out_lens = self.audio_adaptor(encoder_out, encoder_out_lens)
+
+            batch_size, token_num, dims = inputs_embeds.shape
+            fake_token_len = kwargs.get("fake_token_len")
+            fake_token_len[fake_token_len < 0] = 0
+            fbank_beg[fbank_beg < 0] = 0
+
+            speech_idx = 0
+            for batch_idx in range(batch_size):
+                for turn_id in range(fbank_beg.shape[1]):
+                    fbank_beg_idx = fbank_beg[batch_idx, turn_id].item()
+                    if fbank_beg_idx > 0:
+                        speech_token_len = fake_token_len[batch_idx, turn_id]
+                        speech_token = encoder_out[speech_idx, :speech_token_len, :]
+
+                        try:
+                            inputs_embeds[
+                                batch_idx,
+                                fbank_beg_idx : fbank_beg_idx + speech_token_len,
+                                :,
+                            ] = speech_token
+                        except Exception as e:
+                            logging.error(f"{str(e)}, {traceback.format_exc()}")
+                            logging.info(
+                                f"batch_idx: {batch_idx}, inputs_embeds: {inputs_embeds.shape}, fbank_beg_idx: {fbank_beg_idx}, speech_token_len: {speech_token_len}, encoder_out: {encoder_out.shape}, encoder_out_lens: {encoder_out_lens}, fake_token_len: {fake_token_len}, speech_lengths: {speech_lengths}"
+                            )
+                            speech_token_len = encoder_out_lens[speech_idx].item()
+                            speech_token = encoder_out[speech_idx, :speech_token_len, :]
+                            inputs_embeds[
+                                batch_idx,
+                                fbank_beg_idx : fbank_beg_idx + speech_token_len,
+                                :,
+                            ] = speech_token
+
+                        speech_idx += 1
+
+            stats["batch_size_speech"] = batch_size_speech
+            stats["batch_size_x_frames"] = frames * batch_size_speech
+            stats["batch_size_real_frames"] = speech_lengths.sum().item()
+            stats["padding_frames"] = stats["batch_size_x_frames"] - stats["batch_size_real_frames"]
+
+        autocast_device_type = resolve_autocast_device_type(next(self.parameters()).device)
+        with torch.autocast(
+            device_type=autocast_device_type,
+            enabled=True if self.llm_dtype != "fp32" else False,
+            dtype=dtype_map[self.llm_dtype],
+        ):
+            labels_ids[labels_ids == -1] = -100
+            attention_mask[attention_mask < 0] = 0
+            model_outputs = self.llm(
+                inputs_embeds=inputs_embeds.to(dtype_map[self.llm_dtype]),
+                attention_mask=attention_mask,
+                labels=labels_ids,
+            )
+            loss = model_outputs.loss
+
+        with torch.no_grad():
+            preds = torch.argmax(model_outputs.logits, -1)
+            acc_att = compute_accuracy(preds[:, :-1], labels_ids[:, 1:], ignore_label=-100)
+            stats["acc"] = acc_att
+
+        stats["loss"] = torch.clone(loss.detach())
+        stats["batch_size"] = batch_size
+
+        stats["batch_size_x_tokens"] = token_num * batch_size
+        stats["batch_size_real_tokens"] = attention_mask.sum().item()
+        stats["padding_tokens"] = stats["batch_size_x_tokens"] - stats["batch_size_real_tokens"]
+
+        dialog_turns = (fbank_beg > 0).sum(-1)
+        dialog_turns_max = torch.max(dialog_turns).int().item()
+        dialog_turns_avg = dialog_turns.sum().item() / batch_size
+        stats["dialog_turns_max"] = dialog_turns_max
+        stats["dialog_turns_avg"] = dialog_turns_avg
+
+        # force_gatherable: to-device and to-tensor if scalar for DataParallel
+        if self.length_normalized_loss:
+            batch_size = int((labels_ids > 0 + 1).sum())
+        loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)
+        return loss, stats, weight
+
+    def forward_export(self, speech, speech_lengths, **kwargs):
+        x, olens = self.audio_encoder(speech, speech_lengths)
+        encoder_out, encoder_out_lens = self.audio_adaptor(x, olens)
+        return encoder_out, encoder_out_lens
+
+    def encode(self, speech, speech_lengths):
+        # audio encoder
+        encoder_out, encoder_out_lens = self.audio_encoder(speech, speech_lengths)
+
+        return encoder_out, encoder_out_lens
+
+    def data_template(self, data):
+        system, user, assistant = [], [], []
+        for i, item in enumerate(data):
+            role = item["role"]
+            content = item["content"]
+            if role == "system":
+                system.append(content)
+            elif role == "user":
+                if "audio" in item:
+                    audio = item["audio"]
+                    content = [content, audio]
+                user.append(content)
+            elif role == "assistant":
+                assistant.append(content)
+
+        system = system * len(user)
+
+        contents = {
+            "system": system,
+            "user": user,
+            "assistant": assistant,
+        }
+
+        return contents
+
+    def data_load_speech(self, contents: dict, tokenizer, frontend, meta_data={}, **kwargs):
+        system = contents["system"]
+        user = contents["user"]
+        assistant = contents["assistant"]
+        pattern = re.compile(r"(<\|startofspeech\|>.*?<\|endofspeech\|>)")
+        do_think = True
+        sys_prompt = True
+        if "dataset_conf" in kwargs:
+            do_think = kwargs["dataset_conf"].get("do_think", True)
+            sys_prompt = kwargs["dataset_conf"].get("sys_prompt", True)
+
+        input_ids, labels, fbank, fbank_lens, fbank_mask, fbank_beg, fake_token_len = (
+            [],
+            [],
+            [],
+            [],
+            [],
+            [],
+            [],
+        )
+        input_source_ids = []
+        for i, (system_prompt, user_prompt, target_out) in enumerate(zip(system, user, assistant)):
+            if i >= kwargs.get("multiturn_num_max", 5):
+                break
+            if len(input_ids) > kwargs.get("max_token_length", 1500):
+                break
+            if isinstance(user_prompt, (list, tuple)):
+                user_prompt, audio = user_prompt
+            if i == 0:
+                if kwargs.get("infer_with_assistant_input", False):
+                    source_input = f"<|im_start|>system\n{system_prompt}<|im_end|>\n<|im_start|>user\n{user_prompt}"
+                    if not sys_prompt:
+                        source_input = f"<|im_start|>user\n{user_prompt}"
+                else:
+                    source_input = f"<|im_start|>system\n{system_prompt}<|im_end|>\n<|im_start|>user\n{user_prompt}<|im_end|>\n<|im_start|>assistant\n"
+                    if not sys_prompt:
+                        source_input = (
+                            f"<|im_start|>user\n{user_prompt}<|im_end|>\n<|im_start|>assistant\n"
+                        )
+            else:
+                if kwargs.get("infer_with_assistant_input", False):
+                    source_input = f"<|im_start|>user\n{user_prompt}"
+                else:
+                    source_input = (
+                        f"<|im_start|>user\n{user_prompt}<|im_end|>\n<|im_start|>assistant\n"
+                    )
+            if not do_think:
+                source_input += "<think>\n\n</think>\n\n"
+            if kwargs.get("prev_text", None) is not None:
+                source_input += kwargs["prev_text"]
+
+            splits = pattern.split(source_input)
+            source_ids = []
+            fbank_mask_i = []
+            fake_token_len_i = 0
+            fbank_beg_i = -1
+            speech, speech_lengths = [], []
+            for k, sub_str in enumerate(splits):
+                if not sub_str.startswith("<|startofspeech|>"):
+                    sub_token = tokenizer.encode(sub_str)
+                    source_ids += sub_token
+                    fbank_mask_i += [0] * len(sub_token)
+                else:
+                    sub_str = sub_str.replace("<|startofspeech|>", "").replace(
+                        "<|endofspeech|>", ""
+                    )
+                    if sub_str.startswith("!"):
+                        sub_str = sub_str[1:]
+                        if sub_str.startswith("!"):  # !!: audio sample point
+                            sub_str = audio
+                        try:
+                            time1 = time.perf_counter()
+                            data_src = load_audio_text_image_video(
+                                sub_str, fs=frontend.fs, **kwargs
+                            )
+                            time2 = time.perf_counter()
+                            meta_data["load_data"] = f"{time2 - time1:0.3f}"
+                        except Exception as e:
+                            logging.error(f"Loading wav failed! {str(e)}, {traceback.format_exc()}")
+
+                        speech, speech_lengths = extract_fbank(
+                            data_src,
+                            data_type=kwargs.get("data_type", "sound"),
+                            frontend=frontend,
+                            is_final=True,
+                        )  # speech: [b, T, d]
+
+                        time3 = time.perf_counter()
+                        meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
+                        meta_data["batch_data_time"] = (
+                            speech_lengths.sum().item()
+                            * frontend.frame_shift
+                            * frontend.lfr_n
+                            / 1000
+                        )
+
+                        if self.use_low_frame_rate:
+                            olens = 1 + (speech_lengths[0].item() - 3 + 2 * 1) // 2
+                            olens = 1 + (olens - 3 + 2 * 1) // 2
+                            fake_token_len_i = (olens - 1) // 2 + 1
+                        else:
+                            fake_token_len_i = speech_lengths[0].item()
+                        fake_token = [0] * fake_token_len_i
+                        fbank_beg_i = len(source_ids)
+                        source_ids += fake_token
+                        fbank_mask_i += [1] * len(fake_token)
+
+            fbank_beg += [fbank_beg_i + len(input_ids)]
+            fake_token_len += [fake_token_len_i]
+            source_mask = [-100] * len(source_ids)
+            target_out = f"{target_out}<|im_end|>"
+            target_ids = tokenizer.encode(target_out)
+            input_source_ids = input_ids + source_ids
+            input_ids += source_ids + target_ids
+            labels += source_mask + target_ids
+            fbank_mask += fbank_mask_i
+            if len(speech) > 0:
+                fbank.append(speech[0, :, :])
+                fbank_lens.append(speech_lengths)
+
+        input_ids = torch.tensor(input_ids, dtype=torch.int64)  # [: self.max_token_length]
+        attention_mask = torch.tensor([1] * len(input_ids), dtype=torch.int32)
+        labels = torch.tensor(labels, dtype=torch.int64)  # [: self.max_token_length]
+
+        fbank_mask = torch.tensor(fbank_mask, dtype=torch.float32)
+        fbank_beg = torch.tensor(fbank_beg, dtype=torch.int32)
+        fake_token_len = torch.tensor(fake_token_len, dtype=torch.int32)
+        source_ids = torch.tensor(input_source_ids, dtype=torch.int64)
+        target_ids = torch.tensor(target_ids, dtype=torch.int64)
+
+        if len(fbank) > 0:
+            speech = torch.nn.utils.rnn.pad_sequence(fbank, batch_first=True, padding_value=0.0)
+            speech_lengths = torch.nn.utils.rnn.pad_sequence(
+                fbank_lens, batch_first=True, padding_value=-1
+            )
+        else:
+            speech = []
+            speech_lengths = []
+        output = {
+            "speech": speech,
+            "speech_lengths": speech_lengths,
+            "fbank_mask": fbank_mask[None, :],
+            "fbank_beg": fbank_beg[None,],
+            "fake_token_len": fake_token_len[None, :],
+            "input_ids": input_ids[None,],
+            "attention_mask": attention_mask[None,],
+            "labels_ids": labels,
+            "source_ids": source_ids[None, :],
+            "target_ids": target_ids[None, :],
+        }
+
+        return output
+
+    def inference_prepare(
+        self,
+        data_in,
+        data_lengths=None,
+        key: list = None,
+        tokenizer=None,
+        frontend=None,
+        **kwargs,
+    ):
+        meta_data = {}
+
+        if isinstance(data_in, (list, tuple)) and len(data_in) > 1:
+            raise NotImplementedError("batch decoding is not implemented")
+
+        contents = self.data_template(data_in[0])
+        output = self.data_load_speech(contents, tokenizer, frontend, meta_data=meta_data, **kwargs)
+        batch = to_device(output, kwargs["device"])
+
+        # audio encoder
+        speech = batch["speech"]
+
+        if len(speech) > 0:
+            if "audio_embedding" in kwargs and "audio_embedding_lens" in kwargs:
+                encoder_out = kwargs["audio_embedding"]
+                encoder_out_lens = kwargs["audio_embedding_lens"]
+            else:
+                speech_lengths = batch["speech_lengths"][:, 0]
+                # fp16
+                if kwargs.get("fp16", False):
+                    speech = speech.to(torch.float16)
+                elif kwargs.get("bf16", False):
+                    speech = speech.to(torch.bfloat16)
+                # audio encoder
+                encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
+
+                # audio_adaptor
+                adaptor_out, adaptor_out_lens = self.audio_adaptor(encoder_out, encoder_out_lens)
+                meta_data["encoder_out"] = encoder_out
+                meta_data["encoder_out_lens"] = encoder_out_lens
+                meta_data["audio_adaptor_out"] = adaptor_out
+                meta_data["audio_adaptor_out_lens"] = adaptor_out_lens
+
+        input_ids = batch["input_ids"]
+        source_ids = batch["source_ids"]
+        fbank_beg = batch["fbank_beg"]
+        fake_token_len = batch["fake_token_len"]
+
+        if not kwargs.get("teacherforcing", False):
+            input_ids = source_ids
+
+        input_ids[input_ids < 0] = 0
+        inputs_embeds = self.llm.model.get_input_embeddings()(input_ids)
+
+        batch_size, token_num, dims = inputs_embeds.shape
+
+        fake_token_len[fake_token_len < 0] = 0
+        fbank_beg[fbank_beg < 0] = 0
+
+        speech_idx = 0
+        for batch_idx in range(batch_size):
+            for turn_id in range(fbank_beg.shape[1]):
+                fbank_beg_idx = fbank_beg[batch_idx, turn_id].item()
+                if fbank_beg_idx > 0:
+                    speech_token_len = fake_token_len[batch_idx, turn_id]
+                    speech_token = adaptor_out[speech_idx, :speech_token_len, :]
+
+                    try:
+                        inputs_embeds[
+                            batch_idx,
+                            fbank_beg_idx : fbank_beg_idx + speech_token_len,
+                            :,
+                        ] = speech_token
+                    except Exception as e:
+                        #
+                        logging.error(f"{str(e)}, {traceback.format_exc()}")
+                        logging.info(
+                            f"batch_idx: {batch_idx}, inputs_embeds: {inputs_embeds.shape}, fbank_beg_idx: {fbank_beg_idx}, speech_token_len: {speech_token_len}, adaptor_out: {adaptor_out.shape}, adaptor_out_lens: {adaptor_out_lens}, fake_token_len: {fake_token_len}, speech_lengths: {speech_lengths}"
+                        )
+                        speech_token_len = adaptor_out_lens[speech_idx].item()
+                        speech_token = adaptor_out[speech_idx, :speech_token_len, :]
+                        inputs_embeds[
+                            batch_idx,
+                            fbank_beg_idx : fbank_beg_idx + speech_token_len,
+                            :,
+                        ] = speech_token
+
+                    speech_idx += 1
+        return inputs_embeds, contents, batch, source_ids, meta_data
+
+    def get_prompt(self, hotwords: list[str], language: str = None, itn: bool = True):
+        if len(hotwords) > 0:
+            hotwords = ", ".join(hotwords)
+            prompt = f"请结合上下文信息，更加准确地完成语音转写任务。如果没有相关信息，我们会留空。\n\n\n**上下文信息：**\n\n\n"
+            prompt += f"热词列表：[{hotwords}]\n"
+        else:
+            prompt = ""
+        if language is None:
+            prompt += "语音转写"
+        else:
+            prompt += f"语音转写成{language}"
+        if not itn:
+            prompt += "，不进行文本规整"
+        return prompt + "："
+
+    def generate_chatml(self, prompt: str, data: Union[str, torch.Tensor]):
+        if isinstance(data, str):
+            return [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": f"{prompt}<|startofspeech|>!{data}<|endofspeech|>"},
+                {"role": "assistant", "content": "null"},
+            ]
+        elif isinstance(data, torch.Tensor):
+            return [
+                {"role": "system", "content": "You are a helpful assistant."},
+                {
+                    "role": "user",
+                    "content": f"{prompt}<|startofspeech|>!!<|endofspeech|>",
+                    "audio": data,
+                },
+                {"role": "assistant", "content": "null"},
+            ]
+
+    def inference(
+        self,
+        data_in,
+        data_lengths=None,
+        key: list = None,
+        tokenizer=None,
+        frontend=None,
+        **kwargs,
+    ):
+        prompt = self.get_prompt(
+            kwargs.get("hotwords", []), kwargs.get("language", None), kwargs.get("itn", True)
+        )
+        data_in = [self.generate_chatml(prompt, data) for data in data_in]
+
+        if key is None:
+            key = []
+            for _ in data_in:
+                chars = string.ascii_letters + string.digits
+                key.append("rand_key_" + "".join(random.choice(chars) for _ in range(13)))
+
+        return self.inference_llm(
+            data_in,
+            data_lengths=data_lengths,
+            key=key,
+            tokenizer=tokenizer,
+            frontend=frontend,
+            **kwargs,
+        )
+
+    def inference_llm(
+        self,
+        data_in,
+        data_lengths=None,
+        key: list = None,
+        tokenizer=None,
+        frontend=None,
+        **kwargs,
+    ):
+        inputs_embeds, contents, batch, source_ids, meta_data = self.inference_prepare(
+            data_in, data_lengths, key, tokenizer, frontend, **kwargs
+        )
+
+        ctc_results = []
+        if self.ctc_decoder is not None:
+            encoder_out = meta_data["encoder_out"]
+            encoder_out_lens = meta_data["encoder_out_lens"]
+            decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)
+            ctc_logits = self.ctc.log_softmax(decoder_out)
+
+            b, n, d = encoder_out.size()
+            if isinstance(key[0], (list, tuple)):
+                key = key[0]
+            if len(key) < b:
+                key = key * b
+            for i in range(b):
+                x = ctc_logits[i, : encoder_out_lens[i].item(), :]
+                yseq = x.argmax(dim=-1)
+                yseq = torch.unique_consecutive(yseq, dim=-1)
+                mask = yseq != self.blank_id
+                token_int = yseq[mask].tolist()
+                # Change integer-ids to tokens
+                text = self.ctc_tokenizer.decode(token_int)
+                ctc_results.append({"key": key[i], "text": text, "ctc_logits": x})
+
+        llm_dtype = kwargs.get("llm_dtype", "fp32")
+        if llm_dtype == "fp32":
+            llm_dtype = "fp16" if kwargs.get("fp16", False) else llm_dtype
+            llm_dtype = "bf16" if kwargs.get("bf16", False) else llm_dtype
+
+        autocast_device_type = resolve_autocast_device_type(kwargs.get("device", "cuda"))
+        with torch.autocast(
+            device_type=autocast_device_type,
+            enabled=True if llm_dtype != "fp32" else False,
+            dtype=dtype_map[llm_dtype],
+        ):
+            label = contents["assistant"][-1]
+            self.llm = self.llm.to(dtype_map[llm_dtype])
+            inputs_embeds = inputs_embeds.to(dtype_map[llm_dtype])
+            llm_kwargs = kwargs.get("llm_kwargs", {})
+            if not kwargs.get("teacherforcing", False):
+                attention_mask = batch.get("attention_mask", None)
+                generated_ids = self.llm.generate(
+                    inputs_embeds=inputs_embeds,
+                    attention_mask=attention_mask,
+                    max_new_tokens=kwargs.get("max_length", 512),
+                    pad_token_id=self.llm.config.pad_token_id or self.llm.config.eos_token_id,
+                    **llm_kwargs,
+                )
+
+                response = tokenizer.batch_decode(
+                    generated_ids,
+                    skip_special_tokens=kwargs.get("skip_special_tokens", True),
+                )[0]
+
+                loss = None
+            else:
+                labels_ids = batch["labels_ids"]
+                labels_ids[labels_ids == -1] = -100
+                attention_mask = batch.get("attention_mask", None)
+                model_outputs = self.llm(
+                    inputs_embeds=inputs_embeds,
+                    attention_mask=attention_mask,
+                    labels=labels_ids,
+                    pad_token_id=self.llm.config.pad_token_id or self.llm.config.eos_token_id,
+                    **llm_kwargs,
+                )
+
+                preds = torch.argmax(model_outputs.logits, -1)[:, source_ids.shape[1] :]
+                response = tokenizer.batch_decode(
+                    preds,
+                    add_special_tokens=False,
+                    skip_special_tokens=kwargs.get("skip_special_tokens", True),
+                )[0]
+                loss = model_outputs.loss.item()
+        response = kwargs.get("prev_text", "") + response
+
+        ibest_writer = None
+        if kwargs.get("output_dir") is not None:
+            if not hasattr(self, "writer"):
+                self.writer = DatadirWriter(kwargs.get("output_dir"))
+            ibest_writer = self.writer[f"{0 + 1}best_recog"]
+
+        results = []
+        response_clean = re.sub(r"[^\w\s\u3000\u4e00-\u9fff]+", "", response)
+        result_i = {
+            "key": key[0],
+            "text": re.sub(r"\s+", " ", response.replace("/sil", " ")),
+            "text_tn": response_clean,
+            "label": label,
+        }
+        if loss is not None:
+            result_i["loss"] = loss
+        results.append(result_i)
+
+        for ctc_result, result in zip(ctc_results, results):
+            result["ctc_text"] = ctc_result["text"].replace("<|nospeech|>", "")
+            target_ids = torch.tensor(
+                self.ctc_tokenizer.encode(result["ctc_text"]), dtype=torch.int64
+            )
+            result["ctc_timestamps"] = forced_align(
+                ctc_result["ctc_logits"], target_ids, self.blank_id
+            )
+            target_ids = torch.tensor(self.ctc_tokenizer.encode(result["text"]), dtype=torch.int64)
+            result["timestamps"] = forced_align(ctc_result["ctc_logits"], target_ids, self.blank_id)
+            for timestamps in [result["timestamps"], result["ctc_timestamps"]]:
+                for timestamp in timestamps:
+                    timestamp["token"] = self.ctc_tokenizer.decode([timestamp["token"]])
+                    timestamp["start_time"] = timestamp["start_time"] * 6 * 10 / 1000
+                    timestamp["end_time"] = timestamp["end_time"] * 6 * 10 / 1000
+
+        if ibest_writer is not None:
+            ibest_writer["text"][key[0]] = response.replace("\n", " ")
+            ibest_writer["label"][key[0]] = label.replace("\n", " ")
+            ibest_writer["text_tn"][key[0]] = response_clean
+
+        return results, meta_data
+
+    @staticmethod
+    def from_pretrained(model: str = None, **kwargs):
+        from funasr import AutoModel
+
+        model, kwargs = AutoModel.build_model(model=model, trust_remote_code=True, **kwargs)
+
+        return model, kwargs

examples/industrial_data_pretraining/fun_asr_nano/realtime_ws_benchmark.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""Benchmark client for serve_realtime_ws.py.
+
+The script replays a 16 kHz mono PCM16 WAV file to one or more realtime
+WebSocket sessions and reports client-observable latency metrics. It does not
+change the service and it does not require soundfile/librosa.
+"""
+
+import argparse
+import asyncio
+import json
+import statistics
+import sys
+import time
+import wave
+from pathlib import Path
+
+try:
+    import websockets
+except ImportError:
+    print("Please install websockets: pip install websockets", file=sys.stderr)
+    sys.exit(1)
+
+
+SAMPLE_RATE = 16000
+SAMPLE_WIDTH_BYTES = 2
+
+
+def load_pcm16_wav(path):
+    """Read a mono 16 kHz PCM16 WAV file and return raw bytes plus duration."""
+    wav_path = Path(path)
+    with wave.open(str(wav_path), "rb") as wf:
+        channels = wf.getnchannels()
+        sample_width = wf.getsampwidth()
+        sample_rate = wf.getframerate()
+        frames = wf.getnframes()
+        if channels != 1 or sample_width != SAMPLE_WIDTH_BYTES or sample_rate != SAMPLE_RATE:
+            raise ValueError(
+                f"{wav_path} must be 16 kHz mono PCM16 WAV; got "
+                f"{sample_rate} Hz, {channels} channel(s), {sample_width * 8}-bit samples"
+            )
+        audio = wf.readframes(frames)
+    return audio, frames / SAMPLE_RATE
+
+
+def percentile(values, pct):
+    if not values:
+        return None
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    index = (len(ordered) - 1) * pct / 100.0
+    lower = int(index)
+    upper = min(lower + 1, len(ordered) - 1)
+    weight = index - lower
+    return ordered[lower] * (1 - weight) + ordered[upper] * weight
+
+
+def round_or_none(value, digits=3):
+    return None if value is None else round(value, digits)
+
+
+async def receive_message(ws, timeout):
+    message = await asyncio.wait_for(ws.recv(), timeout=timeout)
+    if isinstance(message, bytes):
+        return {"_binary_bytes": len(message)}
+    try:
+        return json.loads(message)
+    except json.JSONDecodeError:
+        return {"_raw": message}
+
+
+async def send_command(ws, command, expected_event, timeout):
+    await ws.send(command)
+    response = await receive_message(ws, timeout)
+    if expected_event and response.get("event") != expected_event:
+        raise RuntimeError(f"{command} expected event={expected_event!r}, got {response}")
+    return response
+
+
+async def recv_results(ws, metrics, audio_started_at, stop_sent_at_ref, timeout):
+    while True:
+        try:
+            data = await receive_message(ws, timeout)
+        except asyncio.TimeoutError:
+            if metrics["final_messages"] == 0 and not metrics["stopped"]:
+                metrics["errors"].append(f"timed out waiting for server message after {timeout}s")
+            return
+        except websockets.exceptions.ConnectionClosed as exc:
+            if metrics["final_messages"] == 0 and not metrics["stopped"]:
+                metrics["errors"].append(f"connection closed while receiving: {exc}")
+            return
+
+        now = time.perf_counter()
+        metrics["messages"] += 1
+
+        event = data.get("event")
+        if event:
+            metrics["events"][event] = metrics["events"].get(event, 0) + 1
+
+        has_result = "sentences" in data or "partial" in data or data.get("is_final") is True
+        if has_result:
+            metrics["result_messages"] += 1
+            if metrics["first_update_ms"] is None:
+                metrics["first_update_ms"] = (now - audio_started_at) * 1000.0
+            if data.get("partial"):
+                metrics["partial_messages"] += 1
+            duration_ms = data.get("duration_ms")
+            if data.get("is_final") is not True and isinstance(duration_ms, (int, float)):
+                metrics["response_lag_ms"].append((now - audio_started_at) * 1000.0 - duration_ms)
+
+        if data.get("is_final") is True:
+            metrics["final_messages"] += 1
+            metrics["final_update_ms"] = (now - audio_started_at) * 1000.0
+            if stop_sent_at_ref["value"] is not None:
+                metrics["final_after_stop_ms"] = (now - stop_sent_at_ref["value"]) * 1000.0
+
+        if event == "stopped":
+            metrics["stopped"] = True
+            return
+
+
+async def run_client(client_id, args, audio_bytes, audio_seconds):
+    chunk_bytes = max(1, int(SAMPLE_RATE * args.chunk_ms / 1000.0)) * SAMPLE_WIDTH_BYTES
+    total_audio_seconds = audio_seconds * args.loops
+    metrics = {
+        "client_id": client_id,
+        "audio_seconds": total_audio_seconds,
+        "chunk_ms": args.chunk_ms,
+        "messages": 0,
+        "result_messages": 0,
+        "partial_messages": 0,
+        "final_messages": 0,
+        "events": {},
+        "first_update_ms": None,
+        "final_update_ms": None,
+        "final_after_stop_ms": None,
+        "response_lag_ms": [],
+        "stopped": False,
+        "errors": [],
+    }
+
+    wall_started_at = time.perf_counter()
+    stop_sent_at_ref = {"value": None}
+
+    try:
+        async with websockets.connect(
+            args.server,
+            ping_interval=None,
+            open_timeout=args.connect_timeout,
+            max_size=args.max_message_size,
+        ) as ws:
+            await send_command(ws, "START", "started", args.recv_timeout)
+            if args.hotwords:
+                await send_command(ws, f"HOTWORDS:{args.hotwords}", "hotwords_set", args.recv_timeout)
+            if args.language:
+                await send_command(ws, f"LANGUAGE:{args.language}", "language_set", args.recv_timeout)
+
+            audio_started_at = time.perf_counter()
+            recv_task = asyncio.create_task(
+                recv_results(ws, metrics, audio_started_at, stop_sent_at_ref, args.recv_timeout)
+            )
+
+            sent_audio_seconds = 0.0
+            for _ in range(args.loops):
+                for offset in range(0, len(audio_bytes), chunk_bytes):
+                    chunk = audio_bytes[offset : offset + chunk_bytes]
+                    await ws.send(chunk)
+                    sent_audio_seconds += len(chunk) / (SAMPLE_RATE * SAMPLE_WIDTH_BYTES)
+                    if args.pace:
+                        target_elapsed = sent_audio_seconds
+                        elapsed = time.perf_counter() - audio_started_at
+                        delay = target_elapsed - elapsed
+                        if delay > 0:
+                            await asyncio.sleep(delay)
+
+            stop_sent_at_ref["value"] = time.perf_counter()
+            metrics["send_seconds"] = stop_sent_at_ref["value"] - audio_started_at
+            await ws.send("STOP")
+            await recv_task
+    except Exception as exc:
+        metrics["errors"].append(str(exc))
+
+    wall_seconds = time.perf_counter() - wall_started_at
+    lags = metrics.pop("response_lag_ms")
+    send_seconds = metrics.get("send_seconds")
+    audio_per_wall = total_audio_seconds / wall_seconds if wall_seconds else None
+    send_audio_per_wall = total_audio_seconds / send_seconds if send_seconds else None
+    response_lag_ms_max = max(lags) if lags else None
+    response_lag_ms_p95 = percentile(lags, 95)
+    metrics.update(
+        {
+            "first_update_ms": round_or_none(metrics["first_update_ms"], 1),
+            "final_update_ms": round_or_none(metrics["final_update_ms"], 1),
+            "final_after_stop_ms": round_or_none(metrics["final_after_stop_ms"], 1),
+            "wall_seconds": round(wall_seconds, 3),
+            "send_seconds": round_or_none(send_seconds, 3),
+            "audio_per_wall": round_or_none(audio_per_wall),
+            "send_audio_per_wall": round_or_none(send_audio_per_wall),
+            "response_lag_ms_max": round_or_none(response_lag_ms_max, 1),
+            "response_lag_ms_p95": round_or_none(response_lag_ms_p95, 1),
+        }
+    )
+    return metrics
+
+
+def summarize(results, elapsed_seconds):
+    total_audio = sum(item["audio_seconds"] for item in results)
+    first_updates = [item["first_update_ms"] for item in results if item["first_update_ms"] is not None]
+    final_after_stop = [
+        item["final_after_stop_ms"] for item in results if item["final_after_stop_ms"] is not None
+    ]
+    lag_p95 = [item["response_lag_ms_p95"] for item in results if item["response_lag_ms_p95"] is not None]
+    errors = sum(len(item["errors"]) for item in results)
+    return {
+        "clients": len(results),
+        "total_audio_seconds": round(total_audio, 3),
+        "elapsed_seconds": round(elapsed_seconds, 3),
+        "aggregate_audio_per_wall": round_or_none(total_audio / elapsed_seconds if elapsed_seconds else None),
+        "first_update_ms_p50": round_or_none(statistics.median(first_updates), 1)
+        if first_updates
+        else None,
+        "first_update_ms_p95": round_or_none(percentile(first_updates, 95), 1),
+        "final_after_stop_ms_p50": round_or_none(statistics.median(final_after_stop), 1)
+        if final_after_stop
+        else None,
+        "final_after_stop_ms_p95": round_or_none(percentile(final_after_stop, 95), 1),
+        "client_response_lag_ms_p95_max": round_or_none(max(lag_p95), 1) if lag_p95 else None,
+        "partial_messages": sum(item["partial_messages"] for item in results),
+        "final_messages": sum(item["final_messages"] for item in results),
+        "errors": errors,
+    }
+
+
+def print_summary(summary, results):
+    print("FunASR realtime WebSocket benchmark")
+    print(f"clients: {summary['clients']}")
+    print(f"total audio seconds: {summary['total_audio_seconds']}")
+    print(f"elapsed seconds: {summary['elapsed_seconds']}")
+    print(f"aggregate audio/wall: {summary['aggregate_audio_per_wall']}x")
+    print(f"first update p50/p95 ms: {summary['first_update_ms_p50']} / {summary['first_update_ms_p95']}")
+    print(
+        "final after STOP p50/p95 ms: "
+        f"{summary['final_after_stop_ms_p50']} / {summary['final_after_stop_ms_p95']}"
+    )
+    print(f"max client response-lag p95 ms: {summary['client_response_lag_ms_p95_max']}")
+    print(f"partial/final messages: {summary['partial_messages']} / {summary['final_messages']}")
+    print(f"errors: {summary['errors']}")
+    if summary["errors"]:
+        for item in results:
+            for error in item["errors"]:
+                print(f"client {item['client_id']} error: {error}")
+
+
+async def async_main(args):
+    audio_bytes, audio_seconds = load_pcm16_wav(args.wav)
+    started_at = time.perf_counter()
+    results = await asyncio.gather(
+        *(run_client(client_id, args, audio_bytes, audio_seconds) for client_id in range(args.clients))
+    )
+    elapsed = time.perf_counter() - started_at
+    summary = summarize(results, elapsed)
+    if args.output_jsonl:
+        output_path = Path(args.output_jsonl)
+        with output_path.open("w", encoding="utf-8") as f:
+            for result in results:
+                f.write(json.dumps({"type": "client", **result}, ensure_ascii=False) + "\n")
+            f.write(json.dumps({"type": "summary", **summary}, ensure_ascii=False) + "\n")
+    print_summary(summary, results)
+    return 1 if summary["errors"] else 0
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Benchmark serve_realtime_ws.py with one or more clients")
+    parser.add_argument("wav", help="16 kHz mono PCM16 WAV file")
+    parser.add_argument("--server", default="ws://localhost:10095", help="WebSocket server URL")
+    parser.add_argument("--clients", type=int, default=1, help="Concurrent clients")
+    parser.add_argument("--loops", type=int, default=1, help="Times each client replays the WAV before STOP")
+    parser.add_argument("--chunk-ms", type=int, default=100, help="PCM frame duration per WebSocket send")
+    parser.add_argument("--language", default="", help="Optional LANGUAGE command value")
+    parser.add_argument("--hotwords", default="", help="Optional HOTWORDS command value, comma separated")
+    parser.add_argument("--no-pace", dest="pace", action="store_false", help="Send as fast as possible")
+    parser.add_argument("--output-jsonl", default="", help="Write per-client metrics and summary JSONL")
+    parser.add_argument("--connect-timeout", type=float, default=10.0, help="Connection timeout seconds")
+    parser.add_argument("--recv-timeout", type=float, default=30.0, help="Timeout waiting for server messages")
+    parser.add_argument("--max-message-size", type=int, default=16 * 1024 * 1024, help="WebSocket max message size")
+    parser.set_defaults(pace=True)
+    args = parser.parse_args()
+    if args.clients < 1:
+        parser.error("--clients must be >= 1")
+    if args.loops < 1:
+        parser.error("--loops must be >= 1")
+    if args.chunk_ms < 10:
+        parser.error("--chunk-ms must be >= 10")
+    return args
+
+
+def main():
+    args = parse_args()
+    try:
+        raise SystemExit(asyncio.run(async_main(args)))
+    except KeyboardInterrupt:
+        print("Interrupted.", file=sys.stderr)
+        raise SystemExit(130)
+    except ValueError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        raise SystemExit(2)
+
+
+if __name__ == "__main__":
+    main()

examples/industrial_data_pretraining/fun_asr_nano/requirements.txt

@@ -0,0 +1,10 @@
+torch>=2.0
+funasr
+websockets>=12.0
+regex
+numpy
+soundfile
+
+# Optional: for Python client microphone mode
+# sounddevice
+# librosa

examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py

@@ -0,0 +1,605 @@
+#!/usr/bin/env python3
+"""Fun-ASR-Nano Streaming WebSocket Server.
+
+Features:
+- Streaming VAD segmentation (fsmn-vad)
+- Per-segment ASR decoding (Fun-ASR-Nano via vLLM)
+- Speaker diarization (eres2netv2 + ClusterBackend)
+- Hotword customization
+- Hallucination detection & prevention
+"""
+
+import asyncio
+import json
+import logging
+import os
+import time
+import argparse
+import numpy as np
+import torch
+import warnings
+import regex
+import websockets
+
+warnings.filterwarnings('ignore')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
+logger = logging.getLogger(__name__)
+
+
+def detect_and_fix_hallucination(text, max_ngram_length=12, max_occurrences=3):
+    """Detect repeated patterns (hallucination) and truncate to keep one occurrence."""
+    if not text or len(text) < max_ngram_length * 2:
+        return text, False
+
+    cleaned = regex.sub(r'\p{P}+', '', text)
+
+    word_pattern = rf'(?<!\S)(?!\d+$)(\w+)(?:\s+\1){{{max_occurrences - 1},}}(?!\S)'
+    if regex.search(word_pattern, cleaned, regex.IGNORECASE):
+        match = regex.search(word_pattern, cleaned, regex.IGNORECASE)
+        repeated = match.group(1)
+        pos = text.find(repeated)
+        if pos >= 0:
+            end_pos = text.find(repeated, pos + len(repeated))
+            if end_pos >= 0:
+                return text[:end_pos + len(repeated)], True
+        return text[:len(text)//2], True
+
+    for length in range(1, max_ngram_length):
+        pattern = rf'(?<!\d)(\S{{{length}}})\1{{{max_occurrences - 1},}}(?!\d)'
+        combined = rf'(?=.*\D){pattern}'
+        match = regex.search(combined, cleaned)
+        if match:
+            repeated = match.group(1)
+            pos = text.find(repeated)
+            if pos >= 0:
+                end_pos = text.find(repeated, pos + len(repeated))
+                if end_pos >= 0:
+                    return text[:end_pos + len(repeated)], True
+            return text[:len(text)//2], True
+
+    return text, False
+
+
+def _clean_asr_text(text):
+    """Remove timestamp tags and artifacts from vLLM output."""
+    import re
+    text = re.sub(r'<[^>]*>', '', text)
+    text = re.sub(r'\[.*?\]', '', text)
+    text = re.sub(r'[Ｏ\[\]&＆|｜]', '', text)
+    text = re.sub(r'/sil|endofbreak|FFFF', '', text)
+    text = re.sub(r'\s+', ' ', text)
+    return text.strip()
+
+
+from funasr.models.fsmn_vad_streaming.dynamic_vad import DynamicStreamingVAD
+
+
+class HybridSpeakerTracker:
+    """Speaker diarization: streaming ClusterBackend + final re-clustering."""
+
+    def __init__(self, spk_model, device, threshold=0.6):
+        self.spk_model = spk_model
+        self.device = device
+        self.threshold = threshold
+        self.speaker_centers = []
+        from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+        from funasr.models.campplus.cluster_backend import ClusterBackend
+        self.sv_chunk = sv_chunk
+        self.postprocess = postprocess
+        self.distribute_spk = distribute_spk
+        self.cluster_backend = ClusterBackend(merge_thr=0.78).to(device)
+        self.all_chunks = []
+        self.all_embeddings = []
+        self.display_map = {}
+        self.next_display_id = 0
+
+    @torch.no_grad()
+    def assign_streaming(self, audio_samples, seg_start_s, seg_end_s, sentence):
+        """Assign speaker ID during streaming using ClusterBackend."""
+        vad_seg = [[seg_start_s, seg_end_s, audio_samples]]
+        chunks = self.sv_chunk(vad_seg)
+        if not chunks:
+            sentence["spk"] = self.next_display_id
+            self.next_display_id += 1
+            return
+
+        self.all_chunks.extend(chunks)
+        speech_list = [ch[2] for ch in chunks]
+        spk_res = self.spk_model.generate(input=speech_list, cache={}, is_final=True)
+        embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)
+        self.all_embeddings.append(embs)
+
+        all_embs = torch.cat(self.all_embeddings, dim=0)
+        labels = self.cluster_backend(all_embs.cpu(), oracle_num=None)
+        if not isinstance(labels, np.ndarray):
+            labels = np.array(labels)
+
+        all_sorted = sorted(self.all_chunks, key=lambda x: x[0])
+        sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu())
+        temp = [{"start": int(seg_start_s*1000), "end": int(seg_end_s*1000), "text": sentence["text"]}]
+        self.distribute_spk(temp, sv_output)
+        raw_spk = temp[0].get("spk", 0)
+
+        if raw_spk not in self.display_map:
+            self.display_map[raw_spk] = self.next_display_id
+            self.next_display_id += 1
+        sentence["spk"] = self.display_map[raw_spk]
+
+    @torch.no_grad()
+    def finalize(self, sentences, min_split_s=3.0):
+        """Final re-clustering for accurate speaker assignment."""
+        if not self.all_embeddings or not sentences:
+            return sentences
+
+        all_embs = torch.cat(self.all_embeddings, dim=0)
+        labels = self.cluster_backend(all_embs.cpu(), oracle_num=None)
+        if not isinstance(labels, np.ndarray):
+            labels = np.array(labels)
+
+        all_sorted = sorted(self.all_chunks, key=lambda x: x[0])
+        sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu())
+
+        for s in sentences:
+            s.pop("spk", None)
+        self.distribute_spk(sentences, sv_output)
+
+        id_map = {}
+        next_id = 0
+        for s in sentences:
+            raw = s.get("spk", 0)
+            if raw not in id_map:
+                id_map[raw] = next_id
+                next_id += 1
+            s["spk"] = id_map[raw]
+
+        final_sentences = []
+        for s in sentences:
+            sub = self._try_split(s, sv_output, id_map, min_split_s)
+            final_sentences.extend(sub)
+
+        return final_sentences
+
+    def _try_split(self, sentence, sv_output, id_map, min_split_s):
+        """Split a sentence if multiple speakers detected within its time range."""
+        sent_start = sentence["start"] / 1000.0
+        sent_end = sentence["end"] / 1000.0
+        text = sentence["text"]
+
+        overlapping = []
+        for sv_start, sv_end, sv_spk in sv_output:
+            o_start = max(sent_start, sv_start)
+            o_end = min(sent_end, sv_end)
+            if o_end > o_start:
+                mapped_spk = id_map.get(int(sv_spk), int(sv_spk))
+                overlapping.append([o_start, o_end, mapped_spk])
+
+        if len(overlapping) <= 1:
+            return [sentence]
+
+        filtered = [overlapping[0]]
+        for i in range(1, len(overlapping)):
+            cur = overlapping[i]
+            prev = filtered[-1]
+            if cur[2] == prev[2]:
+                filtered[-1] = [prev[0], cur[1], prev[2]]
+            elif (cur[1] - cur[0]) < min_split_s:
+                filtered[-1] = [prev[0], cur[1], prev[2]]
+            else:
+                filtered.append(cur)
+
+        merged = [filtered[0]]
+        for i in range(1, len(filtered)):
+            if (merged[-1][1] - merged[-1][0]) < min_split_s:
+                merged[-1] = [merged[-1][0], filtered[i][1], filtered[i][2]]
+            else:
+                merged.append(filtered[i])
+        if len(merged) > 1 and (merged[-1][1] - merged[-1][0]) < min_split_s:
+            merged[-2] = [merged[-2][0], merged[-1][1], merged[-2][2]]
+            merged.pop()
+
+        if len(merged) <= 1:
+            return [sentence]
+
+        total_dur = sum(m[1] - m[0] for m in merged)
+        sub_sentences = []
+        char_pos = 0
+        for i, (m_start, m_end, m_spk) in enumerate(merged):
+            if i == len(merged) - 1:
+                sub_text = text[char_pos:]
+            else:
+                n_chars = max(1, int(len(text) * (m_end - m_start) / total_dur))
+                sub_text = text[char_pos:char_pos + n_chars]
+                char_pos += n_chars
+            if sub_text.strip():
+                sub_sentences.append({"text": sub_text.strip(), "start": int(m_start*1000), "end": int(m_end*1000), "spk": m_spk})
+
+        return sub_sentences if sub_sentences else [sentence]
+
+    def reset(self):
+        self.speaker_centers = []
+        self.all_chunks = []
+        self.all_embeddings = []
+        self.display_map = {}
+        self.next_display_id = 0
+
+
+class RealtimeASRSession:
+    """Manages a single streaming ASR session."""
+
+    def __init__(self, vllm_engine, asr_kwargs, vad, spk_tracker=None, sample_rate=16000, chunk_ms=960,
+                 partial_window_sec=15.0):
+        self.vllm_engine = vllm_engine
+        self.asr_kwargs = asr_kwargs
+        self.vad = vad
+        self.sample_rate = sample_rate
+        self.chunk_samples = int(sample_rate * chunk_ms / 1000)
+        self.first_chunk_samples = int(sample_rate * 480 / 1000)
+        # Bound the interim (partial) re-decode window. While a speech segment has
+        # not yet hit a VAD pause it keeps growing, and the partial path re-encodes
+        # it from the start on every chunk -> O(L^2) total re-encoding for a
+        # length-L segment. Under concurrency that saturates the GPU and long-segment
+        # requests time out. Capping the partial window to the most recent
+        # `partial_window_sec` seconds makes interim re-decoding ~O(L) per segment
+        # without changing the final result (completed segments are always decoded
+        # in full by _decode_segment / the is_final path). Set <=0 to disable.
+        self.partial_window_samples = int(sample_rate * partial_window_sec) if partial_window_sec and partial_window_sec > 0 else 0
+        self.first_decode_done = False
+
+        self.audio_buffer = np.array([], dtype=np.float32)
+        self.vad_fed_samples = 0
+        self.prev_text = ""
+        self.last_partial_text = ""
+        self.last_partial_start_ms = 0
+        self.last_decode_samples = 0
+        self.locked_sentences = []
+        self.prev_seg_text = ""
+        self.spk_tracker = spk_tracker
+        self.use_context = True
+        self.is_active = False
+
+    def add_audio(self, pcm_bytes):
+        audio_int16 = np.frombuffer(pcm_bytes, dtype=np.int16)
+        audio_float = audio_int16.astype(np.float32) / 32768.0
+        self.audio_buffer = np.concatenate([self.audio_buffer, audio_float])
+
+        new_audio = self.audio_buffer[self.vad_fed_samples:]
+        if len(new_audio) > 0:
+            new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False)
+            self.vad_fed_samples = len(self.audio_buffer)
+
+            for seg in new_confirmed:
+                seg_text = self._decode_segment(seg)
+                self.prev_text = ""
+                if not seg_text.strip():
+                    continue
+                self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})
+                if self.spk_tracker:
+                    s0 = int(seg[0] * self.sample_rate / 1000)
+                    s1 = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))
+                    self.spk_tracker.assign_streaming(self.audio_buffer[s0:s1], seg[0]/1000, seg[1]/1000, self.locked_sentences[-1])
+                logger.info(f"Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:40]}\"")
+
+    def should_decode(self):
+        threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples
+        return (len(self.audio_buffer) - self.last_decode_samples) >= threshold
+
+    @torch.no_grad()
+    def decode(self, is_final=False):
+        if len(self.audio_buffer) < self.chunk_samples:
+            return self._build_response(is_final)
+
+        if is_final:
+            remaining = self.audio_buffer[self.vad_fed_samples:]
+            if len(remaining) > 0:
+                new_confirmed = self.vad.feed(torch.from_numpy(remaining).float(), is_final=True)
+                self.vad_fed_samples = len(self.audio_buffer)
+                for seg in new_confirmed:
+                    seg_text = self._decode_segment(seg)
+                    if not seg_text.strip():
+                        continue
+                    self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})
+
+            if self.vad.current_speech_start is not None:
+                end_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)
+                seg = [self.vad.current_speech_start, end_ms]
+                seg_text = self._decode_segment(seg)
+                if seg_text.strip():
+                    self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})
+                self.vad.current_speech_start = None
+
+            if self.spk_tracker and self.locked_sentences:
+                self.locked_sentences = self.spk_tracker.finalize(self.locked_sentences)
+            return self._build_response(is_final)
+
+        if self.vad.current_speech_start is not None:
+            seg_audio, partial_start_ms = self.get_partial_decode_audio()
+        else:
+            self.last_decode_samples = len(self.audio_buffer)
+            self.last_partial_text = ""
+            return self._build_response(is_final)
+
+        if len(seg_audio) < self.chunk_samples // 2:
+            return self._build_response(is_final)
+
+        audio_tensor = torch.from_numpy(seg_audio).float()
+        try:
+            results = self.vllm_engine.generate(
+                inputs=[audio_tensor],
+                hotwords=self.asr_kwargs.get("hotwords"),
+                language=self.asr_kwargs.get("language"),
+                max_new_tokens=200,
+            )
+            text = results[0]["text"] if results else ""
+            text = _clean_asr_text(text)
+        except Exception as e:
+            logger.error(f"ASR error: {e}")
+            return self._build_response(is_final)
+
+        text, hallucinated = detect_and_fix_hallucination(text)
+        if hallucinated:
+            self.prev_text = ""
+
+        self.last_decode_samples = len(self.audio_buffer)
+        self.last_partial_text = text
+        self.last_partial_start_ms = partial_start_ms
+        if text.strip() and not self.first_decode_done:
+            self.first_decode_done = True
+
+        tokenizer = self.vllm_engine._engine.tokenizer
+        encoded = tokenizer.encode(text)
+        if len(encoded) > 5:
+            self.prev_text = tokenizer.decode(encoded[:-5], skip_special_tokens=True)
+        else:
+            self.prev_text = ""
+
+        return self._build_response(is_final)
+
+    def get_partial_decode_audio(self):
+        """Return the bounded audio window used for unstable partial decoding."""
+        seg_start_sample = int(self.vad.current_speech_start * self.sample_rate / 1000)
+        decode_start_sample = seg_start_sample
+
+        if self.partial_window_samples:
+            min_start = len(self.audio_buffer) - self.partial_window_samples
+            if min_start > decode_start_sample:
+                decode_start_sample = min_start
+
+        decode_start_sample = max(0, decode_start_sample)
+        start_ms = int(decode_start_sample * 1000 / self.sample_rate)
+        return self.audio_buffer[decode_start_sample:], start_ms
+
+    @torch.no_grad()
+    def _decode_segment(self, seg):
+        """Decode a completed VAD segment via vLLM."""
+        start_sample = int(seg[0] * self.sample_rate / 1000)
+        end_sample = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))
+        seg_audio = self.audio_buffer[start_sample:end_sample]
+        if len(seg_audio) < 1600:
+            return ""
+        audio_tensor = torch.from_numpy(seg_audio).float()
+        try:
+            results = self.vllm_engine.generate(
+                inputs=[audio_tensor],
+                hotwords=self.asr_kwargs.get("hotwords"),
+                language=self.asr_kwargs.get("language"),
+                max_new_tokens=512,
+            )
+            text = results[0]["text"] if results else ""
+            text = _clean_asr_text(text)
+            self.prev_seg_text = text
+            return text
+        except Exception as e:
+            logger.error(f"Segment decode error: {e}")
+            return ""
+
+    def _build_response(self, is_final):
+        duration_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)
+        sentences = list(self.locked_sentences)
+        partial = self.last_partial_text
+        if partial:
+            partial_start = self.last_partial_start_ms
+        elif self.vad.current_speech_start is not None:
+            partial_start = self.vad.current_speech_start
+        else:
+            partial_start = duration_ms
+
+        if is_final:
+            return {"sentences": sentences, "partial": "", "partial_start_ms": 0,
+                    "duration_ms": duration_ms, "is_final": True}
+        return {"sentences": sentences, "partial": partial,
+                "partial_start_ms": partial_start,
+                "duration_ms": duration_ms, "is_final": False}
+
+    def reset(self):
+        self.audio_buffer = np.array([], dtype=np.float32)
+        self.vad_fed_samples = 0
+        self.first_decode_done = False
+        self.vad.reset()
+        self.prev_text = ""
+        self.last_partial_text = ""
+        self.last_partial_start_ms = 0
+        self.last_decode_samples = 0
+        self.locked_sentences = []
+        if self.spk_tracker:
+            self.spk_tracker.reset()
+
+
+_vllm_engine = None
+_asr_kwargs = None
+_vad_model = None
+_spk_model = None
+
+
+def load_models(args):
+    global _vllm_engine, _asr_kwargs, _vad_model, _spk_model
+    if _vllm_engine is None:
+        from funasr import AutoModel
+        from funasr.auto.auto_model_vllm import AutoModelVLLM
+
+        logger.info(f"Loading ASR (vLLM): {args.model}")
+        _vllm_engine = AutoModelVLLM(
+            model=args.model, hub=args.hub, device=args.device,
+            dtype=getattr(args, 'dtype', 'bf16'),
+            tensor_parallel_size=getattr(args, 'tensor_parallel_size', 1),
+            gpu_memory_utilization=getattr(args, 'gpu_memory_utilization', 0.8),
+            max_model_len=getattr(args, 'max_model_len', 2048),
+        )
+
+        _asr_kwargs = {}
+        hw_file = getattr(args, 'hotword_file', '热词列表')
+        if hw_file and os.path.isfile(hw_file):
+            with open(hw_file, "r", encoding="utf-8") as hf:
+                hotwords = [line.strip() for line in hf if line.strip()]
+            _asr_kwargs["hotwords"] = hotwords
+            logger.info(f"Loaded {len(hotwords)} hotwords from '{hw_file}'")
+
+        if getattr(args, 'language', None):
+            _asr_kwargs["language"] = args.language
+            logger.info(f"Language: {args.language}")
+
+        logger.info("Loading VAD: fsmn-vad (streaming)")
+        _vad_model = AutoModel(model="fsmn-vad", device=args.device, disable_update=True)
+
+        if getattr(args, "enable_spk", False):
+            logger.info(f"Loading SPK: {args.spk_model}")
+            _spk_model = AutoModel(model=args.spk_model, device=args.device, disable_update=True)
+        else:
+            _spk_model = None
+            logger.info("SPK disabled; use --enable-spk to include speaker diarization")
+
+        logger.info("All models ready!")
+    return _vllm_engine, _asr_kwargs, _vad_model, _spk_model
+
+
+def create_speaker_tracker(spk_model, args):
+    if not getattr(args, "enable_spk", False) or spk_model is None:
+        return None
+    return HybridSpeakerTracker(spk_model, args.device)
+
+
+async def handle_client(websocket, args):
+    vllm_engine, asr_kwargs, vad_model, spk_model = load_models(args)
+    vad = DynamicStreamingVAD(vad_model)
+    spk_tracker = create_speaker_tracker(spk_model, args)
+    session = RealtimeASRSession(
+        vllm_engine,
+        asr_kwargs,
+        vad,
+        spk_tracker=spk_tracker,
+        partial_window_sec=getattr(args, 'partial_window_sec', 15.0),
+    )
+    logger.info(f"Client connected: {websocket.remote_address}")
+
+    decode_interval = args.decode_interval
+    last_decode_time = 0
+
+    try:
+        async for message in websocket:
+            if isinstance(message, str):
+                cmd = message.strip()
+                if cmd.upper() == "START":
+                    session.reset()
+                    session.is_active = True
+                    await websocket.send(json.dumps({"event": "started"}))
+                    logger.info("Session started")
+                elif cmd.upper().startswith("HOTWORDS:"):
+                    hw_str = cmd[9:]
+                    hotwords = [w.strip() for w in hw_str.split(",") if w.strip()]
+                    session.asr_kwargs = dict(session.asr_kwargs)
+                    session.asr_kwargs["hotwords"] = hotwords
+                    await websocket.send(json.dumps({"event": "hotwords_set", "hotwords": hotwords}))
+                    logger.info(f"Hotwords set: {len(hotwords)} words")
+                elif cmd.upper().startswith("LANGUAGE:"):
+                    lang = cmd[9:].strip()
+                    session.asr_kwargs = dict(session.asr_kwargs)
+                    session.asr_kwargs["language"] = lang if lang else None
+                    await websocket.send(json.dumps({"event": "language_set", "language": lang}))
+                    logger.info(f"Language set: {lang}")
+                elif cmd.upper() == "STOP":
+                    if session.is_active and len(session.audio_buffer) > 0:
+                        result = session.decode(is_final=True)
+                        await websocket.send(json.dumps(result))
+                        logger.info(f"Final: {len(result['sentences'])} sentences")
+                        session.is_active = False
+                    await websocket.send(json.dumps({"event": "stopped"}))
+            elif isinstance(message, bytes) and session.is_active:
+                session.add_audio(message)
+                now = time.time()
+                if now - last_decode_time >= decode_interval and session.should_decode():
+                    result = session.decode(is_final=False)
+                    await websocket.send(json.dumps(result))
+                    last_decode_time = now
+
+    except websockets.exceptions.ConnectionClosed:
+        logger.info("Client disconnected")
+    except Exception as e:
+        logger.error(f"Error: {e}", exc_info=True)
+
+
+def _positive_or_none(value):
+    return None if value <= 0 else value
+
+
+def build_websocket_serve_kwargs(args):
+    return {
+        "max_size": args.ws_max_size,
+        "ping_interval": _positive_or_none(args.ws_ping_interval),
+        "ping_timeout": _positive_or_none(args.ws_ping_timeout),
+        "close_timeout": args.ws_close_timeout,
+    }
+
+
+async def main(args):
+    load_models(args)
+    logger.info(f"Server on ws://0.0.0.0:{args.port}")
+    serve_kwargs = build_websocket_serve_kwargs(args)
+    logger.info(
+        "WebSocket options: "
+        f"max_size={serve_kwargs['max_size']}, "
+        f"ping_interval={serve_kwargs['ping_interval']}, "
+        f"ping_timeout={serve_kwargs['ping_timeout']}, "
+        f"close_timeout={serve_kwargs['close_timeout']}"
+    )
+    async with websockets.serve(
+        lambda ws: handle_client(ws, args), "0.0.0.0", args.port, **serve_kwargs,
+    ):
+        await asyncio.Future()
+
+
+def build_arg_parser():
+    parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming WebSocket Server")
+    parser.add_argument("--port", type=int, default=10095)
+    parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")
+    parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])
+    parser.add_argument("--device", type=str, default="cuda:0")
+    parser.add_argument("--use-context", action="store_true", default=True)
+    parser.add_argument("--no-context", dest="use_context", action="store_false")
+    parser.add_argument("--decode-interval", type=float, default=0.48)
+    parser.add_argument("--partial-window-sec", type=float, default=15.0,
+                        help="Cap the interim partial re-decode window to the most recent N seconds. "
+                             "A long ongoing speech segment is otherwise re-encoded from its start on "
+                             "every chunk (O(L^2)), which saturates the GPU under concurrency and times "
+                             "out long-segment requests. Lower it (e.g. 8-10) for high-concurrency "
+                             "self-hosting; <=0 disables (legacy behaviour). Final transcripts are unaffected.")
+    parser.add_argument("--enable-spk", action="store_true", help="Enable streaming speaker diarization.")
+    parser.add_argument("--spk-model", type=str, default="iic/speech_eres2netv2_sv_zh-cn_16k-common")
+    parser.add_argument("--hotword-file", type=str, default="热词列表")
+    parser.add_argument("--language", type=str, default=None, help="Language hint (e.g. 中文, English, 日本語)")
+    parser.add_argument("--dtype", type=str, default="bf16", choices=["bf16", "fp16", "fp32"])
+    parser.add_argument("--tensor-parallel-size", type=int, default=1)
+    parser.add_argument("--gpu-memory-utilization", type=float, default=0.8)
+    parser.add_argument("--max-model-len", type=int, default=2048)
+    parser.add_argument("--ws-ping-interval", type=float, default=20.0,
+                        help="WebSocket ping interval in seconds; <=0 disables keepalive pings.")
+    parser.add_argument("--ws-ping-timeout", type=float, default=20.0,
+                        help="WebSocket ping timeout in seconds; <=0 disables ping timeout.")
+    parser.add_argument("--ws-close-timeout", type=float, default=10.0,
+                        help="WebSocket close handshake timeout in seconds.")
+    parser.add_argument("--ws-max-size", type=int, default=10 * 1024 * 1024,
+                        help="Maximum incoming WebSocket message size in bytes.")
+    return parser
+
+
+if __name__ == "__main__":
+    args = build_arg_parser().parse_args()
+    asyncio.run(main(args))

examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py

@@ -0,0 +1,432 @@
+#!/usr/bin/env python3
+"""Fun-ASR-Nano vLLM Inference Server.
+
+Unified server with three interfaces:
+- HTTP REST: POST /asr (file upload)
+- WebSocket: ws://host:port/ws (streaming audio)
+- OpenAI API: POST /v1/audio/transcriptions (Whisper-compatible)
+
+All endpoints share the same vLLM engine + dynamic VAD + SPK + timestamps.
+
+Usage:
+    CUDA_VISIBLE_DEVICES=0 python serve_vllm.py --port 8000
+    CUDA_VISIBLE_DEVICES=0 python serve_vllm.py --port 8000 --model FunAudioLLM/Fun-ASR-Nano-2512
+"""
+
+import asyncio
+import argparse
+import io
+import json
+import logging
+import os
+import re
+import time
+import tempfile
+
+import numpy as np
+import soundfile as sf
+import torch
+import warnings
+
+warnings.filterwarnings('ignore')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
+logger = logging.getLogger(__name__)
+
+def truncate_repetition(text, min_repeat_len=3, max_repeats=3):
+    """Detect and truncate repetitive patterns in ASR output."""
+    if not text or len(text) < 20:
+        return text
+    n = len(text)
+    for length in range(min_repeat_len, min(n // max_repeats, 30)):
+        for start in range(n - length * max_repeats):
+            chunk = text[start:start + length]
+            if text[start:start + length * max_repeats] == chunk * max_repeats:
+                return text[:start + length]
+    return text
+
+
+
+try:
+    from fastapi import FastAPI, File, UploadFile, Form, WebSocket, WebSocketDisconnect
+    from fastapi.responses import JSONResponse
+    import uvicorn
+except ImportError:
+    raise ImportError("pip install fastapi uvicorn python-multipart")
+
+from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+from funasr.models.fsmn_vad_streaming.dynamic_vad import DynamicStreamingVAD
+from funasr import AutoModel
+
+
+# ============================================================
+# Global state
+# ============================================================
+_engine = None
+_vad_model = None
+_spk_model = None
+_args = None
+
+
+def prepare_audio_for_inference(audio_data, sr, target_sr=16000):
+    """Return mono float32 audio at target_sr for ASR inference."""
+    audio_data = np.asarray(audio_data)
+    if audio_data.ndim > 1:
+        channel_axis = -1 if audio_data.shape[-1] <= audio_data.shape[0] else 0
+        audio_data = audio_data.mean(axis=channel_axis)
+
+    if sr != target_sr:
+        import librosa
+        audio_data = librosa.resample(audio_data, orig_sr=sr, target_sr=target_sr)
+        sr = target_sr
+
+    return audio_data.astype(np.float32), sr
+
+
+def load_engine(args):
+    global _engine, _vad_model, _spk_model, _args
+    _args = args
+    if _engine is None:
+        logger.info(f"Loading vLLM engine: {args.model}")
+        _engine = FunASRNanoVLLM.from_pretrained(
+            model=args.model, hub=args.hub, device=args.device, dtype=args.dtype,
+            max_model_len=args.max_model_len,
+            gpu_memory_utilization=args.gpu_memory_utilization,
+        )
+        logger.info(f"Loading VAD: {args.vad_model}")
+        _vad_model = AutoModel(model=args.vad_model, device=args.device, disable_update=True)
+        if args.spk_model:
+            logger.info(f"Loading SPK: {args.spk_model}")
+            _spk_model = AutoModel(model=args.spk_model, device=args.device, disable_update=True)
+        else:
+            logger.info("SPK disabled")
+        logger.info("All models ready!")
+
+
+def process_audio(audio_data, sr=16000, language=None, hotwords=None, 
+                  use_vad=True, use_spk=False, use_timestamp=True):
+    """Core processing: VAD segment → vLLM ASR → timestamps → SPK."""
+    audio_data, sr = prepare_audio_for_inference(audio_data, sr)
+
+    # VAD segmentation
+    if use_vad and len(audio_data) > sr * 1:
+        vad_res = _vad_model.generate(input=audio_data, fs=sr)
+        segments = vad_res[0]["value"]
+    else:
+        segments = [[0, int(len(audio_data) * 1000 / sr)]]
+
+    if not segments:
+        return {"text": "", "segments": [], "duration": len(audio_data) / sr}
+
+    # Extract segment audio
+    seg_audios = []
+    seg_times = []
+    for seg in segments:
+        s0 = int(seg[0] * sr / 1000)
+        s1 = int(seg[1] * sr / 1000)
+        seg_audio = audio_data[s0:s1]
+        if len(seg_audio) > sr * 0.3:
+            seg_audios.append(seg_audio)
+            seg_times.append((seg[0], seg[1]))
+
+    if not seg_audios:
+        return {"text": "", "segments": [], "duration": len(audio_data) / sr}
+
+    # vLLM batch ASR
+    gen_kwargs = {"max_new_tokens": 500}
+    if language:
+        gen_kwargs["language"] = language
+    if hotwords:
+        gen_kwargs["hotwords"] = hotwords
+
+    results = _engine.generate(inputs=seg_audios, **gen_kwargs)
+
+    # Build segments with timestamps
+    output_segments = []
+    full_text_parts = []
+
+    for i, (r, (start_ms, end_ms)) in enumerate(zip(results, seg_times)):
+        r["text"] = truncate_repetition(r["text"])
+        seg_info = {
+            "text": r["text"],
+            "start": start_ms / 1000,
+            "end": end_ms / 1000,
+        }
+        if use_timestamp and "timestamps" in r:
+            # Offset timestamps by segment start
+            offset = start_ms / 1000
+            seg_info["words"] = [
+                {"word": ts["token"], "start": ts["start_time"] + offset, "end": ts["end_time"] + offset}
+                for ts in r["timestamps"]
+            ]
+        output_segments.append(seg_info)
+        full_text_parts.append(r["text"])
+
+    # SPK diarization
+    if use_spk and _spk_model is not None:
+        from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+        from funasr.models.campplus.cluster_backend import ClusterBackend
+
+        vad_segs = [[st, et, audio_data[int(st*sr):int(et*sr)]] 
+                    for st, et in [(s["start"], s["end"]) for s in output_segments]]
+        chunks = sv_chunk(vad_segs)
+        if chunks:
+            speech_list = [ch[2] for ch in chunks]
+            spk_res = _spk_model.generate(input=speech_list, cache={}, is_final=True)
+            embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)
+            cluster = ClusterBackend(merge_thr=0.78).to(_args.device)
+            labels = cluster(embs.cpu(), oracle_num=None)
+            if not isinstance(labels, np.ndarray):
+                labels = np.array(labels)
+            all_sorted = sorted(chunks, key=lambda x: x[0])
+            sv_output = postprocess(all_sorted, None, labels, embs.cpu())
+            sentences = [{"text": s["text"], "start": int(s["start"]*1000), "end": int(s["end"]*1000)} 
+                        for s in output_segments]
+            distribute_spk(sentences, sv_output)
+            for i, s in enumerate(sentences):
+                output_segments[i]["speaker"] = f"SPK{s.get('spk', 0)}"
+
+    return {
+        "text": " ".join(full_text_parts),
+        "segments": output_segments,
+        "duration": len(audio_data) / sr,
+    }
+
+
+def build_openai_verbose_json(result, language=None):
+    """Build OpenAI-compatible verbose_json while preserving FunASR extensions."""
+    segments = []
+    for i, seg in enumerate(result["segments"]):
+        item = {
+            "id": i,
+            "start": seg["start"],
+            "end": seg["end"],
+            "text": seg["text"],
+            "words": seg.get("words", []),
+        }
+        if "speaker" in seg:
+            item["speaker"] = seg["speaker"]
+        segments.append(item)
+
+    return {
+        "task": "transcribe",
+        "language": language or "zh",
+        "duration": result["duration"],
+        "text": result["text"],
+        "segments": segments,
+    }
+
+
+# ============================================================
+# FastAPI App
+# ============================================================
+app = FastAPI(title="Fun-ASR-Nano vLLM Server", version="1.0")
+
+
+@app.on_event("startup")
+async def startup():
+    load_engine(_args)
+
+
+# --- HTTP REST: POST /asr ---
+@app.post("/asr")
+async def asr_endpoint(
+    file: UploadFile = File(...),
+    language: str = Form(default=None),
+    hotwords: str = Form(default=""),
+    spk: bool = Form(default=False),
+    timestamp: bool = Form(default=True),
+):
+    """ASR with file upload. Returns text + segments + timestamps + speaker."""
+    content = await file.read()
+    audio_data, sr = sf.read(io.BytesIO(content))
+
+    hw_list = [w.strip() for w in hotwords.split(",") if w.strip()] if hotwords else None
+
+    t0 = time.perf_counter()
+    result = process_audio(audio_data, sr=sr, language=language, 
+                          hotwords=hw_list, use_spk=spk, use_timestamp=timestamp)
+    t1 = time.perf_counter()
+
+    result["processing_time"] = round(t1 - t0, 3)
+    result["rtf"] = round((t1 - t0) / result["duration"], 4) if result["duration"] > 0 else 0
+    return JSONResponse(content=result)
+
+
+# --- OpenAI API: POST /v1/audio/transcriptions ---
+@app.post("/v1/audio/transcriptions")
+async def openai_transcriptions(
+    file: UploadFile = File(...),
+    model: str = Form(default="fun-asr-nano"),
+    language: str = Form(default=None),
+    response_format: str = Form(default="json"),
+    timestamp_granularities: str = Form(default="word"),
+    spk: bool = Form(default=False),
+):
+    """OpenAI Whisper-compatible transcription API (extended with spk support)."""
+    content = await file.read()
+    audio_data, sr = sf.read(io.BytesIO(content))
+
+    use_ts = "word" in timestamp_granularities or "segment" in timestamp_granularities
+    result = process_audio(audio_data, sr=sr, language=language, use_spk=spk, use_timestamp=use_ts)
+
+    if response_format == "text":
+        return JSONResponse(content=result["text"])
+    elif response_format == "verbose_json":
+        return JSONResponse(content=build_openai_verbose_json(result, language=language))
+    else:
+        return JSONResponse(content={"text": result["text"]})
+
+
+# --- WebSocket: ws://host:port/ws ---
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    """Streaming WebSocket ASR with dynamic VAD + SPK."""
+    await websocket.accept()
+    logger.info(f"WebSocket connected: {websocket.client}")
+
+    vad = DynamicStreamingVAD(_vad_model)
+    audio_buffer = np.array([], dtype=np.float32)
+    locked_sentences = []
+    language = None
+    hotwords = None
+    use_spk = False
+    is_active = False
+
+    try:
+        while True:
+            message = await websocket.receive()
+
+            if "text" in message:
+                cmd = message["text"].strip()
+                if cmd.upper() == "START":
+                    vad.reset()
+                    audio_buffer = np.array([], dtype=np.float32)
+                    locked_sentences = []
+                    is_active = True
+                    await websocket.send_json({"event": "started"})
+                elif cmd.upper().startswith("LANGUAGE:"):
+                    language = cmd[9:].strip() or None
+                    await websocket.send_json({"event": "language_set", "language": language})
+                elif cmd.upper().startswith("HOTWORDS:"):
+                    hotwords = [w.strip() for w in cmd[9:].split(",") if w.strip()]
+                    await websocket.send_json({"event": "hotwords_set", "hotwords": hotwords})
+                elif cmd.upper().startswith("SPK:"):
+                    use_spk = cmd[4:].strip().lower() in ("true", "1", "on", "yes")
+                    await websocket.send_json({"event": "spk_set", "spk": use_spk})
+                elif cmd.upper() == "STOP":
+                    if is_active and len(audio_buffer) > 0:
+                        # Final: process remaining audio
+                        final_segs = vad.finalize()
+                        for seg in final_segs:
+                            seg_audio = audio_buffer[int(seg[0]*16):int(seg[1]*16)]
+                            if len(seg_audio) > 8000:
+                                gen_kw = {"max_new_tokens": 500}
+                                if language: gen_kw["language"] = language
+                                if hotwords: gen_kw["hotwords"] = hotwords
+                                res = _engine.generate(inputs=[seg_audio], **gen_kw)
+                                if res[0]["text"].strip():
+                                    locked_sentences.append({
+                                        "text": res[0]["text"], "start": seg[0], "end": seg[1]
+                                    })
+
+                        # Handle ongoing speech
+                        if vad.is_speaking:
+                            end_ms = int(len(audio_buffer) * 1000 / 16000)
+                            start_ms = int(vad.current_speech_start) if hasattr(vad, 'current_speech_start') and vad.current_speech_start else 0
+                            seg_audio = audio_buffer[int(start_ms*16):]
+                            if len(seg_audio) > 8000:
+                                gen_kw = {"max_new_tokens": 500}
+                                if language: gen_kw["language"] = language
+                                if hotwords: gen_kw["hotwords"] = hotwords
+                                res = _engine.generate(inputs=[seg_audio], **gen_kw)
+                                if res[0]["text"].strip():
+                                    locked_sentences.append({
+                                        "text": res[0]["text"], "start": start_ms, "end": end_ms
+                                    })
+
+                        # SPK: run full clustering on all sentences (only if enabled)
+                        if use_spk and locked_sentences and _spk_model is not None:
+                            try:
+                                from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+                                from funasr.models.campplus.cluster_backend import ClusterBackend
+                                vad_segs = [[s["start"]/1000, s["end"]/1000, 
+                                            audio_buffer[int(s["start"]*16):int(s["end"]*16)]]
+                                           for s in locked_sentences]
+                                chunks = sv_chunk(vad_segs)
+                                if chunks:
+                                    speech_list = [ch[2] for ch in chunks]
+                                    spk_res = _spk_model.generate(input=speech_list, cache={}, is_final=True)
+                                    import torch as _torch
+                                    embs = _torch.cat([r["spk_embedding"] for r in spk_res], dim=0)
+                                    cluster = ClusterBackend(merge_thr=0.78).to(_args.device)
+                                    labels = cluster(embs.cpu(), oracle_num=None)
+                                    if not isinstance(labels, np.ndarray):
+                                        labels = np.array(labels)
+                                    all_sorted = sorted(chunks, key=lambda x: x[0])
+                                    sv_output = postprocess(all_sorted, None, labels, embs.cpu())
+                                    spk_sents = [{"text": s["text"], "start": int(s["start"]), "end": int(s["end"])}
+                                                for s in locked_sentences]
+                                    distribute_spk(spk_sents, sv_output)
+                                    for i, ss in enumerate(spk_sents):
+                                        locked_sentences[i]["spk"] = ss.get("spk", 0)
+                            except Exception as e:
+                                logger.warning(f"SPK failed: {e}")
+
+                        await websocket.send_json({
+                            "sentences": locked_sentences,
+                            "is_final": True,
+                            "duration_ms": int(len(audio_buffer) * 1000 / 16000),
+                        })
+                        is_active = False
+                    await websocket.send_json({"event": "stopped"})
+
+            elif "bytes" in message and is_active:
+                pcm = np.frombuffer(message["bytes"], dtype=np.int16).astype(np.float32) / 32768.0
+                audio_buffer = np.concatenate([audio_buffer, pcm])
+
+                # Feed VAD
+                new_confirmed = vad.feed(torch.from_numpy(pcm).float())
+                for seg in new_confirmed:
+                    seg_audio = audio_buffer[int(seg[0]*16):int(seg[1]*16)]
+                    if len(seg_audio) > 8000:
+                        gen_kw = {"max_new_tokens": 500}
+                        if language: gen_kw["language"] = language
+                        if hotwords: gen_kw["hotwords"] = hotwords
+                        res = _engine.generate(inputs=[seg_audio], **gen_kw)
+                        if res[0]["text"].strip():
+                            locked_sentences.append({
+                                "text": res[0]["text"], "start": seg[0], "end": seg[1]
+                            })
+
+                # Send partial update
+                await websocket.send_json({
+                    "sentences": locked_sentences,
+                    "is_final": False,
+                    "duration_ms": int(len(audio_buffer) * 1000 / 16000),
+                })
+
+    except WebSocketDisconnect:
+        logger.info("WebSocket disconnected")
+    except Exception as e:
+        logger.error(f"WebSocket error: {e}", exc_info=True)
+
+
+# ============================================================
+# Main
+# ============================================================
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Fun-ASR-Nano vLLM Server")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--host", type=str, default="0.0.0.0")
+    parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")
+    parser.add_argument("--hub", type=str, default="ms")
+    parser.add_argument("--device", type=str, default="cuda:0")
+    parser.add_argument("--dtype", type=str, default="bf16")
+    parser.add_argument("--max-model-len", type=int, default=4096)
+    parser.add_argument("--gpu-memory-utilization", type=float, default=0.5)
+    parser.add_argument("--vad-model", type=str, default="fsmn-vad", help="VAD model name or local path")
+    parser.add_argument("--spk-model", type=str, default="iic/speech_eres2netv2_sv_zh-cn_16k-common", help="Speaker model name or local path (set empty to disable)")
+    _args = parser.parse_args()
+
+    load_engine(_args)
+    uvicorn.run(app, host=_args.host, port=_args.port)

examples/industrial_data_pretraining/fun_asr_nano/tools/format5res.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+# Author: Mengze Chen
+
+import re
+import sys
+
+
+def scoreformat(name, line, flag=1):
+    newline = ""
+    for i in range(0, len(line)):
+        curr = line[i]
+        currEn = False
+        if curr == "":
+            continue
+        if (
+            (curr >= "\u0041" and curr <= "\u005a")  # eng
+            or (curr >= "\u0061" and curr <= "\u007a")  # eng
+            or (curr >= "\u0000" and curr <= "\u007f")  # de fr es it
+            or (curr >= "\u0400" and curr <= "\u04ff")  # ru
+            or (curr >= "\u0100" and curr <= "\u017f")  # latin1
+            or (curr >= "\u0080" and curr <= "\u00ff")  # latin2
+            or curr == "'"
+        ) and (curr < "\u0030" or curr > "\u0039"):
+            currEn = True
+        if i == 0:
+            newline = newline + curr
+        else:
+            if lastEn == True and currEn == True:
+                newline = newline + curr
+            else:
+                newline = newline + " " + curr
+        if flag == -1:
+            lastEn = False
+        else:
+            lastEn = currEn
+    ret = re.sub("[ ]{1,}", " ", newline)
+    ret = ret
+    if name == "":
+        ret = ret
+    else:
+        if flag <= 0:
+            ret = ret + " " + "(" + name + ")"
+        else:
+            ret = name + "\t" + ret
+    return ret
+
+
+def recoformat(line):
+    newline = ""
+    en_flag = 0  # 0: no-english   1 : english   2: former
+    for i in range(0, len(line)):
+        word = line[i]
+        if ord(word) == 32:
+            if en_flag == 0:
+                continue
+            else:
+                en_flag = 0
+                newline += " "
+        if (word >= "\u4e00" and word <= "\u9fa5") or (word >= "\u0030" and word <= "\u0039"):
+            if en_flag == 1:
+                newline += " " + word
+            else:
+                newline += word
+            en_flag = 0
+        elif (
+            (word >= "\u0041" and word <= "\u005a")  # eng
+            or (word >= "\u0061" and word <= "\u007a")  # eng
+            or (word >= "\u0000" and word <= "\u007f")  # de fr es it
+            or (word >= "\u0400" and word <= "\u04ff")  # ru
+            or (word >= "\u0100" and word <= "\u017f")  # latin1
+            or (word >= "\u0080" and word <= "\u00ff")  # latin2
+            or word == "'"
+        ):
+            if en_flag == 0:
+                newline += " " + ("" if (word == "'") else word)
+            else:
+                newline += word
+            en_flag = 1
+        else:
+            newline += " " + word
+    newline = newline
+    newline = re.sub("[ ]{1,}", " ", newline)
+    newline = newline
+    return newline
+
+
+def numbersingle(line):
+    chnu = ["零", "一", "二", "两", "三", "四", "五", "六", "七", "八", "九", "点"]
+    newline = ""
+    for id in range(len(line)):
+        if re.findall(r"\.", line[id]):
+            if re.findall(r"\.\s*$", line[id]):
+                newline += "."
+            else:
+                newline += chnu[10]
+        elif re.search(r"0", line[id]):
+            if id > 0 and id < len(line) - 1:
+                if (
+                    re.search(r"\d", line[id - 1])
+                    and (not re.search(r"\d", line[id + 1]))
+                    and (not re.search(r"0", line[id - 1]))
+                ):
+                    if id > 2 and len(line) > 2 and (not re.search(r"\d", line[id - 1])):
+                        newline = newline[:-1]
+                        newline += chnu[int(line[id - 1])] + "十"
+                    else:
+                        newline += chnu[int(line[id])]
+                else:
+                    newline += chnu[int(line[id])]
+            else:
+                newline += chnu[int(line[id])]
+        elif re.search(r"\d", line[id]):
+            newline += chnu[int(line[id])]
+        else:
+            newline += line[id]
+    return newline
+
+
+def ch_number2digit(line):
+    number_flag = 0
+    zero_flag = 0
+    bits = {
+        "零": "1",
+        "十": "2",
+        "百": "3",
+        "千": "4",
+        "万": "5",
+        "十万": "6",
+        "百万": "7",
+        "千万": "8",
+    }
+    chsh = {
+        "一": "1",
+        "二": "2",
+        "三": "3",
+        "四": "4",
+        "五": "5",
+        "六": "6",
+        "七": "7",
+        "八": "8",
+        "九": "9",
+        "两": "2",
+        "幺": "1",
+    }
+    unit = {"里": "1", "克": "1", "米": "1"}
+    newline = ""
+    digit = []
+    bit = []
+    onebit = ""
+    for i in range(len(line)):
+        if ord(line[i]) == 32:
+            newline += " "
+            continue
+        if line[i] in chsh:
+            number_flag = 1
+            if line[i] == "两":
+                if (i == len(line) - 1) or ((line[i + 1] not in chsh.keys()) and (line[i + 1] not in bits.keys())):
+                    number_flag = -1
+            if number_flag == 1:
+                digit.append(chsh[line[i]])
+
+        elif "十" == line[i] and number_flag == 0:
+            number_flag = 2
+            digit.append("1")
+            bit.append(line[i])
+        elif "十" == line[i] and number_flag == 3:
+            digit.append("1")
+            bit.append(line[i])
+        elif ("零" == line[i]) and (number_flag == 0 or number_flag == 1):
+            digit.append("0")
+        elif ("零" == line[i]) and number_flag == 3:
+            zero_flag = 1
+        elif number_flag == 1 and line[i] in bits:
+            number_flag = 3
+            if line[i] == "千":
+                if i < len(line) - 1:
+                    if line[i + 1] in unit:
+                        number_flag = -1
+            if number_flag == 3:
+                onebit = line[i]
+                bit.append(onebit)
+        elif number_flag == 3 and line[i] in bits:
+            onebit = bit[-1] + line[i]
+            if onebit in bits:
+                bit[-1] = onebit
+            else:
+                number_flag = -2
+        else:
+            number_flag = -1
+        if len(digit) > 0 and number_flag == -1:
+            number_flag = -2
+        if i == (len(line) - 1) and number_flag >= 0:
+            number_flag = -1
+        if number_flag < 0:
+            newdigit = ""
+            if len(digit) > 0:  # and (len(digit) == len(bit))):
+                if len(bit) == 1 and zero_flag == 0 and bit[0] == "百" and len(bit) != len(digit):
+                    bit.append("十")
+                if len(digit) == (len(bit) + 1):
+                    bit.append("零")
+                if len(digit) == len(bit):
+                    for m in range(len(digit))[-1::-1]:
+                        if int(bits[bit[m]]) == int(len(newdigit) + 1):
+                            newdigit += digit[m]
+                        else:
+                            nu = int(bits[bit[m]]) - len(newdigit) - 1
+                            for n in range(nu):
+                                newdigit += "0"
+                            newdigit += digit[m]
+                    for z in range(len(newdigit))[-1::-1]:
+                        newline += newdigit[z]
+                else:
+                    newline += "".join(digit)
+                bit = []
+                digit = []
+                zero_flag = 0
+            else:
+                newline += line[i]
+            if number_flag == -2:
+                newline += line[i]
+            number_flag = 0
+    return newline
+
+
+def special(line):
+    newline = ""
+    for e in range(len(line)):
+        if ord(line[e]) == 247:
+            newline += "除以"
+        elif ord(line[e]) == 215:
+            newline += "乘以"
+        elif ord(line[e]) == 61:
+            newline += "等于"
+        elif ord(line[e]) == 43:
+            newline += "加"
+        elif ord(line[e]) == 45:
+            newline += "负"
+        elif ord(line[e]) == 8451:
+            newline += "摄氏度"
+        elif ord(line[e]) == 13217:
+            newline += "平方米"
+        elif ord(line[e]) == 8240 or ord(line[e]) == 65130:
+            newline += "%"
+        elif ord(line[e]) == 46:
+            newline += "点"
+        elif ord(line[e]) == 176:
+            newline += "度"
+            angel = 1
+        elif ord(line[e]) == 8242 and angel == 1:
+            newline += "分"
+        else:
+            newline += line[e]
+    return newline
+
+
+def all_convert(content):
+    content = recoformat(content)
+    content = numbersingle(content)
+    content = ch_number2digit(content)
+    content = special(content)
+    content = scoreformat("", content)
+    return content
+
+
+if __name__ == "__main__":
+    if len(sys.argv[1:]) < 1:
+        sys.stderr.write("Usage:\n .py  reco.result\n")
+        sys.stderr.write(" reco.result:   id<tab>recoresult\n")
+        sys.exit(1)
+    f = open(sys.argv[1])
+    flag = 0
+    if len(sys.argv[1:]) > 1:
+        flag = int(sys.argv[2])
+    for line in f.readlines():
+        if not line:
+            continue
+        line = line.rstrip()
+        tmp = line.split("\t")
+        if len(tmp) < 2:
+            tmp = line.split(",")
+            if len(tmp) < 2:
+                tmp = line.split(" ", 1)
+                if len(tmp) < 2:
+                    name = tmp[0]
+                    content = ""
+                    print(content)
+                    continue
+        name = tmp[0]
+        content = tmp[1]
+        name = re.sub("\.pcm", "", name)
+        name = re.sub("\.wav", "", name)
+        content = recoformat(content)
+        content = numbersingle(content)
+        content = ch_number2digit(content)
+        content = special(content)
+        content = scoreformat(name, content, flag)
+        print(content)
+    f.close()

examples/industrial_data_pretraining/fun_asr_nano/tools/scp2jsonl.py

@@ -0,0 +1,137 @@
+import hydra
+import json
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from io import BytesIO
+from typing import Dict, Optional, Tuple
+from urllib.request import urlopen
+
+import soundfile as sf
+from modelscope import AutoTokenizer
+from tqdm import tqdm
+from omegaconf import DictConfig, OmegaConf, ListConfig
+
+
+class LineProcessor:
+    def __init__(self, tokenizer):
+        self.tokenizer = tokenizer
+        self.lock = threading.Lock()
+
+    def process_line(self, line_pair: Tuple[str, str]) -> Optional[Dict]:
+        line1, line2 = line_pair
+
+        line1, line2 = line1.strip(), line2.strip()
+        if not line1 or not line2:
+            return None
+
+        parts1, parts2 = line1.split(maxsplit=1), line2.split(maxsplit=1)
+        if len(parts1) != 2 or len(parts2) != 2:
+            return None
+
+        utt1, utt2 = parts1[0], parts2[0]
+        wav_path, text = parts1[1], parts2[1]
+
+        if utt1 != utt2:
+            return {"error": f"UTT mismatch: {utt1} vs {utt2}"}
+
+        try:
+            if wav_path.startswith("http"):
+                response = urlopen(wav_path)
+                if response.status != 200:
+                    return {"error": f"WAV not found: {wav_path}"}
+                audio_file = BytesIO(response.read())
+                duration = sf.info(audio_file).duration
+            else:
+                if not os.path.exists(wav_path):
+                    return {"error": f"WAV not found: {wav_path}"}
+                duration = sf.info(wav_path).duration
+
+            data = {
+                "messages": [
+                    {"role": "system", "content": "You are a helpful assistant."},
+                    {
+                        "role": "user",
+                        "content": f"语音转写：<|startofspeech|>!{wav_path}<|endofspeech|>",
+                    },
+                    {"role": "assistant", "content": text},
+                ],
+                "speech_length": int((duration * 1000 - 25) // 10 + 1),
+                "text_length": len(self.tokenizer.tokenize(text)),
+            }
+            return {"success": data, "utt": utt1}
+
+        except Exception as e:
+            return {"error": f"Error processing {wav_path}: {str(e)}"}
+
+
+@hydra.main(config_name=None, version_base=None)
+def main_hydra(cfg: DictConfig):
+    def to_plain_list(cfg_item):
+        if isinstance(cfg_item, ListConfig):
+            return OmegaConf.to_container(cfg_item, resolve=True)
+        elif isinstance(cfg_item, DictConfig):
+            return {k: to_plain_list(v) for k, v in cfg_item.items()}
+        else:
+            return cfg_item
+
+    kwargs = to_plain_list(cfg)
+    scp_file = kwargs["scp_file"]
+    transcript_file = kwargs["transcript_file"]
+    max_workers = kwargs.get("max_workers", os.cpu_count())
+    jsonl_file = kwargs["jsonl_file"]
+
+    with open(scp_file, "r") as f1, open(transcript_file, "r") as f2:
+        scp_lines = f1.readlines()
+        transcript_lines = f2.readlines()
+
+    if len(scp_lines) != len(transcript_lines):
+        print(f"Warning: Line count mismatch - scp: {len(scp_lines)}, transcript: {len(transcript_lines)}")
+
+    tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
+    processor = LineProcessor(tokenizer)
+
+    data_pairs = list(zip(scp_lines, transcript_lines))
+
+    processed_count = 0
+    failed_count = 0
+    error_messages = []
+
+    with tqdm(total=len(data_pairs), desc="Processing") as pbar:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            with open(jsonl_file, "w") as f_out:
+                futures = {executor.submit(processor.process_line, pair): i for i, pair in enumerate(data_pairs)}
+
+                for future in as_completed(futures):
+                    result = future.result()
+
+                    if result and "success" in result:
+                        with processor.lock:
+                            json.dump(result["success"], f_out, ensure_ascii=False)
+                            f_out.write("\n")
+                        processed_count += 1
+                    elif result and "error" in result:
+                        failed_count += 1
+                        error_messages.append(result["error"])
+
+                    pbar.update(1)
+                    pbar.set_postfix({"processed": processed_count, "failed": failed_count})
+
+    print(f"\nProcessing completed:")
+    print(f"  Total lines: {len(data_pairs)}")
+    print(f"  Successfully processed: {processed_count}")
+    print(f"  Failed: {failed_count}")
+
+    if error_messages and len(error_messages) <= 10:
+        print(f"\nSample errors:")
+        for error in error_messages[:10]:
+            print(f"  - {error}")
+    elif error_messages:
+        print(f"\nFirst 10 errors:")
+        for error in error_messages[:10]:
+            print(f"  - {error}")
+        print(f"  ... and {len(error_messages) - 10} more errors")
+
+
+if __name__ == "__main__":
+    main_hydra()

examples/industrial_data_pretraining/fun_asr_nano/tools/utils.py

@@ -0,0 +1,57 @@
+from itertools import groupby
+
+import soundfile as sf
+import torch
+import torchaudio
+import torchaudio.functional as F
+
+
+def load_audio(wav_path, rate: int = None, offset: float = 0, duration: float = None):
+    with sf.SoundFile(wav_path) as f:
+        start_frame = int(offset * f.samplerate)
+        if duration is None:
+            frames_to_read = f.frames - start_frame
+        else:
+            frames_to_read = int(duration * f.samplerate)
+        f.seek(start_frame)
+        audio_data = f.read(frames_to_read, dtype="float32")
+    audio_tensor = torch.from_numpy(audio_data)
+    if rate is not None and f.samplerate != rate:
+        if audio_tensor.ndim == 1:
+            audio_tensor = audio_tensor.unsqueeze(0)
+        else:
+            audio_tensor = audio_tensor.T
+        resampler = torchaudio.transforms.Resample(orig_freq=f.samplerate, new_freq=rate)
+        audio_tensor = resampler(audio_tensor)
+        if audio_tensor.shape[0] == 1:
+            audio_tensor = audio_tensor.squeeze(0)
+    return audio_tensor, rate if rate is not None else f.samplerate
+
+
+def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0):
+    items = []
+    try:
+        # The current version only supports batch_size==1.
+        log_probs, targets = log_probs.unsqueeze(0).cpu(), targets.unsqueeze(0).cpu()
+        assert log_probs.shape[1] >= targets.shape[1]
+        alignments, scores = F.forced_align(log_probs, targets, blank=blank)
+        alignments, scores = alignments[0], torch.exp(scores[0]).tolist()
+        # use enumerate to keep track of the original indices, then group by token value
+        for token, group in groupby(enumerate(alignments), key=lambda item: item[1]):
+            if token == blank:
+                continue
+            group = list(group)
+            start = group[0][0]
+            end = start + len(group)
+            score = max(scores[start:end])
+            items.append(
+                {
+                    "token": token.item(),
+                    "start_time": start,
+                    "end_time": end,
+                    "score": round(score, 3),
+                }
+            )
+    except:
+        pass
+    return items

examples/industrial_data_pretraining/fun_asr_nano/tools/whisper_mix_normalize.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/python
+# Author: Mengze Chen
+
+import re
+import sys
+
+import cn_tn as cn_tn
+import format5res as cn_itn
+import pyopenjtalk
+import zhconv
+from whisper_normalizer.basic import BasicTextNormalizer
+from whisper_normalizer.english import EnglishTextNormalizer
+
+basic_normalizer = BasicTextNormalizer()
+english_normalizer = EnglishTextNormalizer()
+
+
+def is_only_chinese_and_english(s):
+    # 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
+    pattern = r"^[\u4e00-\u9fa5A-Za-z0-9,\.!\?:;，。！？：；、%\'\s\-\~]+$"
+    # 使用正则表达式进行匹配
+    return re.match(pattern, s) is not None
+
+
+def is_only_english(s):
+    # 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
+    pattern = r"^[A-Za-z0-9,\.!\?:;，。！？：；、%\'\s\-\~]+$"
+    # 使用正则表达式进行匹配
+    return re.match(pattern, s) is not None
+
+
+def is_number(s):
+    # 定义正则表达式模式，匹配中文字符范围和英文字母（包括大小写）
+    pattern = r"^[0-9,\.!\?:;，。！？：；、%\'\s]+$"
+    # 使用正则表达式进行匹配
+    return re.match(pattern, s) is not None
+
+
+def safe_ja_g2p(text, kana=True, max_length=100):
+    if len(text) > max_length:
+        # 如果文本过长，分段处理
+        parts = []
+        for i in range(0, len(text), max_length):
+            part = text[i : i + max_length]
+            try:
+                converted = pyopenjtalk.g2p(part, kana=kana)
+                parts.append(converted)
+            except:
+                parts.append(part)  # 如果转换失败，使用原文本
+        return " ".join(parts)
+    else:
+        try:
+            return pyopenjtalk.g2p(text, kana=kana)
+        except:
+            return text  # 如果转换失败，返回原文本
+
+
+def normalize_text(srcfn, dstfn, kana=False):
+    with open(srcfn, "r") as f_read, open(dstfn, "w") as f_write:
+        all_lines = f_read.readlines()
+        for line in all_lines:
+            line = line.strip()
+            line_arr = line.split(maxsplit=1)
+            if len(line_arr) < 1:
+                continue
+            if len(line_arr) == 1:
+                line_arr.append("")
+            key = line_arr[0]
+            line_arr[1] = re.sub(r"=", " ", line_arr[1])
+            line_arr[1] = re.sub(r"\(", " ", line_arr[1])
+            line_arr[1] = re.sub(r"\)", " ", line_arr[1])
+            # From Chongjia Ni
+            if kana:
+                line_arr[1] = safe_ja_g2p(line_arr[1], kana=True, max_length=100)
+
+            line_arr = f"{key}\t{line_arr[1]}".split()
+            conts = []
+            language_bak = ""
+            part = []
+            for i in range(1, len(line_arr)):
+                out_part = ""
+                chn_eng_bool = is_only_chinese_and_english(line_arr[i])
+                eng_bool = is_only_english(line_arr[i])
+                num_bool = is_number(line_arr[i])
+                if eng_bool and not num_bool:
+                    language = "en"
+                elif chn_eng_bool:
+                    language = "chn_en"
+                else:
+                    language = "not_chn_en"
+                if language == language_bak or language_bak == "":
+                    part.append(line_arr[i])
+                    language_bak = language
+                else:
+                    if language_bak == "en":
+                        out_part1 = english_normalizer(" ".join(part))
+                        out_part = cn_itn.scoreformat("", out_part1)
+                    elif language_bak == "chn_en":
+                        out_part1 = english_normalizer(" ".join(part))
+                        out_part2 = cn_tn.normalize_nsw(out_part1)
+                        out_part3 = cn_itn.all_convert(out_part2)
+                        out_part = zhconv.convert(out_part3, "zh-cn")
+                    else:
+                        out_part1 = basic_normalizer(" ".join(part))
+                        out_part2 = cn_tn.normalize_nsw(out_part1)
+                        out_part3 = cn_itn.all_convert(out_part2)
+                        out_part = zhconv.convert(out_part3, "zh-cn")
+                    conts.append(out_part)
+                    language_bak = language
+                    part = []
+                    part.append(line_arr[i])
+                if i == len(line_arr) - 1:
+                    if language == "en":
+                        out_part1 = english_normalizer(" ".join(part))
+                        out_part = cn_itn.scoreformat("", out_part1)
+                    elif language == "chn_en":
+                        out_part1 = english_normalizer(" ".join(part))
+                        out_part2 = cn_tn.normalize_nsw(out_part1)
+                        out_part3 = cn_itn.all_convert(out_part2)
+                        out_part = zhconv.convert(out_part3, "zh-cn")
+                    else:
+                        out_part1 = basic_normalizer(" ".join(part))
+                        out_part2 = cn_tn.normalize_nsw(out_part1)
+                        out_part3 = cn_itn.all_convert(out_part2)
+                        out_part = zhconv.convert(out_part3, "zh-cn")
+                    conts.append(out_part)
+
+            f_write.write("{0}\t{1}\n".format(key, " ".join(conts).strip()))
+
+
+if __name__ == "__main__":
+    srcfn = sys.argv[1]
+    dstfn = sys.argv[2]
+    normalize_text(srcfn, dstfn, True if len(sys.argv) > 3 else False)

examples/industrial_data_pretraining/fun_asr_nano/热词列表

@@ -0,0 +1,4 @@
+兔子
+窝边草
+土匪
+师爷

examples/industrial_data_pretraining/glm_asr/demo.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# Requirements: pip install transformers>=5.0.0 torchaudio
+
+from funasr import AutoModel
+
+# Initialize GLM-ASR model
+# hub: "ms" for ModelScope (default), "hf" for HuggingFace
+model = AutoModel(
+    model="zai-org/GLM-ASR-Nano-2512",
+    hub="hf",
+    device="cuda:0",
+    dtype="bf16",
+)
+
+# Chinese speech recognition
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav",
+)
+print("Chinese:", res[0]["text"])
+
+# English speech recognition
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav",
+)
+print("English:", res[0]["text"])

examples/industrial_data_pretraining/glm_asr/demo_ms.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+# Requirements: pip install transformers>=5.0.0 torchaudio
+
+from funasr import AutoModel
+
+# Download from ModelScope (recommended for Chinese users)
+# ModelScope model id: ZhipuAI/GLM-ASR-Nano-2512
+model = AutoModel(
+    model="ZhipuAI/GLM-ASR-Nano-2512",
+    hub="ms",
+    device="cuda:0",
+    dtype="bf16",
+)
+
+res = model.generate(
+    input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav",
+)
+print(res[0]["text"])

examples/industrial_data_pretraining/lcbnet/compute_wer_details.py

@@ -0,0 +1,735 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+from enum import Enum
+import re, sys, unicodedata
+import codecs
+import argparse
+from tqdm import tqdm
+import os
+import pdb
+
+remove_tag = False
+spacelist = [" ", "\t", "\r", "\n"]
+puncts = [
+    "!",
+    ",",
+    "?",
+    "、",
+    "。",
+    "！",
+    "，",
+    "；",
+    "？",
+    "：",
+    "「",
+    "」",
+    "︰",
+    "『",
+    "』",
+    "《",
+    "》",
+]
+
+
+class Code(Enum):
+    match = 1
+    substitution = 2
+    insertion = 3
+    deletion = 4
+
+
+class WordError(object):
+    def __init__(self):
+        self.errors = {
+            Code.substitution: 0,
+            Code.insertion: 0,
+            Code.deletion: 0,
+        }
+        self.ref_words = 0
+
+    def get_wer(self):
+        assert self.ref_words != 0
+        errors = (
+            self.errors[Code.substitution]
+            + self.errors[Code.insertion]
+            + self.errors[Code.deletion]
+        )
+        return 100.0 * errors / self.ref_words
+
+    def get_result_string(self):
+        return (
+            f"error_rate={self.get_wer():.4f}, "
+            f"ref_words={self.ref_words}, "
+            f"subs={self.errors[Code.substitution]}, "
+            f"ins={self.errors[Code.insertion]}, "
+            f"dels={self.errors[Code.deletion]}"
+        )
+
+
+def characterize(string):
+    res = []
+    i = 0
+    while i < len(string):
+        char = string[i]
+        if char in puncts:
+            i += 1
+            continue
+        cat1 = unicodedata.category(char)
+        # https://unicodebook.readthedocs.io/unicode.html#unicode-categories
+        if cat1 == "Zs" or cat1 == "Cn" or char in spacelist:  # space or not assigned
+            i += 1
+            continue
+        if cat1 == "Lo":  # letter-other
+            res.append(char)
+            i += 1
+        else:
+            # some input looks like: <unk><noise>, we want to separate it to two words.
+            sep = " "
+            if char == "<":
+                sep = ">"
+            j = i + 1
+            while j < len(string):
+                c = string[j]
+                if ord(c) >= 128 or (c in spacelist) or (c == sep):
+                    break
+                j += 1
+            if j < len(string) and string[j] == ">":
+                j += 1
+            res.append(string[i:j])
+            i = j
+    return res
+
+
+def stripoff_tags(x):
+    if not x:
+        return ""
+    chars = []
+    i = 0
+    T = len(x)
+    while i < T:
+        if x[i] == "<":
+            while i < T and x[i] != ">":
+                i += 1
+            i += 1
+        else:
+            chars.append(x[i])
+            i += 1
+    return "".join(chars)
+
+
+def normalize(sentence, ignore_words, cs, split=None):
+    """sentence, ignore_words are both in unicode"""
+    new_sentence = []
+    for token in sentence:
+        x = token
+        if not cs:
+            x = x.upper()
+        if x in ignore_words:
+            continue
+        if remove_tag:
+            x = stripoff_tags(x)
+        if not x:
+            continue
+        if split and x in split:
+            new_sentence += split[x]
+        else:
+            new_sentence.append(x)
+    return new_sentence
+
+
+class Calculator:
+    def __init__(self):
+        self.data = {}
+        self.space = []
+        self.cost = {}
+        self.cost["cor"] = 0
+        self.cost["sub"] = 1
+        self.cost["del"] = 1
+        self.cost["ins"] = 1
+
+    def calculate(self, lab, rec):
+        # Initialization
+        lab.insert(0, "")
+        rec.insert(0, "")
+        while len(self.space) < len(lab):
+            self.space.append([])
+        for row in self.space:
+            for element in row:
+                element["dist"] = 0
+                element["error"] = "non"
+            while len(row) < len(rec):
+                row.append({"dist": 0, "error": "non"})
+        for i in range(len(lab)):
+            self.space[i][0]["dist"] = i
+            self.space[i][0]["error"] = "del"
+        for j in range(len(rec)):
+            self.space[0][j]["dist"] = j
+            self.space[0][j]["error"] = "ins"
+        self.space[0][0]["error"] = "non"
+        for token in lab:
+            if token not in self.data and len(token) > 0:
+                self.data[token] = {"all": 0, "cor": 0, "sub": 0, "ins": 0, "del": 0}
+        for token in rec:
+            if token not in self.data and len(token) > 0:
+                self.data[token] = {"all": 0, "cor": 0, "sub": 0, "ins": 0, "del": 0}
+        # Computing edit distance
+        for i, lab_token in enumerate(lab):
+            for j, rec_token in enumerate(rec):
+                if i == 0 or j == 0:
+                    continue
+                min_dist = sys.maxsize
+                min_error = "none"
+                dist = self.space[i - 1][j]["dist"] + self.cost["del"]
+                error = "del"
+                if dist < min_dist:
+                    min_dist = dist
+                    min_error = error
+                dist = self.space[i][j - 1]["dist"] + self.cost["ins"]
+                error = "ins"
+                if dist < min_dist:
+                    min_dist = dist
+                    min_error = error
+                if lab_token == rec_token.replace("<BIAS>", ""):
+                    dist = self.space[i - 1][j - 1]["dist"] + self.cost["cor"]
+                    error = "cor"
+                else:
+                    dist = self.space[i - 1][j - 1]["dist"] + self.cost["sub"]
+                    error = "sub"
+                if dist < min_dist:
+                    min_dist = dist
+                    min_error = error
+                self.space[i][j]["dist"] = min_dist
+                self.space[i][j]["error"] = min_error
+        # Tracing back
+        result = {
+            "lab": [],
+            "rec": [],
+            "code": [],
+            "all": 0,
+            "cor": 0,
+            "sub": 0,
+            "ins": 0,
+            "del": 0,
+        }
+        i = len(lab) - 1
+        j = len(rec) - 1
+        while True:
+            if self.space[i][j]["error"] == "cor":  # correct
+                if len(lab[i]) > 0:
+                    self.data[lab[i]]["all"] = self.data[lab[i]]["all"] + 1
+                    self.data[lab[i]]["cor"] = self.data[lab[i]]["cor"] + 1
+                    result["all"] = result["all"] + 1
+                    result["cor"] = result["cor"] + 1
+                result["lab"].insert(0, lab[i])
+                result["rec"].insert(0, rec[j])
+                result["code"].insert(0, Code.match)
+                i = i - 1
+                j = j - 1
+            elif self.space[i][j]["error"] == "sub":  # substitution
+                if len(lab[i]) > 0:
+                    self.data[lab[i]]["all"] = self.data[lab[i]]["all"] + 1
+                    self.data[lab[i]]["sub"] = self.data[lab[i]]["sub"] + 1
+                    result["all"] = result["all"] + 1
+                    result["sub"] = result["sub"] + 1
+                result["lab"].insert(0, lab[i])
+                result["rec"].insert(0, rec[j])
+                result["code"].insert(0, Code.substitution)
+                i = i - 1
+                j = j - 1
+            elif self.space[i][j]["error"] == "del":  # deletion
+                if len(lab[i]) > 0:
+                    self.data[lab[i]]["all"] = self.data[lab[i]]["all"] + 1
+                    self.data[lab[i]]["del"] = self.data[lab[i]]["del"] + 1
+                    result["all"] = result["all"] + 1
+                    result["del"] = result["del"] + 1
+                result["lab"].insert(0, lab[i])
+                result["rec"].insert(0, "")
+                result["code"].insert(0, Code.deletion)
+                i = i - 1
+            elif self.space[i][j]["error"] == "ins":  # insertion
+                if len(rec[j]) > 0:
+                    self.data[rec[j]]["ins"] = self.data[rec[j]]["ins"] + 1
+                    result["ins"] = result["ins"] + 1
+                result["lab"].insert(0, "")
+                result["rec"].insert(0, rec[j])
+                result["code"].insert(0, Code.insertion)
+                j = j - 1
+            elif self.space[i][j]["error"] == "non":  # starting point
+                break
+            else:  # shouldn't reach here
+                print(
+                    "this should not happen , i = {i} , j = {j} , error = {error}".format(
+                        i=i, j=j, error=self.space[i][j]["error"]
+                    )
+                )
+        return result
+
+    def overall(self):
+        result = {"all": 0, "cor": 0, "sub": 0, "ins": 0, "del": 0}
+        for token in self.data:
+            result["all"] = result["all"] + self.data[token]["all"]
+            result["cor"] = result["cor"] + self.data[token]["cor"]
+            result["sub"] = result["sub"] + self.data[token]["sub"]
+            result["ins"] = result["ins"] + self.data[token]["ins"]
+            result["del"] = result["del"] + self.data[token]["del"]
+        return result
+
+    def cluster(self, data):
+        result = {"all": 0, "cor": 0, "sub": 0, "ins": 0, "del": 0}
+        for token in data:
+            if token in self.data:
+                result["all"] = result["all"] + self.data[token]["all"]
+                result["cor"] = result["cor"] + self.data[token]["cor"]
+                result["sub"] = result["sub"] + self.data[token]["sub"]
+                result["ins"] = result["ins"] + self.data[token]["ins"]
+                result["del"] = result["del"] + self.data[token]["del"]
+        return result
+
+    def keys(self):
+        return list(self.data.keys())
+
+
+def width(string):
+    return sum(1 + (unicodedata.east_asian_width(c) in "AFW") for c in string)
+
+
+def default_cluster(word):
+    unicode_names = [unicodedata.name(char) for char in word]
+    for i in reversed(range(len(unicode_names))):
+        if unicode_names[i].startswith("DIGIT"):  # 1
+            unicode_names[i] = "Number"  # 'DIGIT'
+        elif unicode_names[i].startswith("CJK UNIFIED IDEOGRAPH") or unicode_names[i].startswith(
+            "CJK COMPATIBILITY IDEOGRAPH"
+        ):
+            # 明 / 郎
+            unicode_names[i] = "Mandarin"  # 'CJK IDEOGRAPH'
+        elif unicode_names[i].startswith("LATIN CAPITAL LETTER") or unicode_names[i].startswith(
+            "LATIN SMALL LETTER"
+        ):
+            # A / a
+            unicode_names[i] = "English"  # 'LATIN LETTER'
+        elif unicode_names[i].startswith("HIRAGANA LETTER"):  # は こ め
+            unicode_names[i] = "Japanese"  # 'GANA LETTER'
+        elif (
+            unicode_names[i].startswith("AMPERSAND")
+            or unicode_names[i].startswith("APOSTROPHE")
+            or unicode_names[i].startswith("COMMERCIAL AT")
+            or unicode_names[i].startswith("DEGREE CELSIUS")
+            or unicode_names[i].startswith("EQUALS SIGN")
+            or unicode_names[i].startswith("FULL STOP")
+            or unicode_names[i].startswith("HYPHEN-MINUS")
+            or unicode_names[i].startswith("LOW LINE")
+            or unicode_names[i].startswith("NUMBER SIGN")
+            or unicode_names[i].startswith("PLUS SIGN")
+            or unicode_names[i].startswith("SEMICOLON")
+        ):
+            # & / ' / @ / ℃ / = / . / - / _ / # / + / ;
+            del unicode_names[i]
+        else:
+            return "Other"
+    if len(unicode_names) == 0:
+        return "Other"
+    if len(unicode_names) == 1:
+        return unicode_names[0]
+    for i in range(len(unicode_names) - 1):
+        if unicode_names[i] != unicode_names[i + 1]:
+            return "Other"
+    return unicode_names[0]
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description="wer cal")
+    parser.add_argument("--ref", type=str, help="Text input path")
+    parser.add_argument("--ref_ocr", type=str, help="Text input path")
+    parser.add_argument("--rec_name", type=str, action="append", default=[])
+    parser.add_argument("--rec_file", type=str, action="append", default=[])
+    parser.add_argument("--verbose", type=int, default=1, help="show")
+    parser.add_argument("--char", type=bool, default=True, help="show")
+    args = parser.parse_args()
+    return args
+
+
+def main(args):
+    cluster_file = ""
+    ignore_words = set()
+    tochar = args.char
+    verbose = args.verbose
+    padding_symbol = " "
+    case_sensitive = False
+    max_words_per_line = sys.maxsize
+    split = None
+
+    if not case_sensitive:
+        ig = set([w.upper() for w in ignore_words])
+        ignore_words = ig
+
+    default_clusters = {}
+    default_words = {}
+    ref_file = args.ref
+    ref_ocr = args.ref_ocr
+    rec_files = args.rec_file
+    rec_names = args.rec_name
+    assert len(rec_files) == len(rec_names)
+
+    # load ocr
+    ref_ocr_dict = {}
+    with codecs.open(ref_ocr, "r", "utf-8") as fh:
+        for line in fh:
+            if "$" in line:
+                line = line.replace("$", " ")
+            if tochar:
+                array = characterize(line)
+            else:
+                array = line.strip().split()
+            if len(array) == 0:
+                continue
+            fid = array[0]
+            ref_ocr_dict[fid] = normalize(array[1:], ignore_words, case_sensitive, split)
+
+    if split and not case_sensitive:
+        newsplit = dict()
+        for w in split:
+            words = split[w]
+            for i in range(len(words)):
+                words[i] = words[i].upper()
+            newsplit[w.upper()] = words
+        split = newsplit
+
+    rec_sets = {}
+    calculators_dict = dict()
+    ub_wer_dict = dict()
+    hotwords_related_dict = dict()  # 记录recall相关的内容
+    for i, hyp_file in enumerate(rec_files):
+        rec_sets[rec_names[i]] = dict()
+        with codecs.open(hyp_file, "r", "utf-8") as fh:
+            for line in fh:
+                if tochar:
+                    array = characterize(line)
+                else:
+                    array = line.strip().split()
+                if len(array) == 0:
+                    continue
+                fid = array[0]
+                rec_sets[rec_names[i]][fid] = normalize(
+                    array[1:], ignore_words, case_sensitive, split
+                )
+
+        calculators_dict[rec_names[i]] = Calculator()
+        ub_wer_dict[rec_names[i]] = {"u_wer": WordError(), "b_wer": WordError(), "wer": WordError()}
+        hotwords_related_dict[rec_names[i]] = {"tp": 0, "tn": 0, "fp": 0, "fn": 0}
+        # tp: 热词在label里，同时在rec里
+        # tn: 热词不在label里，同时不在rec里
+        # fp: 热词不在label里，但是在rec里
+        # fn: 热词在label里，但是不在rec里
+
+    # record wrong label but in ocr
+    wrong_rec_but_in_ocr_dict = {}
+    for rec_name in rec_names:
+        wrong_rec_but_in_ocr_dict[rec_name] = 0
+
+    _file_total_len = 0
+    with os.popen("cat {} | wc -l".format(ref_file)) as pipe:
+        _file_total_len = int(pipe.read().strip())
+
+    # compute error rate on the interaction of reference file and hyp file
+    for line in tqdm(open(ref_file, "r", encoding="utf-8"), total=_file_total_len):
+        if tochar:
+            array = characterize(line)
+        else:
+            array = line.rstrip("\n").split()
+        if len(array) == 0:
+            continue
+        fid = array[0]
+        lab = normalize(array[1:], ignore_words, case_sensitive, split)
+
+        if verbose:
+            print("\nutt: %s" % fid)
+
+        ocr_text = ref_ocr_dict[fid]
+        ocr_set = set(ocr_text)
+        print("ocr: {}".format(" ".join(ocr_text)))
+        list_match = []  # 指label里面在ocr里面的内容
+        list_not_mathch = []
+        tmp_error = 0
+        tmp_match = 0
+        for index in range(len(lab)):
+            # text_list.append(uttlist[index+1])
+            if lab[index] not in ocr_set:
+                tmp_error += 1
+                list_not_mathch.append(lab[index])
+            else:
+                tmp_match += 1
+                list_match.append(lab[index])
+        print("label in ocr: {}".format(" ".join(list_match)))
+
+        # for each reco file
+        base_wrong_ocr_wer = None
+        ocr_wrong_ocr_wer = None
+
+        for rec_name in rec_names:
+            rec_set = rec_sets[rec_name]
+            if fid not in rec_set:
+                continue
+            rec = rec_set[fid]
+
+            # print(rec)
+            for word in rec + lab:
+                if word not in default_words:
+                    default_cluster_name = default_cluster(word)
+                    if default_cluster_name not in default_clusters:
+                        default_clusters[default_cluster_name] = {}
+                    if word not in default_clusters[default_cluster_name]:
+                        default_clusters[default_cluster_name][word] = 1
+                    default_words[word] = default_cluster_name
+
+            result = calculators_dict[rec_name].calculate(lab.copy(), rec.copy())
+            if verbose:
+                if result["all"] != 0:
+                    wer = (
+                        float(result["ins"] + result["sub"] + result["del"]) * 100.0 / result["all"]
+                    )
+                else:
+                    wer = 0.0
+            print("WER(%s): %4.2f %%" % (rec_name, wer), end=" ")
+            print(
+                "N=%d C=%d S=%d D=%d I=%d"
+                % (result["all"], result["cor"], result["sub"], result["del"], result["ins"])
+            )
+
+            # print(result['rec'])
+            wrong_rec_but_in_ocr = []
+            for idx in range(len(result["lab"])):
+                if result["lab"][idx] != "":
+                    if result["lab"][idx] != result["rec"][idx].replace("<BIAS>", ""):
+                        if result["lab"][idx] in list_match:
+                            wrong_rec_but_in_ocr.append(result["lab"][idx])
+                            wrong_rec_but_in_ocr_dict[rec_name] += 1
+            print("wrong_rec_but_in_ocr: {}".format(" ".join(wrong_rec_but_in_ocr)))
+
+            if rec_name == "base":
+                base_wrong_ocr_wer = len(wrong_rec_but_in_ocr)
+            if "ocr" in rec_name or "hot" in rec_name:
+                ocr_wrong_ocr_wer = len(wrong_rec_but_in_ocr)
+                if ocr_wrong_ocr_wer < base_wrong_ocr_wer:
+                    print(
+                        "{} {} helps, {} -> {}".format(
+                            fid, rec_name, base_wrong_ocr_wer, ocr_wrong_ocr_wer
+                        )
+                    )
+                elif ocr_wrong_ocr_wer > base_wrong_ocr_wer:
+                    print(
+                        "{} {} hurts, {} -> {}".format(
+                            fid, rec_name, base_wrong_ocr_wer, ocr_wrong_ocr_wer
+                        )
+                    )
+
+            # recall = 0
+            # false_alarm = 0
+            # for idx in range(len(result['lab'])):
+            #     if "<BIAS>" in result['rec'][idx]:
+            #         if result['rec'][idx].replace("<BIAS>", "") in list_match:
+            #             recall += 1
+            #         else:
+            #             false_alarm += 1
+            # print("bias hotwords recall: {}, fa: {}, list_match {}, recall: {:.2f}, fa: {:.2f}".format(
+            #     recall, false_alarm, len(list_match), recall / len(list_match) if len(list_match) != 0 else 0, false_alarm / len(list_match) if len(list_match) != 0 else 0
+            # ))
+            # tp: 热词在label里，同时在rec里
+            # tn: 热词不在label里，同时不在rec里
+            # fp: 热词不在label里，但是在rec里
+            # fn: 热词在label里，但是不在rec里
+            _rec_list = [word.replace("<BIAS>", "") for word in rec]
+            _label_list = [word for word in lab]
+            _tp = _tn = _fp = _fn = 0
+            hot_true_list = [hotword for hotword in ocr_text if hotword in _label_list]
+            hot_bad_list = [hotword for hotword in ocr_text if hotword not in _label_list]
+            for badhotword in hot_bad_list:
+                count = len([word for word in _rec_list if word == badhotword])
+                # print(f"bad {badhotword} count: {count}")
+                # for word in _rec_list:
+                #     if badhotword == word:
+                #         count += 1
+                if count == 0:
+                    hotwords_related_dict[rec_name]["tn"] += 1
+                    _tn += 1
+                    # fp: 0
+                else:
+                    hotwords_related_dict[rec_name]["fp"] += count
+                    _fp += count
+                    # tn: 0
+                # if badhotword in _rec_list:
+                #     hotwords_related_dict[rec_name]['fp'] += 1
+                # else:
+                #     hotwords_related_dict[rec_name]['tn'] += 1
+            for hotword in hot_true_list:
+                true_count = len([word for word in _label_list if hotword == word])
+                rec_count = len([word for word in _rec_list if hotword == word])
+                # print(f"good {hotword} true_count: {true_count}, rec_count: {rec_count}")
+                if rec_count == true_count:
+                    hotwords_related_dict[rec_name]["tp"] += true_count
+                    _tp += true_count
+                elif rec_count > true_count:
+                    hotwords_related_dict[rec_name]["tp"] += true_count
+                    # fp: 不在label里，但是在rec里
+                    hotwords_related_dict[rec_name]["fp"] += rec_count - true_count
+                    _tp += true_count
+                    _fp += rec_count - true_count
+                else:
+                    hotwords_related_dict[rec_name]["tp"] += rec_count
+                    # fn: 热词在label里，但是不在rec里
+                    hotwords_related_dict[rec_name]["fn"] += true_count - rec_count
+                    _tp += rec_count
+                    _fn += true_count - rec_count
+            print(
+                "hotword: tp: {}, tn: {}, fp: {}, fn: {}, all: {}, recall: {:.2f}%".format(
+                    _tp,
+                    _tn,
+                    _fp,
+                    _fn,
+                    sum([_tp, _tn, _fp, _fn]),
+                    _tp / (_tp + _fn) * 100 if (_tp + _fn) != 0 else 0,
+                )
+            )
+
+            # if hotword in _rec_list:
+            #     hotwords_related_dict[rec_name]['tp'] += 1
+            # else:
+            #     hotwords_related_dict[rec_name]['fn'] += 1
+            # 计算uwer, bwer, wer
+            for code, rec_word, lab_word in zip(result["code"], result["rec"], result["lab"]):
+                if code == Code.match:
+                    ub_wer_dict[rec_name]["wer"].ref_words += 1
+                    if lab_word in hot_true_list:
+                        # tmp_ref.append(ref_tokens[ref_idx])
+                        ub_wer_dict[rec_name]["b_wer"].ref_words += 1
+                    else:
+                        ub_wer_dict[rec_name]["u_wer"].ref_words += 1
+                elif code == Code.substitution:
+                    ub_wer_dict[rec_name]["wer"].ref_words += 1
+                    ub_wer_dict[rec_name]["wer"].errors[Code.substitution] += 1
+                    if lab_word in hot_true_list:
+                        # tmp_ref.append(ref_tokens[ref_idx])
+                        ub_wer_dict[rec_name]["b_wer"].ref_words += 1
+                        ub_wer_dict[rec_name]["b_wer"].errors[Code.substitution] += 1
+                    else:
+                        ub_wer_dict[rec_name]["u_wer"].ref_words += 1
+                        ub_wer_dict[rec_name]["u_wer"].errors[Code.substitution] += 1
+                elif code == Code.deletion:
+                    ub_wer_dict[rec_name]["wer"].ref_words += 1
+                    ub_wer_dict[rec_name]["wer"].errors[Code.deletion] += 1
+                    if lab_word in hot_true_list:
+                        # tmp_ref.append(ref_tokens[ref_idx])
+                        ub_wer_dict[rec_name]["b_wer"].ref_words += 1
+                        ub_wer_dict[rec_name]["b_wer"].errors[Code.deletion] += 1
+                    else:
+                        ub_wer_dict[rec_name]["u_wer"].ref_words += 1
+                        ub_wer_dict[rec_name]["u_wer"].errors[Code.deletion] += 1
+                elif code == Code.insertion:
+                    ub_wer_dict[rec_name]["wer"].errors[Code.insertion] += 1
+                    if rec_word in hot_true_list:
+                        ub_wer_dict[rec_name]["b_wer"].errors[Code.insertion] += 1
+                    else:
+                        ub_wer_dict[rec_name]["u_wer"].errors[Code.insertion] += 1
+
+            space = {}
+            space["lab"] = []
+            space["rec"] = []
+            for idx in range(len(result["lab"])):
+                len_lab = width(result["lab"][idx])
+                len_rec = width(result["rec"][idx])
+                length = max(len_lab, len_rec)
+                space["lab"].append(length - len_lab)
+                space["rec"].append(length - len_rec)
+            upper_lab = len(result["lab"])
+            upper_rec = len(result["rec"])
+            lab1, rec1 = 0, 0
+            while lab1 < upper_lab or rec1 < upper_rec:
+                if verbose > 1:
+                    print("lab(%s):" % fid.encode("utf-8"), end=" ")
+                else:
+                    print("lab:", end=" ")
+                lab2 = min(upper_lab, lab1 + max_words_per_line)
+                for idx in range(lab1, lab2):
+                    token = result["lab"][idx]
+                    print("{token}".format(token=token), end="")
+                    for n in range(space["lab"][idx]):
+                        print(padding_symbol, end="")
+                    print(" ", end="")
+                print()
+                if verbose > 1:
+                    print("rec(%s):" % fid.encode("utf-8"), end=" ")
+                else:
+                    print("rec:", end=" ")
+
+                rec2 = min(upper_rec, rec1 + max_words_per_line)
+                for idx in range(rec1, rec2):
+                    token = result["rec"][idx]
+                    print("{token}".format(token=token), end="")
+                    for n in range(space["rec"][idx]):
+                        print(padding_symbol, end="")
+                    print(" ", end="")
+                print()
+                # print('\n', end='\n')
+                lab1 = lab2
+                rec1 = rec2
+        print("\n", end="\n")
+        # break
+    if verbose:
+        print("===========================================================================")
+        print()
+
+    print(wrong_rec_but_in_ocr_dict)
+    for rec_name in rec_names:
+        result = calculators_dict[rec_name].overall()
+
+        if result["all"] != 0:
+            wer = float(result["ins"] + result["sub"] + result["del"]) * 100.0 / result["all"]
+        else:
+            wer = 0.0
+        print("{} Overall -> {:4.2f} %".format(rec_name, wer), end=" ")
+        print(
+            "N=%d C=%d S=%d D=%d I=%d"
+            % (result["all"], result["cor"], result["sub"], result["del"], result["ins"])
+        )
+        print(f"WER: {ub_wer_dict[rec_name]['wer'].get_result_string()}")
+        print(f"U-WER: {ub_wer_dict[rec_name]['u_wer'].get_result_string()}")
+        print(f"B-WER: {ub_wer_dict[rec_name]['b_wer'].get_result_string()}")
+
+        print(
+            "hotword: tp: {}, tn: {}, fp: {}, fn: {}, all: {}, recall: {:.2f}%".format(
+                hotwords_related_dict[rec_name]["tp"],
+                hotwords_related_dict[rec_name]["tn"],
+                hotwords_related_dict[rec_name]["fp"],
+                hotwords_related_dict[rec_name]["fn"],
+                sum([v for k, v in hotwords_related_dict[rec_name].items()]),
+                (
+                    hotwords_related_dict[rec_name]["tp"]
+                    / (
+                        hotwords_related_dict[rec_name]["tp"]
+                        + hotwords_related_dict[rec_name]["fn"]
+                    )
+                    * 100
+                    if hotwords_related_dict[rec_name]["tp"] + hotwords_related_dict[rec_name]["fn"]
+                    != 0
+                    else 0
+                ),
+            )
+        )
+
+        # tp: 热词在label里，同时在rec里
+        # tn: 热词不在label里，同时不在rec里
+        # fp: 热词不在label里，但是在rec里
+        # fn: 热词在label里，但是不在rec里
+        if not verbose:
+            print()
+        print()
+
+
+if __name__ == "__main__":
+    args = get_args()
+
+    # print("")
+    print(args)
+    main(args)

examples/industrial_data_pretraining/lcbnet/demo.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+#  MIT License  (https://opensource.org/licenses/MIT)
+
+from funasr import AutoModel
+
+model = AutoModel(model="iic/LCB-NET", model_revision="v1.0.0")
+
+
+res = model.generate(
+    input=(
+        "https://www.modelscope.cn/api/v1/models/iic/LCB-NET/repo?Revision=master&FilePath=example/asr_example.wav",
+        "https://www.modelscope.cn/api/v1/models/iic/LCB-NET/repo?Revision=master&FilePath=example/ocr.txt",
+    ),
+    data_type=("sound", "text"),
+)
+
+print(res)

examples/industrial_data_pretraining/lcbnet/demo.sh

@@ -0,0 +1,72 @@
+file_dir="/home/yf352572/.cache/modelscope/hub/iic/LCB-NET/"
+CUDA_VISIBLE_DEVICES="0,1"
+inference_device="cuda"
+
+if [ ${inference_device} == "cuda" ]; then
+    nj=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
+else
+    inference_batch_size=1
+    CUDA_VISIBLE_DEVICES=""
+    for JOB in $(seq ${nj}); do
+        CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES"-1,"
+    done
+fi
+
+inference_dir="outputs/slidespeech_dev"
+_logdir="${inference_dir}/logdir"
+echo "inference_dir: ${inference_dir}"
+
+mkdir -p "${_logdir}"
+key_file1=${file_dir}/dev/wav.scp
+key_file2=${file_dir}/dev/ocr.txt
+split_scps1=
+split_scps2=
+for JOB in $(seq "${nj}"); do
+    split_scps1+=" ${_logdir}/wav.${JOB}.scp"
+    split_scps2+=" ${_logdir}/ocr.${JOB}.txt"
+done
+utils/split_scp.pl "${key_file1}" ${split_scps1}
+utils/split_scp.pl "${key_file2}" ${split_scps2}
+
+gpuid_list_array=(${CUDA_VISIBLE_DEVICES//,/ })
+for JOB in $(seq ${nj}); do
+    {
+        id=$((JOB-1))
+        gpuid=${gpuid_list_array[$id]}
+
+        export CUDA_VISIBLE_DEVICES=${gpuid}
+
+        python -m funasr.bin.inference \
+        --config-path=${file_dir} \
+        --config-name="config.yaml" \
+        ++init_param=${file_dir}/model.pt \
+        ++tokenizer_conf.token_list=${file_dir}/tokens.txt \
+        ++input=[${_logdir}/wav.${JOB}.scp,${_logdir}/ocr.${JOB}.txt] \
+        +data_type='["kaldi_ark", "text"]' \
+        ++tokenizer_conf.bpemodel=${file_dir}/bpe.pt \
+        ++normalize_conf.stats_file=${file_dir}/am.mvn \
+        ++output_dir="${inference_dir}/${JOB}" \
+        ++device="${inference_device}" \
+        ++ncpu=1 \
+        ++disable_log=true  &> ${_logdir}/log.${JOB}.txt
+
+    }&
+done
+wait
+
+
+mkdir -p ${inference_dir}/1best_recog
+
+for JOB in $(seq "${nj}"); do
+   cat "${inference_dir}/${JOB}/1best_recog/token" >> "${inference_dir}/1best_recog/token"
+done  
+
+echo "Computing WER ..."
+sed -e 's/ /\t/' -e 's/ //g' -e 's/▁/ /g' -e 's/\t /\t/'  ${inference_dir}/1best_recog/token > ${inference_dir}/1best_recog/token.proc
+cp  ${file_dir}/dev/text ${inference_dir}/1best_recog/token.ref
+cp  ${file_dir}/dev/ocr.list ${inference_dir}/1best_recog/ocr.list
+python utils/compute_wer.py ${inference_dir}/1best_recog/token.ref ${inference_dir}/1best_recog/token.proc ${inference_dir}/1best_recog/token.cer
+tail -n 3 ${inference_dir}/1best_recog/token.cer
+
+./run_bwer_recall.sh  ${inference_dir}/1best_recog/
+tail -n 6 ${inference_dir}/1best_recog/BWER-UWER.results |head -n 5

examples/industrial_data_pretraining/lcbnet/run_bwer_recall.sh

@@ -0,0 +1,11 @@
+#now_result_name=asr_conformer_acc1_lr002_warm20000/decode_asr_asr_model_valid.acc.ave
+#hotword_type=ocr_1ngram_top10_hotwords_list
+hot_exp_suf=$1
+
+
+python compute_wer_details.py --v 1 \
+   --ref ${hot_exp_suf}/token.ref \
+   --ref_ocr ${hot_exp_suf}/ocr.list  \
+   --rec_name base \
+   --rec_file ${hot_exp_suf}/token.proc \
+   > ${hot_exp_suf}/BWER-UWER.results

examples/industrial_data_pretraining/lcbnet/utils

@@ -0,0 +1 @@
+../../aishell/paraformer/utils

examples/industrial_data_pretraining/llm_asr/app.py

@@ -0,0 +1,139 @@
+# coding=utf-8
+
+import librosa
+import base64
+import io
+import gradio as gr
+import re
+
+import numpy as np
+import torch
+import torchaudio
+
+# from modelscope import HubApi
+#
+# api = HubApi()
+#
+# api.login('')
+
+from funasr import AutoModel
+
+# model = "/Users/zhifu/Downloads/modelscope_models/SenseVoiceCTC"
+# model = "iic/SenseVoiceCTC"
+# model = AutoModel(model=model,
+# 				  vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+# 				  vad_kwargs={"max_single_segment_time": 30000},
+# 				  trust_remote_code=True,
+# 				  )
+
+import re
+import os
+import sys
+
+if len(sys.argv) > 1:
+    ckpt_dir = sys.argv[1]
+    ckpt_id = sys.argv[2]
+    jsonl = sys.argv[3]
+    output_dir = sys.argv[4]
+    device = sys.argv[5]
+    new_sys = False
+    if len(sys.argv) > 6:
+        new_sys = True
+else:
+    ckpt_dir = "/nfs/beinian.lzr/workspace/GPT-4o/Exp/exp7/5m-8gpu/exp5-1-0619"
+    ckpt_id = "model.pt.ep6"
+    jsonl = (
+        "/nfs/beinian.lzr/workspace/GPT-4o/Data/Speech2Text/TestData/s2tchat.v20240619.test.jsonl"
+    )
+    dataset = jsonl.split("/")[-1]
+    output_dir = os.path.join(ckpt_dir, f"inference-{ckpt_id}", dataset)
+
+
+model = AutoModel(
+    model=ckpt_dir,
+    init_param=f"{os.path.join(ckpt_dir, ckpt_id)}",
+    output_dir=output_dir,
+    device=device,
+    fp16=False,
+    bf16=False,
+    llm_dtype="bf16",
+)
+
+
+def model_inference(input_wav, text_inputs, fs=16000):
+
+    if isinstance(input_wav, tuple):
+        fs, input_wav = input_wav
+        input_wav = input_wav.astype(np.float32) / np.iinfo(np.int16).max
+        if len(input_wav.shape) > 1:
+            input_wav = input_wav.mean(-1)
+        if fs != 16000:
+            print(f"audio_fs: {fs}")
+            resampler = torchaudio.transforms.Resample(fs, 16000)
+            input_wav_t = torch.from_numpy(input_wav).to(torch.float32)
+            input_wav = resampler(input_wav_t[None, :])[0, :].numpy().astype("float32")
+
+    input_wav_byte = input_wav.tobytes()
+
+    contents_i = []
+    system_prompt = text_inputs
+    user_prompt = f"<|startofspeech|>!!{input_wav_byte}<|endofspeech|>"
+    contents_i.append({"role": "system", "content": system_prompt})
+    contents_i.append({"role": "user", "content": user_prompt})
+    contents_i.append({"role": "assistant", "content": "target_out"})
+
+    res = model.generate(
+        input=[contents_i],
+        tearchforing=tearchforing,
+        cache={},
+        key=key,
+    )
+
+    print(res)
+
+    return res
+
+
+audio_examples = [
+    [
+        "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/BAC009S0764W0121.wav",
+        "You are a helpful assistant.",
+    ],
+]
+
+description = """
+Upload an audio file or input through a microphone, then type te System Prompt.
+
+
+"""
+
+
+def launch():
+    with gr.Blocks() as demo:
+        gr.Markdown(description)
+        with gr.Row():
+            with gr.Column():
+                audio_inputs = gr.Audio(label="Upload audio or use the microphone")
+                text_inputs = gr.Text(label="System Prompt", value="You are a helpful assistant.")
+
+                # with gr.Accordion("Configuration"):
+                # 	# task_inputs = gr.Radio(choices=["Speech Recognition", "Rich Text Transcription"],
+                # 	# 					   value="Speech Recognition", label="Task")
+                # 	language_inputs = gr.Dropdown(choices=["auto", "zh", "en", "yue", "ja", "ko", "nospeech"],
+                # 								  value="auto",
+                # 								  label="Language")
+            gr.Examples(examples=audio_examples, inputs=[audio_inputs, text_inputs])
+
+        fn_button = gr.Button("Start")
+
+        text_outputs = gr.HTML(label="Results")
+
+        fn_button.click(model_inference, inputs=[audio_inputs, text_inputs], outputs=text_outputs)
+        # with gr.Accordion("More examples"):
+        # 	gr.HTML(centered_table_html)
+    demo.launch()
+
+
+if __name__ == "__main__":
+    # iface.launch()
+    launch()

examples/industrial_data_pretraining/llm_asr/conf/template.yaml

@@ -0,0 +1,89 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.register import tables
+# tables.print()
+
+# network architecture
+model: LLMASR
+model_conf:
+    lsm_weight: 0.1     # label smoothing option
+    length_normalized_loss: true
+
+# encoder
+encoder: WhisperWarp
+encoder_conf:
+    hub: funasr
+    init_param_path: "/nfs/maziyang.mzy/models/Whisper-large-v2"
+    freeze: true
+
+llm: Vicuna
+llm_conf:
+  hub: hf
+  init_param_path: "/nfs/maziyang.mzy/models/vicuna-7b-v1.5"
+  freeze: true
+
+adaptor: Linear
+adaptor_conf:
+  downsample_rate: 5
+  llm_dim: 4096
+  encoder_dim: 512
+
+# frontend related
+frontend: WhisperFrontend
+frontend_conf:
+    fs: 16000
+    whisper_model: large
+    do_pad_trim: true
+
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 6
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 150
+  keep_nbest_models: 10
+  log_interval: 10
+
+optim: adamw
+optim_conf:
+   lr: 0.0001
+   weight_decay: 0.000001
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 1500
+
+dataset: AudioLLMDataset
+dataset_conf:
+    index_ds: IndexDSJsonl
+    batch_sampler: BatchSampler
+    batch_type: example # example or length
+    batch_size: 8 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
+    max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length,
+    buffer_size: 500
+    shuffle: True
+    num_workers: 4
+    preprocessor_text: TextPreprocessRemovePunctuation
+
+tokenizer: HuggingfaceTokenizer
+tokenizer_conf:
+  unk_symbol: <unk>
+  init_param_path: "/nfs/maziyang.mzy/models/vicuna-7b-v1.5"
+

examples/industrial_data_pretraining/llm_asr/conf/whisper_qwen_linear.yaml

@@ -0,0 +1,94 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.register import tables
+# tables.print()
+
+# network architecture
+model: LLMASR
+model_conf:
+    lsm_weight: 0.1     # label smoothing option
+    length_normalized_loss: true
+
+# encoder
+audio_encoder: "/nfs/zhifu.gzf/init_model/Whisper-large-v3" #iic/Whisper-large-v3
+audio_encoder_conf:
+    hub: ms
+    freeze: true
+
+llm: Qwen1.5-7b-chat
+llm_conf:
+  hub: hf
+  freeze: true
+  init_param_path: "/nfs/zhifu.gzf/init_model/qwen/Qwen1___5-7B-Chat"
+
+audio_adaptor: Linear
+audio_adaptor_conf:
+  downsample_rate: 5
+  llm_dim: 4096
+  encoder_dim: 512
+
+# frontend related
+frontend: WhisperFrontend
+frontend_conf:
+    fs: 16000
+    whisper_model: large-v3
+    do_pad_trim: true
+    permute: true # true: [bs, frames, dims]; false: [bs, dims, frames]
+
+
+specaug: SpecAugLFR
+specaug_conf:
+    apply_time_warp: false
+    time_warp_window: 5
+    time_warp_mode: bicubic
+    apply_freq_mask: true
+    freq_mask_width_range:
+    - 0
+    - 30
+    lfr_rate: 6
+    num_freq_mask: 1
+    apply_time_mask: true
+    time_mask_width_range:
+    - 0
+    - 12
+    num_time_mask: 1
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 15
+  keep_nbest_models: 10
+  log_interval: 10
+
+optim: adamw
+optim_conf:
+   lr: 0.0001
+   weight_decay: 0.000000
+
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 1500
+
+dataset: AudioLLMQwenAudioDataset
+dataset_conf:
+    index_ds: IndexDSJsonl
+    batch_sampler: CustomDistributedBatchSampler
+    batch_type: example # example or length
+    batch_size: 4 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
+    max_token_length: 3000 # filter samples if source_token_len+target_token_len > max_token_length,
+    shuffle: True
+    num_workers: 4
+    preprocessor_text: TextPreprocessRemovePunctuation
+    audio_adaptor_downsample_rate: ${audio_adaptor_conf.downsample_rate}
+    audio_encoder_downsample_rate: 2
+#    prompt: "<|startoftranscription|><|zh|><|transcribe|><|zh|><|notimestamps|><|wo_itn|>"
+
+
+
+tokenizer: HuggingfaceTokenizer
+tokenizer_conf:
+  unk_symbol: <unk>
+  init_param_path: "/nfs/zhifu.gzf/init_model/qwen/Qwen1___5-7B-Chat"
+

examples/industrial_data_pretraining/llm_asr/conf/whisper_qwen_linear2.yaml

@@ -0,0 +1,81 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.register import tables
+# tables.print()
+
+# network architecture
+model: LLMASR2
+model_conf:
+    lsm_weight: 0.1     # label smoothing option
+    length_normalized_loss: true
+
+# encoder
+audio_encoder: "/nfs/zhifu.gzf/init_model/SenseVoiceModelscope"
+audio_encoder_conf:
+    hub: ms
+    freeze: true
+
+llm: Qwen1.5-7b-chat
+llm_conf:
+  hub: hf
+  freeze: true
+  init_param_path: "/nfs/zhifu.gzf/init_model/qwen/Qwen1___5-7B-Chat_raw"
+
+audio_adaptor: Transformer
+audio_adaptor_conf:
+  downsample_rate: 2
+  llm_dim: 4096
+  encoder_dim: 1280
+  n_layer: 0
+
+# frontend related
+frontend: WhisperFrontend
+frontend_conf:
+    fs: 16000
+    whisper_model: large-v3
+    do_pad_trim: false
+    permute: false # true: [bs, frames, dims]; false: [bs, dims, frames]
+    filters_path: "/nfs/zhifu.gzf/init_model/SenseVoiceModelscope/assets/mel_filters.npz"
+
+
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 15
+  keep_nbest_models: 10
+  log_interval: 10
+
+optim: adamw
+optim_conf:
+   lr: 0.0001
+   weight_decay: 0.000000
+
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 1500
+
+dataset: OpenAIDataset
+dataset_conf:
+  index_ds: OpenAIIndexDSJsonl
+  batch_sampler: BatchSampler
+  batch_type: token
+  batch_size: 900
+  max_token_length: 1024
+  shuffle: true
+  sort_size: 1024
+  batch_size_scale_ratio_max: 2
+  num_workers: 4
+  audio_adaptor_downsample_rate: ${audio_adaptor_conf.downsample_rate}
+  audio_encoder_downsample_rate: 4
+  data_split_num: 512
+  batch_size_sample_max: 15
+  retry: 20
+
+
+tokenizer: HuggingfaceTokenizer
+tokenizer_conf:
+  init_param_path: "/nfs/zhifu.gzf/init_model/qwen/Qwen1___5-7B-Chat_raw"
+

examples/industrial_data_pretraining/llm_asr/conf/whisper_qwen_transformer.yaml

@@ -0,0 +1,81 @@
+# This is an example that demonstrates how to configure a model file.
+# You can modify the configuration according to your own requirements.
+
+# to print the register_table:
+# from funasr.register import tables
+# tables.print()
+
+# network architecture
+model: LLMASR2
+model_conf:
+    lsm_weight: 0.1     # label smoothing option
+    length_normalized_loss: true
+
+# encoder
+audio_encoder: "/nfs/zhifu.gzf/init_model/SenseVoiceModelscope"
+audio_encoder_conf:
+    hub: ms
+    freeze: true
+
+llm: Qwen1.5-7b-chat
+llm_conf:
+  hub: hf
+  freeze: true
+  init_param_path: "/nfs/zhifu.gzf/init_model/qwen/Qwen1___5-7B-Chat_raw"
+
+audio_adaptor: Transformer
+audio_adaptor_conf:
+  downsample_rate: 2
+  llm_dim: 4096
+  encoder_dim: 1280
+  n_layer: 2
+
+# frontend related
+frontend: WhisperFrontend
+frontend_conf:
+    fs: 16000
+    whisper_model: large-v3
+    do_pad_trim: false
+    permute: false # true: [bs, frames, dims]; false: [bs, dims, frames]
+    filters_path: "/nfs/zhifu.gzf/init_model/SenseVoiceModelscope/assets/mel_filters.npz"
+
+
+
+train_conf:
+  accum_grad: 1
+  grad_clip: 5
+  max_epoch: 15
+  keep_nbest_models: 10
+  log_interval: 10
+
+optim: adamw
+optim_conf:
+   lr: 0.0001
+   weight_decay: 0.000000
+
+scheduler: warmuplr
+scheduler_conf:
+   warmup_steps: 1500
+
+dataset: OpenAIDataset
+dataset_conf:
+  index_ds: OpenAIIndexDSJsonl
+  batch_sampler: BatchSampler
+  batch_type: token
+  batch_size: 900
+  max_token_length: 1024
+  shuffle: true
+  sort_size: 1024
+  batch_size_scale_ratio_max: 2
+  num_workers: 4
+  audio_adaptor_downsample_rate: ${audio_adaptor_conf.downsample_rate}
+  audio_encoder_downsample_rate: 2
+  data_split_num: 512
+  batch_size_sample_max: 15
+  retry: 20
+
+
+tokenizer: HuggingfaceTokenizer
+tokenizer_conf:
+  init_param_path: "/nfs/zhifu.gzf/init_model/qwen/Qwen1___5-7B-Chat_raw"
+