// funasr_vad.h — single-header FSMN-VAD for the funasr ggml runtime. // Exposes funasr_vad_segments(): 16k mono wav -> speech segments [start_ms,end_ms]. // Front end (80-mel fbank + LFR m5n1 + CMVN) and FSMN encoder validated bit-exact vs // PyTorch fsmn-vad; the host state machine reproduces E2EVadModel (DEFAULT_SILENCE_SCHEDULE, // chunk-stepped) to within 1 frame (10ms) of fsmn-vad.generate on the 184-clip set. #pragma once #include "ggml.h" #include "ggml-cpu.h" #include "ggml-alloc.h" #include "ggml-backend.h" #include "gguf.h" #include #include #include #include #include #include #include #include #ifndef M_PI #define M_PI 3.14159265358979323846 // not guaranteed by on MSVC #endif namespace funasr_vad_impl { static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80; static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f; static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);} static void fftc(std::vector&re,std::vector&im,int n){for(int i=1,j=0;i>1;for(;j&b;b>>=1)j^=b;j^=b;if(i> fbank80(std::vector wav){ for(auto&v:wav)v*=32768.0f; std::vectorwin(WINLEN); for(int i=0;i>fb(NMEL,std::vector(NB,0.0f)); for(int m=0;mL&&mf>feat(T,std::vector(NMEL)); std::vectorre(NFFT),im(NFFT),fr(WINLEN);const float fl=1.1920929e-07f; for(int t=0;t0;i--)fr[i]-=PREEMPH*fr[i-1];fr[0]-=PREEMPH*fr[0]; for(int i=0;i0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]);feat[t][m]=logf(e>fl?e:fl);}} return feat; } static std::vector lfr(const std::vector>&feat,int m,int n,int&T_out){ int T=feat.size(); if(T<1){T_out=0;return {};} // empty (audio shorter than one frame) int D=NMEL,pad=(m-1)/2; int Tl=(T+n-1)/n; std::vector> pf; pf.reserve(T+pad+m); for(int i=0;i out((size_t)Tl*m*D); for(int i=0;it; ggml_tensor*g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"vad: missing %s\n",n.c_str());return nullptr;}return it->second;}}; static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;} } // namespace funasr_vad_impl // Run FSMN-VAD on a 16k mono float waveform; fills segs with [start_ms,end_ms] speech spans. // max_seg_ms caps a single segment (e.g. 30000); pass <=0 to use 60000 (model default). inline bool funasr_vad_segments(const std::string& gguf_path, const std::vector& wav, int max_seg_ms, std::vector>& segs, int nthreads=8){ using namespace funasr_vad_impl; segs.clear(); vad m; gguf_init_params ip={false,&m.ctx}; gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),ip); if(!gg){fprintf(stderr,"vad: cannot load %s\n",gguf_path.c_str());return false;} auto rd=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);}; int idim=rd("vad.input_dim",400),pd=rd("vad.proj_dim",128),nl=rd("vad.fsmn_layers",4),lorder=rd("vad.lorder",20), od=rd("vad.output_dim",248),lm=rd("vad.lfr_m",5),ln=rd("vad.lfr_n",1); for(int i=0;i no speech float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data; for(int t=0;t ctx holds only tensor/graph metadata (the real compute buffer is // allocated by gallocr below), so a few MB is plenty regardless of clip length. ggml_init_params cp={(size_t)16*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp); ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,idim,T); ggml_set_input(x); ggml_tensor*h=lin(c,m.g("encoder.in_linear1.linear.weight"),m.g("encoder.in_linear1.linear.bias"),x); h=lin(c,m.g("encoder.in_linear2.linear.weight"),m.g("encoder.in_linear2.linear.bias"),h); h=ggml_relu(c,h); for(int i=0;i already contiguous, no ggml_cont needed for(int j=0;jnb[1],(size_t)j*zp->nb[1]);auto wj=ggml_view_1d(c,fk,pd,(size_t)j*fk->nb[1]);acc=ggml_add(c,acc,ggml_mul(c,sl,wj));} ggml_tensor*a=lin(c,m.g(p+"affine.linear.weight"),m.g(p+"affine.linear.bias"),acc); h=ggml_relu(c,a);} h=lin(c,m.g("encoder.out_linear1.linear.weight"),m.g("encoder.out_linear1.linear.bias"),h); h=lin(c,m.g("encoder.out_linear2.linear.weight"),m.g("encoder.out_linear2.linear.bias"),h); h=ggml_soft_max(c,h); ggml_set_output(h); ggml_cgraph*gf=ggml_new_graph(c); ggml_build_forward_expand(gf,h); ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf); ggml_backend_tensor_set(x,feats.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,nthreads); bool ok=ggml_backend_graph_compute(be,gf)==GGML_STATUS_SUCCESS; std::vector sc((size_t)od*T); if(ok)ggml_backend_tensor_get(h,sc.data(),0,ggml_nbytes(h)); ggml_gallocr_free(ga);ggml_free(c);ggml_backend_free(be);if(m.ctx)ggml_free(m.ctx); if(!ok)return false; // ===== E2EVadModel state machine (host) -> speech segments [start_ms,end_ms] ===== const int FR=10; // ms per frame (frame_in_ms) const int win=20; // window_size_ms 200 / 10 const int s2s=15, sp2s=15; // sil_to_speech / speech_to_sil thres (150/10) const int lookahead_end=100/FR; // lookahead_time_end_point 100/10 = 10 (do_extend) int max_seg = (max_seg_ms>0 ? max_seg_ms : 60000)/FR; // max_single_segment frames const int start_lookback = win + 200/FR; // LatencyFrmNumAtStartPoint = 40 // End-silence threshold from DEFAULT_SILENCE_SCHEDULE, stepped at chunk boundaries (chunk_size // 60000ms = 6000 frames). At each boundary, if mid-segment (InSpeech) or in_speech latched, // accumulated_ms += 60000; threshold = schedule(accumulated). Reset to 0 on each segment emit. const int CHUNK=60000/FR; int acc=0, insp=0; int max_end_sil, end_lookback; auto recompute=[&](){ int s; if(acc<=10000)s=2000; else if(acc<=20000)s=1000; else if(acc<=30000)s=800; else if(acc<=40000)s=600; else if(acc<=50000)s=400; else if(acc<=60000)s=200; else s=100; int ms=s-150; if(ms<0)ms=0; max_end_sil=ms/FR; end_lookback=max_end_sil-lookahead_end-1; if(end_lookback<0)end_lookback=0; }; recompute(); std::vector wbuf(win,0); int wpos=0,wsum=0,pre=0; int st=0, cstart=-1, csil=0, prev_end=0; auto reset=[&](){ std::fill(wbuf.begin(),wbuf.end(),0); wpos=0; wsum=0; pre=0; csil=0; st=0; cstart=-1; acc=0; insp=0; }; auto emit=[&](int s,int e){ if(sT)e=T; if(e>s){segs.push_back({s,e}); prev_end=e;} }; for(int t=0;t0 && t%CHUNK==0){ if(st==1||insp){acc+=60000; insp=1;} recompute(); } float sil=sc[(size_t)t*od+0]; int fs = ((1.0f-sil) >= sil + 0.5f) ? 1 : 0; // speech_noise_thres=0.5 wsum -= wbuf[wpos]; wsum += fs; wbuf[wpos]=fs; wpos=(wpos+1)%win; int ch; if(pre==0 && wsum>=s2s){pre=1; ch=3;} else if(pre==1 && wsum<=sp2s){pre=0; ch=1;} else ch = pre==0?0:2; if(ch==3){ csil=0; if(st==0){ cstart=t-start_lookback; if(cstartmax_seg){ emit(cstart,t); reset(); } } else if(ch==1||ch==2){ csil=0; if(st==1 && t-cstart+1>max_seg){ emit(cstart,t); reset(); } } else { csil++; if(st==1){ if(csil>=max_end_sil){ emit(cstart, t-end_lookback); reset(); } else if(t-cstart+1>max_seg){ emit(cstart,t); reset(); } } } } if(st==1) emit(cstart,T); // convert frame indices -> ms for(auto&s:segs){ s.first*=FR; s.second*=FR; } return true; }