chore: import upstream snapshot with attribution

2026-07-13 13:25:10 +08:00
commit c397331b1e
3684 changed files with 990993 additions and 0 deletions
@@ -0,0 +1,118 @@
+#!/usr/bin/perl
+# Copyright 2010-2011  Microsoft Corporation
+#                2013  Johns Hopkins University (author: Daniel Povey)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Adds disambiguation symbols to a lexicon.
+# Outputs still in the normal lexicon format.
+# Disambig syms are numbered #1, #2, #3, etc. (#0 
+# reserved for symbol in grammar).
+# Outputs the number of disambig syms to the standard output.
+# With the --pron-probs option, expects the second field
+# of each lexicon line to be a pron-prob.
+
+$pron_probs = 0;
+
+if ($ARGV[0] eq "--pron-probs") {
+  $pron_probs = 1;
+  shift @ARGV;
+}
+
+if(@ARGV != 2) {
+    die "Usage: add_lex_disambig.pl [--pron-probs] lexicon.txt lexicon_disambig.txt "
+}
+
+
+$lexfn = shift @ARGV;
+$lexoutfn = shift @ARGV;
+
+open(L, "<$lexfn") || die "Error opening lexicon $lexfn";
+
+# (1)  Read in the lexicon.
+@L = ( );
+while(<L>) {
+    @A = split(" ", $_);
+    push @L, join(" ", @A);
+}
+
+# (2) Work out the count of each phone-sequence in the
+# lexicon.
+
+foreach $l (@L) {
+    @A = split(" ", $l);
+    shift @A; # Remove word.
+    if ($pron_probs) {
+      $p = shift @A;
+      if (!($p > 0.0 && $p <= 1.0)) { die "Bad lexicon line $l (expecting pron-prob as second field)"; }
+    }
+    $count{join(" ",@A)}++;
+}
+
+# (3) For each left sub-sequence of each phone-sequence, note down
+# that exists (for identifying prefixes of longer strings).
+
+foreach $l (@L) {
+    @A = split(" ", $l);
+    shift @A; # Remove word.
+    if ($pron_probs) { shift @A; } # remove pron-prob.
+    while(@A > 0) {
+        pop @A;  # Remove last phone
+        $issubseq{join(" ",@A)} = 1;
+    }
+}
+
+# (4) For each entry in the lexicon:
+#  if the phone sequence is unique and is not a
+#  prefix of another word, no diambig symbol.
+#  Else output #1, or #2, #3, ... if the same phone-seq
+#  has already been assigned a disambig symbol.
+
+
+open(O, ">$lexoutfn") || die "Opening lexicon file $lexoutfn for writing.\n";
+
+$max_disambig = 0;
+foreach $l (@L) {
+    @A = split(" ", $l);
+    $word = shift @A;
+    if ($pron_probs) { $pron_prob = shift @A; }
+    $phnseq = join(" ",@A);
+    if(!defined $issubseq{$phnseq}
+       && $count{$phnseq} == 1) {
+        ; # Do nothing.
+    } else {
+        if($phnseq eq "") { # need disambig symbols for the empty string
+            # that are not use anywhere else.
+            $max_disambig++;
+            $reserved{$max_disambig} = 1;
+            $phnseq = "#$max_disambig";
+        } else {
+            $curnumber = $disambig_of{$phnseq};
+            if(!defined{$curnumber}) { $curnumber = 0; }
+            $curnumber++; # now 1 or 2, ... 
+            while(defined $reserved{$curnumber} ) { $curnumber++; } # skip over reserved symbols
+            if($curnumber > $max_disambig) {
+                $max_disambig = $curnumber;
+            }
+            $disambig_of{$phnseq} = $curnumber;
+            $phnseq = $phnseq . " #" . $curnumber;
+         }
+    }
+    if ($pron_probs) {  print O "$word\t$pron_prob\t$phnseq\n"; }
+    else {  print O "$word\t$phnseq\n"; }
+}
+
+print $max_disambig . "\n";
+
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Collect TLG.fst
+lm_dir=$1
+tgt_dir=$2
+
+tlg=${lm_dir}/lang/TLG.fst
+
+[ ! -f $tlg ] && echo No TLG file $tlg && exit 1;
+
+rm -rf $tgt_dir
+mkdir -p $tgt_dir
+cp -r $tlg ${tgt_dir}/TLG.fst
+
+# Generate configuration file
+wd_file=${lm_dir}/lang/words.txt
+cfg_file=${tgt_dir}/config.yaml
+
+[ ! -f $wd_file ] && echo No words list $wd_file && exit 1;
+
+cat $wd_file | awk '{print $1}' | awk '
+  BEGIN {
+    print "token_list:";
+  }
+  {
+    printf("- \"%s\"\n", $1);
+  }
+  END {
+  }' > $cfg_file || exit 1;
+
@@ -0,0 +1,84 @@
+#!/bin/bash
+# Copyright 2015       Yajie Miao    (Carnegie Mellon University)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+# This script compiles the lexicon and CTC tokens into FSTs. FST compiling slightly differs between the
+# phoneme and character-based lexicons. 
+
+. fst/parse_options.sh 
+
+if [ $# -ne 3 ]; then 
+  echo "usage: utils/compile_dict_token.sh <dict-src-dir> <tmp-dir> <lang-dir>"
+  echo "e.g.: utils/compile_dict_token.sh data/local/dict_phn data/local/lang_phn_tmp data/lang_phn"
+  echo "<dict-src-dir> should contain the following files:"
+  echo "lexicon.out units.txt"
+  echo "options: "
+  exit 1;
+fi
+
+srcdir=$1
+tmpdir=$2
+dir=$3
+mkdir -p $tmpdir $dir
+
+[ -f path.sh ] && . ./path.sh
+
+cp $srcdir/units.txt $dir
+
+# Add probabilities to lexicon entries. There is in fact no point of doing this here since all the entries have 1.0.
+# But utils/make_lexicon_fst.pl requires a probabilistic version, so we just leave it as it is. 
+perl -ape 's/(\S+\s+)(.+)/${1}1.0\t$2/;' < $srcdir/lexicon.out > $tmpdir/lexiconp.txt || exit 1;
+
+# Add disambiguation symbols to the lexicon. This is necessary for determinizing the composition of L.fst and G.fst.
+# Without these symbols, determinization will fail. 
+ndisambig=`fst/add_lex_disambig.pl $tmpdir/lexiconp.txt $tmpdir/lexiconp_disambig.txt`
+ndisambig=$[$ndisambig+1];
+
+( for n in `seq 0 $ndisambig`; do echo '#'$n; done ) > $tmpdir/disambig.list
+
+# Get the full list of CTC tokens used in FST. These tokens include <eps>, the blank <blk>, the actual labels (e.g.,
+# phonemes), and the disambiguation symbols. 
+cat $srcdir/units.txt | awk '{print $1}' > $tmpdir/units.list
+(echo '<eps>') | cat - $tmpdir/units.list $tmpdir/disambig.list | awk '{print $1 " " (NR-1)}' > $dir/tokens.txt
+
+# Compile the tokens into FST
+fst/ctc_token_fst.py $dir/tokens.txt | fstcompile --isymbols=$dir/tokens.txt --osymbols=$dir/tokens.txt \
+   --keep_isymbols=false --keep_osymbols=false | fstarcsort --sort_type=olabel > $dir/T.fst || exit 1;
+
+# Encode the words with indices. Will be used in lexicon and language model FST compiling. 
+cat $tmpdir/lexiconp.txt | awk '{print $1}' | sort | uniq  | awk '
+  BEGIN {
+    print "<eps> 0";
+  } 
+  {
+    printf("%s %d\n", $1, NR);
+  }
+  END {
+    printf("#0 %d\n", NR+1);
+    printf("<s> %d\n", NR+2);
+    printf("</s> %d\n", NR+3);
+  }' > $dir/words.txt || exit 1;
+
+# Now compile the lexicon FST. Depending on the size of your lexicon, it may take some time. 
+token_disambig_symbol=`grep \#0 $dir/tokens.txt | awk '{print $2}'`
+word_disambig_symbol=`grep \#0 $dir/words.txt | awk '{print $2}'`
+
+fst/make_lexicon_fst.pl --pron-probs $tmpdir/lexiconp_disambig.txt 0 "sil" '#'$ndisambig | \
+  fstcompile --isymbols=$dir/tokens.txt --osymbols=$dir/words.txt \
+  --keep_isymbols=false --keep_osymbols=false |   \
+  fstaddselfloops  "echo $token_disambig_symbol |" "echo $word_disambig_symbol |" | \
+  fstarcsort --sort_type=olabel > $dir/L.fst || exit 1;
+
+echo "Dict and token FSTs compiling succeeded"
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+# Apache 2.0
+
+import sys
+
+with open(sys.argv[1], "r") as fread:
+    print("0 0 <blank> <eps>")
+    nodeX = 1
+    for entry in fread.readlines():
+        entry = entry.replace("\n", "").strip()
+        fields = entry.split(" ")
+        phone = fields[0]
+        if phone == "<eps>" or phone == "<blank>":
+            continue
+        if "#" in phone:
+            print(str(0) + " " + str(0) + " " + "<eps>" + " " + phone)
+        else:
+            print(str(0) + " " + str(nodeX) + " " + phone + " " + phone)
+            print(str(nodeX) + " " + str(nodeX) + " " + phone + " <eps>")
+            print(str(nodeX) + " " + str(0) + " " + "<eps> <eps>")
+        nodeX += 1
+    print("0")
@@ -0,0 +1,23 @@
+#!/usr/bin/perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+# This script replaces epsilon with #0 on the input side only, of the G.fst
+# acceptor.  
+
+while(<>){
+    s:^(\d+\s+\d+\s+)\<eps\>(\s+):$1#0$2:;
+    print;
+}
@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+
+import sys
+
+# sys.argv[1]: lm dict
+# sys.argv[2]: lexicon file
+# sys.argv[3]: lexicon file for corpus.dict
+
+lex_dict = {}
+with open(sys.argv[2], "r", encoding="utf8") as fin:
+    for line in fin:
+        words = line.strip().split("\t")
+        if len(words) != 2:
+            continue
+        lex_dict[words[0]] = words[1]
+
+with open(sys.argv[1], "r", encoding="utf8") as fin, open(
+    sys.argv[3], "w", encoding="utf8"
+) as fout:
+    for line in fin:
+        word = line.strip()
+        if word == "<s>" or word == "</s>":
+            continue
+        word_lex = ""
+        if word in lex_dict:
+            word_lex = lex_dict[word]
+        else:
+            for i in range(len(word)):
+                if word[i] in lex_dict:
+                    word_lex += " " + lex_dict[word[i]]
+                else:
+                    word_lex += " <unk>"
+
+        fout.write("{}\t{}\n".format(word, word_lex.strip()))
@@ -0,0 +1,34 @@
+#!/bin/bash 
+#
+
+if [ -f path.sh ]; then . path.sh; fi
+
+lm_dir=$1
+tgt_lang=$2
+
+arpa_lm=${lm_dir}/lm.arpa
+[ ! -f $arpa_lm ] && echo No such file $arpa_lm && exit 1;
+
+# Compose the language model to FST
+cat "$arpa_lm" | \
+   grep -v '<s> <s>' | \
+   grep -v '</s> <s>' | \
+   grep -v '</s> </s>' | \
+   grep -v -i '<unk>' | \
+   arpa2fst --read-symbol-table=$tgt_lang/words.txt --keep-symbols=true - | fstprint | \
+   fst/eps2disambig.pl | fst/s2eps.pl | fstcompile --isymbols=$tgt_lang/words.txt \
+     --osymbols=$tgt_lang/words.txt  --keep_isymbols=false --keep_osymbols=false | \
+    fstrmepsilon | fstarcsort --sort_type=ilabel > $tgt_lang/G.fst
+
+
+echo  "Checking how stochastic G is (the first of these numbers should be small):"
+fstisstochastic $tgt_lang/G.fst 
+
+# Compose the token, lexicon and language-model FST into the final decoding graph
+fsttablecompose $tgt_lang/L.fst $tgt_lang/G.fst | fstdeterminizestar --use-log=true | \
+    fstminimizeencoded | fstarcsort --sort_type=ilabel > $tgt_lang/LG.fst || exit 1;
+fsttablecompose $tgt_lang/T.fst $tgt_lang/LG.fst > $tgt_lang/TLG.fst || exit 1;
+
+echo "Composing decoding graph TLG.fst succeeded"
+rm -r $tgt_lang/LG.fst   # We don't need to keep this intermediate FST
+
@@ -0,0 +1,148 @@
+#!/usr/bin/perl -w
+# Copyright 2010-2011  Microsoft Corporation
+#                2013  Johns Hopkins University (author: Daniel Povey)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# makes lexicon FST, in text form, from lexicon (pronunciation probabilities optional).
+
+$pron_probs = 0;
+
+if ($ARGV[0] eq "--pron-probs") {
+  $pron_probs = 1;
+  shift @ARGV;
+}
+
+if (@ARGV != 1 && @ARGV != 3 && @ARGV != 4) {
+  print STDERR
+    "Usage: make_lexicon_fst.pl [--pron-probs] lexicon.txt [silprob silphone [sil_disambig_sym]] >lexiconfst.txt
+Creates a lexicon FST that transduces phones to words, and may allow optional silence. 
+Note: ordinarily, each line of lexicon.txt is: word phone1 phone2 ... phoneN; if the --pron-probs option is
+used, each line is: word pronunciation-probability phone1 phone2 ... phoneN.  The probability 'prob' will
+typically be between zero and one, and note that it's generally helpful to normalize so the largest one
+for each word is 1.0, but this is your responsibility.  The silence disambiguation symbol, e.g. something
+like #5, is used only when creating a lexicon with disambiguation symbols, e.g. L_disambig.fst, and was
+introduced to fix a particular case of non-determinism of decoding graphs.\n";
+  exit(1);
+}
+
+$lexfn = shift @ARGV;
+if (@ARGV == 0) {
+  $silprob = 0.0;
+} elsif (@ARGV == 2) { 
+  ($silprob,$silphone) = @ARGV;
+} else {
+  ($silprob,$silphone,$sildisambig) = @ARGV;
+}
+if ($silprob != 0.0) {
+  $silprob < 1.0 || die "Sil prob cannot be >= 1.0";
+  $silcost = -log($silprob);
+  $nosilcost = -log(1.0 - $silprob);
+}
+
+
+open(L, "<$lexfn") || die "Error opening lexicon $lexfn";
+
+
+if ( $silprob == 0.0 ) { # No optional silences: just have one (loop+final) state which is numbered zero.
+  $loopstate = 0;
+  $nextstate = 1;               # next unallocated state.
+  while (<L>) {
+    @A = split(" ", $_);
+    @A == 0 && die "Empty lexicon line.";
+    $w = shift @A;
+    if (! $pron_probs) {
+      $pron_cost = 0.0;
+    } else {
+      $pron_prob = shift @A;
+      if (! defined $pron_prob || !($pron_prob > 0.0 && $pron_prob <= 1.0)) {
+        die "Bad pronunciation probability in line $_";
+      }
+      $pron_cost = -log($pron_prob);
+    }
+    if ($pron_cost != 0.0) { $pron_cost_string = "\t$pron_cost"; } else { $pron_cost_string = ""; }
+    
+    $s = $loopstate;
+    $word_or_eps = $w;
+    while (@A > 0) {
+      $p = shift @A;
+      if (@A > 0) {
+        $ns = $nextstate++;
+      } else {
+        $ns = $loopstate;
+      }
+      print "$s\t$ns\t$p\t$word_or_eps$pron_cost_string\n";
+      $word_or_eps = "<eps>";
+      $pron_cost_string = ""; # so we only print it on the first arc of the word.
+      $s = $ns;
+    }
+  }
+  print "$loopstate\t0\n";      # final-cost.
+} else {                        # have silence probs.
+  $startstate = 0;
+  $loopstate = 1;
+  $silstate = 2;   # state from where we go to loopstate after emitting silence.
+  print "$startstate\t$loopstate\t<eps>\t<eps>\t$nosilcost\n"; # no silence.
+  if (!defined $sildisambig) {
+    print "$startstate\t$loopstate\t$silphone\t<eps>\t$silcost\n"; # silence.
+    print "$silstate\t$loopstate\t$silphone\t<eps>\n";             # no cost.
+    $nextstate = 3;
+  } else {
+    $disambigstate = 3;
+    $nextstate = 4;
+    print "$startstate\t$disambigstate\t$silphone\t<eps>\t$silcost\n"; # silence.
+    print "$silstate\t$disambigstate\t$silphone\t<eps>\n"; # no cost.
+    print "$disambigstate\t$loopstate\t$sildisambig\t<eps>\n"; # silence disambiguation symbol.
+  }
+  while (<L>) {
+    @A = split(" ", $_);
+    $w = shift @A;
+    if (! $pron_probs) {
+      $pron_cost = 0.0;
+    } else {
+      $pron_prob = shift @A;
+      if (! defined $pron_prob || !($pron_prob > 0.0 && $pron_prob <= 1.0)) {
+        die "Bad pronunciation probability in line $_";
+      }
+      $pron_cost = -log($pron_prob);
+    }
+    if ($pron_cost != 0.0) { $pron_cost_string = "\t$pron_cost"; } else { $pron_cost_string = ""; }
+    $s = $loopstate;
+    $word_or_eps = $w;
+    while (@A > 0) {
+      $p = shift @A;
+      if (@A > 0) {
+        $ns = $nextstate++;
+        print "$s\t$ns\t$p\t$word_or_eps$pron_cost_string\n";
+        $word_or_eps = "<eps>";
+        $pron_cost_string = ""; $pron_cost = 0.0; # so we only print it the 1st time.
+        $s = $ns;
+      } else {
+        if (!defined($silphone) || $p ne $silphone) {
+          # This is non-deterministic but relatively compact,
+          # and avoids epsilons.
+          $local_nosilcost = $nosilcost + $pron_cost;
+          $local_silcost = $silcost + $pron_cost;
+          print "$s\t$loopstate\t$p\t$word_or_eps\t$local_nosilcost\n";
+          print "$s\t$silstate\t$p\t$word_or_eps\t$local_silcost\n";
+        } else {
+          # no point putting opt-sil after silence word.
+          print "$s\t$loopstate\t$p\t$word_or_eps$pron_cost_string\n";
+        }
+      }
+    }
+  }
+  print "$loopstate\t0\n";      # final-cost.
+}
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the 
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line 
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### No we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help 
+    # message and exit.  Scripts should put help messages in $help_message
+  --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+	  else printf "$help_message\n" 1>&2 ; fi; 
+	  exit 0 ;; 
+  --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+       exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar), 
+    # then work out the variable name as $name, which will equal "foo_bar".
+  --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`; 
+    # Next we test whether the variable in question is undefned-- if so it's 
+    # an invalid option and we die.  Note: $0 evaluates to the name of the 
+    # enclosing script.
+    # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+    # is undefined.  We then have to wrap this test inside "eval" because 
+    # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      
+      oldval="`eval echo \\$$name`";
+    # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then 
+	was_bool=true;
+      else 
+	was_bool=false;
+      fi
+
+    # Set the variable to the right value-- the escaped quotes make it work if
+    # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\"; 
+        
+    # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a 
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.
@@ -0,0 +1,27 @@
+#!/usr/bin/perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+# This script replaces <s> and </s> with <eps> (on both input and output sides),
+# for the G.fst acceptor.
+
+while(<>){
+    @A = split(" ", $_);
+    if ( @A >= 4 ) {
+        if ($A[2] eq "<s>" || $A[2] eq "</s>") { $A[2] = "<eps>"; }
+        if ($A[3] eq "<s>" || $A[3] eq "</s>") { $A[3] = "<eps>"; }
+    }
+    print join("\t", @A) . "\n";
+}
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+## Make sure that srilm is installed
+
+dir=lm
+mkdir -p $dir
+[ -f path.sh ] && . ./path.sh
+
+# Prepare data, the format of the text should be:
+# BAC009S0002W0122 而 对 楼市 成交 抑制 作用 最 大 的 限 购
+# BAC009S0002W0123 也 成为 地方 政府 的 眼中 钉 
+corpus=lm/text
+
+# generate lm dict
+cat $corpus | awk '{for(n=2;n<=NF;n++) print tolower($n); }' | \
+   cat - <(echo "<unk>";echo "<s>"; echo "</s>") | \
+   sort | uniq -c | sort -nr | awk '{print $2}' > $dir/corpus.dict || exit 1;
+
+# train ngram
+cat $corpus | awk '{for(n=2;n<=NF;n++){ printf tolower($n); if(n<NF) printf " "; else print ""; }}' > $dir/train
+
+ngram-count -text $dir/train -order 4 -limit-vocab -vocab $dir/corpus.dict -unk \
+  -kndiscount -interpolate -gt1min 1 -gt2min 1 -gt3min 2  -gt4min 2 -lm $dir/lm.arpa
+
+
@@ -0,0 +1,3 @@
+export FUNASR_RUNTIME_ROOT=`pwd`/../
+export PATH=$FUNASR_RUNTIME_ROOT/onnxruntime/build/bin:$PWD:$PATH
+export LC_ALL=C
@@ -0,0 +1,21 @@
+chmod +x fst/*
+[ -f path.sh ] && . ./path.sh
+
+# download train corpus and lexicon
+wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/requirements/lm.tar.gz
+tar -zxvf lm.tar.gz
+
+# train lm, make sure that srilm is installed
+bash fst/train_lms.sh
+
+# generate lexicon
+python3 fst/generate_lexicon.py lm/corpus.dict lm/lexicon.txt lm/lexicon.out 
+
+# Compile the lexicon and token FSTs
+fst/compile_dict_token.sh  lm lm/tmp lm/lang
+
+# Compile the language-model FST and the final decoding graph TLG.fst
+fst/make_decode_graph.sh lm lm/lang || exit 1;
+
+# Collect resource files required for decoding
+fst/collect_resource_file.sh lm lm/resource
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey);
+#                 Arnab Ghoshal, Karel Vesely
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Parse command-line options.
+# To be sourced by another script (as in ". parse_options.sh").
+# Option format is: --option-name arg
+# and shell variable "option_name" gets set to value "arg."
+# The exception is --help, which takes no arguments, but prints the
+# $help_message variable (if defined).
+
+
+###
+### The --config file options have lower priority to command line
+### options, so we need to import them first...
+###
+
+# Now import all the configs specified by command-line, in left-to-right order
+for ((argpos=1; argpos<$#; argpos++)); do
+  if [ "${!argpos}" == "--config" ]; then
+    argpos_plus1=$((argpos+1))
+    config=${!argpos_plus1}
+    [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
+    . $config  # source the config file.
+  fi
+done
+
+
+###
+### Now we process the command line options
+###
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    # If the enclosing script is called with --help option, print the help
+    # message and exit.  Scripts should put help messages in $help_message
+    --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
+      else printf "$help_message\n" 1>&2 ; fi;
+      exit 0 ;;
+    --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
+      exit 1 ;;
+    # If the first command-line argument begins with "--" (e.g. --foo-bar),
+    # then work out the variable name as $name, which will equal "foo_bar".
+    --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
+      # Next we test whether the variable in question is undefned-- if so it's
+      # an invalid option and we die.  Note: $0 evaluates to the name of the
+      # enclosing script.
+      # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
+      # is undefined.  We then have to wrap this test inside "eval" because
+      # foo_bar is itself inside a variable ($name).
+      eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+
+      oldval="`eval echo \\$$name`";
+      # Work out whether we seem to be expecting a Boolean argument.
+      if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval $name=\"$2\";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+  *) break;
+  esac
+done
+
+
+# Check for an empty argument to the --cmd option, which can easily occur as a
+# result of scripting errors.
+[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
+
+
+true; # so this script returns exit code 0.