Change target binary library name and copy header

* methods get_model_by_name, get_model_by_lang, get_model were added into the model class

* importing modules changed to using components; introduced constant MODELS_HOME_DIR; simplified code

* added new model folders into init; changed samples and transcriber bin for new mode loader

* changed back in cli.py lang arg to args.lang

* added 3 directories instead of 1 to check for models

* cli.py: added 3 args instead of 1 for model; __init__.py: changed script get_model_path for run get_model_by_name/lang inside current directory

* deleted default env var

* cli.py: changed arg_name; __init__.py: changed const name, changed model loading only for last directory

* deleted unused method

* changed by_name, by_lang methods, added download_model method

* deleted env variable initialization

* deleted print()

* deteled unused modules

* added progress_bar, added folder AppData/Local/vosk for model search

* changed download_model methond; added my_hook method

android/build.gradle

@@ -10,7 +10,7 @@ buildscript {
 }
 
 allprojects {
-    version = '0.3.38'
+    version = '0.3.41'
 }
 
 subprojects {

csharp/demo/VoskDemo.csproj

@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Vosk" Version="0.3.31" />
+    <PackageReference Include="Vosk" Version="0.3.41" />
   </ItemGroup>
 
 </Project>

csharp/nuget/Vosk.nuspec

@@ -2,13 +2,14 @@
 <package>
   <metadata>
     <id>Vosk</id>
-    <version>0.3.38</version>
+    <version>0.3.41</version>
     <authors>Alpha Cephei Inc</authors>
     <owners>Alpha Cephei Inc</owners>
     <license type="expression">Apache-2.0</license>
     <projectUrl>https://alphacephei.com/vosk/</projectUrl>
     <requireLicenseAcceptance>false</requireLicenseAcceptance>
-    <description>Vosk is an offline open source speech recognition toolkit. It enables speech recognition models for 16 languages and dialects - English, Indian English, German, French, Spanish, Portuguese, Chinese, Russian, Turkish, Vietnamese, Italian, Dutch, Catalan, Arabic, Greek, Farsi.
+    <title>Vosk Speech Recognition Toolkit</title>
+    <description>Vosk is an offline open source speech recognition toolkit. It enables speech recognition models for 20+ languages and dialects - English, Indian English, German, French, Spanish, Portuguese, Chinese, Russian, Turkish, Vietnamese, Italian, Dutch, Catalan, Arabic, Greek, Farsi, Filipino, Ukrainian, Kazakh, Swedish, Japanese, Esperanto, Hindi, Czech. More to come.
 
 Vosk models are small (50 Mb) but provide continuous large vocabulary transcription, zero-latency response with streaming API, reconfigurable vocabulary and speaker identification.
 
@@ -18,7 +19,8 @@ Vosk supplies speech recognition for chatbots, smart home appliances, virtual as
 
 Vosk scales from small devices like Raspberry Pi or Android smartphone to big clusters.</description>
     <releaseNotes>See for details https://github.com/alphacep/vosk-api/releases</releaseNotes>
-    <copyright>Copyright 2020 Alpha Cephei Inc</copyright>
+    <repository type="git" url="https://github.com/alphacep/vosk-api.git" branch="master"/>
+    <copyright>Copyright 2020-2050 Alpha Cephei Inc</copyright>
     <tags>speech recognition voice stt asr speech-to-text ai offline privacy</tags>
     <dependencies>
       <group targetFramework=".NETStandard2.0"/>

csharp/nuget/build/Vosk.targets

@@ -2,7 +2,7 @@
   <ItemGroup>
     <NativeLibs Include="$(MSBuildThisFileDirectory)\lib\linux-x64\*.so" Condition="'$([MSBuild]::IsOsPlatform(Linux))'" />
     <NativeLibs Include="$(MSBuildThisFileDirectory)\lib\win-x64\*.dll" Condition="'$([MSBuild]::IsOsPlatform(Windows))'" />
-    <NativeLibs Include="$(MSBuildThisFileDirectory)\lib\osx-x64\*.dylib" Condition="'$([MSBuild]::IsOsPlatform(OSX))'" />
+    <NativeLibs Include="$(MSBuildThisFileDirectory)\lib\osx-universal\*.dylib" Condition="'$([MSBuild]::IsOsPlatform(OSX))'" />
     <None Include="@(NativeLibs)">
       <Link>%(FileName)%(Extension)</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

java/demo/build.gradle

@@ -12,5 +12,5 @@ repositories {
 
 dependencies {
     implementation group: 'net.java.dev.jna', name: 'jna', version: '5.7.0'
-    implementation group: 'com.alphacephei', name: 'vosk', version: '0.3.38+'
+    implementation group: 'com.alphacephei', name: 'vosk', version: '0.3.41+'
 }

java/lib/build.gradle

@@ -16,7 +16,7 @@ repositories {
 
 archivesBaseName = 'vosk'
 group = 'com.alphacephei'
-version = '0.3.38'
+version = '0.3.41'
 
 mavenPublish {
     group = 'com.alphacephei'

nodejs/index.js

@@ -76,7 +76,7 @@ if (os.platform() == 'win32') {
 
     soname = path.join(__dirname, "lib", "win-x86_64", "libvosk.dll")
 } else if (os.platform() == 'darwin') {
-    soname = path.join(__dirname, "lib", "osx-x86_64", "libvosk.dylib")
+    soname = path.join(__dirname, "lib", "osx-universal", "libvosk.dylib")
 } else {
     soname = path.join(__dirname, "lib", "linux-x86_64", "libvosk.so")
 }

nodejs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vosk",
-  "version": "0.3.38",
+  "version": "0.3.41",
   "description": "Node binding for continuous offline voice recoginition with Vosk library.",
   "repository": {
     "type": "git",

python/example/test_alternatives.py

@@ -8,16 +8,12 @@ import json
 
 SetLogLevel(0)
 
-if not os.path.exists("model"):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as 'model' in the current folder.")
-    exit (1)
-
 wf = wave.open(sys.argv[1], "rb")
 if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
     print ("Audio file must be WAV format mono PCM.")
     exit (1)
 
-model = Model("model")
+model = Model(lang="en-us")
 rec = KaldiRecognizer(model, wf.getframerate())
 rec.SetMaxAlternatives(10)
 rec.SetWords(True)

python/example/test_empty.py

@@ -4,7 +4,7 @@ from vosk import Model, KaldiRecognizer
 import sys
 import json
 
-model = Model("model")
+model = Model(lang="en-us")
 rec = KaldiRecognizer(model, 8000)
 
 res = json.loads(rec.FinalResult())

python/example/test_ffmpeg.py

@@ -8,12 +8,8 @@ import subprocess
 
 SetLogLevel(0)
 
-if not os.path.exists("model"):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as 'model' in the current folder.")
-    exit (1)
-
 sample_rate=16000
-model = Model("model")
+model = Model(lang="en-us")
 rec = KaldiRecognizer(model, sample_rate)
 
 process = subprocess.Popen(['ffmpeg', '-loglevel', 'quiet', '-i',

python/example/test_gpu_batch.py

@@ -30,7 +30,7 @@ while True:
     for i, fd in enumerate(fds):
         if i in ended:
             continue
-        data = fd.read(8000);
+        data = fd.read(8000)
         if len(data) == 0:
             recs[i].FinishStream()
             ended.add(i)

python/example/test_microphone.py

@@ -37,9 +37,6 @@ parser = argparse.ArgumentParser(
 parser.add_argument(
     '-f', '--filename', type=str, metavar='FILENAME',
     help='audio file to store recording to')
-parser.add_argument(
-    '-m', '--model', type=str, metavar='MODEL_PATH',
-    help='Path to the model')
 parser.add_argument(
     '-d', '--device', type=int_or_str,
     help='input device (numeric ID or substring)')
@@ -48,18 +45,12 @@ parser.add_argument(
 args = parser.parse_args(remaining)
 
 try:
-    if args.model is None:
-        args.model = "model"
-    if not os.path.exists(args.model):
-        print ("Please download a model for your language from https://alphacephei.com/vosk/models")
-        print ("and unpack as 'model' in the current folder.")
-        parser.exit(0)
     if args.samplerate is None:
         device_info = sd.query_devices(args.device, 'input')
         # soundfile expects an int, sounddevice provides a float:
         args.samplerate = int(device_info['default_samplerate'])
 
-    model = vosk.Model(args.model)
+    model = vosk.Model(lang="en-us")
 
     if args.filename:
         dump_fn = open(args.filename, "wb")

python/example/test_nlsml.py

@@ -7,16 +7,12 @@ import wave
 
 SetLogLevel(0)
 
-if not os.path.exists("model"):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as 'model' in the current folder.")
-    exit (1)
-
 wf = wave.open(sys.argv[1], "rb")
 if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
     print ("Audio file must be WAV format mono PCM.")
     exit (1)
 
-model = Model("model")
+model = Model(lang="en-us")
 rec = KaldiRecognizer(model, wf.getframerate())
 rec.SetMaxAlternatives(10)
 rec.SetNLSML(True)

python/example/test_reset.py

@@ -8,16 +8,12 @@ import json
 
 SetLogLevel(0)
 
-if not os.path.exists("model"):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as 'model' in the current folder.")
-    exit (1)
-
 wf = wave.open(sys.argv[1], "rb")
 if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
     print ("Audio file must be WAV format mono PCM.")
     exit (1)
 
-model = Model("model")
+model = Model(lang="en-us")
 rec = KaldiRecognizer(model, wf.getframerate())
 
 while True:

python/example/test_simple.py

@@ -5,18 +5,20 @@ import sys
 import os
 import wave
 
+# You can set log level to -1 to disable debug messages
 SetLogLevel(0)
 
-if not os.path.exists("model"):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as 'model' in the current folder.")
-    exit (1)
-
 wf = wave.open(sys.argv[1], "rb")
 if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
     print ("Audio file must be WAV format mono PCM.")
     exit (1)
 
-model = Model("model")
+model = Model(lang="en-us")
+
+# You can also init model by name or with a folder path
+# model = Model(model_name="vosk-model-en-us-0.21")
+# model = Model("models/en")
+
 rec = KaldiRecognizer(model, wf.getframerate())
 rec.SetWords(True)
 rec.SetPartialWords(True)

python/example/test_speaker.py

@@ -7,13 +7,8 @@ import json
 import os
 import numpy as np
 
-model_path = "model"
 spk_model_path = "model-spk"
 
-if not os.path.exists(model_path):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as {} in the current folder.".format(model_path))
-    exit (1)
-
 if not os.path.exists(spk_model_path):
     print ("Please download the speaker model from https://alphacephei.com/vosk/models and unpack as {} in the current folder.".format(spk_model_path))
     exit (1)
@@ -24,7 +19,7 @@ if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE
     exit (1)
 
 # Large vocabulary free form recognition
-model = Model(model_path)
+model = Model(lang="en-us")
 spk_model = SpkModel(spk_model_path)
 #rec = KaldiRecognizer(model, wf.getframerate(), spk_model)
 rec = KaldiRecognizer(model, wf.getframerate())

python/example/test_srt.py

@@ -11,12 +11,8 @@ import datetime
 
 SetLogLevel(-1)
 
-if not os.path.exists("model"):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as 'model' in the current folder.")
-    exit (1)
-
 sample_rate=16000
-model = Model("model")
+model = Model(lang="en-us")
 rec = KaldiRecognizer(model, sample_rate)
 rec.SetWords(True)
 

python/example/test_text.py

@@ -5,12 +5,7 @@ import sys
 import json
 import os
 
-if not os.path.exists("model"):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as 'model' in the current folder.")
-    exit (1)
-
-
-model = Model("model")
+model = Model(lang="en-us")
 
 # Large vocabulary free form recognition
 rec = KaldiRecognizer(model, 16000)

python/example/test_webvtt.py

@@ -10,13 +10,8 @@ import textwrap
 
 SetLogLevel(-1)
 
-if not os.path.exists('model'):
-    print('Please download the model from https://alphacephei.com/vosk/models'
-          ' and unpack as `model` in the current folder.')
-    exit(1)
-
 sample_rate = 16000
-model = Model('model')
+model = Model(lang="en-us")
 rec = KaldiRecognizer(model, sample_rate)
 rec.SetWords(True)
 

python/example/test_words.py

@@ -5,16 +5,12 @@ import sys
 import os
 import wave
 
-if not os.path.exists("model"):
-    print ("Please download the model from https://alphacephei.com/vosk/models and unpack as 'model' in the current folder.")
-    exit (1)
-
 wf = wave.open(sys.argv[1], "rb")
 if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
     print ("Audio file must be WAV format mono PCM.")
     exit (1)
 
-model = Model("model")
+model = Model(lang="en-us")
 
 # You can also specify the possible word or phrase list as JSON list, the order doesn't have to be strict
 rec = KaldiRecognizer(model, wf.getframerate(), '["oh one two three four five six seven eight nine zero", "[unk]"]')

python/setup.py

@@ -25,7 +25,7 @@ else:
         def get_tag(self):
             abi = 'none'
             if system == 'Darwin':
-                oses = 'macosx_10_6_universal'
+                oses = 'macosx_10_6_universal2'
             elif system == 'Windows' and architecture == '32bit':
                 oses = 'win32'
             elif system == 'Windows' and architecture == '64bit':
@@ -44,7 +44,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="vosk",
-    version="0.3.38",
+    version="0.3.41",
     author="Alpha Cephei Inc",
     author_email="contact@alphacephei.com",
     description="Offline open source speech recognition API based on Kaldi and Vosk",
@@ -54,7 +54,7 @@ setuptools.setup(
     packages=setuptools.find_packages(),
     package_data = {'vosk': ['*.so', '*.dll', '*.dyld']},
     entry_points = {
-        'console_scripts': ['vosk-transcriber=vosk.transcriber.cli:cli'],
+        'console_scripts': ['vosk-transcriber=vosk.transcriber.cli:main'],
     },
     include_package_data=True,
     classifiers=[
@@ -68,7 +68,7 @@ setuptools.setup(
     cmdclass=cmdclass,
     python_requires='>=3',
     zip_safe=False, # Since we load so file from the filesystem, we can not run from zip file
-    setup_requires=['cffi>=1.0'],
-    install_requires=['cffi>=1.0'],
+    setup_requires=['cffi>=1.0', 'requests', 'tqdm'],
+    install_requires=['cffi>=1.0', 'requests', 'tqdm'],
     cffi_modules=['vosk_builder.py:ffibuilder'],
 )

python/vosk/__init__.py

@@ -1,7 +1,18 @@
 import os
 import sys
 
+import requests
+from urllib.request import urlretrieve
+from zipfile import ZipFile
+from re import match
+from pathlib import Path
 from .vosk_cffi import ffi as _ffi
+from tqdm import tqdm
+
+# Remote location of the models and local folders
+MODEL_PRE_URL = 'https://alphacephei.com/vosk/models/'
+MODEL_LIST_URL = MODEL_PRE_URL + 'model-list.json'
+MODEL_DIRS = [os.getenv('VOSK_MODEL_PATH'), Path('/usr/share/vosk'), Path.home() / 'AppData/Local/vosk', Path.home() / '.cache/vosk']
 
 def open_dll():
     dlldir = os.path.abspath(os.path.dirname(__file__))
@@ -20,11 +31,24 @@ def open_dll():
 
 _c = open_dll()
 
+def list_models():
+    response = requests.get(MODEL_LIST_URL)
+    for model in response.json():
+        print(model['name']) 
+
+def list_languages():
+    response = requests.get(MODEL_LIST_URL)
+    languages = set([m['lang'] for m in response.json()])
+    for lang in languages:
+        print (lang)
+
 class Model(object):
-
-    def __init__(self, model_path):
-        self._handle = _c.vosk_model_new(model_path.encode('utf-8'))
-
+    def __init__(self, model_path=None, model_name=None, lang=None):
+        if model_path != None:
+            self._handle = _c.vosk_model_new(model_path.encode('utf-8'))
+        else:
+            model_path = self.get_model_path(model_name, lang)
+            self._handle = _c.vosk_model_new(model_path.encode('utf-8'))
         if self._handle == _ffi.NULL:
             raise Exception("Failed to create a model")
 
@@ -34,6 +58,68 @@ class Model(object):
     def vosk_model_find_word(self, word):
         return _c.vosk_model_find_word(self._handle, word.encode('utf-8'))
 
+    def get_model_path(self, model_name, lang):
+        if model_name is None:
+            model_path = self.get_model_by_lang(lang)
+        else:
+            model_path = self.get_model_by_name(model_name)
+        return str(model_path)
+
+    def get_model_by_name(self, model_name):
+        for directory in MODEL_DIRS:
+            if directory is None or not Path(directory).exists():
+                continue
+            model_file_list = os.listdir(directory)
+            model_file = [model for model in model_file_list if model == model_name]
+            if model_file != []:
+                return Path(directory, model_file[0])
+        response = requests.get(MODEL_LIST_URL)
+        result_model = [model['name'] for model in response.json() if model['name'] == model_name]
+        if result_model == []:
+            raise Exception("model name %s does not exist" % (model_name))
+        else:
+            self.download_model(Path(directory, result_model[0]))
+            return Path(directory, result_model[0])
+
+    def get_model_by_lang(self, lang):
+        for directory in MODEL_DIRS:
+            if directory is None or not Path(directory).exists():
+                continue
+            model_file_list = os.listdir(directory)
+            model_file = [model for model in model_file_list if match(f"vosk-model(-small)?-{lang}", model)]
+            if model_file != []:
+                return Path(directory, model_file[0])
+        response = requests.get(MODEL_LIST_URL)
+        result_model = [model['name'] for model in response.json() if model['lang'] == lang and model['type'] == 'small' and model['obsolete'] == 'false']
+        if result_model == []:
+            raise Exception("lang %s does not exist" % (lang))
+        else:
+            self.download_model(Path(directory, result_model[0]))
+            return Path(directory, result_model[0])
+
+    def download_model(self, model_name):
+        if not MODEL_DIRS[3].exists():
+            MODEL_DIRS[3].mkdir()
+        with tqdm(unit='B', unit_scale=True, unit_divisor=1024, miniters=1,
+                desc=(MODEL_PRE_URL + str(model_name.name) + '.zip').split('/')[-1]) as t:
+            reporthook = self.download_progress_hook(t)
+            urlretrieve(MODEL_PRE_URL + str(model_name.name) + '.zip', str(model_name) + '.zip', 
+                reporthook=reporthook, data=None)
+            t.total = t.n
+            with ZipFile(str(model_name) + '.zip', 'r') as model_ref:
+                model_ref.extractall(model_name.parent)
+            Path(str(model_name) + '.zip').unlink()
+
+    def download_progress_hook(self, t):
+        last_b = [0]
+        def update_to(b=1, bsize=1, tsize=None):
+            if tsize not in (None, -1):
+                t.total = tsize
+            displayed = t.update((b - last_b[0]) * bsize)
+            last_b[0] = b
+            return displayed
+        return update_to
+
 class SpkModel(object):
 
     def __init__(self, model_path):

python/vosk/transcriber/cli.py

@@ -3,103 +3,76 @@
 import logging
 import argparse
 
-from datetime import datetime as dt
-from vosk.transcriber.transcriber import Transcriber
-from multiprocessing.dummy import Pool
 from pathlib import Path
-
+from vosk import list_models, list_languages
+from vosk.transcriber.transcriber import Transcriber
 
 parser = argparse.ArgumentParser(
         description = 'Transcribe audio file and save result in selected format')
 parser.add_argument(
-        '-model', type=str,
+        '--model', '-m', type=str,
         help='model path')
 parser.add_argument(
-        '-list_models', default=False, action='store_true', 
+        '--list-models', default=False, action='store_true', 
         help='list available models')
 parser.add_argument(
-        '-list_languages', default=False, action='store_true',
+        '--list-languages', default=False, action='store_true',
         help='list available languages')
 parser.add_argument(
-        '-model_name',  default='vosk-model-small-en-us-0.15', type=str,
+        '--model-name', '-n', type=str,
         help='select model by name')
 parser.add_argument(
-        '-lang', type=str,
+        '--lang', '-l', default='en-us', type=str,
         help='select model by language')
 parser.add_argument(
-        '-input', type=str,
+        '--input', '-i', type=str,
         help='audiofile')
 parser.add_argument(
-        '-output', default='', type=str,
+        '--output', '-o', default='', type=str,
         help='optional output filename path')
 parser.add_argument(
-        '-otype', '--outputtype', default='txt', type=str,
+        '--output-type', '-t', default='txt', type=str,
         help='optional arg output data type')
 parser.add_argument(
-        '--log', default='INFO',
+        '--log-level', default='INFO',
         help='logging level')
 
-args = parser.parse_args()
-log_level = args.log.upper()
-logging.getLogger().setLevel(log_level)
-logging.info('checking args')
+def main():
 
-def get_results(inputdata):
-    logging.info('converting audiofile to 16K sampled wav')
-    stream = transcriber.resample_ffmpeg(inputdata[0])
-    logging.info('starting transcription')
-    final_result, tot_samples = transcriber.transcribe(model, stream, args)
-    logging.info('complete')
-    if args.output:
-        with open(inputdata[1], 'w', encoding='utf-8') as fh:
-            fh.write(final_result)
-        logging.info('output written to %s' % (inputdata[1]))
-    else:
-        print(final_result)
-    return final_result, tot_samples
+    args = parser.parse_args()
+    log_level = args.log_level.upper()
+    logging.getLogger().setLevel(log_level)
 
-def main(args):
-    global model
-    global transcriber
-    transcriber = Transcriber()
-    transcriber.check_args(args)
-    if args.input:
-        model = transcriber.get_model(args)
-        if Path(args.input).is_dir() and Path(args.output).is_dir():
-            task_list = transcriber.get_task_list(args)
-            with Pool() as pool:
-                for final_result, tot_samples in pool.map(get_results, file_list):
-                    return final_result, tot_samples
-        else:
-            if Path(args.input).is_file():
-                task = (args.input, args.output)
-                final_result, tot_samples = get_results(task)
-            elif not Path(args.input).exists():
-                logging.info('File %s does not exist, please select an existing file' % (args.input))
-                exit(1)
-            elif not Path(args.output).exists():
-                logging.info('Folder %s does not exist, please select an existing folder' % (args.output))
-                exit(1)
-        return final_result, tot_samples
-    else:
+    if args.list_models == True:
+        list_models()
+        return
+
+    if args.list_languages == True:
+        list_languages()
+        return
+
+    if not args.input:
         logging.info('Please specify input file or directory')
         exit(1)
 
-def get_start_time():
-    start_time = dt.now()
-    return start_time
+    if not Path(args.input).exists():
+        logging.info('File %s does not exist, please specify an existing file/directory' % (args.input))
+        exit(1)
 
-def get_end_time(start_time):
-    script_time = str(dt.now() - start_time)
-    seconds = script_time[5:8].strip('0')
-    mcseconds = script_time[8:].strip('0')
-    return script_time.strip(':0'), seconds.rstrip('.'), mcseconds
+    if args.output !='' and not Path(args.output).exists():
+        logging.info('Output %s does not exist, please specify an existing file' % (args.output))
+        exit(1)
 
-def cli():  # entrypoint used in setup.py
-    start_time = get_start_time()
-    tot_samples = main(args)[1]
-    diff_end_start, sec, mcsec = get_end_time(start_time)
-    logging.info(f'''Execution time: {sec} sec, {mcsec} mcsec; xRT: {format(tot_samples / 16000.0 / float(diff_end_start), '.3f')}''')
+    transcriber = Transcriber(args)
+
+    if Path(args.input).is_dir() and Path(args.output).is_dir():
+        transcriber.process_dir(args)
+        return
+    elif Path(args.input).is_file() and (args.output=='' or Path(args.output).is_file()):
+        transcriber.process_file(args)
+    else:
+        logging.info('Wrong arguments, input and output must be same type')
+        exit(1)
 
 if __name__ == "__main__":
     main()

python/vosk/transcriber/transcriber.py

@@ -1,30 +1,25 @@
 import json
 import subprocess
-import time
-import requests
-import urllib.request
-import zipfile
 import srt
 import datetime
 import os
-import re
 import logging
 
-from vosk import KaldiRecognizer, Model
 from pathlib import Path
-
-
-WORDS_PER_LINE = 7
-MODEL_PRE_URL = 'https://alphacephei.com/vosk/models/'
-MODEL_LIST_URL = MODEL_PRE_URL + 'model-list.json'
+from timeit import default_timer as timer
+from vosk import KaldiRecognizer, Model
 
 class Transcriber:
 
-    def get_result_and_tot_samples(self, rec, process):
+    def __init__(self, args):
+        self.model = Model(model_path=args.model, model_name=args.model_name, lang=args.lang)
+        self.args = args
+
+    def recognize_stream(self, rec, stream):
         tot_samples = 0
         result = []
         while True:
-            data = process.stdout.read(4000)
+            data = stream.stdout.read(4000)
             if len(data) == 0:
                 break
             if rec.AcceptWaveform(data):
@@ -33,29 +28,27 @@ class Transcriber:
         result.append(json.loads(rec.FinalResult()))
         return result, tot_samples
 
-    def transcribe(self, model, process, args):
-        rec = KaldiRecognizer(model, 16000)
-        rec.SetWords(True)
-        result, tot_samples = self.get_result_and_tot_samples(rec, process)
+    def format_result(self, result, words_per_line=7):
         final_result = ''
-        if args.outputtype == 'srt':
+        if self.args.output_type == 'srt':
             subs = []
             for i, res in enumerate(result):
                 if not 'result' in res:
                     continue
                 words = res['result']
-                for j in range(0, len(words), WORDS_PER_LINE):
-                    line = words[j : j + WORDS_PER_LINE]
+                for j in range(0, len(words), words_per_line):
+                    line = words[j : j + words_per_line]
                     s = srt.Subtitle(index=len(subs),
                             content = ' '.join([l['word'] for l in line]),
                             start=datetime.timedelta(seconds=line[0]['start']),
                             end=datetime.timedelta(seconds=line[-1]['end']))
                     subs.append(s)
             final_result = srt.compose(subs)
-        elif args.outputtype == 'txt':
+        elif self.args.output_type == 'txt':
             for part in result:
                 final_result += part['text'] + ' '
-        return final_result, tot_samples
+        return final_result
+
 
     def resample_ffmpeg(self, infile):
         stream = subprocess.Popen(
@@ -65,69 +58,32 @@ class Transcriber:
             stdout=subprocess.PIPE)
         return stream
 
-    def get_task_list(self, args):
-        task_list = [(Path(args.input, fn), Path(args.output, Path(fn).stem).with_suffix('.' + args.outputtype)) for fn in os.listdir(args.input)]
-        return task_list
 
-    def list_models(self):
-        response = requests.get(MODEL_LIST_URL)
-        [print(model['name']) for model in response.json()]
-        exit(1)
+    def process_entry(self, inputdata):
+        logging.info(f'Recognizing {inputdata[0]}')
 
-    def list_languages(self):
-        response = requests.get(MODEL_LIST_URL)
-        list_languages = set([language['lang'] for language in response.json()])
-        print(*list_languages, sep='\n')
-        exit(1)
+        rec = KaldiRecognizer(self.model, 16000)
+        rec.SetWords(True)
 
-    def check_args(self, args):
-        if args.list_models == True:
-            self.list_models()
-        elif args.list_languages == True:
-            self.list_languages()
+        stream = self.resample_ffmpeg(inputdata[0])
+        result, tot_samples = self.recognize_stream(rec, stream)
+        final_result = self.format_result(result)
 
-    def get_model_by_name(self, args, models_path):
-        if not Path.is_dir(Path(models_path, args.model_name)):
-            response = requests.get(MODEL_LIST_URL)
-            result = [model['name'] for model in response.json() if model['name'] == args.model_name]
-            if result == []:
-                logging.info('model name "%s" does not exist, request -list_models to see available models' % (args.model_name))
-                exit(1)
-            else:
-                result = result[0]
+        if inputdata[1] != '':
+            with open(inputdata[1], 'w', encoding='utf-8') as fh:
+                fh.write(final_result)
         else:
-            result = args.model_name
-        return result
+            print(final_result)
+        return final_result, tot_samples
 
-    def get_model_by_lang(self, args, models_path):
-        model_file_list = os.listdir(models_path)
-        model_file = [model for model in model_file_list if re.match(f"vosk-model(-small)?-{args.lang}", model)]
-        if model_file == []:
-            response = requests.get(MODEL_LIST_URL)
-            result = [model['name'] for model in response.json() if model['lang'] == args.lang and model['type'] == 'small' and model['obsolete'] == 'false']
-            if result == []:
-                logging.info('language "%s" does not exist, request -list_languages to see available languages' % (args.lang))
-                exit(1)
-            else:
-                result = result[0]
-        else:
-            result = model_file[0]
-        return result
 
-    def get_model(self, args):
-        models_path = Path.home() / '.cache' / 'vosk'
-        if not Path.is_dir(models_path):
-            Path.mkdir(models_path)
-        if args.lang == None:
-            model_name = self.get_model_by_name(args, models_path)
-        else:
-            model_name = self.get_model_by_lang(args, models_path)
-        model_location = models_path / model_name
-        if not model_location.exists():
-            model_zip = str(model_location) + '.zip'
-            urllib.request.urlretrieve(MODEL_PRE_URL + model_name + '.zip', model_zip)
-            with zipfile.ZipFile(model_zip, 'r') as model_ref:
-                model_ref.extractall(models_path)
-            Path.unlink(Path(model_zip))
-        model = Model(str(model_location))
-        return model
+    def process_directory(self,args):
+        task_list = [(Path(args.input, fn), Path(args.output, Path(fn).stem).with_suffix('.' + args.output_type)) for fn in os.listdir(args.input)]
+        with Pool() as pool:
+            pool.map(self.process_entry, file_list)
+
+    def process_file(self, args):
+        start_time = timer()
+        final_result, tot_samples = self.process_entry([args.input, args.output])
+        elapsed = timer() - start_time
+        logging.info(f'''Execution time: {elapsed:.3f} sec; xRT: {format(tot_samples / 16000.0 / float(elapsed), '.3f')}''')

travis/Dockerfile.dockcross

@@ -21,16 +21,20 @@ RUN apt-get update && \
         python3-cffi \
     && rm -rf /var/lib/apt/lists/*
 
-ARG OPENBLAS_ARCH=ARMV7
-ARG ARM_HARDWARE_OPTS="-mfloat-abi=hard -mfpu=neon"
+ARG OPENBLAS_ARGS=
 RUN cd /opt \
     && git clone -b vosk --single-branch https://github.com/alphacep/kaldi \
     && cd kaldi/tools \
     && git clone -b v0.3.20 --single-branch https://github.com/xianyi/OpenBLAS \
     && git clone -b v3.2.1  --single-branch https://github.com/alphacep/clapack \
-    && make -C OpenBLAS ONLY_CBLAS=1 TARGET="${OPENBLAS_ARCH}" HOSTCC=gcc USE_LOCKING=1 USE_THREAD=0 all \
+    && echo ${OPENBLAS_ARGS} \
+    && make -C OpenBLAS ONLY_CBLAS=1 ${OPENBLAS_ARGS} HOSTCC=gcc USE_LOCKING=1 USE_THREAD=0 all \
     && make -C OpenBLAS PREFIX=$(pwd)/OpenBLAS/install install \
-    && mkdir -p clapack/BUILD && cd clapack/BUILD && cmake .. && make -j 10 && find . -name "*.a" | xargs cp -t ../../OpenBLAS/install/lib \
+    && mkdir -p clapack/BUILD && cd clapack/BUILD && cmake .. \
+    && make -j 10 -C F2CLIBS \
+    && make -j 10 -C BLAS \
+    && make -j 10 -C SRC \
+    && find . -name "*.a" | xargs cp -t ../../OpenBLAS/install/lib \
     && cd /opt/kaldi/tools \
     && git clone --single-branch https://github.com/alphacep/openfst openfst \
     && cd openfst \
@@ -39,7 +43,6 @@ RUN cd /opt \
     && make -j 10 && make install \
     && cd /opt/kaldi/src \
     && sed -i "s:TARGET_ARCH=\"\`uname -m\`\":TARGET_ARCH=$(echo $CROSS_TRIPLE|cut -d - -f 1):g" configure \
-    && sed -i "s:-mfloat-abi=hard -mfpu=neon:${ARM_HARDWARE_OPTS}:g" makefiles/linux_openblas_arm.mk \
     && sed -i "s: -O1 : -O3 :g" makefiles/linux_openblas_arm.mk \
     && ./configure --mathlib=OPENBLAS_CLAPACK --shared --use-cuda=no \
     && make -j 10 online2 lm rnnlm \

travis/build-dockcross.sh

@@ -3,8 +3,10 @@
 set -e
 set -x
 
-docker build --build-arg="DOCKCROSS_IMAGE=alphacep/dockcross-linux-armv7" --build-arg="OPENBLAS_ARCH=ARMV7" --file Dockerfile.dockcross --tag alphacep/kaldi-dockcross-armv7:latest .
-docker build --build-arg="DOCKCROSS_IMAGE=dockcross/linux-arm64" --build-arg="OPENBLAS_ARCH=ARMV8" --file Dockerfile.dockcross --tag alphacep/kaldi-dockcross-arm64:latest .
+docker build --build-arg="DOCKCROSS_IMAGE=alphacep/dockcross-linux-armv7" --build-arg="OPENBLAS_ARGS=TARGET=ARMV7" --file Dockerfile.dockcross --tag alphacep/kaldi-dockcross-armv7:latest .
+docker build --build-arg="DOCKCROSS_IMAGE=dockcross/linux-arm64" --build-arg="OPENBLAS_ARGS=TARGET=ARMV8" --file Dockerfile.dockcross --tag alphacep/kaldi-dockcross-arm64:latest .
+docker build --build-arg="DOCKCROSS_IMAGE=dockcross/linux-x86" --build-arg="OPENBLAS_ARGS=TARGET=CORE2\ DYNAMIC_ARCH=1" --file Dockerfile.dockcross --tag alphacep/kaldi-dockcross-x86:latest .
 
 docker run --rm -v /home/shmyrev/travis/vosk-api/:/io alphacep/kaldi-dockcross-armv7 /io/travis/build-wheels-dockcross.sh
 docker run --rm -v /home/shmyrev/travis/vosk-api/:/io alphacep/kaldi-dockcross-arm64 /io/travis/build-wheels-dockcross.sh
+docker run --rm -v /home/shmyrev/travis/vosk-api/:/io alphacep/kaldi-dockcross-x86 /io/travis/build-wheels-dockcross.sh

travis/build-wheels-dockcross.sh

@@ -5,7 +5,7 @@ set -e -x
 cd /opt
 git clone https://github.com/alphacep/vosk-api
 cd /opt/vosk-api/src
-KALDI_ROOT=/opt/kaldi make -j $(nproc)
+KALDI_ROOT=/opt/kaldi EXTRA_LDFLAGS="-latomic" make -j $(nproc)
 
 # Decide architecture name
 export VOSK_SOURCE=/opt/vosk-api
@@ -16,11 +16,15 @@ case $CROSS_TRIPLE in
     *aarch64-*)
         export VOSK_ARCHITECTURE=aarch64
         ;;
+    *i686-*)
+        export VOSK_ARCHITECTURE=x86
+        ;;
 esac
 
 # Copy library to output folder
-mkdir -p /io/wheelhouse/linux-$VOSK_ARCHITECTURE
-cp /opt/vosk-api/src/*.so /io/wheelhouse/linux-$VOSK_ARCHITECTURE
+mkdir -p /io/wheelhouse/vosk-linux-$VOSK_ARCHITECTURE
+cp /opt/vosk-api/src/*.so /opt/vosk-api/src/vosk_api.h /io/wheelhouse/vosk-linux-$VOSK_ARCHITECTURE
 
 # Build wheel
-pip3 wheel /opt/vosk-api/python --no-deps -w /io/wheelhouse
+python3 -m pip install requests tqdm
+python3 -m pip wheel /opt/vosk-api/python --no-deps -w /io/wheelhouse

travis/build-wheels-win.sh

@@ -13,8 +13,8 @@ cp /usr/lib/gcc/x86_64-w64-mingw32/*-posix/libgcc_s_seh-1.dll /opt/vosk-api/src
 cp /usr/x86_64-w64-mingw32/lib/libwinpthread-1.dll /opt/vosk-api/src
 
 # Copy dlls to output folder
-mkdir -p /io/wheelhouse/win64
-cp /opt/vosk-api/src/*.{dll,lib} /io/wheelhouse/win64
+mkdir -p /io/wheelhouse/vosk-win64
+cp /opt/vosk-api/src/*.{dll,lib} /opt/vosk-api/src/vosk_api.h /io/wheelhouse/vosk-win64
 
 # Build wheel and put to the output folder
 export VOSK_SOURCE=/opt/vosk-api

travis/build-wheels-win32.sh

@@ -13,8 +13,8 @@ cp /usr/lib/gcc/i686-w64-mingw32/*-posix/libgcc_s_sjlj-1.dll /opt/vosk-api/src
 cp /usr/i686-w64-mingw32/lib/libwinpthread-1.dll /opt/vosk-api/src
 
 # Copy dlls to output folder
-mkdir -p /io/wheelhouse/win32
-cp /opt/vosk-api/src/*.{dll,lib} /io/wheelhouse/win32
+mkdir -p /io/wheelhouse/vosk-win32
+cp /opt/vosk-api/src/*.{dll,lib} /opt/vosk-api/src/vosk_api.h /io/wheelhouse/vosk-win32
 
 # Build wheel and put to the output folder
 export VOSK_SOURCE=/opt/vosk-api

travis/build-wheels.sh

@@ -8,8 +8,8 @@ cd vosk-api/src
 KALDI_ROOT=/opt/kaldi OPENFST_ROOT=/opt/kaldi/tools/openfst OPENBLAS_ROOT=/opt/kaldi/tools/OpenBLAS/install make -j $(nproc)
 
 # Copy dlls to output folder
-mkdir -p /io/wheelhouse/linux
-cp /opt/vosk-api/src/*.so /io/wheelhouse/linux
+mkdir -p /io/wheelhouse/vosk-linux-x86_64
+cp /opt/vosk-api/src/*.so /opt/vosk-api/src/vosk_api.h /io/wheelhouse/vosk-linux-x86_64
 
 # Build wheel and put to the output folder
 mkdir -p /opt/wheelhouse

webjs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vosk-js",
-  "version": "0.3.29",
+  "version": "0.3.41",
   "description": "Node binding for continuous voice recoginition through vosk-api.",
   "repository": {
     "type": "git",