chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,639 @@
|
||||
[MAIN]
|
||||
|
||||
# Analyse import fallback blocks. This can be used to support both Python 2 and
|
||||
# 3 compatible code, which means that the block might have code that exists
|
||||
# only in one or another interpreter, leading to false positives when analysed.
|
||||
analyse-fallback-blocks=no
|
||||
|
||||
# Load and enable all available extensions. Use --list-extensions to see a list
|
||||
# all available extensions.
|
||||
#enable-all-extensions=
|
||||
|
||||
# In error mode, messages with a category besides ERROR or FATAL are
|
||||
# suppressed, and no reports are done by default. Error mode is compatible with
|
||||
# disabling specific errors.
|
||||
#errors-only=
|
||||
|
||||
# Always return a 0 (non-error) status code, even if lint errors are found.
|
||||
# This is primarily useful in continuous integration scripts.
|
||||
#exit-zero=
|
||||
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code.
|
||||
extension-pkg-allow-list=
|
||||
|
||||
# A comma-separated list of package or module names from where C extensions may
|
||||
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||
# run arbitrary code. (This is an alternative name to extension-pkg-allow-list
|
||||
# for backward compatibility.)
|
||||
extension-pkg-whitelist=
|
||||
|
||||
# Return non-zero exit code if any of these messages/categories are detected,
|
||||
# even if score is above --fail-under value. Syntax same as enable. Messages
|
||||
# specified are enabled, while categories only check already-enabled messages.
|
||||
fail-on=
|
||||
|
||||
# Specify a score threshold under which the program will exit with error.
|
||||
fail-under=10
|
||||
|
||||
# Interpret the stdin as a python script, whose filename needs to be passed as
|
||||
# the module_or_package argument.
|
||||
#from-stdin=
|
||||
|
||||
# Files or directories to be skipped. They should be base names, not paths.
|
||||
ignore=CVS
|
||||
|
||||
# Add files or directories matching the regular expressions patterns to the
|
||||
# ignore-list. The regex matches against paths and can be in Posix or Windows
|
||||
# format. Because '\' represents the directory delimiter on Windows systems, it
|
||||
# can't be used as an escape character.
|
||||
ignore-paths=
|
||||
|
||||
# Files or directories matching the regular expression patterns are skipped.
|
||||
# The regex matches against base names, not paths. The default value ignores
|
||||
# Emacs file locks
|
||||
ignore-patterns=^\.#
|
||||
|
||||
# List of module names for which member attributes should not be checked
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis). It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=
|
||||
|
||||
# Python code to execute, usually for sys.path manipulation such as
|
||||
# pygtk.require().
|
||||
#init-hook=
|
||||
|
||||
# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
|
||||
# number of processors available to use, and will cap the count on Windows to
|
||||
# avoid hangs.
|
||||
jobs=1
|
||||
|
||||
# Control the amount of potential inferred values when inferring a single
|
||||
# object. This can help the performance when dealing with large functions or
|
||||
# complex, nested conditions.
|
||||
limit-inference-results=100
|
||||
|
||||
# List of plugins (as comma separated values of python module names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=yes
|
||||
|
||||
# Minimum Python version to use for version dependent checks. Will default to
|
||||
# the version used to run pylint.
|
||||
py-version=3.7
|
||||
|
||||
# Discover python modules and packages in the file system subtree.
|
||||
recursive=no
|
||||
|
||||
# When enabled, pylint would attempt to guess common misconfiguration and emit
|
||||
# user-friendly hints instead of false-positive error messages.
|
||||
suggestion-mode=yes
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
unsafe-load-any-extension=no
|
||||
|
||||
# In verbose mode, extra non-checker-related info will be displayed.
|
||||
#verbose=
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Python expression which should return a score less than or equal to 10. You
|
||||
# have access to the variables 'fatal', 'error', 'warning', 'refactor',
|
||||
# 'convention', and 'info' which contain the number of messages in each
|
||||
# category, as well as 'statement' which is the total number of statements
|
||||
# analyzed. This score is used by the global evaluation report (RP0004).
|
||||
evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))
|
||||
|
||||
# Template used to display messages. This is a python new-style format string
|
||||
# used to format the message information. See doc for all details.
|
||||
msg-template=
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, json
|
||||
# and msvs (visual studio). You can also give a reporter class, e.g.
|
||||
# mypackage.mymodule.MyReporterClass.
|
||||
#output-format=
|
||||
|
||||
# Tells whether to display a full report or only the messages.
|
||||
reports=no
|
||||
|
||||
# Activate the evaluation score.
|
||||
score=yes
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||
# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE,
|
||||
# UNDEFINED.
|
||||
confidence=HIGH,
|
||||
CONTROL_FLOW,
|
||||
INFERENCE,
|
||||
INFERENCE_FAILURE,
|
||||
UNDEFINED
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once). You can also use "--disable=all" to
|
||||
# disable everything first and then re-enable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use "--disable=all --enable=classes
|
||||
# --disable=W".
|
||||
disable=raw-checker-failed,
|
||||
bad-inline-option,
|
||||
locally-disabled,
|
||||
file-ignored,
|
||||
suppressed-message,
|
||||
useless-suppression,
|
||||
deprecated-pragma,
|
||||
use-symbolic-message-instead,
|
||||
missing-module-docstring, # added
|
||||
missing-class-docstring, # added
|
||||
missing-function-docstring, # added
|
||||
invalid-name, # added
|
||||
broad-except, # added
|
||||
wrong-import-order, # added
|
||||
consider-using-f-string, # added
|
||||
logging-format-interpolation, # added
|
||||
no-member, # added
|
||||
consider-using-with, # added
|
||||
unused-argument,
|
||||
no-else-return, # added
|
||||
too-few-public-methods, # added
|
||||
import-error, # added
|
||||
protected-access, # added
|
||||
unused-variable, # added
|
||||
R0801 # added (similarity of two scripts)
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once). See also the "--disable" option for examples.
|
||||
enable=c-extension-no-member
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# Naming style matching correct argument names.
|
||||
argument-naming-style=snake_case
|
||||
|
||||
# Regular expression matching correct argument names. Overrides argument-
|
||||
# naming-style. If left empty, argument names will be checked with the set
|
||||
# naming style.
|
||||
#argument-rgx=
|
||||
|
||||
# Naming style matching correct attribute names.
|
||||
attr-naming-style=snake_case
|
||||
|
||||
# Regular expression matching correct attribute names. Overrides attr-naming-
|
||||
# style. If left empty, attribute names will be checked with the set naming
|
||||
# style.
|
||||
#attr-rgx=
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma.
|
||||
bad-names=foo,
|
||||
bar,
|
||||
baz,
|
||||
toto,
|
||||
tutu,
|
||||
tata
|
||||
|
||||
# Bad variable names regexes, separated by a comma. If names match any regex,
|
||||
# they will always be refused
|
||||
bad-names-rgxs=
|
||||
|
||||
# Naming style matching correct class attribute names.
|
||||
class-attribute-naming-style=any
|
||||
|
||||
# Regular expression matching correct class attribute names. Overrides class-
|
||||
# attribute-naming-style. If left empty, class attribute names will be checked
|
||||
# with the set naming style.
|
||||
#class-attribute-rgx=
|
||||
|
||||
# Naming style matching correct class constant names.
|
||||
class-const-naming-style=UPPER_CASE
|
||||
|
||||
# Regular expression matching correct class constant names. Overrides class-
|
||||
# const-naming-style. If left empty, class constant names will be checked with
|
||||
# the set naming style.
|
||||
#class-const-rgx=
|
||||
|
||||
# Naming style matching correct class names.
|
||||
class-naming-style=PascalCase
|
||||
|
||||
# Regular expression matching correct class names. Overrides class-naming-
|
||||
# style. If left empty, class names will be checked with the set naming style.
|
||||
#class-rgx=
|
||||
|
||||
# Naming style matching correct constant names.
|
||||
const-naming-style=UPPER_CASE
|
||||
|
||||
# Regular expression matching correct constant names. Overrides const-naming-
|
||||
# style. If left empty, constant names will be checked with the set naming
|
||||
# style.
|
||||
#const-rgx=
|
||||
|
||||
# Minimum line length for functions/classes that require docstrings, shorter
|
||||
# ones are exempt.
|
||||
docstring-min-length=-1
|
||||
|
||||
# Naming style matching correct function names.
|
||||
function-naming-style=snake_case
|
||||
|
||||
# Regular expression matching correct function names. Overrides function-
|
||||
# naming-style. If left empty, function names will be checked with the set
|
||||
# naming style.
|
||||
#function-rgx=
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma.
|
||||
good-names=i,
|
||||
j,
|
||||
k,
|
||||
ex,
|
||||
Run,
|
||||
_,
|
||||
x,
|
||||
y,
|
||||
nx,
|
||||
ny
|
||||
|
||||
# Good variable names regexes, separated by a comma. If names match any regex,
|
||||
# they will always be accepted
|
||||
good-names-rgxs=
|
||||
|
||||
# Include a hint for the correct naming format with invalid-name.
|
||||
include-naming-hint=no
|
||||
|
||||
# Naming style matching correct inline iteration names.
|
||||
inlinevar-naming-style=any
|
||||
|
||||
# Regular expression matching correct inline iteration names. Overrides
|
||||
# inlinevar-naming-style. If left empty, inline iteration names will be checked
|
||||
# with the set naming style.
|
||||
#inlinevar-rgx=
|
||||
|
||||
# Naming style matching correct method names.
|
||||
method-naming-style=snake_case
|
||||
|
||||
# Regular expression matching correct method names. Overrides method-naming-
|
||||
# style. If left empty, method names will be checked with the set naming style.
|
||||
#method-rgx=
|
||||
|
||||
# Naming style matching correct module names.
|
||||
module-naming-style=snake_case
|
||||
|
||||
# Regular expression matching correct module names. Overrides module-naming-
|
||||
# style. If left empty, module names will be checked with the set naming style.
|
||||
#module-rgx=
|
||||
|
||||
# Colon-delimited sets of names that determine each other's naming style when
|
||||
# the name regexes allow several styles.
|
||||
name-group=
|
||||
|
||||
# Regular expression which should only match function or class names that do
|
||||
# not require a docstring.
|
||||
no-docstring-rgx=^_
|
||||
|
||||
# List of decorators that produce properties, such as abc.abstractproperty. Add
|
||||
# to this list to register other decorators that produce valid properties.
|
||||
# These decorators are taken in consideration only for invalid-name.
|
||||
property-classes=abc.abstractproperty
|
||||
|
||||
# Regular expression matching correct type variable names. If left empty, type
|
||||
# variable names will be checked with the set naming style.
|
||||
#typevar-rgx=
|
||||
|
||||
# Naming style matching correct variable names.
|
||||
variable-naming-style=snake_case
|
||||
|
||||
# Regular expression matching correct variable names. Overrides variable-
|
||||
# naming-style. If left empty, variable names will be checked with the set
|
||||
# naming style.
|
||||
#variable-rgx=
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid defining new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
# Tells whether unused global variables should be treated as a violation.
|
||||
allow-global-unused-variables=yes
|
||||
|
||||
# List of names allowed to shadow builtins
|
||||
allowed-redefined-builtins=
|
||||
|
||||
# List of strings which can identify a callback function by name. A callback
|
||||
# name must start or end with one of those strings.
|
||||
callbacks=cb_,
|
||||
_cb
|
||||
|
||||
# A regular expression matching the name of dummy variables (i.e. expected to
|
||||
# not be used).
|
||||
dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
|
||||
|
||||
# Argument names that match this expression will be ignored.
|
||||
ignored-argument-names=_.*|^ignored_|^unused_
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# List of qualified module names which can have objects that can redefine
|
||||
# builtins.
|
||||
redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Comments are removed from the similarity computation
|
||||
ignore-comments=yes
|
||||
|
||||
# Docstrings are removed from the similarity computation
|
||||
ignore-docstrings=yes
|
||||
|
||||
# Imports are removed from the similarity computation
|
||||
ignore-imports=yes
|
||||
|
||||
# Signatures are removed from the similarity computation
|
||||
ignore-signatures=yes
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=FIXME,
|
||||
XXX,
|
||||
TODO
|
||||
|
||||
# Regular expression of note tags to take in consideration.
|
||||
notes-rgx=
|
||||
|
||||
|
||||
[STRING]
|
||||
|
||||
# This flag controls whether inconsistent-quotes generates a warning when the
|
||||
# character used as a quote delimiter is used inconsistently within a module.
|
||||
check-quote-consistency=no
|
||||
|
||||
# This flag controls whether the implicit-str-concat should generate a warning
|
||||
# on implicit string concatenation in sequences defined over several lines.
|
||||
check-str-concat-over-line-jumps=no
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# List of modules that can be imported at any level, not just the top level
|
||||
# one.
|
||||
allow-any-import-level=
|
||||
|
||||
# Allow wildcard imports from modules that define __all__.
|
||||
allow-wildcard-with-all=no
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma.
|
||||
deprecated-modules=
|
||||
|
||||
# Output a graph (.gv or any supported image format) of external dependencies
|
||||
# to the given file (report RP0402 must not be disabled).
|
||||
ext-import-graph=
|
||||
|
||||
# Output a graph (.gv or any supported image format) of all (i.e. internal and
|
||||
# external) dependencies to the given file (report RP0402 must not be
|
||||
# disabled).
|
||||
import-graph=
|
||||
|
||||
# Output a graph (.gv or any supported image format) of internal dependencies
|
||||
# to the given file (report RP0402 must not be disabled).
|
||||
int-import-graph=
|
||||
|
||||
# Force import order to recognize a module as part of the standard
|
||||
# compatibility libraries.
|
||||
known-standard-library=
|
||||
|
||||
# Force import order to recognize a module as part of a third party library.
|
||||
known-third-party=enchant
|
||||
|
||||
# Couples of modules and preferred modules, separated by a comma.
|
||||
preferred-modules=
|
||||
|
||||
|
||||
[EXCEPTIONS]
|
||||
|
||||
# Exceptions that will emit a warning when caught.
|
||||
overgeneral-exceptions=BaseException,
|
||||
Exception
|
||||
|
||||
|
||||
[DESIGN]
|
||||
|
||||
# List of regular expressions of class ancestor names to ignore when counting
|
||||
# public methods (see R0903)
|
||||
exclude-too-few-public-methods=
|
||||
|
||||
# List of qualified class names to ignore when counting class parents (see
|
||||
# R0901)
|
||||
ignored-parents=
|
||||
|
||||
# Maximum number of arguments for function / method.
|
||||
max-args=5
|
||||
|
||||
# Maximum number of attributes for a class (see R0902).
|
||||
max-attributes=7
|
||||
|
||||
# Maximum number of boolean expressions in an if statement (see R0916).
|
||||
max-bool-expr=5
|
||||
|
||||
# Maximum number of branch for function / method body.
|
||||
max-branches=12
|
||||
|
||||
# Maximum number of locals for function / method body.
|
||||
max-locals=15
|
||||
|
||||
# Maximum number of parents for a class (see R0901).
|
||||
max-parents=7
|
||||
|
||||
# Maximum number of public methods for a class (see R0904).
|
||||
max-public-methods=20
|
||||
|
||||
# Maximum number of return / yield for function / method body.
|
||||
max-returns=6
|
||||
|
||||
# Maximum number of statements in function / method body.
|
||||
max-statements=50
|
||||
|
||||
# Minimum number of public methods for a class (see R0903).
|
||||
min-public-methods=2
|
||||
|
||||
|
||||
[METHOD_ARGS]
|
||||
|
||||
# List of qualified names (i.e., library.method) which require a timeout
|
||||
# parameter e.g. 'requests.api.get,requests.api.post'
|
||||
timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request
|
||||
|
||||
|
||||
[LOGGING]
|
||||
|
||||
# The type of string formatting that logging methods do. `old` means using %
|
||||
# formatting, `new` is for `{}` formatting.
|
||||
logging-format-style=old
|
||||
|
||||
# Logging modules to check that the string format arguments are in logging
|
||||
# function parameter format.
|
||||
logging-modules=logging
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||
expected-line-ending-format=
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||
|
||||
# Number of spaces of indent required inside a hanging or continued line.
|
||||
indent-after-paren=4
|
||||
|
||||
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||
# tab).
|
||||
indent-string=' '
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=100
|
||||
|
||||
# Maximum number of lines in a module.
|
||||
max-module-lines=1000
|
||||
|
||||
# Allow the body of a class to be on the same line as the declaration if body
|
||||
# contains single statement.
|
||||
single-line-class-stmt=no
|
||||
|
||||
# Allow the body of an if to be on the same line as the test if there is no
|
||||
# else.
|
||||
single-line-if-stmt=no
|
||||
|
||||
|
||||
[REFACTORING]
|
||||
|
||||
# Maximum number of nested blocks for function / method body
|
||||
max-nested-blocks=5
|
||||
|
||||
# Complete name of functions that never returns. When checking for
|
||||
# inconsistent-return-statements if a never returning function is called then
|
||||
# it will be considered as an explicit return statement and no message will be
|
||||
# printed.
|
||||
never-returning-functions=sys.exit,argparse.parse_error
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# List of decorators that produce context managers, such as
|
||||
# contextlib.contextmanager. Add to this list to register other decorators that
|
||||
# produce valid context managers.
|
||||
contextmanager-decorators=contextlib.contextmanager
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=
|
||||
|
||||
# Tells whether to warn about missing members when the owner of the attribute
|
||||
# is inferred to be None.
|
||||
ignore-none=yes
|
||||
|
||||
# This flag controls whether pylint should warn about no-member and similar
|
||||
# checks whenever an opaque object is returned when inferring. The inference
|
||||
# can return multiple potential results while evaluating a Python object, but
|
||||
# some branches might not be evaluated, which results in partial inference. In
|
||||
# that case, it might be useful to still emit no-member and other checks for
|
||||
# the rest of the inferred objects.
|
||||
ignore-on-opaque-inference=yes
|
||||
|
||||
# List of symbolic message names to ignore for Mixin members.
|
||||
ignored-checks-for-mixins=no-member,
|
||||
not-async-context-manager,
|
||||
not-context-manager,
|
||||
attribute-defined-outside-init
|
||||
|
||||
# List of class names for which member attributes should not be checked (useful
|
||||
# for classes with dynamically set attributes). This supports the use of
|
||||
# qualified names.
|
||||
ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace
|
||||
|
||||
# Show a hint with possible names when a member name was not found. The aspect
|
||||
# of finding the hint is based on edit distance.
|
||||
missing-member-hint=yes
|
||||
|
||||
# The minimum edit distance a name should have in order to be considered a
|
||||
# similar match for a missing member name.
|
||||
missing-member-hint-distance=1
|
||||
|
||||
# The total number of similar names that should be taken in consideration when
|
||||
# showing a hint for a missing member.
|
||||
missing-member-max-choices=1
|
||||
|
||||
# Regex pattern to define which classes are considered mixins.
|
||||
mixin-class-rgx=.*[Mm]ixin
|
||||
|
||||
# List of decorators that change the signature of a decorated function.
|
||||
signature-mutators=
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# Warn about protected attribute access inside special methods
|
||||
check-protected-access-in-special-methods=no
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,
|
||||
__new__,
|
||||
setUp,
|
||||
__post_init__
|
||||
|
||||
# List of member names, which should be excluded from the protected access
|
||||
# warning.
|
||||
exclude-protected=_asdict,
|
||||
_fields,
|
||||
_replace,
|
||||
_source,
|
||||
_make
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls
|
||||
|
||||
# List of valid names for the first argument in a metaclass class method.
|
||||
valid-metaclass-classmethod-first-arg=cls
|
||||
|
||||
|
||||
[SPELLING]
|
||||
|
||||
# Limits count of emitted suggestions for spelling mistakes.
|
||||
max-spelling-suggestions=4
|
||||
|
||||
# Spelling dictionary name. Available dictionaries: none. To make it work,
|
||||
# install the 'python-enchant' package.
|
||||
spelling-dict=
|
||||
|
||||
# List of comma separated words that should be considered directives if they
|
||||
# appear at the beginning of a comment and should not be checked.
|
||||
spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:
|
||||
|
||||
# List of comma separated words that should not be checked.
|
||||
spelling-ignore-words=
|
||||
|
||||
# A path to a file that contains the private dictionary; one word per line.
|
||||
spelling-private-dict-file=
|
||||
|
||||
# Tells whether to store unknown words to the private dictionary (see the
|
||||
# --spelling-private-dict-file option) instead of raising a message.
|
||||
spelling-store-unknown-words=no
|
||||
@@ -0,0 +1,25 @@
|
||||
This is a Python module for Vosk.
|
||||
|
||||
Vosk is an offline open source speech recognition toolkit. It enables
|
||||
speech recognition for 20+ languages and dialects - English, Indian
|
||||
English, German, French, Spanish, Portuguese, Chinese, Russian, Turkish,
|
||||
Vietnamese, Italian, Dutch, Catalan, Arabic, Greek, Farsi, Filipino,
|
||||
Ukrainian, Kazakh, Swedish, Japanese, Esperanto, Hindi, Czech, Polish.
|
||||
More to come.
|
||||
|
||||
Vosk models are small (50 Mb) but provide continuous large vocabulary
|
||||
transcription, zero-latency response with streaming API, reconfigurable
|
||||
vocabulary and speaker identification.
|
||||
|
||||
Vosk supplies speech recognition for chatbots, smart home appliances,
|
||||
virtual assistants. It can also create subtitles for movies,
|
||||
transcription for lectures and interviews.
|
||||
|
||||
Vosk scales from small devices like Raspberry Pi or Android smartphone to
|
||||
big clusters.
|
||||
|
||||
# Documentation
|
||||
|
||||
For installation instructions, examples and documentation visit [Vosk
|
||||
Website](https://alphacephei.com/vosk). See also our project on
|
||||
[Github](https://github.com/alphacep/vosk-api).
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,558 @@
|
||||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "Vosk Adaptation",
|
||||
"provenance": [],
|
||||
"collapsed_sections": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
},
|
||||
"accelerator": "GPU",
|
||||
"gpuClass": "standard"
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "URzWMmv50-Ba",
|
||||
"outputId": "0e096a99-74dd-42e2-efb1-9cba784c3664"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"/content\n",
|
||||
"--2022-08-17 09:48:52-- https://alphacephei.com/vosk-colab/kaldi.tar.gz\n",
|
||||
"Resolving alphacephei.com (alphacephei.com)... 188.40.21.16, 2a01:4f8:13a:279f::2\n",
|
||||
"Connecting to alphacephei.com (alphacephei.com)|188.40.21.16|:443... connected.\n",
|
||||
"HTTP request sent, awaiting response... 200 OK\n",
|
||||
"Length: 809174554 (772M) [application/octet-stream]\n",
|
||||
"Saving to: ‘kaldi.tar.gz’\n",
|
||||
"\n",
|
||||
"kaldi.tar.gz 100%[===================>] 771.69M 20.3MB/s in 40s \n",
|
||||
"\n",
|
||||
"2022-08-17 09:49:33 (19.4 MB/s) - ‘kaldi.tar.gz’ saved [809174554/809174554]\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%cd /content\n",
|
||||
"!wget -c https://alphacephei.com/vosk-colab/kaldi.tar.gz\n",
|
||||
"!tar xzf kaldi.tar.gz"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"%cd /content/kaldi/egs/ac\n",
|
||||
"!wget -c https://alphacephei.com/vosk-colab/vosk-model-small-en-us-0.15-compile-colab.tar.gz\n",
|
||||
"!rm -rf vosk-model-small-en-us-0.15-compile-colab\n",
|
||||
"!tar xf vosk-model-small-en-us-0.15-compile-colab.tar.gz"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "-065p7WC2SHh",
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"outputId": "241c7473-7464-48d5-b48d-dc6e3bf4971d"
|
||||
},
|
||||
"execution_count": 8,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"/content/kaldi/egs/ac\n",
|
||||
"--2022-08-17 10:28:26-- https://alphacephei.com/vosk-colab/vosk-model-small-en-us-0.15-compile-colab.tar.gz\n",
|
||||
"Resolving alphacephei.com (alphacephei.com)... 188.40.21.16, 2a01:4f8:13a:279f::2\n",
|
||||
"Connecting to alphacephei.com (alphacephei.com)|188.40.21.16|:443... connected.\n",
|
||||
"HTTP request sent, awaiting response... 200 OK\n",
|
||||
"Length: 59618100 (57M) [application/octet-stream]\n",
|
||||
"Saving to: ‘vosk-model-small-en-us-0.15-compile-colab.tar.gz’\n",
|
||||
"\n",
|
||||
"vosk-model-small-en 100%[===================>] 56.86M 18.6MB/s in 3.6s \n",
|
||||
"\n",
|
||||
"2022-08-17 10:28:30 (15.7 MB/s) - ‘vosk-model-small-en-us-0.15-compile-colab.tar.gz’ saved [59618100/59618100]\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"%cd /content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab\n",
|
||||
"!ls\n",
|
||||
"!cat compile-graph.sh\n",
|
||||
"!bash compile-graph.sh"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "wuDjvNbd2sf9",
|
||||
"outputId": "34a1d2fe-d443-4574-e25d-824e38eb3a78"
|
||||
},
|
||||
"execution_count": 9,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab\n",
|
||||
"compile-graph.sh data_test decode.sh\texp\t local path.sh steps\n",
|
||||
"conf\t\t db\t dict.py\tget_vocab.py mfcc RESULTS utils\n",
|
||||
"#!/bin/bash\n",
|
||||
"\n",
|
||||
"set -x\n",
|
||||
"\n",
|
||||
". path.sh\n",
|
||||
"\n",
|
||||
"pip3 install phonetisaurus\n",
|
||||
"\n",
|
||||
"rm -rf data\n",
|
||||
"rm -rf exp/tdnn/lgraph\n",
|
||||
"rm -rf exp/tdnn/lgraph_orig\n",
|
||||
"\n",
|
||||
"mkdir -p data/dict\n",
|
||||
"cp db/phone/* data/dict\n",
|
||||
"./dict.py > data/dict/lexicon.txt\n",
|
||||
"\n",
|
||||
"python3 ./get_vocab.py > data/mix.vocab\n",
|
||||
"ngramsymbols data/mix.vocab data/mix.syms\n",
|
||||
"farcompilestrings --fst_type=compact --symbols=data/mix.syms --keep_symbols --unknown_symbol=\"[unk]\" db/extra.txt data/extra.far\n",
|
||||
"ngramcount --order=3 data/extra.far - |\n",
|
||||
" ngramprint --integers | grep -v \"<unk>\" | ngramread |\n",
|
||||
" ngramshrink --method=count_prune --count_pattern=\"3+:3\" |\n",
|
||||
" ngrammake --method=witten_bell - data/extra.mod\n",
|
||||
"gunzip -c db/en-50k-0.4-android.lm.gz | ngramread --renormalize_arpa --ARPA --symbols=data/mix.syms - data/en-us.mod\n",
|
||||
"ngrammerge --method=\"bayes_model_merge\" --normalize --alpha=0.95 --beta=0.05 data/en-us.mod data/extra.mod data/en-us-mix.mod\n",
|
||||
"ngramprint --ARPA data/en-us-mix.mod | gzip -c > data/en-us-mix.lm.gz\n",
|
||||
"\n",
|
||||
"# Prune for the first stage if needed\n",
|
||||
"# ngramshrink --method=relative_entropy --theta=2e-8 data/en-us-mix.mod data/en-us-mix-prune.mod\n",
|
||||
"# ngramprint --ARPA data/en-us-mix-prune.mod | gzip -c > data/en-us-mix-small.lm.gz\n",
|
||||
"\n",
|
||||
"utils/prepare_lang.sh data/dict \"[unk]\" data/lang_local data/lang\n",
|
||||
"utils/format_lm.sh data/lang db/en-50k-0.4-android.lm.gz data/dict/lexicon.txt data/lang_test\n",
|
||||
"utils/format_lm.sh data/lang data/en-us-mix.lm.gz data/dict/lexicon.txt data/lang_test_adapt\n",
|
||||
"\n",
|
||||
"utils/mkgraph.sh --self-loop-scale 1.0 data/lang_test exp/tdnn exp/tdnn/graph\n",
|
||||
"utils/mkgraph.sh --self-loop-scale 1.0 data/lang_test_adapt exp/tdnn exp/tdnn/graph_adapt\n",
|
||||
"\n",
|
||||
"# Lookahead part goes OOM\n",
|
||||
"#utils/mkgraph_lookahead.sh \\\n",
|
||||
"# --self-loop-scale 1.0 data/lang \\\n",
|
||||
"# exp/tdnn data/en-us-mix.lm.gz exp/tdnn/lgraph\n",
|
||||
"#utils/mkgraph_lookahead.sh \\\n",
|
||||
"# --self-loop-scale 1.0 data/lang \\\n",
|
||||
"# exp/tdnn db/en-50k-0.4-android.lm.gz exp/tdnn/lgraph_orig\n",
|
||||
"+ . path.sh\n",
|
||||
"+++ pwd\n",
|
||||
"++ export KALDI_ROOT=/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../..\n",
|
||||
"++ KALDI_ROOT=/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../..\n",
|
||||
"++ export PATH=/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/utils:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/openfst/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/fstbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/gmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/featbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/lm:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/sgmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/sgmm2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/fgmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/latbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnetbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnet2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/online2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/ivectorbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/lmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/chainbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnet3bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab:/opt/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tools/node/bin:/tools/google-cloud-sdk/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/sph2pipe_v2.5\n",
|
||||
"++ PATH=/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/utils:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/openfst/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/fstbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/gmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/featbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/lm:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/sgmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/sgmm2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/fgmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/latbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnetbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnet2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/online2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/ivectorbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/lmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/chainbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnet3bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab:/opt/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tools/node/bin:/tools/google-cloud-sdk/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/sph2pipe_v2.5\n",
|
||||
"++ export PATH=/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/ngram-1.3.7/src/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/utils:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/openfst/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/fstbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/gmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/featbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/lm:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/sgmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/sgmm2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/fgmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/latbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnetbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnet2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/online2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/ivectorbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/lmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/chainbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnet3bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab:/opt/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tools/node/bin:/tools/google-cloud-sdk/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/sph2pipe_v2.5\n",
|
||||
"++ PATH=/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/ngram-1.3.7/src/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/utils:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/openfst/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/fstbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/gmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/featbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/lm:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/sgmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/sgmm2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/fgmmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/latbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnetbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnet2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/online2bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/ivectorbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/lmbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/chainbin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../src/nnet3bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab:/opt/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/tools/node/bin:/tools/google-cloud-sdk/bin:/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/sph2pipe_v2.5\n",
|
||||
"++ export LD_LIBRARY_PATH=/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/openfst/lib/fst/\n",
|
||||
"++ LD_LIBRARY_PATH=/content/kaldi/egs/ac/vosk-model-small-en-us-0.15-compile-colab/../../../tools/openfst/lib/fst/\n",
|
||||
"++ export LC_ALL=C\n",
|
||||
"++ LC_ALL=C\n",
|
||||
"+ pip3 install phonetisaurus\n",
|
||||
"Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
|
||||
"Requirement already satisfied: phonetisaurus in /usr/local/lib/python3.7/dist-packages (0.3.0)\n",
|
||||
"+ rm -rf data\n",
|
||||
"+ rm -rf exp/tdnn/lgraph\n",
|
||||
"+ rm -rf exp/tdnn/lgraph_orig\n",
|
||||
"+ mkdir -p data/dict\n",
|
||||
"+ cp db/phone/extra_questions.txt db/phone/nonsilence_phones.txt db/phone/optional_silence.txt db/phone/silence_phones.txt data/dict\n",
|
||||
"+ ./dict.py\n",
|
||||
"+ python3 ./get_vocab.py\n",
|
||||
"+ ngramsymbols data/mix.vocab data/mix.syms\n",
|
||||
"+ farcompilestrings --fst_type=compact --symbols=data/mix.syms --keep_symbols '--unknown_symbol=[unk]' db/extra.txt data/extra.far\n",
|
||||
"+ ngramcount --order=3 data/extra.far -\n",
|
||||
"+ ngrammake --method=witten_bell - data/extra.mod\n",
|
||||
"+ ngramshrink --method=count_prune --count_pattern=3+:3\n",
|
||||
"+ ngramread\n",
|
||||
"+ ngramprint --integers\n",
|
||||
"+ grep -v '<unk>'\n",
|
||||
"+ ngramread --renormalize_arpa --ARPA --symbols=data/mix.syms - data/en-us.mod\n",
|
||||
"+ gunzip -c db/en-50k-0.4-android.lm.gz\n",
|
||||
"+ ngrammerge --method=bayes_model_merge --normalize --alpha=0.95 --beta=0.05 data/en-us.mod data/extra.mod data/en-us-mix.mod\n",
|
||||
"+ ngramprint --ARPA data/en-us-mix.mod\n",
|
||||
"+ gzip -c\n",
|
||||
"+ utils/prepare_lang.sh data/dict '[unk]' data/lang_local data/lang\n",
|
||||
"utils/prepare_lang.sh data/dict [unk] data/lang_local data/lang\n",
|
||||
"Checking data/dict/silence_phones.txt ...\n",
|
||||
"--> reading data/dict/silence_phones.txt\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> data/dict/silence_phones.txt is OK\n",
|
||||
"\n",
|
||||
"Checking data/dict/optional_silence.txt ...\n",
|
||||
"--> reading data/dict/optional_silence.txt\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> data/dict/optional_silence.txt is OK\n",
|
||||
"\n",
|
||||
"Checking data/dict/nonsilence_phones.txt ...\n",
|
||||
"--> reading data/dict/nonsilence_phones.txt\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> data/dict/nonsilence_phones.txt is OK\n",
|
||||
"\n",
|
||||
"Checking disjoint: silence_phones.txt, nonsilence_phones.txt\n",
|
||||
"--> disjoint property is OK.\n",
|
||||
"\n",
|
||||
"Checking data/dict/lexicon.txt\n",
|
||||
"--> reading data/dict/lexicon.txt\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> data/dict/lexicon.txt is OK\n",
|
||||
"\n",
|
||||
"Checking data/dict/extra_questions.txt ...\n",
|
||||
"--> data/dict/extra_questions.txt is empty (this is OK)\n",
|
||||
"--> SUCCESS [validating dictionary directory data/dict]\n",
|
||||
"\n",
|
||||
"**Creating data/dict/lexiconp.txt from data/dict/lexicon.txt\n",
|
||||
"fstaddselfloops data/lang/phones/wdisambig_phones.int data/lang/phones/wdisambig_words.int \n",
|
||||
"prepare_lang.sh: validating output directory\n",
|
||||
"utils/validate_lang.pl data/lang\n",
|
||||
"Checking existence of separator file\n",
|
||||
"separator file data/lang/subword_separator.txt is empty or does not exist, deal in word case.\n",
|
||||
"Checking data/lang/phones.txt ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> data/lang/phones.txt is OK\n",
|
||||
"\n",
|
||||
"Checking words.txt: #0 ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> data/lang/words.txt is OK\n",
|
||||
"\n",
|
||||
"Checking disjoint: silence.txt, nonsilence.txt, disambig.txt ...\n",
|
||||
"--> silence.txt and nonsilence.txt are disjoint\n",
|
||||
"--> silence.txt and disambig.txt are disjoint\n",
|
||||
"--> disambig.txt and nonsilence.txt are disjoint\n",
|
||||
"--> disjoint property is OK\n",
|
||||
"\n",
|
||||
"Checking sumation: silence.txt, nonsilence.txt, disambig.txt ...\n",
|
||||
"--> found no unexplainable phones in phones.txt\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/context_indep.{txt, int, csl} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 10 entry/entries in data/lang/phones/context_indep.txt\n",
|
||||
"--> data/lang/phones/context_indep.int corresponds to data/lang/phones/context_indep.txt\n",
|
||||
"--> data/lang/phones/context_indep.csl corresponds to data/lang/phones/context_indep.txt\n",
|
||||
"--> data/lang/phones/context_indep.{txt, int, csl} are OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/nonsilence.{txt, int, csl} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 156 entry/entries in data/lang/phones/nonsilence.txt\n",
|
||||
"--> data/lang/phones/nonsilence.int corresponds to data/lang/phones/nonsilence.txt\n",
|
||||
"--> data/lang/phones/nonsilence.csl corresponds to data/lang/phones/nonsilence.txt\n",
|
||||
"--> data/lang/phones/nonsilence.{txt, int, csl} are OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/silence.{txt, int, csl} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 10 entry/entries in data/lang/phones/silence.txt\n",
|
||||
"--> data/lang/phones/silence.int corresponds to data/lang/phones/silence.txt\n",
|
||||
"--> data/lang/phones/silence.csl corresponds to data/lang/phones/silence.txt\n",
|
||||
"--> data/lang/phones/silence.{txt, int, csl} are OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/optional_silence.{txt, int, csl} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 1 entry/entries in data/lang/phones/optional_silence.txt\n",
|
||||
"--> data/lang/phones/optional_silence.int corresponds to data/lang/phones/optional_silence.txt\n",
|
||||
"--> data/lang/phones/optional_silence.csl corresponds to data/lang/phones/optional_silence.txt\n",
|
||||
"--> data/lang/phones/optional_silence.{txt, int, csl} are OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/disambig.{txt, int, csl} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 14 entry/entries in data/lang/phones/disambig.txt\n",
|
||||
"--> data/lang/phones/disambig.int corresponds to data/lang/phones/disambig.txt\n",
|
||||
"--> data/lang/phones/disambig.csl corresponds to data/lang/phones/disambig.txt\n",
|
||||
"--> data/lang/phones/disambig.{txt, int, csl} are OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/roots.{txt, int} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 41 entry/entries in data/lang/phones/roots.txt\n",
|
||||
"--> data/lang/phones/roots.int corresponds to data/lang/phones/roots.txt\n",
|
||||
"--> data/lang/phones/roots.{txt, int} are OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/sets.{txt, int} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 41 entry/entries in data/lang/phones/sets.txt\n",
|
||||
"--> data/lang/phones/sets.int corresponds to data/lang/phones/sets.txt\n",
|
||||
"--> data/lang/phones/sets.{txt, int} are OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/extra_questions.{txt, int} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 9 entry/entries in data/lang/phones/extra_questions.txt\n",
|
||||
"--> data/lang/phones/extra_questions.int corresponds to data/lang/phones/extra_questions.txt\n",
|
||||
"--> data/lang/phones/extra_questions.{txt, int} are OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/phones/word_boundary.{txt, int} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 166 entry/entries in data/lang/phones/word_boundary.txt\n",
|
||||
"--> data/lang/phones/word_boundary.int corresponds to data/lang/phones/word_boundary.txt\n",
|
||||
"--> data/lang/phones/word_boundary.{txt, int} are OK\n",
|
||||
"\n",
|
||||
"Checking optional_silence.txt ...\n",
|
||||
"--> reading data/lang/phones/optional_silence.txt\n",
|
||||
"--> data/lang/phones/optional_silence.txt is OK\n",
|
||||
"\n",
|
||||
"Checking disambiguation symbols: #0 and #1\n",
|
||||
"--> data/lang/phones/disambig.txt has \"#0\" and \"#1\"\n",
|
||||
"--> data/lang/phones/disambig.txt is OK\n",
|
||||
"\n",
|
||||
"Checking topo ...\n",
|
||||
"\n",
|
||||
"Checking word_boundary.txt: silence.txt, nonsilence.txt, disambig.txt ...\n",
|
||||
"--> data/lang/phones/word_boundary.txt doesn't include disambiguation symbols\n",
|
||||
"--> data/lang/phones/word_boundary.txt is the union of nonsilence.txt and silence.txt\n",
|
||||
"--> data/lang/phones/word_boundary.txt is OK\n",
|
||||
"\n",
|
||||
"Checking word-level disambiguation symbols...\n",
|
||||
"--> data/lang/phones/wdisambig.txt exists (newer prepare_lang.sh)\n",
|
||||
"Checking word_boundary.int and disambig.int\n",
|
||||
"--> generating a 98 word/subword sequence\n",
|
||||
"--> resulting phone sequence from L.fst corresponds to the word sequence\n",
|
||||
"--> L.fst is OK\n",
|
||||
"--> generating a 49 word/subword sequence\n",
|
||||
"--> resulting phone sequence from L_disambig.fst corresponds to the word sequence\n",
|
||||
"--> L_disambig.fst is OK\n",
|
||||
"\n",
|
||||
"Checking data/lang/oov.{txt, int} ...\n",
|
||||
"--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
|
||||
"--> text contains only allowed whitespaces\n",
|
||||
"--> 1 entry/entries in data/lang/oov.txt\n",
|
||||
"--> data/lang/oov.int corresponds to data/lang/oov.txt\n",
|
||||
"--> data/lang/oov.{txt, int} are OK\n",
|
||||
"\n",
|
||||
"--> data/lang/L.fst is olabel sorted\n",
|
||||
"--> data/lang/L_disambig.fst is olabel sorted\n",
|
||||
"--> SUCCESS [validating lang directory data/lang]\n",
|
||||
"+ utils/format_lm.sh data/lang db/en-50k-0.4-android.lm.gz data/dict/lexicon.txt data/lang_test\n",
|
||||
"Converting 'db/en-50k-0.4-android.lm.gz' to FST\n",
|
||||
"arpa2fst --disambig-symbol=#0 --read-symbol-table=data/lang_test/words.txt - data/lang_test/G.fst \n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:Read():arpa-file-parser.cc:94) Reading \\data\\ section.\n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:Read():arpa-file-parser.cc:149) Reading \\1-grams: section.\n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:Read():arpa-file-parser.cc:149) Reading \\2-grams: section.\n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:Read():arpa-file-parser.cc:149) Reading \\3-grams: section.\n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:RemoveRedundantStates():arpa-lm-compiler.cc:359) Reduced num-states from 1217362 to 185036\n",
|
||||
"fstisstochastic data/lang_test/G.fst \n",
|
||||
"0.476411 -3.03779\n",
|
||||
"Succeeded in formatting LM: 'db/en-50k-0.4-android.lm.gz'\n",
|
||||
"+ utils/format_lm.sh data/lang data/en-us-mix.lm.gz data/dict/lexicon.txt data/lang_test_adapt\n",
|
||||
"Converting 'data/en-us-mix.lm.gz' to FST\n",
|
||||
"arpa2fst --disambig-symbol=#0 --read-symbol-table=data/lang_test_adapt/words.txt - data/lang_test_adapt/G.fst \n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:Read():arpa-file-parser.cc:94) Reading \\data\\ section.\n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:Read():arpa-file-parser.cc:149) Reading \\1-grams: section.\n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:Read():arpa-file-parser.cc:149) Reading \\2-grams: section.\n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:Read():arpa-file-parser.cc:149) Reading \\3-grams: section.\n",
|
||||
"LOG (arpa2fst[5.5.1046~1-76cd5]:RemoveRedundantStates():arpa-lm-compiler.cc:359) Reduced num-states from 1217646 to 185095\n",
|
||||
"fstisstochastic data/lang_test_adapt/G.fst \n",
|
||||
"6.81902e-07 -3.03779\n",
|
||||
"Succeeded in formatting LM: 'data/en-us-mix.lm.gz'\n",
|
||||
"+ utils/mkgraph.sh --self-loop-scale 1.0 data/lang_test exp/tdnn exp/tdnn/graph\n",
|
||||
"tree-info exp/tdnn/tree \n",
|
||||
"tree-info exp/tdnn/tree \n",
|
||||
"fstdeterminizestar --use-log=true \n",
|
||||
"fsttablecompose data/lang_test/L_disambig.fst data/lang_test/G.fst \n",
|
||||
"fstminimizeencoded \n",
|
||||
"fstpushspecial \n",
|
||||
"fstisstochastic data/lang_test/tmp/LG.fst \n",
|
||||
"-0.145498 -0.146281\n",
|
||||
"[info]: LG not stochastic.\n",
|
||||
"fstcomposecontext --context-size=2 --central-position=1 --read-disambig-syms=data/lang_test/phones/disambig.int --write-disambig-syms=data/lang_test/tmp/disambig_ilabels_2_1.int data/lang_test/tmp/ilabels_2_1.905 data/lang_test/tmp/LG.fst \n",
|
||||
"fstisstochastic data/lang_test/tmp/CLG_2_1.fst \n",
|
||||
"-0.145498 -0.146281\n",
|
||||
"[info]: CLG not stochastic.\n",
|
||||
"make-h-transducer --disambig-syms-out=exp/tdnn/graph/disambig_tid.int --transition-scale=1.0 data/lang_test/tmp/ilabels_2_1 exp/tdnn/tree exp/tdnn/final.mdl \n",
|
||||
"fstrmepslocal \n",
|
||||
"fsttablecompose exp/tdnn/graph/Ha.fst data/lang_test/tmp/CLG_2_1.fst \n",
|
||||
"fstdeterminizestar --use-log=true \n",
|
||||
"fstminimizeencoded \n",
|
||||
"fstrmsymbols exp/tdnn/graph/disambig_tid.int \n",
|
||||
"fstisstochastic exp/tdnn/graph/HCLGa.fst \n",
|
||||
"-0.109817 -0.571742\n",
|
||||
"HCLGa is not stochastic\n",
|
||||
"add-self-loops --self-loop-scale=1.0 --reorder=true exp/tdnn/final.mdl exp/tdnn/graph/HCLGa.fst \n",
|
||||
"fstisstochastic exp/tdnn/graph/HCLG.fst \n",
|
||||
"1.90465e-09 -0.415046\n",
|
||||
"[info]: final HCLG is not stochastic.\n",
|
||||
"+ utils/mkgraph.sh --self-loop-scale 1.0 data/lang_test_adapt exp/tdnn exp/tdnn/graph_adapt\n",
|
||||
"tree-info exp/tdnn/tree \n",
|
||||
"tree-info exp/tdnn/tree \n",
|
||||
"fstdeterminizestar --use-log=true \n",
|
||||
"fsttablecompose data/lang_test_adapt/L_disambig.fst data/lang_test_adapt/G.fst \n",
|
||||
"fstminimizeencoded \n",
|
||||
"fstpushspecial \n",
|
||||
"fstisstochastic data/lang_test_adapt/tmp/LG.fst \n",
|
||||
"-0.148474 -0.149181\n",
|
||||
"[info]: LG not stochastic.\n",
|
||||
"fstcomposecontext --context-size=2 --central-position=1 --read-disambig-syms=data/lang_test_adapt/phones/disambig.int --write-disambig-syms=data/lang_test_adapt/tmp/disambig_ilabels_2_1.int data/lang_test_adapt/tmp/ilabels_2_1.979 data/lang_test_adapt/tmp/LG.fst \n",
|
||||
"fstisstochastic data/lang_test_adapt/tmp/CLG_2_1.fst \n",
|
||||
"-0.148474 -0.149181\n",
|
||||
"[info]: CLG not stochastic.\n",
|
||||
"make-h-transducer --disambig-syms-out=exp/tdnn/graph_adapt/disambig_tid.int --transition-scale=1.0 data/lang_test_adapt/tmp/ilabels_2_1 exp/tdnn/tree exp/tdnn/final.mdl \n",
|
||||
"fstrmepslocal \n",
|
||||
"fsttablecompose exp/tdnn/graph_adapt/Ha.fst data/lang_test_adapt/tmp/CLG_2_1.fst \n",
|
||||
"fstdeterminizestar --use-log=true \n",
|
||||
"fstminimizeencoded \n",
|
||||
"fstrmsymbols exp/tdnn/graph_adapt/disambig_tid.int \n",
|
||||
"fstisstochastic exp/tdnn/graph_adapt/HCLGa.fst \n",
|
||||
"-0.113907 -0.5857\n",
|
||||
"HCLGa is not stochastic\n",
|
||||
"add-self-loops --self-loop-scale=1.0 --reorder=true exp/tdnn/final.mdl exp/tdnn/graph_adapt/HCLGa.fst \n",
|
||||
"fstisstochastic exp/tdnn/graph_adapt/HCLG.fst \n",
|
||||
"1.90465e-09 -0.423618\n",
|
||||
"[info]: final HCLG is not stochastic.\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"!cat decode.sh\n",
|
||||
"!bash decode.sh"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "Sl3QBI1MXpc-",
|
||||
"outputId": "affac8a3-782f-4000-e31f-81bfed47a37a"
|
||||
},
|
||||
"execution_count": 10,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"#!/bin/bash\n",
|
||||
"\n",
|
||||
". path.sh\n",
|
||||
"\n",
|
||||
"steps/make_mfcc.sh --nj 10 data_test/test_small exp/make_mfcc/test mfcc\n",
|
||||
"steps/compute_cmvn_stats.sh data_test/test_small exp/make_mfcc/test mfcc\n",
|
||||
"utils/fix_data_dir.sh data_test/test_small\n",
|
||||
"\n",
|
||||
"steps/online/nnet2/extract_ivectors_online.sh --nj 4 \\\n",
|
||||
" data_test/test_small exp/extractor \\\n",
|
||||
" exp/ivectors_test\n",
|
||||
"\n",
|
||||
"steps/nnet3/decode.sh --nj 4 \\\n",
|
||||
" --acwt 1.0 --post-decode-acwt 10.0 \\\n",
|
||||
" --online-ivector-dir exp/ivectors_test \\\n",
|
||||
" exp/tdnn/graph_adapt data_test/test_small exp/tdnn/decode_test_adapt\n",
|
||||
"\n",
|
||||
"steps/nnet3/decode.sh --nj 4 \\\n",
|
||||
" --acwt 1.0 --post-decode-acwt 10.0 \\\n",
|
||||
" --online-ivector-dir exp/ivectors_test \\\n",
|
||||
" exp/tdnn/graph data_test/test_small exp/tdnn/decode_test\n",
|
||||
"\n",
|
||||
"#steps/nnet3/decode_lookahead.sh --nj 4 \\\n",
|
||||
"# --acwt 1.0 --post-decode-acwt 10.0 \\\n",
|
||||
"# --online-ivector-dir exp/ivectors_test \\\n",
|
||||
"# exp/tdnn/lgraph data_test/test_small exp/tdnn/decode_test_adapt\n",
|
||||
"#steps/nnet3/decode_lookahead.sh --nj 4 \\\n",
|
||||
"# --acwt 1.0 --post-decode-acwt 10.0 \\\n",
|
||||
"# --online-ivector-dir exp/ivectors_test \\\n",
|
||||
"# exp/tdnn/lgraph_orig data_test/test_small exp/tdnn/decode_test\n",
|
||||
"steps/make_mfcc.sh --nj 10 data_test/test_small exp/make_mfcc/test mfcc\n",
|
||||
"steps/make_mfcc.sh: moving data_test/test_small/feats.scp to data_test/test_small/.backup\n",
|
||||
"utils/validate_data_dir.sh: Successfully validated data-directory data_test/test_small\n",
|
||||
"steps/make_mfcc.sh: [info]: no segments file exists: assuming wav.scp indexed by utterance.\n",
|
||||
"steps/make_mfcc.sh: Succeeded creating MFCC features for test_small\n",
|
||||
"steps/compute_cmvn_stats.sh data_test/test_small exp/make_mfcc/test mfcc\n",
|
||||
"Succeeded creating CMVN stats for test_small\n",
|
||||
"fix_data_dir.sh: kept all 50 utterances.\n",
|
||||
"fix_data_dir.sh: old files are kept in data_test/test_small/.backup\n",
|
||||
"steps/online/nnet2/extract_ivectors_online.sh --nj 4 data_test/test_small exp/extractor exp/ivectors_test\n",
|
||||
"steps/online/nnet2/extract_ivectors_online.sh: extracting iVectors\n",
|
||||
"steps/online/nnet2/extract_ivectors_online.sh: combining iVectors across jobs\n",
|
||||
"steps/online/nnet2/extract_ivectors_online.sh: done extracting (online) iVectors to exp/ivectors_test using the extractor in exp/extractor.\n",
|
||||
"steps/nnet3/decode.sh --nj 4 --acwt 1.0 --post-decode-acwt 10.0 --online-ivector-dir exp/ivectors_test exp/tdnn/graph_adapt data_test/test_small exp/tdnn/decode_test_adapt\n",
|
||||
"steps/nnet2/check_ivectors_compatible.sh: WARNING: One of the directories do not contain iVector ID.\n",
|
||||
"steps/nnet2/check_ivectors_compatible.sh: WARNING: That means it's you who's reponsible for keeping \n",
|
||||
"steps/nnet2/check_ivectors_compatible.sh: WARNING: the directories compatible\n",
|
||||
"steps/nnet3/decode.sh: feature type is raw\n",
|
||||
"steps/diagnostic/analyze_lats.sh --cmd run.pl --iter final exp/tdnn/graph_adapt exp/tdnn/decode_test_adapt\n",
|
||||
"steps/diagnostic/analyze_lats.sh: see stats in exp/tdnn/decode_test_adapt/log/analyze_alignments.log\n",
|
||||
"Overall, lattice depth (10,50,90-percentile)=(1,1,4) and mean=2.4\n",
|
||||
"steps/diagnostic/analyze_lats.sh: see stats in exp/tdnn/decode_test_adapt/log/analyze_lattice_depth_stats.log\n",
|
||||
"score best paths\n",
|
||||
"local/score.sh --cmd run.pl data_test/test_small exp/tdnn/graph_adapt exp/tdnn/decode_test_adapt\n",
|
||||
"local/score.sh: scoring with word insertion penalty=0.0,0.5,1.0\n",
|
||||
"score confidence and timing with sclite\n",
|
||||
"Decoding done.\n",
|
||||
"steps/nnet3/decode.sh --nj 4 --acwt 1.0 --post-decode-acwt 10.0 --online-ivector-dir exp/ivectors_test exp/tdnn/graph data_test/test_small exp/tdnn/decode_test\n",
|
||||
"steps/nnet2/check_ivectors_compatible.sh: WARNING: One of the directories do not contain iVector ID.\n",
|
||||
"steps/nnet2/check_ivectors_compatible.sh: WARNING: That means it's you who's reponsible for keeping \n",
|
||||
"steps/nnet2/check_ivectors_compatible.sh: WARNING: the directories compatible\n",
|
||||
"steps/nnet3/decode.sh: feature type is raw\n",
|
||||
"steps/diagnostic/analyze_lats.sh --cmd run.pl --iter final exp/tdnn/graph exp/tdnn/decode_test\n",
|
||||
"steps/diagnostic/analyze_lats.sh: see stats in exp/tdnn/decode_test/log/analyze_alignments.log\n",
|
||||
"Overall, lattice depth (10,50,90-percentile)=(1,5,23) and mean=10.4\n",
|
||||
"steps/diagnostic/analyze_lats.sh: see stats in exp/tdnn/decode_test/log/analyze_lattice_depth_stats.log\n",
|
||||
"score best paths\n",
|
||||
"local/score.sh --cmd run.pl data_test/test_small exp/tdnn/graph exp/tdnn/decode_test\n",
|
||||
"local/score.sh: scoring with word insertion penalty=0.0,0.5,1.0\n",
|
||||
"score confidence and timing with sclite\n",
|
||||
"Decoding done.\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"!bash RESULTS"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "ABtcNyUDX4S8",
|
||||
"outputId": "d5e50be7-3293-4a59-94b8-9bfa46736481",
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
}
|
||||
},
|
||||
"execution_count": 11,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"%WER 11.77 [ 107 / 909, 13 ins, 7 del, 87 sub ] exp/tdnn/decode_test/wer_7_1.0\n",
|
||||
"%WER 0.22 [ 2 / 909, 0 ins, 1 del, 1 sub ] exp/tdnn/decode_test_adapt/wer_10_1.0\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
Binary file not shown.
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import wave
|
||||
import sys
|
||||
import json
|
||||
|
||||
from vosk import Model, KaldiRecognizer, SetLogLevel
|
||||
|
||||
SetLogLevel(0)
|
||||
|
||||
wf = wave.open(sys.argv[1], "rb")
|
||||
if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
|
||||
print("Audio file must be WAV format mono PCM.")
|
||||
sys.exit(1)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
rec = KaldiRecognizer(model, wf.getframerate())
|
||||
rec.SetMaxAlternatives(10)
|
||||
rec.SetWords(True)
|
||||
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
print(json.loads(rec.Result()))
|
||||
else:
|
||||
print(json.loads(rec.PartialResult()))
|
||||
|
||||
print(json.loads(rec.FinalResult()))
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
|
||||
from vosk import Model, KaldiRecognizer
|
||||
|
||||
model = Model(lang="en-us")
|
||||
rec = KaldiRecognizer(model, 8000)
|
||||
|
||||
res = json.loads(rec.FinalResult())
|
||||
print(res)
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import wave
|
||||
import sys
|
||||
|
||||
from vosk import Model, KaldiRecognizer, SetLogLevel, EndpointerMode
|
||||
|
||||
# You can set log level to -1 to disable debug messages
|
||||
SetLogLevel(0)
|
||||
|
||||
wf = wave.open(sys.argv[1], "rb")
|
||||
if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
|
||||
print("Audio file must be WAV format mono PCM.")
|
||||
sys.exit(1)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
|
||||
# You can also init model by name or with a folder path
|
||||
# model = Model(model_name="vosk-model-en-us-0.21")
|
||||
# model = Model("models/en")
|
||||
|
||||
rec = KaldiRecognizer(model, wf.getframerate())
|
||||
rec.SetWords(True)
|
||||
rec.SetPartialWords(True)
|
||||
rec.SetEndpointerMode(EndpointerMode.VERY_LONG)
|
||||
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
print(rec.Result())
|
||||
else:
|
||||
print(rec.PartialResult())
|
||||
|
||||
print(rec.FinalResult())
|
||||
|
||||
|
||||
wf = wave.open(sys.argv[1], "rb")
|
||||
if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
|
||||
print("Audio file must be WAV format mono PCM.")
|
||||
sys.exit(1)
|
||||
|
||||
rec.SetEndpointerDelays(0.5, 0.3, 10.0)
|
||||
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
print(rec.Result())
|
||||
else:
|
||||
print(rec.PartialResult())
|
||||
|
||||
print(rec.FinalResult())
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from vosk import Model, KaldiRecognizer, SetLogLevel
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
|
||||
SetLogLevel(0)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
rec = KaldiRecognizer(model, SAMPLE_RATE)
|
||||
|
||||
with subprocess.Popen(["ffmpeg", "-loglevel", "quiet", "-i",
|
||||
sys.argv[1],
|
||||
"-ar", str(SAMPLE_RATE) , "-ac", "1", "-f", "s16le", "-"],
|
||||
stdout=subprocess.PIPE) as process:
|
||||
|
||||
while True:
|
||||
data = process.stdout.read(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
print(rec.Result())
|
||||
else:
|
||||
print(rec.PartialResult())
|
||||
|
||||
print(rec.FinalResult())
|
||||
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import json
|
||||
|
||||
from vosk import BatchModel, BatchRecognizer, GpuInit
|
||||
from timeit import default_timer as timer
|
||||
|
||||
TOT_SAMPLES = 0
|
||||
|
||||
GpuInit()
|
||||
|
||||
model = BatchModel("model")
|
||||
|
||||
with open(sys.argv[1]) as fn:
|
||||
fnames = fn.readlines()
|
||||
fds = [open(x.strip(), "rb") for x in fnames]
|
||||
uids = [fname.strip().split("/")[-1][:-4] for fname in fnames]
|
||||
recs = [BatchRecognizer(model, 16000) for x in fnames]
|
||||
results = [""] * len(fnames)
|
||||
|
||||
ended = set()
|
||||
|
||||
start_time = timer()
|
||||
|
||||
while True:
|
||||
|
||||
# Feed in the data
|
||||
for i, fd in enumerate(fds):
|
||||
if i in ended:
|
||||
continue
|
||||
data = fd.read(8000)
|
||||
if len(data) == 0:
|
||||
recs[i].FinishStream()
|
||||
ended.add(i)
|
||||
continue
|
||||
recs[i].AcceptWaveform(data)
|
||||
TOT_SAMPLES += len(data)
|
||||
|
||||
# Wait for results from CUDA
|
||||
model.Wait()
|
||||
|
||||
# Retrieve and add results
|
||||
for i, fd in enumerate(fds):
|
||||
res = recs[i].Result()
|
||||
if len(res) != 0:
|
||||
results[i] = results[i] + " " + json.loads(res)["text"]
|
||||
|
||||
if len(ended) == len(fds):
|
||||
break
|
||||
|
||||
end_time = timer()
|
||||
|
||||
for i, res in enumerate(results):
|
||||
print(uids[i], res.strip())
|
||||
|
||||
print("Processed %.3f seconds of audio in %.3f seconds (%.3f xRT)"
|
||||
% (TOT_SAMPLES / 16000.0 / 2,
|
||||
end_time - start_time,
|
||||
(TOT_SAMPLES / 16000.0 / 2 / (end_time - start_time))),
|
||||
file=sys.stderr)
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import gradio as gr
|
||||
|
||||
from vosk import KaldiRecognizer, Model
|
||||
|
||||
model = Model(lang="en-us")
|
||||
|
||||
def transcribe(stream, new_chunk):
|
||||
|
||||
sample_rate, audio_data = new_chunk
|
||||
audio_data = audio_data.tobytes()
|
||||
|
||||
if stream is None:
|
||||
rec = KaldiRecognizer(model, sample_rate)
|
||||
result = []
|
||||
else:
|
||||
rec, result = stream
|
||||
|
||||
if rec.AcceptWaveform(audio_data):
|
||||
text_result = json.loads(rec.Result())["text"]
|
||||
if text_result != "":
|
||||
result.append(text_result)
|
||||
partial_result = ""
|
||||
else:
|
||||
partial_result = json.loads(rec.PartialResult())["partial"] + " "
|
||||
|
||||
return (rec, result), "\n".join(result) + "\n" + partial_result
|
||||
|
||||
gr.Interface(
|
||||
fn=transcribe,
|
||||
inputs=[
|
||||
"state", gr.Audio(sources=["microphone"], type="numpy", streaming=True),
|
||||
],
|
||||
outputs=[
|
||||
"state", "text",
|
||||
],
|
||||
live=True).launch(share=True)
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import wave
|
||||
import sys
|
||||
|
||||
from vosk import Processor
|
||||
|
||||
proc = Processor("ru_itn_tagger.fst", "ru_itn_verbalizer.fst")
|
||||
print (proc.process("у нас десять яблок"))
|
||||
print (proc.process("у нас десять яблок и десять миллилитров воды точка"))
|
||||
print (proc.process("мы пришли в восемь часов пять минут"))
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# prerequisites: as described in https://alphacephei.com/vosk/install and also python module `sounddevice` (simply run command `pip install sounddevice`)
|
||||
# Example usage using Dutch (nl) recognition model: `python test_microphone.py -m nl`
|
||||
# For more help run: `python test_microphone.py -h`
|
||||
|
||||
import argparse
|
||||
import queue
|
||||
import sys
|
||||
import sounddevice as sd
|
||||
|
||||
from vosk import Model, KaldiRecognizer
|
||||
|
||||
q = queue.Queue()
|
||||
|
||||
def int_or_str(text):
|
||||
"""Helper function for argument parsing."""
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError:
|
||||
return text
|
||||
|
||||
def callback(indata, frames, time, status):
|
||||
"""This is called (from a separate thread) for each audio block."""
|
||||
if status:
|
||||
print(status, file=sys.stderr)
|
||||
q.put(bytes(indata))
|
||||
|
||||
parser = argparse.ArgumentParser(add_help=False)
|
||||
parser.add_argument(
|
||||
"-l", "--list-devices", action="store_true",
|
||||
help="show list of audio devices and exit")
|
||||
args, remaining = parser.parse_known_args()
|
||||
if args.list_devices:
|
||||
print(sd.query_devices())
|
||||
parser.exit(0)
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
parents=[parser])
|
||||
parser.add_argument(
|
||||
"-f", "--filename", type=str, metavar="FILENAME",
|
||||
help="audio file to store recording to")
|
||||
parser.add_argument(
|
||||
"-d", "--device", type=int_or_str,
|
||||
help="input device (numeric ID or substring)")
|
||||
parser.add_argument(
|
||||
"-r", "--samplerate", type=int, help="sampling rate")
|
||||
parser.add_argument(
|
||||
"-m", "--model", type=str, help="language model; e.g. en-us, fr, nl; default is en-us")
|
||||
args = parser.parse_args(remaining)
|
||||
|
||||
try:
|
||||
if args.samplerate is None:
|
||||
device_info = sd.query_devices(args.device, "input")
|
||||
# soundfile expects an int, sounddevice provides a float:
|
||||
args.samplerate = int(device_info["default_samplerate"])
|
||||
|
||||
if args.model is None:
|
||||
model = Model(lang="en-us")
|
||||
else:
|
||||
model = Model(lang=args.model)
|
||||
|
||||
if args.filename:
|
||||
dump_fn = open(args.filename, "wb")
|
||||
else:
|
||||
dump_fn = None
|
||||
|
||||
with sd.RawInputStream(samplerate=args.samplerate, blocksize = 8000, device=args.device,
|
||||
dtype="int16", channels=1, callback=callback):
|
||||
print("#" * 80)
|
||||
print("Press Ctrl+C to stop the recording")
|
||||
print("#" * 80)
|
||||
|
||||
rec = KaldiRecognizer(model, args.samplerate)
|
||||
while True:
|
||||
data = q.get()
|
||||
if rec.AcceptWaveform(data):
|
||||
print(rec.Result())
|
||||
else:
|
||||
print(rec.PartialResult())
|
||||
if dump_fn is not None:
|
||||
dump_fn.write(data)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nDone")
|
||||
parser.exit(0)
|
||||
except Exception as e:
|
||||
parser.exit(type(e).__name__ + ": " + str(e))
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import wave
|
||||
import sys
|
||||
|
||||
from vosk import Model, KaldiRecognizer, SetLogLevel
|
||||
|
||||
SetLogLevel(0)
|
||||
|
||||
wf = wave.open(sys.argv[1], "rb")
|
||||
if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
|
||||
print("Audio file must be WAV format mono PCM.")
|
||||
sys.exit(1)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
rec = KaldiRecognizer(model, wf.getframerate())
|
||||
rec.SetMaxAlternatives(10)
|
||||
rec.SetNLSML(True)
|
||||
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
print(rec.Result())
|
||||
|
||||
print(rec.FinalResult())
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import wave
|
||||
import sys
|
||||
import json
|
||||
|
||||
from vosk import Model, KaldiRecognizer, SetLogLevel
|
||||
|
||||
SetLogLevel(0)
|
||||
|
||||
wf = wave.open(sys.argv[1], "rb")
|
||||
if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
|
||||
print("Audio file must be WAV format mono PCM.")
|
||||
sys.exit(1)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
rec = KaldiRecognizer(model, wf.getframerate())
|
||||
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
print(rec.Result())
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
jres = json.loads(rec.PartialResult())
|
||||
print(jres)
|
||||
|
||||
if jres["partial"] == "one zero zero zero":
|
||||
print("We can reset recognizer here and start over")
|
||||
rec.Reset()
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import wave
|
||||
import sys
|
||||
|
||||
from vosk import Model, KaldiRecognizer, SetLogLevel
|
||||
|
||||
# You can set log level to -1 to disable debug messages
|
||||
SetLogLevel(0)
|
||||
|
||||
wf = wave.open(sys.argv[1], "rb")
|
||||
if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
|
||||
print("Audio file must be WAV format mono PCM.")
|
||||
sys.exit(1)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
|
||||
# You can also init model by name or with a folder path
|
||||
# model = Model(model_name="vosk-model-en-us-0.21")
|
||||
# model = Model("models/en")
|
||||
|
||||
rec = KaldiRecognizer(model, wf.getframerate())
|
||||
rec.SetWords(True)
|
||||
rec.SetPartialWords(True)
|
||||
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
print(rec.Result())
|
||||
else:
|
||||
print(rec.PartialResult())
|
||||
|
||||
print(rec.FinalResult())
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import wave
|
||||
import json
|
||||
import numpy as np
|
||||
|
||||
from vosk import Model, KaldiRecognizer, SpkModel
|
||||
|
||||
SPK_MODEL_PATH = "model-spk"
|
||||
|
||||
if not os.path.exists(SPK_MODEL_PATH):
|
||||
print("Please download the speaker model from "
|
||||
"https://alphacephei.com/vosk/models and unpack as {SPK_MODEL_PATH} "
|
||||
"in the current folder.")
|
||||
sys.exit(1)
|
||||
|
||||
wf = wave.open(sys.argv[1], "rb")
|
||||
if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
|
||||
print("Audio file must be WAV format mono PCM.")
|
||||
sys.exit(1)
|
||||
|
||||
# Large vocabulary free form recognition
|
||||
model = Model(lang="en-us")
|
||||
spk_model = SpkModel(SPK_MODEL_PATH)
|
||||
#rec = KaldiRecognizer(model, wf.getframerate(), spk_model)
|
||||
rec = KaldiRecognizer(model, wf.getframerate())
|
||||
rec.SetSpkModel(spk_model)
|
||||
|
||||
# We compare speakers with cosine distance.
|
||||
# We can keep one or several fingerprints for the speaker in a database
|
||||
# to distingusih among users.
|
||||
spk_sig = [-1.110417,0.09703002,1.35658,0.7798632,-0.305457,-0.339204,0.6186931,
|
||||
-0.4521213,0.3982236,-0.004530723,0.7651616,0.6500852,-0.6664245,0.1361499,
|
||||
0.1358056,-0.2887807,-0.1280468,-0.8208137,-1.620276,-0.4628615,0.7870904,
|
||||
-0.105754,0.9739769,-0.3258137,-0.7322628,-0.6212429,-0.5531687,-0.7796484,
|
||||
0.7035915,1.056094,-0.4941756,-0.6521456,-0.2238328,-0.003737517,0.2165709,
|
||||
1.200186,-0.7737719,0.492015,1.16058,0.6135428,-0.7183084,0.3153541,0.3458071,
|
||||
-1.418189,-0.9624157,0.4168292,-1.627305,0.2742135,-0.6166027,0.1962581,
|
||||
-0.6406527,0.4372789,-0.4296024,0.4898657,-0.9531326,-0.2945702,0.7879696,
|
||||
-1.517101,-0.9344181,-0.5049928,-0.005040941,-0.4637912,0.8223695,-1.079849,
|
||||
0.8871287,-0.9732434,-0.5548235,1.879138,-1.452064,-0.1975368,1.55047,
|
||||
0.5941782,-0.52897,1.368219,0.6782904,1.202505,-0.9256122,-0.9718158,
|
||||
-0.9570228,-0.5563112,-1.19049,-1.167985,2.606804,-2.261825,0.01340385,
|
||||
0.2526799,-1.125458,-1.575991,-0.363153,0.3270262,1.485984,-1.769565,
|
||||
1.541829,0.7293826,0.1743717,-0.4759418,1.523451,-2.487134,-1.824067,
|
||||
-0.626367,0.7448186,-1.425648,0.3524166,-0.9903384,3.339342,0.4563958,
|
||||
-0.2876643,1.521635,0.9508078,-0.1398541,0.3867955,-0.7550205,0.6568405,
|
||||
0.09419366,-1.583935,1.306094,-0.3501927,0.1794427,-0.3768163,0.9683866,
|
||||
-0.2442541,-1.696921,-1.8056,-0.6803037,-1.842043,0.3069353,0.9070363,-0.486526]
|
||||
|
||||
def cosine_dist(x, y):
|
||||
nx = np.array(x)
|
||||
ny = np.array(y)
|
||||
return 1 - np.dot(nx, ny) / np.linalg.norm(nx) / np.linalg.norm(ny)
|
||||
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
res = json.loads(rec.Result())
|
||||
print("Text:", res["text"])
|
||||
if "spk" in res:
|
||||
print("X-vector:", res["spk"])
|
||||
print("Speaker distance:", cosine_dist(spk_sig, res["spk"]),
|
||||
"based on", res["spk_frames"], "frames")
|
||||
|
||||
print("Note that second distance is not very reliable because utterance is too short. "
|
||||
"Utterances longer than 4 seconds give better xvector")
|
||||
|
||||
res = json.loads(rec.FinalResult())
|
||||
print("Text:", res["text"])
|
||||
if "spk" in res:
|
||||
print("X-vector:", res["spk"])
|
||||
print("Speaker distance:", cosine_dist(spk_sig, res["spk"]),
|
||||
"based on", res["spk_frames"], "frames")
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from vosk import Model, KaldiRecognizer, SetLogLevel
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
|
||||
SetLogLevel(-1)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
rec = KaldiRecognizer(model, SAMPLE_RATE)
|
||||
rec.SetWords(True)
|
||||
|
||||
with subprocess.Popen(["ffmpeg", "-loglevel", "quiet", "-i",
|
||||
sys.argv[1],
|
||||
"-ar", str(SAMPLE_RATE) , "-ac", "1", "-f", "s16le", "-"],
|
||||
stdout=subprocess.PIPE).stdout as stream:
|
||||
|
||||
print(rec.SrtResult(stream))
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import json
|
||||
|
||||
from vosk import Model, KaldiRecognizer
|
||||
|
||||
model = Model(lang="en-us")
|
||||
|
||||
# Large vocabulary free form recognition
|
||||
rec = KaldiRecognizer(model, 16000)
|
||||
|
||||
# You can also specify the possible word list
|
||||
#rec = KaldiRecognizer(model, 16000, "zero oh one two three four five six seven eight nine")
|
||||
|
||||
with open(sys.argv[1], "rb") as wf:
|
||||
wf.read(44) # skip header
|
||||
|
||||
while True:
|
||||
data = wf.read(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
res = json.loads(rec.Result())
|
||||
print(res["text"])
|
||||
|
||||
res = json.loads(rec.FinalResult())
|
||||
print(res["text"])
|
||||
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import subprocess
|
||||
import json
|
||||
import textwrap
|
||||
|
||||
from webvtt import WebVTT, Caption
|
||||
from vosk import Model, KaldiRecognizer, SetLogLevel
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
WORDS_PER_LINE = 7
|
||||
|
||||
SetLogLevel(-1)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
rec = KaldiRecognizer(model, SAMPLE_RATE)
|
||||
rec.SetWords(True)
|
||||
|
||||
|
||||
def timestring(seconds):
|
||||
minutes = seconds / 60
|
||||
seconds = seconds % 60
|
||||
hours = int(minutes / 60)
|
||||
minutes = int(minutes % 60)
|
||||
return "%i:%02i:%06.3f" % (hours, minutes, seconds)
|
||||
|
||||
|
||||
def transcribe():
|
||||
command = ["ffmpeg", "-nostdin", "-loglevel", "quiet", "-i", sys.argv[1],
|
||||
"-ar", str(SAMPLE_RATE), "-ac", "1", "-f", "s16le", "-"]
|
||||
with subprocess.Popen(command, stdout=subprocess.PIPE) as process:
|
||||
|
||||
results = []
|
||||
while True:
|
||||
data = process.stdout.read(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
results.append(rec.Result())
|
||||
results.append(rec.FinalResult())
|
||||
|
||||
vtt = WebVTT()
|
||||
for _, res in enumerate(results):
|
||||
words = json.loads(res).get("result")
|
||||
if not words:
|
||||
continue
|
||||
|
||||
start = timestring(words[0]["start"])
|
||||
end = timestring(words[-1]["end"])
|
||||
content = " ".join([w["word"] for w in words])
|
||||
|
||||
caption = Caption(start, end, textwrap.fill(content))
|
||||
vtt.captions.append(caption)
|
||||
|
||||
# save or return webvtt
|
||||
if len(sys.argv) > 2:
|
||||
vtt.save(sys.argv[2])
|
||||
else:
|
||||
print(vtt.content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not 1 < len(sys.argv) < 4:
|
||||
print("Usage: {} audiofile [output file]".format(sys.argv[0]))
|
||||
sys.exit(1)
|
||||
transcribe()
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import wave
|
||||
import sys
|
||||
|
||||
from vosk import Model, KaldiRecognizer
|
||||
|
||||
wf = wave.open(sys.argv[1], "rb")
|
||||
if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != "NONE":
|
||||
print("Audio file must be WAV format mono PCM.")
|
||||
sys.exit(1)
|
||||
|
||||
model = Model(lang="en-us")
|
||||
|
||||
# You can also specify the possible word or phrase list as JSON list,
|
||||
# the order doesn't have to be strict
|
||||
rec = KaldiRecognizer(model,
|
||||
wf.getframerate(),
|
||||
'["oh one two three", "four five six", "seven eight nine zero", "[unk]"]')
|
||||
|
||||
while True:
|
||||
data = wf.readframes(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
print(rec.Result())
|
||||
rec.SetGrammar('["one zero one two three oh", "four five six", "seven eight nine zero", "[unk]"]')
|
||||
else:
|
||||
print(rec.PartialResult())
|
||||
|
||||
print(rec.FinalResult())
|
||||
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
import setuptools
|
||||
import shutil
|
||||
import glob
|
||||
import platform
|
||||
|
||||
# Figure out environment for cross-compile
|
||||
vosk_source = os.getenv("VOSK_SOURCE", os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||
"..")))
|
||||
system = os.environ.get('VOSK_SYSTEM', platform.system())
|
||||
architecture = os.environ.get('VOSK_ARCHITECTURE', platform.architecture()[0])
|
||||
machine = os.environ.get('VOSK_MACHINE', platform.machine())
|
||||
|
||||
# Copy precompmilled libraries
|
||||
for lib in glob.glob(os.path.join(vosk_source, "src/lib*.*")):
|
||||
print ("Adding library", lib)
|
||||
shutil.copy(lib, "vosk")
|
||||
|
||||
# Create OS-dependent, but Python-independent wheels.
|
||||
try:
|
||||
from wheel.bdist_wheel import bdist_wheel
|
||||
except ImportError:
|
||||
cmdclass = {}
|
||||
else:
|
||||
class bdist_wheel_tag_name(bdist_wheel):
|
||||
def get_tag(self):
|
||||
abi = 'none'
|
||||
if system == 'Darwin':
|
||||
oses = 'macosx_10_6_universal2'
|
||||
elif system == 'Windows' and architecture == '32bit':
|
||||
oses = 'win32'
|
||||
elif system == 'Windows' and architecture == '64bit':
|
||||
oses = 'win_amd64'
|
||||
elif system == 'Linux' and machine == 'aarch64' and architecture == '64bit':
|
||||
oses = 'manylinux2014_aarch64'
|
||||
elif system == 'Linux':
|
||||
oses = 'linux_' + machine
|
||||
else:
|
||||
raise TypeError("Unknown build environment")
|
||||
return 'py3', abi, oses
|
||||
cmdclass = {'bdist_wheel': bdist_wheel_tag_name}
|
||||
|
||||
with open("README.md", "rb") as fh:
|
||||
long_description = fh.read().decode("utf-8")
|
||||
|
||||
setuptools.setup(
|
||||
name="vosk",
|
||||
version="0.3.75",
|
||||
author="Alpha Cephei Inc",
|
||||
author_email="contact@alphacephei.com",
|
||||
description="Offline open source speech recognition API based on Kaldi and Vosk",
|
||||
long_description=long_description,
|
||||
long_description_content_type="text/markdown",
|
||||
url="https://github.com/alphacep/vosk-api",
|
||||
packages=setuptools.find_packages(),
|
||||
package_data = {'vosk': ['*.so', '*.dll', '*.dyld']},
|
||||
entry_points = {
|
||||
'console_scripts': ['vosk-transcriber=vosk.transcriber.cli:main'],
|
||||
},
|
||||
include_package_data=True,
|
||||
classifiers=[
|
||||
'Programming Language :: Python :: 3',
|
||||
'License :: OSI Approved :: Apache Software License',
|
||||
'Operating System :: Microsoft :: Windows',
|
||||
'Operating System :: POSIX :: Linux',
|
||||
'Operating System :: MacOS :: MacOS X',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules'
|
||||
],
|
||||
cmdclass=cmdclass,
|
||||
python_requires='>=3',
|
||||
zip_safe=False, # Since we load so file from the filesystem, we can not run from zip file
|
||||
setup_requires=['cffi>=1.0', 'requests', 'tqdm', 'srt', 'websockets'],
|
||||
install_requires=['cffi>=1.0', 'requests', 'tqdm', 'srt', 'websockets'],
|
||||
cffi_modules=['vosk_builder.py:ffibuilder'],
|
||||
)
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import wave
|
||||
import json
|
||||
import sys
|
||||
|
||||
from multiprocessing.dummy import Pool
|
||||
from vosk import Model, KaldiRecognizer
|
||||
|
||||
model = Model("en-us")
|
||||
|
||||
def recognize(line):
|
||||
uid, fn = line.split()
|
||||
wf = wave.open(fn, "rb")
|
||||
rec = KaldiRecognizer(model, wf.getframerate())
|
||||
|
||||
text = ""
|
||||
while True:
|
||||
data = wf.readframes(1000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if rec.AcceptWaveform(data):
|
||||
jres = json.loads(rec.Result())
|
||||
text = text + " " + jres["text"]
|
||||
jres = json.loads(rec.FinalResult())
|
||||
text = text + " " + jres["text"]
|
||||
return uid + text
|
||||
|
||||
def main():
|
||||
p = Pool(8)
|
||||
texts = p.map(recognize, open(sys.argv[1], encoding="utf-8").readlines())
|
||||
print ("\n".join(texts))
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,303 @@
|
||||
import os
|
||||
import sys
|
||||
import srt
|
||||
import datetime
|
||||
import json
|
||||
import enum
|
||||
|
||||
import requests
|
||||
from urllib.request import urlretrieve
|
||||
from zipfile import ZipFile
|
||||
from re import match
|
||||
from pathlib import Path
|
||||
from .vosk_cffi import ffi as _ffi
|
||||
from tqdm import tqdm
|
||||
|
||||
# Remote location of the models and local folders
|
||||
MODEL_PRE_URL = "https://alphacephei.com/vosk/models/"
|
||||
MODEL_LIST_URL = MODEL_PRE_URL + "model-list.json"
|
||||
MODEL_DIRS = [os.getenv("VOSK_MODEL_PATH"), Path("/usr/share/vosk"),
|
||||
Path.home() / "AppData/Local/vosk", Path.home() / ".cache/vosk"]
|
||||
|
||||
def open_dll():
|
||||
dlldir = os.path.abspath(os.path.dirname(__file__))
|
||||
if sys.platform == "win32":
|
||||
# We want to load dependencies too
|
||||
os.environ["PATH"] = dlldir + os.pathsep + os.environ["PATH"]
|
||||
if hasattr(os, "add_dll_directory"):
|
||||
os.add_dll_directory(dlldir)
|
||||
return _ffi.dlopen(os.path.join(dlldir, "libvosk.dll"))
|
||||
elif sys.platform == "linux":
|
||||
return _ffi.dlopen(os.path.join(dlldir, "libvosk.so"))
|
||||
elif sys.platform == "darwin":
|
||||
return _ffi.dlopen(os.path.join(dlldir, "libvosk.dyld"))
|
||||
else:
|
||||
raise TypeError("Unsupported platform")
|
||||
|
||||
_c = open_dll()
|
||||
|
||||
def list_models():
|
||||
response = requests.get(MODEL_LIST_URL, timeout=10)
|
||||
for model in response.json():
|
||||
print(model["name"])
|
||||
|
||||
def list_languages():
|
||||
response = requests.get(MODEL_LIST_URL, timeout=10)
|
||||
languages = {m["lang"] for m in response.json()}
|
||||
for lang in languages:
|
||||
print (lang)
|
||||
|
||||
class Model:
|
||||
def __init__(self, model_path=None, model_name=None, lang=None):
|
||||
if model_path is not None:
|
||||
self._handle = _c.vosk_model_new(model_path.encode("utf-8"))
|
||||
else:
|
||||
model_path = self.get_model_path(model_name, lang)
|
||||
self._handle = _c.vosk_model_new(model_path.encode("utf-8"))
|
||||
if self._handle == _ffi.NULL:
|
||||
raise Exception("Failed to create a model")
|
||||
|
||||
def __del__(self):
|
||||
if _c is not None:
|
||||
_c.vosk_model_free(self._handle)
|
||||
|
||||
def vosk_model_find_word(self, word):
|
||||
return _c.vosk_model_find_word(self._handle, word.encode("utf-8"))
|
||||
|
||||
def get_model_path(self, model_name, lang):
|
||||
if model_name is None:
|
||||
model_path = self.get_model_by_lang(lang)
|
||||
else:
|
||||
model_path = self.get_model_by_name(model_name)
|
||||
return str(model_path)
|
||||
|
||||
def get_model_by_name(self, model_name):
|
||||
for directory in MODEL_DIRS:
|
||||
if directory is None or not Path(directory).exists():
|
||||
continue
|
||||
model_file_list = os.listdir(directory)
|
||||
model_file = [model for model in model_file_list if model == model_name]
|
||||
if model_file != []:
|
||||
return Path(directory, model_file[0])
|
||||
response = requests.get(MODEL_LIST_URL, timeout=10)
|
||||
result_model = [model["name"] for model in response.json() if model["name"] == model_name]
|
||||
if result_model == []:
|
||||
print("model name %s does not exist" % (model_name))
|
||||
sys.exit(1)
|
||||
else:
|
||||
self.download_model(Path(directory, result_model[0]))
|
||||
return Path(directory, result_model[0])
|
||||
|
||||
def get_model_by_lang(self, lang):
|
||||
for directory in MODEL_DIRS:
|
||||
if directory is None or not Path(directory).exists():
|
||||
continue
|
||||
model_file_list = os.listdir(directory)
|
||||
model_file = [model for model in model_file_list if
|
||||
match(r"vosk-model(-small)?-{}".format(lang), model)]
|
||||
if model_file != []:
|
||||
return Path(directory, model_file[0])
|
||||
response = requests.get(MODEL_LIST_URL, timeout=10)
|
||||
result_model = [model["name"] for model in response.json() if
|
||||
model["lang"] == lang and model["type"] == "small" and model["obsolete"] == "false"]
|
||||
if result_model == []:
|
||||
print("lang %s does not exist" % (lang))
|
||||
sys.exit(1)
|
||||
else:
|
||||
self.download_model(Path(directory, result_model[0]))
|
||||
return Path(directory, result_model[0])
|
||||
|
||||
def download_model(self, model_name):
|
||||
if not (model_name.parent).exists():
|
||||
(model_name.parent).mkdir(parents=True)
|
||||
with tqdm(unit="B", unit_scale=True, unit_divisor=1024, miniters=1,
|
||||
desc=(MODEL_PRE_URL + str(model_name.name) + ".zip").rsplit("/",
|
||||
maxsplit=1)[-1]) as t:
|
||||
reporthook = self.download_progress_hook(t)
|
||||
urlretrieve(MODEL_PRE_URL + str(model_name.name) + ".zip",
|
||||
str(model_name) + ".zip", reporthook=reporthook, data=None)
|
||||
t.total = t.n
|
||||
with ZipFile(str(model_name) + ".zip", "r") as model_ref:
|
||||
model_ref.extractall(model_name.parent)
|
||||
Path(str(model_name) + ".zip").unlink()
|
||||
|
||||
def download_progress_hook(self, t):
|
||||
last_b = [0]
|
||||
def update_to(b=1, bsize=1, tsize=None):
|
||||
if tsize not in (None, -1):
|
||||
t.total = tsize
|
||||
displayed = t.update((b - last_b[0]) * bsize)
|
||||
last_b[0] = b
|
||||
return displayed
|
||||
return update_to
|
||||
|
||||
class SpkModel:
|
||||
|
||||
def __init__(self, model_path):
|
||||
self._handle = _c.vosk_spk_model_new(model_path.encode("utf-8"))
|
||||
|
||||
if self._handle == _ffi.NULL:
|
||||
raise Exception("Failed to create a speaker model")
|
||||
|
||||
def __del__(self):
|
||||
_c.vosk_spk_model_free(self._handle)
|
||||
|
||||
class EndpointerMode(enum.Enum):
|
||||
DEFAULT = 0
|
||||
SHORT = 1
|
||||
LONG = 2
|
||||
VERY_LONG = 3
|
||||
|
||||
class KaldiRecognizer:
|
||||
|
||||
def __init__(self, *args):
|
||||
if len(args) == 2:
|
||||
self._handle = _c.vosk_recognizer_new(args[0]._handle, args[1])
|
||||
elif len(args) == 3 and isinstance(args[2], SpkModel):
|
||||
self._handle = _c.vosk_recognizer_new_spk(args[0]._handle,
|
||||
args[1], args[2]._handle)
|
||||
elif len(args) == 3 and isinstance(args[2], str):
|
||||
self._handle = _c.vosk_recognizer_new_grm(args[0]._handle,
|
||||
args[1], args[2].encode("utf-8"))
|
||||
else:
|
||||
raise TypeError("Unknown arguments")
|
||||
|
||||
if self._handle == _ffi.NULL:
|
||||
raise Exception("Failed to create a recognizer")
|
||||
|
||||
def __del__(self):
|
||||
_c.vosk_recognizer_free(self._handle)
|
||||
|
||||
def SetMaxAlternatives(self, max_alternatives):
|
||||
_c.vosk_recognizer_set_max_alternatives(self._handle, max_alternatives)
|
||||
|
||||
def SetWords(self, enable_words):
|
||||
_c.vosk_recognizer_set_words(self._handle, 1 if enable_words else 0)
|
||||
|
||||
def SetPartialWords(self, enable_partial_words):
|
||||
_c.vosk_recognizer_set_partial_words(self._handle, 1 if enable_partial_words else 0)
|
||||
|
||||
def SetNLSML(self, enable_nlsml):
|
||||
_c.vosk_recognizer_set_nlsml(self._handle, 1 if enable_nlsml else 0)
|
||||
|
||||
def SetEndpointerMode(self, mode):
|
||||
_c.vosk_recognizer_set_endpointer_mode(self._handle, mode.value)
|
||||
|
||||
def SetEndpointerDelays(self, t_start_max, t_end, t_max):
|
||||
_c.vosk_recognizer_set_endpointer_delays(self._handle, t_start_max, t_end, t_max)
|
||||
|
||||
def SetSpkModel(self, spk_model):
|
||||
_c.vosk_recognizer_set_spk_model(self._handle, spk_model._handle)
|
||||
|
||||
def SetGrammar(self, grammar):
|
||||
_c.vosk_recognizer_set_grm(self._handle, grammar.encode("utf-8"))
|
||||
|
||||
def AcceptWaveform(self, data):
|
||||
res = _c.vosk_recognizer_accept_waveform(self._handle, data, len(data))
|
||||
if res < 0:
|
||||
raise Exception("Failed to process waveform")
|
||||
return res
|
||||
|
||||
def Result(self):
|
||||
return _ffi.string(_c.vosk_recognizer_result(self._handle)).decode("utf-8")
|
||||
|
||||
def PartialResult(self):
|
||||
return _ffi.string(_c.vosk_recognizer_partial_result(self._handle)).decode("utf-8")
|
||||
|
||||
def FinalResult(self):
|
||||
return _ffi.string(_c.vosk_recognizer_final_result(self._handle)).decode("utf-8")
|
||||
|
||||
def Reset(self):
|
||||
return _c.vosk_recognizer_reset(self._handle)
|
||||
|
||||
def SrtResult(self, stream, words_per_line = 7):
|
||||
results = []
|
||||
|
||||
while True:
|
||||
data = stream.read(4000)
|
||||
if len(data) == 0:
|
||||
break
|
||||
if self.AcceptWaveform(data):
|
||||
results.append(self.Result())
|
||||
results.append(self.FinalResult())
|
||||
|
||||
subs = []
|
||||
for res in results:
|
||||
jres = json.loads(res)
|
||||
if not "result" in jres:
|
||||
continue
|
||||
words = jres["result"]
|
||||
for j in range(0, len(words), words_per_line):
|
||||
line = words[j : j + words_per_line]
|
||||
s = srt.Subtitle(index=len(subs),
|
||||
content=" ".join([l["word"] for l in line]),
|
||||
start=datetime.timedelta(seconds=line[0]["start"]),
|
||||
end=datetime.timedelta(seconds=line[-1]["end"]))
|
||||
subs.append(s)
|
||||
|
||||
return srt.compose(subs)
|
||||
|
||||
def SetLogLevel(level):
|
||||
return _c.vosk_set_log_level(level)
|
||||
|
||||
|
||||
def GpuInit():
|
||||
_c.vosk_gpu_init()
|
||||
|
||||
|
||||
def GpuThreadInit():
|
||||
_c.vosk_gpu_thread_init()
|
||||
|
||||
class BatchModel:
|
||||
|
||||
def __init__(self, model_path, *args):
|
||||
self._handle = _c.vosk_batch_model_new(model_path.encode('utf-8'))
|
||||
|
||||
if self._handle == _ffi.NULL:
|
||||
raise Exception("Failed to create a model")
|
||||
|
||||
def __del__(self):
|
||||
_c.vosk_batch_model_free(self._handle)
|
||||
|
||||
def Wait(self):
|
||||
_c.vosk_batch_model_wait(self._handle)
|
||||
|
||||
class BatchRecognizer:
|
||||
|
||||
def __init__(self, *args):
|
||||
self._handle = _c.vosk_batch_recognizer_new(args[0]._handle, args[1])
|
||||
|
||||
if self._handle == _ffi.NULL:
|
||||
raise Exception("Failed to create a recognizer")
|
||||
|
||||
def __del__(self):
|
||||
_c.vosk_batch_recognizer_free(self._handle)
|
||||
|
||||
def AcceptWaveform(self, data):
|
||||
res = _c.vosk_batch_recognizer_accept_waveform(self._handle, data, len(data))
|
||||
|
||||
def Result(self):
|
||||
ptr = _c.vosk_batch_recognizer_front_result(self._handle)
|
||||
res = _ffi.string(ptr).decode("utf-8")
|
||||
_c.vosk_batch_recognizer_pop(self._handle)
|
||||
return res
|
||||
|
||||
def FinishStream(self):
|
||||
_c.vosk_batch_recognizer_finish_stream(self._handle)
|
||||
|
||||
def GetPendingChunks(self):
|
||||
return _c.vosk_batch_recognizer_get_pending_chunks(self._handle)
|
||||
|
||||
class Processor:
|
||||
|
||||
def __init__(self, *args):
|
||||
self._handle = _c.vosk_text_processor_new(args[0].encode('utf-8'), args[1].encode('utf-8'))
|
||||
|
||||
if self._handle == _ffi.NULL:
|
||||
raise Exception("Failed to create processor")
|
||||
|
||||
def __del__(self):
|
||||
_c.vosk_text_processor_free(self._handle)
|
||||
|
||||
def process(self, text):
|
||||
return _ffi.string(_c.vosk_text_processor_itn(self._handle, text.encode('utf-8'))).decode('utf-8')
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
|
||||
from pathlib import Path
|
||||
from vosk import list_models, list_languages
|
||||
from vosk.transcriber.transcriber import Transcriber
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description = "Transcribe audio file and save result in selected format")
|
||||
parser.add_argument(
|
||||
"--model", "-m", type=str,
|
||||
help="model path")
|
||||
parser.add_argument(
|
||||
"--server", "-s", type=str,
|
||||
help="use server for recognition. For example ws://localhost:2700")
|
||||
parser.add_argument(
|
||||
"--list-models", default=False, action="store_true",
|
||||
help="list available models")
|
||||
parser.add_argument(
|
||||
"--list-languages", default=False, action="store_true",
|
||||
help="list available languages")
|
||||
parser.add_argument(
|
||||
"--model-name", "-n", type=str,
|
||||
help="select model by name")
|
||||
parser.add_argument(
|
||||
"--lang", "-l", default="en-us", type=str,
|
||||
help="select model by language")
|
||||
parser.add_argument(
|
||||
"--input", "-i", type=str,
|
||||
help="audiofile")
|
||||
parser.add_argument(
|
||||
"--output", "-o", default="", type=str,
|
||||
help="optional output filename path")
|
||||
parser.add_argument(
|
||||
"--output-type", "-t", default="txt", type=str,
|
||||
help="optional arg output data type")
|
||||
parser.add_argument(
|
||||
"--tasks", "-ts", default=10, type=int,
|
||||
help="number of parallel recognition tasks")
|
||||
parser.add_argument(
|
||||
"--log-level", default="INFO",
|
||||
help="logging level")
|
||||
|
||||
def main():
|
||||
|
||||
args = parser.parse_args()
|
||||
log_level = args.log_level.upper()
|
||||
logging.getLogger().setLevel(log_level)
|
||||
|
||||
if args.list_models is True:
|
||||
list_models()
|
||||
return
|
||||
|
||||
if args.list_languages is True:
|
||||
list_languages()
|
||||
return
|
||||
|
||||
if not args.input:
|
||||
logging.info("Please specify input file or directory")
|
||||
sys.exit(1)
|
||||
|
||||
if not Path(args.input).exists():
|
||||
logging.info("File/folder {args.input} does not exist, "\
|
||||
"please specify an existing file/directory")
|
||||
sys.exit(1)
|
||||
|
||||
transcriber = Transcriber(args)
|
||||
|
||||
if Path(args.input).is_dir():
|
||||
task_list = [(Path(args.input, fn),
|
||||
Path(args.output,
|
||||
Path(fn).stem).with_suffix("." + args.output_type)) for fn in os.listdir(args.input)]
|
||||
elif Path(args.input).is_file():
|
||||
if args.output == "":
|
||||
task_list = [(Path(args.input), args.output)]
|
||||
else:
|
||||
task_list = [(Path(args.input), Path(args.output))]
|
||||
else:
|
||||
logging.info("Wrong arguments")
|
||||
sys.exit(1)
|
||||
|
||||
transcriber.process_task_list(task_list)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,201 @@
|
||||
import json
|
||||
import logging
|
||||
import asyncio
|
||||
import websockets
|
||||
import srt
|
||||
import datetime
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
from vosk import KaldiRecognizer, Model
|
||||
from queue import Queue
|
||||
from timeit import default_timer as timer
|
||||
from multiprocessing.dummy import Pool
|
||||
|
||||
CHUNK_SIZE = 4000
|
||||
SAMPLE_RATE = 16000.0
|
||||
|
||||
class Transcriber:
|
||||
|
||||
def __init__(self, args):
|
||||
self.model = Model(model_path=args.model, model_name=args.model_name, lang=args.lang)
|
||||
self.args = args
|
||||
self.queue = Queue()
|
||||
|
||||
def recognize_stream(self, rec, stream):
|
||||
tot_samples = 0
|
||||
result = []
|
||||
|
||||
while True:
|
||||
data = stream.stdout.read(CHUNK_SIZE)
|
||||
|
||||
if len(data) == 0:
|
||||
break
|
||||
|
||||
tot_samples += len(data)
|
||||
if rec.AcceptWaveform(data):
|
||||
jres = json.loads(rec.Result())
|
||||
logging.info(jres)
|
||||
result.append(jres)
|
||||
else:
|
||||
jres = json.loads(rec.PartialResult())
|
||||
if jres["partial"] != "":
|
||||
logging.info(jres)
|
||||
|
||||
jres = json.loads(rec.FinalResult())
|
||||
result.append(jres)
|
||||
|
||||
return result, tot_samples
|
||||
|
||||
async def recognize_stream_server(self, proc):
|
||||
async with websockets.connect(self.args.server) as websocket:
|
||||
tot_samples = 0
|
||||
result = []
|
||||
|
||||
await websocket.send('{ "config" : { "sample_rate" : %f } }' % (SAMPLE_RATE))
|
||||
while True:
|
||||
data = await proc.stdout.read(CHUNK_SIZE)
|
||||
tot_samples += len(data)
|
||||
if len(data) == 0:
|
||||
break
|
||||
await websocket.send(data)
|
||||
jres = json.loads(await websocket.recv())
|
||||
logging.info(jres)
|
||||
if not "partial" in jres:
|
||||
result.append(jres)
|
||||
await websocket.send('{"eof" : 1}')
|
||||
jres = json.loads(await websocket.recv())
|
||||
logging.info(jres)
|
||||
result.append(jres)
|
||||
|
||||
return result, tot_samples
|
||||
|
||||
|
||||
def format_result(self, result, words_per_line=7):
|
||||
processed_result = ""
|
||||
if self.args.output_type == "srt":
|
||||
subs = []
|
||||
|
||||
for _, res in enumerate(result):
|
||||
if not "result" in res:
|
||||
continue
|
||||
words = res["result"]
|
||||
|
||||
for j in range(0, len(words), words_per_line):
|
||||
line = words[j : j + words_per_line]
|
||||
s = srt.Subtitle(index=len(subs),
|
||||
content = " ".join([l["word"] for l in line]),
|
||||
start=datetime.timedelta(seconds=line[0]["start"]),
|
||||
end=datetime.timedelta(seconds=line[-1]["end"]))
|
||||
subs.append(s)
|
||||
processed_result = srt.compose(subs)
|
||||
|
||||
elif self.args.output_type == "txt":
|
||||
for part in result:
|
||||
if part["text"] != "":
|
||||
processed_result += part["text"] + "\n"
|
||||
|
||||
elif self.args.output_type == "json":
|
||||
monologues = {"schemaVersion":"2.0", "monologues":[], "text":[]}
|
||||
for part in result:
|
||||
if part["text"] != "":
|
||||
monologues["text"] += [part["text"]]
|
||||
for _, res in enumerate(result):
|
||||
if not "result" in res:
|
||||
continue
|
||||
monologue = { "speaker": {"id": "unknown", "name": None}, "start": 0, "end": 0, "terms": []}
|
||||
monologue["start"] = res["result"][0]["start"]
|
||||
monologue["end"] = res["result"][-1]["end"]
|
||||
monologue["terms"] = [{"confidence": t["conf"], "start": t["start"], "end": t["end"], "text": t["word"], "type": "WORD" } for t in res["result"]]
|
||||
monologues["monologues"].append(monologue)
|
||||
processed_result = json.dumps(monologues)
|
||||
return processed_result
|
||||
|
||||
def resample_ffmpeg(self, infile):
|
||||
cmd = shlex.split("ffmpeg -nostdin -loglevel quiet "
|
||||
"-i \'{}\' -ar {} -ac 1 -f s16le -".format(str(infile), SAMPLE_RATE))
|
||||
stream = subprocess.Popen(cmd, stdout=subprocess.PIPE)
|
||||
return stream
|
||||
|
||||
async def resample_ffmpeg_async(self, infile):
|
||||
cmd = "ffmpeg -nostdin -loglevel quiet "\
|
||||
"-i \'{}\' -ar {} -ac 1 -f s16le -".format(str(infile), SAMPLE_RATE)
|
||||
return await asyncio.create_subprocess_shell(cmd, stdout=subprocess.PIPE)
|
||||
|
||||
async def server_worker(self):
|
||||
while True:
|
||||
try:
|
||||
input_file, output_file = self.queue.get_nowait()
|
||||
except Exception:
|
||||
break
|
||||
|
||||
logging.info("Recognizing {}".format(input_file))
|
||||
start_time = timer()
|
||||
proc = await self.resample_ffmpeg_async(input_file)
|
||||
result, tot_samples = await self.recognize_stream_server(proc)
|
||||
await proc.wait()
|
||||
|
||||
# Bad input, continue
|
||||
if tot_samples == 0:
|
||||
self.queue.task_done()
|
||||
continue
|
||||
|
||||
processed_result = self.format_result(result)
|
||||
if output_file != "":
|
||||
logging.info("File {} processing complete".format(output_file))
|
||||
with open(output_file, "w", encoding="utf-8") as fh:
|
||||
fh.write(processed_result)
|
||||
else:
|
||||
print(processed_result)
|
||||
|
||||
elapsed = timer() - start_time
|
||||
logging.info("Execution time: {:.3f} sec; "\
|
||||
"xRT {:.3f}".format(elapsed, float(elapsed) * (2 * SAMPLE_RATE) / tot_samples))
|
||||
self.queue.task_done()
|
||||
|
||||
def pool_worker(self, inputdata):
|
||||
logging.info("Recognizing {}".format(inputdata[0]))
|
||||
start_time = timer()
|
||||
|
||||
try:
|
||||
stream = self.resample_ffmpeg(inputdata[0])
|
||||
except FileNotFoundError as e:
|
||||
print(e, "Missing FFMPEG, please install and try again")
|
||||
return
|
||||
except Exception as e:
|
||||
logging.info(e)
|
||||
return
|
||||
|
||||
rec = KaldiRecognizer(self.model, SAMPLE_RATE)
|
||||
rec.SetWords(True)
|
||||
result, tot_samples = self.recognize_stream(rec, stream)
|
||||
if tot_samples == 0:
|
||||
return
|
||||
|
||||
processed_result = self.format_result(result)
|
||||
if inputdata[1] != "":
|
||||
logging.info("File {} processing complete".format(inputdata[1]))
|
||||
with open(inputdata[1], "w", encoding="utf-8") as fh:
|
||||
fh.write(processed_result)
|
||||
else:
|
||||
print(processed_result)
|
||||
|
||||
elapsed = timer() - start_time
|
||||
logging.info("Execution time: {:.3f} sec; "\
|
||||
"xRT {:.3f}".format(elapsed, float(elapsed) * (2 * SAMPLE_RATE) / tot_samples))
|
||||
|
||||
async def process_task_list_server(self, task_list):
|
||||
for x in task_list:
|
||||
self.queue.put(x)
|
||||
workers = [asyncio.create_task(self.server_worker()) for i in range(self.args.tasks)]
|
||||
await asyncio.gather(*workers)
|
||||
|
||||
def process_task_list_pool(self, task_list):
|
||||
with Pool() as pool:
|
||||
pool.map(self.pool_worker, task_list)
|
||||
|
||||
def process_task_list(self, task_list):
|
||||
if self.args.server is None:
|
||||
self.process_task_list_pool(task_list)
|
||||
else:
|
||||
asyncio.run(self.process_task_list_server(task_list))
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
from cffi import FFI
|
||||
|
||||
vosk_root=os.environ.get("VOSK_SOURCE", "..")
|
||||
cpp_command = "cpp " + vosk_root + "/src/vosk_api.h"
|
||||
|
||||
ffibuilder = FFI()
|
||||
ffibuilder.set_source("vosk.vosk_cffi", None)
|
||||
ffibuilder.cdef(os.popen(cpp_command).read())
|
||||
|
||||
if __name__ == '__main__':
|
||||
ffibuilder.compile(verbose=True)
|
||||
Reference in New Issue
Block a user