nvidia-nemo--speech/nemo/collections/asr/inference/utils/itn_utils.py

# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import re
from collections import OrderedDict

from nemo.collections.asr.inference.utils.constants import DEFAULT_SEMIOTIC_CLASS


# Compile regex pattern once at module level for better performance
TOKEN_PATTERN = re.compile(r'tokens \{.*?(?=tokens \{|$)', re.DOTALL)


def get_semiotic_class(tokens: list[OrderedDict]) -> str:
    """
    Returns the semiotic class of the given tokens.
    """
    return list(tokens[0]["tokens"].keys())[0]


def split_text(text: str, sep: str = " ") -> tuple[list, int]:
    """
    Splits the text into words based on the separator.
    Args:
        text: (str) input text
        sep: (str) separator to split the text
    Returns:
        words: (list) list of words
        n_words: (int) number of words
    """
    words = [w for w in text.split(sep) if w]
    return words, len(words)


def find_tokens(text: str) -> list[str]:
    """
    Find the start and end positions of token blocks in the given text.
    Args:
        text: (str) input text containing token blocks
    Returns:
        token_blocks: (list[str]) list of token blocks
    """

    # Use compiled regex to find all token blocks in a single pass
    token_blocks = TOKEN_PATTERN.findall(text)

    # Strip whitespace from each block
    return [block.strip() for block in token_blocks]


def get_trivial_alignment(N: int, i_shift: int = 0, o_shift: int = 0) -> list[tuple]:
    """
    Returns a trivial word alignment for N input words.
    Args:
        N: (int) number of input words
        i_shift: (int) input shift
        o_shift: (int) output shift
    Returns:
        (list) Returns a trivial word alignment
    """
    return [([i + i_shift], [i + o_shift], DEFAULT_SEMIOTIC_CLASS) for i in range(N)]


def fallback_to_trivial_alignment(
    input_words: list[str], i_shift: int = 0, o_shift: int = 0
) -> tuple[list[str], list[str], list[tuple]]:
    """
    Returns a trivial word alignment for the input words.
    Args:
        input_words: (list[str]) list of input words
        i_shift: (int) input shift
        o_shift: (int) output shift
    Returns:
        (tuple) Returns a tuple of input words, output words, and a trivial word alignment
    """
    return input_words, input_words.copy(), get_trivial_alignment(N=len(input_words), i_shift=i_shift, o_shift=o_shift)