nvidia-nemo--speech/tests/collections/asr/inference/test_itn.py

# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest

nemo_text_processing = pytest.importorskip("nemo_text_processing", reason="Requires nemo_text_processing")

from nemo.collections.asr.inference.itn.inverse_normalizer import AlignmentPreservingInverseNormalizer


@pytest.fixture(scope="module")
def en_itn_model():
    return AlignmentPreservingInverseNormalizer(
        lang="en", input_case=AlignmentPreservingInverseNormalizer.LOWER_CASED, cache_dir=None
    )


@pytest.fixture(scope="module")
def de_itn_model():
    return AlignmentPreservingInverseNormalizer(
        lang="de", input_case=AlignmentPreservingInverseNormalizer.LOWER_CASED, cache_dir=None
    )


@pytest.fixture(scope="module")
def es_itn_model():
    return AlignmentPreservingInverseNormalizer(
        lang="es", input_case=AlignmentPreservingInverseNormalizer.LOWER_CASED, cache_dir=None
    )


class TestAlignmentPreservingInverseNormalizer:

    @pytest.mark.unit
    def test_word_alignment_cardinal_en(self, en_itn_model):
        text = "zzz minus twenty five thousand thirty seven zzz"
        iwords, owords, alignment = en_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["zzz", "minus", "twenty", "five", "thousand", "thirty", "seven", "zzz"]
        assert owords == ["zzz", "-25037", "zzz"]
        assert alignment == [([0], [0], "name"), ([1, 2, 3, 4, 5, 6], [1], "cardinal"), ([7], [2], "name")]

    @pytest.mark.unit
    def test_word_alignment_time_en(self, en_itn_model):
        text = "zzz eleven fifty five p m zzz"
        iwords, owords, alignment = en_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["zzz", "eleven", "fifty", "five", "p", "m", "zzz"]
        assert owords == ["zzz", "11:55", "p.m.", "zzz"]
        assert alignment == [([0], [0], "name"), ([1, 2, 3, 4, 5], [1, 2], "time"), ([6], [3], "name")]

    @pytest.mark.unit
    def test_word_alignment_money_en(self, en_itn_model):
        text = "zzz two hundred fifty dollars zzz"
        iwords, owords, alignment = en_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["zzz", "two", "hundred", "fifty", "dollars", "zzz"]
        assert owords == ["zzz", "$250", "zzz"]
        assert alignment == [([0], [0], "name"), ([1, 2, 3, 4], [1], "money"), ([5], [2], "name")]

    @pytest.mark.unit
    def test_word_alignment_combo_en(self, en_itn_model):
        text = "eleven twenty seven fifty seven october twenty fourth nineteen seventy"
        iwords, owords, alignment = en_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == [
            "eleven",
            "twenty",
            "seven",
            "fifty",
            "seven",
            "october",
            "twenty",
            "fourth",
            "nineteen",
            "seventy",
        ]
        assert owords == ["1120", "07:57", "october", "24", "1970"]
        assert alignment == [([0, 1], [0], "date"), ([2, 3, 4], [1], "time"), ([5, 6, 7, 8, 9], [2, 3, 4], "date")]

    @pytest.mark.unit
    def test_word_alignment_measure_en(self, en_itn_model):
        text = "it is two hundred fifty meters long"
        iwords, owords, alignment = en_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["it", "is", "two", "hundred", "fifty", "meters", "long"]
        assert owords == ["it", "is", "250", "m", "long"]
        assert alignment == [
            ([0], [0], "name"),
            ([1], [1], "name"),
            ([2, 3, 4, 5], [2, 3], "measure"),
            ([6], [4], "name"),
        ]

    @pytest.mark.unit
    def test_word_alignment_sterling_en(self, en_itn_model):
        text = "trade turnover of three million pounds sterling"
        iwords, owords, alignment = en_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["trade", "turnover", "of", "three", "million", "pounds", "sterling"]
        assert owords == ["trade", "turnover", "of", "£3", "million"]
        assert alignment == [
            ([0], [0], "name"),
            ([1], [1], "name"),
            ([2], [2], "name"),
            ([3, 4, 5, 6], [3, 4], "money"),
        ]

    @pytest.mark.unit
    def test_word_alignment_time_de(self, de_itn_model):
        text = "zzz drei uhr zwanzig zzz"
        iwords, owords, alignment = de_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["zzz", "drei", "uhr", "zwanzig", "zzz"]
        assert owords == ['zzz', '03:20', 'Uhr', 'zzz']
        assert alignment == [([0], [0], "name"), ([1, 2, 3], [1, 2], "time"), ([4], [3], "name")]

    @pytest.mark.unit
    def test_word_alignment_money_de(self, de_itn_model):
        text = "zzz zwei hundert fünfzig dollar zzz"
        iwords, owords, alignment = de_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["zzz", "zwei", "hundert", "fünfzig", "dollar", "zzz"]
        assert owords == ["zzz", "$250", "zzz"]
        assert alignment == [([0], [0], "name"), ([1, 2, 3, 4], [1], "money"), ([5], [2], "name")]

    @pytest.mark.unit
    def test_word_alignment_cardinal_de(self, de_itn_model):
        text = "zzz minus fünfundzwanzigtausendsiebenunddreißig zzz"
        iwords, owords, alignment = de_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["zzz", "minus", "fünfundzwanzigtausendsiebenunddreißig", "zzz"]
        assert owords == ["zzz", "-25037", "zzz"]
        assert alignment == [([0], [0], "name"), ([1, 2], [1], "cardinal"), ([3], [2], "name")]

    @pytest.mark.unit
    def test_word_alignment_measure_de(self, de_itn_model):
        text = "es ist zweihundertfünfzig meter lang"
        iwords, owords, alignment = de_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == ["es", "ist", "zweihundertfünfzig", "meter", "lang"]
        assert owords == ["es", "ist", "250", "m", "lang"]
        assert alignment == [([0], [0], "name"), ([1], [1], "name"), ([2, 3], [2, 3], "measure"), ([4], [4], "name")]

    @pytest.mark.unit
    def test_word_alignment_combo_es(self, es_itn_model):
        text = "un mil intereses al diez por ciento a la semana estándar derecho diez por ciento"
        iwords, owords, alignment = es_itn_model.get_word_alignment(text, sep=" ")
        assert iwords == [
            'un',
            'mil',
            'intereses',
            'al',
            'diez',
            'por',
            'ciento',
            'a',
            'la',
            'semana',
            'estándar',
            'derecho',
            'diez',
            'por',
            'ciento',
        ]
        assert owords == ['1000', 'intereses', 'al', '10', '%', 'a', 'la', 'semana', 'estándar', 'derecho', '10', '%']
        assert alignment == [
            ([0, 1], [0], 'cardinal'),
            ([2], [1], 'name'),
            ([3], [2], 'name'),
            ([4, 5, 6], [3, 4], 'measure'),
            ([7], [5], 'name'),
            ([8], [6], 'name'),
            ([9], [7], 'name'),
            ([10], [8], 'name'),
            ([11], [9], 'name'),
            ([12, 13, 14], [10, 11], 'measure'),
        ]