chore: import upstream snapshot with attribution

scripts/asr_eou/add_eob_labels.py

@@ -0,0 +1,222 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A script to add EOB labels to a manifest file.
+
+Example usage:
+
+```bash
+python add_eob_labels.py /path/to/manifest/dir
+```
+where output will be saved in the same directory with `-eob` suffix added to the filename.
+"""
+
+import argparse
+import json
+from pathlib import Path
+from string import punctuation
+
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description="Add `is_backchannel` labels to manifest files.")
+parser.add_argument(
+    "input_manifest",
+    type=str,
+    help="Path to the input manifest file to be cleaned.",
+)
+parser.add_argument(
+    "-o",
+    "--output",
+    type=str,
+    default=None,
+    help="Path to the output manifest file after cleaning.",
+)
+parser.add_argument(
+    "-p",
+    "--pattern",
+    type=str,
+    default="*.json",
+    help="Pattern to match files in the input directory.",
+)
+
+
+def read_manifest(manifest_path):
+    manifest = []
+    with open(manifest_path, 'r') as f:
+        for line in f.readlines():
+            line = line.strip()
+            if line:
+                manifest.append(json.loads(line))
+    return manifest
+
+
+def write_manifest(manifest_path, manifest):
+    with open(manifest_path, 'w') as f:
+        for item in manifest:
+            f.write(json.dumps(item) + '\n')
+
+
+def clean_text(text):
+    text = text.translate(str.maketrans('', '', punctuation)).lower().strip()
+    valid_chars = "abcdefghijklmnopqrstuvwxyz'"
+    text = ''.join([c for c in text if c in valid_chars or c.isspace() or c == "'"])
+    return " ".join(text.split()).strip()
+
+
+backchannel_phrases = [
+    'absolutely',
+    'ah',
+    'all right',
+    'alright',
+    'but yeah',
+    'definitely',
+    'exactly',
+    'go ahead',
+    'good',
+    'great',
+    'great thanks',
+    'ha ha',
+    'hi',
+    'i know',
+    'i know right',
+    'i see',
+    'indeed',
+    'interesting',
+    'mhmm',
+    'mhmm mhmm',
+    'mhmm right',
+    'mhmm yeah',
+    'mhmm yes',
+    'nice',
+    'of course',
+    'oh',
+    'oh dear',
+    'oh man',
+    'oh okay',
+    'oh wow',
+    'oh yes',
+    'ok',
+    'ok thanks',
+    'okay',
+    'okay okay',
+    'okay thanks',
+    'perfect',
+    'really',
+    'right',
+    'right exactly',
+    'right right',
+    'right yeah',
+    'so yeah',
+    'sounds good',
+    'sure',
+    'thank you',
+    'thanks',
+    "that's awesome",
+    'thats right',
+    'thats true',
+    'true',
+    'uh-huh',
+    'uh-huh yeah',
+    'uhhuh',
+    'um-humm',
+    'well',
+    'what',
+    'wow',
+    'yeah',
+    'yeah i know',
+    'yeah i see',
+    'yeah mhmm',
+    'yeah okay',
+    'yeah right',
+    'yeah uh-huh',
+    'yeah yeah',
+    'yep',
+    'yes',
+    'yes please',
+    'yes yes',
+    'you know',
+    "you're right",
+]
+
+backchannel_phrases_nopc = [clean_text(phrase) for phrase in backchannel_phrases]
+
+
+def check_if_backchannel(text):
+    """
+    Check if the text is a backchannel phrase.
+    """
+    # Remove punctuation and convert to lowercase
+    text = clean_text(text)
+    # Check if the text is in the list of backchannel phrases
+    return text in backchannel_phrases_nopc
+
+
+def add_eob_labels(manifest_path):
+    """
+    Add EOB labels to a manifest file.
+    Args:
+        manifest_path: Path to the manifest file.
+
+    Returns:
+        manifest: List of dictionaries with the EOB label added.
+        num_eob: Number of EOB labels added.
+    """
+    num_eob = 0
+    manifest = read_manifest(manifest_path)
+    for i, item in enumerate(manifest):
+        text = item['text']
+        # Check if the text is a backchannel phrase
+        is_backchannel = check_if_backchannel(text)
+        # Add the EOB label to the text
+        if is_backchannel:
+            item['is_backchannel'] = True
+            num_eob += 1
+        else:
+            item['is_backchannel'] = False
+        manifest[i] = item
+    return manifest, num_eob
+
+
+def main():
+    args = parser.parse_args()
+    input_manifest = Path(args.input_manifest)
+
+    if input_manifest.is_dir():
+        manifest_list = list(input_manifest.glob(args.pattern))
+        if not manifest_list:
+            raise ValueError(f"No files found in {input_manifest} matching pattern `{args.pattern}`")
+    else:
+        manifest_list = [input_manifest]
+
+    if args.output is None:
+        output_dir = input_manifest if input_manifest.is_dir() else input_manifest.parent
+    else:
+        output_dir = Path(args.output)
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+    total_num_eob = 0
+    print(f"Processing {len(manifest_list)} manifest files...")
+    for manifest_path in tqdm(manifest_list, total=len(manifest_list)):
+        output_file = output_dir / f"{manifest_path.stem}-eob.json"
+        new_manifest, num_eob = add_eob_labels(manifest_path)
+        total_num_eob += num_eob
+        write_manifest(output_file, new_manifest)
+        print(f"Processed {manifest_path} and saved to {output_file}. Number of EOB labels added: {num_eob}")
+
+    print(f"Total number of EOB labels added: {total_num_eob}")
+
+
+if __name__ == "__main__":
+    main()

scripts/asr_eou/clean_manifest.py

@@ -0,0 +1,670 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A rule-based text cleaning script for preparing text for ASR-EOU model training.
+
+Example usage:
+
+```bash
+python clean_manifest.py \
+    /path/to/manifest/dir \
+    -o /path/to/output/dir
+```
+
+"""
+
+import argparse
+import re
+import unicodedata
+from datetime import datetime
+from pathlib import Path
+from string import punctuation
+
+from num2words import num2words
+from whisper_normalizer.english import EnglishTextNormalizer
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+
+punctuations = punctuation.replace("'", "")
+
+text_normalizer = EnglishTextNormalizer()
+
+parser = argparse.ArgumentParser(description="Clean manifest file")
+parser.add_argument(
+    "input_manifest",
+    type=str,
+    help="Path to the input manifest file to be cleaned.",
+)
+parser.add_argument(
+    "-o",
+    "--output",
+    type=str,
+    default=None,
+    help="Path to the output manifest file after cleaning.",
+)
+parser.add_argument(
+    "-lower",
+    "--lowercase",
+    type=bool,
+    default=False,
+    help="Whether to convert the text to lowercase.",
+)
+parser.add_argument(
+    "-drop",
+    "--remove_punc",
+    type=bool,
+    default=False,
+    help="Whether to remove punctuation from the text.",
+)
+parser.add_argument(
+    "--normalize",
+    type=bool,
+    default=False,
+    help="Whether to normalize the text using Whisper text normalizer.",
+)
+parser.add_argument(
+    "-n2w",
+    "--replace_numbers",
+    type=bool,
+    default=True,
+    help="Whether to replace numbers with words.",
+)
+parser.add_argument(
+    "-p",
+    "--pattern",
+    type=str,
+    default="**/*.json",
+    help="Pattern to match files in the input directory.",
+)
+parser.add_argument(
+    "-t",
+    "--text_field",
+    type=str,
+    default="text",
+    help="Field in the manifest to clean. Default is 'text'.",
+)
+parser.add_argument(
+    "--auto_pc",
+    action="store_true",
+    help="If set, will add auto capitalization and punctuation at the end of the text.",
+)
+parser.add_argument(
+    "--format",
+    default="asr",
+    choices=["asr", "conv"],
+    help="Format of the manifest. Default is 'asr'.",
+)
+parser.add_argument(
+    "--keep_name",
+    action="store_true",
+    help="If set, will keep the original name of the manifest file.",
+)
+
+# Spoken representations
+
+MONTHS = [
+    "",
+    "January",
+    "February",
+    "March",
+    "April",
+    "May",
+    "June",
+    "July",
+    "August",
+    "September",
+    "October",
+    "November",
+    "December",
+]
+
+ORDINALS = {
+    1: "first",
+    2: "second",
+    3: "third",
+    4: "fourth",
+    5: "fifth",
+    6: "sixth",
+    7: "seventh",
+    8: "eighth",
+    9: "ninth",
+    10: "tenth",
+    11: "eleventh",
+    12: "twelfth",
+    13: "thirteenth",
+    14: "fourteenth",
+    15: "fifteenth",
+    16: "sixteenth",
+    17: "seventeenth",
+    18: "eighteenth",
+    19: "nineteenth",
+    20: "twentieth",
+    21: "twenty first",
+    22: "twenty second",
+    23: "twenty third",
+    24: "twenty fourth",
+    25: "twenty fifth",
+    26: "twenty sixth",
+    27: "twenty seventh",
+    28: "twenty eighth",
+    29: "twenty ninth",
+    30: "thirtieth",
+    31: "thirty first",
+}
+
+
+def speak_year(year: int) -> str:
+    if 2000 <= year <= 2099:
+        return f"twenty {speak_number(year % 100)}"
+    elif 1900 <= year <= 1999:
+        return f"nineteen {speak_number(year % 100)}"
+    else:
+        return str(year)
+
+
+def speak_number(n: int) -> str:
+    num_words = {
+        0: "zero",
+        1: "one",
+        2: "two",
+        3: "three",
+        4: "four",
+        5: "five",
+        6: "six",
+        7: "seven",
+        8: "eight",
+        9: "nine",
+        10: "ten",
+        11: "eleven",
+        12: "twelve",
+        13: "thirteen",
+        14: "fourteen",
+        15: "fifteen",
+        16: "sixteen",
+        17: "seventeen",
+        18: "eighteen",
+        19: "nineteen",
+        20: "twenty",
+        30: "thirty",
+        40: "forty",
+        50: "fifty",
+        60: "sixty",
+        70: "seventy",
+        80: "eighty",
+        90: "ninety",
+    }
+    if n <= 20:
+        return num_words[n]
+    elif n < 100:
+        tens, ones = divmod(n, 10)
+        return f"{num_words[tens * 10]} {num_words[ones]}" if ones else num_words[tens * 10]
+    else:
+        return str(n)
+
+
+def _parse_numeric_date(date_str: str, *, dayfirst: bool):
+    separator = "/" if "/" in date_str else "-"
+    parts = date_str.split(separator)
+
+    if len(parts) == 3 and len(parts[0]) == 4:
+        year, month, day = [int(part) for part in parts]
+    elif len(parts) == 3:
+        first, second, year = [int(part) for part in parts]
+        day, month = (first, second) if dayfirst else (second, first)
+    elif len(parts) == 2:
+        first, second = [int(part) for part in parts]
+        year = datetime.now().year
+        day, month = (first, second) if dayfirst else (second, first)
+    else:
+        raise ValueError(date_str)
+
+    if year < 100:
+        year += 2000 if year < 69 else 1900
+
+    return datetime(year, month, day)
+
+
+def parse_with_auto_dayfirst(date_str: str):
+    try:
+        parsed_us = _parse_numeric_date(date_str, dayfirst=False)
+    except Exception:
+        parsed_us = None
+
+    try:
+        parsed_eu = _parse_numeric_date(date_str, dayfirst=True)
+    except Exception:
+        parsed_eu = None
+
+    if parsed_us is None:
+        return parsed_eu
+    if parsed_eu is None:
+        return parsed_us
+
+    first = int(date_str.split("/" if "/" in date_str else "-")[0])
+    if first > 12 and len(str(first)) != 4:
+        return parsed_eu
+
+    # Default fallback assumes US style for ambiguous numeric dates.
+    return parsed_us
+
+
+def date_to_spoken_string(date_str: str) -> str:
+    parsed = parse_with_auto_dayfirst(date_str)
+    if not parsed:
+        return None
+
+    month = MONTHS[parsed.month]
+    day = ORDINALS[parsed.day]
+    spoken = f"{month} {day} {speak_year(parsed.year)}"
+
+    return spoken
+
+
+def replace_dates_in_text(text: str) -> str:
+    # Regex pattern to match common date formats like:
+    # 5/22, 05/22/2025, 22/05/2025, 2025-05-22
+    date_pattern = r'\b(?:\d{1,4}[-/])?\d{1,2}[-/]\d{1,4}\b'
+
+    def replace_match(match):
+        date_str = match.group(0)
+        spoken = date_to_spoken_string(date_str)
+        return spoken if spoken else date_str
+
+    return re.sub(date_pattern, replace_match, text)
+
+
+def convert_to_spoken(text: str) -> str:
+
+    text = replace_dates_in_text(text)  # Convert dates to spoken form
+
+    # Mapping of metric units to spoken forms
+    unit_map = {
+        "kg": "kilograms",
+        "g": "grams",
+        "mg": "milligrams",
+        "l": "liters",
+        "ml": "milliliters",
+        "cm": "centimeters",
+        "mm": "millimeters",
+        "m": "meters",
+        "km": "kilometers",
+        "°c": "degrees celsius",
+        "°f": "degrees fahrenheit",
+        "oz": "ounces",
+        "lb": "pounds",
+        "lbs": "pounds",
+    }
+
+    # Replace metric units like "12kg" or "5 ml"
+    def replace_metric(match):
+        number = match.group(1)
+        unit = match.group(2).lower()
+        spoken_unit = unit_map.get(unit, unit)
+        return f"{number} {spoken_unit}"
+
+    # Replace time like "5am" or "6PM"
+    def replace_ampm(match):
+        hour = match.group(1)
+        meridiem = match.group(2).lower()
+        return f"{hour} {'a m' if meridiem == 'am' else 'p m'}"
+
+    # Replace time like "1:30pm"
+    def replace_colon_time(match):
+        hour = match.group(1)
+        minute = match.group(2)
+        meridiem = match.group(3).lower()
+        return f"{hour} {minute} {'a m' if meridiem == 'am' else 'p m'}"
+
+    # Convert feet and inches like 5'11" to "5 feet 11 inches"
+    def replace_feet_inches(match):
+        feet = match.group(1)
+        inches = match.group(2)
+        return f"{feet} feet {inches} inches"
+
+    # Convert just feet (e.g., 6') to "6 feet"
+    def replace_feet_only(match):
+        feet = match.group(1)
+        return f"{feet} feet"
+
+    # Convert just inches (e.g., 10") to "10 inches"
+    def replace_inches_only(match):
+        inches = match.group(1)
+        return f"{inches} inches"
+
+    # Apply replacements
+    # First: time with colon (e.g., 1:30pm)
+    text = re.sub(r'\b(\d{1,2}):(\d{2})(am|pm)\b', replace_colon_time, text, flags=re.IGNORECASE)
+
+    # Then: basic am/pm (e.g., 5am)
+    text = re.sub(r'\b(\d{1,2})(am|pm)\b', replace_ampm, text, flags=re.IGNORECASE)
+
+    # Then: replace 1st, 2nd, 3rd, etc
+    text = text.replace("1st", "first")
+    text = text.replace("2nd", "second")
+    text = text.replace("3rd", "third")
+    text = text.replace("@", " at ")
+
+    # Finally: metric units
+    text = re.sub(
+        r'\b(\d+(?:\.\d+)?)\s?(kg|g|mg|l|ml|cm|mm|m|km|°c|°f|oz|lbs?|LB|LBS?)\b',
+        replace_metric,
+        text,
+        flags=re.IGNORECASE,
+    )
+    text = re.sub(r'\b(\d+)\'(\d+)"', replace_feet_inches, text)  # e.g., 5'11"
+    text = re.sub(r'\b(\d+)\'', replace_feet_only, text)  # e.g., 6'
+    text = re.sub(r'(\d+)"', replace_inches_only, text)  # e.g., 10"
+
+    return text
+
+
+def replace_numbers_with_words(text):
+    def convert_number(match):
+        num_str = match.group()
+        original = num_str
+
+        # Remove dollar sign
+        is_dollar = False
+        if num_str.startswith('$'):
+            is_dollar = True
+            num_str = num_str[1:]
+        elif num_str.endswith('$'):
+            is_dollar = True
+            num_str = num_str[:-1]
+
+        # Remove commas
+        num_str = num_str.replace(',', '')
+
+        try:
+            if '.' in num_str:
+                # Convert decimal number
+                integer_part, decimal_part = num_str.split('.')
+                words = num2words(int(integer_part)) + ' point ' + ' '.join(num2words(int(d)) for d in decimal_part)
+            else:
+                words = num2words(int(num_str))
+            if is_dollar:
+                words += ' dollars'
+            return words + " "
+        except Exception:
+            return original  # Return original if conversion fails
+
+    # Pattern matches: $3,000 or 3,000.45 or 1234
+    pattern = re.compile(r'\$?\d{1,3}(?:,\d{3})*(?:\.\d+)?|\$?\d+(?:\.\d+)?')
+    result = pattern.sub(convert_number, text)
+    result = result.replace("$", " dollars ")  # Handle dollar sign separately
+
+    def merge_th(text: str) -> str:
+        # merge th with the preceding digit
+        candidates = ["four th ", "five th ", "six th ", "seven th ", "eight th ", "nine th "]
+        for key in candidates:
+            if key in text:
+                if "five" in key:
+                    target = "fifth "
+                else:
+                    target = f"{key.split(' ')[0]}th "
+                text = text.replace(key, target)
+            elif text.endswith(key.strip()):
+                if "five" in key:
+                    target = "fifth"
+                else:
+                    target = f"{key.split(' ')[0]}th"
+                text = text.replace(key.strip(), target)
+        return text
+
+    result = merge_th(result)
+    result = " ".join(result.split())  # Remove extra spaces
+    return result
+
+
+def unicode_to_ascii(text: str) -> str:
+    """
+    Converts text with accented or special Latin characters (e.g., ó, ñ, ū, ō)
+    into their closest ASCII equivalents.
+    """
+    # Normalize the string to NFKD to separate base characters from diacritics
+    normalized = unicodedata.normalize('NFKD', text)
+
+    # Encode to ASCII bytes, ignoring characters that can't be converted
+    ascii_bytes = normalized.encode('ascii', 'ignore')
+
+    # Decode back to string
+    ascii_text = ascii_bytes.decode('ascii')
+
+    return ascii_text
+
+
+def drop_punctuations(text: str) -> str:
+    """
+    Clean the text by removing invalid characters and converting to lowercase.
+
+    :param text: Input text.
+    :return: Cleaned text.
+    """
+    valid_chars = "abcdefghijklmnopqrstuvwxyz'"
+    text = text.lower()
+    text = unicode_to_ascii(text)
+    text = text.replace(":", " ")
+    text = text.replace("-", " ")
+    text = ''.join([c for c in text if c in valid_chars or c.isspace()])
+    text = ' '.join(text.split()).strip()
+    return text
+
+
+def clean_label(_str: str) -> str:
+    """
+    Remove unauthorized characters in a string, lower it and remove unneeded spaces
+    """
+    # replace_with_space = [char for char in '/?*\",.:=?_{|}~¨«·»¡¿„…‧‹›≪≫!:;ː→']
+    replace_with_blank = [char for char in '`¨´‘’“”`ʻ‘’“"‘”']
+    replace_with_apos = [char for char in '‘’ʻ‘’‘'] + ["\u2019"]
+    _str = _str.strip()
+    for i in replace_with_blank:
+        _str = _str.replace(i, "")
+    for i in replace_with_apos:
+        _str = _str.replace(i, "'")
+
+    text = _str
+    text = text.replace("\u2103", "celsius")
+    text = text.replace("\u2109", "fahrenheit")
+    text = text.replace("\u00b0", "degrees")
+    text = text.replace("\u2019", "'")
+    text = text.replace("\\", ".")
+    text = text.replace("\n", " ")
+    text = text.replace("\r", " ")
+    text = text.replace("\t", " ")
+
+    ret = " ".join(text.split())
+    return ret
+
+
+def ends_with_punctuation(s: str) -> bool:
+    # Strip trailing whitespace
+    s = s.rstrip()
+
+    # consider this set to be punctuation that's acceptable to end a sentence with
+    puncturation_chars = [",", ".", ":", ";", "?", "!", "-", "—", "–", "…"]
+
+    # If string is empty after stripping, return False
+    if not s:
+        return False
+
+    # Get the last character
+    last_char = s[-1]
+
+    # Return True if the last character is punctuation, otherwise False
+    return last_char in puncturation_chars
+
+
+def add_period_if_needed(text: str) -> str:
+    """
+    Add a period at the end of the text if it does not already end with one.
+    """
+    if not ends_with_punctuation(text):
+        text += "."
+    return text.strip()
+
+
+def capitalize_self_i(text):
+    # Replace standalone lowercase "i" with "I"
+    # Handles "i", "i.", "i?", "i'll", "i'm", etc.
+    return re.sub(r'\b(i)(?=[\s.,!?;:\'\"-]|$)', r'I', text)
+
+
+def add_space_after_punctuation(text):
+    # Add a space after punctuation if it's not already followed by one or by the end of the string
+    return re.sub(r'([,\.?;:])(?=\S)', r'\1 ', text)
+
+
+def add_auto_capitalization(text):
+    if text.lower() != text:
+        # If the text is not all lowercase, we assume it has some capitalization
+        return text
+
+    # Remove space before punctuation (.,!?;:)
+    text = re.sub(r'\s+([.,!?;:])', r'\1', text)
+
+    # Capitalize the first letter of each sentence
+    def capitalize_sentences(match):
+        return match.group(1) + match.group(2).upper()
+
+    # Ensure first character is capitalized
+    text = text.strip()
+    if text:
+        text = text[0].upper() + text[1:]
+
+    text = capitalize_self_i(text)
+    text = add_space_after_punctuation(text)
+    # Capitalize after sentence-ending punctuation followed by space(s)
+    text = re.sub(r'([.!?]\s+)([a-z])', capitalize_sentences, text)
+    return text
+
+
+def unicode_to_ascii(text: str) -> str:
+    """
+    Converts text with accented or special Latin characters (e.g., ó, ñ, ū, ō)
+    into their closest ASCII equivalents.
+    """
+    # Normalize the string to NFKD to separate base characters from diacritics
+    normalized = unicodedata.normalize('NFKD', text)
+
+    # Encode to ASCII bytes, ignoring characters that can't be converted
+    ascii_bytes = normalized.encode('ascii', 'ignore')
+
+    # Decode back to string
+    ascii_text = ascii_bytes.decode('ascii')
+
+    return ascii_text
+
+
+def clean_text(text: str, args) -> str:
+    """
+    Clean the text based on the provided arguments.
+    """
+    text = unicode_to_ascii(text)
+    if args.normalize:
+        text = text_normalizer(text)
+    if args.replace_numbers:
+        text = convert_to_spoken(text)
+        text = replace_numbers_with_words(text)
+    if args.lowercase:
+        text = text.lower()
+    if args.remove_punc:
+        text = text.replace("-", " ")
+        text = text.replace("_", " ")
+        text = text.translate(str.maketrans("", "", punctuations))
+        text = drop_punctuations(text)
+    if args.auto_pc:
+        text = add_auto_capitalization(text)
+    return clean_label(text)
+
+
+def clean_asr_manifest(manifest, text_field, args):
+    for i, item in enumerate(manifest):
+        text = str(item[text_field])
+        manifest[i][f"origin_{text_field}"] = text
+        manifest[i][text_field] = clean_text(text, args)
+    return manifest
+
+
+def clean_conv_manifest(manifest, text_field, args):
+    new_manifest = []
+    for i, item in enumerate(manifest):
+        conversations = []
+        for turn in item["conversations"]:
+            conversations.append(
+                {
+                    "role": turn["role"],
+                    "value": clean_text(turn["value"], args),
+                    "type": turn.get("type", "text"),
+                }
+            )
+        item["conversations"] = conversations
+        new_manifest.append(item)
+    return manifest
+
+
+def main(args):
+    text_field = args.text_field
+    manifest_files = Path(args.input_manifest)
+    if manifest_files.is_dir():
+        manifest_files = list(manifest_files.glob(args.pattern))
+    elif manifest_files.is_file():
+        manifest_files = [manifest_files]
+    else:
+        raise ValueError(f"Invalid input manifest path: {args.input_manifest}")
+
+    for manifest_file in manifest_files:
+        print(f"Processing manifest file: {manifest_file}")
+        postfix = "-cleaned"
+        postfix += "_norm" if args.normalize else ""
+        postfix += "_n2w" if args.replace_numbers else ""
+        if args.lowercase and args.remove_punc:
+            postfix += "_noPC"
+        else:
+            postfix += "_lc" if args.lowercase else ""
+            postfix += "_np" if args.remove_punc else ""
+        postfix += "_aPC" if args.auto_pc else ""
+
+        output_manifest = manifest_file.with_name(f"{manifest_file.stem}{postfix}{manifest_file.suffix}")
+
+        if args.output:
+            if args.output.endswith(".json"):
+                if len(manifest_files) > 1:
+                    raise ValueError("Output path must be a directory when processing multiple manifest files.")
+                output_manifest = Path(args.output)
+            else:
+                output_dir = Path(args.output)
+                output_dir.mkdir(parents=True, exist_ok=True)
+                if args.keep_name:
+                    output_manifest = output_dir / manifest_file.name
+                else:
+                    output_manifest = output_dir / output_manifest.name
+
+        manifest = read_manifest(str(manifest_file))
+
+        if args.format == "asr":
+            manifest = clean_asr_manifest(manifest, text_field, args)
+        elif args.format == "conv":
+            manifest = clean_conv_manifest(manifest, text_field, args)
+        else:
+            raise ValueError(f"Unsupported manifest format: {args.format}")
+
+        write_manifest(str(output_manifest), manifest)
+        print(f"Cleaned manifest saved to {output_manifest}")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    main(args)

scripts/asr_eou/conf/data.yaml

@@ -0,0 +1,46 @@
+
+output_dir: ???
+
+data:
+  pattern: "*.json"
+  manifest_filepath: ???
+  tarred_audio_filepaths: null
+  sample_rate: 16000
+  max_duration: 30 # you may need to update it for your dataset
+  min_duration: 0.1
+  batch_duration: 300  # you may disable batch_duration by setting it to `null`
+  batch_size: null 
+  shuffle: false
+  seed: 42
+  num_workers: 8
+  pin_memory: true
+  quadratic_duration: 30
+  num_buckets: 30
+  num_cuts_for_bins_estimate: 10000
+  bucket_buffer_size: 10000
+  shuffle_buffer_size: 10000
+
+  random_padding:
+      prob: 1.0
+      min_pad_duration: 0.0  # minimum duration of pre/post padding in seconds
+      max_pad_duration: 3.0  # maximum duration of pre/post padding in seconds
+      max_total_duration: 40.0  # maximum total duration of the padded audio in seconds
+      pad_distribution: 'constant'  # distribution of padding duration, 'uniform' or 'normal' or 'constant'
+      pre_pad_duration: 0.2
+      post_pad_duration: 3.0
+
+  augmentor:
+    white_noise:
+      prob: 0.0
+      min_level: -90
+      max_level: -40
+    gain:
+      prob: 0.0
+      min_gain_dbfs: -10.0
+      max_gain_dbfs: 10.0
+    noise:
+      prob: 1.0
+      manifest_path: ???
+      min_snr_db: 0
+      max_snr_db: 20
+      max_gain_db: 300.0

scripts/asr_eou/eval_eou_metrics.py

@@ -0,0 +1,176 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script calculates the EOU metrics using predictions and references in SegLST format.
+
+Example usage:
+
+The PREDICTION_ROOT and REFERENCE_ROOT directories should have the following structure:
+
+<PREDICTION_ROOT>:
+->dataset1/
+    -> sample1.json
+    -> sample2.json
+->dataset2/
+    -> sample1.json
+    -> sample2.json
+
+<REFERENCE_ROOT>:
+->dataset1/
+    -> sample1.json
+    -> sample2.json
+->dataset2/
+    -> sample1.json
+    -> sample2.json
+
+
+each sample.json should contain a list of dictionaries with the following fields:
+{
+    "session_id": str,
+    "start_time": float,  # start time in seconds
+    "end_time": float,  # end time in seconds
+    "words": str,  # transcription of the utterance
+    "audio_filepath": str,  # only in prediction
+    "eou_prob": float, # only in prediction, probability of EOU in range [0.1]
+    "eou_pred": bool, # only in prediction
+    "full_text": str, # only in prediction, which is the full transcription up to the end_time
+}
+    
+```bash
+python eval_eou_metrics.py \
+    --prediction $PREDICTION_ROOT \
+    --reference $REFERENCE_ROOT  \
+    --multiple
+```
+"""
+
+
+import argparse
+import json
+from pathlib import Path
+from typing import List
+
+from nemo.collections.asr.parts.utils.eou_utils import EOUResult, aggregate_eou_metrics, evaluate_eou
+
+parser = argparse.ArgumentParser(description="Evaluate end of utterance predictions against reference labels.")
+parser.add_argument(
+    "-p",
+    "--prediction",
+    type=str,
+    required=True,
+    help="Path to the directory containing the predictions.",
+)
+parser.add_argument(
+    "-r",
+    "--reference",
+    type=str,
+    required=True,
+    help="Path to the directory containing the groundtruth.",
+)
+parser.add_argument(
+    "--eob",
+    action="store_true",
+    help="Whether to evaluate end of backchannel predictions.",
+)
+parser.add_argument(
+    "--ignore_eob",
+    action="store_true",
+    help="Whether to ignore end of backchannel predictions.",
+)
+parser.add_argument(
+    "--multiple",
+    action="store_true",
+    help="Whether to evaluate multiple datasets.",
+)
+
+
+def load_segLST(directory: str, use_eob: bool = False, ignore_eob: bool = False) -> dict:
+    json_files = list(Path(directory).glob("*.json"))
+    segLST = {}
+    for json_file in json_files:
+        key = json_file.stem
+        with open(json_file, 'r') as f:
+            data = json.load(f)
+            assert isinstance(data, list), f"Data in {json_file} is not a list."
+            if not ignore_eob:
+                # get the data with the correct eob label
+                data = [x for x in data if (x.get("is_backchannel", False) == use_eob)]
+            segLST[key] = data
+    return segLST
+
+
+def evaluate_eou_predictions(
+    prediction_dir: str, reference_dir: str, use_eob: bool = False, ignore_eob: bool = False
+) -> List[EOUResult]:
+    prediction_segLST = load_segLST(prediction_dir, use_eob, ignore_eob)
+    reference_segLST = load_segLST(reference_dir, use_eob, ignore_eob)
+
+    eou_metrics = []
+    for key, reference in reference_segLST.items():
+        if key not in prediction_segLST:
+            raise ValueError(f"Key {key} in reference not found in predictions.")
+        prediction = prediction_segLST[key]
+        eou_result = evaluate_eou(
+            prediction=prediction, reference=reference, threshold=None, collar=0.0, do_sorting=True
+        )
+        eou_metrics.append(eou_result)
+
+    results = aggregate_eou_metrics(eou_metrics)
+
+    # add prefix to the keys of the results
+    prefix = Path(reference_dir).stem
+    prefix += "_eob" if use_eob else "_eou"
+    results = {f"{prefix}_{k}": v for k, v in results.items()}
+
+    return results
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    prediction_dir = Path(args.prediction)
+    reference_dir = Path(args.reference)
+
+    if not prediction_dir.is_dir():
+        raise ValueError(f"Prediction directory {prediction_dir} does not exist or is not a directory.")
+    if not reference_dir.is_dir():
+        raise ValueError(f"Reference directory {reference_dir} does not exist or is not a directory.")
+
+    if args.multiple:
+        # get all subdirectories in the prediction and reference directories
+        prediction_dirs = sorted([x for x in prediction_dir.glob("*/") if x.is_dir()])
+        reference_dirs = sorted([x for x in reference_dir.glob("*/") if x.is_dir()])
+        if len(prediction_dirs) != len(reference_dirs):
+            raise ValueError(
+                f"Number of prediction directories {len(prediction_dirs)} must match number of reference directories {len(reference_dirs)}."
+            )
+    else:
+        prediction_dirs = [prediction_dir]
+        reference_dirs = [reference_dir]
+
+    for ref_dir, pred_dir in zip(reference_dirs, prediction_dirs):
+        if args.multiple and ref_dir.stem != pred_dir.stem:
+            raise ValueError(
+                f"Reference directory {ref_dir} and prediction directory {pred_dir} must have the same name."
+            )
+        results = evaluate_eou_predictions(
+            prediction_dir=str(pred_dir), reference_dir=str(ref_dir), use_eob=args.eob, ignore_eob=args.ignore_eob
+        )
+        # Print the results
+        print("==========================================")
+        print(f"Evaluation Results for: {pred_dir} against {ref_dir}")
+        for key, value in results.items():
+            print(f"{key}: {value:.4f}")
+        print("==========================================")

scripts/asr_eou/generate_noisy_eval_data.py

@@ -0,0 +1,224 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+This script is used to generate noisy evaluation data for ASR and end of utterance detection.
+
+Example usage with a single manifest input:
+python generate_noisy_eval_data.py \
+    --config-path conf/ \
+    --config-name data \
+    output_dir=/path/to/output \
+    data.manifest_filepath=/path/to/manifest.json \
+    data.seed=42 \
+    data.noise.manifest_path /path/to/noise_manifest.json
+
+
+Example usage with multiple manifests matching a pattern:
+python generate_noisy_eval_data.py \
+    --config-path conf/ \
+    --config-name data \
+    output_dir=/path/to/output/dir \
+    data.manifest_filepath=/path/to/manifest/dir/ \
+    data.pattern="*.json" \
+    data.seed=42 \
+    data.noise.manifest_path /path/to/noise_manifest.json
+
+"""
+
+from copy import deepcopy
+from pathlib import Path
+from shutil import rmtree
+
+import librosa
+import lightning.pytorch as pl
+import numpy as np
+import soundfile as sf
+import torch
+import yaml
+from lhotse.cut import MixedCut
+from omegaconf import DictConfig, ListConfig, OmegaConf, open_dict
+from tqdm import tqdm
+
+from nemo.collections.asr.data.audio_to_eou_label_lhotse import LhotseSpeechToTextBpeEOUDataset
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+from nemo.collections.common.data.lhotse import get_lhotse_dataloader_from_config
+from nemo.collections.common.parts.preprocessing import parsers
+from nemo.core.config import hydra_runner
+from nemo.utils import logging
+
+
+@hydra_runner(config_path="conf/", config_name="data")
+def main(cfg: DictConfig):
+    """
+    Generate noisy evaluation data for ASR and end of utterance detection.
+
+    Args:
+        cfg: DictConfig object containing the configuration.
+    """
+    # Seed everything for reproducibility
+    seed = cfg.data.get('seed', None)
+    if seed is None:
+        seed = np.random.randint(0, 2**32 - 1)
+        logging.info(f'No seed provided, using random seed: {seed}')
+    logging.info(f'Setting random seed to {seed}')
+    with open_dict(cfg):
+        cfg.data.seed = seed
+    logging.info(f'Hydra config: {OmegaConf.to_yaml(cfg)}')
+    pl.seed_everything(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    np.random.seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+    # Patch data config
+    with open_dict(cfg.data):
+        cfg.data.force_finite = True
+        cfg.data.force_map_dataset = True
+        cfg.data.shuffle = False
+        cfg.data.check_tokenizer = False  # No need to check tokenizer in LhotseSpeechToTextBpeEOUDataset
+
+    # Make output directory
+    output_dir = Path(cfg.output_dir)
+    if output_dir.exists() and cfg.get('overwrite', False):
+        logging.info(f'Removing existing output directory: {output_dir}')
+        rmtree(output_dir)
+    if not output_dir.exists():
+        logging.info(f'Creating output directory: {output_dir}')
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Dump the config to the output directory
+    config = OmegaConf.to_container(cfg, resolve=True)
+    with open(output_dir / 'config.yaml', 'w') as f:
+        yaml.dump(config, f)
+    logging.info(f'Config dumped to {output_dir / "config.yaml"}')
+
+    if isinstance(cfg.data.manifest_filepath, (list, ListConfig)):
+        manifest_list = [Path(x) for x in cfg.data.manifest_filepath]
+    else:
+        input_manifest_file = Path(cfg.data.manifest_filepath)
+        if input_manifest_file.is_dir():
+            pattern = cfg.data.get('pattern', '*.json')
+            manifest_list = list(input_manifest_file.glob(pattern))
+            if not manifest_list:
+                raise ValueError(f"No files found in {input_manifest_file} matching pattern `{pattern}`")
+        else:
+            manifest_list = [Path(x) for x in str(input_manifest_file).split(",")]
+
+    logging.info(f'Found {len(manifest_list)} manifest files to process...')
+
+    for i, manifest_file in enumerate(manifest_list):
+        logging.info(f'[{i+1}/{len(manifest_list)}] Processing {manifest_file}...')
+        data_cfg = deepcopy(cfg.data)
+        data_cfg.manifest_filepath = str(manifest_file)
+        process_manifest(data_cfg, output_dir)
+
+
+def process_manifest(data_cfg: DictConfig, output_dir: Path):
+    """
+    Process a manifest file and generate noisy evaluation data.
+
+    Args:
+        data_cfg: Configuration.
+        output_dir: Output directory.
+    """
+    # Load the input manifest
+    input_manifest = read_manifest(data_cfg.manifest_filepath)
+    logging.info(f'Found {len(input_manifest)} items in input manifest: {data_cfg.manifest_filepath}')
+    manifest_parent_dir = Path(data_cfg.manifest_filepath).parent
+    if Path(input_manifest[0]["audio_filepath"]).is_absolute():
+        output_audio_dir = output_dir / 'wav'
+        flatten_audio_path = True
+    else:
+        output_audio_dir = output_dir
+        flatten_audio_path = False
+
+    if "random_padding" in data_cfg and data_cfg.random_padding.pad_distribution == "constant":
+        is_constant_padding = True
+        pre_pad_dur = data_cfg.random_padding.pre_pad_duration
+    else:
+        is_constant_padding = False
+        pre_pad_dur = None
+
+    # Load the dataset
+    tokenizer = parsers.make_parser()  # dummy tokenizer
+    dataset = LhotseSpeechToTextBpeEOUDataset(cfg=data_cfg, tokenizer=tokenizer, return_cuts=True)
+
+    dataloader = get_lhotse_dataloader_from_config(
+        config=data_cfg,
+        global_rank=0,
+        world_size=1,
+        dataset=dataset,
+        tokenizer=tokenizer,
+    )
+
+    # Generate noisy evaluation data
+    manifest = []
+    for i, batch in enumerate(tqdm(dataloader, desc="Generating noisy evaluation data")):
+        audio_batch, audio_len_batch, cuts_batch = batch
+        audio_batch = audio_batch.cpu().numpy()
+        audio_len_batch = audio_len_batch.cpu().numpy()
+
+        for j in range(len(cuts_batch)):
+            cut = cuts_batch[j]
+            if isinstance(cut, MixedCut):
+                cut = cut.first_non_padding_cut
+
+            manifest_item = {}
+            for k, v in cut.custom.items():
+                if k == "dataloading_info":
+                    continue
+                manifest_item[k] = v
+            audio = audio_batch[j][: audio_len_batch[j]]
+            audio_file = cut.recording.sources[0].source
+
+            if flatten_audio_path:
+                output_audio_file = output_audio_dir / str(audio_file).replace('/', '_')[:255]  # type: Path
+            else:
+                output_audio_file = output_audio_dir / Path(audio_file).relative_to(manifest_parent_dir)  # type: Path
+
+            output_audio_file.parent.mkdir(parents=True, exist_ok=True)
+            sf.write(output_audio_file, audio, dataset.sample_rate)
+
+            manifest_item["audio_filepath"] = str(output_audio_file.relative_to(output_audio_dir))
+            manifest_item["offset"] = 0
+            manifest_item["duration"] = audio.shape[0] / dataset.sample_rate
+
+            if is_constant_padding:
+                # Adjust the sou_time and eou_time for constant padding
+                if 'sou_time' in manifest_item and 'eou_time' in manifest_item:
+                    if not isinstance(manifest_item['sou_time'], list):
+                        manifest_item['sou_time'] = manifest_item['sou_time'] + pre_pad_dur
+                        manifest_item['eou_time'] = manifest_item['eou_time'] + pre_pad_dur
+                    else:
+                        manifest_item['sou_time'] = [x + pre_pad_dur for x in manifest_item['sou_time']]
+                        manifest_item['eou_time'] = [x + pre_pad_dur for x in manifest_item['eou_time']]
+                else:
+                    # add sou_time and eou_time to the manifest item
+                    manifest_item['sou_time'] = pre_pad_dur
+                    manifest_item['eou_time'] = pre_pad_dur + librosa.get_duration(filename=audio_file)
+
+            manifest.append(manifest_item)
+
+    # Write the output manifest
+    output_manifest_file = output_dir / Path(data_cfg.manifest_filepath).name
+    write_manifest(output_manifest_file, manifest)
+    logging.info(f'Output manifest written to {output_manifest_file}')
+
+
+if __name__ == "__main__":
+    main()

scripts/asr_eou/tokenizers/add_special_tokens_to_sentencepiece.py

@@ -0,0 +1,214 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+import logging
+import sys
+import tempfile
+from argparse import ArgumentParser
+from pathlib import Path
+
+import sentencepiece as spm
+
+from nemo.collections.asr.data.audio_to_eou_label_lhotse import EOB_STRING, EOU_STRING
+from nemo.core.connectors.save_restore_connector import SaveRestoreConnector
+
+try:
+    import sentencepiece_model_pb2 as spt
+except (ImportError, ModuleNotFoundError):
+    raise Exception("Ensure that sentencepiece_model_pb2.py has been generated from the protoc compiler")
+
+
+SPECIAL_TOKENS = [EOU_STRING, EOB_STRING]
+
+"""Utility to add special tokens to existing sentencepiece models.
+
+Generate sentencepiece_model_pb2.py in the directory of this script before running
+To generate run `protoc --python_out=<path_to_NeMo>/scripts/asr_end_of_utterance/tokenizers sentencepiece_model.proto`
+inside the src folder in sentencepiece repo
+Refer: https://github.com/google/sentencepiece/issues/121
+
+Usage:
+python add_special_tokens_to_sentencepiece.py \
+    --input_file your_model.nemo \
+    --output_dir /path/to/new/tokenizer_dir/
+"""
+
+
+parser = ArgumentParser(description="Add special tokens to sentencepiece model")
+parser.add_argument(
+    "-i",
+    "--input_file",
+    type=str,
+    required=True,
+    help="Path to nemo model file, or sentencepiece model file",
+)
+parser.add_argument(
+    "-o",
+    "--output_dir",
+    type=str,
+    required=True,
+    help="Path to output directory for new tokenizer",
+)
+parser.add_argument(
+    "--tokens",
+    type=str,
+    nargs='+',
+    help="Special tokens to add to tokenizer",
+    default=SPECIAL_TOKENS,
+)
+parser.add_argument(
+    "--extract_only",
+    action="store_true",
+    help="Extract tokenizer without adding special tokens",
+)
+
+
+def extract_nemo_tokenizer(nemo_filepath: str, output_dir: Path) -> str:
+    """
+    Extract a tokenizer from a Nemo file.
+    Args:
+        nemo_filepath: Path to the Nemo file.
+        output_dir: Path to the output directory.
+
+    Returns:
+        tokenizer: Path to the tokenizer file.
+    """
+    SaveRestoreConnector._unpack_nemo_file(path2file=nemo_filepath, out_folder=output_dir)
+    tokenizer = None
+    for file in Path(output_dir).glob("**/*"):
+        if file.is_file() and file.name.endswith("tokenizer.model"):
+            tokenizer = file
+            break
+    if tokenizer is None:
+        raise ValueError(f"Tokenizer not found in {output_dir}: {os.listdir(output_dir)}")
+    return str(tokenizer.absolute())
+
+
+def edit_spt_model(input_file, output_dir, tokens, is_userdefined, extract_only=False):
+    """
+    Edit a sentencepiece model to add special tokens.
+    Args:
+        input_file: Path to the input sentencepiece model file.
+        output_dir: Path to the output directory.
+        tokens: List of special tokens to add.
+        is_userdefined: Whether the special tokens are user-defined.
+        extract_only: Whether to extract the tokenizer only.
+    """
+
+    if extract_only:
+        logging.info("Extracting tokenizer only, no special tokens will be added.")
+
+    output_dir = Path(output_dir)
+
+    if output_dir.exists():
+        logging.warning(f"Output directory {output_dir} already exists. Overwriting it.")
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    output_file = str(output_dir / "tokenizer.model")
+
+    token_type = 3
+    if is_userdefined:
+        token_type = 4
+
+    model = spt.ModelProto()
+    with open(input_file, 'rb') as f:
+        model.ParseFromString(f.read())
+
+    if not extract_only:
+        for token in tokens:
+            piece = model.SentencePiece(piece=token, score=0.0, type=token_type)
+            if piece in model.pieces:
+                logging.error(f"Special Token '{token}' already exists in the input model!")
+                sys.exit(1)
+            model.pieces.append(piece)
+
+    sp = spm.SentencePieceProcessor()
+    sp.LoadFromSerializedProto(model.SerializeToString())
+
+    if not extract_only:
+        try:
+            for token in tokens:
+                id = sp.piece_to_id(token)
+                logging.info(f"Created token '{token}' at ID {id}")
+            logging.info(f"New tokenizer vocab size: {sp.get_piece_size()}")
+        except Exception:
+            logging.error(
+                "Could not appropriately configure new tokenizer. Verify if the special tokens already exist."
+            )
+            sys.exit(1)
+
+    with open(output_file, 'wb') as outf:
+        outf.write(model.SerializeToString())
+    logging.info(f"Created new tokenizer at: {output_file}")
+
+    # Write the vocab to file
+    vocab_file = str(output_dir / "tokenizer.vocab")
+    with open(vocab_file, "w", encoding="utf-8") as f:
+        for i in range(sp.get_piece_size()):
+            piece = sp.id_to_piece(i)
+            score = sp.get_score(i)  # Optional: only available if using newer SentencePiece versions
+            f.write(f"{piece}\t{score}\n")  # Format follows the original vocab format
+    logging.info(f"Created new tokenizer vocab at: {vocab_file}")
+
+    special_tokens = ["<s>", "</s>", "<pad>", "<unk>"]
+    special_tokens.extend(tokens)
+    vocab_txt_file = str(output_dir / "vocab.txt")
+    with open(vocab_txt_file, "w", encoding="utf-8") as f:
+        for i in range(sp.get_piece_size()):
+            piece = sp.id_to_piece(i)
+            if piece in special_tokens:
+                # skip special tokens
+                continue
+            token = piece[1:] if piece.startswith("▁") else f"##{piece}"
+            if len(token) > 0:
+                f.write(f"{token}\n")  # Format follows the original vocab format
+    logging.info(f"Created new tokenizer vocab at: {vocab_txt_file}")
+
+
+def inject_special_tokens(input_file, output_dir, tokens, is_userdefined=True, extract_only=False):
+    """
+    Inject special tokens into a sentencepiece model.
+    NOTE: is_userdefined should be set to True in order for ASR model to work with the new special tokens properly.
+
+    Args:
+        input_file: Path to the input sentencepiece model file.
+        output_dir: Path to the output directory.
+        tokens: List of special tokens to add.
+        is_userdefined: Whether the special tokens are user-defined.
+        extract_only: Whether to extract the tokenizer only.
+    """
+
+    if not os.path.exists(input_file):
+        raise ValueError(f"Input file {input_file} does not exist")
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Check if input file is a Nemo file
+        if input_file.endswith(".nemo"):
+            input_file = extract_nemo_tokenizer(input_file, temp_dir)
+            logging.info(f"Extracted tokenizer from Nemo file: {input_file}")
+        else:
+            input_file = os.path.abspath(input_file)
+            logging.info(f"Using input file: {input_file}")
+
+        edit_spt_model(input_file, output_dir, tokens, is_userdefined, extract_only=extract_only)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+    args = parser.parse_args()
+    inject_special_tokens(args.input_file, args.output_dir, args.tokens, extract_only=args.extract_only)

scripts/asr_eou/transcribe_speech_sharded.py

@@ -0,0 +1,343 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add the examples/asr directory to the Python path so that we can import the transcribe_speech.py file
+import sys
+from pathlib import Path
+
+nemo_root = Path(__file__).parent.parent.parent
+asr_examples_dir = nemo_root / "examples" / "asr"
+sys.path.insert(0, str(asr_examples_dir))
+
+from collections import defaultdict
+from copy import deepcopy
+from dataclasses import dataclass
+from math import ceil
+from pathlib import Path
+from typing import List
+
+from omegaconf import ListConfig
+from tqdm import tqdm
+from transcribe_speech import TranscriptionConfig as SingleTranscribeConfig  # type: ignore
+from transcribe_speech import main as single_transcribe_main  # type: ignore
+
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest
+from nemo.core.config import hydra_runner
+from nemo.utils import logging
+
+"""
+Transcribe audio manifests on distributed GPUs. Useful for transcription of moderate amounts of audio data.
+This script also supports splitting the manifest into chunks and merging the results back together.
+This script is a modified version of `transcribe_speech.py` that only takes manifest files as input.
+It is useful for transcribing a large amount of audio data that does not fit into a single job.
+
+# Arguments
+  model_path: path to .nemo ASR checkpoint
+  pretrained_name: name of pretrained ASR model (from NGC registry)
+  dataset_manifest: path to dataset JSON manifest file (in NeMo formats), can be a comma-separated list of manifest files
+                    or a directory containing manifest files
+  pattern: pattern to glob the manifest files if `dataset_manifest` is a directory
+  output_dir: directory to write the transcriptions
+
+  compute_langs: Bool to request language ID information (if the model supports it)
+  timestamps: Bool to request greedy time stamp information (if the model supports it) by default None 
+
+  (Optionally: You can limit the type of timestamp computations using below overrides)
+  ctc_decoding.ctc_timestamp_type="all"  # (default all, can be [all, char, word, segment])
+  rnnt_decoding.rnnt_timestamp_type="all"  # (default all, can be [all, char, word, segment])
+
+  output_filename: Output filename where the transcriptions will be written
+  batch_size: batch size during inference
+  presort_manifest: sorts the provided manifest by audio length for faster inference (default: True)
+
+  cuda: Optional int to enable or disable execution of model on certain CUDA device.
+  allow_mps: Bool to allow using MPS (Apple Silicon M-series GPU) device if available
+  amp: Bool to decide if Automatic Mixed Precision should be used during inference
+  audio_type: Str filetype of the audio. Supported = wav, flac, mp3
+
+  overwrite_transcripts: Bool which when set allows repeated transcriptions to overwrite previous results.
+
+  ctc_decoding: Decoding sub-config for CTC. Refer to documentation for specific values.
+  rnnt_decoding: Decoding sub-config for RNNT. Refer to documentation for specific values.
+
+  calculate_wer: Bool to decide whether to calculate wer/cer at end of this script
+  clean_groundtruth_text: Bool to clean groundtruth text
+  langid: Str used for convert_num_to_words during groundtruth cleaning
+  use_cer: Bool to use Character Error Rate (CER)  or Word Error Rate (WER)
+
+  calculate_rtfx: Bool to calculate the RTFx throughput to transcribe the input dataset.
+
+# Usage
+ASR model can be specified by either "model_path" or "pretrained_name".
+append_pred - optional. Allows you to add more than one prediction to an existing .json
+pred_name_postfix - optional. The name you want to be written for the current model
+Results are returned in a JSON manifest file.
+
+```bash
+CUDA_VISIBLE_DEVICES=1 python transcribe_speech_distributed.py \
+    model_path=<path to .nemo ASR checkpoint> \
+    dataset_manifest="<remove or path to manifest>" \
+    output_dir="<output directory>" \
+    output_filename="<remove or specify output filename>" \
+    clean_groundtruth_text=True \
+    langid='en' \
+    batch_size=32 \
+    timestamps=False \
+    compute_langs=False \
+    amp=True \
+    append_pred=False \
+    pred_name_postfix="<remove or use another model name for output filename>" \
+    split_size=10000 \
+    num_nodes=1 \
+    node_idx=0 \
+    num_gpus_per_node=1 \
+    gpu_idx=0
+```
+
+If you use Slurm, you can use this params to configure the script:
+```bash
+    gpu_idx=\$SLURM_LOCALID \
+    num_gpus_per_node=\$SLURM_GPUS_ON_NODE \
+    num_nodes=\$SLURM_JOB_NUM_NODES \
+    node_idx=\$SLURM_NODEID
+```
+
+"""
+
+
+@dataclass
+class TranscriptionConfig(SingleTranscribeConfig):
+    """
+    Transcription Configuration for audio to text transcription.
+    """
+
+    # General configs
+    pattern: str = "*.json"
+    output_dir: str = "transcribe_output/"
+
+    # Distributed config
+    num_nodes: int = 1  # total number of nodes
+    node_idx: int = 0  # index of the current node
+    num_gpus_per_node: int = 1  # number of GPUs per node
+    gpu_idx: int = 0  # index of the current GPU
+    bind_gpu_to_cuda: bool = (
+        False  # If False, the script will just do .cuda() on the model, otherwise it will do .to(f"cuda:{gpu_idx}")
+    )
+
+    # handle long manifest
+    split_size: int = -1  # -1 means no split, otherwise split the manifest into chunks of this size
+
+
+def get_unfinished_manifest(manifest_list: List[Path], output_dir: Path):
+    """
+    Get the manifest files that have not finished processing yet, including those that are partly processed.
+
+    Args:
+        manifest_list: List of manifest files to process.
+        output_dir: Directory to write the transcriptions.
+
+    Returns:
+        List of manifest files that have not finished processing yet.
+    """
+    unfinished = []
+    for manifest_file in manifest_list:
+        output_manifest_file = output_dir / manifest_file.name
+        if not output_manifest_file.exists():
+            unfinished.append(manifest_file)
+    return sorted(unfinished)
+
+
+def get_manifest_for_current_rank(
+    manifest_list: List[Path], gpu_id: int = 0, num_gpu: int = 1, node_idx: int = 0, num_node: int = 1
+):
+    """
+    Get the manifest files for the current rank.
+
+    Args:
+        manifest_list: List of manifest files to process.
+        gpu_id: ID of the current GPU.
+        num_gpu: Number of GPUs per node.
+        node_idx: Index of the current node.
+        num_node: Total number of nodes.
+
+    Returns:
+        List of manifest files for the current rank.
+    """
+    node_manifest_list = []
+    assert num_node > 0, f"num_node ({num_node}) must be greater than 0"
+    assert num_gpu > 0, f"num_gpu ({num_gpu}) must be greater than 0"
+    assert 0 <= gpu_id < num_gpu, f"gpu_id ({gpu_id}) must be in range [0, {num_gpu})"
+    assert 0 <= node_idx < num_node, f"node_idx ({node_idx}) must be in range [0, {num_node})"
+    for i, manifest_file in enumerate(manifest_list):
+        if (i + node_idx) % num_node == 0:
+            node_manifest_list.append(manifest_file)
+
+    gpu_manifest_list = []
+    for i, manifest_file in enumerate(node_manifest_list):
+        if (i + gpu_id) % num_gpu == 0:
+            gpu_manifest_list.append(manifest_file)
+    return gpu_manifest_list
+
+
+def maybe_split_manifest(manifest_list: List[Path], cfg: TranscriptionConfig) -> List[Path]:
+    """
+    Split the manifest files into chunks of the specified size.
+
+    Args:
+        manifest_list: List of manifest files to process.
+        cfg: Configuration.
+
+    Returns:
+        List of sharded manifest files.
+    """
+    if cfg.split_size is None or cfg.split_size <= 0:
+        return manifest_list
+
+    all_sharded_manifest_files = []
+    sharded_manifest_dir = Path(cfg.output_dir) / "sharded_manifest_todo"
+    sharded_manifest_dir.mkdir(parents=True, exist_ok=True)
+
+    sharded_manifest_done_dir = Path(cfg.output_dir) / "sharded_manifest_done"
+    sharded_manifest_done_dir.mkdir(parents=True, exist_ok=True)
+    cfg.output_dir = sharded_manifest_done_dir
+
+    logging.info(f"Splitting {len(manifest_list)} manifest files by every {cfg.split_size} samples.")
+    for manifest_file in tqdm(manifest_list, total=len(manifest_list), desc="Splitting manifest files"):
+        manifest = read_manifest(manifest_file)
+
+        num_chunks = ceil(len(manifest) / cfg.split_size)
+        for i in range(num_chunks):
+            chunk_manifest = manifest[i * cfg.split_size : (i + 1) * cfg.split_size]
+            sharded_manifest_file = sharded_manifest_dir / f"{manifest_file.stem}--tpart_{i}.json"
+            write_manifest(sharded_manifest_file, chunk_manifest)
+            all_sharded_manifest_files.append(sharded_manifest_file)
+
+    return all_sharded_manifest_files
+
+
+def maybe_merge_manifest(cfg: TranscriptionConfig):
+    """
+    Merge the sharded manifest files back into the original manifest files and write them to the output directory.
+
+    Args:
+        cfg: Configuration.
+
+    Returns:
+        None.
+    """
+    if cfg.split_size is None or cfg.split_size <= 0:
+        return
+
+    # only merge manifest on the first GPU of the first node
+    if not (cfg.gpu_idx == 0 and cfg.node_idx == 0):
+        return
+
+    sharded_manifest_dir = Path(cfg.output_dir)
+    sharded_manifests = list(sharded_manifest_dir.glob("*--tpart_*.json"))
+    if not sharded_manifests:
+        logging.info(f"No sharded manifest files found in {sharded_manifest_dir}")
+        return
+
+    logging.info(f"Merging {len(sharded_manifests)} sharded manifest files.")
+    manifest_dict = defaultdict(list)
+    for sharded_manifest in sharded_manifests:
+        data_name = sharded_manifest.stem.split("--tpart_")[0]
+        manifest_dict[data_name].append(sharded_manifest)
+
+    output_dir = Path(cfg.output_dir).parent
+    for data_name, sharded_manifest_list in tqdm(
+        manifest_dict.items(), total=len(manifest_dict), desc="Merging manifest files"
+    ):
+        merged_manifest = []
+        for sharded_manifest in sharded_manifest_list:
+            manifest = read_manifest(sharded_manifest)
+            merged_manifest.extend(manifest)
+        output_manifest = output_dir / f"{data_name}.json"
+        write_manifest(output_manifest, merged_manifest)
+    logging.info(f"Merged manifest files saved to {output_dir}")
+
+
+@hydra_runner(config_name="TranscriptionConfig", schema=TranscriptionConfig)
+def run_distributed_transcribe(cfg: TranscriptionConfig):
+    """
+    Run distributed transcription with the given configuration.
+    """
+    logging.info(f"Running distributed transcription with config: {cfg}")
+
+    if cfg.dataset_manifest is None:
+        raise ValueError("`dataset_manifest` is required")
+
+    # load the manifest
+    if isinstance(cfg.dataset_manifest, str) and "," in cfg.dataset_manifest:
+        manifest_list = cfg.dataset_manifest.split(",")
+    elif isinstance(cfg.dataset_manifest, (ListConfig, list)):
+        manifest_list = cfg.dataset_manifest
+    else:
+        input_manifest = Path(cfg.dataset_manifest)
+        if input_manifest.is_dir():
+            manifest_list = list(input_manifest.glob(cfg.pattern))
+        elif input_manifest.is_file():
+            manifest_list = [input_manifest]
+        else:
+            raise ValueError(f"Invalid manifest file or directory: {input_manifest}")
+
+    if not manifest_list:
+        raise ValueError(f"No manifest files found matching pattern: {cfg.pattern} in {input_manifest}")
+
+    manifest_list = maybe_split_manifest(manifest_list, cfg)
+    original_manifest_list = list(manifest_list)
+    logging.info(f"Found {len(manifest_list)} manifest files.")
+
+    output_dir = Path(cfg.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    unfinished_manifest = get_unfinished_manifest(manifest_list, output_dir=output_dir)
+    if not unfinished_manifest:
+        maybe_merge_manifest(cfg)
+        logging.info("All manifest files have been processed. Exiting.")
+        return
+    logging.info(f"Found {len(unfinished_manifest)} unfinished manifest files.")
+
+    manifest_list = get_manifest_for_current_rank(
+        unfinished_manifest,
+        gpu_id=cfg.gpu_idx,
+        num_gpu=cfg.num_gpus_per_node,
+        node_idx=cfg.node_idx,
+        num_node=cfg.num_nodes,
+    )
+    if not manifest_list:
+        logging.info(f"No manifest files found for GPU {cfg.gpu_idx} on node {cfg.node_idx}. Exiting.")
+        return
+
+    logging.info(f"Processing {len(manifest_list)} manifest files with GPU {cfg.gpu_idx} on node {cfg.node_idx}.")
+
+    cfg.cuda = cfg.gpu_idx if cfg.bind_gpu_to_cuda else None
+    for manifest_file in tqdm(manifest_list):
+        logging.info(f"Processing {manifest_file}...")
+        output_filename = output_dir / Path(manifest_file).name
+        curr_cfg = deepcopy(cfg)
+        curr_cfg.dataset_manifest = str(manifest_file)
+        curr_cfg.output_filename = str(output_filename)
+
+        single_transcribe_main(curr_cfg)
+
+    # check if all manifest files have been processed
+    unfinished_manifest = get_unfinished_manifest(original_manifest_list, output_dir=output_dir)
+    if not unfinished_manifest:
+        maybe_merge_manifest(cfg)
+        logging.info("All manifest files have been processed. Exiting.")
+        return
+
+
+if __name__ == '__main__':
+    run_distributed_transcribe()  # noqa pylint: disable=no-value-for-parameter