nvidia-nemo--speech/nemo/collections/asr/inference/utils/per_stream_biasing.py

# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import annotations

from typing import TYPE_CHECKING, Any, Sequence

import numpy as np
import torch
from torch import Tensor

from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
from nemo.utils import logging

if TYPE_CHECKING:
    from nemo.collections.asr.parts.context_biasing.biasing_multi_model import GPUBiasingMultiModelBase


def build_multi_biasing_ids_np(
    states: Sequence[Any],
    biasing_multi_model: GPUBiasingMultiModelBase,
    tokenizer: TokenizerSpec,
) -> np.ndarray:
    """Build per-stream biasing model ids; ``-1`` means no biasing for that stream."""
    ids_np = np.full([len(states)], fill_value=-1, dtype=np.int64)
    for i, state in enumerate(states):
        if not state.has_biasing_request():
            continue

        biasing_cfg = state.options.biasing_cfg
        model_id = biasing_cfg.multi_model_id
        if model_id is not None and not biasing_multi_model.model2active[model_id].item():
            biasing_cfg.multi_model_id = None
            model_id = None

        if model_id is None:
            if biasing_cfg.auto_manage_multi_model:
                with torch.inference_mode():
                    biasing_cfg.add_to_multi_model(tokenizer=tokenizer, biasing_multi_model=biasing_multi_model)
                model_id = biasing_cfg.multi_model_id
            else:
                logging.warning("Biasing request is not empty, not auto managed and not compiled. Skipping")
                continue

        ids_np[i] = model_id
    return ids_np


def multi_biasing_ids_tensor_from_states(
    states: Sequence[Any],
    device: torch.device,
    *,
    per_stream_biasing_enabled: bool,
) -> Tensor | None:
    """Build decode-time biasing ids from ``state.options.biasing_cfg`` (after registration)."""
    if not per_stream_biasing_enabled:
        return None

    ids_np = np.full([len(states)], fill_value=-1, dtype=np.int64)
    for i, state in enumerate(states):
        if not state.has_biasing_request():
            continue
        model_id = state.options.biasing_cfg.multi_model_id
        if model_id is None:
            logging.warning(f"Boosting tree requested in index {i}, not compiled, skipping")
            continue
        ids_np[i] = model_id

    if (ids_np < 0).all():
        return None
    return torch.from_numpy(ids_np).to(device=device)


def release_all_biasing_models(biasing_multi_model: GPUBiasingMultiModelBase, states: Sequence[Any]) -> None:
    """Remove every active biasing model and clear per-stream ``multi_model_id`` bookkeeping."""
    active_model_ids = [
        model_id
        for model_id in range(biasing_multi_model.num_models)
        if biasing_multi_model.model2active[model_id].item()
    ]
    with torch.inference_mode():
        for model_id in sorted(active_model_ids, reverse=True):
            biasing_multi_model.remove_model(model_id)
    for state in states:
        if state.has_biasing_request():
            state.options.biasing_cfg.multi_model_id = None


def release_auto_managed_stream_biasing(state: Any, biasing_multi_model: GPUBiasingMultiModelBase) -> None:
    """Drop an auto-managed biasing model when a single stream ends."""
    if not state.has_biasing_request():
        return
    if state.options.biasing_cfg.auto_manage_multi_model:
        state.options.biasing_cfg.remove_from_multi_model(biasing_multi_model)