chore: import upstream snapshot with attribution

examples/voice_agent/server/backchannel_phrases.yaml

@@ -0,0 +1,78 @@
+- "absolutely"
+- "ah"
+- "all right"
+- "alright"
+- "but yeah"
+- "cool"
+- "definitely"
+- "exactly"
+- "go ahead"
+- "good"
+- "great"
+- "great thanks"
+- "ha ha"
+- "hmm"
+- "humm"
+- "huh"
+- "i know"
+- "i know right"
+- "i see"
+- "indeed"
+- "interesting"
+- "mhmm"
+- "mhmm mhmm"
+- "mhmm right"
+- "mhmm yeah"
+- "mhmm yes"
+- "mm hmm"
+- "mmhmm"
+- "nice"
+- "of course"
+- "oh"
+- "oh dear"
+- "oh man"
+- "oh okay"
+- "oh wow"
+- "oh yes"
+- "ok"
+- "ok thanks"
+- "okay"
+- "okay okay"
+- "okay thanks"
+- "perfect"
+- "really"
+- "right"
+- "right exactly"
+- "right right"
+- "right yeah"
+- "so yeah"
+- "sounds good"
+- "sure"
+- "sure thing"
+- "thank you"
+- "thanks"
+- "that's awesome"
+- "thats right"
+- "thats true"
+- "true"
+- "uh huh"
+- "uh-huh"
+- "uh-huh yeah"
+- "uhhuh"
+- "uhhuh okay"
+- "um-humm"
+- "well"
+- "what"
+- "wow"
+- "yeah"
+- "yeah i know"
+- "yeah i see"
+- "yeah mhmm"
+- "yeah okay"
+- "yeah right"
+- "yeah uh-huh"
+- "yeah yeah"
+- "yep"
+- "yes"
+- "yes please"
+- "yes yes"

examples/voice_agent/server/example_prompts/fast-bite.txt

@@ -0,0 +1,52 @@
+Fast Bites Lunch Menu
+
+Burgers and Sandwiches:
+1. Classic Cheeseburger – $5.99
+   Juicy beef patty, cheddar cheese, pickles, ketchup & mustard on a toasted bun.
+   - Make it a double cheeseburger by adding another patty - $1.50
+2. Crispy Chicken Sandwich – $6.49
+   Fried chicken filet, lettuce, mayo, and pickles on a brioche bun.
+3. Veggie Wrap – $5.49
+   Grilled vegetables, hummus, lettuce, and tomato in a spinach wrap.
+
+Combo Deals (includes small fries and fountain soda)
+4. Cheeseburger Combo – $8.99
+5. Chicken Sandwich Combo – $9.49
+6. Veggie Wrap Combo – $8.49
+
+Sides:
+7. French Fries
+ - Small - $2.49
+ - Medium - $3.49
+ - Large - $4.49
+8. Chicken Nuggets
+ - 4 pieces - $3.29
+ - 8 pieces -  $5.99
+ - 12 pieces -  $8.99
+9. Side Salad -  $2.99
+
+Drinks:
+10. Fountain Soda (16 oz, choices: Coke, Diet Coke, Sprite, Fanta) – $1.99
+11. Iced Tea or Lemonade – $2.29
+12. Bottled Water – $1.49
+
+You are a helpful assistant named Lisa that helps customers order food from the lunch menu.
+Start by greeting the user warmly and introducing yourself within one sentence "Hi welcome to Fast Bites! I'm Lisa, what can I help you with?".
+Your answer should be concise and to the point.
+Do not include the whole lunch menu in your response, only include the items that are relevant to the user's question.
+If the user asks about a specific item, you should include the price of that item. 
+If the user asks about the menu, you should include the entire lunch menu. 
+If the user asks about the prices, you should include the prices of the items. 
+If the user asks about the location, you should include the location of the restaurant (123 Main St, Anytown, USA). 
+If the user asks about the hours, you should include the hours of the restaurant (11:00 AM - 9:00 PM). 
+When a user asks for the total price of the order, you should include the total price of the order. 
+When the conversation is done, you should say "Thank you for your order! Your total is <total_price>. Please come back soon!", where <total_price> is the total price of the orders of all speakers. 
+If a speaker finishes their order and you don't know their name, you should ask them for their name and associate it with their order. 
+When introducing an item from the menu, you should include the name of the item and the price.
+Stick strictly to the lunch menu and do not make up any items.
+You might also see speaker tags (<speaker_0>, <speaker_1>, etc.) in the user context. 
+You should respond to the user based on the speaker tag and the context of that speaker. 
+Do not include the speaker tags in your response, use them only to identify the speaker.
+If there are multiple speakers, you should handle the order of each speaker separately and not mix up the speakers. 
+Do not respond only with "Hi" or "Hi there", you should focus on the task of taking the order and not just greeting the user.
+/no_think

examples/voice_agent/server/example_prompts/simple_chatbot.txt

@@ -0,0 +1,4 @@
+You are a helpful AI agent named Lisa. 
+Start by greeting the user warmly and introducing yourself within one sentence. 
+Your answer should be concise and to the point.
+/no_think

examples/voice_agent/server/example_prompts/simple_chatbot_diar.txt

@@ -0,0 +1,8 @@
+You are a helpful AI agent named Lisa. 
+Start by greeting the user warmly and introducing yourself within one sentence. 
+Your answer should be concise and to the point. 
+You might also see speaker tags (<speaker_0>, <speaker_1>, etc.) in the user context. 
+You should respond to the user based on the speaker tag and the context of that speaker. 
+Do not include the speaker tags in your response, use them only to identify the speaker.
+If a speaker provides their name, use their name when addressing their requests.
+/no_think

examples/voice_agent/server/model_registry.yaml

@@ -0,0 +1,38 @@
+# Model registry configuration
+# Centralized registry of available models and their corresponding configuration files
+#
+# "reasoning_supported=true" means that there's a another llm yaml file with the same name but addidional "_think.yaml" suffix,
+# which is tested to work as expected, otherwise you need to mamually modify the llm yaml to enable their reasoning mode.
+
+llm_models:
+  "nvidia/NVIDIA-Nemotron-Nano-9B-v2":
+    yaml_id: "nemotron_nano_v2.yaml"
+    reasoning_supported: false
+    
+  "meta-llama/Llama-3.1-8B-Instruct":
+    yaml_id: "llama3.1-8B-instruct.yaml"
+    reasoning_supported: false
+
+  "Qwen/Qwen2.5-7B-Instruct":
+    yaml_id: "qwen2.5-7B.yaml"
+    reasoning_supported: false
+    
+  "Qwen/Qwen3-8B":
+    yaml_id: "qwen3-8B.yaml"
+    reasoning_supported: true
+
+  "hf_llm_generic":
+    yaml_id: "hf_llm_generic.yaml"
+    reasoning_supported: false
+
+tts_models:
+  "fastpitch-hifigan":
+    yaml_id: "nemo_fastpitch-hifigan.yaml"
+
+  "hexgrad/Kokoro-82M":
+    yaml_id: "kokoro_82M.yaml"
+
+stt_models:
+  "stt_en_fastconformer_hybrid_large_streaming_80ms":
+    yaml_id: "nemo_cache_aware_streaming.yaml"
+    type: "nemo"

examples/voice_agent/server/parsers/nano_v3_reasoning_parser.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code adapted from https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/resolve/main/nano_v3_reasoning_parser.py
+
+from vllm.reasoning.abs_reasoning_parsers import ReasoningParserManager
+from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
+
+
+@ReasoningParserManager.register_module("nano_v3")
+class NanoV3ReasoningParser(DeepSeekR1ReasoningParser):
+    def extract_reasoning(self, model_output, request):
+        reasoning_content, final_content = super().extract_reasoning(model_output, request)
+        if (
+            hasattr(request, "chat_template_kwargs")
+            and request.chat_template_kwargs
+            and request.chat_template_kwargs.get("enable_thinking") is False
+            and final_content is None
+        ):
+            reasoning_content, final_content = final_content, reasoning_content
+
+        return reasoning_content, final_content

examples/voice_agent/server/parsers/nemotron_toolcall_parser_streaming.py

@@ -0,0 +1,637 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Code adapted from https://huggingface.co/nvidia/NVIDIA-Nemotron-Nano-9B-v2/resolve/main/nemotron_toolcall_parser_streaming.py
+
+import json
+from collections.abc import Sequence
+from random import choices
+from string import ascii_letters, digits
+from typing import Optional, Union
+
+import partial_json_parser
+import regex as re
+from partial_json_parser.core.options import Allow
+from pydantic import Field
+from vllm.entrypoints.openai.protocol import (
+    ChatCompletionRequest,
+    DeltaFunctionCall,
+    DeltaMessage,
+    DeltaToolCall,
+    ExtractedToolCallInformation,
+    FunctionCall,
+    ToolCall,
+)
+from vllm.logger import init_logger
+from vllm.tokenizers.mistral import MistralTokenizer
+from vllm.tool_parsers.abstract_tool_parser import ToolParser, ToolParserManager
+from vllm.transformers_utils.tokenizer import AnyTokenizer
+
+logger = init_logger(__name__)
+
+ALPHANUMERIC = ascii_letters + digits
+
+
+class NemotronToolCall(ToolCall):
+    id: str = Field(default_factory=lambda: NemotronToolCall.generate_random_id())
+
+    @staticmethod
+    def generate_random_id():
+        return "".join(choices(ALPHANUMERIC, k=9))
+
+    @staticmethod
+    def is_valid_id(id: str) -> bool:
+        return id.isalnum() and len(id) == 9
+
+
+def _is_fn_name_regex_support(model_tokenizer: AnyTokenizer) -> bool:
+    return isinstance(model_tokenizer, MistralTokenizer) and model_tokenizer.version >= 11
+
+
+@ToolParserManager.register_module("nemotron_json")
+class NemotronToolParser(ToolParser):
+    """
+    Streaming tool call parser specifically designed for the Nemotron-Nano-V2 model.
+
+    This parser functions as an active reconstruction engine, managing the realtime
+    transition from text generation to structured tool execution. Its primary responsibilities
+    during token streaming include:
+
+    - Interception: Detects and consumes the `<TOOLCALL>` control tokens to switch parsing modes.
+    - Buffering: Manages a lookahead buffer to prevent ambiguous partial tags (like `<TOO`)
+                 from leaking to the user.
+    - Restoration: Utilizes `partial_json_parser` to reconstruct valid objects from incomplete
+                   JSON fragments.
+    - Differentiation: Computes the precise "delta" between the current and previous JSON states
+                       to ensure monotonic streaming.
+    - Sanitization: Strips premature auto-completed closing characters (e.g., `}`)
+                    to prevent malformed updates.
+
+    Configuration:
+        Activate this parser in the vLLM server by setting the following mandatory arguments:
+        - `--enable-auto-tool-choice`
+        - `--tool-call-parser nemotron_json`
+    """
+
+    def __init__(self, tokenizer: AnyTokenizer):
+        super().__init__(tokenizer)
+        # initialize properties used for state when parsing tool calls in
+        # streaming mode
+        self.prev_tool_call_arr: list[dict] = []
+        self.current_tool_id: int = -1
+        self.current_tool_name_sent: bool = False
+        self.streamed_args_for_tool: list[str] = []  # map what has been streamed for each tool so far to a list
+        self.tool_args_emitted: list[bool] = []
+        self.bot_token = "<TOOLCALL>"
+        self.bot_token_id = self.vocab.get(self.bot_token)
+        logger.info(f"Nemotron Tool Parser: bot_token: {self.bot_token}, bot_token_id: {self.bot_token_id}")
+        self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL)
+        if _is_fn_name_regex_support(self.model_tokenizer):
+            self.fn_name_regex = re.compile(r'([a-zA-Z0-9_-]+)(\{[\s\S]*?\})(?=\s*$|,|\s)', re.DOTALL)
+        else:
+            self.fn_name_regex = None
+
+        # Buffer for partial tag sequences to disambiguate between normal content and
+        # a forthcoming <TOOLCALL> or </TOOLCALL> tag in streaming.
+        self._pending_tag_buffer: str = ""
+
+    def _reset_state(self) -> None:
+        """
+        Reset the parser state for a new request.
+        This is used to prevent state corruption across multiple sequential requests.
+        """
+        self.prev_tool_call_arr: list[dict] = []
+        self.current_tool_id: int = -1
+        self.current_tool_name_sent: bool = False
+        self.streamed_args_for_tool: list[str] = []
+        self.tool_args_emitted: list[bool] = []
+        self._pending_tag_buffer: str = ""
+
+    @staticmethod
+    def _strip_trailing_auto_closers(chunk: str) -> str:
+        """
+        Remove parser auto-completed closing braces/brackets plus trailing whitespace.
+        These should be flushed only when a tool call completes to avoid duplicate
+        argument fragments.
+
+        Args:
+            chunk (str):
+                The chunk of text to strip.
+        Return:
+            (str): The chunk of text with trailing auto-completed closing braces/brackets
+                   plus trailing whitespace removed.
+        """
+        idx = len(chunk)
+        while idx > 0 and chunk[idx - 1] in " \t\r\n}]":
+            idx -= 1
+        # Remove trailing non-escaped double quotes (partial JSON auto-closes strings)
+        while idx > 0 and chunk[idx - 1] == '"':
+            # keep escaped quotes (\"), only strip bare ones
+            if idx - 2 >= 0 and chunk[idx - 2] == '\\':
+                break
+            idx -= 1
+        return chunk[:idx]
+
+    @staticmethod
+    def _common_prefix_len(left: str, right: str) -> int:
+        """
+        Calculate the length of the longest initial substring shared by two strings.
+
+        This utility is used to determine how much of the tool arguments have already
+        been streamed to the client, allowing the system to send only the new 'delta'.
+
+        Args:
+            left (str): The first string to compare (typically the full current arguments).
+            right (str): The second string to compare (typically the previously streamed arguments).
+
+        Returns:
+            int: The count of identical characters starting from index 0.
+                 Returns 0 if the strings share no common prefix.
+        """
+        max_len = min(len(left), len(right))
+        idx = 0
+        while idx < max_len and left[idx] == right[idx]:
+            idx += 1
+        return idx
+
+    def _compute_arguments_delta(self, cur_arguments_json: str, end_of_call: bool) -> str:
+        """
+        Determine the incremental suffix to stream for the current tool call.
+        Ensures we only emit monotonic chunks by trimming our tracked prefix to
+        the longest common prefix with the latest JSON snapshot.
+
+        Args:
+            cur_arguments_json (str):
+                The current arguments JSON in string format.
+            end_of_call (bool):
+                Whether the current tool call is the last one in the array.
+
+        Return:
+            (str): The incremental suffix to stream for the current tool call.
+        """
+        tool_idx = self.current_tool_id
+        if tool_idx < 0 or tool_idx >= len(self.streamed_args_for_tool):
+            if tool_idx < 0:
+                logger.debug(f"current_tool_id is negative ({tool_idx}), no tool designated yet")
+            else:
+                logger.warning(
+                    f"tool_idx ({tool_idx}) is out of bounds for streamed_args_for_tool "
+                    f"(length: {len(self.streamed_args_for_tool)})"
+                )
+            return ""
+
+        streamed_prefix = self.streamed_args_for_tool[tool_idx]
+        had_any = self.tool_args_emitted[tool_idx] if tool_idx < len(self.tool_args_emitted) else False
+
+        lcp_len = self._common_prefix_len(cur_arguments_json, streamed_prefix)
+        if lcp_len != len(streamed_prefix):
+            streamed_prefix = streamed_prefix[:lcp_len]
+            self.streamed_args_for_tool[tool_idx] = streamed_prefix
+
+        if (
+            not had_any
+            and not end_of_call
+            and lcp_len == 0
+            and cur_arguments_json.endswith('": ""}')
+            and '": ""' in cur_arguments_json
+        ):
+            closing_pos = cur_arguments_json.rfind('": ""}')
+            if closing_pos != -1:
+                arguments_delta = cur_arguments_json[: closing_pos + 4]
+            else:
+                arguments_delta = cur_arguments_json
+        else:
+            arguments_delta = cur_arguments_json[lcp_len:]
+
+        if not arguments_delta:
+            return ""
+
+        if not end_of_call:
+            arguments_delta = self._strip_trailing_auto_closers(arguments_delta)
+
+        if not had_any and not end_of_call and arguments_delta and arguments_delta.endswith('}'):
+            arguments_delta = arguments_delta[:-1]
+            if arguments_delta.endswith('"'):
+                arguments_delta = arguments_delta[:-1]
+
+        return arguments_delta
+
+    def _visible_delta_outside_tool(
+        self, delta_text: str, start_token: Optional[str], end_token: Optional[str]
+    ) -> str:
+        """
+        Filters incoming streaming text to hide incomplete or complete tool call tags.
+
+        This method acts as a buffer for the streaming response. It consumes and holds
+        characters that resemble the start of `start_token` or `end_token` (e.g., "<", "<T", "<TOO").
+
+        - If the buffer eventually matches the full token exactly (e.g., "<TOOLCALL>"),
+          the buffer is discarded (suppressed).
+        - If the buffer diverges from the expected tokens (e.g., user types "<Think>"),
+          the buffered text is released (flushed) alongside the current character.
+        - Regular text that does not start with "<" passes through immediately.
+
+        Args:
+            delta_text (str):
+                The new chunk of text generated by the model in this streaming step.
+            start_token (Optional[str]):
+                The opening tag to suppress (e.g., "<TOOLCALL>"). If None, no start tag is tracked.
+            end_token (Optional[str]):
+                The closing tag to suppress (e.g., "</TOOLCALL>"). If None, no end tag is tracked.
+
+        Returns:
+            str: The portion of `delta_text` (plus any previously buffered ambiguous characters)
+            that has been confirmed as *not* being part of a tool call tag.
+        """
+        if not delta_text:
+            return delta_text
+
+        visible: list[str] = []
+        for ch in delta_text:
+            if self._pending_tag_buffer or ch == '<':
+                self._pending_tag_buffer += ch
+
+                if start_token and start_token.startswith(self._pending_tag_buffer):
+                    if self._pending_tag_buffer == start_token:
+                        self._pending_tag_buffer = ""
+                    continue
+
+                if end_token and end_token.startswith(self._pending_tag_buffer):
+                    if self._pending_tag_buffer == end_token:
+                        self._pending_tag_buffer = ""
+                    continue
+
+                # Not a tool tag; flush buffered characters as normal content.
+                visible.append(self._pending_tag_buffer)
+                self._pending_tag_buffer = ""
+            else:
+                visible.append(ch)
+
+        return "".join(visible)
+
+    def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
+        if not isinstance(self.model_tokenizer, MistralTokenizer) and request.tools and request.tool_choice != 'none':
+            # Do not skip special tokens when using chat template
+            # with Mistral parser as TOOL_CALL token is needed
+            # for tool detection.
+            # Note: we don't want skip_special_tokens=False
+            # with MistralTokenizer as it is incompatible
+            request.skip_special_tokens = False
+        return request
+
+    def extract_tool_calls(
+        self,
+        model_output: str,
+        request: ChatCompletionRequest,
+    ) -> ExtractedToolCallInformation:
+        """
+        Parses a complete (non-streaming) model response to extract tool execution instructions.
+
+        This method attempts to convert the raw text output from the model into structured
+        `NemotronToolCall` objects. It employs a robust two-stage parsing strategy:
+        - Direct JSON Parsing: First attempts to parse the content following the
+           `<TOOLCALL>` token as valid JSON.
+        - Regex Fallback: If direct parsing fails (e.g., due to extra text or noise),
+           it uses a regular expression to locate and extract the specific JSON array pattern.
+
+        Args:
+            model_output (str): The full text generated by the model.
+            request (ChatCompletionRequest): The original request object (used for context if needed).
+
+        Returns:
+            ExtractedToolCallInformation: An object containing the parsed list of tool calls
+            and any preceding text content. If parsing fails entirely, it returns the raw
+            content as a standard text message.
+        """
+        # Reset state for each new non-streaming request
+        self._reset_state()
+
+        # case -- if a tool call token is not present, return a text response
+        if self.bot_token not in model_output:
+            return ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)
+
+        # first remove the BOT token
+        tool_content = model_output.replace(self.bot_token, "").strip()
+
+        try:
+            # we first try to directly load the json as parsing very nested
+            # jsons is difficult
+            try:
+                if self.fn_name_regex:
+                    matches = self.fn_name_regex.findall(tool_content)
+
+                    function_call_arr = []
+                    for match in matches:
+                        fn_name = match[0]
+                        args = match[1]
+
+                        # fn_name is encoded outside serialized json dump
+                        # only arguments are serialized
+                        function_call_arr.append({"name": fn_name, "arguments": json.loads(args)})
+                else:
+                    function_call_arr = json.loads(tool_content)
+            except json.JSONDecodeError:
+                # use a regex to find the part corresponding to the tool call.
+                # NOTE: This use case should not happen if the model is trained
+                # correctly. It's an easy possible fix so it's included, but
+                # can be brittle for very complex / highly nested tool calls
+                matches = self.tool_call_regex.findall(tool_content)
+                if not matches:
+                    raise ValueError(f"No tool call pattern found in: {tool_content[:100]} ...")
+                raw_tool_call = matches[0]
+                function_call_arr = json.loads(raw_tool_call)
+
+            # Tool Call
+            tool_calls: list[NemotronToolCall] = [
+                NemotronToolCall(
+                    type="function",
+                    function=FunctionCall(
+                        name=raw_function_call["name"],
+                        # function call args are JSON but as a string
+                        arguments=json.dumps(raw_function_call["arguments"], ensure_ascii=False),
+                    ),
+                )
+                for raw_function_call in function_call_arr
+            ]
+
+            # get any content before  the tool call
+            content = model_output.split(self.bot_token)[0]
+            return ExtractedToolCallInformation(
+                tools_called=True, tool_calls=tool_calls, content=content if len(content) > 0 else None
+            )
+
+        except Exception:
+            logger.exception("Error in extracting tool call from response.")
+            # return information to just treat the tool call as regular JSON
+            return ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=tool_content)
+
+    def extract_tool_calls_streaming(
+        self,
+        previous_text: str,
+        current_text: str,
+        delta_text: str,
+        previous_token_ids: Sequence[int],
+        current_token_ids: Sequence[int],
+        delta_token_ids: Sequence[int],
+        request: ChatCompletionRequest,
+    ) -> Union[DeltaMessage, None]:
+        """
+        Parses the raw text stream to identify and extract tool calls in real-time.
+        This method monitors for the `<TOOLCALL>` trigger to switch parsing modes, buffers
+        ambiguous tag prefixes to prevent leakage, and utilizes partial JSON parsing to
+        compute and emit precise incremental updates (deltas) for tool arguments while
+        suppressing auto-generated artifacts.
+
+        Args:
+            previous_text (str): (Placeholder) The generated text prior to the current step.
+            current_text (str): The total generated text including the new token.
+            delta_text (str): The specific text chunk generated in this step.
+            previous_token_ids (Sequence[int]): (Placeholder) Token IDs for previous text.
+            current_token_ids (Sequence[int]): (Placeholder) Token IDs for current text.
+            delta_token_ids (Sequence[int]): (Placeholder) Token IDs for the delta.
+            request (ChatCompletionRequest): (Placeholder) The original client request object.
+
+        Returns:
+            Union[DeltaMessage, None]: A `DeltaMessage` containing visible content or
+            tool call updates, or `None` if the output is currently buffered or unchanged.
+        """
+        # Reset state at the start of a new streaming request
+        # Detect new request: if we have stale state but previous_text indicates this is a fresh start
+        if not previous_text and (
+            self.current_tool_id != -1 or self.prev_tool_call_arr or self.streamed_args_for_tool
+        ):
+            logger.debug("Detected new streaming request, resetting parser state")
+            self._reset_state()
+
+        # if candidates tool call tokens are in the tokens generated so far, that
+        # means we're parsing as tool calls now. Suppress streaming if we are
+        # currently generating any prefix of the start or end tag.
+        visible_delta_text = delta_text
+        try:
+            start_token = self.bot_token
+            end_token = f"</{self.bot_token[1:]}" if self.bot_token.startswith('<') else None
+
+            visible_delta_text = self._visible_delta_outside_tool(delta_text, start_token, end_token)
+        except Exception:
+            # Fallback to conservative checks in case of any issues
+            if (
+                current_text.endswith('<')
+                or current_text.endswith('<T')
+                or current_text.endswith('<TO')
+                or current_text.endswith('<TOOL')
+                or current_text.endswith('<TOOLCALL')
+            ):
+                return None
+
+        # if the tool call token is not in the tokens generated so far, append
+        # output to contents since it's not a tool
+        if self.bot_token not in current_text:
+            if visible_delta_text:
+                return DeltaMessage(content=visible_delta_text)
+            # still waiting on a potential tag, so emit nothing yet
+            return None
+
+        # bit mask flags for partial JSON parsing. If the name hasn't been
+        # sent yet, don't allow sending
+        # an incomplete string since OpenAI only ever (as far as I have
+        # seen) allows sending the entire tool/ function name at once.
+        flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR
+        end_of_call: bool = False
+        try:
+
+            # replace BOT token with empty string, and convert single quotes
+            # to double to allow parsing as JSON since mistral uses single
+            # quotes instead of double for tool calls
+            parsable_arr = current_text.split(self.bot_token)[-1]
+
+            # Check if we're at the end of the tool call
+            if '</TOOLCALL>' in parsable_arr:
+                end_of_call = True
+                parsable_arr = parsable_arr.split('</TOOLCALL>')[0]
+
+            # tool calls are generated in an array, so do partial JSON
+            # parsing on the entire array
+            try:
+                tool_call_arr: list[dict] = partial_json_parser.loads(parsable_arr, flags)
+            except (partial_json_parser.core.exceptions.MalformedJSON, json.JSONDecodeError, ValueError):
+                return None
+
+            current_tool_call: dict = (
+                tool_call_arr[self.current_tool_id]
+                if len(tool_call_arr) > 0 and self.current_tool_id >= 0 and self.current_tool_id < len(tool_call_arr)
+                else {}
+            )
+
+            # case -- if no tokens have been streamed for the tool, e.g.
+            #   only the array brackets, stream nothing
+            if len(tool_call_arr) == 0:
+                return None
+
+            # case: we are starting a new tool in the array
+            #   -> array has > 0 length AND length has moved past cursor
+            elif len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1:
+
+                # if we're moving on to a new call, first make sure we
+                # haven't missed anything in the previous one that was
+                # auto-generated due to JSON completions, but wasn't
+                # streamed to the client yet.
+                if self.current_tool_id >= 0 and self.current_tool_id < len(self.streamed_args_for_tool):
+                    diff: Union[str, None] = current_tool_call.get("arguments")
+
+                    if diff:
+                        diff = json.dumps(diff, ensure_ascii=False).replace(
+                            self.streamed_args_for_tool[self.current_tool_id], ""
+                        )
+                        delta = DeltaMessage(
+                            tool_calls=[
+                                DeltaToolCall(
+                                    index=self.current_tool_id,
+                                    function=DeltaFunctionCall(arguments=diff).model_dump(exclude_none=True),
+                                )
+                            ]
+                        )
+                        self.streamed_args_for_tool[self.current_tool_id] += diff
+                    else:
+                        delta = None
+                else:
+                    delta = None
+                # re-set stuff pertaining to progress in the current tool
+                self.current_tool_id = len(tool_call_arr) - 1
+                self.current_tool_name_sent = False
+                self.streamed_args_for_tool.append("")
+                self.tool_args_emitted.append(False)
+                return delta
+
+            # case: update an existing tool - this is handled below
+
+            # if the current tool name hasn't been sent, send if available
+            # - otherwise send nothing
+            if not self.current_tool_name_sent:
+                function_name = current_tool_call.get("name")
+                if function_name:
+
+                    delta = DeltaMessage(
+                        tool_calls=[
+                            DeltaToolCall(
+                                index=self.current_tool_id,
+                                type="function",
+                                id=NemotronToolCall.generate_random_id(),
+                                function=DeltaFunctionCall(name=function_name).model_dump(exclude_none=True),
+                            )
+                        ]
+                    )
+                    self.current_tool_name_sent = True
+                else:
+                    delta = None
+
+            # now we know we're on the same tool call and we're streaming
+            # arguments
+            else:
+                if self.current_tool_id < 0 or self.current_tool_id >= len(self.prev_tool_call_arr):
+                    prev_arguments = None
+                else:
+                    prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get("arguments")
+                cur_arguments = current_tool_call.get("arguments")
+
+                if not cur_arguments and not prev_arguments:
+                    delta = None
+                elif not cur_arguments and prev_arguments:
+                    logger.error("INVARIANT - impossible to have arguments reset " "mid-arguments")
+                    delta = None
+                elif cur_arguments:
+                    cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False)
+                    arguments_delta = self._compute_arguments_delta(cur_arguments_json, end_of_call)
+                    if arguments_delta:
+                        delta = DeltaMessage(
+                            tool_calls=[
+                                DeltaToolCall(
+                                    index=self.current_tool_id,
+                                    function=DeltaFunctionCall(arguments=arguments_delta).model_dump(
+                                        exclude_none=True
+                                    ),
+                                )
+                            ]
+                        )
+                        if self.current_tool_id >= 0 and self.current_tool_id < len(self.streamed_args_for_tool):
+                            self.streamed_args_for_tool[self.current_tool_id] += arguments_delta
+                        else:
+                            logger.warning(
+                                f"current_tool_id ({self.current_tool_id}) is out of bounds for streamed_args_for_tool "
+                                f"(length: {len(self.streamed_args_for_tool)})"
+                            )
+                        if self.current_tool_id >= 0 and self.current_tool_id < len(self.tool_args_emitted):
+                            self.tool_args_emitted[self.current_tool_id] = True
+                        else:
+                            logger.warning(
+                                f"current_tool_id ({self.current_tool_id}) is out of bounds for tool_args_emitted "
+                                f"(length: {len(self.tool_args_emitted)})"
+                            )
+                    else:
+                        # Do not flush final JSON here; let the serving layer
+                        # compute a minimal remaining suffix on finish.
+                        delta = None
+                else:
+                    # End-of-call or equal state; do not force a final flush here.
+                    delta = None
+
+            # check to see if the name is defined and has been sent. if so,
+            # stream the name - otherwise keep waiting
+            # finish by setting old and returning None as base case
+            self.prev_tool_call_arr = tool_call_arr
+            # If we've reached the end of a tool call, flush any remaining
+            # suffix (including a final '}') that hasn't been streamed yet.
+            if end_of_call and self.current_tool_id >= 0:
+                try:
+                    cur_arguments = current_tool_call.get("arguments")
+                    if cur_arguments is not None:
+                        cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
+                        remaining_suffix = self._compute_arguments_delta(cur_args_json, end_of_call=True)
+
+                        # Only send remaining suffix if it's non-empty and contains meaningful content
+                        # (not just whitespace or single characters like closing braces)
+                        if remaining_suffix and remaining_suffix.strip():
+                            extra = DeltaToolCall(
+                                index=self.current_tool_id,
+                                function=DeltaFunctionCall(arguments=remaining_suffix).model_dump(exclude_none=True),
+                            )
+                            if delta is None:
+                                delta = DeltaMessage(tool_calls=[extra])
+                            else:
+                                if getattr(delta, "tool_calls", None):
+                                    delta.tool_calls.append(extra)
+                                else:
+                                    delta.tool_calls = [extra]
+                            if self.current_tool_id >= 0 and self.current_tool_id < len(self.streamed_args_for_tool):
+                                self.streamed_args_for_tool[self.current_tool_id] += remaining_suffix
+                            else:
+                                logger.warning(
+                                    f"current_tool_id ({self.current_tool_id}) is out of bounds for streamed_args_for_tool "
+                                    f"(length: {len(self.streamed_args_for_tool)})"
+                                )
+                            if self.current_tool_id >= 0 and self.current_tool_id < len(self.tool_args_emitted):
+                                self.tool_args_emitted[self.current_tool_id] = True
+                            else:
+                                logger.warning(
+                                    f"current_tool_id ({self.current_tool_id}) is out of bounds for tool_args_emitted "
+                                    f"(length: {len(self.tool_args_emitted)})"
+                                )
+                except Exception as e:
+                    # Failure to flush the remaining arguments suffix is non-fatal; log for debugging.
+                    logger.warning(f"Error in flushing remaining suffix for tool call: {e}")
+
+            return delta
+
+        except Exception as e:
+            logger.exception(f"Error trying to handle streaming tool call: {e}")
+            return None

examples/voice_agent/server/server.py

@@ -0,0 +1,456 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import asyncio
+import copy
+import os
+import signal
+import sys
+from contextlib import asynccontextmanager
+from datetime import datetime
+from typing import Any, Dict
+
+import uvicorn
+from dotenv import load_dotenv
+from fastapi import FastAPI, WebSocket
+from fastapi.middleware.cors import CORSMiddleware
+from loguru import logger
+from omegaconf import OmegaConf
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.frames.frames import EndTaskFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.processors.frameworks.rtvi import RTVIAction, RTVIConfig, RTVIObserverParams, RTVIProcessor
+from pipecat.serializers.protobuf import ProtobufFrameSerializer
+from websocket_url import build_websocket_url
+
+from nemo.agents.voice_agent.pipecat.processors.frameworks.rtvi import RTVIObserver
+from nemo.agents.voice_agent.pipecat.services.nemo.audio_logger import AudioLogger, RTVIAudioLoggerObserver
+from nemo.agents.voice_agent.pipecat.services.nemo.diar import NemoDiarService
+from nemo.agents.voice_agent.pipecat.services.nemo.llm import get_llm_service_from_config
+from nemo.agents.voice_agent.pipecat.services.nemo.stt import NemoSTTService
+from nemo.agents.voice_agent.pipecat.services.nemo.tts import get_tts_service_from_config
+from nemo.agents.voice_agent.pipecat.services.nemo.turn_taking import NeMoTurnTakingService
+from nemo.agents.voice_agent.pipecat.transports.network.websocket_server import (
+    WebsocketServerParams,
+    WebsocketServerTransport,
+)
+from nemo.agents.voice_agent.utils.config_manager import ConfigManager
+from nemo.agents.voice_agent.utils.tool_calling.basic_tools import tool_get_city_weather
+from nemo.agents.voice_agent.utils.tool_calling.mixins import register_direct_tools_to_llm
+
+# Load environment variables
+load_dotenv(override=True)
+
+
+def setup_logging():
+    # Configure loguru to output to both console and file
+    logger.remove()  # Remove default handler
+    logger.add(
+        sys.stderr,
+        format="<green>{time:YYYY-MM-DD HH:mm:ss.SSSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+        level="DEBUG",
+    )
+
+    logger.add("bot_server.log", rotation="1 day", level="DEBUG")
+
+
+setup_logging()
+
+# Global flag for graceful shutdown
+shutdown_event = asyncio.Event()
+
+# Initialize configuration manager
+config_manager = ConfigManager(
+    server_base_path=os.path.dirname(__file__), server_config_path=os.environ.get("SERVER_CONFIG_PATH", None)
+)
+server_config = config_manager.get_server_config()
+
+logger.info(f"Server config: {OmegaConf.to_container(server_config, resolve=True)}")
+
+# Access configuration parameters from ConfigManager
+SAMPLE_RATE = config_manager.SAMPLE_RATE
+RAW_AUDIO_FRAME_LEN_IN_SECS = config_manager.RAW_AUDIO_FRAME_LEN_IN_SECS
+SYSTEM_PROMPT = config_manager.SYSTEM_PROMPT
+SYSTEM_ROLE = config_manager.SYSTEM_ROLE
+
+# Transport configuration
+TRANSPORT_AUDIO_OUT_10MS_CHUNKS = config_manager.TRANSPORT_AUDIO_OUT_10MS_CHUNKS
+RECORD_AUDIO_DATA = server_config.transport.get("record_audio_data", False)
+AUDIO_LOG_DIR = server_config.transport.get("audio_log_dir", "./audio_logs")
+SERVER_HOST = os.getenv("SERVER_HOST", "0.0.0.0")
+SERVER_PUBLIC_HOST = os.getenv("SERVER_PUBLIC_HOST", "127.0.0.1")
+WEBSOCKET_SCHEME = os.getenv("WEBSOCKET_SCHEME", "ws")
+WEBSOCKET_PORT = int(os.getenv("WEBSOCKET_PORT", 8765))
+FASTAPI_PORT = int(os.getenv("FASTAPI_PORT", 7860))
+
+# VAD configuration
+vad_params = config_manager.get_vad_params()
+
+# STT configuration
+STT_MODEL = config_manager.STT_MODEL
+STT_DEVICE = config_manager.STT_DEVICE
+stt_params = config_manager.get_stt_params()
+
+# Diarization configuration
+DIAR_MODEL = config_manager.DIAR_MODEL
+USE_DIAR = config_manager.USE_DIAR
+diar_params = config_manager.get_diar_params()
+
+# Turn taking configuration
+TURN_TAKING_BACKCHANNEL_PHRASES_PATH = config_manager.TURN_TAKING_BACKCHANNEL_PHRASES_PATH
+TURN_TAKING_MAX_BUFFER_SIZE = config_manager.TURN_TAKING_MAX_BUFFER_SIZE
+TURN_TAKING_BOT_STOP_DELAY = config_manager.TURN_TAKING_BOT_STOP_DELAY
+
+# TTS configuration
+TTS_TYPE = config_manager.server_config.tts.type
+
+
+def signal_handler(signum, frame):
+    """Handle shutdown signals gracefully"""
+    logger.info(f"Received signal {signum}, initiating graceful shutdown...")
+    shutdown_event.set()
+
+
+async def run_bot_websocket_server(host: str = "0.0.0.0", port: int = None):
+    """
+    NO-TIMEOUT CONFIGURATION:
+    - session_timeout=None: Disables WebSocket session timeout
+    - idle_timeout=None: Disables pipeline idle timeout
+    - asyncio.wait_for(timeout=None): No timeout on pipeline runner
+    - Server will run indefinitely until manually stopped (Ctrl+C)
+    """
+    if port is None:
+        port = WEBSOCKET_PORT
+    logger.info(f"Starting websocket server on {host}:{port}")
+    logger.info(f"Server configured to run indefinitely with no timeouts, use Ctrl+C to quit.")
+
+    # Set up signal handlers for graceful shutdown
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    logger.info("Initializing WebSocket server transport...")
+    logger.info("Server configured to run indefinitely with no timeouts")
+    # Initialize AudioLogger if recording is enabled
+    audio_logger = None
+    if RECORD_AUDIO_DATA:
+        session_id = f"session_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        audio_logger = AudioLogger(
+            log_dir=AUDIO_LOG_DIR,
+            session_id=session_id,
+            enabled=True,
+        )
+        logger.info(f"AudioLogger initialized for session: {session_id} at {AUDIO_LOG_DIR}")
+
+    vad_analyzer = SileroVADAnalyzer(
+        sample_rate=SAMPLE_RATE,
+        params=vad_params,
+    )
+    logger.info("VAD analyzer initialized")
+
+    ws_transport = WebsocketServerTransport(
+        params=WebsocketServerParams(
+            serializer=ProtobufFrameSerializer(),
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            add_wav_header=False,
+            vad_analyzer=vad_analyzer,
+            session_timeout=None,  # Disable session timeout
+            audio_in_sample_rate=SAMPLE_RATE,
+            can_create_user_frames=False,
+            audio_out_10ms_chunks=TRANSPORT_AUDIO_OUT_10MS_CHUNKS,
+        ),
+        host=host,
+        port=port,
+    )
+
+    logger.info("Initializing STT service...")
+
+    stt = NemoSTTService(
+        model=STT_MODEL,
+        device=STT_DEVICE,
+        params=stt_params,
+        sample_rate=SAMPLE_RATE,
+        audio_passthrough=True,
+        backend="legacy",
+        decoder_type="rnnt",
+        audio_logger=audio_logger,
+    )
+    logger.info("STT service initialized")
+
+    if USE_DIAR:
+        diar = NemoDiarService(
+            model=DIAR_MODEL,
+            device=STT_DEVICE,
+            params=diar_params,
+            sample_rate=SAMPLE_RATE,
+            backend="legacy",
+            enabled=USE_DIAR,
+        )
+        logger.info("Diarization service initialized")
+    else:
+        diar = None
+
+    turn_taking = NeMoTurnTakingService(
+        use_diar=USE_DIAR,
+        max_buffer_size=TURN_TAKING_MAX_BUFFER_SIZE,
+        bot_stop_delay=TURN_TAKING_BOT_STOP_DELAY,
+        backchannel_phrases=TURN_TAKING_BACKCHANNEL_PHRASES_PATH,
+        audio_logger=audio_logger,
+    )
+    logger.info("Turn taking service initialized")
+
+    if TTS_TYPE == "nemo":
+        tts = get_tts_service_from_config(config_manager.server_config.tts, audio_logger)
+    else:
+        raise ValueError(f"Invalid TTS type: {TTS_TYPE}")
+
+    logger.info("TTS service initialized")
+
+    # Setup logging again to avoid logger from being overwritten during setting up the pipeline components
+    setup_logging()
+
+    # Put LLM in the end of model initialization to reduce the chance of running out of HBM memory
+    logger.info("Initializing LLM service...")
+    llm = get_llm_service_from_config(server_config.llm)
+    logger.info("LLM service initialized")
+
+    messages = [
+        {
+            "role": SYSTEM_ROLE,
+            "content": SYSTEM_PROMPT,
+        }
+    ]
+    inject_dummy_user_message = server_config.llm.get("inject_dummy_user_message", False)
+    if inject_dummy_user_message:
+        messages.append(
+            {
+                "role": "user",
+                "content": "Hello, who are you?",
+            }
+        )
+    context = OpenAILLMContext(messages=messages)
+
+    if server_config.llm.get("enable_tool_calling", False):
+        logger.info("Tools calling for LLM is enabled by config, registering tools...")
+        register_direct_tools_to_llm(llm=llm, context=context, tool_mixins=[tts], tools=[tool_get_city_weather])
+    else:
+        logger.info("Tools calling for LLM is disabled by config, skipping tool registration.")
+
+    original_messages = copy.deepcopy(context.get_messages())
+    original_context = copy.deepcopy(context)
+    original_context.set_llm_adapter(llm.get_llm_adapter())
+
+    context_aggregator = llm.create_context_aggregator(context)
+    user_context_aggregator = context_aggregator.user()
+    assistant_context_aggregator = context_aggregator.assistant()
+
+    # RTVI events for Pipecat client UI
+    rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
+
+    # Add reset action to RTVI processor
+    async def reset_context_handler(rtvi_processor: RTVIProcessor, service: str, arguments: dict[str, Any]) -> bool:
+        """Reset both user and assistant context aggregators"""
+        logger.info("Resetting conversation context...")
+        try:
+            user_context_aggregator.reset()
+            assistant_context_aggregator.reset()
+            user_context_aggregator.set_messages(copy.deepcopy(original_messages))
+            assistant_context_aggregator.set_messages(copy.deepcopy(original_messages))
+            tts.reset()
+            if diar is not None:
+                diar.reset()
+            turn_taking.reset()
+            logger.info("Conversation context reset successfully")
+            return True
+        except Exception as e:
+            logger.error(f"Error resetting context: {e}")
+            return False
+
+    reset_action = RTVIAction(
+        service="context",
+        action="reset",
+        result="bool",
+        arguments=[],
+        handler=reset_context_handler,
+    )
+    rtvi.register_action(reset_action)
+
+    logger.info("Setting up pipeline...")
+
+    pipeline = [
+        ws_transport.input(),
+        rtvi,
+        stt,
+    ]
+
+    if USE_DIAR:
+        pipeline.append(diar)
+
+    pipeline.extend(
+        [turn_taking, user_context_aggregator, llm, tts, ws_transport.output(), assistant_context_aggregator]
+    )
+
+    pipeline = Pipeline(pipeline)
+
+    rtvi_params = RTVIObserverParams(bot_llm_enabled=False)
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            allow_interruptions=True,
+            enable_metrics=False,
+            enable_usage_metrics=False,
+            send_initial_empty_metrics=True,
+            report_only_initial_ttfb=True,
+            idle_timeout=None,  # Disable idle timeout
+        ),
+        observers=[
+            RTVIObserver(rtvi, params=rtvi_params),
+            RTVIAudioLoggerObserver(audio_logger=audio_logger),
+        ],
+        idle_timeout_secs=None,
+        cancel_on_idle_timeout=False,
+    )
+
+    # Track task state
+    task_running = True
+
+    # Setup logging again to avoid logger from being overwritten during setting up the pipeline components
+    setup_logging()
+
+    @rtvi.event_handler("on_client_ready")
+    async def on_client_ready(rtvi: RTVIProcessor):
+        logger.info("Pipecat client ready.")
+        await rtvi.set_bot_ready()
+        # Kick off the conversation.
+        try:
+            await task.queue_frames([user_context_aggregator.get_context_frame()])
+        except Exception as e:
+            logger.error(f"Error queuing context frame: {e}")
+
+    @ws_transport.event_handler("on_client_connected")
+    async def on_client_connected(transport, client):
+        logger.info(f"Pipecat Client connected from {client.remote_address}")
+        # Reset RTVI state for new connection
+        rtvi._client_ready = False
+        rtvi._bot_ready = False
+
+    @ws_transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(transport, client):
+        logger.info(f"Pipecat Client disconnected from {client.remote_address}")
+        # Finalize audio logger session if enabled
+        if audio_logger:
+            audio_logger.finalize_session()
+            logger.info("Audio logger session finalized")
+        # Don't cancel the task immediately - let it handle the disconnection gracefully
+        # The task will continue running and can accept new connections
+        # Only send an EndTaskFrame to clean up the current session
+        if task_running:
+            try:
+                await task.queue_frames([EndTaskFrame()])
+            except Exception as e:
+                # Don't log warnings for normal connection closures
+                if "ConnectionClosedOK" not in str(e) and "1005" not in str(e):
+                    logger.warning(f"Error sending EndTaskFrame: {e}")
+                else:
+                    logger.info(f"Normal connection closure: {e}")
+
+    @ws_transport.event_handler("on_session_timeout")
+    async def on_session_timeout(transport, client):
+        logger.info(f"Session timeout for {client.remote_address}")
+        # Don't cancel the task - keep server running indefinitely
+        logger.info("Session timeout occurred but keeping server running")
+        # Note: With session_timeout=None, this handler should never be called
+
+    logger.info("Starting pipeline runner...")
+
+    try:
+        runner = PipelineRunner()
+        # Run the task until shutdown is requested
+        await asyncio.wait_for(runner.run(task), timeout=None)  # No timeout - run indefinitely
+    except asyncio.TimeoutError:
+        logger.info("Pipeline runner timeout (should not happen with no timeout)")
+    except Exception as e:
+        logger.error(f"Pipeline runner error: {e}")
+        task_running = False
+    finally:
+        # Finalize audio logger on shutdown
+        if audio_logger:
+            audio_logger.finalize_session()
+            logger.info("Audio logger session finalized on shutdown")
+        logger.info("Pipeline runner stopped")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Handles FastAPI startup and shutdown."""
+    yield  # Run app
+
+
+# Initialize FastAPI app with lifespan manager
+app = FastAPI(lifespan=lifespan)
+
+# Configure CORS to allow requests from any origin
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    await websocket.accept()
+    print("WebSocket connection accepted")
+    try:
+        # TODO: [heh] Implement FastAPI websocket endpoint
+        # await run_bot_fastapi_server(websocket)
+        raise NotImplementedError("FastAPI websocket endpoint is not implemented")
+    except Exception as e:
+        print(f"Exception in run_bot: {e}")
+
+
+@app.post("/connect")
+async def bot_connect() -> Dict[Any, Any]:
+    print("Received /connect request")
+    ws_url = build_websocket_url(SERVER_PUBLIC_HOST, WEBSOCKET_PORT, WEBSOCKET_SCHEME)
+    print(f"Returning WebSocket URL: {ws_url}")
+    return {"ws_url": ws_url}
+
+
+async def main():
+    """Main function to run both websocket server and FastAPI server concurrently."""
+    logger.info(f"Starting servers - WebSocket on port {WEBSOCKET_PORT}, FastAPI on port {FASTAPI_PORT}")
+    tasks = []
+    try:
+        # Start websocket server
+        tasks.append(run_bot_websocket_server(host=SERVER_HOST, port=WEBSOCKET_PORT))
+
+        # Start FastAPI server
+        config = uvicorn.Config(app, host=SERVER_HOST, port=FASTAPI_PORT)
+        server = uvicorn.Server(config)
+        tasks.append(server.serve())
+
+        await asyncio.gather(*tasks)
+    except asyncio.CancelledError:
+        logger.info("Tasks cancelled (probably due to shutdown).")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/voice_agent/server/server_configs/NVIDIA_NeMo_models.yaml

@@ -0,0 +1,49 @@
+# This is an example config for setting up a NeMo Voice Agent server only with NVIDIA NeMo models.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+# STT, LLM and TTS models have standalone configs in the folder "server/server_configs/{stt,llm,tts}_configs".
+# Specify the type and an a model identifier to automatically configure the model.
+
+transport:
+  audio_out_10ms_chunks: 10  # use 4 as websocket default, but increasing to a larger number might have less glitches in TTS output
+
+vad:
+  type: silero
+  confidence: 0.6  # VAD threshold for detecting speech versus non-speech
+  start_secs: 0.1  # min amount of speech to trigger UserStartSpeaking
+  stop_secs: 1.2  # min amount of silence to trigger UserStopSpeaking
+  min_volume: 0.4  # Microphone volumn threshold for VAD
+
+stt:
+  type: nemo # choices in ['nemo'] currently only NeMo is supported
+  model: "stt_en_fastconformer_hybrid_large_streaming_80ms"
+  model_config: "./server_configs/stt_configs/nemo_cache_aware_streaming.yaml"
+  device: "cuda"
+
+diar:
+  type: nemo
+  enabled: true # set to false to disable
+  model: "nvidia/diar_streaming_sortformer_4spk-v2"
+  device: "cuda"
+  threshold: 0.4  # threshold value used to determine if a speaker exists or not, setting it to a lower value will increaset the sensitivity of the model
+  frame_len_in_secs: 0.08  # default for Sortformer, do not change unless using other architechtures
+
+turn_taking:
+  backchannel_phrases_path: "./server/backchannel_phrases.yaml"  # set it to the actual path of the file, or specify a list of backchannel phrases here
+  max_buffer_size: 2  # num of words more than this amount will interrupt the LLM immediately if not backchannel phrases
+  bot_stop_delay: 0.5  # a delay in seconds allowed between server and client audio output, so that the BotStopSpeaking signal is handled not too far away from the actual time that the user hears all audio output
+
+llm:
+  type: auto  # choices in ['auto', 'hf', 'vllm'], if `auto`, it will try to use vllm and fall back to hf if vllm not available
+  model: "nvidia/NVIDIA-Nemotron-Nano-9B-v2"  # model name for HF models, will be used via `AutoModelForCausalLM.from_pretrained()`
+  model_config: "./server_configs/llm_configs/nemotron_nano_v2.yaml"
+  device: "cuda"
+  enable_reasoning: false  # it's best to turn-off reasoning for lowest latency
+  # `system_prompt` is used as the sytem prompt to the LLM, please refer to differnt LLM webpage for spcial functions like enabling/disabling thinking
+  # system_prompt: /path/to/prompt.txt  # or use path to a txt file that contains a long prompt, for example in `../example_prompts/fast_bite.txt`
+  system_prompt: "You are a helpful AI agent named Lisa. Start by greeting the user warmly and introducing yourself within one sentence. Your answer should be concise and to the point. You might also see speaker tags (<speaker_0>, <speaker_1>, etc.) in the user context. You should respond to the user based on the speaker tag and the context of that speaker. Do not include the speaker tags in your response, use them only to identify the speaker."
+
+tts:
+  type: nemo # choices in ['nemo', 'kokoro']
+  model: fastpitch-hifigan
+  model_config: "./server_configs/tts_configs/nemo_fastpitch-hifigan.yaml"
+  device: "cuda"

examples/voice_agent/server/server_configs/default.yaml

@@ -0,0 +1,63 @@
+# This is an example config for setting up a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+# STT, LLM and TTS models have standalone configs in the folder "server/server_configs/{stt,llm,tts}_configs".
+# Specify the type and an a model identifier to automatically configure the model.
+
+transport:
+  audio_out_10ms_chunks: 10  # use 4 as websocket default, but increasing to a larger number might have less glitches in TTS output
+  record_audio_data: false
+  audio_log_dir: "./audio_logs"
+
+vad:
+  type: silero
+  confidence: 0.6  # VAD threshold for detecting speech versus non-speech
+  start_secs: 0.1  # min amount of speech to trigger UserStartSpeaking
+  stop_secs: 1.2  # min amount of silence to trigger UserStopSpeaking
+  min_volume: 0.4  # Microphone volumn threshold for VAD
+
+stt:
+  type: nemo # choices in ['nemo'] currently only NeMo is supported
+  model: "nvidia/parakeet_realtime_eou_120m-v1"
+  # model: "nvidia/nemotron-speech-streaming-en-0.6b"
+  model_config: "./server_configs/stt_configs/nemo_cache_aware_streaming.yaml"
+  device: "cuda"
+
+diar:
+  type: nemo
+  enabled: true # set to false to disable
+  model: "nvidia/diar_streaming_sortformer_4spk-v2.1"
+  device: "cuda"
+  threshold: 0.4  # threshold value used to determine if a speaker exists or not, setting it to a lower value will increaset the sensitivity of the model
+  frame_len_in_secs: 0.08  # default for Sortformer, do not change unless using other architechtures
+
+turn_taking:
+  backchannel_phrases_path: "./server/backchannel_phrases.yaml"  # set it to the actual path of the file, or specify a list of backchannel phrases here
+  max_buffer_size: 2  # num of words more than this amount will interrupt the LLM immediately if not backchannel phrases
+  bot_stop_delay: 0.5  # a delay in seconds allowed between server and client audio output, so that the BotStopSpeaking signal is handled not too far away from the actual time that the user hears all audio output
+
+llm:
+  type: auto  # choices in ['auto', 'hf', 'vllm'], if `auto`, it will try to use vllm and fall back to hf if vllm not available
+  model: "nvidia/NVIDIA-Nemotron-Nano-9B-v2"  # model name for HF models, will be used via `AutoModelForCausalLM.from_pretrained()`
+  model_config: "./server_configs/llm_configs/nemotron_nano_v2.yaml"
+  # model: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
+  # model_config: "./server_configs/llm_configs/nemotron_nano_v3.yaml"
+  # if `model_config` is not specified, and the llm.model is not in model_registry.yaml, it will use `llm_configs/hf_llm_generic.yaml`
+  # model: "Qwen/Qwen2.5-7B-Instruct" 
+  # model_config: "./server_configs/llm_configs/qwen2.5-7B.yaml"
+  # model: "Qwen/Qwen3-8B"
+  # model_config: "./server_configs/llm_configs/qwen3-8B.yaml"
+  # model: meta-llama/Llama-3.1-8B-Instruct
+  device: "cuda"
+  enable_reasoning: false  # it's best to turn-off reasoning for lowest latency, setting it to True will use the same config ending with `_think.yaml` instead
+  # `system_prompt` is used as the sytem prompt to the LLM, please refer to differnt LLM webpage for spcial functions like enabling/disabling thinking
+  # system_prompt: /path/to/prompt.txt  # or use path to a txt file that contains a long prompt, for example in `../example_prompts/fast_bite.txt`
+  system_prompt: "You are a helpful AI agent named Lisa. Start by greeting the user with 'Hi, I'm Lisa, your helpful AI assistant. How can I help you today?'. Keep your responses concise and conversational since they will be spoken aloud. Avoid special characters. Use only simple, plain text sentences. Always punctuate your responses using standard sentence punctuation: commas, periods, question marks, exclamation points, etc. Always spell out numbers as words. You might also see speaker tags (<speaker_0>, <speaker_1>, etc.) in the user context. You should respond to the user based on the speaker tag and the context of that speaker. Do not include the speaker tags in your response, use them only to identify the speaker. Avoid using emoji in your response."
+
+tts:
+  type: nemo
+  model: "kokoro"
+  model_config: "./server_configs/tts_configs/kokoro_82M.yaml"
+  device: "cuda"
+  ignore_strings:  # strings/characters to ignore in TTS
+    - "*"
+    - "<unk>"

examples/voice_agent/server/server_configs/llm_configs/hf_llm_generic.yaml

@@ -0,0 +1,57 @@
+# This is an example config for setting up a generic HuggingFace LLM for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+# type: auto  # choices in ['auto', 'hf', 'vllm']
+# device: "cuda"
+dtype: bfloat16  # torch.dtype for LLM
+system_role: "system"  # role for system prompt, set it to `user` for models that do not support system prompt
+system_prompt_suffix: "/no_think"  # a string that would be appended to the system prompt, used to enable/disable thinking
+enable_tool_calling: False  # Tool calling is not supported by default, since different LLMs have diffferent implementations
+
+##############################
+## Common generation config ##
+##############################
+temperature: 0.7  # LLM sampling params
+top_k: 20  # LLM sampling params
+top_p: 0.8  # LLM sampling params
+min_p: 0.0  # LLM sampling params
+max_new_tokens: 256  # max num of output tokens from LLM
+##############################
+##### HuggingFace config #####
+##############################
+# Please refer to the model page of each HF LLM model to set following params properly.
+# kwargs that will be passed into tokenizer.apply_chat_template() function
+apply_chat_template_kwargs:  
+  add_generation_prompt: true  # This is required in most cases, do not change unless you're sure of it
+  tokenize: false  # This is required, do not change
+# kwargs that will be passed into model.generate() function of HF models
+generation_kwargs:  
+  temperature: ${llm.temperature}  # LLM sampling params
+  top_k: ${llm.top_k}  # LLM sampling params
+  top_p: ${llm.top_p}  # LLM sampling params
+  min_p: ${llm.min_p} # LLM sampling params
+  max_new_tokens: ${llm.max_new_tokens}  # max num of output tokens from LLM
+  do_sample: true # enable sampling
+##############################
+######## vLLM config #########
+##############################
+api_key: "EMPTY"
+base_url: "http://localhost:8000/v1"  
+# Set `start_vllm_on_init` to automatically start vllm server if it's not manually started yet
+start_vllm_on_init: true
+# Specifying vllm_server_params with the parameters you want to pass to the vllm server command `vllm serve $model $vllm_server_params`
+# Refer to each LLM's model page for details on the recommended parameters
+# It's recommended to stay with `--max-num-seqs` 1 as the voice agent currently supports one connection at a time.
+# You can try increasing the model's max context len `--max-model-len` if GPU memory allows, or decrease it if GPU OOM occurs.
+vllm_server_params: "--trust-remote-code --max-num-seqs 1 --gpu-memory-utilization 0.85"  
+# `params` are the inference parameters that would be passed into OpenAI API, 
+# please put additional model-specific parameters in `extra`
+vllm_generation_params:
+  frequency_penalty: 0.0  # Penalty for frequent tokens (-2.0 to 2.0).
+  presence_penalty: 0.0  # Penalty for new tokens (-2.0 to 2.0).
+  seed: 42  # Random seed for deterministic outputs.
+  temperature: ${llm.temperature}  # Sampling temperature (0.0 to 2.0).
+  top_k: null  # Top-k sampling parameter (currently ignored by OpenAI).
+  top_p: ${llm.top_p}  # Top-p (nucleus) sampling parameter (0.0 to 1.0).
+  max_completion_tokens: ${llm.max_new_tokens}  # max number of tokens to generate
+  extra: null  # additional model specific params can be specified in dict format

examples/voice_agent/server/server_configs/llm_configs/llama3.1-8B-instruct.yaml

@@ -0,0 +1,58 @@
+# This is an example config for setting up Qwen2.5-7B model for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+# type: auto  # choices in ['auto', 'hf', 'vllm']
+# model: meta-llama/Llama-3.1-8B-Instruct
+# device: "cuda"
+dtype: bfloat16  # torch.dtype for LLM
+system_role: "system"  # role for system prompt, set it to `user` for models that do not support system prompt
+system_prompt_suffix: null  # a string that would be appended to the system prompt, used to enable/disable thinking
+enable_tool_calling: False  # Tool calling is not supported yet in this specific configuration of the model
+
+##############################
+## Common generation config ##
+##############################
+temperature: 0.7  # LLM sampling params
+top_k: 20  # LLM sampling params
+top_p: 0.8  # LLM sampling params
+min_p: 0.0  # LLM sampling params
+max_new_tokens: 256  # max num of output tokens from LLM
+##############################
+##### HuggingFace config #####
+##############################
+# Please refer to the model page of each HF LLM model to set following params properly.
+# kwargs that will be passed into tokenizer.apply_chat_template() function
+apply_chat_template_kwargs:  
+  add_generation_prompt: true  # This is required in most cases, do not change unless you're sure of it
+  tokenize: false  # This is required, do not change
+# kwargs that will be passed into model.generate() function of HF models
+generation_kwargs:  
+  temperature: ${llm.temperature}  # LLM sampling params
+  top_k: ${llm.top_k}  # LLM sampling params
+  top_p: ${llm.top_p}  # LLM sampling params
+  min_p: ${llm.min_p} # LLM sampling params
+  max_new_tokens: ${llm.max_new_tokens}  # max num of output tokens from LLM
+  do_sample: true # enable sampling
+##############################
+######## vLLM config #########
+##############################
+api_key: "EMPTY"
+base_url: "http://localhost:8000/v1"  
+# Set `start_vllm_on_init` to automatically start vllm server if it's not manually started yet
+start_vllm_on_init: true
+# Specifying vllm_server_params with the parameters you want to pass to the vllm server command `vllm serve $model $vllm_server_params`
+# Refer to each LLM's model page for details on the recommended parameters
+# It's recommended to stay with `--max-num-seqs` 1 as the voice agent currently supports one connection at a time.
+# You can try increasing the model's max context len `--max-model-len` if GPU memory allows, or decrease it if GPU OOM occurs.
+vllm_server_params: "--trust-remote-code --max-num-seqs 1 --gpu-memory-utilization 0.85"  
+# `params` are the inference parameters that would be passed into OpenAI API, 
+# please put additional model-specific parameters in `extra`
+vllm_generation_params:
+  frequency_penalty: 0.0  # Penalty for frequent tokens (-2.0 to 2.0).
+  presence_penalty: 0.0  # Penalty for new tokens (-2.0 to 2.0).
+  seed: 42  # Random seed for deterministic outputs.
+  temperature: ${llm.temperature}  # Sampling temperature (0.0 to 2.0).
+  top_k: null  # Top-k sampling parameter (currently ignored by OpenAI).
+  top_p: ${llm.top_p}  # Top-p (nucleus) sampling parameter (0.0 to 1.0).
+  max_completion_tokens: ${llm.max_new_tokens}  # max number of tokens to generate
+  extra: null  # additional model specific params can be specified in dict format

examples/voice_agent/server/server_configs/llm_configs/nemotron_nano_v2.yaml

@@ -0,0 +1,58 @@
+# This is an example config for setting up nemotron_nano_v2 model for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+# model: "nvidia/NVIDIA-Nemotron-Nano-9B-v2"  # model name for HF models, will be used via `AutoModelForCausalLM.from_pretrained()`
+type: vllm  # Overwrite to vllm to enable tool calling, the HF backend currently doesn't support tools
+dtype: bfloat16  # torch.dtype for LLM
+device: "cuda"
+system_role: "system"  # role for system prompt, set it to `user` for models that do not support system prompt
+system_prompt_suffix: "Before responding to the user, check if the user request requires using external tools, and use the tools if they match with the user's intention. Otherwise, use your internal knowledge to answer the user's question. Do not use tools for casual conversation or when the tools don't fit the use cases. You should still try to address the user's request when it's not related to the provided tools. If you are provided with a set of tools, use them only when needed, do not limit your capabilities to the scope of the tools. If the purpose of a tool matches well with a user's request, always try to call the tool first. Conversation history should not limit your behavior on whether you can use tools. You must answer questions not related to the tools. /no_think"  # a string that would be appended to the system prompt, `/think` and `/no_think` are used to enable/disable thinking
+enable_tool_calling: True  # set to True since the vllm config below supports tool calling
+
+##############################
+## Common generation config ##
+##############################
+temperature: 0.7  # LLM sampling params
+top_k: 20  # LLM sampling params
+top_p: 0.9  # LLM sampling params
+min_p: 0.0  # LLM sampling params
+max_new_tokens: 256  # max num of output tokens from LLM
+##############################
+##### HuggingFace config #####
+##############################
+# Please refer to the model page of each HF LLM model to set following params properly.
+# kwargs that will be passed into tokenizer.apply_chat_template() function
+apply_chat_template_kwargs:  
+  add_generation_prompt: true  # This is required in most cases, do not change unless you're sure of it
+  tokenize: false  # This is required, do not change
+# kwargs that will be passed into model.generate() function of HF models
+generation_kwargs:  
+  temperature: ${llm.temperature}  # LLM sampling params
+  top_k: ${llm.top_k}  # LLM sampling params
+  top_p: ${llm.top_p}  # LLM sampling params
+  min_p: ${llm.min_p} # LLM sampling params
+  max_new_tokens: ${llm.max_new_tokens}  # max num of output tokens from LLM
+  do_sample: true # enable sampling
+##############################
+######## vLLM config #########
+##############################
+api_key: "EMPTY"
+base_url: "http://localhost:8000/v1"  
+# Set `start_vllm_on_init` to automatically start vllm server if it's not manually started yet
+start_vllm_on_init: true
+# Specifying vllm_server_params with the parameters you want to pass to the vllm server command `vllm serve $model $vllm_server_params`
+# Refer to each LLM's model page for details on the recommended parameters.
+# It's recommended to stay with `--max-num-seqs` 1 as the voice agent currently supports one connection at a time.
+# You can try increasing the model's max context len `--max-model-len` if GPU memory allows, or decrease it if GPU OOM occurs.
+vllm_server_params: "--trust-remote-code --enable-prefix-caching --max-num-seqs 1 --gpu-memory-utilization 0.85 --max-model-len 8192 --enable-auto-tool-choice --tool-parser-plugin server/parsers/nemotron_toolcall_parser_streaming.py --tool-call-parser nemotron_json --mamba_ssm_cache_dtype float32"  
+# `params` are the inference parameters that would be passed into OpenAI API, 
+# please put additional model-specific parameters in `extra`
+vllm_generation_params:
+  frequency_penalty: 0.0  # Penalty for frequent tokens (-2.0 to 2.0).
+  presence_penalty: 0.0  # Penalty for new tokens (-2.0 to 2.0).
+  seed: 42  # Random seed for deterministic outputs.
+  temperature: ${llm.temperature}  # Sampling temperature (0.0 to 2.0).
+  top_k: null  # Top-k sampling parameter (currently ignored by OpenAI).
+  top_p: ${llm.top_p}  # Top-p (nucleus) sampling parameter (0.0 to 1.0).
+  max_completion_tokens: ${llm.max_new_tokens}  # max number of tokens to generate
+  extra: null  # additional model specific params can be specified in dict format

examples/voice_agent/server/server_configs/llm_configs/nemotron_nano_v3.yaml

@@ -0,0 +1,62 @@
+# This is an example config for setting up nemotron_nano_v3 model for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+# model: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"  # model name for HF models, will be used via `AutoModelForCausalLM.from_pretrained()`
+type: vllm  # Overwrite to vllm to enable tool calling, the HF backend currently doesn't support tools
+dtype: auto  # torch.dtype for LLM, `auto` is only available for vllm
+device: "cuda"
+system_role: "system"  # role for system prompt, set it to `user` for models that do not support system prompt
+system_prompt_suffix: "Before responding to the user, check if the user request requires using external tools, and use the tools if they match with the user's intention. Otherwise, use your internal knowledge to answer the user's question. Do not use tools for casual conversation or when the tools don't fit the use cases. You should still try to address the user's request when it's not related to the provided tools. If you are provided with a set of tools, use them only when needed, do not limit your capabilities to the scope of the tools. If the purpose of a tool matches well with a user's request, always try to call the tool first. Conversation history should not limit your behavior on whether you can use tools. You must answer questions not related to the tools. Avoid any emoji in your response."  
+enable_tool_calling: true  # set to True since the vllm config below supports tool calling
+inject_dummy_user_message: true
+
+##############################
+## Common generation config ##
+##############################
+temperature: 0.6  # LLM sampling params
+top_k: 20  # LLM sampling params, NO effect for vllm
+top_p: 0.95  # LLM sampling params
+min_p: 0.0  # LLM sampling params
+max_new_tokens: 256  # max num of output tokens from LLM
+##############################
+##### HuggingFace config #####
+##############################
+# Please refer to the model page of each HF LLM model to set following params properly.
+# kwargs that will be passed into tokenizer.apply_chat_template() function
+apply_chat_template_kwargs:  
+  add_generation_prompt: true  # This is required in most cases, do not change unless you're sure of it
+  tokenize: false  # This is required, do not change
+# kwargs that will be passed into model.generate() function of HF models
+generation_kwargs:  
+  temperature: ${llm.temperature}  # LLM sampling params
+  top_k: ${llm.top_k}  # LLM sampling params
+  top_p: ${llm.top_p}  # LLM sampling params
+  min_p: ${llm.min_p} # LLM sampling params
+  max_new_tokens: ${llm.max_new_tokens}  # max num of output tokens from LLM
+  do_sample: true # enable sampling
+##############################
+######## vLLM config #########
+##############################
+api_key: "EMPTY"
+base_url: "http://localhost:8000/v1"  
+# Set `start_vllm_on_init` to automatically start vllm server if it's not manually started yet
+start_vllm_on_init: false
+# Specifying vllm_server_params with the parameters you want to pass to the vllm server command `vllm serve $model $vllm_server_params`
+# Refer to each LLM's model page for details on the recommended parameters.
+# It's recommended to stay with `--max-num-seqs` 1 as the voice agent currently supports one connection at a time.
+# You can try increasing the model's max context len `--max-model-len` if GPU memory allows, or decrease it if GPU OOM occurs.
+vllm_server_params: "--trust-remote-code --tensor-parallel-size 2 --enable-prefix-caching --max-num-seqs 1 --gpu-memory-utilization 0.8 --max-model-len 8192 --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser-plugin server/parsers/nano_v3_reasoning_parser.py --reasoning-parser nano_v3"  
+# `params` are the inference parameters that would be passed into OpenAI API, 
+# please put additional model-specific parameters in `extra`
+vllm_generation_params:
+  frequency_penalty: 0.0  # Penalty for frequent tokens (-2.0 to 2.0).
+  presence_penalty: 0.0  # Penalty for new tokens (-2.0 to 2.0).
+  seed: 42  # Random seed for deterministic outputs.
+  temperature: ${llm.temperature}  # Sampling temperature (0.0 to 2.0).
+  top_k: null  # Top-k sampling parameter (currently ignored by OpenAI).
+  top_p: ${llm.top_p}  # Top-p (nucleus) sampling parameter (0.0 to 1.0).
+  max_completion_tokens: ${llm.max_new_tokens}  # max number of tokens to generate
+  extra:  # additional model specific params can be specified in dict format
+    extra_body:
+      chat_template_kwargs:
+        enable_thinking: False  # disable thinking

examples/voice_agent/server/server_configs/llm_configs/qwen2.5-7B.yaml

@@ -0,0 +1,58 @@
+# This is an example config for setting up Qwen2.5-7B model for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+# type: hf  # choices in ['auto', 'hf', 'vllm']
+# model: Qwen/Qwen2.5-7B-Instruct
+# device: "cuda"
+dtype: bfloat16  # torch.dtype for LLM
+system_role: "system"  # role for system prompt, set it to `user` for models that do not support system prompt
+system_prompt_suffix: null  # a string that would be appended to the system prompt, used to enable/disable thinking
+enable_tool_calling: False  # Tool calling is not supported yet in this specific configuration of the model
+
+##############################
+## Common generation config ##
+##############################
+temperature: 0.7  # LLM sampling params
+top_k: 20  # LLM sampling params
+top_p: 0.8  # LLM sampling params
+min_p: 0.0  # LLM sampling params
+max_new_tokens: 256  # max num of output tokens from LLM
+##############################
+##### HuggingFace config #####
+##############################
+# Please refer to the model page of each HF LLM model to set following params properly.
+# kwargs that will be passed into tokenizer.apply_chat_template() function
+apply_chat_template_kwargs:  
+  add_generation_prompt: true  # This is required in most cases, do not change unless you're sure of it
+  tokenize: false  # This is required, do not change
+# kwargs that will be passed into model.generate() function of HF models
+generation_kwargs:  
+  temperature: ${llm.temperature}  # LLM sampling params
+  top_k: ${llm.top_k}  # LLM sampling params
+  top_p: ${llm.top_p}  # LLM sampling params
+  min_p: ${llm.min_p} # LLM sampling params
+  max_new_tokens: ${llm.max_new_tokens}  # max num of output tokens from LLM
+  do_sample: true # enable sampling
+##############################
+######## vLLM config #########
+##############################
+api_key: "EMPTY"
+base_url: "http://localhost:8000/v1"  
+# Set `start_vllm_on_init` to automatically start vllm server if it's not manually started yet
+start_vllm_on_init: true
+# Specifying vllm_server_params with the parameters you want to pass to the vllm server command `vllm serve $model $vllm_server_params`
+# Refer to each LLM's model page for details on the recommended parameters
+# It's recommended to stay with `--max-num-seqs` 1 as the voice agent currently supports one connection at a time.
+# You can try increasing the model's max context len `--max-model-len` if GPU memory allows, or decrease it if GPU OOM occurs.
+vllm_server_params: "--trust-remote-code --max-num-seqs 1 --gpu-memory-utilization 0.85"  
+# `params` are the inference parameters that would be passed into OpenAI API, 
+# please put additional model-specific parameters in `extra`
+vllm_generation_params:
+  frequency_penalty: 0.0  # Penalty for frequent tokens (-2.0 to 2.0).
+  presence_penalty: 0.0  # Penalty for new tokens (-2.0 to 2.0).
+  seed: 42  # Random seed for deterministic outputs.
+  temperature: ${llm.temperature}  # Sampling temperature (0.0 to 2.0).
+  top_k: null  # Top-k sampling parameter (currently ignored by OpenAI).
+  top_p: ${llm.top_p}  # Top-p (nucleus) sampling parameter (0.0 to 1.0).
+  max_completion_tokens: ${llm.max_new_tokens}  # max number of tokens to generate
+  extra: null  # additional model specific params can be specified in dict format

examples/voice_agent/server/server_configs/llm_configs/qwen3-8B.yaml

@@ -0,0 +1,58 @@
+# This is an example config for setting up Qwen3-8B model for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+# type: auto  # choices in ['auto', 'hf', 'vllm']
+# model: "Qwen/Qwen3-8B"  # model name for HF models, will be used via `AutoModelForCausalLM.from_pretrained()`
+# device: "cuda"
+dtype: bfloat16  # torch.dtype for LLM
+system_role: "system"  # role for system prompt, set it to `user` for models that do not support system prompt
+system_prompt_suffix: "/no_think"  # a string that would be appended to the system prompt, used to enable/disable thinking
+enable_tool_calling: False  # Tool calling is not supported yet in this specific configuration of the model
+
+##############################
+## Common generation config ##
+##############################
+temperature: 0.7  # LLM sampling params
+top_k: 20  # LLM sampling params
+top_p: 0.8  # LLM sampling params
+min_p: 0.0  # LLM sampling params
+max_new_tokens: 256  # max num of output tokens from LLM
+##############################
+##### HuggingFace config #####
+##############################
+# Please refer to the model page of each HF LLM model to set following params properly.
+# kwargs that will be passed into tokenizer.apply_chat_template() function
+apply_chat_template_kwargs:  
+  add_generation_prompt: true  # This is required in most cases, do not change unless you're sure of it
+  tokenize: false  # This is required, do not change
+# kwargs that will be passed into model.generate() function of HF models
+generation_kwargs:  
+  temperature: ${llm.temperature}  # LLM sampling params
+  top_k: ${llm.top_k}  # LLM sampling params
+  top_p: ${llm.top_p}  # LLM sampling params
+  min_p: ${llm.min_p} # LLM sampling params
+  max_new_tokens: ${llm.max_new_tokens}  # max num of output tokens from LLM
+  do_sample: true # enable sampling
+##############################
+######## vLLM config #########
+##############################
+api_key: "EMPTY"
+base_url: "http://localhost:8000/v1"  
+# Set `start_vllm_on_init` to automatically start vllm server if it's not manually started yet
+start_vllm_on_init: true
+# Specifying vllm_server_params with the parameters you want to pass to the vllm server command `vllm serve $model $vllm_server_params`
+# Refer to each LLM's model page for details on the recommended parameters
+# It's recommended to stay with `--max-num-seqs` 1 as the voice agent currently supports one connection at a time.
+# You can try increasing the model's max context len `--max-model-len` if GPU memory allows, or decrease it if GPU OOM occurs.
+vllm_server_params: "--trust-remote-code --max-num-seqs 1 --gpu-memory-utilization 0.85"  
+# `params` are the inference parameters that would be passed into OpenAI API, 
+# please put additional model-specific parameters in `extra`
+vllm_generation_params:
+  frequency_penalty: 0.0  # Penalty for frequent tokens (-2.0 to 2.0).
+  presence_penalty: 0.0  # Penalty for new tokens (-2.0 to 2.0).
+  seed: 42  # Random seed for deterministic outputs.
+  temperature: ${llm.temperature}  # Sampling temperature (0.0 to 2.0).
+  top_k: null  # Top-k sampling parameter (currently ignored by OpenAI).
+  top_p: ${llm.top_p}  # Top-p (nucleus) sampling parameter (0.0 to 1.0).
+  max_completion_tokens: ${llm.max_new_tokens}  # max number of tokens to generate
+  extra: null  # additional model specific params can be specified in dict format

examples/voice_agent/server/server_configs/llm_configs/qwen3-8B_think.yaml

@@ -0,0 +1,58 @@
+# This is an example config for setting up Qwen3-8B model in thinking mode for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+# type: auto  # choices in ['auto', 'hf', 'vllm']
+# model: "Qwen/Qwen3-8B"  # model name for HF models, will be used via `AutoModelForCausalLM.from_pretrained()`
+# device: "cuda"
+dtype: bfloat16  # torch.dtype for LLM
+system_role: "system"  # role for system prompt, set it to `user` for models that do not support system prompt
+system_prompt_suffix: "/think"  # a string that would be appended to the system prompt, used to enable/disable thinking
+enable_tool_calling: False  # Tool calling is not supported yet in this specific configuration of the model
+
+##############################
+## Common generation config ##
+##############################
+temperature: 0.7  # LLM sampling params
+top_k: 20  # LLM sampling params
+top_p: 0.8  # LLM sampling params
+min_p: 0.0  # LLM sampling params
+max_new_tokens: 256  # max num of output tokens from LLM
+##############################
+##### HuggingFace config #####
+##############################
+# Please refer to the model page of each HF LLM model to set following params properly.
+# kwargs that will be passed into tokenizer.apply_chat_template() function
+apply_chat_template_kwargs:  
+  add_generation_prompt: true  # This is required in most cases, do not change unless you're sure of it
+  tokenize: false  # This is required, do not change
+# kwargs that will be passed into model.generate() function of HF models
+generation_kwargs:  
+  temperature: ${llm.temperature}  # LLM sampling params
+  top_k: ${llm.top_k}  # LLM sampling params
+  top_p: ${llm.top_p}  # LLM sampling params
+  min_p: ${llm.min_p} # LLM sampling params
+  max_new_tokens: ${llm.max_new_tokens}  # max num of output tokens from LLM
+  do_sample: true # enable sampling
+##############################
+######## vLLM config #########
+##############################
+api_key: "EMPTY"
+base_url: "http://localhost:8000/v1"  
+# Set `start_vllm_on_init` to automatically start vllm server if it's not manually started yet
+start_vllm_on_init: true
+# Specifying vllm_server_params with the parameters you want to pass to the vllm server command `vllm serve $model $vllm_server_params`
+# Refer to each LLM's model page for details on the recommended parameters
+# It's recommended to stay with `--max-num-seqs` 1 as the voice agent currently supports one connection at a time.
+# You can try increasing the model's max context len `--max-model-len` if GPU memory allows, or decrease it if GPU OOM occurs.
+vllm_server_params: "--trust-remote-code --max-num-seqs 1 --gpu-memory-utilization 0.85"  
+# `params` are the inference parameters that would be passed into OpenAI API, 
+# please put additional model-specific parameters in `extra`
+vllm_generation_params:
+  frequency_penalty: 0.0  # Penalty for frequent tokens (-2.0 to 2.0).
+  presence_penalty: 0.0  # Penalty for new tokens (-2.0 to 2.0).
+  seed: 42  # Random seed for deterministic outputs.
+  temperature: ${llm.temperature}  # Sampling temperature (0.0 to 2.0).
+  top_k: null  # Top-k sampling parameter (currently ignored by OpenAI).
+  top_p: ${llm.top_p}  # Top-p (nucleus) sampling parameter (0.0 to 1.0).
+  max_completion_tokens: ${llm.max_new_tokens}  # max number of tokens to generate
+  extra: null  # additional model specific params can be specified in dict format

examples/voice_agent/server/server_configs/stt_configs/nemo_cache_aware_streaming.yaml

@@ -0,0 +1,6 @@
+# This is an example config for setting up a NeMo cache-aware streaming ASR model for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+  att_context_size: [70,1] # left and right attention context sizes for streaming ASR
+  frame_len_in_secs: 0.08  # default for FastConformer, do not change unless using other architechtures
+  audio_chunk_size_in_secs: 0.08 

examples/voice_agent/server/server_configs/tts_configs/kokoro_82M.yaml

@@ -0,0 +1,15 @@
+# This is an example config for setting up a NeMo FastPitch-Hifigan TTS model for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+main_model_id: "hexgrad/Kokoro-82M"
+sub_model_id: "af_heart" # "af_heart" "af_bella" "am_fenrir" "am_michael"
+device: "cuda"
+speed: 1.25 # Speaking rate
+extra_separator:  # a list of additional punctuations to chunk LLM response into segments for faster TTS output, e.g., ",". Set to `null` to use default behavior 
+  - ','
+  - '\n'
+  - "."
+  - "?"
+  - "!"
+  - ";"
+think_tokens: ["<think>", "</think>"]  # specify them to avoid TTS for thinking process, set to `null` to allow thinking out loud

examples/voice_agent/server/server_configs/tts_configs/magpie_tts_multilingual_357m.yaml

@@ -0,0 +1,17 @@
+# This is an example config for setting up a nvidia/magpie_tts_multilingual_357m TTS model for a NeMo Voice Agent server.
+# Please refer to https://huggingface.co/nvidia/magpie_tts_multilingual_357m for more details
+
+main_model_id: "nvidia/magpie_tts_multilingual_357m"
+sub_model_id: null
+language: "en"
+speaker: "Sofia"  # choices in ["Sofia", "Aria", "John", "Jason", "Leo"]
+apply_TN: false  # whether to apply text normalization
+device: "cuda"
+extra_separator:  # a list of additional punctuations to chunk LLM response into segments for faster TTS output, e.g., ",". Set to `null` to use default behavior 
+  - ','
+  - '\n'
+  - "."
+  - "?"
+  - "!"
+  - ";"
+think_tokens: ["<think>", "</think>"]  # specify them to avoid TTS for thinking process, set to `null` to allow thinking out loud

examples/voice_agent/server/server_configs/tts_configs/nemo_fastpitch-hifigan.yaml

@@ -0,0 +1,14 @@
+# This is an example config for setting up a NeMo FastPitch-Hifigan TTS model for a NeMo Voice Agent server.
+# Please refer to https://github.com/NVIDIA-NeMo/NeMo/tree/main/examples/voice_agent/README.md for more details
+
+main_model_id: "nvidia/tts_en_fastpitch"
+sub_model_id: "nvidia/tts_hifigan"
+device: "cuda"
+extra_separator:  # a list of additional punctuations to chunk LLM response into segments for faster TTS output, e.g., ",". Set to `null` to use default behavior 
+  - ','
+  - '\n'
+  - "."
+  - "?"
+  - "!"
+  - ";"
+think_tokens: ["<think>", "</think>"]  # specify them to avoid TTS for thinking process, set to `null` to allow thinking out loud

examples/voice_agent/server/websocket_url.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from urllib.parse import urlsplit
+
+
+def _normalize_websocket_scheme(scheme: str) -> str:
+    scheme = scheme.strip().lower()
+    if scheme not in {"ws", "wss"}:
+        raise ValueError("WEBSOCKET_SCHEME must be either 'ws' or 'wss'")
+    return scheme
+
+
+def _normalize_websocket_host(host: str) -> str:
+    host = host.strip()
+    if not host:
+        raise ValueError("SERVER_PUBLIC_HOST must not be empty")
+    if "://" in host:
+        parsed_host = urlsplit(host).hostname
+        if not parsed_host:
+            raise ValueError("SERVER_PUBLIC_HOST must include a host name")
+        host = parsed_host
+    return host
+
+
+def build_websocket_url(host: str, port: int, scheme: str = "ws") -> str:
+    """Build the client-facing WebSocket URL from trusted server configuration."""
+    scheme = _normalize_websocket_scheme(scheme)
+    host = _normalize_websocket_host(host)
+    if ":" in host and not host.startswith("["):
+        host = f"[{host}]"
+    return f"{scheme}://{host}:{port}"