chore: import upstream snapshot with attribution

examples/realtime/app/README.md

@@ -0,0 +1,60 @@
+# Realtime Demo App
+
+A web-based realtime voice assistant demo with a FastAPI backend and HTML/JS frontend.
+
+## Installation
+
+Install the required dependencies:
+
+```bash
+uv add fastapi uvicorn websockets
+```
+
+## Usage
+
+Start the application with a single command:
+
+```bash
+cd examples/realtime/app && uv run python server.py
+```
+
+Then open your browser to: http://localhost:8000
+
+### Debugging Realtime usage
+
+Set `LOG_LEVEL=DEBUG` to log the raw `response.done` usage, the typed per-response usage with modality details, and the cumulative session usage:
+
+```bash
+cd examples/realtime/app && LOG_LEVEL=DEBUG uv run python server.py
+```
+
+The debug logs include concise summaries for server, model, session, history, tool, handoff, error, and usage events. Audio frames and high-volume delta events are omitted, and transcript content is not logged. Uvicorn and WebSocket protocol logging remain at INFO so `LOG_LEVEL=DEBUG` does not dump wire payloads.
+
+## Customization
+
+To use the same UI with your own agents, edit `agent.py` and ensure get_starting_agent() returns the right starting agent for your use case.
+
+## How to Use
+
+1. Click **Connect** to establish a realtime session
+2. Audio capture starts automatically - just speak naturally
+3. Click the **Mic On/Off** button to mute/unmute your microphone
+4. To send an image, enter an optional prompt and click **🖼️ Send Image** (select a file)
+5. Watch the conversation unfold in the left pane (image thumbnails are shown)
+6. Monitor raw events in the right pane (click to expand/collapse)
+7. Click **Disconnect** when done
+
+### Human-in-the-loop approvals
+
+- The seat update tool now requires approval. When the agent wants to run it, the browser shows a `window.confirm` dialog so you can allow or deny the tool call before it executes.
+
+## Architecture
+
+-   **Backend**: FastAPI server with WebSocket connections for real-time communication
+-   **Session Management**: Each connection gets a unique session with the OpenAI Realtime API
+- **Image Inputs**: The UI uploads images and the server forwards a `conversation.item.create` event with `input_image` (plus optional `input_text`), followed by `response.create` to start the model response. The messages pane renders image bubbles for `input_image` content.
+-   **Audio Processing**: 24kHz mono audio capture and playback
+-   **Event Handling**: Full event stream processing with transcript generation
+-   **Frontend**: Vanilla JavaScript with clean, responsive CSS
+
+The demo showcases the core patterns for building realtime voice applications with the OpenAI Agents SDK.

examples/realtime/app/agent.py

@@ -0,0 +1,107 @@
+import asyncio
+
+from agents import function_tool
+from agents.extensions.handoff_prompt import RECOMMENDED_PROMPT_PREFIX
+from agents.realtime import RealtimeAgent, realtime_handoff
+
+"""
+When running the UI example locally, you can edit this file to change the setup. THe server
+will use the agent returned from get_starting_agent() as the starting agent."""
+
+### TOOLS
+
+
+@function_tool(
+    name_override="faq_lookup_tool", description_override="Lookup frequently asked questions."
+)
+async def faq_lookup_tool(question: str) -> str:
+    # Simulate a slow API call
+    await asyncio.sleep(3)
+
+    q = question.lower()
+    if "wifi" in q or "wi-fi" in q:
+        return "We have free wifi on the plane, join Airline-Wifi"
+    elif "bag" in q or "baggage" in q:
+        return (
+            "You are allowed to bring one bag on the plane. "
+            "It must be under 50 pounds and 22 inches x 14 inches x 9 inches."
+        )
+    elif "seats" in q or "plane" in q:
+        return (
+            "There are 120 seats on the plane. "
+            "There are 22 business class seats and 98 economy seats. "
+            "Exit rows are rows 4 and 16. "
+            "Rows 5-8 are Economy Plus, with extra legroom. "
+        )
+    return "I'm sorry, I don't know the answer to that question."
+
+
+@function_tool(needs_approval=True)
+async def update_seat(confirmation_number: str, new_seat: str) -> str:
+    """
+    Update the seat for a given confirmation number.
+
+    Args:
+        confirmation_number: The confirmation number for the flight.
+        new_seat: The new seat to update to.
+    """
+    return f"Updated seat to {new_seat} for confirmation number {confirmation_number}"
+
+
+@function_tool
+def get_weather(city: str) -> str:
+    """Get the weather in a city."""
+    return f"The weather in {city} is sunny."
+
+
+faq_agent = RealtimeAgent(
+    name="FAQ Agent",
+    handoff_description="A helpful agent that can answer questions about the airline.",
+    instructions=f"""{RECOMMENDED_PROMPT_PREFIX}
+    You are an FAQ agent. If you are speaking to a customer, you probably were transferred to from the triage agent.
+    Use the following routine to support the customer.
+    # Routine
+    1. Identify the last question asked by the customer.
+    2. Use the faq lookup tool to answer the question. Do not rely on your own knowledge.
+    3. If you cannot answer the question, transfer back to the triage agent.""",
+    tools=[faq_lookup_tool],
+)
+
+seat_booking_agent = RealtimeAgent(
+    name="Seat Booking Agent",
+    handoff_description="A helpful agent that can update a seat on a flight.",
+    instructions=f"""{RECOMMENDED_PROMPT_PREFIX}
+    You are a seat booking agent. If you are speaking to a customer, you probably were transferred to from the triage agent.
+    Use the following routine to support the customer.
+    # Routine
+    1. Ask for their confirmation number.
+    2. Ask the customer what their desired seat number is.
+    3. Use the update seat tool to update the seat on the flight.
+    If the customer asks a question that is not related to the routine, transfer back to the triage agent. """,
+    tools=[update_seat],
+)
+
+triage_agent = RealtimeAgent(
+    name="Triage Agent",
+    handoff_description="A triage agent that can delegate a customer's request to the appropriate agent.",
+    instructions=(
+        f"{RECOMMENDED_PROMPT_PREFIX} "
+        "You are a helpful triaging agent. You can use your tools to delegate questions to other appropriate agents."
+    ),
+    tools=[get_weather],
+    handoffs=[
+        realtime_handoff(faq_agent, tool_name_override="transfer_to_faq_agent"),
+        realtime_handoff(seat_booking_agent, tool_name_override="transfer_to_seat_booking_agent"),
+    ],
+)
+
+faq_agent.handoffs.append(
+    realtime_handoff(triage_agent, tool_name_override="transfer_to_triage_agent")
+)
+seat_booking_agent.handoffs.append(
+    realtime_handoff(triage_agent, tool_name_override="transfer_to_triage_agent")
+)
+
+
+def get_starting_agent() -> RealtimeAgent:
+    return triage_agent

examples/realtime/app/server.py

@@ -0,0 +1,590 @@
+import asyncio
+import base64
+import json
+import logging
+import os
+import struct
+from contextlib import asynccontextmanager
+from dataclasses import asdict
+from typing import TYPE_CHECKING, Any
+
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+from typing_extensions import assert_never
+
+from agents.realtime import RealtimeRunner, RealtimeSession, RealtimeSessionEvent
+from agents.realtime.config import RealtimeUserInputMessage
+from agents.realtime.items import RealtimeItem
+from agents.realtime.model import RealtimeModelConfig
+from agents.realtime.model_events import (
+    RealtimeModelItemUpdatedEvent,
+    RealtimeModelRawServerEvent,
+    RealtimeModelUsageEvent,
+)
+from agents.realtime.model_inputs import RealtimeModelSendRawMessage
+
+# Import TwilioHandler class - handle both module and package use cases
+if TYPE_CHECKING:
+    # For type checking, use the relative import
+    from .agent import get_starting_agent
+else:
+    # At runtime, try both import styles
+    try:
+        # Try relative import first (when used as a package)
+        from .agent import get_starting_agent
+    except ImportError:
+        # Fall back to direct import (when run as a script)
+        from agent import get_starting_agent
+
+
+_requested_log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+_log_level = getattr(logging, _requested_log_level, logging.INFO)
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+logger.setLevel(_log_level)
+
+
+class RealtimeWebSocketManager:
+    def __init__(self):
+        self.active_sessions: dict[str, RealtimeSession] = {}
+        self.session_contexts: dict[str, Any] = {}
+        self.websockets: dict[str, WebSocket] = {}
+
+    async def connect(self, websocket: WebSocket, session_id: str):
+        await websocket.accept()
+        self.websockets[session_id] = websocket
+
+        agent = get_starting_agent()
+        runner = RealtimeRunner(agent)
+        # If you want to customize the runner behavior, you can pass options:
+        # runner_config = RealtimeRunConfig(async_tool_calls=False)
+        # runner = RealtimeRunner(agent, config=runner_config)
+        model_config: RealtimeModelConfig = {
+            "initial_model_settings": {
+                "model_name": "gpt-realtime-2.1",
+                "turn_detection": {
+                    "type": "server_vad",
+                    "prefix_padding_ms": 300,
+                    "silence_duration_ms": 500,
+                    "interrupt_response": True,
+                    "create_response": True,
+                },
+            },
+        }
+        session_context = await runner.run(model_config=model_config)
+        session = await session_context.__aenter__()
+        self.active_sessions[session_id] = session
+        self.session_contexts[session_id] = session_context
+
+        # Start event processing task
+        asyncio.create_task(self._process_events(session_id))
+
+    async def disconnect(self, session_id: str):
+        if session_id in self.session_contexts:
+            await self.session_contexts[session_id].__aexit__(None, None, None)
+            del self.session_contexts[session_id]
+        if session_id in self.active_sessions:
+            del self.active_sessions[session_id]
+        if session_id in self.websockets:
+            del self.websockets[session_id]
+
+    async def send_audio(self, session_id: str, audio_bytes: bytes):
+        if session_id in self.active_sessions:
+            await self.active_sessions[session_id].send_audio(audio_bytes)
+
+    async def send_client_event(self, session_id: str, event: dict[str, Any]):
+        """Send a raw client event to the underlying realtime model."""
+        session = self.active_sessions.get(session_id)
+        if not session:
+            return
+        await session.model.send_event(
+            RealtimeModelSendRawMessage(
+                message={
+                    "type": event["type"],
+                    "other_data": {k: v for k, v in event.items() if k != "type"},
+                }
+            )
+        )
+
+    async def send_user_message(self, session_id: str, message: RealtimeUserInputMessage):
+        """Send a structured user message via the higher-level API (supports input_image)."""
+        session = self.active_sessions.get(session_id)
+        if not session:
+            return
+        await session.send_message(message)  # delegates to RealtimeModelSendUserInput path
+
+    async def approve_tool_call(self, session_id: str, call_id: str, *, always: bool = False):
+        """Approve a pending tool call for a session."""
+        session = self.active_sessions.get(session_id)
+        if not session:
+            return
+        await session.approve_tool_call(call_id, always=always)
+
+    async def reject_tool_call(self, session_id: str, call_id: str, *, always: bool = False):
+        """Reject a pending tool call for a session."""
+        session = self.active_sessions.get(session_id)
+        if not session:
+            return
+        await session.reject_tool_call(call_id, always=always)
+
+    async def interrupt(self, session_id: str) -> None:
+        """Interrupt current model playback/response for a session."""
+        session = self.active_sessions.get(session_id)
+        if not session:
+            return
+        await session.interrupt()
+
+    async def _process_events(self, session_id: str):
+        try:
+            session = self.active_sessions[session_id]
+            websocket = self.websockets[session_id]
+
+            async for event in session:
+                self._log_debug_event(session_id, event)
+                event_data = await self._serialize_event(event)
+                await websocket.send_text(json.dumps(event_data))
+        except Exception as e:
+            print(e)
+            logger.error(f"Error processing events for session {session_id}: {e}")
+
+    def _log_debug_event(self, session_id: str, event: RealtimeSessionEvent) -> None:
+        """Log useful event summaries without noisy audio or delta payloads."""
+        if not logger.isEnabledFor(logging.DEBUG):
+            return
+
+        if event.type == "audio":
+            return
+        if event.type == "audio_end":
+            return
+        if event.type == "audio_interrupted":
+            return
+
+        if event.type == "raw_model_event":
+            self._log_debug_model_event(session_id, event)
+            return
+
+        event_summary: dict[str, Any] = {"type": event.type}
+        if event.type == "agent_start":
+            event_summary["agent"] = event.agent.name
+        elif event.type == "agent_end":
+            event_summary["agent"] = event.agent.name
+        elif event.type == "handoff":
+            event_summary["from_agent"] = event.from_agent.name
+            event_summary["to_agent"] = event.to_agent.name
+        elif event.type == "tool_start":
+            event_summary["tool"] = event.tool.name
+        elif event.type == "tool_end":
+            event_summary["tool"] = event.tool.name
+        elif event.type == "tool_approval_required":
+            event_summary.update(
+                {
+                    "agent": event.agent.name,
+                    "tool": event.tool.name,
+                    "call_id": event.call_id,
+                }
+            )
+        elif event.type == "history_updated":
+            event_summary["item_count"] = len(event.history)
+            if event.history:
+                event_summary["last_item"] = self._item_debug_summary(event.history[-1])
+        elif event.type == "history_added":
+            event_summary["item"] = self._item_debug_summary(event.item)
+        elif event.type == "guardrail_tripped":
+            event_summary["guardrails"] = [
+                result.guardrail.name for result in event.guardrail_results
+            ]
+        elif event.type == "error":
+            event_summary["error"] = str(event.error)
+        elif event.type == "input_audio_timeout_triggered":
+            pass
+        else:
+            assert_never(event)
+
+        logger.debug("Realtime session event session_id=%s event=%s", session_id, event_summary)
+
+    def _log_debug_model_event(self, session_id: str, event: Any) -> None:
+        model_event = event.data
+        if model_event.type in {"audio", "transcript_delta"}:
+            return
+
+        if isinstance(model_event, RealtimeModelRawServerEvent):
+            raw_event = model_event.data
+            if not isinstance(raw_event, dict):
+                return
+
+            raw_type = raw_event.get("type")
+            if isinstance(raw_type, str) and raw_type.endswith(".delta"):
+                return
+
+            raw_summary: dict[str, Any] = {
+                "type": raw_type,
+                "event_id": raw_event.get("event_id"),
+            }
+            response = raw_event.get("response")
+            if isinstance(response, dict):
+                raw_summary["response_id"] = response.get("id")
+                raw_summary["response_status"] = response.get("status")
+            item = raw_event.get("item")
+            if isinstance(item, dict):
+                raw_summary["item_id"] = item.get("id")
+                raw_summary["item_type"] = item.get("type")
+            else:
+                raw_summary["item_id"] = raw_event.get("item_id")
+
+            raw_summary = {key: value for key, value in raw_summary.items() if value is not None}
+
+            if raw_type == "response.done":
+                raw_summary["usage"] = response.get("usage") if isinstance(response, dict) else None
+                logger.debug(
+                    "Realtime raw response completed session_id=%s event=%s",
+                    session_id,
+                    raw_summary,
+                )
+            else:
+                logger.debug(
+                    "Realtime raw server event session_id=%s event=%s",
+                    session_id,
+                    raw_summary,
+                )
+            return
+
+        if isinstance(model_event, RealtimeModelUsageEvent):
+            self._log_debug_usage_event(session_id, event, model_event)
+            return
+
+        model_summary: dict[str, Any] = {"type": model_event.type}
+        for field_name in (
+            "item_id",
+            "response_id",
+            "call_id",
+            "name",
+            "status",
+            "content_index",
+        ):
+            value = getattr(model_event, field_name, None)
+            if value is not None:
+                model_summary[field_name] = value
+        if isinstance(model_event, RealtimeModelItemUpdatedEvent):
+            model_summary["item"] = self._item_debug_summary(model_event.item)
+
+        logger.debug(
+            "Realtime model event session_id=%s event=%s",
+            session_id,
+            model_summary,
+        )
+
+    def _log_debug_usage_event(
+        self,
+        session_id: str,
+        event: Any,
+        model_event: RealtimeModelUsageEvent,
+    ) -> None:
+        response_usage = model_event.usage
+        cumulative_usage = event.info.context.usage
+        logger.debug(
+            "Realtime typed response usage session_id=%s aggregate=%s "
+            "input_details=%s output_details=%s",
+            session_id,
+            {
+                "requests": response_usage.requests,
+                "input_tokens": response_usage.input_tokens,
+                "output_tokens": response_usage.output_tokens,
+                "total_tokens": response_usage.total_tokens,
+                "cached_input_tokens": response_usage.input_tokens_details.cached_tokens,
+            },
+            (
+                asdict(model_event.input_tokens_details)
+                if model_event.input_tokens_details is not None
+                else None
+            ),
+            (
+                asdict(model_event.output_tokens_details)
+                if model_event.output_tokens_details is not None
+                else None
+            ),
+        )
+        logger.debug(
+            "Realtime cumulative session usage session_id=%s aggregate=%s",
+            session_id,
+            {
+                "requests": cumulative_usage.requests,
+                "input_tokens": cumulative_usage.input_tokens,
+                "output_tokens": cumulative_usage.output_tokens,
+                "total_tokens": cumulative_usage.total_tokens,
+                "cached_input_tokens": cumulative_usage.input_tokens_details.cached_tokens,
+            },
+        )
+
+    @staticmethod
+    def _item_debug_summary(item: RealtimeItem) -> dict[str, Any]:
+        content = getattr(item, "content", None)
+        return {
+            "item_id": item.item_id,
+            "type": item.type,
+            "role": getattr(item, "role", None),
+            "status": getattr(item, "status", None),
+            "content_types": (
+                [getattr(part, "type", type(part).__name__) for part in content]
+                if isinstance(content, list)
+                else []
+            ),
+        }
+
+    def _sanitize_history_item(self, item: RealtimeItem) -> dict[str, Any]:
+        """Remove large binary payloads from history items while keeping transcripts."""
+        item_dict = item.model_dump()
+        content = item_dict.get("content")
+        if isinstance(content, list):
+            sanitized_content: list[Any] = []
+            for part in content:
+                if isinstance(part, dict):
+                    sanitized_part = part.copy()
+                    if sanitized_part.get("type") in {"audio", "input_audio"}:
+                        sanitized_part.pop("audio", None)
+                    sanitized_content.append(sanitized_part)
+                else:
+                    sanitized_content.append(part)
+            item_dict["content"] = sanitized_content
+        return item_dict
+
+    async def _serialize_event(self, event: RealtimeSessionEvent) -> dict[str, Any]:
+        base_event: dict[str, Any] = {
+            "type": event.type,
+        }
+
+        if event.type == "agent_start":
+            base_event["agent"] = event.agent.name
+        elif event.type == "agent_end":
+            base_event["agent"] = event.agent.name
+        elif event.type == "handoff":
+            base_event["from"] = event.from_agent.name
+            base_event["to"] = event.to_agent.name
+        elif event.type == "tool_start":
+            base_event["tool"] = event.tool.name
+        elif event.type == "tool_end":
+            base_event["tool"] = event.tool.name
+            base_event["output"] = str(event.output)
+        elif event.type == "tool_approval_required":
+            base_event["tool"] = event.tool.name
+            base_event["call_id"] = event.call_id
+            base_event["arguments"] = event.arguments
+            base_event["agent"] = event.agent.name
+        elif event.type == "audio":
+            base_event["audio"] = base64.b64encode(event.audio.data).decode("utf-8")
+        elif event.type == "audio_interrupted":
+            pass
+        elif event.type == "audio_end":
+            pass
+        elif event.type == "history_updated":
+            base_event["history"] = [self._sanitize_history_item(item) for item in event.history]
+        elif event.type == "history_added":
+            # Provide the added item so the UI can render incrementally.
+            try:
+                base_event["item"] = self._sanitize_history_item(event.item)
+            except Exception:
+                base_event["item"] = None
+        elif event.type == "guardrail_tripped":
+            base_event["guardrail_results"] = [
+                {"name": result.guardrail.name} for result in event.guardrail_results
+            ]
+        elif event.type == "raw_model_event":
+            base_event["raw_model_event"] = {
+                "type": event.data.type,
+            }
+        elif event.type == "error":
+            base_event["error"] = str(event.error) if hasattr(event, "error") else "Unknown error"
+        elif event.type == "input_audio_timeout_triggered":
+            pass
+        else:
+            assert_never(event)
+
+        return base_event
+
+
+manager = RealtimeWebSocketManager()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    yield
+
+
+app = FastAPI(lifespan=lifespan)
+
+
+@app.websocket("/ws/{session_id}")
+async def websocket_endpoint(websocket: WebSocket, session_id: str):
+    await manager.connect(websocket, session_id)
+    image_buffers: dict[str, dict[str, Any]] = {}
+    try:
+        while True:
+            data = await websocket.receive_text()
+            message = json.loads(data)
+
+            if message["type"] == "audio":
+                # Convert int16 array to bytes
+                int16_data = message["data"]
+                audio_bytes = struct.pack(f"{len(int16_data)}h", *int16_data)
+                await manager.send_audio(session_id, audio_bytes)
+            elif message["type"] == "image":
+                logger.info("Received image message from client (session %s).", session_id)
+                # Build a conversation.item.create with input_image (and optional input_text)
+                data_url = message.get("data_url")
+                prompt_text = message.get("text") or "Please describe this image."
+                if data_url:
+                    logger.info(
+                        "Forwarding image (structured message) to Realtime API (len=%d).",
+                        len(data_url),
+                    )
+                    user_msg: RealtimeUserInputMessage = {
+                        "type": "message",
+                        "role": "user",
+                        "content": (
+                            [
+                                {"type": "input_image", "image_url": data_url, "detail": "high"},
+                                {"type": "input_text", "text": prompt_text},
+                            ]
+                            if prompt_text
+                            else [{"type": "input_image", "image_url": data_url, "detail": "high"}]
+                        ),
+                    }
+                    await manager.send_user_message(session_id, user_msg)
+                    # Acknowledge to client UI
+                    await websocket.send_text(
+                        json.dumps(
+                            {
+                                "type": "client_info",
+                                "info": "image_enqueued",
+                                "size": len(data_url),
+                            }
+                        )
+                    )
+                else:
+                    await websocket.send_text(
+                        json.dumps(
+                            {
+                                "type": "error",
+                                "error": "No data_url for image message.",
+                            }
+                        )
+                    )
+            elif message["type"] == "commit_audio":
+                # Force close the current input audio turn
+                await manager.send_client_event(session_id, {"type": "input_audio_buffer.commit"})
+            elif message["type"] == "image_start":
+                img_id = str(message.get("id"))
+                image_buffers[img_id] = {
+                    "text": message.get("text") or "Please describe this image.",
+                    "chunks": [],
+                }
+                await websocket.send_text(
+                    json.dumps({"type": "client_info", "info": "image_start_ack", "id": img_id})
+                )
+            elif message["type"] == "image_chunk":
+                img_id = str(message.get("id"))
+                chunk = message.get("chunk", "")
+                if img_id in image_buffers:
+                    image_buffers[img_id]["chunks"].append(chunk)
+                    if len(image_buffers[img_id]["chunks"]) % 10 == 0:
+                        await websocket.send_text(
+                            json.dumps(
+                                {
+                                    "type": "client_info",
+                                    "info": "image_chunk_ack",
+                                    "id": img_id,
+                                    "count": len(image_buffers[img_id]["chunks"]),
+                                }
+                            )
+                        )
+            elif message["type"] == "image_end":
+                img_id = str(message.get("id"))
+                buf = image_buffers.pop(img_id, None)
+                if buf is None:
+                    await websocket.send_text(
+                        json.dumps({"type": "error", "error": "Unknown image id for image_end."})
+                    )
+                else:
+                    data_url = "".join(buf["chunks"]) if buf["chunks"] else None
+                    prompt_text = buf["text"]
+                    if data_url:
+                        logger.info(
+                            "Forwarding chunked image (structured message) to Realtime API (len=%d).",
+                            len(data_url),
+                        )
+                        user_msg2: RealtimeUserInputMessage = {
+                            "type": "message",
+                            "role": "user",
+                            "content": (
+                                [
+                                    {
+                                        "type": "input_image",
+                                        "image_url": data_url,
+                                        "detail": "high",
+                                    },
+                                    {"type": "input_text", "text": prompt_text},
+                                ]
+                                if prompt_text
+                                else [
+                                    {"type": "input_image", "image_url": data_url, "detail": "high"}
+                                ]
+                            ),
+                        }
+                        await manager.send_user_message(session_id, user_msg2)
+                        await websocket.send_text(
+                            json.dumps(
+                                {
+                                    "type": "client_info",
+                                    "info": "image_enqueued",
+                                    "id": img_id,
+                                    "size": len(data_url),
+                                }
+                            )
+                        )
+                    else:
+                        await websocket.send_text(
+                            json.dumps({"type": "error", "error": "Empty image."})
+                        )
+            elif message["type"] == "tool_approval_decision":
+                call_id = message.get("call_id")
+                approve = bool(message.get("approve"))
+                always = bool(message.get("always", False))
+                if not call_id:
+                    await websocket.send_text(
+                        json.dumps(
+                            {
+                                "type": "error",
+                                "error": "Missing call_id for tool approval decision.",
+                            }
+                        )
+                    )
+                    continue
+                if approve:
+                    await manager.approve_tool_call(session_id, call_id, always=always)
+                else:
+                    await manager.reject_tool_call(session_id, call_id, always=always)
+            elif message["type"] == "interrupt":
+                await manager.interrupt(session_id)
+
+    except WebSocketDisconnect:
+        await manager.disconnect(session_id)
+
+
+app.mount("/", StaticFiles(directory="static", html=True), name="static")
+
+
+@app.get("/")
+async def read_index():
+    return FileResponse("static/index.html")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=8000,
+        # Increased WebSocket frame size to comfortably handle image data URLs.
+        ws_max_size=16 * 1024 * 1024,
+    )

examples/realtime/app/static/app.js

@@ -0,0 +1,713 @@
+class RealtimeDemo {
+    constructor() {
+        this.ws = null;
+        this.isConnected = false;
+        this.isMuted = false;
+        this.isCapturing = false;
+        this.audioContext = null;
+        this.captureSource = null;
+        this.captureNode = null;
+        this.stream = null;
+        this.sessionId = this.generateSessionId();
+
+        this.isPlayingAudio = false;
+        this.playbackAudioContext = null;
+        this.playbackNode = null;
+        this.playbackInitPromise = null;
+        this.pendingPlaybackChunks = [];
+        this.playbackFadeSec = 0.02; // ~20ms fade to reduce clicks
+        this.messageNodes = new Map(); // item_id -> DOM node
+        this.seenItemIds = new Set(); // item_id set for append-only syncing
+
+        this.initializeElements();
+        this.setupEventListeners();
+    }
+
+    initializeElements() {
+        this.connectBtn = document.getElementById('connectBtn');
+        this.muteBtn = document.getElementById('muteBtn');
+        this.imageBtn = document.getElementById('imageBtn');
+        this.imageInput = document.getElementById('imageInput');
+        this.imagePrompt = document.getElementById('imagePrompt');
+        this.status = document.getElementById('status');
+        this.messagesContent = document.getElementById('messagesContent');
+        this.eventsContent = document.getElementById('eventsContent');
+        this.toolsContent = document.getElementById('toolsContent');
+    }
+
+    setupEventListeners() {
+        this.connectBtn.addEventListener('click', () => {
+            if (this.isConnected) {
+                this.disconnect();
+            } else {
+                this.connect();
+            }
+        });
+
+        this.muteBtn.addEventListener('click', () => {
+            this.toggleMute();
+        });
+
+        // Image upload
+        this.imageBtn.addEventListener('click', (e) => {
+            e.preventDefault();
+            e.stopPropagation();
+            console.log('Send Image clicked');
+            // Programmatically open the hidden file input
+            this.imageInput.click();
+        });
+
+        this.imageInput.addEventListener('change', async (e) => {
+            console.log('Image input change fired');
+            const file = e.target.files && e.target.files[0];
+            if (!file) return;
+            await this._handlePickedFile(file);
+            this.imageInput.value = '';
+        });
+
+        this._handlePickedFile = async (file) => {
+            try {
+                const dataUrl = await this.prepareDataURL(file);
+                const promptText = (this.imagePrompt && this.imagePrompt.value) || '';
+                // Send to server; server forwards to Realtime API.
+                // Use chunked frames to avoid WS frame limits.
+                if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+                    console.log('Interrupting and sending image (chunked) to server WebSocket');
+                    // Stop any current audio locally and tell model to interrupt
+                    this.stopAudioPlayback();
+                    this.ws.send(JSON.stringify({ type: 'interrupt' }));
+                    const id = 'img_' + Math.random().toString(36).slice(2);
+                    const CHUNK = 60_000; // ~60KB per frame
+                    this.ws.send(JSON.stringify({ type: 'image_start', id, text: promptText }));
+                    for (let i = 0; i < dataUrl.length; i += CHUNK) {
+                        const chunk = dataUrl.slice(i, i + CHUNK);
+                        this.ws.send(JSON.stringify({ type: 'image_chunk', id, chunk }));
+                    }
+                    this.ws.send(JSON.stringify({ type: 'image_end', id }));
+                } else {
+                    console.warn('Not connected; image will not be sent. Click Connect first.');
+                }
+                // Add to UI immediately for better feedback
+                console.log('Adding local user image bubble');
+                this.addUserImageMessage(dataUrl, promptText);
+            } catch (err) {
+                console.error('Failed to process image:', err);
+            }
+        };
+    }
+
+    generateSessionId() {
+        return 'session_' + Math.random().toString(36).substr(2, 9);
+    }
+
+    async connect() {
+        try {
+            this.ws = new WebSocket(`ws://localhost:8000/ws/${this.sessionId}`);
+
+            this.ws.onopen = () => {
+                this.isConnected = true;
+                this.updateConnectionUI();
+                this.startContinuousCapture();
+            };
+
+            this.ws.onmessage = (event) => {
+                const data = JSON.parse(event.data);
+                this.handleRealtimeEvent(data);
+            };
+
+            this.ws.onclose = () => {
+                this.isConnected = false;
+                this.updateConnectionUI();
+            };
+
+            this.ws.onerror = (error) => {
+                console.error('WebSocket error:', error);
+            };
+
+        } catch (error) {
+            console.error('Failed to connect:', error);
+        }
+    }
+
+    disconnect() {
+        if (this.ws) {
+            this.ws.close();
+        }
+        this.stopContinuousCapture();
+    }
+
+    updateConnectionUI() {
+        if (this.isConnected) {
+            this.connectBtn.textContent = 'Disconnect';
+            this.connectBtn.className = 'connect-btn connected';
+            this.status.textContent = 'Connected';
+            this.status.className = 'status connected';
+            this.muteBtn.disabled = false;
+        } else {
+            this.connectBtn.textContent = 'Connect';
+            this.connectBtn.className = 'connect-btn disconnected';
+            this.status.textContent = 'Disconnected';
+            this.status.className = 'status disconnected';
+            this.muteBtn.disabled = true;
+        }
+    }
+
+    toggleMute() {
+        this.isMuted = !this.isMuted;
+        this.updateMuteUI();
+    }
+
+    updateMuteUI() {
+        if (this.isMuted) {
+            this.muteBtn.textContent = '🔇 Mic Off';
+            this.muteBtn.className = 'mute-btn muted';
+        } else {
+            this.muteBtn.textContent = '🎤 Mic On';
+            this.muteBtn.className = 'mute-btn unmuted';
+            if (this.isCapturing) {
+                this.muteBtn.classList.add('active');
+            }
+        }
+    }
+
+    readFileAsDataURL(file) {
+        return new Promise((resolve, reject) => {
+            const reader = new FileReader();
+            reader.onload = () => resolve(reader.result);
+            reader.onerror = reject;
+            reader.readAsDataURL(file);
+        });
+    }
+
+    async prepareDataURL(file) {
+        const original = await this.readFileAsDataURL(file);
+        try {
+            const img = new Image();
+            img.decoding = 'async';
+            const loaded = new Promise((res, rej) => {
+                img.onload = () => res();
+                img.onerror = rej;
+            });
+            img.src = original;
+            await loaded;
+
+            const maxDim = 1024;
+            const maxSide = Math.max(img.width, img.height);
+            const scale = maxSide > maxDim ? (maxDim / maxSide) : 1;
+            const w = Math.max(1, Math.round(img.width * scale));
+            const h = Math.max(1, Math.round(img.height * scale));
+
+            const canvas = document.createElement('canvas');
+            canvas.width = w; canvas.height = h;
+            const ctx = canvas.getContext('2d');
+            ctx.drawImage(img, 0, 0, w, h);
+            return canvas.toDataURL('image/jpeg', 0.85);
+        } catch (e) {
+            console.warn('Image resize failed; sending original', e);
+            return original;
+        }
+    }
+
+    async startContinuousCapture() {
+        if (!this.isConnected || this.isCapturing) return;
+
+        // Check if getUserMedia is available
+        if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+            throw new Error('getUserMedia not available. Please use HTTPS or localhost.');
+        }
+
+        try {
+            this.stream = await navigator.mediaDevices.getUserMedia({
+                audio: {
+                    sampleRate: 24000,
+                    channelCount: 1,
+                    echoCancellation: true,
+                    noiseSuppression: true
+                }
+            });
+
+            this.audioContext = new AudioContext({ sampleRate: 24000, latencyHint: 'interactive' });
+            if (this.audioContext.state === 'suspended') {
+                try { await this.audioContext.resume(); } catch {}
+            }
+
+            if (!this.audioContext.audioWorklet) {
+                throw new Error('AudioWorklet API not supported in this browser.');
+            }
+
+            await this.audioContext.audioWorklet.addModule('audio-recorder.worklet.js');
+
+            this.captureSource = this.audioContext.createMediaStreamSource(this.stream);
+            this.captureNode = new AudioWorkletNode(this.audioContext, 'pcm-recorder');
+
+            this.captureNode.port.onmessage = (event) => {
+                if (this.isMuted) return;
+                if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
+
+                const chunk = event.data instanceof ArrayBuffer ? new Int16Array(event.data) : event.data;
+                if (!chunk || !(chunk instanceof Int16Array) || chunk.length === 0) return;
+
+                this.ws.send(JSON.stringify({
+                    type: 'audio',
+                    data: Array.from(chunk)
+                }));
+            };
+
+            this.captureSource.connect(this.captureNode);
+            this.captureNode.connect(this.audioContext.destination);
+
+            this.isCapturing = true;
+            this.updateMuteUI();
+
+        } catch (error) {
+            console.error('Failed to start audio capture:', error);
+        }
+    }
+
+    stopContinuousCapture() {
+        if (!this.isCapturing) return;
+
+        this.isCapturing = false;
+
+        if (this.captureSource) {
+            try { this.captureSource.disconnect(); } catch {}
+            this.captureSource = null;
+        }
+
+        if (this.captureNode) {
+            this.captureNode.port.onmessage = null;
+            try { this.captureNode.disconnect(); } catch {}
+            this.captureNode = null;
+        }
+
+        if (this.audioContext) {
+            this.audioContext.close();
+            this.audioContext = null;
+        }
+
+        if (this.stream) {
+            this.stream.getTracks().forEach(track => track.stop());
+            this.stream = null;
+        }
+
+        this.updateMuteUI();
+    }
+
+    handleRealtimeEvent(event) {
+        // Add to raw events pane
+        this.addRawEvent(event);
+
+        // Add to tools panel if it's a tool or handoff event
+        if (event.type === 'tool_start' || event.type === 'tool_end' || event.type === 'handoff' || event.type === 'tool_approval_required') {
+            this.addToolEvent(event);
+        }
+
+        // Handle specific event types
+        switch (event.type) {
+            case 'audio':
+                this.playAudio(event.audio);
+                break;
+            case 'audio_interrupted':
+                this.stopAudioPlayback();
+                break;
+            case 'input_audio_timeout_triggered':
+                // Ask server to commit the input buffer to expedite model response
+                if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+                    this.ws.send(JSON.stringify({ type: 'commit_audio' }));
+                }
+                break;
+            case 'history_updated':
+                this.syncMissingFromHistory(event.history);
+                this.updateLastMessageFromHistory(event.history);
+                break;
+            case 'history_added':
+                // Append just the new item without clearing the thread.
+                if (event.item) {
+                    this.addMessageFromItem(event.item);
+                }
+                break;
+            case 'tool_approval_required':
+                this.promptForToolApproval(event);
+                break;
+        }
+    }
+    updateLastMessageFromHistory(history) {
+        if (!history || !Array.isArray(history) || history.length === 0) return;
+        // Find the last message item in history
+        let last = null;
+        for (let i = history.length - 1; i >= 0; i--) {
+            const it = history[i];
+            if (it && it.type === 'message') { last = it; break; }
+        }
+        if (!last) return;
+        const itemId = last.item_id;
+
+        // Extract a text representation (for assistant transcript updates)
+        let text = '';
+        if (Array.isArray(last.content)) {
+            for (const part of last.content) {
+                if (!part || typeof part !== 'object') continue;
+                if (part.type === 'text' && part.text) text += part.text;
+                else if (part.type === 'input_text' && part.text) text += part.text;
+                else if ((part.type === 'input_audio' || part.type === 'audio') && part.transcript) text += part.transcript;
+            }
+        }
+
+        const node = this.messageNodes.get(itemId);
+        if (!node) {
+            // If we haven't rendered this item yet, append it now.
+            this.addMessageFromItem(last);
+            return;
+        }
+
+        // Update only the text content of the bubble, preserving any images already present.
+        const bubble = node.querySelector('.message-bubble');
+        if (bubble && text && text.trim()) {
+            // If there's an <img>, keep it and only update the trailing caption/text node.
+            const hasImg = !!bubble.querySelector('img');
+            if (hasImg) {
+                // Ensure there is a caption div after the image
+                let cap = bubble.querySelector('.image-caption');
+                if (!cap) {
+                    cap = document.createElement('div');
+                    cap.className = 'image-caption';
+                    cap.style.marginTop = '0.5rem';
+                    bubble.appendChild(cap);
+                }
+                cap.textContent = text.trim();
+            } else {
+                bubble.textContent = text.trim();
+            }
+            this.scrollToBottom();
+        }
+    }
+
+    syncMissingFromHistory(history) {
+        if (!history || !Array.isArray(history)) return;
+        for (const item of history) {
+            if (!item || item.type !== 'message') continue;
+            const id = item.item_id;
+            if (!id) continue;
+            if (!this.seenItemIds.has(id)) {
+                this.addMessageFromItem(item);
+            }
+        }
+    }
+
+    addMessageFromItem(item) {
+        try {
+            if (!item || item.type !== 'message') return;
+            const role = item.role;
+            let content = '';
+            let imageUrls = [];
+
+            if (Array.isArray(item.content)) {
+                for (const contentPart of item.content) {
+                    if (!contentPart || typeof contentPart !== 'object') continue;
+                    if (contentPart.type === 'text' && contentPart.text) {
+                        content += contentPart.text;
+                    } else if (contentPart.type === 'input_text' && contentPart.text) {
+                        content += contentPart.text;
+                    } else if (contentPart.type === 'input_audio' && contentPart.transcript) {
+                        content += contentPart.transcript;
+                    } else if (contentPart.type === 'audio' && contentPart.transcript) {
+                        content += contentPart.transcript;
+                    } else if (contentPart.type === 'input_image') {
+                        const url = contentPart.image_url || contentPart.url;
+                        if (typeof url === 'string' && url) imageUrls.push(url);
+                    }
+                }
+            }
+
+            let node = null;
+            if (imageUrls.length > 0) {
+                for (const url of imageUrls) {
+                    node = this.addImageMessage(role, url, content.trim());
+                }
+            } else if (content && content.trim()) {
+                node = this.addMessage(role, content.trim());
+            }
+            if (node && item.item_id) {
+                this.messageNodes.set(item.item_id, node);
+                this.seenItemIds.add(item.item_id);
+            }
+        } catch (e) {
+            console.error('Failed to add message from item:', e, item);
+        }
+    }
+
+    addMessage(type, content) {
+        const messageDiv = document.createElement('div');
+        messageDiv.className = `message ${type}`;
+
+        const bubbleDiv = document.createElement('div');
+        bubbleDiv.className = 'message-bubble';
+        bubbleDiv.textContent = content;
+
+        messageDiv.appendChild(bubbleDiv);
+        this.messagesContent.appendChild(messageDiv);
+        this.scrollToBottom();
+
+        return messageDiv;
+    }
+
+    addImageMessage(role, imageUrl, caption = '') {
+        const messageDiv = document.createElement('div');
+        messageDiv.className = `message ${role}`;
+
+        const bubbleDiv = document.createElement('div');
+        bubbleDiv.className = 'message-bubble';
+
+        const img = document.createElement('img');
+        img.src = imageUrl;
+        img.alt = 'Uploaded image';
+        img.style.maxWidth = '220px';
+        img.style.borderRadius = '8px';
+        img.style.display = 'block';
+
+        bubbleDiv.appendChild(img);
+        if (caption) {
+            const cap = document.createElement('div');
+            cap.textContent = caption;
+            cap.style.marginTop = '0.5rem';
+            bubbleDiv.appendChild(cap);
+        }
+
+        messageDiv.appendChild(bubbleDiv);
+        this.messagesContent.appendChild(messageDiv);
+        this.scrollToBottom();
+
+        return messageDiv;
+    }
+
+    addUserImageMessage(imageUrl, caption = '') {
+        return this.addImageMessage('user', imageUrl, caption);
+    }
+
+    addRawEvent(event) {
+        const eventDiv = document.createElement('div');
+        eventDiv.className = 'event';
+
+        const headerDiv = document.createElement('div');
+        headerDiv.className = 'event-header';
+        headerDiv.innerHTML = `
+            <span>${event.type}</span>
+            <span>▼</span>
+        `;
+
+        const contentDiv = document.createElement('div');
+        contentDiv.className = 'event-content collapsed';
+        contentDiv.textContent = JSON.stringify(event, null, 2);
+
+        headerDiv.addEventListener('click', () => {
+            const isCollapsed = contentDiv.classList.contains('collapsed');
+            contentDiv.classList.toggle('collapsed');
+            headerDiv.querySelector('span:last-child').textContent = isCollapsed ? '▲' : '▼';
+        });
+
+        eventDiv.appendChild(headerDiv);
+        eventDiv.appendChild(contentDiv);
+        this.eventsContent.appendChild(eventDiv);
+
+        // Auto-scroll events pane
+        this.eventsContent.scrollTop = this.eventsContent.scrollHeight;
+    }
+
+    addToolEvent(event) {
+        const eventDiv = document.createElement('div');
+        eventDiv.className = 'event';
+
+        let title = '';
+        let description = '';
+        let eventClass = '';
+
+        if (event.type === 'handoff') {
+            title = `🔄 Handoff`;
+            description = `From ${event.from} to ${event.to}`;
+            eventClass = 'handoff';
+        } else if (event.type === 'tool_start') {
+            title = `🔧 Tool Started`;
+            description = `Running ${event.tool}`;
+            eventClass = 'tool';
+        } else if (event.type === 'tool_end') {
+            title = `✅ Tool Completed`;
+            description = `${event.tool}: ${event.output || 'No output'}`;
+            eventClass = 'tool';
+        } else if (event.type === 'tool_approval_required') {
+            title = `⏸️ Approval Needed`;
+            description = `Waiting on ${event.tool}`;
+            eventClass = 'tool';
+        } else if (event.type === 'tool_approval_decision') {
+            title = event.approved ? '✅ Approved' : '❌ Rejected';
+            description = `${event.tool} (${event.call_id || 'call'})`;
+            eventClass = 'tool';
+        }
+
+        eventDiv.innerHTML = `
+            <div class="event-header ${eventClass}">
+                <div>
+                    <div style="font-weight: 600; margin-bottom: 2px;">${title}</div>
+                    <div style="font-size: 0.8rem; opacity: 0.8;">${description}</div>
+                </div>
+                <span style="font-size: 0.7rem; opacity: 0.6;">${new Date().toLocaleTimeString()}</span>
+            </div>
+        `;
+
+        this.toolsContent.appendChild(eventDiv);
+
+        // Auto-scroll tools pane
+        this.toolsContent.scrollTop = this.toolsContent.scrollHeight;
+    }
+
+    promptForToolApproval(event) {
+        const args = event.arguments || '';
+        const preview = args ? `${args.slice(0, 180)}${args.length > 180 ? '…' : ''}` : '';
+        const message = `Allow tool "${event.tool}" to run?${preview ? `\nArgs: ${preview}` : ''}`;
+        const approved = window.confirm(message);
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(JSON.stringify({
+                type: 'tool_approval_decision',
+                call_id: event.call_id,
+                approve: approved
+            }));
+        }
+        this.addToolEvent({
+            type: 'tool_approval_decision',
+            tool: event.tool,
+            call_id: event.call_id,
+            approved
+        });
+    }
+
+    async playAudio(audioBase64) {
+        try {
+            if (!audioBase64 || audioBase64.length === 0) {
+                console.warn('Received empty audio data, skipping playback');
+                return;
+            }
+
+            const int16Array = this.decodeBase64ToInt16(audioBase64);
+            if (!int16Array || int16Array.length === 0) {
+                console.warn('Audio chunk has no samples, skipping');
+                return;
+            }
+
+            this.pendingPlaybackChunks.push(int16Array);
+            await this.ensurePlaybackNode();
+            this.flushPendingPlaybackChunks();
+
+        } catch (error) {
+            console.error('Failed to play audio:', error);
+            this.pendingPlaybackChunks = [];
+        }
+    }
+
+    async ensurePlaybackNode() {
+        if (this.playbackNode) {
+            return;
+        }
+
+        if (!this.playbackInitPromise) {
+            this.playbackInitPromise = (async () => {
+                if (!this.playbackAudioContext) {
+                    this.playbackAudioContext = new AudioContext({ sampleRate: 24000, latencyHint: 'interactive' });
+                }
+
+                if (this.playbackAudioContext.state === 'suspended') {
+                    try { await this.playbackAudioContext.resume(); } catch {}
+                }
+
+                if (!this.playbackAudioContext.audioWorklet) {
+                    throw new Error('AudioWorklet API not supported in this browser.');
+                }
+
+                await this.playbackAudioContext.audioWorklet.addModule('audio-playback.worklet.js');
+
+                this.playbackNode = new AudioWorkletNode(this.playbackAudioContext, 'pcm-playback', { outputChannelCount: [1] });
+                this.playbackNode.port.onmessage = (event) => {
+                    const message = event.data;
+                    if (!message || typeof message !== 'object') return;
+                    if (message.type === 'drained') {
+                        this.isPlayingAudio = false;
+                    }
+                };
+
+                // Provide initial configuration for fades.
+                const fadeSamples = Math.floor(this.playbackAudioContext.sampleRate * this.playbackFadeSec);
+                this.playbackNode.port.postMessage({ type: 'config', fadeSamples });
+
+                this.playbackNode.connect(this.playbackAudioContext.destination);
+            })().catch((error) => {
+                this.playbackInitPromise = null;
+                throw error;
+            });
+        }
+
+        await this.playbackInitPromise;
+    }
+
+    flushPendingPlaybackChunks() {
+        if (!this.playbackNode) {
+            return;
+        }
+
+        while (this.pendingPlaybackChunks.length > 0) {
+            const chunk = this.pendingPlaybackChunks.shift();
+            if (!chunk || !(chunk instanceof Int16Array) || chunk.length === 0) {
+                continue;
+            }
+
+            try {
+                this.playbackNode.port.postMessage(
+                    { type: 'chunk', payload: chunk.buffer },
+                    [chunk.buffer]
+                );
+                this.isPlayingAudio = true;
+            } catch (error) {
+                console.error('Failed to enqueue audio chunk to worklet:', error);
+            }
+        }
+    }
+
+    decodeBase64ToInt16(audioBase64) {
+        try {
+            const binaryString = atob(audioBase64);
+            const length = binaryString.length;
+            const bytes = new Uint8Array(length);
+            for (let i = 0; i < length; i++) {
+                bytes[i] = binaryString.charCodeAt(i);
+            }
+            return new Int16Array(bytes.buffer);
+        } catch (error) {
+            console.error('Failed to decode audio chunk:', error);
+            return null;
+        }
+    }
+
+    stopAudioPlayback() {
+        console.log('Stopping audio playback due to interruption');
+
+        this.pendingPlaybackChunks = [];
+
+        if (this.playbackNode) {
+            try {
+                this.playbackNode.port.postMessage({ type: 'stop' });
+            } catch (error) {
+                console.error('Failed to notify playback worklet to stop:', error);
+            }
+        }
+
+        this.isPlayingAudio = false;
+
+        console.log('Audio playback stopped and queue cleared');
+    }
+
+    scrollToBottom() {
+        this.messagesContent.scrollTop = this.messagesContent.scrollHeight;
+    }
+}
+
+// Initialize the demo when the page loads
+document.addEventListener('DOMContentLoaded', () => {
+    new RealtimeDemo();
+});

examples/realtime/app/static/audio-playback.worklet.js

@@ -0,0 +1,120 @@
+class PCMPlaybackProcessor extends AudioWorkletProcessor {
+    constructor() {
+        super();
+
+        this.buffers = [];
+        this.currentBuffer = null;
+        this.currentIndex = 0;
+        this.isCurrentlyPlaying = false;
+        this.fadeSamples = Math.round(sampleRate * 0.02);
+
+        this.port.onmessage = (event) => {
+            const message = event.data;
+            if (!message || typeof message !== 'object') return;
+
+            if (message.type === 'chunk') {
+                const payload = message.payload;
+                if (!(payload instanceof ArrayBuffer)) {
+                    return;
+                }
+
+                const int16Data = new Int16Array(payload);
+                if (int16Data.length === 0) {
+                    return;
+                }
+
+                const scale = 1 / 32768;
+                const floatData = new Float32Array(int16Data.length);
+                for (let i = 0; i < int16Data.length; i++) {
+                    floatData[i] = Math.max(-1, Math.min(1, int16Data[i] * scale));
+                }
+
+                if (!this.hasPendingAudio()) {
+                    const fadeSamples = Math.min(this.fadeSamples, floatData.length);
+                    for (let i = 0; i < fadeSamples; i++) {
+                        const gain = fadeSamples <= 1 ? 1 : (i / fadeSamples);
+                        floatData[i] *= gain;
+                    }
+                }
+
+                this.buffers.push(floatData);
+
+            } else if (message.type === 'stop') {
+                this.reset();
+                this.port.postMessage({ type: 'drained' });
+
+            } else if (message.type === 'config') {
+                const fadeSamples = message.fadeSamples;
+                if (Number.isFinite(fadeSamples) && fadeSamples >= 0) {
+                    this.fadeSamples = fadeSamples >>> 0;
+                }
+            }
+        };
+    }
+
+    reset() {
+        this.buffers = [];
+        this.currentBuffer = null;
+        this.currentIndex = 0;
+        this.isCurrentlyPlaying = false;
+    }
+
+    hasPendingAudio() {
+        if (this.currentBuffer && this.currentIndex < this.currentBuffer.length) {
+            return true;
+        }
+        return this.buffers.length > 0;
+    }
+
+    pullSample() {
+        if (this.currentBuffer && this.currentIndex < this.currentBuffer.length) {
+            return this.currentBuffer[this.currentIndex++];
+        }
+
+        if (this.currentBuffer && this.currentIndex >= this.currentBuffer.length) {
+            this.currentBuffer = null;
+            this.currentIndex = 0;
+        }
+
+        while (this.buffers.length > 0) {
+            this.currentBuffer = this.buffers.shift();
+            this.currentIndex = 0;
+            if (this.currentBuffer && this.currentBuffer.length > 0) {
+                return this.currentBuffer[this.currentIndex++];
+            }
+        }
+
+        this.currentBuffer = null;
+        this.currentIndex = 0;
+        return 0;
+    }
+
+    process(inputs, outputs) {
+        const output = outputs[0];
+        if (!output || output.length === 0) {
+            return true;
+        }
+
+        const channel = output[0];
+        let wroteSamples = false;
+
+        for (let i = 0; i < channel.length; i++) {
+            const sample = this.pullSample();
+            channel[i] = sample;
+            if (sample !== 0) {
+                wroteSamples = true;
+            }
+        }
+
+        if (this.hasPendingAudio()) {
+            this.isCurrentlyPlaying = true;
+        } else if (!wroteSamples && this.isCurrentlyPlaying) {
+            this.isCurrentlyPlaying = false;
+            this.port.postMessage({ type: 'drained' });
+        }
+
+        return true;
+    }
+}
+
+registerProcessor('pcm-playback', PCMPlaybackProcessor);

examples/realtime/app/static/audio-recorder.worklet.js

@@ -0,0 +1,56 @@
+class PCMRecorderProcessor extends AudioWorkletProcessor {
+    constructor() {
+        super();
+        this.chunkSize = 4096;
+        this.buffer = new Int16Array(this.chunkSize);
+        this.offset = 0;
+        this.pendingFrames = 0;
+        this.maxPendingFrames = 10;
+    }
+
+    flushBuffer() {
+        if (this.offset === 0) {
+            return;
+        }
+
+        const chunk = new Int16Array(this.offset);
+        chunk.set(this.buffer.subarray(0, this.offset));
+        this.port.postMessage(chunk, [chunk.buffer]);
+
+        this.offset = 0;
+        this.pendingFrames = 0;
+    }
+
+    process(inputs) {
+        const input = inputs[0];
+        if (!input || input.length === 0) {
+            return true;
+        }
+
+        const channel = input[0];
+        if (!channel || channel.length === 0) {
+            return true;
+        }
+
+        for (let i = 0; i < channel.length; i++) {
+            let sample = channel[i];
+            sample = Math.max(-1, Math.min(1, sample));
+            this.buffer[this.offset++] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+
+            if (this.offset === this.chunkSize) {
+                this.flushBuffer();
+            }
+        }
+
+        if (this.offset > 0) {
+            this.pendingFrames += 1;
+            if (this.pendingFrames >= this.maxPendingFrames) {
+                this.flushBuffer();
+            }
+        }
+
+        return true;
+    }
+}
+
+registerProcessor('pcm-recorder', PCMRecorderProcessor);

examples/realtime/app/static/index.html

@@ -0,0 +1,299 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Realtime Demo</title>
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+        
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            background: #f8f9fa;
+            height: 100vh;
+            display: flex;
+            flex-direction: column;
+        }
+        
+        .header {
+            background: white;
+            padding: 1rem;
+            border-bottom: 1px solid #e1e5e9;
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+        }
+        
+        .connect-btn {
+            padding: 0.5rem 1rem;
+            border: none;
+            border-radius: 6px;
+            cursor: pointer;
+            font-weight: 500;
+            transition: background-color 0.2s;
+        }
+        
+        .connect-btn.disconnected {
+            background: #0066cc;
+            color: white;
+        }
+        
+        .connect-btn.connected {
+            background: #dc3545;
+            color: white;
+        }
+        
+        .connect-btn:hover {
+            opacity: 0.9;
+        }
+        
+        .main {
+            flex: 1;
+            display: flex;
+            gap: 1rem;
+            padding: 1rem;
+            height: calc(100vh - 80px);
+        }
+        
+        .messages-pane {
+            flex: 2;
+            background: white;
+            border-radius: 8px;
+            display: flex;
+            flex-direction: column;
+            overflow: hidden;
+        }
+        
+        .messages-header {
+            padding: 1rem;
+            border-bottom: 1px solid #e1e5e9;
+            font-weight: 600;
+        }
+        
+        .messages-content {
+            flex: 1;
+            overflow-y: auto;
+            padding: 1rem;
+        }
+        
+        .message {
+            margin-bottom: 1rem;
+            display: flex;
+        }
+        
+        .message.user {
+            justify-content: flex-end;
+        }
+        
+        .message.assistant {
+            justify-content: flex-start;
+        }
+        
+        .message-bubble {
+            max-width: 70%;
+            padding: 0.75rem 1rem;
+            border-radius: 18px;
+            word-wrap: break-word;
+        }
+        
+        .message.user .message-bubble {
+            background: #0066cc;
+            color: white;
+        }
+        
+        .message.assistant .message-bubble {
+            background: #f1f3f4;
+            color: #333;
+        }
+        
+        .right-column {
+            flex: 1;
+            display: flex;
+            flex-direction: column;
+            gap: 1rem;
+        }
+        
+        .events-pane {
+            flex: 2;
+            background: white;
+            border-radius: 8px;
+            display: flex;
+            flex-direction: column;
+            overflow: hidden;
+        }
+        
+        .tools-pane {
+            flex: 1;
+            background: white;
+            border-radius: 8px;
+            display: flex;
+            flex-direction: column;
+            overflow: hidden;
+        }
+        
+        .events-header, .tools-header {
+            padding: 1rem;
+            border-bottom: 1px solid #e1e5e9;
+            font-weight: 600;
+        }
+        
+        .events-content, .tools-content {
+            flex: 1;
+            overflow-y: auto;
+            padding: 0.5rem;
+        }
+        
+        .event {
+            border: 1px solid #e1e5e9;
+            border-radius: 6px;
+            margin-bottom: 0.5rem;
+        }
+        
+        .event-header {
+            padding: 0.75rem;
+            background: #f8f9fa;
+            cursor: pointer;
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            font-family: monospace;
+            font-size: 0.85rem;
+        }
+        
+        .event-header:hover {
+            background: #e9ecef;
+        }
+        
+        .tools-content .event-header {
+            cursor: default;
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+        }
+        
+        .tools-content .event-header.handoff {
+            background: #f3e8ff;
+            border-left: 4px solid #8b5cf6;
+        }
+        
+        .tools-content .event-header.tool {
+            background: #fef3e2;
+            border-left: 4px solid #f59e0b;
+        }
+        
+        .event-content {
+            padding: 0.75rem;
+            background: white;
+            border-top: 1px solid #e1e5e9;
+            font-family: monospace;
+            font-size: 0.8rem;
+            white-space: pre-wrap;
+            max-height: 200px;
+            overflow-y: auto;
+        }
+        
+        .event-content.collapsed {
+            display: none;
+        }
+        
+        .controls {
+            padding: 1rem;
+            border-top: 1px solid #e1e5e9;
+            background: #f8f9fa;
+            display: flex;
+            gap: 0.5rem;
+            align-items: center;
+        }
+        
+        .mute-btn {
+            padding: 0.5rem 1rem;
+            border: none;
+            border-radius: 6px;
+            cursor: pointer;
+            font-weight: 500;
+            transition: all 0.2s;
+        }
+        
+        .mute-btn.unmuted {
+            background: #28a745;
+            color: white;
+        }
+        
+        .mute-btn.muted {
+            background: #dc3545;
+            color: white;
+        }
+        
+        .mute-btn.active {
+            animation: pulse 1s infinite;
+        }
+        
+        @keyframes pulse {
+            0% { opacity: 1; }
+            50% { opacity: 0.7; }
+            100% { opacity: 1; }
+        }
+        
+        .status {
+            font-size: 0.9rem;
+            color: #6c757d;
+        }
+        
+        .connected {
+            color: #28a745;
+        }
+        
+        .disconnected {
+            color: #dc3545;
+        }
+    </style>
+</head>
+<body>
+    <div class="header">
+        <h1>Realtime Demo</h1>
+        <button id="connectBtn" class="connect-btn disconnected">Connect</button>
+    </div>
+    
+    <div class="main">
+        <div class="messages-pane">
+            <div class="messages-header">
+                Conversation
+            </div>
+            <div id="messagesContent" class="messages-content">
+                <!-- Messages will appear here -->
+            </div>
+            <div class="controls">
+                <button id="muteBtn" class="mute-btn unmuted" disabled>🎤 Mic On</button>
+                <input id="imagePrompt" type="text" placeholder="Optional prompt for image" style="flex: 1; padding: 0.5rem; border: 1px solid #e1e5e9; border-radius: 6px;" />
+                <input id="imageInput" type="file" accept="image/*" aria-hidden="true" style="position:absolute;left:-9999px;width:1px;height:1px;opacity:0;" />
+                <button id="imageBtn" type="button" class="mute-btn unmuted" style="background:#6f42c1; user-select:none;">🖼️ Send Image</button>
+                <span id="status" class="status disconnected">Disconnected</span>
+            </div>
+        </div>
+        
+        <div class="right-column">
+            <div class="events-pane">
+                <div class="events-header">
+                    Event stream
+                </div>
+                <div id="eventsContent" class="events-content">
+                    <!-- Events will appear here -->
+                </div>
+            </div>
+            
+            <div class="tools-pane">
+                <div class="tools-header">
+                    Tools & Handoffs
+                </div>
+                <div id="toolsContent" class="tools-content">
+                    <!-- Tools and handoffs will appear here -->
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <script src="app.js"></script>
+</body>
+</html>

examples/realtime/cli/demo.py

@@ -0,0 +1,381 @@
+import asyncio
+import queue
+import sys
+import threading
+from typing import Any
+
+import numpy as np
+import sounddevice as sd
+
+from agents import function_tool
+from agents.realtime import (
+    RealtimeAgent,
+    RealtimePlaybackTracker,
+    RealtimeRunner,
+    RealtimeSession,
+    RealtimeSessionEvent,
+)
+from agents.realtime.model import RealtimeModelConfig
+
+# Audio configuration
+CHUNK_LENGTH_S = 0.04  # 40ms aligns with realtime defaults
+SAMPLE_RATE = 24000
+FORMAT = np.int16
+CHANNELS = 1
+ENERGY_THRESHOLD = 0.015  # RMS threshold for barge‑in while assistant is speaking
+PREBUFFER_CHUNKS = 3  # initial jitter buffer (~120ms with 40ms chunks)
+FADE_OUT_MS = 12  # short fade to avoid clicks when interrupting
+PLAYBACK_ECHO_MARGIN = 0.002  # extra energy above playback echo required to count as speech
+
+# Set up logging for OpenAI agents SDK
+# logging.basicConfig(
+#     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+# )
+# logger.logger.setLevel(logging.ERROR)
+
+
+@function_tool
+def get_weather(city: str) -> str:
+    """Get the weather in a city."""
+    return f"The weather in {city} is sunny."
+
+
+agent = RealtimeAgent(
+    name="Assistant",
+    instructions="You always greet the user with 'Top of the morning to you'.",
+    tools=[get_weather],
+)
+
+
+def _truncate_str(s: str, max_length: int) -> str:
+    if len(s) > max_length:
+        return s[:max_length] + "..."
+    return s
+
+
+class NoUIDemo:
+    def __init__(self) -> None:
+        self.session: RealtimeSession | None = None
+        self.audio_stream: sd.InputStream | None = None
+        self.audio_player: sd.OutputStream | None = None
+        self.recording = False
+
+        # Playback tracker lets the model know our real playback progress
+        self.playback_tracker = RealtimePlaybackTracker()
+
+        # Audio output state for callback system
+        # Store tuples: (samples_np, item_id, content_index)
+        # Use an unbounded queue to avoid drops that sound like skipped words.
+        self.output_queue: queue.Queue[Any] = queue.Queue(maxsize=0)
+        self.interrupt_event = threading.Event()
+        self.current_audio_chunk: tuple[np.ndarray[Any, np.dtype[Any]], str, int] | None = None
+        self.chunk_position = 0
+        self.bytes_per_sample = np.dtype(FORMAT).itemsize
+
+        # Jitter buffer and fade-out state
+        self.prebuffering = True
+        self.prebuffer_target_chunks = PREBUFFER_CHUNKS
+        self.fading = False
+        self.fade_total_samples = 0
+        self.fade_done_samples = 0
+        self.fade_samples = int(SAMPLE_RATE * (FADE_OUT_MS / 1000.0))
+        self.playback_rms = 0.0  # smoothed playback energy to filter out echo
+
+    def _output_callback(self, outdata, frames: int, time, status) -> None:
+        """Callback for audio output - handles continuous audio stream from server."""
+        if status:
+            print(f"Output callback status: {status}")
+
+        # Handle interruption with a short fade-out to prevent clicks.
+        if self.interrupt_event.is_set():
+            outdata.fill(0)
+            if self.current_audio_chunk is None:
+                # Nothing to fade, just flush everything and reset.
+                while not self.output_queue.empty():
+                    try:
+                        self.output_queue.get_nowait()
+                    except queue.Empty:
+                        break
+                self.prebuffering = True
+                self.interrupt_event.clear()
+                return
+
+            # Prepare fade parameters
+            if not self.fading:
+                self.fading = True
+                self.fade_done_samples = 0
+                # Remaining samples in the current chunk
+                remaining_in_chunk = len(self.current_audio_chunk[0]) - self.chunk_position
+                self.fade_total_samples = min(self.fade_samples, max(0, remaining_in_chunk))
+
+            samples, item_id, content_index = self.current_audio_chunk
+            samples_filled = 0
+            while (
+                samples_filled < len(outdata) and self.fade_done_samples < self.fade_total_samples
+            ):
+                remaining_output = len(outdata) - samples_filled
+                remaining_fade = self.fade_total_samples - self.fade_done_samples
+                n = min(remaining_output, remaining_fade)
+
+                src = samples[self.chunk_position : self.chunk_position + n].astype(np.float32)
+                # Linear ramp from current level down to 0 across remaining fade samples
+                idx = np.arange(
+                    self.fade_done_samples, self.fade_done_samples + n, dtype=np.float32
+                )
+                gain = 1.0 - (idx / float(self.fade_total_samples))
+                ramped = np.clip(src * gain, -32768.0, 32767.0).astype(np.int16)
+                outdata[samples_filled : samples_filled + n, 0] = ramped
+                self._update_playback_rms(ramped)
+
+                # Optionally report played bytes (ramped) to playback tracker
+                try:
+                    self.playback_tracker.on_play_bytes(
+                        item_id=item_id, item_content_index=content_index, bytes=ramped.tobytes()
+                    )
+                except Exception:
+                    pass
+
+                samples_filled += n
+                self.chunk_position += n
+                self.fade_done_samples += n
+
+            # If fade completed, flush the remaining audio and reset state
+            if self.fade_done_samples >= self.fade_total_samples:
+                self.current_audio_chunk = None
+                self.chunk_position = 0
+                while not self.output_queue.empty():
+                    try:
+                        self.output_queue.get_nowait()
+                    except queue.Empty:
+                        break
+                self.fading = False
+                self.prebuffering = True
+                self.interrupt_event.clear()
+            return
+
+        # Fill output buffer from queue and current chunk
+        outdata.fill(0)  # Start with silence
+        samples_filled = 0
+
+        while samples_filled < len(outdata):
+            # If we don't have a current chunk, try to get one from queue
+            if self.current_audio_chunk is None:
+                try:
+                    # Respect a small jitter buffer before starting playback
+                    if (
+                        self.prebuffering
+                        and self.output_queue.qsize() < self.prebuffer_target_chunks
+                    ):
+                        break
+                    self.prebuffering = False
+                    self.current_audio_chunk = self.output_queue.get_nowait()
+                    self.chunk_position = 0
+                except queue.Empty:
+                    # No more audio data available - this causes choppiness
+                    # Uncomment next line to debug underruns:
+                    # print(f"Audio underrun: {samples_filled}/{len(outdata)} samples filled")
+                    break
+
+            # Copy data from current chunk to output buffer
+            remaining_output = len(outdata) - samples_filled
+            samples, item_id, content_index = self.current_audio_chunk
+            remaining_chunk = len(samples) - self.chunk_position
+            samples_to_copy = min(remaining_output, remaining_chunk)
+
+            if samples_to_copy > 0:
+                chunk_data = samples[self.chunk_position : self.chunk_position + samples_to_copy]
+                # More efficient: direct assignment for mono audio instead of reshape
+                outdata[samples_filled : samples_filled + samples_to_copy, 0] = chunk_data
+                self._update_playback_rms(chunk_data)
+                samples_filled += samples_to_copy
+                self.chunk_position += samples_to_copy
+
+                # Inform playback tracker about played bytes
+                try:
+                    self.playback_tracker.on_play_bytes(
+                        item_id=item_id,
+                        item_content_index=content_index,
+                        bytes=chunk_data.tobytes(),
+                    )
+                except Exception:
+                    pass
+
+                # If we've used up the entire chunk, reset for next iteration
+                if self.chunk_position >= len(samples):
+                    self.current_audio_chunk = None
+                    self.chunk_position = 0
+
+    async def run(self) -> None:
+        print("Connecting, may take a few seconds...")
+
+        # Initialize audio player with callback
+        chunk_size = int(SAMPLE_RATE * CHUNK_LENGTH_S)
+        self.audio_player = sd.OutputStream(
+            channels=CHANNELS,
+            samplerate=SAMPLE_RATE,
+            dtype=FORMAT,
+            callback=self._output_callback,
+            blocksize=chunk_size,  # Match our chunk timing for better alignment
+        )
+        self.audio_player.start()
+
+        try:
+            runner = RealtimeRunner(agent)
+            # Attach playback tracker and enable server‑side interruptions + auto response.
+            model_config: RealtimeModelConfig = {
+                "playback_tracker": self.playback_tracker,
+                "initial_model_settings": {
+                    "model_name": "gpt-realtime-2.1",
+                    "turn_detection": {
+                        "type": "semantic_vad",
+                        "interrupt_response": True,
+                        "create_response": True,
+                    },
+                },
+            }
+            async with await runner.run(model_config=model_config) as session:
+                self.session = session
+                print("Connected. Starting audio recording...")
+
+                # Start audio recording
+                await self.start_audio_recording()
+                print("Audio recording started. You can start speaking - expect lots of logs!")
+
+                # Process session events
+                async for event in session:
+                    await self._on_event(event)
+
+        finally:
+            # Clean up audio player
+            if self.audio_player and self.audio_player.active:
+                self.audio_player.stop()
+            if self.audio_player:
+                self.audio_player.close()
+
+        print("Session ended")
+
+    async def start_audio_recording(self) -> None:
+        """Start recording audio from the microphone."""
+        # Set up audio input stream
+        self.audio_stream = sd.InputStream(
+            channels=CHANNELS,
+            samplerate=SAMPLE_RATE,
+            dtype=FORMAT,
+        )
+
+        self.audio_stream.start()
+        self.recording = True
+
+        # Start audio capture task
+        asyncio.create_task(self.capture_audio())
+
+    async def capture_audio(self) -> None:
+        """Capture audio from the microphone and send to the session."""
+        if not self.audio_stream or not self.session:
+            return
+
+        # Buffer size in samples
+        read_size = int(SAMPLE_RATE * CHUNK_LENGTH_S)
+
+        try:
+            while self.recording:
+                # Check if there's enough data to read
+                if self.audio_stream.read_available < read_size:
+                    await asyncio.sleep(0.01)
+                    continue
+
+                # Read audio data
+                data, _ = self.audio_stream.read(read_size)
+
+                # Convert numpy array to bytes
+                audio_bytes = data.tobytes()
+
+                # Smart barge‑in: if assistant audio is playing, send only if mic has speech.
+                assistant_playing = (
+                    self.current_audio_chunk is not None or not self.output_queue.empty()
+                )
+                if assistant_playing:
+                    # Compute RMS energy to detect speech while assistant is talking
+                    samples = data.reshape(-1)
+                    mic_rms = self._compute_rms(samples)
+                    # Require the mic to be louder than the echo of the assistant playback.
+                    playback_gate = max(
+                        ENERGY_THRESHOLD,
+                        self.playback_rms * 0.6 + PLAYBACK_ECHO_MARGIN,
+                    )
+                    if mic_rms >= playback_gate:
+                        # Locally flush queued assistant audio for snappier interruption.
+                        self.interrupt_event.set()
+                        await self.session.send_audio(audio_bytes)
+                else:
+                    await self.session.send_audio(audio_bytes)
+
+                # Yield control back to event loop
+                await asyncio.sleep(0)
+
+        except Exception as e:
+            print(f"Audio capture error: {e}")
+        finally:
+            if self.audio_stream and self.audio_stream.active:
+                self.audio_stream.stop()
+            if self.audio_stream:
+                self.audio_stream.close()
+
+    async def _on_event(self, event: RealtimeSessionEvent) -> None:
+        """Handle session events."""
+        try:
+            if event.type == "agent_start":
+                print(f"Agent started: {event.agent.name}")
+            elif event.type == "agent_end":
+                print(f"Agent ended: {event.agent.name}")
+            elif event.type == "handoff":
+                print(f"Handoff from {event.from_agent.name} to {event.to_agent.name}")
+            elif event.type == "tool_start":
+                print(f"Tool started: {event.tool.name}")
+            elif event.type == "tool_end":
+                print(f"Tool ended: {event.tool.name}; output: {event.output}")
+            elif event.type == "audio_end":
+                print("Audio ended")
+            elif event.type == "audio":
+                # Enqueue audio for callback-based playback with metadata
+                np_audio = np.frombuffer(event.audio.data, dtype=np.int16)
+                # Non-blocking put; queue is unbounded, so drops won’t occur.
+                self.output_queue.put_nowait((np_audio, event.item_id, event.content_index))
+            elif event.type == "audio_interrupted":
+                print("Audio interrupted")
+                # Begin graceful fade + flush in the audio callback and rebuild jitter buffer.
+                self.prebuffering = True
+                self.interrupt_event.set()
+            elif event.type == "error":
+                print(f"Error: {event.error}")
+            elif event.type == "history_updated":
+                pass  # Skip these frequent events
+            elif event.type == "history_added":
+                pass  # Skip these frequent events
+            elif event.type == "raw_model_event":
+                print(f"Raw model event: {_truncate_str(str(event.data), 200)}")
+            else:
+                print(f"Unknown event type: {event.type}")
+        except Exception as e:
+            print(f"Error processing event: {_truncate_str(str(e), 200)}")
+
+    def _compute_rms(self, samples: np.ndarray[Any, np.dtype[Any]]) -> float:
+        """Compute RMS energy for int16 samples normalized to [-1, 1]."""
+        if samples.size == 0:
+            return 0.0
+        x = samples.astype(np.float32) / 32768.0
+        return float(np.sqrt(np.mean(x * x)))
+
+    def _update_playback_rms(self, samples: np.ndarray[Any, np.dtype[Any]]) -> None:
+        """Keep a smoothed estimate of playback energy to filter out echo feedback."""
+        sample_rms = self._compute_rms(samples)
+        self.playback_rms = 0.9 * self.playback_rms + 0.1 * sample_rms
+
+
+if __name__ == "__main__":
+    demo = NoUIDemo()
+    try:
+        asyncio.run(demo.run())
+    except KeyboardInterrupt:
+        print("\nExiting...")
+        sys.exit(0)

examples/realtime/twilio/README.md

@@ -0,0 +1,86 @@
+# Realtime Twilio Integration
+
+This example demonstrates how to connect the OpenAI Realtime API to a phone call using Twilio's Media Streams. The server handles incoming phone calls and streams audio between Twilio and the OpenAI Realtime API, enabling real-time voice conversations with an AI agent over the phone.
+
+## Prerequisites
+
+-   Python 3.10+
+-   OpenAI API key with [Realtime API](https://platform.openai.com/docs/guides/realtime) access
+-   [Twilio](https://www.twilio.com/docs/voice) account with a phone number
+-   A tunneling service like [ngrok](https://ngrok.com/) to expose your local server
+
+## Setup
+
+1. **Start the server:**
+
+    ```bash
+    uv run server.py
+    ```
+
+    The server will start on port 8000 by default.
+
+2. **Expose the server publicly, e.g. via ngrok:**
+
+    ```bash
+    ngrok http 8000
+    ```
+
+    Note the public URL (e.g., `https://abc123.ngrok.io`)
+
+3. **Configure your Twilio phone number:**
+    - Log into your Twilio Console
+    - Select your phone number
+    - Set the webhook URL for incoming calls to: `https://your-ngrok-url.ngrok.io/incoming-call`
+    - Set the HTTP method to POST
+
+## Usage
+
+1. Call your Twilio phone number
+2. You'll hear: "Hello! You're now connected to an AI assistant. You can start talking!"
+3. Start speaking - the AI will respond in real-time
+4. The assistant has access to tools like weather information and current time
+
+## How It Works
+
+1. **Incoming Call**: When someone calls your Twilio number, Twilio makes a request to `/incoming-call`
+2. **TwiML Response**: The server returns TwiML that:
+    - Plays a greeting message
+    - Connects the call to a WebSocket stream at `/media-stream`
+3. **WebSocket Connection**: Twilio establishes a WebSocket connection for bidirectional audio streaming
+4. **Transport Layer**: The `TwilioRealtimeTransportLayer` class owns the WebSocket message handling:
+    - Takes ownership of the Twilio WebSocket after initial handshake
+    - Runs its own message loop to process all Twilio messages
+    - Handles protocol differences between Twilio and OpenAI
+    - Automatically sets G.711 μ-law audio format for Twilio compatibility
+    - Manages audio chunk tracking for interruption support
+    - Wraps the OpenAI realtime model instead of subclassing it
+5. **Audio Processing**:
+    - Audio from the caller is base64 decoded and sent to OpenAI Realtime API
+    - Audio responses from OpenAI are base64 encoded and sent back to Twilio
+    - Twilio plays the audio to the caller
+
+## Configuration
+
+-   **Port**: Set `PORT` environment variable (default: 8000)
+-   **OpenAI API Key**: Set `OPENAI_API_KEY` environment variable
+-   **Agent Instructions**: Modify the `RealtimeAgent` configuration in `server.py`
+-   **Tools**: Add or modify function tools in `server.py`
+
+## Troubleshooting
+
+-   **WebSocket connection issues**: Ensure your ngrok URL is correct and publicly accessible
+-   **Audio quality**: Twilio streams audio in mulaw format at 8kHz, which may affect quality
+-   **Latency**: Network latency between Twilio, your server, and OpenAI affects response time
+-   **Logs**: Check the console output for detailed connection and error logs
+
+## Architecture
+
+```
+Phone Call → Twilio → WebSocket → TwilioRealtimeTransportLayer → OpenAI Realtime API
+                                              ↓
+                                      RealtimeAgent with Tools
+                                              ↓
+                           Audio Response → Twilio → Phone Call
+```
+
+The `TwilioRealtimeTransportLayer` acts as a bridge between Twilio's Media Streams and OpenAI's Realtime API, handling the protocol differences and audio format conversions. It wraps the OpenAI realtime model to provide a clean interface for Twilio integration.

examples/realtime/twilio/requirements.txt

@@ -0,0 +1,5 @@
+openai-agents
+fastapi
+uvicorn[standard]
+websockets
+python-dotenv

examples/realtime/twilio/server.py

@@ -0,0 +1,80 @@
+import os
+from typing import TYPE_CHECKING
+
+from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import PlainTextResponse
+
+# Import TwilioHandler class - handle both module and package use cases
+if TYPE_CHECKING:
+    # For type checking, use the relative import
+    from .twilio_handler import TwilioHandler
+else:
+    # At runtime, try both import styles
+    try:
+        # Try relative import first (when used as a package)
+        from .twilio_handler import TwilioHandler
+    except ImportError:
+        # Fall back to direct import (when run as a script)
+        from twilio_handler import TwilioHandler
+
+
+class TwilioWebSocketManager:
+    def __init__(self):
+        self.active_handlers: dict[str, TwilioHandler] = {}
+
+    async def new_session(self, websocket: WebSocket) -> TwilioHandler:
+        """Create and configure a new session."""
+        print("Creating twilio handler")
+
+        handler = TwilioHandler(websocket)
+        return handler
+
+    # In a real app, you'd also want to clean up/close the handler when the call ends
+
+
+manager = TwilioWebSocketManager()
+app = FastAPI()
+
+
+@app.get("/")
+async def root():
+    return {"message": "Twilio Media Stream Server is running!"}
+
+
+@app.post("/incoming-call")
+@app.get("/incoming-call")
+async def incoming_call(request: Request):
+    """Handle incoming Twilio phone calls"""
+    host = request.headers.get("Host")
+
+    twiml_response = f"""<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+    <Say>Hello! You're now connected to an AI assistant. You can start talking!</Say>
+    <Connect>
+        <Stream url="wss://{host}/media-stream" />
+    </Connect>
+</Response>"""
+    return PlainTextResponse(content=twiml_response, media_type="text/xml")
+
+
+@app.websocket("/media-stream")
+async def media_stream_endpoint(websocket: WebSocket):
+    """WebSocket endpoint for Twilio Media Streams"""
+
+    try:
+        handler = await manager.new_session(websocket)
+        await handler.start()
+
+        await handler.wait_until_done()
+
+    except WebSocketDisconnect:
+        print("WebSocket disconnected")
+    except Exception as e:
+        print(f"WebSocket error: {e}")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    port = int(os.getenv("PORT", 8000))
+    uvicorn.run(app, host="0.0.0.0", port=port)

examples/realtime/twilio/twilio_handler.py

@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import os
+import time
+from datetime import datetime
+from typing import Any
+
+from fastapi import WebSocket
+
+from agents import function_tool
+from agents.realtime import (
+    RealtimeAgent,
+    RealtimePlaybackTracker,
+    RealtimeRunner,
+    RealtimeSession,
+    RealtimeSessionEvent,
+)
+
+
+@function_tool
+def get_weather(city: str) -> str:
+    """Get the weather in a city."""
+    return f"The weather in {city} is sunny."
+
+
+@function_tool
+def get_current_time() -> str:
+    """Get the current time."""
+    return f"The current time is {datetime.now().strftime('%H:%M:%S')}"
+
+
+agent = RealtimeAgent(
+    name="Twilio Assistant",
+    instructions=(
+        "You are a helpful assistant that starts every conversation with a creative greeting. "
+        "Keep responses concise and friendly since this is a phone conversation."
+    ),
+    tools=[get_weather, get_current_time],
+)
+
+
+class TwilioHandler:
+    def __init__(self, twilio_websocket: WebSocket):
+        self.twilio_websocket = twilio_websocket
+        self._message_loop_task: asyncio.Task[None] | None = None
+        self.session: RealtimeSession | None = None
+        self.playback_tracker = RealtimePlaybackTracker()
+
+        # Audio chunking (matches CLI demo)
+        self.CHUNK_LENGTH_S = 0.05  # 50ms chunks
+        self.SAMPLE_RATE = 8000  # Twilio g711_ulaw at 8kHz
+        self.BUFFER_SIZE_BYTES = int(self.SAMPLE_RATE * self.CHUNK_LENGTH_S)  # ~400 bytes per 50ms
+
+        self._stream_sid: str | None = None
+        self._audio_buffer: bytearray = bytearray()
+        self._last_buffer_send_time = time.time()
+
+        # Playback tracking for outbound audio
+        self._mark_counter = 0
+        self._mark_data: dict[
+            str, tuple[str, int, int]
+        ] = {}  # mark_id -> (item_id, content_index, byte_count)
+
+        # ---- Deterministic startup warm-up (preferred over sleep) ----
+        # Buffer the first N chunks before sending to OpenAI; then mark warmed.
+        try:
+            self.STARTUP_BUFFER_CHUNKS = max(0, int(os.getenv("TWILIO_STARTUP_BUFFER_CHUNKS", "3")))
+        except Exception:
+            self.STARTUP_BUFFER_CHUNKS = 3
+
+        self._startup_buffer = bytearray()
+        self._startup_warmed = (
+            self.STARTUP_BUFFER_CHUNKS == 0
+        )  # if 0, considered warmed immediately
+
+        # Optional delay (defaults 0.0 because buffering is preferred)
+        try:
+            self.STARTUP_DELAY_S = float(os.getenv("TWILIO_STARTUP_DELAY_S", "0.0"))
+        except Exception:
+            self.STARTUP_DELAY_S = 0.0
+
+    async def start(self) -> None:
+        """Start the session."""
+        runner = RealtimeRunner(agent)
+        api_key = os.getenv("OPENAI_API_KEY")
+        if not api_key:
+            raise ValueError("OPENAI_API_KEY environment variable is required")
+
+        self.session = await runner.run(
+            model_config={
+                "api_key": api_key,
+                "initial_model_settings": {
+                    "model_name": "gpt-realtime-2.1",
+                    "input_audio_format": "g711_ulaw",
+                    "output_audio_format": "g711_ulaw",
+                    "turn_detection": {
+                        "type": "semantic_vad",
+                        "interrupt_response": True,
+                        "create_response": True,
+                    },
+                },
+                "playback_tracker": self.playback_tracker,
+            }
+        )
+
+        await self.session.enter()
+
+        await self.twilio_websocket.accept()
+        print("Twilio WebSocket connection accepted")
+
+        # Optional tiny delay (kept configurable; default 0.0)
+        if self.STARTUP_DELAY_S > 0:
+            await asyncio.sleep(self.STARTUP_DELAY_S)
+
+        # Start loops after handshake
+        self._realtime_session_task = asyncio.create_task(self._realtime_session_loop())
+        self._message_loop_task = asyncio.create_task(self._twilio_message_loop())
+        self._buffer_flush_task = asyncio.create_task(self._buffer_flush_loop())
+
+    async def wait_until_done(self) -> None:
+        """Wait until the session is done."""
+        assert self._message_loop_task is not None
+        await self._message_loop_task
+
+    async def _realtime_session_loop(self) -> None:
+        """Listen for events from the realtime session."""
+        assert self.session is not None
+        try:
+            async for event in self.session:
+                await self._handle_realtime_event(event)
+        except Exception as e:
+            print(f"Error in realtime session loop: {e}")
+
+    async def _twilio_message_loop(self) -> None:
+        """Listen for messages from Twilio WebSocket and handle them."""
+        try:
+            while True:
+                message_text = await self.twilio_websocket.receive_text()
+                message = json.loads(message_text)
+                await self._handle_twilio_message(message)
+        except json.JSONDecodeError as e:
+            print(f"Failed to parse Twilio message as JSON: {e}")
+        except Exception as e:
+            print(f"Error in Twilio message loop: {e}")
+
+    async def _handle_realtime_event(self, event: RealtimeSessionEvent) -> None:
+        """Handle events from the realtime session."""
+        if event.type == "audio":
+            base64_audio = base64.b64encode(event.audio.data).decode("utf-8")
+            await self.twilio_websocket.send_text(
+                json.dumps(
+                    {
+                        "event": "media",
+                        "streamSid": self._stream_sid,
+                        "media": {"payload": base64_audio},
+                    }
+                )
+            )
+
+            # Send mark event for playback tracking
+            self._mark_counter += 1
+            mark_id = str(self._mark_counter)
+            self._mark_data[mark_id] = (
+                event.audio.item_id,
+                event.audio.content_index,
+                len(event.audio.data),
+            )
+
+            await self.twilio_websocket.send_text(
+                json.dumps(
+                    {
+                        "event": "mark",
+                        "streamSid": self._stream_sid,
+                        "mark": {"name": mark_id},
+                    }
+                )
+            )
+
+        elif event.type == "audio_interrupted":
+            print("Sending audio interrupted to Twilio")
+            await self.twilio_websocket.send_text(
+                json.dumps({"event": "clear", "streamSid": self._stream_sid})
+            )
+        elif event.type == "audio_end":
+            print("Audio end")
+        elif event.type == "raw_model_event":
+            pass
+        else:
+            pass
+
+    async def _handle_twilio_message(self, message: dict[str, Any]) -> None:
+        """Handle incoming messages from Twilio Media Stream."""
+        try:
+            event = message.get("event")
+
+            if event == "connected":
+                print("Twilio media stream connected")
+            elif event == "start":
+                start_data = message.get("start", {})
+                self._stream_sid = start_data.get("streamSid")
+                print(f"Media stream started with SID: {self._stream_sid}")
+            elif event == "media":
+                await self._handle_media_event(message)
+            elif event == "mark":
+                await self._handle_mark_event(message)
+            elif event == "stop":
+                print("Media stream stopped")
+        except Exception as e:
+            print(f"Error handling Twilio message: {e}")
+
+    async def _handle_media_event(self, message: dict[str, Any]) -> None:
+        """Handle audio data from Twilio - buffer it before sending to OpenAI."""
+        media = message.get("media", {})
+        payload = media.get("payload", "")
+
+        if payload:
+            try:
+                # Decode base64 audio from Twilio (µ-law format)
+                ulaw_bytes = base64.b64decode(payload)
+
+                # Add original µ-law to buffer for OpenAI (they expect µ-law)
+                self._audio_buffer.extend(ulaw_bytes)
+
+                # Send buffered audio if we have enough data for one chunk
+                if len(self._audio_buffer) >= self.BUFFER_SIZE_BYTES:
+                    await self._flush_audio_buffer()
+
+            except Exception as e:
+                print(f"Error processing audio from Twilio: {e}")
+
+    async def _handle_mark_event(self, message: dict[str, Any]) -> None:
+        """Handle mark events from Twilio to update playback tracker."""
+        try:
+            mark_data = message.get("mark", {})
+            mark_id = mark_data.get("name", "")
+
+            if mark_id in self._mark_data:
+                item_id, item_content_index, byte_count = self._mark_data[mark_id]
+                audio_bytes = b"\x00" * byte_count  # Placeholder bytes for tracker
+                self.playback_tracker.on_play_bytes(item_id, item_content_index, audio_bytes)
+                print(
+                    f"Playback tracker updated: {item_id}, index {item_content_index}, {byte_count} bytes"
+                )
+                del self._mark_data[mark_id]
+
+        except Exception as e:
+            print(f"Error handling mark event: {e}")
+
+    async def _flush_audio_buffer(self) -> None:
+        """Send buffered audio to OpenAI with deterministic startup warm-up."""
+        if not self._audio_buffer or not self.session:
+            return
+
+        try:
+            buffer_data = bytes(self._audio_buffer)
+            self._audio_buffer.clear()
+            self._last_buffer_send_time = time.time()
+
+            # During startup, accumulate first N chunks before sending anything
+            if not self._startup_warmed:
+                self._startup_buffer.extend(buffer_data)
+
+                # target bytes = N chunks * bytes-per-chunk
+                target_bytes = self.BUFFER_SIZE_BYTES * max(0, self.STARTUP_BUFFER_CHUNKS)
+
+                if len(self._startup_buffer) >= target_bytes:
+                    # Warm-up complete: flush all buffered data in order
+                    await self.session.send_audio(bytes(self._startup_buffer))
+                    self._startup_buffer.clear()
+                    self._startup_warmed = True
+                else:
+                    # Not enough yet; keep buffering and return
+                    return
+            else:
+                # Already warmed: send immediately
+                await self.session.send_audio(buffer_data)
+
+        except Exception as e:
+            print(f"Error sending buffered audio to OpenAI: {e}")
+
+    async def _buffer_flush_loop(self) -> None:
+        """Periodically flush audio buffer to prevent stale data."""
+        try:
+            while True:
+                await asyncio.sleep(self.CHUNK_LENGTH_S)  # check every 50ms
+
+                # If buffer has data and it's been too long since last send, flush it
+                current_time = time.time()
+                if (
+                    self._audio_buffer
+                    and current_time - self._last_buffer_send_time > self.CHUNK_LENGTH_S * 2
+                ):
+                    await self._flush_audio_buffer()
+
+        except Exception as e:
+            print(f"Error in buffer flush loop: {e}")

examples/realtime/twilio_sip/README.md

@@ -0,0 +1,54 @@
+# Twilio SIP Realtime Example
+
+This example shows how to handle OpenAI Realtime SIP calls with the Agents SDK. Incoming calls are accepted through the Realtime Calls API, a triage agent answers with a fixed greeting, and handoffs route the caller to specialist agents (FAQ lookup and record updates) similar to the realtime UI demo.
+
+## Prerequisites
+
+- Python 3.10+
+- An OpenAI API key with Realtime API access
+- A configured webhook secret for your OpenAI project
+- A Twilio account with a phone number and Elastic SIP Trunking enabled
+- A public HTTPS endpoint for local development (for example, [ngrok](https://ngrok.com/))
+
+## Configure OpenAI
+
+1. In [platform settings](https://platform.openai.com/settings) select your project.
+2. Create a webhook pointing to `https://<your-public-host>/openai/webhook` with "realtime.call.incoming" event type and note the signing secret. The example verifies each webhook with `OPENAI_WEBHOOK_SECRET`.
+
+## Configure Twilio Elastic SIP Trunking
+
+1. Create (or edit) an Elastic SIP trunk.
+2. On the **Origination** tab, add an origination SIP URI of `sip:proj_<your_project_id>@sip.api.openai.com;transport=tls` so Twilio sends inbound calls to OpenAI. (The Termination tab always ends with `.pstn.twilio.com`, so leave it unchanged.)
+3. Add at least one phone number to the trunk so inbound calls are forwarded to OpenAI.
+
+## Setup
+
+1. Install dependencies:
+   ```bash
+   uv pip install -r examples/realtime/twilio_sip/requirements.txt
+   ```
+2. Export required environment variables:
+   ```bash
+   export OPENAI_API_KEY="sk-..."
+   export OPENAI_WEBHOOK_SECRET="whsec_..."
+   ```
+3. (Optional) Adjust the multi-agent logic in `examples/realtime/twilio_sip/agents.py` if you want to change the specialist agents or tools.
+4. Run the FastAPI server:
+   ```bash
+   uv run uvicorn examples.realtime.twilio_sip.server:app --host 0.0.0.0 --port 8000
+   ```
+5. Expose the server publicly (example with ngrok):
+   ```bash
+   ngrok http 8000
+   ```
+
+## Test a Call
+
+1. Place a call to the Twilio number attached to the SIP trunk.
+2. Twilio sends the call to `sip.api.openai.com`; OpenAI fires `realtime.call.incoming`, which this example accepts.
+3. The triage agent greets the caller, then either keeps the conversation or hands off to:
+   - **FAQ Agent** – answers common questions via `faq_lookup_tool`.
+   - **Records Agent** – writes short notes using `update_customer_record`.
+4. The background task attaches to the call and logs transcripts plus basic events in the console.
+
+You can edit `server.py` to change instructions, add tools, or integrate with internal systems once the SIP session is active.

examples/realtime/twilio_sip/__init__.py

@@ -0,0 +1 @@
+"""OpenAI Realtime SIP example package."""

examples/realtime/twilio_sip/agents.py

@@ -0,0 +1,94 @@
+"""Realtime agent definitions shared by the Twilio SIP example."""
+
+from __future__ import annotations
+
+import asyncio
+
+from agents import function_tool
+from agents.extensions.handoff_prompt import RECOMMENDED_PROMPT_PREFIX
+from agents.realtime import RealtimeAgent, realtime_handoff
+
+# --- Tools -----------------------------------------------------------------
+
+
+WELCOME_MESSAGE = "Hello, this is ABC customer service. How can I help you today?"
+
+
+@function_tool(
+    name_override="faq_lookup_tool", description_override="Lookup frequently asked questions."
+)
+async def faq_lookup_tool(question: str) -> str:
+    """Fetch FAQ answers for the caller."""
+
+    await asyncio.sleep(3)
+
+    q = question.lower()
+    if "plan" in q or "wifi" in q or "wi-fi" in q:
+        return "We provide complimentary Wi-Fi. Join the ABC-Customer network."  # demo data
+    if "billing" in q or "invoice" in q:
+        return "Your latest invoice is available in the ABC portal under Billing > History."
+    if "hours" in q or "support" in q:
+        return "Human support agents are available 24/7; transfer to the specialist if needed."
+    return "I'm not sure about that. Let me transfer you back to the triage agent."
+
+
+@function_tool
+async def update_customer_record(customer_id: str, note: str) -> str:
+    """Record a short note about the caller."""
+
+    await asyncio.sleep(1)
+    return f"Recorded note for {customer_id}: {note}"
+
+
+# --- Agents ----------------------------------------------------------------
+
+
+faq_agent = RealtimeAgent(
+    name="FAQ Agent",
+    handoff_description="Handles frequently asked questions and general account inquiries.",
+    instructions=f"""{RECOMMENDED_PROMPT_PREFIX}
+    You are an FAQ specialist. Always rely on the faq_lookup_tool for answers and keep replies
+    concise. If the caller needs hands-on help, transfer back to the triage agent.
+    """,
+    tools=[faq_lookup_tool],
+)
+
+records_agent = RealtimeAgent(
+    name="Records Agent",
+    handoff_description="Updates customer records with brief notes and confirmation numbers.",
+    instructions=f"""{RECOMMENDED_PROMPT_PREFIX}
+    You handle structured updates. Confirm the customer's ID, capture their request in a short
+    note, and use the update_customer_record tool. For anything outside data updates, return to the
+    triage agent.
+    """,
+    tools=[update_customer_record],
+)
+
+triage_agent = RealtimeAgent(
+    name="Triage Agent",
+    handoff_description="Greets callers and routes them to the most appropriate specialist.",
+    instructions=(
+        f"{RECOMMENDED_PROMPT_PREFIX} "
+        "Always begin the call by saying exactly: '"
+        f"{WELCOME_MESSAGE}' "
+        "before collecting details. Once the greeting is complete, gather context and hand off to "
+        "the FAQ or Records agents when appropriate."
+    ),
+    handoffs=[
+        realtime_handoff(faq_agent, tool_name_override="transfer_to_faq_agent"),
+        realtime_handoff(records_agent, tool_name_override="transfer_to_records_agent"),
+    ],
+)
+
+faq_agent.handoffs.append(
+    realtime_handoff(triage_agent, tool_name_override="transfer_to_triage_agent")
+)
+records_agent.handoffs.append(
+    realtime_handoff(triage_agent, tool_name_override="transfer_to_triage_agent")
+)
+
+
+def get_starting_agent() -> RealtimeAgent:
+    """Return the agent used to start each realtime call."""
+
+    return triage_agent

examples/realtime/twilio_sip/requirements.txt

@@ -0,0 +1,3 @@
+fastapi>=0.120.0
+openai>=2.2,<3
+uvicorn[standard]>=0.38.0

examples/realtime/twilio_sip/server.py

@@ -0,0 +1,211 @@
+"""Minimal FastAPI server for handling OpenAI Realtime SIP calls with Twilio."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+
+import websockets
+from fastapi import FastAPI, HTTPException, Request, Response
+from openai import APIStatusError, AsyncOpenAI, InvalidWebhookSignatureError
+
+from agents.realtime.config import RealtimeSessionModelSettings
+from agents.realtime.items import (
+    AssistantAudio,
+    AssistantMessageItem,
+    AssistantText,
+    InputText,
+    UserMessageItem,
+)
+from agents.realtime.model_inputs import RealtimeModelSendRawMessage
+from agents.realtime.openai_realtime import OpenAIRealtimeSIPModel
+from agents.realtime.runner import RealtimeRunner
+
+from .agents import WELCOME_MESSAGE, get_starting_agent
+
+logging.basicConfig(level=logging.INFO)
+
+logger = logging.getLogger("twilio_sip_example")
+
+
+def _get_env(name: str) -> str:
+    value = os.getenv(name)
+    if not value:
+        raise RuntimeError(f"Missing environment variable: {name}")
+    return value
+
+
+OPENAI_API_KEY = _get_env("OPENAI_API_KEY")
+OPENAI_WEBHOOK_SECRET = _get_env("OPENAI_WEBHOOK_SECRET")
+
+client = AsyncOpenAI(api_key=OPENAI_API_KEY, webhook_secret=OPENAI_WEBHOOK_SECRET)
+
+# Build the multi-agent graph (triage + specialist agents) from agents.py.
+assistant_agent = get_starting_agent()
+
+app = FastAPI()
+
+# Track background tasks so repeated webhooks do not spawn duplicates.
+active_call_tasks: dict[str, asyncio.Task[None]] = {}
+
+
+async def accept_call(call_id: str) -> None:
+    """Accept the incoming SIP call and configure the realtime session."""
+
+    # The starting agent uses static instructions, so we can forward them directly to the accept
+    # call payload. If someone swaps in a dynamic prompt, fall back to a sensible default.
+    instructions_payload = (
+        assistant_agent.instructions
+        if isinstance(assistant_agent.instructions, str)
+        else "You are a helpful triage agent for ABC customer service."
+    )
+
+    try:
+        # AsyncOpenAI does not yet expose high-level helpers like client.realtime.calls.accept, so
+        # we call the REST endpoint directly via client.post(). Keep this until the SDK grows an
+        # async helper.
+        await client.post(
+            f"/realtime/calls/{call_id}/accept",
+            body={
+                "type": "realtime",
+                "model": "gpt-realtime-2.1",
+                "instructions": instructions_payload,
+            },
+            cast_to=dict,
+        )
+    except APIStatusError as exc:
+        if exc.status_code == 404:
+            # Twilio occasionally retries webhooks after the caller hangs up; treat as a no-op so
+            # the webhook still returns 200.
+            logger.warning(
+                "Call %s no longer exists when attempting accept (404). Skipping.", call_id
+            )
+            return
+
+        detail = exc.message
+        if exc.response is not None:
+            try:
+                detail = exc.response.text
+            except Exception:  # noqa: BLE001
+                detail = str(exc.response)
+
+        logger.error("Failed to accept call %s: %s %s", call_id, exc.status_code, detail)
+        raise HTTPException(status_code=500, detail="Failed to accept call") from exc
+
+    logger.info("Accepted call %s", call_id)
+
+
+async def observe_call(call_id: str) -> None:
+    """Attach to the realtime session and log conversation events."""
+
+    runner = RealtimeRunner(assistant_agent, model=OpenAIRealtimeSIPModel())
+
+    try:
+        initial_model_settings: RealtimeSessionModelSettings = {
+            "turn_detection": {
+                "type": "semantic_vad",
+                "interrupt_response": True,
+            }
+        }
+        async with await runner.run(
+            model_config={
+                "call_id": call_id,
+                "initial_model_settings": initial_model_settings,
+            }
+        ) as session:
+            # Trigger an initial greeting so callers hear the agent right away.
+            # Issue a response.create immediately after the WebSocket attaches so the model speaks
+            # before the caller says anything. Using the raw client message ensures zero latency
+            # and avoids threading the greeting through history.
+            await session.model.send_event(
+                RealtimeModelSendRawMessage(
+                    message={
+                        "type": "response.create",
+                        "other_data": {
+                            "response": {
+                                "instructions": (
+                                    "Say exactly '"
+                                    f"{WELCOME_MESSAGE}"
+                                    "' now before continuing the conversation."
+                                )
+                            }
+                        },
+                    }
+                )
+            )
+
+            async for event in session:
+                if event.type == "history_added":
+                    item = event.item
+                    if isinstance(item, UserMessageItem):
+                        for user_content in item.content:
+                            if isinstance(user_content, InputText) and user_content.text:
+                                logger.info("Caller: %s", user_content.text)
+                    elif isinstance(item, AssistantMessageItem):
+                        for assistant_content in item.content:
+                            if (
+                                isinstance(assistant_content, AssistantText)
+                                and assistant_content.text
+                            ):
+                                logger.info("Assistant (text): %s", assistant_content.text)
+                            elif (
+                                isinstance(assistant_content, AssistantAudio)
+                                and assistant_content.transcript
+                            ):
+                                logger.info(
+                                    "Assistant (audio transcript): %s",
+                                    assistant_content.transcript,
+                                )
+                elif event.type == "error":
+                    logger.error("Realtime session error: %s", event.error)
+
+    except websockets.exceptions.ConnectionClosedError:
+        # Callers hanging up causes the WebSocket to close without a frame; log at info level so it
+        # does not surface as an error.
+        logger.info("Realtime WebSocket closed for call %s", call_id)
+    except Exception as exc:  # noqa: BLE001 - demo logging only
+        logger.exception("Error while observing call %s", call_id, exc_info=exc)
+    finally:
+        logger.info("Call %s ended", call_id)
+        active_call_tasks.pop(call_id, None)
+
+
+def _track_call_task(call_id: str) -> None:
+    existing = active_call_tasks.get(call_id)
+    if existing:
+        if not existing.done():
+            logger.info(
+                "Call %s already has an active observer; ignoring duplicate webhook delivery.",
+                call_id,
+            )
+            return
+        # Remove completed tasks so a new observer can start for a fresh call.
+        active_call_tasks.pop(call_id, None)
+
+    task = asyncio.create_task(observe_call(call_id))
+    active_call_tasks[call_id] = task
+
+
+@app.post("/openai/webhook")
+async def openai_webhook(request: Request) -> Response:
+    body = await request.body()
+
+    try:
+        event = client.webhooks.unwrap(body, request.headers)
+    except InvalidWebhookSignatureError as exc:
+        raise HTTPException(status_code=400, detail="Invalid webhook signature") from exc
+
+    if event.type == "realtime.call.incoming":
+        call_id = event.data.call_id
+        await accept_call(call_id)
+        _track_call_task(call_id)
+        return Response(status_code=200)
+
+    # Ignore other webhook event types for brevity.
+    return Response(status_code=200)
+
+
+@app.get("/")
+async def healthcheck() -> dict[str, str]:
+    return {"status": "ok"}