chore: import upstream snapshot with attribution

2026-07-13 12:39:17 +08:00
commit 4ed4e9ff99
1368 changed files with 334957 additions and 0 deletions
@@ -0,0 +1,86 @@
+# Realtime Twilio Integration
+
+This example demonstrates how to connect the OpenAI Realtime API to a phone call using Twilio's Media Streams. The server handles incoming phone calls and streams audio between Twilio and the OpenAI Realtime API, enabling real-time voice conversations with an AI agent over the phone.
+
+## Prerequisites
+
+-   Python 3.10+
+-   OpenAI API key with [Realtime API](https://platform.openai.com/docs/guides/realtime) access
+-   [Twilio](https://www.twilio.com/docs/voice) account with a phone number
+-   A tunneling service like [ngrok](https://ngrok.com/) to expose your local server
+
+## Setup
+
+1. **Start the server:**
+
+    ```bash
+    uv run server.py
+    ```
+
+    The server will start on port 8000 by default.
+
+2. **Expose the server publicly, e.g. via ngrok:**
+
+    ```bash
+    ngrok http 8000
+    ```
+
+    Note the public URL (e.g., `https://abc123.ngrok.io`)
+
+3. **Configure your Twilio phone number:**
+    - Log into your Twilio Console
+    - Select your phone number
+    - Set the webhook URL for incoming calls to: `https://your-ngrok-url.ngrok.io/incoming-call`
+    - Set the HTTP method to POST
+
+## Usage
+
+1. Call your Twilio phone number
+2. You'll hear: "Hello! You're now connected to an AI assistant. You can start talking!"
+3. Start speaking - the AI will respond in real-time
+4. The assistant has access to tools like weather information and current time
+
+## How It Works
+
+1. **Incoming Call**: When someone calls your Twilio number, Twilio makes a request to `/incoming-call`
+2. **TwiML Response**: The server returns TwiML that:
+    - Plays a greeting message
+    - Connects the call to a WebSocket stream at `/media-stream`
+3. **WebSocket Connection**: Twilio establishes a WebSocket connection for bidirectional audio streaming
+4. **Transport Layer**: The `TwilioRealtimeTransportLayer` class owns the WebSocket message handling:
+    - Takes ownership of the Twilio WebSocket after initial handshake
+    - Runs its own message loop to process all Twilio messages
+    - Handles protocol differences between Twilio and OpenAI
+    - Automatically sets G.711 μ-law audio format for Twilio compatibility
+    - Manages audio chunk tracking for interruption support
+    - Wraps the OpenAI realtime model instead of subclassing it
+5. **Audio Processing**:
+    - Audio from the caller is base64 decoded and sent to OpenAI Realtime API
+    - Audio responses from OpenAI are base64 encoded and sent back to Twilio
+    - Twilio plays the audio to the caller
+
+## Configuration
+
+-   **Port**: Set `PORT` environment variable (default: 8000)
+-   **OpenAI API Key**: Set `OPENAI_API_KEY` environment variable
+-   **Agent Instructions**: Modify the `RealtimeAgent` configuration in `server.py`
+-   **Tools**: Add or modify function tools in `server.py`
+
+## Troubleshooting
+
+-   **WebSocket connection issues**: Ensure your ngrok URL is correct and publicly accessible
+-   **Audio quality**: Twilio streams audio in mulaw format at 8kHz, which may affect quality
+-   **Latency**: Network latency between Twilio, your server, and OpenAI affects response time
+-   **Logs**: Check the console output for detailed connection and error logs
+
+## Architecture
+
+```
+Phone Call → Twilio → WebSocket → TwilioRealtimeTransportLayer → OpenAI Realtime API
+                                              ↓
+                                      RealtimeAgent with Tools
+                                              ↓
+                           Audio Response → Twilio → Phone Call
+```
+
+The `TwilioRealtimeTransportLayer` acts as a bridge between Twilio's Media Streams and OpenAI's Realtime API, handling the protocol differences and audio format conversions. It wraps the OpenAI realtime model to provide a clean interface for Twilio integration.
@@ -0,0 +1,5 @@
+openai-agents
+fastapi
+uvicorn[standard]
+websockets
+python-dotenv
@@ -0,0 +1,80 @@
+import os
+from typing import TYPE_CHECKING
+
+from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import PlainTextResponse
+
+# Import TwilioHandler class - handle both module and package use cases
+if TYPE_CHECKING:
+    # For type checking, use the relative import
+    from .twilio_handler import TwilioHandler
+else:
+    # At runtime, try both import styles
+    try:
+        # Try relative import first (when used as a package)
+        from .twilio_handler import TwilioHandler
+    except ImportError:
+        # Fall back to direct import (when run as a script)
+        from twilio_handler import TwilioHandler
+
+
+class TwilioWebSocketManager:
+    def __init__(self):
+        self.active_handlers: dict[str, TwilioHandler] = {}
+
+    async def new_session(self, websocket: WebSocket) -> TwilioHandler:
+        """Create and configure a new session."""
+        print("Creating twilio handler")
+
+        handler = TwilioHandler(websocket)
+        return handler
+
+    # In a real app, you'd also want to clean up/close the handler when the call ends
+
+
+manager = TwilioWebSocketManager()
+app = FastAPI()
+
+
+@app.get("/")
+async def root():
+    return {"message": "Twilio Media Stream Server is running!"}
+
+
+@app.post("/incoming-call")
+@app.get("/incoming-call")
+async def incoming_call(request: Request):
+    """Handle incoming Twilio phone calls"""
+    host = request.headers.get("Host")
+
+    twiml_response = f"""<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+    <Say>Hello! You're now connected to an AI assistant. You can start talking!</Say>
+    <Connect>
+        <Stream url="wss://{host}/media-stream" />
+    </Connect>
+</Response>"""
+    return PlainTextResponse(content=twiml_response, media_type="text/xml")
+
+
+@app.websocket("/media-stream")
+async def media_stream_endpoint(websocket: WebSocket):
+    """WebSocket endpoint for Twilio Media Streams"""
+
+    try:
+        handler = await manager.new_session(websocket)
+        await handler.start()
+
+        await handler.wait_until_done()
+
+    except WebSocketDisconnect:
+        print("WebSocket disconnected")
+    except Exception as e:
+        print(f"WebSocket error: {e}")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    port = int(os.getenv("PORT", 8000))
+    uvicorn.run(app, host="0.0.0.0", port=port)
@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import os
+import time
+from datetime import datetime
+from typing import Any
+
+from fastapi import WebSocket
+
+from agents import function_tool
+from agents.realtime import (
+    RealtimeAgent,
+    RealtimePlaybackTracker,
+    RealtimeRunner,
+    RealtimeSession,
+    RealtimeSessionEvent,
+)
+
+
+@function_tool
+def get_weather(city: str) -> str:
+    """Get the weather in a city."""
+    return f"The weather in {city} is sunny."
+
+
+@function_tool
+def get_current_time() -> str:
+    """Get the current time."""
+    return f"The current time is {datetime.now().strftime('%H:%M:%S')}"
+
+
+agent = RealtimeAgent(
+    name="Twilio Assistant",
+    instructions=(
+        "You are a helpful assistant that starts every conversation with a creative greeting. "
+        "Keep responses concise and friendly since this is a phone conversation."
+    ),
+    tools=[get_weather, get_current_time],
+)
+
+
+class TwilioHandler:
+    def __init__(self, twilio_websocket: WebSocket):
+        self.twilio_websocket = twilio_websocket
+        self._message_loop_task: asyncio.Task[None] | None = None
+        self.session: RealtimeSession | None = None
+        self.playback_tracker = RealtimePlaybackTracker()
+
+        # Audio chunking (matches CLI demo)
+        self.CHUNK_LENGTH_S = 0.05  # 50ms chunks
+        self.SAMPLE_RATE = 8000  # Twilio g711_ulaw at 8kHz
+        self.BUFFER_SIZE_BYTES = int(self.SAMPLE_RATE * self.CHUNK_LENGTH_S)  # ~400 bytes per 50ms
+
+        self._stream_sid: str | None = None
+        self._audio_buffer: bytearray = bytearray()
+        self._last_buffer_send_time = time.time()
+
+        # Playback tracking for outbound audio
+        self._mark_counter = 0
+        self._mark_data: dict[
+            str, tuple[str, int, int]
+        ] = {}  # mark_id -> (item_id, content_index, byte_count)
+
+        # ---- Deterministic startup warm-up (preferred over sleep) ----
+        # Buffer the first N chunks before sending to OpenAI; then mark warmed.
+        try:
+            self.STARTUP_BUFFER_CHUNKS = max(0, int(os.getenv("TWILIO_STARTUP_BUFFER_CHUNKS", "3")))
+        except Exception:
+            self.STARTUP_BUFFER_CHUNKS = 3
+
+        self._startup_buffer = bytearray()
+        self._startup_warmed = (
+            self.STARTUP_BUFFER_CHUNKS == 0
+        )  # if 0, considered warmed immediately
+
+        # Optional delay (defaults 0.0 because buffering is preferred)
+        try:
+            self.STARTUP_DELAY_S = float(os.getenv("TWILIO_STARTUP_DELAY_S", "0.0"))
+        except Exception:
+            self.STARTUP_DELAY_S = 0.0
+
+    async def start(self) -> None:
+        """Start the session."""
+        runner = RealtimeRunner(agent)
+        api_key = os.getenv("OPENAI_API_KEY")
+        if not api_key:
+            raise ValueError("OPENAI_API_KEY environment variable is required")
+
+        self.session = await runner.run(
+            model_config={
+                "api_key": api_key,
+                "initial_model_settings": {
+                    "model_name": "gpt-realtime-2.1",
+                    "input_audio_format": "g711_ulaw",
+                    "output_audio_format": "g711_ulaw",
+                    "turn_detection": {
+                        "type": "semantic_vad",
+                        "interrupt_response": True,
+                        "create_response": True,
+                    },
+                },
+                "playback_tracker": self.playback_tracker,
+            }
+        )
+
+        await self.session.enter()
+
+        await self.twilio_websocket.accept()
+        print("Twilio WebSocket connection accepted")
+
+        # Optional tiny delay (kept configurable; default 0.0)
+        if self.STARTUP_DELAY_S > 0:
+            await asyncio.sleep(self.STARTUP_DELAY_S)
+
+        # Start loops after handshake
+        self._realtime_session_task = asyncio.create_task(self._realtime_session_loop())
+        self._message_loop_task = asyncio.create_task(self._twilio_message_loop())
+        self._buffer_flush_task = asyncio.create_task(self._buffer_flush_loop())
+
+    async def wait_until_done(self) -> None:
+        """Wait until the session is done."""
+        assert self._message_loop_task is not None
+        await self._message_loop_task
+
+    async def _realtime_session_loop(self) -> None:
+        """Listen for events from the realtime session."""
+        assert self.session is not None
+        try:
+            async for event in self.session:
+                await self._handle_realtime_event(event)
+        except Exception as e:
+            print(f"Error in realtime session loop: {e}")
+
+    async def _twilio_message_loop(self) -> None:
+        """Listen for messages from Twilio WebSocket and handle them."""
+        try:
+            while True:
+                message_text = await self.twilio_websocket.receive_text()
+                message = json.loads(message_text)
+                await self._handle_twilio_message(message)
+        except json.JSONDecodeError as e:
+            print(f"Failed to parse Twilio message as JSON: {e}")
+        except Exception as e:
+            print(f"Error in Twilio message loop: {e}")
+
+    async def _handle_realtime_event(self, event: RealtimeSessionEvent) -> None:
+        """Handle events from the realtime session."""
+        if event.type == "audio":
+            base64_audio = base64.b64encode(event.audio.data).decode("utf-8")
+            await self.twilio_websocket.send_text(
+                json.dumps(
+                    {
+                        "event": "media",
+                        "streamSid": self._stream_sid,
+                        "media": {"payload": base64_audio},
+                    }
+                )
+            )
+
+            # Send mark event for playback tracking
+            self._mark_counter += 1
+            mark_id = str(self._mark_counter)
+            self._mark_data[mark_id] = (
+                event.audio.item_id,
+                event.audio.content_index,
+                len(event.audio.data),
+            )
+
+            await self.twilio_websocket.send_text(
+                json.dumps(
+                    {
+                        "event": "mark",
+                        "streamSid": self._stream_sid,
+                        "mark": {"name": mark_id},
+                    }
+                )
+            )
+
+        elif event.type == "audio_interrupted":
+            print("Sending audio interrupted to Twilio")
+            await self.twilio_websocket.send_text(
+                json.dumps({"event": "clear", "streamSid": self._stream_sid})
+            )
+        elif event.type == "audio_end":
+            print("Audio end")
+        elif event.type == "raw_model_event":
+            pass
+        else:
+            pass
+
+    async def _handle_twilio_message(self, message: dict[str, Any]) -> None:
+        """Handle incoming messages from Twilio Media Stream."""
+        try:
+            event = message.get("event")
+
+            if event == "connected":
+                print("Twilio media stream connected")
+            elif event == "start":
+                start_data = message.get("start", {})
+                self._stream_sid = start_data.get("streamSid")
+                print(f"Media stream started with SID: {self._stream_sid}")
+            elif event == "media":
+                await self._handle_media_event(message)
+            elif event == "mark":
+                await self._handle_mark_event(message)
+            elif event == "stop":
+                print("Media stream stopped")
+        except Exception as e:
+            print(f"Error handling Twilio message: {e}")
+
+    async def _handle_media_event(self, message: dict[str, Any]) -> None:
+        """Handle audio data from Twilio - buffer it before sending to OpenAI."""
+        media = message.get("media", {})
+        payload = media.get("payload", "")
+
+        if payload:
+            try:
+                # Decode base64 audio from Twilio (µ-law format)
+                ulaw_bytes = base64.b64decode(payload)
+
+                # Add original µ-law to buffer for OpenAI (they expect µ-law)
+                self._audio_buffer.extend(ulaw_bytes)
+
+                # Send buffered audio if we have enough data for one chunk
+                if len(self._audio_buffer) >= self.BUFFER_SIZE_BYTES:
+                    await self._flush_audio_buffer()
+
+            except Exception as e:
+                print(f"Error processing audio from Twilio: {e}")
+
+    async def _handle_mark_event(self, message: dict[str, Any]) -> None:
+        """Handle mark events from Twilio to update playback tracker."""
+        try:
+            mark_data = message.get("mark", {})
+            mark_id = mark_data.get("name", "")
+
+            if mark_id in self._mark_data:
+                item_id, item_content_index, byte_count = self._mark_data[mark_id]
+                audio_bytes = b"\x00" * byte_count  # Placeholder bytes for tracker
+                self.playback_tracker.on_play_bytes(item_id, item_content_index, audio_bytes)
+                print(
+                    f"Playback tracker updated: {item_id}, index {item_content_index}, {byte_count} bytes"
+                )
+                del self._mark_data[mark_id]
+
+        except Exception as e:
+            print(f"Error handling mark event: {e}")
+
+    async def _flush_audio_buffer(self) -> None:
+        """Send buffered audio to OpenAI with deterministic startup warm-up."""
+        if not self._audio_buffer or not self.session:
+            return
+
+        try:
+            buffer_data = bytes(self._audio_buffer)
+            self._audio_buffer.clear()
+            self._last_buffer_send_time = time.time()
+
+            # During startup, accumulate first N chunks before sending anything
+            if not self._startup_warmed:
+                self._startup_buffer.extend(buffer_data)
+
+                # target bytes = N chunks * bytes-per-chunk
+                target_bytes = self.BUFFER_SIZE_BYTES * max(0, self.STARTUP_BUFFER_CHUNKS)
+
+                if len(self._startup_buffer) >= target_bytes:
+                    # Warm-up complete: flush all buffered data in order
+                    await self.session.send_audio(bytes(self._startup_buffer))
+                    self._startup_buffer.clear()
+                    self._startup_warmed = True
+                else:
+                    # Not enough yet; keep buffering and return
+                    return
+            else:
+                # Already warmed: send immediately
+                await self.session.send_audio(buffer_data)
+
+        except Exception as e:
+            print(f"Error sending buffered audio to OpenAI: {e}")
+
+    async def _buffer_flush_loop(self) -> None:
+        """Periodically flush audio buffer to prevent stale data."""
+        try:
+            while True:
+                await asyncio.sleep(self.CHUNK_LENGTH_S)  # check every 50ms
+
+                # If buffer has data and it's been too long since last send, flush it
+                current_time = time.time()
+                if (
+                    self._audio_buffer
+                    and current_time - self._last_buffer_send_time > self.CHUNK_LENGTH_S * 2
+                ):
+                    await self._flush_audio_buffer()
+
+        except Exception as e:
+            print(f"Error in buffer flush loop: {e}")