chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,60 @@
|
||||
# Realtime Demo App
|
||||
|
||||
A web-based realtime voice assistant demo with a FastAPI backend and HTML/JS frontend.
|
||||
|
||||
## Installation
|
||||
|
||||
Install the required dependencies:
|
||||
|
||||
```bash
|
||||
uv add fastapi uvicorn websockets
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Start the application with a single command:
|
||||
|
||||
```bash
|
||||
cd examples/realtime/app && uv run python server.py
|
||||
```
|
||||
|
||||
Then open your browser to: http://localhost:8000
|
||||
|
||||
### Debugging Realtime usage
|
||||
|
||||
Set `LOG_LEVEL=DEBUG` to log the raw `response.done` usage, the typed per-response usage with modality details, and the cumulative session usage:
|
||||
|
||||
```bash
|
||||
cd examples/realtime/app && LOG_LEVEL=DEBUG uv run python server.py
|
||||
```
|
||||
|
||||
The debug logs include concise summaries for server, model, session, history, tool, handoff, error, and usage events. Audio frames and high-volume delta events are omitted, and transcript content is not logged. Uvicorn and WebSocket protocol logging remain at INFO so `LOG_LEVEL=DEBUG` does not dump wire payloads.
|
||||
|
||||
## Customization
|
||||
|
||||
To use the same UI with your own agents, edit `agent.py` and ensure get_starting_agent() returns the right starting agent for your use case.
|
||||
|
||||
## How to Use
|
||||
|
||||
1. Click **Connect** to establish a realtime session
|
||||
2. Audio capture starts automatically - just speak naturally
|
||||
3. Click the **Mic On/Off** button to mute/unmute your microphone
|
||||
4. To send an image, enter an optional prompt and click **🖼️ Send Image** (select a file)
|
||||
5. Watch the conversation unfold in the left pane (image thumbnails are shown)
|
||||
6. Monitor raw events in the right pane (click to expand/collapse)
|
||||
7. Click **Disconnect** when done
|
||||
|
||||
### Human-in-the-loop approvals
|
||||
|
||||
- The seat update tool now requires approval. When the agent wants to run it, the browser shows a `window.confirm` dialog so you can allow or deny the tool call before it executes.
|
||||
|
||||
## Architecture
|
||||
|
||||
- **Backend**: FastAPI server with WebSocket connections for real-time communication
|
||||
- **Session Management**: Each connection gets a unique session with the OpenAI Realtime API
|
||||
- **Image Inputs**: The UI uploads images and the server forwards a `conversation.item.create` event with `input_image` (plus optional `input_text`), followed by `response.create` to start the model response. The messages pane renders image bubbles for `input_image` content.
|
||||
- **Audio Processing**: 24kHz mono audio capture and playback
|
||||
- **Event Handling**: Full event stream processing with transcript generation
|
||||
- **Frontend**: Vanilla JavaScript with clean, responsive CSS
|
||||
|
||||
The demo showcases the core patterns for building realtime voice applications with the OpenAI Agents SDK.
|
||||
@@ -0,0 +1,107 @@
|
||||
import asyncio
|
||||
|
||||
from agents import function_tool
|
||||
from agents.extensions.handoff_prompt import RECOMMENDED_PROMPT_PREFIX
|
||||
from agents.realtime import RealtimeAgent, realtime_handoff
|
||||
|
||||
"""
|
||||
When running the UI example locally, you can edit this file to change the setup. THe server
|
||||
will use the agent returned from get_starting_agent() as the starting agent."""
|
||||
|
||||
### TOOLS
|
||||
|
||||
|
||||
@function_tool(
|
||||
name_override="faq_lookup_tool", description_override="Lookup frequently asked questions."
|
||||
)
|
||||
async def faq_lookup_tool(question: str) -> str:
|
||||
# Simulate a slow API call
|
||||
await asyncio.sleep(3)
|
||||
|
||||
q = question.lower()
|
||||
if "wifi" in q or "wi-fi" in q:
|
||||
return "We have free wifi on the plane, join Airline-Wifi"
|
||||
elif "bag" in q or "baggage" in q:
|
||||
return (
|
||||
"You are allowed to bring one bag on the plane. "
|
||||
"It must be under 50 pounds and 22 inches x 14 inches x 9 inches."
|
||||
)
|
||||
elif "seats" in q or "plane" in q:
|
||||
return (
|
||||
"There are 120 seats on the plane. "
|
||||
"There are 22 business class seats and 98 economy seats. "
|
||||
"Exit rows are rows 4 and 16. "
|
||||
"Rows 5-8 are Economy Plus, with extra legroom. "
|
||||
)
|
||||
return "I'm sorry, I don't know the answer to that question."
|
||||
|
||||
|
||||
@function_tool(needs_approval=True)
|
||||
async def update_seat(confirmation_number: str, new_seat: str) -> str:
|
||||
"""
|
||||
Update the seat for a given confirmation number.
|
||||
|
||||
Args:
|
||||
confirmation_number: The confirmation number for the flight.
|
||||
new_seat: The new seat to update to.
|
||||
"""
|
||||
return f"Updated seat to {new_seat} for confirmation number {confirmation_number}"
|
||||
|
||||
|
||||
@function_tool
|
||||
def get_weather(city: str) -> str:
|
||||
"""Get the weather in a city."""
|
||||
return f"The weather in {city} is sunny."
|
||||
|
||||
|
||||
faq_agent = RealtimeAgent(
|
||||
name="FAQ Agent",
|
||||
handoff_description="A helpful agent that can answer questions about the airline.",
|
||||
instructions=f"""{RECOMMENDED_PROMPT_PREFIX}
|
||||
You are an FAQ agent. If you are speaking to a customer, you probably were transferred to from the triage agent.
|
||||
Use the following routine to support the customer.
|
||||
# Routine
|
||||
1. Identify the last question asked by the customer.
|
||||
2. Use the faq lookup tool to answer the question. Do not rely on your own knowledge.
|
||||
3. If you cannot answer the question, transfer back to the triage agent.""",
|
||||
tools=[faq_lookup_tool],
|
||||
)
|
||||
|
||||
seat_booking_agent = RealtimeAgent(
|
||||
name="Seat Booking Agent",
|
||||
handoff_description="A helpful agent that can update a seat on a flight.",
|
||||
instructions=f"""{RECOMMENDED_PROMPT_PREFIX}
|
||||
You are a seat booking agent. If you are speaking to a customer, you probably were transferred to from the triage agent.
|
||||
Use the following routine to support the customer.
|
||||
# Routine
|
||||
1. Ask for their confirmation number.
|
||||
2. Ask the customer what their desired seat number is.
|
||||
3. Use the update seat tool to update the seat on the flight.
|
||||
If the customer asks a question that is not related to the routine, transfer back to the triage agent. """,
|
||||
tools=[update_seat],
|
||||
)
|
||||
|
||||
triage_agent = RealtimeAgent(
|
||||
name="Triage Agent",
|
||||
handoff_description="A triage agent that can delegate a customer's request to the appropriate agent.",
|
||||
instructions=(
|
||||
f"{RECOMMENDED_PROMPT_PREFIX} "
|
||||
"You are a helpful triaging agent. You can use your tools to delegate questions to other appropriate agents."
|
||||
),
|
||||
tools=[get_weather],
|
||||
handoffs=[
|
||||
realtime_handoff(faq_agent, tool_name_override="transfer_to_faq_agent"),
|
||||
realtime_handoff(seat_booking_agent, tool_name_override="transfer_to_seat_booking_agent"),
|
||||
],
|
||||
)
|
||||
|
||||
faq_agent.handoffs.append(
|
||||
realtime_handoff(triage_agent, tool_name_override="transfer_to_triage_agent")
|
||||
)
|
||||
seat_booking_agent.handoffs.append(
|
||||
realtime_handoff(triage_agent, tool_name_override="transfer_to_triage_agent")
|
||||
)
|
||||
|
||||
|
||||
def get_starting_agent() -> RealtimeAgent:
|
||||
return triage_agent
|
||||
@@ -0,0 +1,590 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import asdict
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from agents.realtime import RealtimeRunner, RealtimeSession, RealtimeSessionEvent
|
||||
from agents.realtime.config import RealtimeUserInputMessage
|
||||
from agents.realtime.items import RealtimeItem
|
||||
from agents.realtime.model import RealtimeModelConfig
|
||||
from agents.realtime.model_events import (
|
||||
RealtimeModelItemUpdatedEvent,
|
||||
RealtimeModelRawServerEvent,
|
||||
RealtimeModelUsageEvent,
|
||||
)
|
||||
from agents.realtime.model_inputs import RealtimeModelSendRawMessage
|
||||
|
||||
# Import TwilioHandler class - handle both module and package use cases
|
||||
if TYPE_CHECKING:
|
||||
# For type checking, use the relative import
|
||||
from .agent import get_starting_agent
|
||||
else:
|
||||
# At runtime, try both import styles
|
||||
try:
|
||||
# Try relative import first (when used as a package)
|
||||
from .agent import get_starting_agent
|
||||
except ImportError:
|
||||
# Fall back to direct import (when run as a script)
|
||||
from agent import get_starting_agent
|
||||
|
||||
|
||||
_requested_log_level = os.getenv("LOG_LEVEL", "INFO").upper()
|
||||
_log_level = getattr(logging, _requested_log_level, logging.INFO)
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(_log_level)
|
||||
|
||||
|
||||
class RealtimeWebSocketManager:
|
||||
def __init__(self):
|
||||
self.active_sessions: dict[str, RealtimeSession] = {}
|
||||
self.session_contexts: dict[str, Any] = {}
|
||||
self.websockets: dict[str, WebSocket] = {}
|
||||
|
||||
async def connect(self, websocket: WebSocket, session_id: str):
|
||||
await websocket.accept()
|
||||
self.websockets[session_id] = websocket
|
||||
|
||||
agent = get_starting_agent()
|
||||
runner = RealtimeRunner(agent)
|
||||
# If you want to customize the runner behavior, you can pass options:
|
||||
# runner_config = RealtimeRunConfig(async_tool_calls=False)
|
||||
# runner = RealtimeRunner(agent, config=runner_config)
|
||||
model_config: RealtimeModelConfig = {
|
||||
"initial_model_settings": {
|
||||
"model_name": "gpt-realtime-2.1",
|
||||
"turn_detection": {
|
||||
"type": "server_vad",
|
||||
"prefix_padding_ms": 300,
|
||||
"silence_duration_ms": 500,
|
||||
"interrupt_response": True,
|
||||
"create_response": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
session_context = await runner.run(model_config=model_config)
|
||||
session = await session_context.__aenter__()
|
||||
self.active_sessions[session_id] = session
|
||||
self.session_contexts[session_id] = session_context
|
||||
|
||||
# Start event processing task
|
||||
asyncio.create_task(self._process_events(session_id))
|
||||
|
||||
async def disconnect(self, session_id: str):
|
||||
if session_id in self.session_contexts:
|
||||
await self.session_contexts[session_id].__aexit__(None, None, None)
|
||||
del self.session_contexts[session_id]
|
||||
if session_id in self.active_sessions:
|
||||
del self.active_sessions[session_id]
|
||||
if session_id in self.websockets:
|
||||
del self.websockets[session_id]
|
||||
|
||||
async def send_audio(self, session_id: str, audio_bytes: bytes):
|
||||
if session_id in self.active_sessions:
|
||||
await self.active_sessions[session_id].send_audio(audio_bytes)
|
||||
|
||||
async def send_client_event(self, session_id: str, event: dict[str, Any]):
|
||||
"""Send a raw client event to the underlying realtime model."""
|
||||
session = self.active_sessions.get(session_id)
|
||||
if not session:
|
||||
return
|
||||
await session.model.send_event(
|
||||
RealtimeModelSendRawMessage(
|
||||
message={
|
||||
"type": event["type"],
|
||||
"other_data": {k: v for k, v in event.items() if k != "type"},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def send_user_message(self, session_id: str, message: RealtimeUserInputMessage):
|
||||
"""Send a structured user message via the higher-level API (supports input_image)."""
|
||||
session = self.active_sessions.get(session_id)
|
||||
if not session:
|
||||
return
|
||||
await session.send_message(message) # delegates to RealtimeModelSendUserInput path
|
||||
|
||||
async def approve_tool_call(self, session_id: str, call_id: str, *, always: bool = False):
|
||||
"""Approve a pending tool call for a session."""
|
||||
session = self.active_sessions.get(session_id)
|
||||
if not session:
|
||||
return
|
||||
await session.approve_tool_call(call_id, always=always)
|
||||
|
||||
async def reject_tool_call(self, session_id: str, call_id: str, *, always: bool = False):
|
||||
"""Reject a pending tool call for a session."""
|
||||
session = self.active_sessions.get(session_id)
|
||||
if not session:
|
||||
return
|
||||
await session.reject_tool_call(call_id, always=always)
|
||||
|
||||
async def interrupt(self, session_id: str) -> None:
|
||||
"""Interrupt current model playback/response for a session."""
|
||||
session = self.active_sessions.get(session_id)
|
||||
if not session:
|
||||
return
|
||||
await session.interrupt()
|
||||
|
||||
async def _process_events(self, session_id: str):
|
||||
try:
|
||||
session = self.active_sessions[session_id]
|
||||
websocket = self.websockets[session_id]
|
||||
|
||||
async for event in session:
|
||||
self._log_debug_event(session_id, event)
|
||||
event_data = await self._serialize_event(event)
|
||||
await websocket.send_text(json.dumps(event_data))
|
||||
except Exception as e:
|
||||
print(e)
|
||||
logger.error(f"Error processing events for session {session_id}: {e}")
|
||||
|
||||
def _log_debug_event(self, session_id: str, event: RealtimeSessionEvent) -> None:
|
||||
"""Log useful event summaries without noisy audio or delta payloads."""
|
||||
if not logger.isEnabledFor(logging.DEBUG):
|
||||
return
|
||||
|
||||
if event.type == "audio":
|
||||
return
|
||||
if event.type == "audio_end":
|
||||
return
|
||||
if event.type == "audio_interrupted":
|
||||
return
|
||||
|
||||
if event.type == "raw_model_event":
|
||||
self._log_debug_model_event(session_id, event)
|
||||
return
|
||||
|
||||
event_summary: dict[str, Any] = {"type": event.type}
|
||||
if event.type == "agent_start":
|
||||
event_summary["agent"] = event.agent.name
|
||||
elif event.type == "agent_end":
|
||||
event_summary["agent"] = event.agent.name
|
||||
elif event.type == "handoff":
|
||||
event_summary["from_agent"] = event.from_agent.name
|
||||
event_summary["to_agent"] = event.to_agent.name
|
||||
elif event.type == "tool_start":
|
||||
event_summary["tool"] = event.tool.name
|
||||
elif event.type == "tool_end":
|
||||
event_summary["tool"] = event.tool.name
|
||||
elif event.type == "tool_approval_required":
|
||||
event_summary.update(
|
||||
{
|
||||
"agent": event.agent.name,
|
||||
"tool": event.tool.name,
|
||||
"call_id": event.call_id,
|
||||
}
|
||||
)
|
||||
elif event.type == "history_updated":
|
||||
event_summary["item_count"] = len(event.history)
|
||||
if event.history:
|
||||
event_summary["last_item"] = self._item_debug_summary(event.history[-1])
|
||||
elif event.type == "history_added":
|
||||
event_summary["item"] = self._item_debug_summary(event.item)
|
||||
elif event.type == "guardrail_tripped":
|
||||
event_summary["guardrails"] = [
|
||||
result.guardrail.name for result in event.guardrail_results
|
||||
]
|
||||
elif event.type == "error":
|
||||
event_summary["error"] = str(event.error)
|
||||
elif event.type == "input_audio_timeout_triggered":
|
||||
pass
|
||||
else:
|
||||
assert_never(event)
|
||||
|
||||
logger.debug("Realtime session event session_id=%s event=%s", session_id, event_summary)
|
||||
|
||||
def _log_debug_model_event(self, session_id: str, event: Any) -> None:
|
||||
model_event = event.data
|
||||
if model_event.type in {"audio", "transcript_delta"}:
|
||||
return
|
||||
|
||||
if isinstance(model_event, RealtimeModelRawServerEvent):
|
||||
raw_event = model_event.data
|
||||
if not isinstance(raw_event, dict):
|
||||
return
|
||||
|
||||
raw_type = raw_event.get("type")
|
||||
if isinstance(raw_type, str) and raw_type.endswith(".delta"):
|
||||
return
|
||||
|
||||
raw_summary: dict[str, Any] = {
|
||||
"type": raw_type,
|
||||
"event_id": raw_event.get("event_id"),
|
||||
}
|
||||
response = raw_event.get("response")
|
||||
if isinstance(response, dict):
|
||||
raw_summary["response_id"] = response.get("id")
|
||||
raw_summary["response_status"] = response.get("status")
|
||||
item = raw_event.get("item")
|
||||
if isinstance(item, dict):
|
||||
raw_summary["item_id"] = item.get("id")
|
||||
raw_summary["item_type"] = item.get("type")
|
||||
else:
|
||||
raw_summary["item_id"] = raw_event.get("item_id")
|
||||
|
||||
raw_summary = {key: value for key, value in raw_summary.items() if value is not None}
|
||||
|
||||
if raw_type == "response.done":
|
||||
raw_summary["usage"] = response.get("usage") if isinstance(response, dict) else None
|
||||
logger.debug(
|
||||
"Realtime raw response completed session_id=%s event=%s",
|
||||
session_id,
|
||||
raw_summary,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"Realtime raw server event session_id=%s event=%s",
|
||||
session_id,
|
||||
raw_summary,
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(model_event, RealtimeModelUsageEvent):
|
||||
self._log_debug_usage_event(session_id, event, model_event)
|
||||
return
|
||||
|
||||
model_summary: dict[str, Any] = {"type": model_event.type}
|
||||
for field_name in (
|
||||
"item_id",
|
||||
"response_id",
|
||||
"call_id",
|
||||
"name",
|
||||
"status",
|
||||
"content_index",
|
||||
):
|
||||
value = getattr(model_event, field_name, None)
|
||||
if value is not None:
|
||||
model_summary[field_name] = value
|
||||
if isinstance(model_event, RealtimeModelItemUpdatedEvent):
|
||||
model_summary["item"] = self._item_debug_summary(model_event.item)
|
||||
|
||||
logger.debug(
|
||||
"Realtime model event session_id=%s event=%s",
|
||||
session_id,
|
||||
model_summary,
|
||||
)
|
||||
|
||||
def _log_debug_usage_event(
|
||||
self,
|
||||
session_id: str,
|
||||
event: Any,
|
||||
model_event: RealtimeModelUsageEvent,
|
||||
) -> None:
|
||||
response_usage = model_event.usage
|
||||
cumulative_usage = event.info.context.usage
|
||||
logger.debug(
|
||||
"Realtime typed response usage session_id=%s aggregate=%s "
|
||||
"input_details=%s output_details=%s",
|
||||
session_id,
|
||||
{
|
||||
"requests": response_usage.requests,
|
||||
"input_tokens": response_usage.input_tokens,
|
||||
"output_tokens": response_usage.output_tokens,
|
||||
"total_tokens": response_usage.total_tokens,
|
||||
"cached_input_tokens": response_usage.input_tokens_details.cached_tokens,
|
||||
},
|
||||
(
|
||||
asdict(model_event.input_tokens_details)
|
||||
if model_event.input_tokens_details is not None
|
||||
else None
|
||||
),
|
||||
(
|
||||
asdict(model_event.output_tokens_details)
|
||||
if model_event.output_tokens_details is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
logger.debug(
|
||||
"Realtime cumulative session usage session_id=%s aggregate=%s",
|
||||
session_id,
|
||||
{
|
||||
"requests": cumulative_usage.requests,
|
||||
"input_tokens": cumulative_usage.input_tokens,
|
||||
"output_tokens": cumulative_usage.output_tokens,
|
||||
"total_tokens": cumulative_usage.total_tokens,
|
||||
"cached_input_tokens": cumulative_usage.input_tokens_details.cached_tokens,
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _item_debug_summary(item: RealtimeItem) -> dict[str, Any]:
|
||||
content = getattr(item, "content", None)
|
||||
return {
|
||||
"item_id": item.item_id,
|
||||
"type": item.type,
|
||||
"role": getattr(item, "role", None),
|
||||
"status": getattr(item, "status", None),
|
||||
"content_types": (
|
||||
[getattr(part, "type", type(part).__name__) for part in content]
|
||||
if isinstance(content, list)
|
||||
else []
|
||||
),
|
||||
}
|
||||
|
||||
def _sanitize_history_item(self, item: RealtimeItem) -> dict[str, Any]:
|
||||
"""Remove large binary payloads from history items while keeping transcripts."""
|
||||
item_dict = item.model_dump()
|
||||
content = item_dict.get("content")
|
||||
if isinstance(content, list):
|
||||
sanitized_content: list[Any] = []
|
||||
for part in content:
|
||||
if isinstance(part, dict):
|
||||
sanitized_part = part.copy()
|
||||
if sanitized_part.get("type") in {"audio", "input_audio"}:
|
||||
sanitized_part.pop("audio", None)
|
||||
sanitized_content.append(sanitized_part)
|
||||
else:
|
||||
sanitized_content.append(part)
|
||||
item_dict["content"] = sanitized_content
|
||||
return item_dict
|
||||
|
||||
async def _serialize_event(self, event: RealtimeSessionEvent) -> dict[str, Any]:
|
||||
base_event: dict[str, Any] = {
|
||||
"type": event.type,
|
||||
}
|
||||
|
||||
if event.type == "agent_start":
|
||||
base_event["agent"] = event.agent.name
|
||||
elif event.type == "agent_end":
|
||||
base_event["agent"] = event.agent.name
|
||||
elif event.type == "handoff":
|
||||
base_event["from"] = event.from_agent.name
|
||||
base_event["to"] = event.to_agent.name
|
||||
elif event.type == "tool_start":
|
||||
base_event["tool"] = event.tool.name
|
||||
elif event.type == "tool_end":
|
||||
base_event["tool"] = event.tool.name
|
||||
base_event["output"] = str(event.output)
|
||||
elif event.type == "tool_approval_required":
|
||||
base_event["tool"] = event.tool.name
|
||||
base_event["call_id"] = event.call_id
|
||||
base_event["arguments"] = event.arguments
|
||||
base_event["agent"] = event.agent.name
|
||||
elif event.type == "audio":
|
||||
base_event["audio"] = base64.b64encode(event.audio.data).decode("utf-8")
|
||||
elif event.type == "audio_interrupted":
|
||||
pass
|
||||
elif event.type == "audio_end":
|
||||
pass
|
||||
elif event.type == "history_updated":
|
||||
base_event["history"] = [self._sanitize_history_item(item) for item in event.history]
|
||||
elif event.type == "history_added":
|
||||
# Provide the added item so the UI can render incrementally.
|
||||
try:
|
||||
base_event["item"] = self._sanitize_history_item(event.item)
|
||||
except Exception:
|
||||
base_event["item"] = None
|
||||
elif event.type == "guardrail_tripped":
|
||||
base_event["guardrail_results"] = [
|
||||
{"name": result.guardrail.name} for result in event.guardrail_results
|
||||
]
|
||||
elif event.type == "raw_model_event":
|
||||
base_event["raw_model_event"] = {
|
||||
"type": event.data.type,
|
||||
}
|
||||
elif event.type == "error":
|
||||
base_event["error"] = str(event.error) if hasattr(event, "error") else "Unknown error"
|
||||
elif event.type == "input_audio_timeout_triggered":
|
||||
pass
|
||||
else:
|
||||
assert_never(event)
|
||||
|
||||
return base_event
|
||||
|
||||
|
||||
manager = RealtimeWebSocketManager()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
|
||||
@app.websocket("/ws/{session_id}")
|
||||
async def websocket_endpoint(websocket: WebSocket, session_id: str):
|
||||
await manager.connect(websocket, session_id)
|
||||
image_buffers: dict[str, dict[str, Any]] = {}
|
||||
try:
|
||||
while True:
|
||||
data = await websocket.receive_text()
|
||||
message = json.loads(data)
|
||||
|
||||
if message["type"] == "audio":
|
||||
# Convert int16 array to bytes
|
||||
int16_data = message["data"]
|
||||
audio_bytes = struct.pack(f"{len(int16_data)}h", *int16_data)
|
||||
await manager.send_audio(session_id, audio_bytes)
|
||||
elif message["type"] == "image":
|
||||
logger.info("Received image message from client (session %s).", session_id)
|
||||
# Build a conversation.item.create with input_image (and optional input_text)
|
||||
data_url = message.get("data_url")
|
||||
prompt_text = message.get("text") or "Please describe this image."
|
||||
if data_url:
|
||||
logger.info(
|
||||
"Forwarding image (structured message) to Realtime API (len=%d).",
|
||||
len(data_url),
|
||||
)
|
||||
user_msg: RealtimeUserInputMessage = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": (
|
||||
[
|
||||
{"type": "input_image", "image_url": data_url, "detail": "high"},
|
||||
{"type": "input_text", "text": prompt_text},
|
||||
]
|
||||
if prompt_text
|
||||
else [{"type": "input_image", "image_url": data_url, "detail": "high"}]
|
||||
),
|
||||
}
|
||||
await manager.send_user_message(session_id, user_msg)
|
||||
# Acknowledge to client UI
|
||||
await websocket.send_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "client_info",
|
||||
"info": "image_enqueued",
|
||||
"size": len(data_url),
|
||||
}
|
||||
)
|
||||
)
|
||||
else:
|
||||
await websocket.send_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "error",
|
||||
"error": "No data_url for image message.",
|
||||
}
|
||||
)
|
||||
)
|
||||
elif message["type"] == "commit_audio":
|
||||
# Force close the current input audio turn
|
||||
await manager.send_client_event(session_id, {"type": "input_audio_buffer.commit"})
|
||||
elif message["type"] == "image_start":
|
||||
img_id = str(message.get("id"))
|
||||
image_buffers[img_id] = {
|
||||
"text": message.get("text") or "Please describe this image.",
|
||||
"chunks": [],
|
||||
}
|
||||
await websocket.send_text(
|
||||
json.dumps({"type": "client_info", "info": "image_start_ack", "id": img_id})
|
||||
)
|
||||
elif message["type"] == "image_chunk":
|
||||
img_id = str(message.get("id"))
|
||||
chunk = message.get("chunk", "")
|
||||
if img_id in image_buffers:
|
||||
image_buffers[img_id]["chunks"].append(chunk)
|
||||
if len(image_buffers[img_id]["chunks"]) % 10 == 0:
|
||||
await websocket.send_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "client_info",
|
||||
"info": "image_chunk_ack",
|
||||
"id": img_id,
|
||||
"count": len(image_buffers[img_id]["chunks"]),
|
||||
}
|
||||
)
|
||||
)
|
||||
elif message["type"] == "image_end":
|
||||
img_id = str(message.get("id"))
|
||||
buf = image_buffers.pop(img_id, None)
|
||||
if buf is None:
|
||||
await websocket.send_text(
|
||||
json.dumps({"type": "error", "error": "Unknown image id for image_end."})
|
||||
)
|
||||
else:
|
||||
data_url = "".join(buf["chunks"]) if buf["chunks"] else None
|
||||
prompt_text = buf["text"]
|
||||
if data_url:
|
||||
logger.info(
|
||||
"Forwarding chunked image (structured message) to Realtime API (len=%d).",
|
||||
len(data_url),
|
||||
)
|
||||
user_msg2: RealtimeUserInputMessage = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": (
|
||||
[
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": data_url,
|
||||
"detail": "high",
|
||||
},
|
||||
{"type": "input_text", "text": prompt_text},
|
||||
]
|
||||
if prompt_text
|
||||
else [
|
||||
{"type": "input_image", "image_url": data_url, "detail": "high"}
|
||||
]
|
||||
),
|
||||
}
|
||||
await manager.send_user_message(session_id, user_msg2)
|
||||
await websocket.send_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "client_info",
|
||||
"info": "image_enqueued",
|
||||
"id": img_id,
|
||||
"size": len(data_url),
|
||||
}
|
||||
)
|
||||
)
|
||||
else:
|
||||
await websocket.send_text(
|
||||
json.dumps({"type": "error", "error": "Empty image."})
|
||||
)
|
||||
elif message["type"] == "tool_approval_decision":
|
||||
call_id = message.get("call_id")
|
||||
approve = bool(message.get("approve"))
|
||||
always = bool(message.get("always", False))
|
||||
if not call_id:
|
||||
await websocket.send_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "error",
|
||||
"error": "Missing call_id for tool approval decision.",
|
||||
}
|
||||
)
|
||||
)
|
||||
continue
|
||||
if approve:
|
||||
await manager.approve_tool_call(session_id, call_id, always=always)
|
||||
else:
|
||||
await manager.reject_tool_call(session_id, call_id, always=always)
|
||||
elif message["type"] == "interrupt":
|
||||
await manager.interrupt(session_id)
|
||||
|
||||
except WebSocketDisconnect:
|
||||
await manager.disconnect(session_id)
|
||||
|
||||
|
||||
app.mount("/", StaticFiles(directory="static", html=True), name="static")
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def read_index():
|
||||
return FileResponse("static/index.html")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(
|
||||
app,
|
||||
host="0.0.0.0",
|
||||
port=8000,
|
||||
# Increased WebSocket frame size to comfortably handle image data URLs.
|
||||
ws_max_size=16 * 1024 * 1024,
|
||||
)
|
||||
@@ -0,0 +1,713 @@
|
||||
class RealtimeDemo {
|
||||
constructor() {
|
||||
this.ws = null;
|
||||
this.isConnected = false;
|
||||
this.isMuted = false;
|
||||
this.isCapturing = false;
|
||||
this.audioContext = null;
|
||||
this.captureSource = null;
|
||||
this.captureNode = null;
|
||||
this.stream = null;
|
||||
this.sessionId = this.generateSessionId();
|
||||
|
||||
this.isPlayingAudio = false;
|
||||
this.playbackAudioContext = null;
|
||||
this.playbackNode = null;
|
||||
this.playbackInitPromise = null;
|
||||
this.pendingPlaybackChunks = [];
|
||||
this.playbackFadeSec = 0.02; // ~20ms fade to reduce clicks
|
||||
this.messageNodes = new Map(); // item_id -> DOM node
|
||||
this.seenItemIds = new Set(); // item_id set for append-only syncing
|
||||
|
||||
this.initializeElements();
|
||||
this.setupEventListeners();
|
||||
}
|
||||
|
||||
initializeElements() {
|
||||
this.connectBtn = document.getElementById('connectBtn');
|
||||
this.muteBtn = document.getElementById('muteBtn');
|
||||
this.imageBtn = document.getElementById('imageBtn');
|
||||
this.imageInput = document.getElementById('imageInput');
|
||||
this.imagePrompt = document.getElementById('imagePrompt');
|
||||
this.status = document.getElementById('status');
|
||||
this.messagesContent = document.getElementById('messagesContent');
|
||||
this.eventsContent = document.getElementById('eventsContent');
|
||||
this.toolsContent = document.getElementById('toolsContent');
|
||||
}
|
||||
|
||||
setupEventListeners() {
|
||||
this.connectBtn.addEventListener('click', () => {
|
||||
if (this.isConnected) {
|
||||
this.disconnect();
|
||||
} else {
|
||||
this.connect();
|
||||
}
|
||||
});
|
||||
|
||||
this.muteBtn.addEventListener('click', () => {
|
||||
this.toggleMute();
|
||||
});
|
||||
|
||||
// Image upload
|
||||
this.imageBtn.addEventListener('click', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
console.log('Send Image clicked');
|
||||
// Programmatically open the hidden file input
|
||||
this.imageInput.click();
|
||||
});
|
||||
|
||||
this.imageInput.addEventListener('change', async (e) => {
|
||||
console.log('Image input change fired');
|
||||
const file = e.target.files && e.target.files[0];
|
||||
if (!file) return;
|
||||
await this._handlePickedFile(file);
|
||||
this.imageInput.value = '';
|
||||
});
|
||||
|
||||
this._handlePickedFile = async (file) => {
|
||||
try {
|
||||
const dataUrl = await this.prepareDataURL(file);
|
||||
const promptText = (this.imagePrompt && this.imagePrompt.value) || '';
|
||||
// Send to server; server forwards to Realtime API.
|
||||
// Use chunked frames to avoid WS frame limits.
|
||||
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
|
||||
console.log('Interrupting and sending image (chunked) to server WebSocket');
|
||||
// Stop any current audio locally and tell model to interrupt
|
||||
this.stopAudioPlayback();
|
||||
this.ws.send(JSON.stringify({ type: 'interrupt' }));
|
||||
const id = 'img_' + Math.random().toString(36).slice(2);
|
||||
const CHUNK = 60_000; // ~60KB per frame
|
||||
this.ws.send(JSON.stringify({ type: 'image_start', id, text: promptText }));
|
||||
for (let i = 0; i < dataUrl.length; i += CHUNK) {
|
||||
const chunk = dataUrl.slice(i, i + CHUNK);
|
||||
this.ws.send(JSON.stringify({ type: 'image_chunk', id, chunk }));
|
||||
}
|
||||
this.ws.send(JSON.stringify({ type: 'image_end', id }));
|
||||
} else {
|
||||
console.warn('Not connected; image will not be sent. Click Connect first.');
|
||||
}
|
||||
// Add to UI immediately for better feedback
|
||||
console.log('Adding local user image bubble');
|
||||
this.addUserImageMessage(dataUrl, promptText);
|
||||
} catch (err) {
|
||||
console.error('Failed to process image:', err);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
generateSessionId() {
|
||||
return 'session_' + Math.random().toString(36).substr(2, 9);
|
||||
}
|
||||
|
||||
async connect() {
|
||||
try {
|
||||
this.ws = new WebSocket(`ws://localhost:8000/ws/${this.sessionId}`);
|
||||
|
||||
this.ws.onopen = () => {
|
||||
this.isConnected = true;
|
||||
this.updateConnectionUI();
|
||||
this.startContinuousCapture();
|
||||
};
|
||||
|
||||
this.ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
this.handleRealtimeEvent(data);
|
||||
};
|
||||
|
||||
this.ws.onclose = () => {
|
||||
this.isConnected = false;
|
||||
this.updateConnectionUI();
|
||||
};
|
||||
|
||||
this.ws.onerror = (error) => {
|
||||
console.error('WebSocket error:', error);
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to connect:', error);
|
||||
}
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
if (this.ws) {
|
||||
this.ws.close();
|
||||
}
|
||||
this.stopContinuousCapture();
|
||||
}
|
||||
|
||||
updateConnectionUI() {
|
||||
if (this.isConnected) {
|
||||
this.connectBtn.textContent = 'Disconnect';
|
||||
this.connectBtn.className = 'connect-btn connected';
|
||||
this.status.textContent = 'Connected';
|
||||
this.status.className = 'status connected';
|
||||
this.muteBtn.disabled = false;
|
||||
} else {
|
||||
this.connectBtn.textContent = 'Connect';
|
||||
this.connectBtn.className = 'connect-btn disconnected';
|
||||
this.status.textContent = 'Disconnected';
|
||||
this.status.className = 'status disconnected';
|
||||
this.muteBtn.disabled = true;
|
||||
}
|
||||
}
|
||||
|
||||
toggleMute() {
|
||||
this.isMuted = !this.isMuted;
|
||||
this.updateMuteUI();
|
||||
}
|
||||
|
||||
updateMuteUI() {
|
||||
if (this.isMuted) {
|
||||
this.muteBtn.textContent = '🔇 Mic Off';
|
||||
this.muteBtn.className = 'mute-btn muted';
|
||||
} else {
|
||||
this.muteBtn.textContent = '🎤 Mic On';
|
||||
this.muteBtn.className = 'mute-btn unmuted';
|
||||
if (this.isCapturing) {
|
||||
this.muteBtn.classList.add('active');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
readFileAsDataURL(file) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(reader.result);
|
||||
reader.onerror = reject;
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
async prepareDataURL(file) {
|
||||
const original = await this.readFileAsDataURL(file);
|
||||
try {
|
||||
const img = new Image();
|
||||
img.decoding = 'async';
|
||||
const loaded = new Promise((res, rej) => {
|
||||
img.onload = () => res();
|
||||
img.onerror = rej;
|
||||
});
|
||||
img.src = original;
|
||||
await loaded;
|
||||
|
||||
const maxDim = 1024;
|
||||
const maxSide = Math.max(img.width, img.height);
|
||||
const scale = maxSide > maxDim ? (maxDim / maxSide) : 1;
|
||||
const w = Math.max(1, Math.round(img.width * scale));
|
||||
const h = Math.max(1, Math.round(img.height * scale));
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = w; canvas.height = h;
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(img, 0, 0, w, h);
|
||||
return canvas.toDataURL('image/jpeg', 0.85);
|
||||
} catch (e) {
|
||||
console.warn('Image resize failed; sending original', e);
|
||||
return original;
|
||||
}
|
||||
}
|
||||
|
||||
async startContinuousCapture() {
|
||||
if (!this.isConnected || this.isCapturing) return;
|
||||
|
||||
// Check if getUserMedia is available
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
throw new Error('getUserMedia not available. Please use HTTPS or localhost.');
|
||||
}
|
||||
|
||||
try {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
sampleRate: 24000,
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true
|
||||
}
|
||||
});
|
||||
|
||||
this.audioContext = new AudioContext({ sampleRate: 24000, latencyHint: 'interactive' });
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
try { await this.audioContext.resume(); } catch {}
|
||||
}
|
||||
|
||||
if (!this.audioContext.audioWorklet) {
|
||||
throw new Error('AudioWorklet API not supported in this browser.');
|
||||
}
|
||||
|
||||
await this.audioContext.audioWorklet.addModule('audio-recorder.worklet.js');
|
||||
|
||||
this.captureSource = this.audioContext.createMediaStreamSource(this.stream);
|
||||
this.captureNode = new AudioWorkletNode(this.audioContext, 'pcm-recorder');
|
||||
|
||||
this.captureNode.port.onmessage = (event) => {
|
||||
if (this.isMuted) return;
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
||||
|
||||
const chunk = event.data instanceof ArrayBuffer ? new Int16Array(event.data) : event.data;
|
||||
if (!chunk || !(chunk instanceof Int16Array) || chunk.length === 0) return;
|
||||
|
||||
this.ws.send(JSON.stringify({
|
||||
type: 'audio',
|
||||
data: Array.from(chunk)
|
||||
}));
|
||||
};
|
||||
|
||||
this.captureSource.connect(this.captureNode);
|
||||
this.captureNode.connect(this.audioContext.destination);
|
||||
|
||||
this.isCapturing = true;
|
||||
this.updateMuteUI();
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to start audio capture:', error);
|
||||
}
|
||||
}
|
||||
|
||||
stopContinuousCapture() {
|
||||
if (!this.isCapturing) return;
|
||||
|
||||
this.isCapturing = false;
|
||||
|
||||
if (this.captureSource) {
|
||||
try { this.captureSource.disconnect(); } catch {}
|
||||
this.captureSource = null;
|
||||
}
|
||||
|
||||
if (this.captureNode) {
|
||||
this.captureNode.port.onmessage = null;
|
||||
try { this.captureNode.disconnect(); } catch {}
|
||||
this.captureNode = null;
|
||||
}
|
||||
|
||||
if (this.audioContext) {
|
||||
this.audioContext.close();
|
||||
this.audioContext = null;
|
||||
}
|
||||
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach(track => track.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
|
||||
this.updateMuteUI();
|
||||
}
|
||||
|
||||
handleRealtimeEvent(event) {
|
||||
// Add to raw events pane
|
||||
this.addRawEvent(event);
|
||||
|
||||
// Add to tools panel if it's a tool or handoff event
|
||||
if (event.type === 'tool_start' || event.type === 'tool_end' || event.type === 'handoff' || event.type === 'tool_approval_required') {
|
||||
this.addToolEvent(event);
|
||||
}
|
||||
|
||||
// Handle specific event types
|
||||
switch (event.type) {
|
||||
case 'audio':
|
||||
this.playAudio(event.audio);
|
||||
break;
|
||||
case 'audio_interrupted':
|
||||
this.stopAudioPlayback();
|
||||
break;
|
||||
case 'input_audio_timeout_triggered':
|
||||
// Ask server to commit the input buffer to expedite model response
|
||||
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
|
||||
this.ws.send(JSON.stringify({ type: 'commit_audio' }));
|
||||
}
|
||||
break;
|
||||
case 'history_updated':
|
||||
this.syncMissingFromHistory(event.history);
|
||||
this.updateLastMessageFromHistory(event.history);
|
||||
break;
|
||||
case 'history_added':
|
||||
// Append just the new item without clearing the thread.
|
||||
if (event.item) {
|
||||
this.addMessageFromItem(event.item);
|
||||
}
|
||||
break;
|
||||
case 'tool_approval_required':
|
||||
this.promptForToolApproval(event);
|
||||
break;
|
||||
}
|
||||
}
|
||||
updateLastMessageFromHistory(history) {
|
||||
if (!history || !Array.isArray(history) || history.length === 0) return;
|
||||
// Find the last message item in history
|
||||
let last = null;
|
||||
for (let i = history.length - 1; i >= 0; i--) {
|
||||
const it = history[i];
|
||||
if (it && it.type === 'message') { last = it; break; }
|
||||
}
|
||||
if (!last) return;
|
||||
const itemId = last.item_id;
|
||||
|
||||
// Extract a text representation (for assistant transcript updates)
|
||||
let text = '';
|
||||
if (Array.isArray(last.content)) {
|
||||
for (const part of last.content) {
|
||||
if (!part || typeof part !== 'object') continue;
|
||||
if (part.type === 'text' && part.text) text += part.text;
|
||||
else if (part.type === 'input_text' && part.text) text += part.text;
|
||||
else if ((part.type === 'input_audio' || part.type === 'audio') && part.transcript) text += part.transcript;
|
||||
}
|
||||
}
|
||||
|
||||
const node = this.messageNodes.get(itemId);
|
||||
if (!node) {
|
||||
// If we haven't rendered this item yet, append it now.
|
||||
this.addMessageFromItem(last);
|
||||
return;
|
||||
}
|
||||
|
||||
// Update only the text content of the bubble, preserving any images already present.
|
||||
const bubble = node.querySelector('.message-bubble');
|
||||
if (bubble && text && text.trim()) {
|
||||
// If there's an <img>, keep it and only update the trailing caption/text node.
|
||||
const hasImg = !!bubble.querySelector('img');
|
||||
if (hasImg) {
|
||||
// Ensure there is a caption div after the image
|
||||
let cap = bubble.querySelector('.image-caption');
|
||||
if (!cap) {
|
||||
cap = document.createElement('div');
|
||||
cap.className = 'image-caption';
|
||||
cap.style.marginTop = '0.5rem';
|
||||
bubble.appendChild(cap);
|
||||
}
|
||||
cap.textContent = text.trim();
|
||||
} else {
|
||||
bubble.textContent = text.trim();
|
||||
}
|
||||
this.scrollToBottom();
|
||||
}
|
||||
}
|
||||
|
||||
syncMissingFromHistory(history) {
|
||||
if (!history || !Array.isArray(history)) return;
|
||||
for (const item of history) {
|
||||
if (!item || item.type !== 'message') continue;
|
||||
const id = item.item_id;
|
||||
if (!id) continue;
|
||||
if (!this.seenItemIds.has(id)) {
|
||||
this.addMessageFromItem(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
addMessageFromItem(item) {
|
||||
try {
|
||||
if (!item || item.type !== 'message') return;
|
||||
const role = item.role;
|
||||
let content = '';
|
||||
let imageUrls = [];
|
||||
|
||||
if (Array.isArray(item.content)) {
|
||||
for (const contentPart of item.content) {
|
||||
if (!contentPart || typeof contentPart !== 'object') continue;
|
||||
if (contentPart.type === 'text' && contentPart.text) {
|
||||
content += contentPart.text;
|
||||
} else if (contentPart.type === 'input_text' && contentPart.text) {
|
||||
content += contentPart.text;
|
||||
} else if (contentPart.type === 'input_audio' && contentPart.transcript) {
|
||||
content += contentPart.transcript;
|
||||
} else if (contentPart.type === 'audio' && contentPart.transcript) {
|
||||
content += contentPart.transcript;
|
||||
} else if (contentPart.type === 'input_image') {
|
||||
const url = contentPart.image_url || contentPart.url;
|
||||
if (typeof url === 'string' && url) imageUrls.push(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let node = null;
|
||||
if (imageUrls.length > 0) {
|
||||
for (const url of imageUrls) {
|
||||
node = this.addImageMessage(role, url, content.trim());
|
||||
}
|
||||
} else if (content && content.trim()) {
|
||||
node = this.addMessage(role, content.trim());
|
||||
}
|
||||
if (node && item.item_id) {
|
||||
this.messageNodes.set(item.item_id, node);
|
||||
this.seenItemIds.add(item.item_id);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to add message from item:', e, item);
|
||||
}
|
||||
}
|
||||
|
||||
addMessage(type, content) {
|
||||
const messageDiv = document.createElement('div');
|
||||
messageDiv.className = `message ${type}`;
|
||||
|
||||
const bubbleDiv = document.createElement('div');
|
||||
bubbleDiv.className = 'message-bubble';
|
||||
bubbleDiv.textContent = content;
|
||||
|
||||
messageDiv.appendChild(bubbleDiv);
|
||||
this.messagesContent.appendChild(messageDiv);
|
||||
this.scrollToBottom();
|
||||
|
||||
return messageDiv;
|
||||
}
|
||||
|
||||
addImageMessage(role, imageUrl, caption = '') {
|
||||
const messageDiv = document.createElement('div');
|
||||
messageDiv.className = `message ${role}`;
|
||||
|
||||
const bubbleDiv = document.createElement('div');
|
||||
bubbleDiv.className = 'message-bubble';
|
||||
|
||||
const img = document.createElement('img');
|
||||
img.src = imageUrl;
|
||||
img.alt = 'Uploaded image';
|
||||
img.style.maxWidth = '220px';
|
||||
img.style.borderRadius = '8px';
|
||||
img.style.display = 'block';
|
||||
|
||||
bubbleDiv.appendChild(img);
|
||||
if (caption) {
|
||||
const cap = document.createElement('div');
|
||||
cap.textContent = caption;
|
||||
cap.style.marginTop = '0.5rem';
|
||||
bubbleDiv.appendChild(cap);
|
||||
}
|
||||
|
||||
messageDiv.appendChild(bubbleDiv);
|
||||
this.messagesContent.appendChild(messageDiv);
|
||||
this.scrollToBottom();
|
||||
|
||||
return messageDiv;
|
||||
}
|
||||
|
||||
addUserImageMessage(imageUrl, caption = '') {
|
||||
return this.addImageMessage('user', imageUrl, caption);
|
||||
}
|
||||
|
||||
addRawEvent(event) {
|
||||
const eventDiv = document.createElement('div');
|
||||
eventDiv.className = 'event';
|
||||
|
||||
const headerDiv = document.createElement('div');
|
||||
headerDiv.className = 'event-header';
|
||||
headerDiv.innerHTML = `
|
||||
<span>${event.type}</span>
|
||||
<span>▼</span>
|
||||
`;
|
||||
|
||||
const contentDiv = document.createElement('div');
|
||||
contentDiv.className = 'event-content collapsed';
|
||||
contentDiv.textContent = JSON.stringify(event, null, 2);
|
||||
|
||||
headerDiv.addEventListener('click', () => {
|
||||
const isCollapsed = contentDiv.classList.contains('collapsed');
|
||||
contentDiv.classList.toggle('collapsed');
|
||||
headerDiv.querySelector('span:last-child').textContent = isCollapsed ? '▲' : '▼';
|
||||
});
|
||||
|
||||
eventDiv.appendChild(headerDiv);
|
||||
eventDiv.appendChild(contentDiv);
|
||||
this.eventsContent.appendChild(eventDiv);
|
||||
|
||||
// Auto-scroll events pane
|
||||
this.eventsContent.scrollTop = this.eventsContent.scrollHeight;
|
||||
}
|
||||
|
||||
addToolEvent(event) {
|
||||
const eventDiv = document.createElement('div');
|
||||
eventDiv.className = 'event';
|
||||
|
||||
let title = '';
|
||||
let description = '';
|
||||
let eventClass = '';
|
||||
|
||||
if (event.type === 'handoff') {
|
||||
title = `🔄 Handoff`;
|
||||
description = `From ${event.from} to ${event.to}`;
|
||||
eventClass = 'handoff';
|
||||
} else if (event.type === 'tool_start') {
|
||||
title = `🔧 Tool Started`;
|
||||
description = `Running ${event.tool}`;
|
||||
eventClass = 'tool';
|
||||
} else if (event.type === 'tool_end') {
|
||||
title = `✅ Tool Completed`;
|
||||
description = `${event.tool}: ${event.output || 'No output'}`;
|
||||
eventClass = 'tool';
|
||||
} else if (event.type === 'tool_approval_required') {
|
||||
title = `⏸️ Approval Needed`;
|
||||
description = `Waiting on ${event.tool}`;
|
||||
eventClass = 'tool';
|
||||
} else if (event.type === 'tool_approval_decision') {
|
||||
title = event.approved ? '✅ Approved' : '❌ Rejected';
|
||||
description = `${event.tool} (${event.call_id || 'call'})`;
|
||||
eventClass = 'tool';
|
||||
}
|
||||
|
||||
eventDiv.innerHTML = `
|
||||
<div class="event-header ${eventClass}">
|
||||
<div>
|
||||
<div style="font-weight: 600; margin-bottom: 2px;">${title}</div>
|
||||
<div style="font-size: 0.8rem; opacity: 0.8;">${description}</div>
|
||||
</div>
|
||||
<span style="font-size: 0.7rem; opacity: 0.6;">${new Date().toLocaleTimeString()}</span>
|
||||
</div>
|
||||
`;
|
||||
|
||||
this.toolsContent.appendChild(eventDiv);
|
||||
|
||||
// Auto-scroll tools pane
|
||||
this.toolsContent.scrollTop = this.toolsContent.scrollHeight;
|
||||
}
|
||||
|
||||
promptForToolApproval(event) {
|
||||
const args = event.arguments || '';
|
||||
const preview = args ? `${args.slice(0, 180)}${args.length > 180 ? '…' : ''}` : '';
|
||||
const message = `Allow tool "${event.tool}" to run?${preview ? `\nArgs: ${preview}` : ''}`;
|
||||
const approved = window.confirm(message);
|
||||
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
|
||||
this.ws.send(JSON.stringify({
|
||||
type: 'tool_approval_decision',
|
||||
call_id: event.call_id,
|
||||
approve: approved
|
||||
}));
|
||||
}
|
||||
this.addToolEvent({
|
||||
type: 'tool_approval_decision',
|
||||
tool: event.tool,
|
||||
call_id: event.call_id,
|
||||
approved
|
||||
});
|
||||
}
|
||||
|
||||
async playAudio(audioBase64) {
|
||||
try {
|
||||
if (!audioBase64 || audioBase64.length === 0) {
|
||||
console.warn('Received empty audio data, skipping playback');
|
||||
return;
|
||||
}
|
||||
|
||||
const int16Array = this.decodeBase64ToInt16(audioBase64);
|
||||
if (!int16Array || int16Array.length === 0) {
|
||||
console.warn('Audio chunk has no samples, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
this.pendingPlaybackChunks.push(int16Array);
|
||||
await this.ensurePlaybackNode();
|
||||
this.flushPendingPlaybackChunks();
|
||||
|
||||
} catch (error) {
|
||||
console.error('Failed to play audio:', error);
|
||||
this.pendingPlaybackChunks = [];
|
||||
}
|
||||
}
|
||||
|
||||
async ensurePlaybackNode() {
|
||||
if (this.playbackNode) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.playbackInitPromise) {
|
||||
this.playbackInitPromise = (async () => {
|
||||
if (!this.playbackAudioContext) {
|
||||
this.playbackAudioContext = new AudioContext({ sampleRate: 24000, latencyHint: 'interactive' });
|
||||
}
|
||||
|
||||
if (this.playbackAudioContext.state === 'suspended') {
|
||||
try { await this.playbackAudioContext.resume(); } catch {}
|
||||
}
|
||||
|
||||
if (!this.playbackAudioContext.audioWorklet) {
|
||||
throw new Error('AudioWorklet API not supported in this browser.');
|
||||
}
|
||||
|
||||
await this.playbackAudioContext.audioWorklet.addModule('audio-playback.worklet.js');
|
||||
|
||||
this.playbackNode = new AudioWorkletNode(this.playbackAudioContext, 'pcm-playback', { outputChannelCount: [1] });
|
||||
this.playbackNode.port.onmessage = (event) => {
|
||||
const message = event.data;
|
||||
if (!message || typeof message !== 'object') return;
|
||||
if (message.type === 'drained') {
|
||||
this.isPlayingAudio = false;
|
||||
}
|
||||
};
|
||||
|
||||
// Provide initial configuration for fades.
|
||||
const fadeSamples = Math.floor(this.playbackAudioContext.sampleRate * this.playbackFadeSec);
|
||||
this.playbackNode.port.postMessage({ type: 'config', fadeSamples });
|
||||
|
||||
this.playbackNode.connect(this.playbackAudioContext.destination);
|
||||
})().catch((error) => {
|
||||
this.playbackInitPromise = null;
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
|
||||
await this.playbackInitPromise;
|
||||
}
|
||||
|
||||
flushPendingPlaybackChunks() {
|
||||
if (!this.playbackNode) {
|
||||
return;
|
||||
}
|
||||
|
||||
while (this.pendingPlaybackChunks.length > 0) {
|
||||
const chunk = this.pendingPlaybackChunks.shift();
|
||||
if (!chunk || !(chunk instanceof Int16Array) || chunk.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
this.playbackNode.port.postMessage(
|
||||
{ type: 'chunk', payload: chunk.buffer },
|
||||
[chunk.buffer]
|
||||
);
|
||||
this.isPlayingAudio = true;
|
||||
} catch (error) {
|
||||
console.error('Failed to enqueue audio chunk to worklet:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
decodeBase64ToInt16(audioBase64) {
|
||||
try {
|
||||
const binaryString = atob(audioBase64);
|
||||
const length = binaryString.length;
|
||||
const bytes = new Uint8Array(length);
|
||||
for (let i = 0; i < length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return new Int16Array(bytes.buffer);
|
||||
} catch (error) {
|
||||
console.error('Failed to decode audio chunk:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
stopAudioPlayback() {
|
||||
console.log('Stopping audio playback due to interruption');
|
||||
|
||||
this.pendingPlaybackChunks = [];
|
||||
|
||||
if (this.playbackNode) {
|
||||
try {
|
||||
this.playbackNode.port.postMessage({ type: 'stop' });
|
||||
} catch (error) {
|
||||
console.error('Failed to notify playback worklet to stop:', error);
|
||||
}
|
||||
}
|
||||
|
||||
this.isPlayingAudio = false;
|
||||
|
||||
console.log('Audio playback stopped and queue cleared');
|
||||
}
|
||||
|
||||
scrollToBottom() {
|
||||
this.messagesContent.scrollTop = this.messagesContent.scrollHeight;
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize the demo when the page loads
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
new RealtimeDemo();
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
class PCMPlaybackProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.buffers = [];
|
||||
this.currentBuffer = null;
|
||||
this.currentIndex = 0;
|
||||
this.isCurrentlyPlaying = false;
|
||||
this.fadeSamples = Math.round(sampleRate * 0.02);
|
||||
|
||||
this.port.onmessage = (event) => {
|
||||
const message = event.data;
|
||||
if (!message || typeof message !== 'object') return;
|
||||
|
||||
if (message.type === 'chunk') {
|
||||
const payload = message.payload;
|
||||
if (!(payload instanceof ArrayBuffer)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int16Data = new Int16Array(payload);
|
||||
if (int16Data.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const scale = 1 / 32768;
|
||||
const floatData = new Float32Array(int16Data.length);
|
||||
for (let i = 0; i < int16Data.length; i++) {
|
||||
floatData[i] = Math.max(-1, Math.min(1, int16Data[i] * scale));
|
||||
}
|
||||
|
||||
if (!this.hasPendingAudio()) {
|
||||
const fadeSamples = Math.min(this.fadeSamples, floatData.length);
|
||||
for (let i = 0; i < fadeSamples; i++) {
|
||||
const gain = fadeSamples <= 1 ? 1 : (i / fadeSamples);
|
||||
floatData[i] *= gain;
|
||||
}
|
||||
}
|
||||
|
||||
this.buffers.push(floatData);
|
||||
|
||||
} else if (message.type === 'stop') {
|
||||
this.reset();
|
||||
this.port.postMessage({ type: 'drained' });
|
||||
|
||||
} else if (message.type === 'config') {
|
||||
const fadeSamples = message.fadeSamples;
|
||||
if (Number.isFinite(fadeSamples) && fadeSamples >= 0) {
|
||||
this.fadeSamples = fadeSamples >>> 0;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.buffers = [];
|
||||
this.currentBuffer = null;
|
||||
this.currentIndex = 0;
|
||||
this.isCurrentlyPlaying = false;
|
||||
}
|
||||
|
||||
hasPendingAudio() {
|
||||
if (this.currentBuffer && this.currentIndex < this.currentBuffer.length) {
|
||||
return true;
|
||||
}
|
||||
return this.buffers.length > 0;
|
||||
}
|
||||
|
||||
pullSample() {
|
||||
if (this.currentBuffer && this.currentIndex < this.currentBuffer.length) {
|
||||
return this.currentBuffer[this.currentIndex++];
|
||||
}
|
||||
|
||||
if (this.currentBuffer && this.currentIndex >= this.currentBuffer.length) {
|
||||
this.currentBuffer = null;
|
||||
this.currentIndex = 0;
|
||||
}
|
||||
|
||||
while (this.buffers.length > 0) {
|
||||
this.currentBuffer = this.buffers.shift();
|
||||
this.currentIndex = 0;
|
||||
if (this.currentBuffer && this.currentBuffer.length > 0) {
|
||||
return this.currentBuffer[this.currentIndex++];
|
||||
}
|
||||
}
|
||||
|
||||
this.currentBuffer = null;
|
||||
this.currentIndex = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
process(inputs, outputs) {
|
||||
const output = outputs[0];
|
||||
if (!output || output.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const channel = output[0];
|
||||
let wroteSamples = false;
|
||||
|
||||
for (let i = 0; i < channel.length; i++) {
|
||||
const sample = this.pullSample();
|
||||
channel[i] = sample;
|
||||
if (sample !== 0) {
|
||||
wroteSamples = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (this.hasPendingAudio()) {
|
||||
this.isCurrentlyPlaying = true;
|
||||
} else if (!wroteSamples && this.isCurrentlyPlaying) {
|
||||
this.isCurrentlyPlaying = false;
|
||||
this.port.postMessage({ type: 'drained' });
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('pcm-playback', PCMPlaybackProcessor);
|
||||
@@ -0,0 +1,56 @@
|
||||
class PCMRecorderProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.chunkSize = 4096;
|
||||
this.buffer = new Int16Array(this.chunkSize);
|
||||
this.offset = 0;
|
||||
this.pendingFrames = 0;
|
||||
this.maxPendingFrames = 10;
|
||||
}
|
||||
|
||||
flushBuffer() {
|
||||
if (this.offset === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const chunk = new Int16Array(this.offset);
|
||||
chunk.set(this.buffer.subarray(0, this.offset));
|
||||
this.port.postMessage(chunk, [chunk.buffer]);
|
||||
|
||||
this.offset = 0;
|
||||
this.pendingFrames = 0;
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (!input || input.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const channel = input[0];
|
||||
if (!channel || channel.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
for (let i = 0; i < channel.length; i++) {
|
||||
let sample = channel[i];
|
||||
sample = Math.max(-1, Math.min(1, sample));
|
||||
this.buffer[this.offset++] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
|
||||
|
||||
if (this.offset === this.chunkSize) {
|
||||
this.flushBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
if (this.offset > 0) {
|
||||
this.pendingFrames += 1;
|
||||
if (this.pendingFrames >= this.maxPendingFrames) {
|
||||
this.flushBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('pcm-recorder', PCMRecorderProcessor);
|
||||
@@ -0,0 +1,299 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Realtime Demo</title>
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: #f8f9fa;
|
||||
height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.header {
|
||||
background: white;
|
||||
padding: 1rem;
|
||||
border-bottom: 1px solid #e1e5e9;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.connect-btn {
|
||||
padding: 0.5rem 1rem;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
font-weight: 500;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
|
||||
.connect-btn.disconnected {
|
||||
background: #0066cc;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.connect-btn.connected {
|
||||
background: #dc3545;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.connect-btn:hover {
|
||||
opacity: 0.9;
|
||||
}
|
||||
|
||||
.main {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
padding: 1rem;
|
||||
height: calc(100vh - 80px);
|
||||
}
|
||||
|
||||
.messages-pane {
|
||||
flex: 2;
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.messages-header {
|
||||
padding: 1rem;
|
||||
border-bottom: 1px solid #e1e5e9;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.messages-content {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
.message {
|
||||
margin-bottom: 1rem;
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.message.user {
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
.message.assistant {
|
||||
justify-content: flex-start;
|
||||
}
|
||||
|
||||
.message-bubble {
|
||||
max-width: 70%;
|
||||
padding: 0.75rem 1rem;
|
||||
border-radius: 18px;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
|
||||
.message.user .message-bubble {
|
||||
background: #0066cc;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.message.assistant .message-bubble {
|
||||
background: #f1f3f4;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.right-column {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.events-pane {
|
||||
flex: 2;
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.tools-pane {
|
||||
flex: 1;
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.events-header, .tools-header {
|
||||
padding: 1rem;
|
||||
border-bottom: 1px solid #e1e5e9;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.events-content, .tools-content {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 0.5rem;
|
||||
}
|
||||
|
||||
.event {
|
||||
border: 1px solid #e1e5e9;
|
||||
border-radius: 6px;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.event-header {
|
||||
padding: 0.75rem;
|
||||
background: #f8f9fa;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
font-family: monospace;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.event-header:hover {
|
||||
background: #e9ecef;
|
||||
}
|
||||
|
||||
.tools-content .event-header {
|
||||
cursor: default;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
}
|
||||
|
||||
.tools-content .event-header.handoff {
|
||||
background: #f3e8ff;
|
||||
border-left: 4px solid #8b5cf6;
|
||||
}
|
||||
|
||||
.tools-content .event-header.tool {
|
||||
background: #fef3e2;
|
||||
border-left: 4px solid #f59e0b;
|
||||
}
|
||||
|
||||
.event-content {
|
||||
padding: 0.75rem;
|
||||
background: white;
|
||||
border-top: 1px solid #e1e5e9;
|
||||
font-family: monospace;
|
||||
font-size: 0.8rem;
|
||||
white-space: pre-wrap;
|
||||
max-height: 200px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.event-content.collapsed {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.controls {
|
||||
padding: 1rem;
|
||||
border-top: 1px solid #e1e5e9;
|
||||
background: #f8f9fa;
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.mute-btn {
|
||||
padding: 0.5rem 1rem;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
font-weight: 500;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.mute-btn.unmuted {
|
||||
background: #28a745;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.mute-btn.muted {
|
||||
background: #dc3545;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.mute-btn.active {
|
||||
animation: pulse 1s infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0% { opacity: 1; }
|
||||
50% { opacity: 0.7; }
|
||||
100% { opacity: 1; }
|
||||
}
|
||||
|
||||
.status {
|
||||
font-size: 0.9rem;
|
||||
color: #6c757d;
|
||||
}
|
||||
|
||||
.connected {
|
||||
color: #28a745;
|
||||
}
|
||||
|
||||
.disconnected {
|
||||
color: #dc3545;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>Realtime Demo</h1>
|
||||
<button id="connectBtn" class="connect-btn disconnected">Connect</button>
|
||||
</div>
|
||||
|
||||
<div class="main">
|
||||
<div class="messages-pane">
|
||||
<div class="messages-header">
|
||||
Conversation
|
||||
</div>
|
||||
<div id="messagesContent" class="messages-content">
|
||||
<!-- Messages will appear here -->
|
||||
</div>
|
||||
<div class="controls">
|
||||
<button id="muteBtn" class="mute-btn unmuted" disabled>🎤 Mic On</button>
|
||||
<input id="imagePrompt" type="text" placeholder="Optional prompt for image" style="flex: 1; padding: 0.5rem; border: 1px solid #e1e5e9; border-radius: 6px;" />
|
||||
<input id="imageInput" type="file" accept="image/*" aria-hidden="true" style="position:absolute;left:-9999px;width:1px;height:1px;opacity:0;" />
|
||||
<button id="imageBtn" type="button" class="mute-btn unmuted" style="background:#6f42c1; user-select:none;">🖼️ Send Image</button>
|
||||
<span id="status" class="status disconnected">Disconnected</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="right-column">
|
||||
<div class="events-pane">
|
||||
<div class="events-header">
|
||||
Event stream
|
||||
</div>
|
||||
<div id="eventsContent" class="events-content">
|
||||
<!-- Events will appear here -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tools-pane">
|
||||
<div class="tools-header">
|
||||
Tools & Handoffs
|
||||
</div>
|
||||
<div id="toolsContent" class="tools-content">
|
||||
<!-- Tools and handoffs will appear here -->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user