class RealtimeDemo { constructor() { this.ws = null; this.isConnected = false; this.isMuted = false; this.isCapturing = false; this.audioContext = null; this.captureSource = null; this.captureNode = null; this.stream = null; this.sessionId = this.generateSessionId(); this.isPlayingAudio = false; this.playbackAudioContext = null; this.playbackNode = null; this.playbackInitPromise = null; this.pendingPlaybackChunks = []; this.playbackFadeSec = 0.02; // ~20ms fade to reduce clicks this.messageNodes = new Map(); // item_id -> DOM node this.seenItemIds = new Set(); // item_id set for append-only syncing this.initializeElements(); this.setupEventListeners(); } initializeElements() { this.connectBtn = document.getElementById('connectBtn'); this.muteBtn = document.getElementById('muteBtn'); this.imageBtn = document.getElementById('imageBtn'); this.imageInput = document.getElementById('imageInput'); this.imagePrompt = document.getElementById('imagePrompt'); this.status = document.getElementById('status'); this.messagesContent = document.getElementById('messagesContent'); this.eventsContent = document.getElementById('eventsContent'); this.toolsContent = document.getElementById('toolsContent'); } setupEventListeners() { this.connectBtn.addEventListener('click', () => { if (this.isConnected) { this.disconnect(); } else { this.connect(); } }); this.muteBtn.addEventListener('click', () => { this.toggleMute(); }); // Image upload this.imageBtn.addEventListener('click', (e) => { e.preventDefault(); e.stopPropagation(); console.log('Send Image clicked'); // Programmatically open the hidden file input this.imageInput.click(); }); this.imageInput.addEventListener('change', async (e) => { console.log('Image input change fired'); const file = e.target.files && e.target.files[0]; if (!file) return; await this._handlePickedFile(file); this.imageInput.value = ''; }); this._handlePickedFile = async (file) => { try { const dataUrl = await this.prepareDataURL(file); const promptText = (this.imagePrompt && this.imagePrompt.value) || ''; // Send to server; server forwards to Realtime API. // Use chunked frames to avoid WS frame limits. if (this.ws && this.ws.readyState === WebSocket.OPEN) { console.log('Interrupting and sending image (chunked) to server WebSocket'); // Stop any current audio locally and tell model to interrupt this.stopAudioPlayback(); this.ws.send(JSON.stringify({ type: 'interrupt' })); const id = 'img_' + Math.random().toString(36).slice(2); const CHUNK = 60_000; // ~60KB per frame this.ws.send(JSON.stringify({ type: 'image_start', id, text: promptText })); for (let i = 0; i < dataUrl.length; i += CHUNK) { const chunk = dataUrl.slice(i, i + CHUNK); this.ws.send(JSON.stringify({ type: 'image_chunk', id, chunk })); } this.ws.send(JSON.stringify({ type: 'image_end', id })); } else { console.warn('Not connected; image will not be sent. Click Connect first.'); } // Add to UI immediately for better feedback console.log('Adding local user image bubble'); this.addUserImageMessage(dataUrl, promptText); } catch (err) { console.error('Failed to process image:', err); } }; } generateSessionId() { return 'session_' + Math.random().toString(36).substr(2, 9); } async connect() { try { this.ws = new WebSocket(`ws://localhost:8000/ws/${this.sessionId}`); this.ws.onopen = () => { this.isConnected = true; this.updateConnectionUI(); this.startContinuousCapture(); }; this.ws.onmessage = (event) => { const data = JSON.parse(event.data); this.handleRealtimeEvent(data); }; this.ws.onclose = () => { this.isConnected = false; this.updateConnectionUI(); }; this.ws.onerror = (error) => { console.error('WebSocket error:', error); }; } catch (error) { console.error('Failed to connect:', error); } } disconnect() { if (this.ws) { this.ws.close(); } this.stopContinuousCapture(); } updateConnectionUI() { if (this.isConnected) { this.connectBtn.textContent = 'Disconnect'; this.connectBtn.className = 'connect-btn connected'; this.status.textContent = 'Connected'; this.status.className = 'status connected'; this.muteBtn.disabled = false; } else { this.connectBtn.textContent = 'Connect'; this.connectBtn.className = 'connect-btn disconnected'; this.status.textContent = 'Disconnected'; this.status.className = 'status disconnected'; this.muteBtn.disabled = true; } } toggleMute() { this.isMuted = !this.isMuted; this.updateMuteUI(); } updateMuteUI() { if (this.isMuted) { this.muteBtn.textContent = 'πŸ”‡ Mic Off'; this.muteBtn.className = 'mute-btn muted'; } else { this.muteBtn.textContent = '🎀 Mic On'; this.muteBtn.className = 'mute-btn unmuted'; if (this.isCapturing) { this.muteBtn.classList.add('active'); } } } readFileAsDataURL(file) { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(reader.result); reader.onerror = reject; reader.readAsDataURL(file); }); } async prepareDataURL(file) { const original = await this.readFileAsDataURL(file); try { const img = new Image(); img.decoding = 'async'; const loaded = new Promise((res, rej) => { img.onload = () => res(); img.onerror = rej; }); img.src = original; await loaded; const maxDim = 1024; const maxSide = Math.max(img.width, img.height); const scale = maxSide > maxDim ? (maxDim / maxSide) : 1; const w = Math.max(1, Math.round(img.width * scale)); const h = Math.max(1, Math.round(img.height * scale)); const canvas = document.createElement('canvas'); canvas.width = w; canvas.height = h; const ctx = canvas.getContext('2d'); ctx.drawImage(img, 0, 0, w, h); return canvas.toDataURL('image/jpeg', 0.85); } catch (e) { console.warn('Image resize failed; sending original', e); return original; } } async startContinuousCapture() { if (!this.isConnected || this.isCapturing) return; // Check if getUserMedia is available if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { throw new Error('getUserMedia not available. Please use HTTPS or localhost.'); } try { this.stream = await navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 24000, channelCount: 1, echoCancellation: true, noiseSuppression: true } }); this.audioContext = new AudioContext({ sampleRate: 24000, latencyHint: 'interactive' }); if (this.audioContext.state === 'suspended') { try { await this.audioContext.resume(); } catch {} } if (!this.audioContext.audioWorklet) { throw new Error('AudioWorklet API not supported in this browser.'); } await this.audioContext.audioWorklet.addModule('audio-recorder.worklet.js'); this.captureSource = this.audioContext.createMediaStreamSource(this.stream); this.captureNode = new AudioWorkletNode(this.audioContext, 'pcm-recorder'); this.captureNode.port.onmessage = (event) => { if (this.isMuted) return; if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return; const chunk = event.data instanceof ArrayBuffer ? new Int16Array(event.data) : event.data; if (!chunk || !(chunk instanceof Int16Array) || chunk.length === 0) return; this.ws.send(JSON.stringify({ type: 'audio', data: Array.from(chunk) })); }; this.captureSource.connect(this.captureNode); this.captureNode.connect(this.audioContext.destination); this.isCapturing = true; this.updateMuteUI(); } catch (error) { console.error('Failed to start audio capture:', error); } } stopContinuousCapture() { if (!this.isCapturing) return; this.isCapturing = false; if (this.captureSource) { try { this.captureSource.disconnect(); } catch {} this.captureSource = null; } if (this.captureNode) { this.captureNode.port.onmessage = null; try { this.captureNode.disconnect(); } catch {} this.captureNode = null; } if (this.audioContext) { this.audioContext.close(); this.audioContext = null; } if (this.stream) { this.stream.getTracks().forEach(track => track.stop()); this.stream = null; } this.updateMuteUI(); } handleRealtimeEvent(event) { // Add to raw events pane this.addRawEvent(event); // Add to tools panel if it's a tool or handoff event if (event.type === 'tool_start' || event.type === 'tool_end' || event.type === 'handoff' || event.type === 'tool_approval_required') { this.addToolEvent(event); } // Handle specific event types switch (event.type) { case 'audio': this.playAudio(event.audio); break; case 'audio_interrupted': this.stopAudioPlayback(); break; case 'input_audio_timeout_triggered': // Ask server to commit the input buffer to expedite model response if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(JSON.stringify({ type: 'commit_audio' })); } break; case 'history_updated': this.syncMissingFromHistory(event.history); this.updateLastMessageFromHistory(event.history); break; case 'history_added': // Append just the new item without clearing the thread. if (event.item) { this.addMessageFromItem(event.item); } break; case 'tool_approval_required': this.promptForToolApproval(event); break; } } updateLastMessageFromHistory(history) { if (!history || !Array.isArray(history) || history.length === 0) return; // Find the last message item in history let last = null; for (let i = history.length - 1; i >= 0; i--) { const it = history[i]; if (it && it.type === 'message') { last = it; break; } } if (!last) return; const itemId = last.item_id; // Extract a text representation (for assistant transcript updates) let text = ''; if (Array.isArray(last.content)) { for (const part of last.content) { if (!part || typeof part !== 'object') continue; if (part.type === 'text' && part.text) text += part.text; else if (part.type === 'input_text' && part.text) text += part.text; else if ((part.type === 'input_audio' || part.type === 'audio') && part.transcript) text += part.transcript; } } const node = this.messageNodes.get(itemId); if (!node) { // If we haven't rendered this item yet, append it now. this.addMessageFromItem(last); return; } // Update only the text content of the bubble, preserving any images already present. const bubble = node.querySelector('.message-bubble'); if (bubble && text && text.trim()) { // If there's an , keep it and only update the trailing caption/text node. const hasImg = !!bubble.querySelector('img'); if (hasImg) { // Ensure there is a caption div after the image let cap = bubble.querySelector('.image-caption'); if (!cap) { cap = document.createElement('div'); cap.className = 'image-caption'; cap.style.marginTop = '0.5rem'; bubble.appendChild(cap); } cap.textContent = text.trim(); } else { bubble.textContent = text.trim(); } this.scrollToBottom(); } } syncMissingFromHistory(history) { if (!history || !Array.isArray(history)) return; for (const item of history) { if (!item || item.type !== 'message') continue; const id = item.item_id; if (!id) continue; if (!this.seenItemIds.has(id)) { this.addMessageFromItem(item); } } } addMessageFromItem(item) { try { if (!item || item.type !== 'message') return; const role = item.role; let content = ''; let imageUrls = []; if (Array.isArray(item.content)) { for (const contentPart of item.content) { if (!contentPart || typeof contentPart !== 'object') continue; if (contentPart.type === 'text' && contentPart.text) { content += contentPart.text; } else if (contentPart.type === 'input_text' && contentPart.text) { content += contentPart.text; } else if (contentPart.type === 'input_audio' && contentPart.transcript) { content += contentPart.transcript; } else if (contentPart.type === 'audio' && contentPart.transcript) { content += contentPart.transcript; } else if (contentPart.type === 'input_image') { const url = contentPart.image_url || contentPart.url; if (typeof url === 'string' && url) imageUrls.push(url); } } } let node = null; if (imageUrls.length > 0) { for (const url of imageUrls) { node = this.addImageMessage(role, url, content.trim()); } } else if (content && content.trim()) { node = this.addMessage(role, content.trim()); } if (node && item.item_id) { this.messageNodes.set(item.item_id, node); this.seenItemIds.add(item.item_id); } } catch (e) { console.error('Failed to add message from item:', e, item); } } addMessage(type, content) { const messageDiv = document.createElement('div'); messageDiv.className = `message ${type}`; const bubbleDiv = document.createElement('div'); bubbleDiv.className = 'message-bubble'; bubbleDiv.textContent = content; messageDiv.appendChild(bubbleDiv); this.messagesContent.appendChild(messageDiv); this.scrollToBottom(); return messageDiv; } addImageMessage(role, imageUrl, caption = '') { const messageDiv = document.createElement('div'); messageDiv.className = `message ${role}`; const bubbleDiv = document.createElement('div'); bubbleDiv.className = 'message-bubble'; const img = document.createElement('img'); img.src = imageUrl; img.alt = 'Uploaded image'; img.style.maxWidth = '220px'; img.style.borderRadius = '8px'; img.style.display = 'block'; bubbleDiv.appendChild(img); if (caption) { const cap = document.createElement('div'); cap.textContent = caption; cap.style.marginTop = '0.5rem'; bubbleDiv.appendChild(cap); } messageDiv.appendChild(bubbleDiv); this.messagesContent.appendChild(messageDiv); this.scrollToBottom(); return messageDiv; } addUserImageMessage(imageUrl, caption = '') { return this.addImageMessage('user', imageUrl, caption); } addRawEvent(event) { const eventDiv = document.createElement('div'); eventDiv.className = 'event'; const headerDiv = document.createElement('div'); headerDiv.className = 'event-header'; headerDiv.innerHTML = ` ${event.type} β–Ό `; const contentDiv = document.createElement('div'); contentDiv.className = 'event-content collapsed'; contentDiv.textContent = JSON.stringify(event, null, 2); headerDiv.addEventListener('click', () => { const isCollapsed = contentDiv.classList.contains('collapsed'); contentDiv.classList.toggle('collapsed'); headerDiv.querySelector('span:last-child').textContent = isCollapsed ? 'β–²' : 'β–Ό'; }); eventDiv.appendChild(headerDiv); eventDiv.appendChild(contentDiv); this.eventsContent.appendChild(eventDiv); // Auto-scroll events pane this.eventsContent.scrollTop = this.eventsContent.scrollHeight; } addToolEvent(event) { const eventDiv = document.createElement('div'); eventDiv.className = 'event'; let title = ''; let description = ''; let eventClass = ''; if (event.type === 'handoff') { title = `πŸ”„ Handoff`; description = `From ${event.from} to ${event.to}`; eventClass = 'handoff'; } else if (event.type === 'tool_start') { title = `πŸ”§ Tool Started`; description = `Running ${event.tool}`; eventClass = 'tool'; } else if (event.type === 'tool_end') { title = `βœ… Tool Completed`; description = `${event.tool}: ${event.output || 'No output'}`; eventClass = 'tool'; } else if (event.type === 'tool_approval_required') { title = `⏸️ Approval Needed`; description = `Waiting on ${event.tool}`; eventClass = 'tool'; } else if (event.type === 'tool_approval_decision') { title = event.approved ? 'βœ… Approved' : '❌ Rejected'; description = `${event.tool} (${event.call_id || 'call'})`; eventClass = 'tool'; } eventDiv.innerHTML = `
${title}
${description}
${new Date().toLocaleTimeString()}
`; this.toolsContent.appendChild(eventDiv); // Auto-scroll tools pane this.toolsContent.scrollTop = this.toolsContent.scrollHeight; } promptForToolApproval(event) { const args = event.arguments || ''; const preview = args ? `${args.slice(0, 180)}${args.length > 180 ? '…' : ''}` : ''; const message = `Allow tool "${event.tool}" to run?${preview ? `\nArgs: ${preview}` : ''}`; const approved = window.confirm(message); if (this.ws && this.ws.readyState === WebSocket.OPEN) { this.ws.send(JSON.stringify({ type: 'tool_approval_decision', call_id: event.call_id, approve: approved })); } this.addToolEvent({ type: 'tool_approval_decision', tool: event.tool, call_id: event.call_id, approved }); } async playAudio(audioBase64) { try { if (!audioBase64 || audioBase64.length === 0) { console.warn('Received empty audio data, skipping playback'); return; } const int16Array = this.decodeBase64ToInt16(audioBase64); if (!int16Array || int16Array.length === 0) { console.warn('Audio chunk has no samples, skipping'); return; } this.pendingPlaybackChunks.push(int16Array); await this.ensurePlaybackNode(); this.flushPendingPlaybackChunks(); } catch (error) { console.error('Failed to play audio:', error); this.pendingPlaybackChunks = []; } } async ensurePlaybackNode() { if (this.playbackNode) { return; } if (!this.playbackInitPromise) { this.playbackInitPromise = (async () => { if (!this.playbackAudioContext) { this.playbackAudioContext = new AudioContext({ sampleRate: 24000, latencyHint: 'interactive' }); } if (this.playbackAudioContext.state === 'suspended') { try { await this.playbackAudioContext.resume(); } catch {} } if (!this.playbackAudioContext.audioWorklet) { throw new Error('AudioWorklet API not supported in this browser.'); } await this.playbackAudioContext.audioWorklet.addModule('audio-playback.worklet.js'); this.playbackNode = new AudioWorkletNode(this.playbackAudioContext, 'pcm-playback', { outputChannelCount: [1] }); this.playbackNode.port.onmessage = (event) => { const message = event.data; if (!message || typeof message !== 'object') return; if (message.type === 'drained') { this.isPlayingAudio = false; } }; // Provide initial configuration for fades. const fadeSamples = Math.floor(this.playbackAudioContext.sampleRate * this.playbackFadeSec); this.playbackNode.port.postMessage({ type: 'config', fadeSamples }); this.playbackNode.connect(this.playbackAudioContext.destination); })().catch((error) => { this.playbackInitPromise = null; throw error; }); } await this.playbackInitPromise; } flushPendingPlaybackChunks() { if (!this.playbackNode) { return; } while (this.pendingPlaybackChunks.length > 0) { const chunk = this.pendingPlaybackChunks.shift(); if (!chunk || !(chunk instanceof Int16Array) || chunk.length === 0) { continue; } try { this.playbackNode.port.postMessage( { type: 'chunk', payload: chunk.buffer }, [chunk.buffer] ); this.isPlayingAudio = true; } catch (error) { console.error('Failed to enqueue audio chunk to worklet:', error); } } } decodeBase64ToInt16(audioBase64) { try { const binaryString = atob(audioBase64); const length = binaryString.length; const bytes = new Uint8Array(length); for (let i = 0; i < length; i++) { bytes[i] = binaryString.charCodeAt(i); } return new Int16Array(bytes.buffer); } catch (error) { console.error('Failed to decode audio chunk:', error); return null; } } stopAudioPlayback() { console.log('Stopping audio playback due to interruption'); this.pendingPlaybackChunks = []; if (this.playbackNode) { try { this.playbackNode.port.postMessage({ type: 'stop' }); } catch (error) { console.error('Failed to notify playback worklet to stop:', error); } } this.isPlayingAudio = false; console.log('Audio playback stopped and queue cleared'); } scrollToBottom() { this.messagesContent.scrollTop = this.messagesContent.scrollHeight; } } // Initialize the demo when the page loads document.addEventListener('DOMContentLoaded', () => { new RealtimeDemo(); });