chore: import upstream snapshot with attribution

examples/voice_agent/client/src/app.ts

@@ -0,0 +1,572 @@
+/**
+ * Copyright (c) 2024–2025, Daily
+ *
+ * SPDX-License-Identifier: BSD 2-Clause License
+ */
+
+/**
+ * RTVI Client Implementation
+ *
+ * This client connects to an RTVI-compatible bot server using WebSocket.
+ *
+ * Requirements:
+ * - A running RTVI bot server (defaults to http://localhost:7860)
+ */
+
+import {
+  RTVIClient,
+  RTVIClientOptions,
+  RTVIEvent,
+} from '@pipecat-ai/client-js';
+import {
+  WebSocketTransport
+} from "@pipecat-ai/websocket-transport";
+
+class WebsocketClientApp {
+  private rtviClient: RTVIClient | null = null;
+  private connectBtn: HTMLButtonElement | null = null;
+  private disconnectBtn: HTMLButtonElement | null = null;
+  private muteBtn: HTMLButtonElement | null = null;
+  private resetBtn: HTMLButtonElement | null = null;
+  private serverSelect: HTMLSelectElement | null = null;
+  private statusSpan: HTMLElement | null = null;
+  private debugLog: HTMLElement | null = null;
+  private volumeBar: HTMLElement | null = null;
+  private volumeText: HTMLElement | null = null;
+  private botAudio: HTMLAudioElement;
+  private isConnecting: boolean = false;
+  private isDisconnecting: boolean = false;
+  private isMuted: boolean = false;
+  private audioContext: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private microphone: MediaStreamAudioSourceNode | null = null;
+  private volumeUpdateInterval: number | null = null;
+  private currentBotMessageElement: HTMLDivElement | null = null;
+  private currentBotMessage: string = '';
+
+  // Server configurations
+  private readonly serverConfigs = {
+    websocket: {
+      name: 'WebSocket Server',
+      baseUrl: `http://${window.location.hostname}:7860`,
+      port: 8765
+    },
+    fastapi: {
+      name: 'FastAPI Server', 
+      baseUrl: `http://${window.location.hostname}:8000`,
+      port: 8000
+    }
+  };
+
+  constructor() {
+    console.log("WebsocketClientApp");
+    this.botAudio = document.createElement('audio');
+    this.botAudio.autoplay = true;
+    //this.botAudio.playsInline = true;
+    document.body.appendChild(this.botAudio);
+
+    this.setupDOMElements();
+    this.setupEventListeners();
+  }
+
+  /**
+   * Set up references to DOM elements and create necessary media elements
+   */
+  private setupDOMElements(): void {
+    this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
+    this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
+    this.muteBtn = document.getElementById('mute-btn') as HTMLButtonElement;
+    this.resetBtn = document.getElementById('reset-btn') as HTMLButtonElement;
+    this.serverSelect = document.getElementById('server-select') as HTMLSelectElement;
+    this.statusSpan = document.getElementById('connection-status');
+    this.debugLog = document.getElementById('debug-log');
+    this.volumeBar = document.getElementById('volume-bar');
+    this.volumeText = document.getElementById('volume-text');
+  }
+
+  /**
+   * Set up event listeners for connect/disconnect buttons
+   */
+  private setupEventListeners(): void {
+    this.connectBtn?.addEventListener('click', () => this.connect());
+    this.disconnectBtn?.addEventListener('click', () => this.disconnect());
+    this.muteBtn?.addEventListener('click', () => this.toggleMute());
+    this.resetBtn?.addEventListener('click', () => this.reset());
+    this.serverSelect?.addEventListener('change', () => this.updateServerUrl());
+  }
+
+  /**
+   * Add a timestamped message to the debug log
+   */
+  private log(message: string): void {
+    if (!this.debugLog) return;
+    const entry = document.createElement('div');
+    entry.textContent = `${new Date().toISOString()} - ${message}`;
+    if (message.startsWith('User: ')) {
+      entry.style.color = '#2196F3';
+    } else if (message.startsWith('Bot: ')) {
+      entry.style.color = '#4CAF50';
+    }
+    this.debugLog.appendChild(entry);
+    this.debugLog.scrollTop = this.debugLog.scrollHeight;
+    console.log(message);
+  }
+
+  /**
+   * Create a bot message element and add it to the debug log
+   */
+  private createBotMessageElement(initialText: string): HTMLDivElement | null {
+    if (!this.debugLog) return null;
+    const entry = document.createElement('div');
+    entry.style.color = '#4CAF50';
+    entry.textContent = `${new Date().toISOString()} - ${initialText}`;
+    this.debugLog.appendChild(entry);
+    this.debugLog.scrollTop = this.debugLog.scrollHeight;
+    return entry;
+  }
+
+  /**
+   * Update the connection status display
+   */
+  private updateStatus(status: string): void {
+    if (this.statusSpan) {
+      this.statusSpan.textContent = status;
+    }
+    this.log(`Status: ${status}`);
+  }
+
+  /**
+   * Check for available media tracks and set them up if present
+   * This is called when the bot is ready or when the transport state changes to ready
+   */
+  setupMediaTracks() {
+    if (!this.rtviClient) return;
+    const tracks = this.rtviClient.tracks();
+    if (tracks.bot?.audio) {
+      this.setupAudioTrack(tracks.bot.audio);
+    }
+  }
+
+  /**
+   * Set up listeners for track events (start/stop)
+   * This handles new tracks being added during the session
+   */
+  setupTrackListeners() {
+    if (!this.rtviClient) {
+      this.log('Cannot setup track listeners: client is null');
+      return;
+    }
+
+    try {
+      // Listen for new tracks starting
+      this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => {
+        // Only handle non-local (bot) tracks
+        if (!participant?.local && track.kind === 'audio') {
+          this.setupAudioTrack(track);
+        }
+      });
+
+      // Listen for tracks stopping
+      this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => {
+        this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`);
+      });
+    } catch (error) {
+      this.log(`Error setting up track listeners: ${error}`);
+    }
+  }
+
+  /**
+   * Set up an audio track for playback
+   * Handles both initial setup and track updates
+   */
+  private setupAudioTrack(track: MediaStreamTrack): void {
+    this.log('Setting up audio track');
+    if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
+      const oldTrack = this.botAudio.srcObject.getAudioTracks()[0];
+      if (oldTrack?.id === track.id) return;
+    }
+    this.botAudio.srcObject = new MediaStream([track]);
+  }
+
+  /**
+   * Initialize and connect to the bot
+   * This sets up the RTVI client, initializes devices, and establishes the connection
+   */
+  public async connect(): Promise<void> {
+    if (this.isConnecting) {
+      this.log('Connection already in progress, ignoring...');
+      return;
+    }
+
+    try {
+      this.isConnecting = true;
+      const startTime = Date.now();
+
+      //const transport = new DailyTransport();
+      const transport = new WebSocketTransport();
+      const RTVIConfig: RTVIClientOptions = {
+        transport,
+        params: {
+          // The baseURL and endpoint of your bot server that the client will connect to
+          baseUrl: this.getSelectedServerConfig().baseUrl,
+          endpoints: { connect: '/connect' },
+        },
+        enableMic: true,
+        enableCam: false,
+        callbacks: {
+          onConnected: () => {
+            this.updateStatus('Connected');
+            if (this.connectBtn) this.connectBtn.disabled = true;
+            if (this.disconnectBtn) this.disconnectBtn.disabled = false;
+            if (this.muteBtn) {
+              this.muteBtn.disabled = false;
+              this.muteBtn.textContent = 'Mute';
+            }
+            if (this.resetBtn) this.resetBtn.disabled = false;
+            if (this.serverSelect) this.serverSelect.disabled = true;
+            // Start volume monitoring when connected
+            if (!this.isMuted) {
+              this.startVolumeMonitoring();
+            }
+          },
+          onDisconnected: () => {
+            // Only handle disconnect if we're not in the middle of error cleanup
+            if (!this.isConnecting) {
+              this.updateStatus('Disconnected');
+              if (this.connectBtn) this.connectBtn.disabled = false;
+              if (this.disconnectBtn) this.disconnectBtn.disabled = true;
+              if (this.muteBtn) {
+                this.muteBtn.disabled = true;
+                this.muteBtn.textContent = 'Mute';
+              }
+              if (this.resetBtn) this.resetBtn.disabled = true;
+              if (this.serverSelect) this.serverSelect.disabled = false;
+              // Stop volume monitoring when disconnected
+              this.stopVolumeMonitoring();
+              this.log('Client disconnected');
+            }
+          },
+          onBotReady: (data) => {
+            this.log(`Bot ready: ${JSON.stringify(data)}`);
+            this.setupMediaTracks();
+          },
+          onUserTranscript: (data) => {
+            if (data.final) {
+              this.log(`User: ${data.text}`);
+            }
+          },
+          onBotTranscript: (data) => {
+            // If no current element exists, create one (fallback in case BOT_LLM_STARTED didn't fire)
+            if (!this.currentBotMessageElement) {
+              this.currentBotMessage = '';
+              this.currentBotMessageElement = this.createBotMessageElement('Bot: ');
+            }
+            
+            // Accumulate the text
+            this.currentBotMessage += data.text;
+            
+            // Update the current element
+            if (this.currentBotMessageElement) {
+              const timestamp = new Date().toISOString();
+              this.currentBotMessageElement.textContent = `${timestamp} - Bot: ${this.currentBotMessage}`;
+              this.debugLog?.scrollTo({ top: this.debugLog.scrollHeight, behavior: 'smooth' });
+            }
+          },
+          onBotLlmStarted: () => {
+            // Only create a new bot message element if the current one has content
+            if (this.currentBotMessage !== '') {
+              this.currentBotMessage = '';
+              this.currentBotMessageElement = this.createBotMessageElement('Bot: ');
+            } else if (!this.currentBotMessageElement) {
+              // Create element if it doesn't exist at all
+              this.currentBotMessage = '';
+              this.currentBotMessageElement = this.createBotMessageElement('Bot: ');
+            }
+          },
+          onMessageError: (error) => console.error('Message error:', error),
+          onError: (error) => console.error('Error:', error),
+        },
+      }
+      
+      // Create the client with error handling
+      try {
+        this.rtviClient = new RTVIClient(RTVIConfig);
+        this.setupTrackListeners();
+      } catch (clientError) {
+        this.log(`Error creating RTVI client: ${clientError}`);
+        throw clientError;
+      }
+
+      this.log('Initializing devices...');
+      await this.rtviClient.initDevices();
+      this.log('Devices initialized successfully');
+
+      this.log('Connecting to bot...');
+      await this.rtviClient.connect();
+
+      const timeTaken = Date.now() - startTime;
+      this.log(`Connection complete, timeTaken: ${timeTaken}`);
+    } catch (error) {
+      this.log(`Error connecting: ${(error as Error).message}`);
+      this.updateStatus('Error');
+      // Clean up if there's an error
+      await this.cleanupOnError();
+    } finally {
+      this.isConnecting = false;
+    }
+  }
+
+  /**
+   * Clean up resources when there's an error during connection
+   */
+  private async cleanupOnError(): Promise<void> {
+    // Set disconnecting flag to prevent onDisconnected callback interference
+    this.isDisconnecting = true;
+    
+    // Store reference to client before it might become null
+    const client = this.rtviClient;
+    
+    if (client) {
+      try {
+        // Check if the client is in a state where disconnect can be called
+        if (typeof client.disconnect === 'function') {
+          await client.disconnect();
+        }
+      } catch (disconnectError) {
+        this.log(`Error during cleanup disconnect: ${disconnectError}`);
+      } finally {
+        // Always reset the client to null to allow reconnection
+        this.rtviClient = null;
+      }
+    } else {
+      this.log('Client was already null during cleanup');
+    }
+    
+    // Reset button states
+    if (this.connectBtn) this.connectBtn.disabled = false;
+    if (this.disconnectBtn) this.disconnectBtn.disabled = true;
+    if (this.muteBtn) {
+      this.muteBtn.disabled = true;
+      this.muteBtn.textContent = 'Mute';
+    }
+    if (this.resetBtn) this.resetBtn.disabled = true;
+    if (this.serverSelect) this.serverSelect.disabled = false;
+    
+    // Stop volume monitoring
+    this.stopVolumeMonitoring();
+    
+    // Clean up bot message state
+    this.currentBotMessage = '';
+    this.currentBotMessageElement = null;
+    
+    // Reset mute state
+    this.isMuted = false;
+    
+    // Reset disconnecting flag
+    this.isDisconnecting = false;
+  }
+
+  /**
+   * Disconnect from the bot and clean up media resources
+   */
+  public async disconnect(): Promise<void> {
+    if (this.isDisconnecting) {
+      this.log('Disconnection already in progress, ignoring...');
+      return;
+    }
+
+    this.isDisconnecting = true;
+    
+    // Store reference to client before it might become null
+    const client = this.rtviClient;
+    
+    if (client) {
+      try {
+        // Check if the client is in a state where disconnect can be called
+        if (typeof client.disconnect === 'function') {
+          await client.disconnect();
+        }
+      } catch (error) {
+        this.log(`Error disconnecting: ${(error as Error).message}`);
+      } finally {
+        // Always clean up resources and reset the client
+        this.rtviClient = null;
+        if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) {
+          this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop());
+          this.botAudio.srcObject = null;
+        }
+      }
+    } else {
+      this.log('Client was already null during disconnect');
+    }
+    
+    // Stop volume monitoring
+    this.stopVolumeMonitoring();
+    
+    // Clean up bot message state
+    this.currentBotMessage = '';
+    this.currentBotMessageElement = null;
+    
+    // Reset mute state
+    this.isMuted = false;
+    
+    this.isDisconnecting = false;
+  }
+
+  /**
+   * Toggle microphone mute/unmute
+   */
+  private toggleMute(): void {
+    if (!this.rtviClient) {
+      this.log('Cannot toggle mute: client is null');
+      return;
+    }
+
+    this.isMuted = !this.isMuted;
+    this.rtviClient.enableMic(!this.isMuted);
+    
+    // Update button text
+    if (this.muteBtn) {
+      this.muteBtn.textContent = this.isMuted ? 'Unmute' : 'Mute';
+    }
+    
+    // Update volume monitoring
+    if (this.isMuted) {
+      this.stopVolumeMonitoring();
+    } else {
+      this.startVolumeMonitoring();
+    }
+    
+    this.log(this.isMuted ? 'Microphone muted' : 'Microphone unmuted');
+  }
+
+  /**
+   * Start monitoring microphone volume
+   */
+  private async startVolumeMonitoring(): Promise<void> {
+    try {
+      if (!this.audioContext) {
+        this.audioContext = new AudioContext();
+      }
+
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      
+      this.analyser = this.audioContext.createAnalyser();
+      this.analyser.fftSize = 256;
+      this.analyser.smoothingTimeConstant = 0.8;
+      
+      this.microphone = this.audioContext.createMediaStreamSource(stream);
+      this.microphone.connect(this.analyser);
+      
+      // Start continuous volume updates
+      this.volumeUpdateInterval = window.setInterval(() => {
+        this.updateVolumeDisplay();
+      }, 100); // Update every 100ms
+      
+      this.log('Volume monitoring started');
+    } catch (error) {
+      this.log(`Error starting volume monitoring: ${error}`);
+    }
+  }
+
+  /**
+   * Stop monitoring microphone volume
+   */
+  private stopVolumeMonitoring(): void {
+    if (this.volumeUpdateInterval) {
+      clearInterval(this.volumeUpdateInterval);
+      this.volumeUpdateInterval = null;
+    }
+    
+    if (this.microphone) {
+      this.microphone.disconnect();
+      this.microphone = null;
+    }
+    
+    // Reset volume display
+    this.updateVolumeDisplay(0);
+    this.log('Volume monitoring stopped');
+  }
+
+  /**
+   * Update the volume display
+   */
+  private updateVolumeDisplay(volume?: number): void {
+    if (!this.volumeBar || !this.volumeText) return;
+
+    if (volume === undefined && this.analyser) {
+      const dataArray = new Uint8Array(this.analyser.frequencyBinCount);
+      this.analyser.getByteFrequencyData(dataArray);
+      
+      // Calculate average volume
+      const average = dataArray.reduce((sum, value) => sum + value, 0) / dataArray.length;
+      volume = (average / 255) * 100;
+    }
+
+    const displayVolume = volume || 0;
+    const clampedVolume = Math.min(100, Math.max(0, displayVolume));
+    
+    this.volumeBar.style.width = `${clampedVolume}%`;
+    this.volumeText.textContent = `${Math.round(clampedVolume)}%`;
+    
+    // Update color based on volume level
+    if (clampedVolume < 30) {
+      this.volumeBar.style.background = '#4caf50'; // Green
+    } else if (clampedVolume < 70) {
+      this.volumeBar.style.background = '#ff9800'; // Orange
+    } else {
+      this.volumeBar.style.background = '#f44336'; // Red
+    }
+  }
+
+  /**
+   * Reset the conversation context by calling the server action
+   */
+  private async reset(): Promise<void> {
+    if (!this.rtviClient) {
+      this.log('Cannot reset: not connected to server');
+      return;
+    }
+
+    try {
+      this.log('Resetting conversation context...');
+      
+      // Call the reset action on the server
+      const result = await this.rtviClient.action({ service: 'context', action: 'reset', arguments: [] });
+      
+      if (result) {
+        this.log('Conversation context reset successfully');
+      } else {
+        this.log('Failed to reset conversation context');
+      }
+    } catch (error) {
+      this.log(`Error resetting context: ${error}`);
+    }
+  }
+
+  private getSelectedServerConfig(): { name: string; baseUrl: string; port: number } {
+    const selectedValue = this.serverSelect?.value || 'websocket';
+    return this.serverConfigs[selectedValue as keyof typeof this.serverConfigs];
+  }
+
+  private updateServerUrl(): void {
+    const selectedConfig = this.getSelectedServerConfig();
+    this.log(`Server changed to: ${selectedConfig.name} (${selectedConfig.baseUrl})`);
+    
+    // If connected, show a message that they need to reconnect
+    if (this.rtviClient) {
+      this.log('Please disconnect and reconnect to use the new server');
+    }
+  }
+}
+
+declare global {
+  interface Window {
+    WebsocketClientApp: typeof WebsocketClientApp;
+  }
+}
+
+window.addEventListener('DOMContentLoaded', () => {
+  window.WebsocketClientApp = WebsocketClientApp;
+  new WebsocketClientApp();
+});

examples/voice_agent/client/src/style.css

@@ -0,0 +1,180 @@
+body {
+    margin: 0;
+    padding: 20px;
+    font-family: Arial, sans-serif;
+    background-color: #f0f0f0;
+}
+
+.container {
+    max-width: 1200px;
+    margin: 0 auto;
+}
+
+.status-bar {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 10px;
+    background-color: #fff;
+    border-radius: 8px;
+    margin-bottom: 20px;
+}
+
+.controls button {
+    padding: 8px 16px;
+    margin-left: 10px;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+}
+
+#connect-btn {
+    background-color: #4caf50;
+    color: white;
+}
+
+#disconnect-btn {
+    background-color: #f44336;
+    color: white;
+}
+
+#mute-btn {
+    background-color: #ff9800;
+    color: white;
+}
+
+#mute-btn:disabled {
+    background-color: #ccc;
+    color: #666;
+}
+
+button:disabled {
+    opacity: 0.5;
+    cursor: not-allowed;
+}
+
+.volume-indicator {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    padding: 10px;
+    background-color: #fff;
+    border-radius: 8px;
+    margin-bottom: 20px;
+}
+
+.volume-label {
+    font-weight: bold;
+    min-width: 120px;
+}
+
+.volume-bar-container {
+    flex: 1;
+    height: 20px;
+    background-color: #e0e0e0;
+    border-radius: 10px;
+    overflow: hidden;
+    position: relative;
+}
+
+.volume-bar {
+    height: 100%;
+    background: linear-gradient(90deg, #4caf50, #ff9800, #f44336);
+    width: 0%;
+    transition: width 0.1s ease;
+    border-radius: 10px;
+}
+
+.volume-text {
+    min-width: 40px;
+    text-align: right;
+    font-weight: bold;
+    font-size: 14px;
+}
+
+.main-content {
+    background-color: #fff;
+    border-radius: 8px;
+    padding: 20px;
+    margin-bottom: 20px;
+}
+
+.bot-container {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+}
+
+#bot-video-container {
+    width: 640px;
+    height: 360px;
+    background-color: #e0e0e0;
+    border-radius: 8px;
+    margin: 20px auto;
+    overflow: hidden;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+}
+
+#bot-video-container video {
+    width: 100%;
+    height: 100%;
+    object-fit: cover;
+}
+
+.debug-panel {
+    background-color: #fff;
+    border-radius: 8px;
+    padding: 20px;
+}
+
+.debug-panel h3 {
+    margin: 0 0 10px 0;
+    font-size: 16px;
+    font-weight: bold;
+}
+
+#debug-log {
+    height: 500px;
+    overflow-y: auto;
+    background-color: #f8f8f8;
+    padding: 10px;
+    border-radius: 4px;
+    font-family: monospace;
+    font-size: 12px;
+    line-height: 1.4;
+}
+
+.server-selection {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    margin-bottom: 10px;
+}
+
+.server-selection label {
+    font-weight: bold;
+    color: #333;
+}
+
+.server-selection select {
+    padding: 6px 12px;
+    border: 1px solid #ccc;
+    border-radius: 4px;
+    background-color: white;
+    font-size: 14px;
+    cursor: pointer;
+}
+
+.server-selection select:focus {
+    outline: none;
+    border-color: #2196F3;
+    box-shadow: 0 0 0 2px rgba(33, 150, 243, 0.2);
+}
+
+.server-selection select:disabled {
+    background-color: #f5f5f5;
+    cursor: not-allowed;
+    opacity: 0.6;
+}