chore: import upstream snapshot with attribution

apps/sim/app/(interfaces)/chat/components/voice-interface/voice-interface.tsx

@@ -0,0 +1,580 @@
+'use client'
+
+import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'
+import { Button, cn } from '@sim/emcn'
+import { createLogger } from '@sim/logger'
+import { Mic, MicOff, Phone } from 'lucide-react'
+import dynamic from 'next/dynamic'
+import { requestJson } from '@/lib/api/client/request'
+import { speechTokenContract } from '@/lib/api/contracts/media/speech'
+import { arrayBufferToBase64, floatTo16BitPCM } from '@/lib/speech/audio'
+import {
+  CHUNK_SEND_INTERVAL_MS,
+  ELEVENLABS_WS_URL,
+  MAX_CHAT_SESSION_MS,
+  SAMPLE_RATE,
+} from '@/lib/speech/config'
+
+const ParticlesVisualization = dynamic(
+  () =>
+    import('@/app/(interfaces)/chat/components/voice-interface/components/particles').then(
+      (mod) => mod.ParticlesVisualization
+    ),
+  { ssr: false }
+)
+
+const logger = createLogger('VoiceInterface')
+
+interface VoiceInterfaceProps {
+  onCallEnd?: () => void
+  onVoiceTranscript?: (transcript: string) => void
+  onVoiceStart?: () => void
+  onVoiceEnd?: () => void
+  onInterrupt?: () => void
+  isStreaming?: boolean
+  isPlayingAudio?: boolean
+  audioContextRef?: RefObject<AudioContext | null>
+  messages?: Array<{ content: string; type: 'user' | 'assistant' }>
+  className?: string
+  chatId?: string
+}
+
+const EMPTY_MESSAGES: Array<{ content: string; type: 'user' | 'assistant' }> = []
+
+export function VoiceInterface({
+  onCallEnd,
+  onVoiceTranscript,
+  onVoiceStart,
+  onVoiceEnd,
+  onInterrupt,
+  isStreaming = false,
+  isPlayingAudio = false,
+  audioContextRef: sharedAudioContextRef,
+  messages = EMPTY_MESSAGES,
+  className,
+  chatId,
+}: VoiceInterfaceProps) {
+  const [state, setState] = useState<'idle' | 'listening' | 'agent_speaking'>('idle')
+  const [isInitialized, setIsInitialized] = useState(false)
+  const [isMuted, setIsMuted] = useState(false)
+  const [audioLevels, setAudioLevels] = useState<number[]>(() => new Array(200).fill(0))
+  const permissionStatusRef = useRef<'prompt' | 'granted' | 'denied'>('prompt')
+  const [currentTranscript, setCurrentTranscript] = useState('')
+
+  const currentStateRef = useRef<'idle' | 'listening' | 'agent_speaking'>('idle')
+  const isCallEndedRef = useRef(false)
+
+  const updateState = useCallback((next: 'idle' | 'listening' | 'agent_speaking') => {
+    setState(next)
+    currentStateRef.current = next
+  }, [])
+
+  const mediaStreamRef = useRef<MediaStream | null>(null)
+  const audioContextRef = useRef<AudioContext | null>(null)
+  const analyserRef = useRef<AnalyserNode | null>(null)
+  const animationFrameRef = useRef<number | null>(null)
+  const isMutedRef = useRef(false)
+
+  const wsRef = useRef<WebSocket | null>(null)
+  const processorRef = useRef<ScriptProcessorNode | null>(null)
+  const pcmBufferRef = useRef<Float32Array[]>([])
+  const sendIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
+  const sessionTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
+  const committedTextRef = useRef('')
+  const lastPartialRef = useRef('')
+  const onVoiceTranscriptRef = useRef(onVoiceTranscript)
+
+  onVoiceTranscriptRef.current = onVoiceTranscript
+
+  const updateIsMuted = useCallback((next: boolean) => {
+    setIsMuted(next)
+    isMutedRef.current = next
+  }, [])
+
+  const stopSendingAudio = useCallback(() => {
+    if (sessionTimerRef.current) {
+      clearTimeout(sessionTimerRef.current)
+      sessionTimerRef.current = null
+    }
+    if (sendIntervalRef.current) {
+      clearInterval(sendIntervalRef.current)
+      sendIntervalRef.current = null
+    }
+    pcmBufferRef.current = []
+  }, [])
+
+  const flushAudioBuffer = useCallback(() => {
+    const ws = wsRef.current
+    if (!ws || ws.readyState !== WebSocket.OPEN) return
+
+    const chunks = pcmBufferRef.current
+    if (chunks.length === 0) return
+    pcmBufferRef.current = []
+
+    let totalLength = 0
+    for (const chunk of chunks) totalLength += chunk.length
+    const merged = new Float32Array(totalLength)
+    let offset = 0
+    for (const chunk of chunks) {
+      merged.set(chunk, offset)
+      offset += chunk.length
+    }
+
+    const pcm16 = floatTo16BitPCM(merged)
+
+    ws.send(
+      JSON.stringify({
+        message_type: 'input_audio_chunk',
+        audio_base_64: arrayBufferToBase64(pcm16),
+        sample_rate: SAMPLE_RATE,
+        commit: false,
+      })
+    )
+  }, [])
+
+  const startSendingAudio = useCallback(() => {
+    if (sendIntervalRef.current) return
+    pcmBufferRef.current = []
+    sendIntervalRef.current = setInterval(flushAudioBuffer, CHUNK_SEND_INTERVAL_MS)
+  }, [flushAudioBuffer])
+
+  const closeWebSocket = useCallback(() => {
+    stopSendingAudio()
+    if (wsRef.current) {
+      if (
+        wsRef.current.readyState === WebSocket.OPEN ||
+        wsRef.current.readyState === WebSocket.CONNECTING
+      ) {
+        wsRef.current.close()
+      }
+      wsRef.current = null
+    }
+  }, [stopSendingAudio])
+
+  const connectWebSocket = useCallback(async (): Promise<boolean> => {
+    try {
+      let tokenData: Awaited<ReturnType<typeof requestJson<typeof speechTokenContract>>>
+      try {
+        tokenData = await requestJson(speechTokenContract, {
+          body: chatId ? { chatId } : {},
+        })
+      } catch (err) {
+        logger.error('Failed to get STT token', err)
+        return false
+      }
+
+      const token = typeof tokenData.token === 'string' ? tokenData.token : undefined
+      if (!token) {
+        logger.error('STT token missing from response')
+        return false
+      }
+
+      const params = new URLSearchParams({
+        token,
+        model_id: 'scribe_v2_realtime',
+        audio_format: 'pcm_16000',
+        commit_strategy: 'vad',
+        vad_silence_threshold_secs: '1.0',
+      })
+
+      const ws = new WebSocket(`${ELEVENLABS_WS_URL}?${params.toString()}`)
+      wsRef.current = ws
+      committedTextRef.current = ''
+
+      return new Promise<boolean>((resolve) => {
+        ws.onopen = () => resolve(true)
+        ws.onerror = () => {
+          logger.error('STT WebSocket connection error')
+          resolve(false)
+        }
+
+        ws.onmessage = (event) => {
+          if (isCallEndedRef.current) return
+
+          try {
+            const msg = JSON.parse(event.data)
+
+            if (msg.message_type === 'partial_transcript') {
+              if (msg.text) {
+                lastPartialRef.current = msg.text
+                setCurrentTranscript(msg.text)
+              }
+            } else if (
+              msg.message_type === 'committed_transcript' ||
+              msg.message_type === 'committed_transcript_with_timestamps'
+            ) {
+              const finalText = msg.text || lastPartialRef.current
+              lastPartialRef.current = ''
+              if (finalText) {
+                committedTextRef.current = committedTextRef.current
+                  ? `${committedTextRef.current} ${finalText}`
+                  : finalText
+                setCurrentTranscript('')
+                onVoiceTranscriptRef.current?.(finalText)
+              }
+            } else if (
+              msg.message_type === 'error' ||
+              msg.message_type === 'auth_error' ||
+              msg.message_type === 'quota_exceeded'
+            ) {
+              logger.error('ElevenLabs STT error', { type: msg.message_type, error: msg.error })
+            }
+          } catch {
+            // Ignore non-JSON messages
+          }
+        }
+
+        ws.onclose = () => {
+          wsRef.current = null
+          if (currentStateRef.current === 'listening' && !isCallEndedRef.current) {
+            stopSendingAudio()
+            updateState('idle')
+          }
+        }
+      })
+    } catch (error) {
+      logger.error('Failed to connect STT WebSocket', error)
+      return false
+    }
+  }, [chatId])
+
+  const setupAudioPipeline = useCallback(async () => {
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          autoGainControl: true,
+          channelCount: 1,
+          sampleRate: SAMPLE_RATE,
+        },
+      })
+
+      permissionStatusRef.current = 'granted'
+      mediaStreamRef.current = stream
+
+      const ac = new AudioContext({ sampleRate: SAMPLE_RATE })
+      audioContextRef.current = ac
+
+      if (ac.state === 'suspended') {
+        await ac.resume()
+      }
+
+      const source = ac.createMediaStreamSource(stream)
+
+      const analyser = ac.createAnalyser()
+      analyser.fftSize = 256
+      analyser.smoothingTimeConstant = 0.8
+      source.connect(analyser)
+      analyserRef.current = analyser
+
+      const processor = ac.createScriptProcessor(4096, 1, 1)
+      processor.onaudioprocess = (e) => {
+        if (!isMutedRef.current && currentStateRef.current === 'listening') {
+          pcmBufferRef.current.push(new Float32Array(e.inputBuffer.getChannelData(0)))
+        }
+      }
+      source.connect(processor)
+      processor.connect(ac.destination)
+      processorRef.current = processor
+
+      const updateVisualization = () => {
+        if (!analyserRef.current) return
+        const bufferLength = analyserRef.current.frequencyBinCount
+        const dataArray = new Uint8Array(bufferLength)
+        analyserRef.current.getByteFrequencyData(dataArray)
+
+        const levels = []
+        for (let i = 0; i < 200; i++) {
+          const dataIndex = Math.floor((i / 200) * bufferLength)
+          const value = dataArray[dataIndex] || 0
+          levels.push((value / 255) * 100)
+        }
+
+        setAudioLevels(levels)
+        animationFrameRef.current = requestAnimationFrame(updateVisualization)
+      }
+      updateVisualization()
+
+      return true
+    } catch (error) {
+      logger.error('Error setting up audio pipeline:', error)
+      permissionStatusRef.current = 'denied'
+      return false
+    }
+  }, [])
+
+  const startListening = useCallback(async () => {
+    if (currentStateRef.current !== 'idle' || isMutedRef.current || isCallEndedRef.current) return
+
+    if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
+      const connected = await connectWebSocket()
+      if (!connected || isCallEndedRef.current) return
+    }
+
+    updateState('listening')
+    setCurrentTranscript('')
+    startSendingAudio()
+
+    sessionTimerRef.current = setTimeout(() => {
+      logger.info('Voice session reached max duration, stopping')
+      stopSendingAudio()
+      closeWebSocket()
+      updateState('idle')
+    }, MAX_CHAT_SESSION_MS)
+  }, [connectWebSocket, updateState, startSendingAudio, stopSendingAudio, closeWebSocket])
+
+  const stopListening = useCallback(() => {
+    stopSendingAudio()
+    updateState('idle')
+    setCurrentTranscript('')
+  }, [updateState, stopSendingAudio])
+
+  useEffect(() => {
+    if (isPlayingAudio && state === 'listening') {
+      stopSendingAudio()
+      closeWebSocket()
+      updateState('agent_speaking')
+      setCurrentTranscript('')
+
+      updateIsMuted(true)
+      if (mediaStreamRef.current) {
+        mediaStreamRef.current.getAudioTracks().forEach((track) => {
+          track.enabled = false
+        })
+      }
+    } else if (!isPlayingAudio && state === 'agent_speaking') {
+      updateState('idle')
+      setCurrentTranscript('')
+
+      updateIsMuted(false)
+      if (mediaStreamRef.current) {
+        mediaStreamRef.current.getAudioTracks().forEach((track) => {
+          track.enabled = true
+        })
+      }
+    }
+  }, [isPlayingAudio, state, updateState, updateIsMuted, stopSendingAudio, closeWebSocket])
+
+  const handleInterrupt = useCallback(() => {
+    if (state === 'agent_speaking') {
+      onInterrupt?.()
+
+      updateIsMuted(false)
+      if (mediaStreamRef.current) {
+        mediaStreamRef.current.getAudioTracks().forEach((track) => {
+          track.enabled = true
+        })
+      }
+
+      updateState('idle')
+      setCurrentTranscript('')
+    }
+  }, [state, onInterrupt, updateState, updateIsMuted])
+
+  const handleCallEnd = useCallback(() => {
+    isCallEndedRef.current = true
+
+    stopSendingAudio()
+    closeWebSocket()
+    updateState('idle')
+    setCurrentTranscript('')
+    updateIsMuted(false)
+
+    if (processorRef.current) {
+      processorRef.current.disconnect()
+      processorRef.current = null
+    }
+
+    if (mediaStreamRef.current) {
+      mediaStreamRef.current.getTracks().forEach((track) => track.stop())
+      mediaStreamRef.current = null
+    }
+
+    if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
+      audioContextRef.current.close().catch(() => {})
+      audioContextRef.current = null
+    }
+
+    if (animationFrameRef.current) {
+      cancelAnimationFrame(animationFrameRef.current)
+      animationFrameRef.current = null
+    }
+
+    onInterrupt?.()
+    onCallEnd?.()
+  }, [onCallEnd, onInterrupt, updateState, updateIsMuted, stopSendingAudio, closeWebSocket])
+
+  useEffect(() => {
+    const handleKeyDown = (event: KeyboardEvent) => {
+      if (event.code === 'Space') {
+        event.preventDefault()
+        handleInterrupt()
+      }
+    }
+
+    document.addEventListener('keydown', handleKeyDown)
+    return () => document.removeEventListener('keydown', handleKeyDown)
+  }, [handleInterrupt])
+
+  const toggleMute = useCallback(() => {
+    if (state === 'agent_speaking') {
+      handleInterrupt()
+      return
+    }
+
+    const newMutedState = !isMuted
+    updateIsMuted(newMutedState)
+
+    if (mediaStreamRef.current) {
+      mediaStreamRef.current.getAudioTracks().forEach((track) => {
+        track.enabled = !newMutedState
+      })
+    }
+
+    if (newMutedState) {
+      stopListening()
+    } else if (state === 'idle') {
+      startListening()
+    }
+  }, [isMuted, state, handleInterrupt, stopListening, startListening, updateIsMuted])
+
+  useEffect(() => {
+    isCallEndedRef.current = false
+    let cancelled = false
+
+    async function init() {
+      const audioOk = await setupAudioPipeline()
+      if (!audioOk || cancelled) return
+
+      const wsOk = await connectWebSocket()
+      if (!wsOk || cancelled) return
+
+      setIsInitialized(true)
+    }
+
+    init()
+
+    return () => {
+      cancelled = true
+    }
+  }, [setupAudioPipeline, connectWebSocket])
+
+  useEffect(() => {
+    if (isInitialized && !isMuted && state === 'idle') {
+      startListening()
+    }
+  }, [isInitialized, isMuted, state, startListening])
+
+  useEffect(() => {
+    return () => {
+      isCallEndedRef.current = true
+
+      stopSendingAudio()
+
+      if (wsRef.current) {
+        wsRef.current.close()
+        wsRef.current = null
+      }
+
+      if (processorRef.current) {
+        processorRef.current.disconnect()
+        processorRef.current = null
+      }
+
+      if (mediaStreamRef.current) {
+        mediaStreamRef.current.getTracks().forEach((track) => track.stop())
+        mediaStreamRef.current = null
+      }
+
+      if (audioContextRef.current) {
+        audioContextRef.current.close()
+        audioContextRef.current = null
+      }
+
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current)
+        animationFrameRef.current = null
+      }
+    }
+  }, [stopSendingAudio])
+
+  const getStatusText = () => {
+    switch (state) {
+      case 'listening':
+        return 'Listening...'
+      case 'agent_speaking':
+        return 'Press Space or tap to interrupt'
+      default:
+        return isInitialized ? 'Ready' : 'Initializing...'
+    }
+  }
+
+  const getButtonContent = () => {
+    if (state === 'agent_speaking') {
+      return (
+        <svg className='size-6' viewBox='0 0 24 24' fill='currentColor'>
+          <rect x='6' y='6' width='12' height='12' rx='2' />
+        </svg>
+      )
+    }
+    return isMuted ? <MicOff className='size-6' /> : <Mic className='size-6' />
+  }
+
+  return (
+    <div
+      className={cn(
+        'light fixed inset-0 z-[100] flex flex-col bg-[var(--bg)] text-[var(--text-primary)]',
+        className
+      )}
+    >
+      <div className='flex flex-1 flex-col items-center justify-center px-8'>
+        <div className='relative mb-16'>
+          <ParticlesVisualization
+            audioLevels={audioLevels}
+            isListening={state === 'listening'}
+            isPlayingAudio={state === 'agent_speaking'}
+            isStreaming={isStreaming}
+            isMuted={isMuted}
+            className='h-80 w-80 md:h-96 md:w-96'
+          />
+        </div>
+
+        <div className='mb-16 flex h-24 items-center justify-center'>
+          {currentTranscript && (
+            <div className='max-w-2xl px-8'>
+              <p className='overflow-hidden text-center text-[var(--text-primary)] text-xl leading-relaxed'>
+                {currentTranscript}
+              </p>
+            </div>
+          )}
+        </div>
+
+        <p className='mb-8 text-center text-[var(--text-muted)] text-lg'>
+          {getStatusText()}
+          {isMuted && <span className='ml-2 text-[var(--text-muted)] text-sm'>(Muted)</span>}
+        </p>
+      </div>
+
+      <div className='px-8 pb-12'>
+        <div className='flex items-center justify-center gap-x-12'>
+          <Button onClick={handleCallEnd} variant='outline' className='size-14 rounded-full p-0'>
+            <Phone className='size-6 rotate-[135deg]' />
+          </Button>
+
+          <Button
+            onClick={toggleMute}
+            variant='outline'
+            disabled={!isInitialized}
+            className={cn(
+              'size-14 rounded-full p-0',
+              isMuted ? 'text-[var(--text-muted)]' : 'text-[var(--text-primary)]'
+            )}
+          >
+            {getButtonContent()}
+          </Button>
+        </div>
+      </div>
+    </div>
+  )
+}