chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,260 @@
'use client'
import { useEffect, useState } from 'react'
import { cn, Input, InputOTP, InputOTPGroup, InputOTPSlot, Label } from '@sim/emcn'
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { quickValidateEmail } from '@/lib/messaging/email/validation'
import { AuthSubmitButton } from '@/app/(auth)/components'
import { AUTH_TEXT_LINK } from '@/app/(auth)/components/auth-button-classes'
import { useChatEmailOtpRequest, useChatEmailOtpVerify } from '@/hooks/queries/chats'
const logger = createLogger('EmailAuth')
interface EmailAuthProps {
identifier: string
}
const validateEmailField = (emailValue: string): string[] => {
const errors: string[] = []
if (!emailValue || !emailValue.trim()) {
errors.push('Email is required.')
return errors
}
const validation = quickValidateEmail(emailValue.trim().toLowerCase())
if (!validation.isValid) {
errors.push(validation.reason || 'Please enter a valid email address.')
}
return errors
}
export default function EmailAuth({ identifier }: EmailAuthProps) {
const [email, setEmail] = useState('')
const [authError, setAuthError] = useState<string | null>(null)
const [emailErrors, setEmailErrors] = useState<string[]>([])
const [showEmailValidationError, setShowEmailValidationError] = useState(false)
const [showOtpVerification, setShowOtpVerification] = useState(false)
const [otpValue, setOtpValue] = useState('')
const [countdown, setCountdown] = useState(0)
const requestOtp = useChatEmailOtpRequest(identifier)
const verifyOtp = useChatEmailOtpVerify(identifier)
useEffect(() => {
if (countdown <= 0) return
const timer = setTimeout(() => setCountdown((c) => c - 1), 1000)
return () => clearTimeout(timer)
}, [countdown])
const handleEmailChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const newEmail = e.target.value
setEmail(newEmail)
const errors = validateEmailField(newEmail)
setEmailErrors(errors)
setShowEmailValidationError(false)
}
const handleSendOtp = async () => {
const emailValidationErrors = validateEmailField(email)
setEmailErrors(emailValidationErrors)
setShowEmailValidationError(emailValidationErrors.length > 0)
if (emailValidationErrors.length > 0) {
return
}
setAuthError(null)
try {
await requestOtp.mutateAsync({ email })
setShowOtpVerification(true)
} catch (error) {
logger.error('Error sending OTP:', error)
setEmailErrors([toError(error).message || 'Failed to send verification code'])
setShowEmailValidationError(true)
}
}
const handleVerifyOtp = async (otp?: string) => {
const codeToVerify = otp || otpValue
if (!codeToVerify || codeToVerify.length !== 6) {
return
}
setAuthError(null)
try {
await verifyOtp.mutateAsync({ email, otp: codeToVerify })
} catch (error) {
logger.error('Error verifying OTP:', error)
setAuthError(toError(error).message || 'Invalid verification code')
}
}
const handleResendOtp = async () => {
setAuthError(null)
setCountdown(30)
try {
await requestOtp.mutateAsync({ email })
setOtpValue('')
} catch (error) {
logger.error('Error resending OTP:', error)
setAuthError(toError(error).message || 'Failed to resend verification code')
setCountdown(0)
}
}
return (
<div className='flex flex-1 items-center justify-center px-4 py-16'>
<div className='w-full max-w-[410px]'>
<div className='flex flex-col items-center justify-center'>
<div className='space-y-1 text-center'>
<h1 className='text-balance text-[40px] text-[var(--text-primary)] leading-[110%] tracking-[-0.02em]'>
{showOtpVerification ? 'Verify Your Email' : 'Email Verification'}
</h1>
<p className='text-[color-mix(in_srgb,var(--text-muted)_60%,transparent)] text-lg leading-[125%] tracking-[0.02em]'>
{showOtpVerification
? `A verification code has been sent to ${email}`
: 'This chat requires email verification'}
</p>
</div>
<div className='mt-8 w-full max-w-[410px]'>
{!showOtpVerification ? (
<form
onSubmit={(e) => {
e.preventDefault()
handleSendOtp()
}}
className='space-y-6'
>
<div className='space-y-2'>
<div className='flex items-center justify-between'>
<Label htmlFor='email'>Email</Label>
</div>
<Input
id='email'
name='email'
placeholder='Enter your email'
required
autoCapitalize='none'
autoComplete='email'
autoCorrect='off'
value={email}
onChange={handleEmailChange}
className={cn(
showEmailValidationError &&
emailErrors.length > 0 &&
'border-[var(--text-error)] focus:border-[var(--text-error)]'
)}
/>
{showEmailValidationError && emailErrors.length > 0 && (
<div className='mt-1 space-y-1 text-[var(--text-error)] text-xs'>
{emailErrors.map((error) => (
<p key={error}>{error}</p>
))}
</div>
)}
</div>
<AuthSubmitButton
type='submit'
loading={requestOtp.isPending}
loadingLabel='Sending Code…'
>
Continue
</AuthSubmitButton>
</form>
) : (
<div className='space-y-6'>
<p className='text-center text-[var(--text-muted)] text-sm'>
Enter the 6-digit code to verify your account. If you don't see it in your inbox,
check your spam folder.
</p>
<div className='flex justify-center'>
<InputOTP
maxLength={6}
value={otpValue}
onChange={(value) => {
setOtpValue(value)
if (value.length === 6) {
handleVerifyOtp(value)
}
}}
disabled={verifyOtp.isPending}
className={cn('gap-2', authError && 'otp-error')}
>
<InputOTPGroup>
{[0, 1, 2, 3, 4, 5].map((index) => (
<InputOTPSlot
key={index}
index={index}
className={cn(authError && 'border-[var(--text-error)]')}
/>
))}
</InputOTPGroup>
</InputOTP>
</div>
{authError && (
<div className='mt-1 space-y-1 text-center text-[var(--text-error)] text-xs'>
<p>{authError}</p>
</div>
)}
<AuthSubmitButton
onClick={() => handleVerifyOtp()}
disabled={otpValue.length !== 6}
loading={verifyOtp.isPending}
loadingLabel='Verifying'
>
Verify Email
</AuthSubmitButton>
<div className='text-center'>
<p className='text-[var(--text-muted)] text-sm'>
Didn't receive a code?{' '}
{countdown > 0 ? (
<span>
Resend in{' '}
<span className='font-medium text-[var(--text-primary)]'>{countdown}s</span>
</span>
) : (
<button
className={AUTH_TEXT_LINK}
onClick={handleResendOtp}
disabled={verifyOtp.isPending || requestOtp.isPending}
>
Resend
</button>
)}
</p>
</div>
<div className='text-center font-light text-sm'>
<button
onClick={() => {
setShowOtpVerification(false)
setOtpValue('')
setAuthError(null)
}}
className={AUTH_TEXT_LINK}
>
Change email
</button>
</div>
</div>
)}
</div>
</div>
</div>
</div>
)
}
@@ -0,0 +1,134 @@
'use client'
import { useState } from 'react'
import { cn, Input, Label } from '@sim/emcn'
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { Eye, EyeOff } from 'lucide-react'
import { AuthSubmitButton } from '@/app/(auth)/components'
import { useChatPasswordAuth } from '@/hooks/queries/chats'
const logger = createLogger('PasswordAuth')
interface PasswordAuthProps {
identifier: string
}
export default function PasswordAuth({ identifier }: PasswordAuthProps) {
const [password, setPassword] = useState('')
const [showPassword, setShowPassword] = useState(false)
const [showValidationError, setShowValidationError] = useState(false)
const [passwordErrors, setPasswordErrors] = useState<string[]>([])
const authenticate = useChatPasswordAuth(identifier)
const handlePasswordChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const newPassword = e.target.value
setPassword(newPassword)
setShowValidationError(false)
setPasswordErrors([])
}
const handleAuthenticate = async () => {
if (!password.trim()) {
setPasswordErrors(['Password is required'])
setShowValidationError(true)
return
}
try {
await authenticate.mutateAsync({ password })
setPassword('')
} catch (error) {
logger.error('Authentication error:', error)
setPasswordErrors([toError(error).message || 'Invalid password. Please try again.'])
setShowValidationError(true)
}
}
return (
<div className='flex flex-1 items-center justify-center px-4 py-16'>
<div className='w-full max-w-[410px]'>
<div className='flex flex-col items-center justify-center'>
<div className='space-y-1 text-center'>
<h1 className='text-balance text-[40px] text-[var(--text-primary)] leading-[110%] tracking-[-0.02em]'>
Password Required
</h1>
<p className='text-[color-mix(in_srgb,var(--text-muted)_60%,transparent)] text-lg leading-[125%] tracking-[0.02em]'>
This chat is password-protected
</p>
</div>
<form
onSubmit={(e) => {
e.preventDefault()
handleAuthenticate()
}}
className='mt-8 w-full max-w-[410px] space-y-6'
>
<div className='space-y-6'>
<div className='flex items-center justify-between'>
<Label htmlFor='password'>Password</Label>
</div>
<div className='relative'>
<div className='relative'>
<Input
id='password'
name='password'
required
type={showPassword ? 'text' : 'password'}
autoCapitalize='none'
autoComplete='new-password'
autoCorrect='off'
placeholder='Enter password'
value={password}
onChange={handlePasswordChange}
className={cn(
'pr-10',
showValidationError &&
passwordErrors.length > 0 &&
'border-[var(--text-error)] focus:border-[var(--text-error)]'
)}
/>
<button
type='button'
onClick={() => setShowPassword(!showPassword)}
className='-translate-y-1/2 absolute top-1/2 right-3 text-[var(--text-muted)] hover:text-[var(--text-primary)]'
aria-label={showPassword ? 'Hide password' : 'Show password'}
>
{showPassword ? <EyeOff size={18} /> : <Eye size={18} />}
</button>
</div>
<div
className={cn(
'absolute right-0 left-0 z-10 grid transition-[grid-template-rows] duration-200 ease-out',
showValidationError && passwordErrors.length > 0
? 'grid-rows-[1fr]'
: 'grid-rows-[0fr]'
)}
aria-live='polite'
>
<div className='overflow-hidden'>
<div className='mt-1 space-y-1 text-[var(--text-error)] text-xs'>
{passwordErrors.map((error) => (
<p key={error}>{error}</p>
))}
</div>
</div>
</div>
</div>
</div>
<AuthSubmitButton
type='submit'
disabled={!password.trim()}
loading={authenticate.isPending}
loadingLabel='Authenticating…'
>
Continue
</AuthSubmitButton>
</form>
</div>
</div>
</div>
)
}
@@ -0,0 +1,30 @@
'use client'
import { Button } from '@sim/emcn'
import { useRouter } from 'next/navigation'
interface ChatErrorStateProps {
error: string
}
export function ChatErrorState({ error }: ChatErrorStateProps) {
const router = useRouter()
return (
<div className='flex flex-1 items-center justify-center px-4 py-16 text-center'>
<div className='flex w-full max-w-[410px] flex-col items-center gap-3'>
<h1 className='text-balance text-[40px] text-[var(--text-primary)] leading-[110%] tracking-[-0.02em]'>
Chat Unavailable
</h1>
<p className='text-[var(--text-muted)] text-lg'>{error}</p>
<Button
variant='primary'
onClick={() => router.push('/workspace')}
className='h-[32px] w-full gap-2 px-2.5 text-sm'
>
Return to Workspace
</Button>
</div>
</div>
)
}
@@ -0,0 +1,76 @@
'use client'
import Image from 'next/image'
import Link from 'next/link'
import { GithubIcon } from '@/components/icons'
import { SimWordmark } from '@/app/(landing)/components/navbar/components'
import { useBrandConfig } from '@/ee/whitelabeling'
interface ChatHeaderProps {
chatConfig: {
title?: string
customizations?: {
headerText?: string
logoUrl?: string
imageUrl?: string
primaryColor?: string
}
} | null
starCount: string
}
export function ChatHeader({ chatConfig, starCount }: ChatHeaderProps) {
const brand = useBrandConfig()
const customImage = chatConfig?.customizations?.imageUrl || chatConfig?.customizations?.logoUrl
return (
<nav
aria-label='Chat navigation'
className='flex w-full items-center justify-between px-4 pt-3 pb-[21px] sm:px-8 sm:pt-[8.5px] md:px-[44px] md:pt-4'
>
<div className='flex items-center gap-[34px]'>
<div className='flex items-center gap-3'>
{customImage && (
<Image
src={customImage}
alt={`${chatConfig?.title || 'Chat'} logo`}
width={24}
height={24}
unoptimized
className='size-6 rounded-md object-cover'
/>
)}
<h2 className='font-medium text-[var(--text-primary)] text-lg'>
{chatConfig?.customizations?.headerText || chatConfig?.title || 'Chat'}
</h2>
</div>
</div>
{!brand.logoUrl && (
<div className='flex items-center gap-4'>
<a
href='https://github.com/simstudioai/sim'
target='_blank'
rel='noopener noreferrer'
className='flex items-center gap-2 text-[var(--text-muted)] transition-colors hover:text-[var(--text-primary)]'
aria-label={`GitHub repository - ${starCount} stars`}
>
<GithubIcon className='size-[16px]' aria-hidden='true' />
<span aria-live='polite'>{starCount}</span>
</a>
{/* Only show Sim logo if no custom branding is set */}
<Link
href='https://sim.ai'
target='_blank'
rel='noopener noreferrer'
aria-label='Sim home'
className='flex items-center'
>
<SimWordmark />
</Link>
</div>
)}
</nav>
)
}
@@ -0,0 +1,9 @@
export { default as EmailAuth } from './auth/email/email-auth'
export { default as PasswordAuth } from './auth/password/password-auth'
export { ChatErrorState } from './error-state/error-state'
export { ChatHeader } from './header/header'
export { ChatInput } from './input/input'
export { ChatLoadingState } from './loading-state/loading-state'
export type { ChatMessage } from './message/message'
export { ChatMessageContainer } from './message-container/message-container'
export { VoiceInterface } from './voice-interface/voice-interface'
@@ -0,0 +1,357 @@
'use client'
import type React from 'react'
import { useCallback, useLayoutEffect, useRef, useState } from 'react'
import { Badge, Button, cn, handleKeyboardActivation, Tooltip } from '@sim/emcn'
import { createLogger } from '@sim/logger'
import { generateId } from '@sim/utils/id'
import { ArrowUp, Mic, Paperclip, X } from 'lucide-react'
import { CHAT_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation'
import { VoiceInput } from '@/app/(interfaces)/chat/components/input/voice-input'
const logger = createLogger('ChatInput')
const MAX_TEXTAREA_HEIGHT = 200
interface AttachedFile {
id: string
name: string
size: number
type: string
file: File
dataUrl?: string
}
export const ChatInput: React.FC<{
onSubmit?: (value: string, isVoiceInput?: boolean, files?: AttachedFile[]) => void
isStreaming?: boolean
onStopStreaming?: () => void
onVoiceStart?: () => void
voiceOnly?: boolean
sttAvailable?: boolean
}> = ({
onSubmit,
isStreaming = false,
onStopStreaming,
onVoiceStart,
voiceOnly = false,
sttAvailable = false,
}) => {
const fileInputRef = useRef<HTMLInputElement>(null)
const textareaRef = useRef<HTMLTextAreaElement>(null)
const [inputValue, setInputValue] = useState('')
const [attachedFiles, setAttachedFiles] = useState<AttachedFile[]>([])
const [uploadErrors, setUploadErrors] = useState<string[]>([])
const [dragCounter, setDragCounter] = useState(0)
const isDragOver = dragCounter > 0
useLayoutEffect(() => {
const el = textareaRef.current
if (!el) return
el.style.height = 'auto'
el.style.height = `${Math.min(el.scrollHeight, MAX_TEXTAREA_HEIGHT)}px`
}, [inputValue])
const handleFileSelect = async (selectedFiles: FileList | null) => {
if (!selectedFiles) return
const newFiles: AttachedFile[] = []
const maxSize = 10 * 1024 * 1024
const maxFiles = 15
for (let i = 0; i < selectedFiles.length; i++) {
if (attachedFiles.length + newFiles.length >= maxFiles) break
const file = selectedFiles[i]
if (file.size > maxSize) {
setUploadErrors((prev) => [...prev, `${file.name} is too large (max 10MB)`])
continue
}
const isDuplicate = attachedFiles.some(
(existing) => existing.name === file.name && existing.size === file.size
)
if (isDuplicate) {
setUploadErrors((prev) => [...prev, `${file.name} already added`])
continue
}
let dataUrl: string | undefined
if (file.type.startsWith('image/')) {
try {
dataUrl = await new Promise<string>((resolve, reject) => {
const reader = new FileReader()
reader.onload = () => resolve(reader.result as string)
reader.onerror = reject
reader.readAsDataURL(file)
})
} catch (error) {
logger.error('Error reading file:', error)
}
}
newFiles.push({
id: generateId(),
name: file.name,
size: file.size,
type: file.type,
file,
dataUrl,
})
}
if (newFiles.length > 0) {
setAttachedFiles((prev) => [...prev, ...newFiles])
setUploadErrors([])
}
}
const handleRemoveFile = useCallback((fileId: string) => {
setAttachedFiles((prev) => prev.filter((f) => f.id !== fileId))
}, [])
const handleSubmit = useCallback(() => {
if (isStreaming) return
if (!inputValue.trim() && attachedFiles.length === 0) return
onSubmit?.(inputValue.trim(), false, attachedFiles)
setInputValue('')
setAttachedFiles([])
setUploadErrors([])
}, [isStreaming, inputValue, attachedFiles, onSubmit])
const handleKeyDown = useCallback(
(e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === 'Enter' && !e.shiftKey && !e.nativeEvent.isComposing) {
e.preventDefault()
handleSubmit()
}
},
[handleSubmit]
)
const focusTextarea = useCallback(() => {
textareaRef.current?.focus()
}, [])
const handleContainerClick = useCallback((e: React.MouseEvent<HTMLDivElement>) => {
if ((e.target as HTMLElement).closest('button')) return
textareaRef.current?.focus()
}, [])
const canSubmit = (inputValue.trim().length > 0 || attachedFiles.length > 0) && !isStreaming
if (voiceOnly) {
return (
<Tooltip.Provider>
<div className='flex items-center justify-center'>
{sttAvailable && (
<Tooltip.Root>
<Tooltip.Trigger asChild>
<div>
<VoiceInput
onVoiceStart={onVoiceStart ?? (() => {})}
disabled={isStreaming}
large={true}
/>
</div>
</Tooltip.Trigger>
<Tooltip.Content side='top'>
<p>Start voice conversation</p>
</Tooltip.Content>
</Tooltip.Root>
)}
</div>
</Tooltip.Provider>
)
}
return (
<Tooltip.Provider>
<div className='fixed right-0 bottom-0 left-0 flex w-full items-center justify-center bg-gradient-to-t from-[var(--bg)] to-transparent px-4 pb-4 md:px-0 md:pb-4'>
<div className='w-full max-w-3xl md:max-w-[748px]'>
{/* Error Messages */}
{uploadErrors.length > 0 && (
<div className='mb-3 flex flex-col gap-2'>
{uploadErrors.map((error, idx) => (
<Badge key={`${error}-${idx}`} variant='red' size='lg' dot className='max-w-full'>
{error}
</Badge>
))}
</div>
)}
{/* Input container */}
<div
role='group'
aria-label='Chat message input'
onClick={handleContainerClick}
onKeyDown={(event) => {
if (event.target !== event.currentTarget) return
handleKeyboardActivation(event, focusTextarea)
}}
className={cn(
'relative z-10 cursor-text rounded-2xl border border-[var(--border-1)] bg-[var(--surface-2)] px-2.5 py-2',
isDragOver && 'border-purple-500'
)}
onDragEnter={(e) => {
e.preventDefault()
e.stopPropagation()
if (!isStreaming) setDragCounter((prev) => prev + 1)
}}
onDragOver={(e) => {
e.preventDefault()
e.stopPropagation()
if (!isStreaming) e.dataTransfer.dropEffect = 'copy'
}}
onDragLeave={(e) => {
e.preventDefault()
e.stopPropagation()
setDragCounter((prev) => Math.max(0, prev - 1))
}}
onDrop={(e) => {
e.preventDefault()
e.stopPropagation()
setDragCounter(0)
if (!isStreaming) handleFileSelect(e.dataTransfer.files)
}}
>
{/* File thumbnails */}
{attachedFiles.length > 0 && (
<div className='mb-1.5 flex flex-wrap gap-1.5'>
{attachedFiles.map((file) => (
<Tooltip.Root key={file.id}>
<Tooltip.Trigger asChild>
<div className='group relative size-[56px] flex-shrink-0 cursor-pointer overflow-hidden rounded-[8px] border border-[var(--border-1)] bg-[var(--surface-3)]'>
{file.dataUrl ? (
<img
src={file.dataUrl}
alt={file.name}
className='h-full w-full object-cover'
/>
) : (
<div className='flex h-full w-full flex-col items-center justify-center gap-0.5 text-[var(--text-muted)]'>
<Paperclip className='size-[18px]' />
<span className='max-w-[48px] truncate px-[2px] text-[9px]'>
{file.name.split('.').pop()}
</span>
</div>
)}
<Button
variant='ghost'
onClick={(e) => {
e.stopPropagation()
handleRemoveFile(file.id)
}}
className='absolute top-[2px] right-[2px] size-[16px] rounded-full bg-black/60 p-0 text-white opacity-0 hover-hover:text-white group-hover:opacity-100'
>
<X className='size-[10px]' />
</Button>
</div>
</Tooltip.Trigger>
<Tooltip.Content side='top'>
<p className='max-w-[200px] truncate'>{file.name}</p>
</Tooltip.Content>
</Tooltip.Root>
))}
</div>
)}
{/* Textarea */}
<textarea
ref={textareaRef}
value={inputValue}
onChange={(e) => setInputValue(e.target.value)}
onKeyDown={handleKeyDown}
placeholder={isDragOver ? 'Drop files here...' : 'Enter a message...'}
rows={1}
className='m-0 h-auto min-h-[24px] w-full resize-none overflow-y-auto overflow-x-hidden border-0 bg-transparent p-1 text-[15px] text-[var(--text-primary)] leading-[24px] caret-[var(--text-primary)] outline-none [-ms-overflow-style:none] [scrollbar-width:none] placeholder:text-[var(--text-muted)] focus-visible:ring-0 focus-visible:ring-offset-0 [&::-webkit-scrollbar]:hidden'
/>
{/* Bottom row */}
<div className='flex items-center justify-between'>
{/* Left: attach */}
<div>
<Tooltip.Root>
<Tooltip.Trigger asChild>
<Button
variant='quiet'
onClick={() => fileInputRef.current?.click()}
disabled={isStreaming || attachedFiles.length >= 15}
className='size-[28px] rounded-full p-0'
>
<Paperclip className='size-[16px]' strokeWidth={2} />
</Button>
</Tooltip.Trigger>
<Tooltip.Content side='top'>
<p>Attach files</p>
</Tooltip.Content>
</Tooltip.Root>
<input
ref={fileInputRef}
type='file'
multiple
accept={CHAT_ACCEPT_ATTRIBUTE}
onChange={(e) => {
handleFileSelect(e.target.files)
if (fileInputRef.current) fileInputRef.current.value = ''
}}
className='hidden'
disabled={isStreaming}
/>
</div>
{/* Right: mic + send */}
<div className='flex items-center gap-1.5'>
{sttAvailable && (
<Tooltip.Root>
<Tooltip.Trigger asChild>
<Button
variant='quiet'
onClick={onVoiceStart}
disabled={isStreaming}
className='size-[28px] rounded-full p-0'
>
<Mic className='size-[16px]' strokeWidth={2} />
</Button>
</Tooltip.Trigger>
<Tooltip.Content side='top'>
<p>Start voice conversation</p>
</Tooltip.Content>
</Tooltip.Root>
)}
{isStreaming ? (
<Button
variant='primary'
onClick={onStopStreaming}
className='size-[28px] rounded-full p-0'
title='Stop generation'
>
<svg
className='block size-[14px] fill-current'
viewBox='0 0 24 24'
xmlns='http://www.w3.org/2000/svg'
>
<rect x='4' y='4' width='16' height='16' rx='3' ry='3' />
</svg>
</Button>
) : (
<Button
variant='primary'
onClick={handleSubmit}
disabled={!canSubmit}
className='size-[28px] rounded-full p-0'
>
<ArrowUp className='block size-[16px]' strokeWidth={2.25} />
</Button>
)}
</div>
</div>
</div>
</div>
</div>
</Tooltip.Provider>
)
}
@@ -0,0 +1,89 @@
'use client'
import { useCallback } from 'react'
import { domAnimation, LazyMotion, m } from 'framer-motion'
import { Mic } from 'lucide-react'
interface VoiceInputProps {
onVoiceStart: () => void
isListening?: boolean
disabled?: boolean
large?: boolean
minimal?: boolean
}
export function VoiceInput({
onVoiceStart,
isListening = false,
disabled = false,
large = false,
minimal = false,
}: VoiceInputProps) {
const handleVoiceClick = useCallback(() => {
if (disabled) return
onVoiceStart()
}, [disabled, onVoiceStart])
if (minimal) {
return (
<button
type='button'
onClick={handleVoiceClick}
disabled={disabled}
className={`flex items-center justify-center rounded-full p-1.5 text-gray-600 transition-colors duration-200 hover:bg-gray-100 md:p-2 ${
disabled ? 'cursor-not-allowed opacity-50' : 'cursor-pointer'
}`}
title='Start voice conversation'
>
<Mic size={16} className='md:h-5 md:w-5' />
</button>
)
}
if (large) {
return (
<div className='flex flex-col items-center'>
<LazyMotion features={domAnimation}>
<m.button
type='button'
onClick={handleVoiceClick}
disabled={disabled}
className={`flex items-center justify-center rounded-full border-2 p-6 transition-all duration-200 ${
isListening
? 'border-red-400 bg-red-500/20 text-red-600 hover:bg-red-500/30'
: 'border-blue-300 bg-blue-500/10 text-blue-600 hover:bg-blue-500/20'
} ${disabled ? 'cursor-not-allowed opacity-50' : 'cursor-pointer'}`}
whileHover={{ scale: 1.05 }}
whileTap={{ scale: 0.95 }}
title='Start voice conversation'
>
<Mic size={32} />
</m.button>
</LazyMotion>
</div>
)
}
return (
<div className='flex items-center'>
<LazyMotion features={domAnimation}>
<m.button
type='button'
onClick={handleVoiceClick}
disabled={disabled}
className={`flex items-center justify-center rounded-full p-2.5 transition-all duration-200 md:p-3 ${
isListening
? 'bg-red-500 text-white hover:bg-red-600'
: 'bg-black text-white hover:bg-zinc-700'
} ${disabled ? 'cursor-not-allowed opacity-50' : 'cursor-pointer'}`}
whileHover={{ scale: 1.05 }}
whileTap={{ scale: 0.95 }}
title='Start voice conversation'
>
<Mic size={16} className='md:hidden' />
<Mic size={18} className='hidden md:block' />
</m.button>
</LazyMotion>
</div>
)
}
@@ -0,0 +1,28 @@
import { Skeleton } from '@sim/emcn'
export function ChatLoadingState() {
return (
<div className='light fixed inset-0 z-[100] flex flex-col bg-[var(--bg)]'>
<div className='flex flex-1 items-center justify-center px-4'>
<div className='w-full max-w-[410px]'>
<div className='flex flex-col items-center justify-center'>
{/* Title skeleton */}
<div className='space-y-2 text-center'>
<Skeleton className='mx-auto h-8 w-32' />
<Skeleton className='mx-auto h-4 w-48' />
</div>
{/* Chat skeleton */}
<div className='mt-8 w-full space-y-8'>
<div className='space-y-2'>
<Skeleton className='h-4 w-16' />
<Skeleton className='h-10 w-full rounded-[10px]' />
</div>
<Skeleton className='h-10 w-full rounded-[10px]' />
</div>
</div>
</div>
</div>
</div>
)
}
@@ -0,0 +1,92 @@
'use client'
import { memo, type RefObject } from 'react'
import { Button } from '@sim/emcn'
import { ArrowDown } from 'lucide-react'
import {
type ChatMessage,
ClientChatMessage,
} from '@/app/(interfaces)/chat/components/message/message'
interface ChatMessageContainerProps {
messages: ChatMessage[]
isLoading: boolean
showScrollButton: boolean
messagesContainerRef: RefObject<HTMLDivElement>
messagesEndRef: RefObject<HTMLDivElement>
scrollToBottom: () => void
scrollToMessage?: (messageId: string) => void
chatConfig: {
description?: string
} | null
}
export const ChatMessageContainer = memo(function ChatMessageContainer({
messages,
isLoading,
showScrollButton,
messagesContainerRef,
messagesEndRef,
scrollToBottom,
scrollToMessage,
chatConfig,
}: ChatMessageContainerProps) {
return (
<div className='relative flex flex-1 flex-col overflow-hidden'>
{/* Scrollable Messages Area */}
<div
ref={messagesContainerRef}
className='absolute inset-0 touch-pan-y overflow-y-auto overscroll-auto scroll-smooth'
>
<div className='mx-auto max-w-3xl px-4 pt-10 pb-20'>
{messages.length === 0 ? (
<div className='flex flex-col items-center justify-center py-10'>
<div className='space-y-2 text-center'>
<h3 className='font-medium text-[var(--text-primary)] text-lg'>
How can I help you today?
</h3>
<p className='text-[var(--text-muted)] text-sm'>
{chatConfig?.description || 'Ask me anything.'}
</p>
</div>
</div>
) : (
messages.map((message) => <ClientChatMessage key={message.id} message={message} />)
)}
{/* Loading indicator (shows only when executing) */}
{isLoading && (
<div className='px-4 py-5'>
<div className='mx-auto max-w-3xl'>
<div className='flex'>
<div className='max-w-[80%]'>
<div className='flex h-6 items-center'>
<div className='loading-dot size-3 rounded-full bg-[var(--text-primary)]' />
</div>
</div>
</div>
</div>
</div>
)}
{/* End of messages marker for scrolling */}
<div ref={messagesEndRef} />
</div>
</div>
{/* Scroll to bottom button - appears when user scrolls up */}
{showScrollButton && (
<div className='-translate-x-1/2 absolute bottom-16 left-1/2 z-20 transform'>
<Button
onClick={scrollToBottom}
size='sm'
className='gap-1 rounded-full px-3 py-1 shadow-lg'
>
<ArrowDown className='size-3.5' />
<span className='sr-only'>Scroll to bottom</span>
</Button>
</div>
)}
</div>
)
})
@@ -0,0 +1,185 @@
'use client'
import { useState } from 'react'
import { Button, cn, Download, Loader } from '@sim/emcn'
import { createLogger } from '@sim/logger'
import { sleep } from '@sim/utils/helpers'
import { Music } from 'lucide-react'
import { DefaultFileIcon, getDocumentIcon } from '@/components/icons/document-icons'
import { isSafeHttpUrl } from '@/lib/core/utils/urls'
import type { ChatFile } from '@/app/(interfaces)/chat/components/message/message'
const logger = createLogger('ChatFileDownload')
interface ChatFileDownloadProps {
file: ChatFile
}
interface ChatFileDownloadAllProps {
files: ChatFile[]
}
function formatFileSize(bytes: number): string {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return `${Math.round((bytes / k ** i) * 10) / 10} ${sizes[i]}`
}
function isAudioFile(mimeType: string, filename: string): boolean {
const audioMimeTypes = [
'audio/mpeg',
'audio/wav',
'audio/mp3',
'audio/ogg',
'audio/webm',
'audio/aac',
'audio/flac',
]
const audioExtensions = ['mp3', 'wav', 'ogg', 'webm', 'aac', 'flac', 'm4a']
const extension = filename.split('.').pop()?.toLowerCase()
return (
audioMimeTypes.some((t) => mimeType.includes(t)) ||
(extension ? audioExtensions.includes(extension) : false)
)
}
function isImageFile(mimeType: string): boolean {
return mimeType.startsWith('image/')
}
function getFileUrl(file: ChatFile): string {
return `/api/files/serve/${encodeURIComponent(file.key)}?context=${file.context || 'execution'}`
}
async function triggerDownload(url: string, filename: string): Promise<void> {
const response = await fetch(url)
if (!response.ok) {
throw new Error(`Failed to fetch file: ${response.status} ${response.statusText}`)
}
const blob = await response.blob()
const blobUrl = URL.createObjectURL(blob)
const link = document.createElement('a')
link.href = blobUrl
link.download = filename
document.body.appendChild(link)
link.click()
document.body.removeChild(link)
URL.revokeObjectURL(blobUrl)
logger.info(`Downloaded: ${filename}`)
}
export function ChatFileDownload({ file }: ChatFileDownloadProps) {
const [isDownloading, setIsDownloading] = useState(false)
const [isHovered, setIsHovered] = useState(false)
const handleDownload = async () => {
if (isDownloading) return
setIsDownloading(true)
try {
logger.info(`Initiating download for file: ${file.name}`)
const url = getFileUrl(file)
await triggerDownload(url, file.name)
} catch (error) {
logger.error(`Failed to download file ${file.name}:`, error)
if (file.url && isSafeHttpUrl(file.url)) {
window.open(file.url, '_blank', 'noopener,noreferrer')
}
} finally {
setIsDownloading(false)
}
}
const renderIcon = () => {
if (isAudioFile(file.type, file.name)) {
return <Music className='size-4 text-purple-500' />
}
if (isImageFile(file.type)) {
const ImageIcon = DefaultFileIcon
return <ImageIcon className='size-5' />
}
const DocumentIcon = getDocumentIcon(file.type, file.name)
return <DocumentIcon className='size-5' />
}
return (
<Button
variant='default'
onClick={handleDownload}
onMouseEnter={() => setIsHovered(true)}
onMouseLeave={() => setIsHovered(false)}
disabled={isDownloading}
className='flex h-auto w-[200px] items-center gap-2 rounded-lg px-3 py-2'
>
<div className='flex size-8 flex-shrink-0 items-center justify-center'>{renderIcon()}</div>
<div className='min-w-0 flex-1 text-left'>
<div className='w-[100px] truncate text-xs'>{file.name}</div>
<div className='text-[var(--text-muted)] text-micro'>{formatFileSize(file.size)}</div>
</div>
<div className='flex-shrink-0'>
{isDownloading ? (
<Loader className='size-3.5' animate />
) : (
<Download
className={cn('size-3.5 transition-opacity', isHovered ? 'opacity-100' : 'opacity-0')}
/>
)}
</div>
</Button>
)
}
export function ChatFileDownloadAll({ files }: ChatFileDownloadAllProps) {
const [isDownloading, setIsDownloading] = useState(false)
if (!files || files.length === 0) return null
const handleDownloadAll = async () => {
if (isDownloading) return
setIsDownloading(true)
try {
logger.info(`Initiating download for ${files.length} files`)
for (let i = 0; i < files.length; i++) {
const file = files[i]
try {
const url = getFileUrl(file)
await triggerDownload(url, file.name)
logger.info(`Downloaded file ${i + 1}/${files.length}: ${file.name}`)
if (i < files.length - 1) {
await sleep(150)
}
} catch (error) {
logger.error(`Failed to download file ${file.name}:`, error)
}
}
} finally {
setIsDownloading(false)
}
}
return (
<Button
variant='ghost-secondary'
onClick={handleDownloadAll}
disabled={isDownloading}
className='p-0'
>
{isDownloading ? (
<Loader className='size-3' animate />
) : (
<Download className='size-3' strokeWidth={2} />
)}
</Button>
)
}
@@ -0,0 +1,175 @@
import React, { type HTMLAttributes, memo, type ReactNode } from 'react'
import { Streamdown } from 'streamdown'
import 'streamdown/styles.css'
import { CopyCodeButton, Tooltip } from '@sim/emcn'
import { extractTextContent } from '@/lib/core/utils/react-node-text'
function LinkWithPreview({ href, children }: { href: string; children: React.ReactNode }) {
return (
<Tooltip.Root delayDuration={300}>
<Tooltip.Trigger asChild>
<a
href={href}
className='text-[var(--text-primary)] underline decoration-dashed underline-offset-4'
target='_blank'
rel='noopener noreferrer'
>
{children}
</a>
</Tooltip.Trigger>
<Tooltip.Content side='top' align='center' sideOffset={5} className='max-w-sm'>
<span className='truncate font-medium text-xs'>{href}</span>
</Tooltip.Content>
</Tooltip.Root>
)
}
const COMPONENTS = {
p: ({ children }: React.HTMLAttributes<HTMLParagraphElement>) => (
<p className='mb-1 font-sans text-[var(--text-primary)] text-base leading-relaxed last:mb-0'>
{children}
</p>
),
h1: ({ children }: React.HTMLAttributes<HTMLHeadingElement>) => (
<h1 className='mt-10 mb-5 font-sans font-semibold text-2xl text-[var(--text-primary)]'>
{children}
</h1>
),
h2: ({ children }: React.HTMLAttributes<HTMLHeadingElement>) => (
<h2 className='mt-8 mb-4 font-sans font-semibold text-[var(--text-primary)] text-xl'>
{children}
</h2>
),
h3: ({ children }: React.HTMLAttributes<HTMLHeadingElement>) => (
<h3 className='mt-7 mb-3 font-sans font-semibold text-[var(--text-primary)] text-lg'>
{children}
</h3>
),
h4: ({ children }: React.HTMLAttributes<HTMLHeadingElement>) => (
<h4 className='mt-5 mb-2 font-sans font-semibold text-[var(--text-primary)] text-base'>
{children}
</h4>
),
ul: ({ children }: React.HTMLAttributes<HTMLUListElement>) => (
<ul
className='mt-1 mb-1 space-y-1 pl-6 font-sans text-[var(--text-primary)]'
style={{ listStyleType: 'disc' }}
>
{children}
</ul>
),
ol: ({ children }: React.HTMLAttributes<HTMLOListElement>) => (
<ol
className='mt-1 mb-1 space-y-1 pl-6 font-sans text-[var(--text-primary)]'
style={{ listStyleType: 'decimal' }}
>
{children}
</ol>
),
li: ({ children }: React.LiHTMLAttributes<HTMLLIElement>) => (
<li className='font-sans text-[var(--text-primary)]' style={{ display: 'list-item' }}>
{children}
</li>
),
pre: ({ children }: HTMLAttributes<HTMLPreElement>) => {
let codeProps: HTMLAttributes<HTMLElement> = {}
let codeContent: ReactNode = children
if (
React.isValidElement<{ className?: string; children?: ReactNode }>(children) &&
children.type === 'code'
) {
const childElement = children as React.ReactElement<{
className?: string
children?: ReactNode
}>
codeProps = { className: childElement.props.className }
codeContent = childElement.props.children
}
return (
<div className='my-6 overflow-hidden rounded-lg border border-[var(--divider)] text-sm'>
<div className='flex items-center justify-between border-[var(--divider)] border-b bg-[var(--surface-4)] px-4 py-1.5'>
<span className='font-sans text-[var(--text-tertiary)] text-xs'>
{codeProps.className?.replace('language-', '') || 'code'}
</span>
<CopyCodeButton
code={extractTextContent(codeContent)}
className='text-[var(--text-tertiary)] hover-hover:bg-[var(--surface-5)] hover-hover:text-[var(--text-secondary)]'
/>
</div>
<pre className='overflow-x-auto bg-[var(--surface-5)] p-4 font-mono text-[var(--text-primary)]'>
{codeContent}
</pre>
</div>
)
},
inlineCode: ({ children }: { children?: React.ReactNode }) => (
<code className='rounded bg-[var(--surface-5)] px-1 py-0.5 font-mono text-[var(--text-primary)] text-inherit'>
{children}
</code>
),
blockquote: ({ children }: React.HTMLAttributes<HTMLQuoteElement>) => (
<blockquote className='my-4 break-words border-[var(--divider)] border-l-2 pl-4 font-sans text-[var(--text-primary)] italic [&>p:first-child]:mt-0 [&>p:last-child]:mb-0 [&>p]:my-2'>
{children}
</blockquote>
),
hr: () => <hr className='my-8 border-[var(--divider)] border-t' />,
a: ({ href, children, ...props }: React.AnchorHTMLAttributes<HTMLAnchorElement>) => (
<LinkWithPreview href={href || '#'} {...props}>
{children}
</LinkWithPreview>
),
table: ({ children }: React.TableHTMLAttributes<HTMLTableElement>) => (
<div className='my-4 w-full overflow-x-auto'>
<table className='min-w-full table-auto border border-[var(--border)] font-sans text-sm'>
{children}
</table>
</div>
),
thead: ({ children }: React.HTMLAttributes<HTMLTableSectionElement>) => (
<thead className='bg-[var(--surface-3)] text-left'>{children}</thead>
),
tbody: ({ children }: React.HTMLAttributes<HTMLTableSectionElement>) => (
<tbody className='divide-y divide-[var(--divider)] bg-[var(--surface-2)]'>{children}</tbody>
),
tr: ({ children }: React.HTMLAttributes<HTMLTableRowElement>) => (
<tr className='border-[var(--divider)] border-b transition-colors hover:bg-[var(--surface-hover)]'>
{children}
</tr>
),
th: ({ children }: React.ThHTMLAttributes<HTMLTableCellElement>) => (
<th className='border-[var(--border)] border-r px-4 py-2 font-medium text-[var(--text-secondary)] last:border-r-0'>
{children}
</th>
),
td: ({ children }: React.TdHTMLAttributes<HTMLTableCellElement>) => (
<td className='break-words border-[var(--border)] border-r px-4 py-2 text-[var(--text-body)] last:border-r-0'>
{children}
</td>
),
img: ({ src, alt, ...props }: React.ImgHTMLAttributes<HTMLImageElement>) => (
<img src={src} alt={alt || 'Image'} className='my-3 h-auto max-w-full rounded-md' {...props} />
),
}
const MarkdownRenderer = memo(function MarkdownRenderer({ content }: { content: string }) {
return (
<div className='space-y-4 break-words font-sans text-[var(--text-primary)] text-base leading-relaxed'>
<Streamdown mode='static' components={COMPONENTS}>
{content.trim()}
</Streamdown>
</div>
)
})
export default MarkdownRenderer
@@ -0,0 +1,43 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
vi.mock('@sim/emcn', () => ({
Duplicate: () => null,
Tooltip: {},
}))
vi.mock('@/app/(interfaces)/chat/components/message/components/file-download', () => ({
ChatFileDownload: () => null,
ChatFileDownloadAll: () => null,
}))
vi.mock('@/app/(interfaces)/chat/components/message/components/markdown-renderer', () => ({
default: () => null,
}))
import { escapeHtml } from '@/app/(interfaces)/chat/components/message/message'
describe('escapeHtml', () => {
it('escapes all five HTML-significant characters', () => {
expect(escapeHtml('&<>"\'')).toBe('&amp;&lt;&gt;&quot;&#39;')
})
it('neutralizes a markup-breakout filename payload', () => {
const payload = '</title><img src=x onerror=alert(document.origin)>'
const escaped = escapeHtml(payload)
expect(escaped).not.toContain('<img')
expect(escaped).not.toContain('</title>')
expect(escaped).toBe('&lt;/title&gt;&lt;img src=x onerror=alert(document.origin)&gt;')
})
it('escapes ampersands first so entities are not double-broken', () => {
expect(escapeHtml('a & b < c')).toBe('a &amp; b &lt; c')
})
it('leaves safe strings untouched', () => {
expect(escapeHtml('report-2026.pdf')).toBe('report-2026.pdf')
expect(escapeHtml('')).toBe('')
})
})
@@ -0,0 +1,282 @@
'use client'
import { memo, useState } from 'react'
import { Button, cn, Duplicate, Tooltip } from '@sim/emcn'
import { Check, File as FileIcon, FileText, Image as ImageIcon } from 'lucide-react'
import {
ChatFileDownload,
ChatFileDownloadAll,
} from '@/app/(interfaces)/chat/components/message/components/file-download'
import MarkdownRenderer from '@/app/(interfaces)/chat/components/message/components/markdown-renderer'
export interface ChatAttachment {
id: string
name: string
type: string
dataUrl: string
size?: number
}
export interface ChatFile {
id: string
name: string
url: string
key: string
size: number
type: string
context?: string
}
export interface ChatMessage {
id: string
content: string | Record<string, unknown>
type: 'user' | 'assistant'
timestamp: Date
isInitialMessage?: boolean
isStreaming?: boolean
attachments?: ChatAttachment[]
files?: ChatFile[]
}
const HTML_ESCAPES: Record<string, string> = {
'&': '&amp;',
'<': '&lt;',
'>': '&gt;',
'"': '&quot;',
"'": '&#39;',
} as const
/**
* Escapes HTML entities so untrusted strings are safe to interpolate into markup.
*/
export function escapeHtml(value: string): string {
return value.replace(/[&<>"']/g, (c) => HTML_ESCAPES[c] || c)
}
/**
* Opens an image attachment preview in a new tab via a blob URL,
* escaping the user-controlled filename and data URL to prevent XSS.
*/
function openAttachmentPreview(name: string, dataUrl: string): void {
const safeName = escapeHtml(name)
const safeUrl = escapeHtml(dataUrl)
const html = `
<!DOCTYPE html>
<html>
<head>
<title>${safeName}</title>
<style>
body { margin: 0; display: flex; justify-content: center; align-items: center; min-height: 100vh; background: #000; }
img { max-width: 100%; max-height: 100vh; object-fit: contain; }
</style>
</head>
<body>
<img src="${safeUrl}" alt="${safeName}" />
</body>
</html>
`
const blob = new Blob([html], { type: 'text/html' })
const blobUrl = URL.createObjectURL(blob)
window.open(blobUrl, '_blank', 'noopener,noreferrer')
setTimeout(() => URL.revokeObjectURL(blobUrl), 60_000)
}
export const ClientChatMessage = memo(
function ClientChatMessage({ message }: { message: ChatMessage }) {
const [isCopied, setIsCopied] = useState(false)
const isJsonObject = typeof message.content === 'object' && message.content !== null
// Since tool calls are now handled via SSE events and stored in message.toolCalls,
// we can use the content directly without parsing
const cleanTextContent = message.content
const content =
message.type === 'user' ? (
<div className='px-4 py-5' data-message-id={message.id}>
<div className='mx-auto max-w-3xl'>
{/* File attachments displayed above the message */}
{message.attachments && message.attachments.length > 0 && (
<div className='mb-2 flex justify-end'>
<div className='flex flex-wrap gap-2'>
{message.attachments.map((attachment) => {
const isImage = attachment.type.startsWith('image/')
const getFileIcon = (type: string) => {
if (type.includes('pdf'))
return <FileText className='size-5 text-[var(--text-muted)] md:size-6' />
if (type.startsWith('image/'))
return <ImageIcon className='size-5 text-[var(--text-muted)] md:size-6' />
if (type.includes('text') || type.includes('json'))
return <FileText className='size-5 text-[var(--text-muted)] md:size-6' />
return <FileIcon className='size-5 text-[var(--text-muted)] md:size-6' />
}
const formatFileSize = (bytes?: number) => {
if (!bytes || bytes === 0) return ''
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return `${Math.round((bytes / k ** i) * 10) / 10} ${sizes[i]}`
}
const isInteractive =
!!attachment.dataUrl?.trim() && attachment.dataUrl.startsWith('data:')
const handleOpenPreview = () => {
const validDataUrl = attachment.dataUrl?.trim()
if (!validDataUrl?.startsWith('data:')) return
openAttachmentPreview(attachment.name, validDataUrl)
}
return (
<div
key={attachment.id}
role={isInteractive ? 'button' : undefined}
aria-disabled={!isInteractive}
tabIndex={isInteractive ? 0 : undefined}
className={cn(
'relative overflow-hidden rounded-2xl border border-[var(--border-1)] bg-[var(--surface-2)]',
isInteractive && 'cursor-pointer',
isImage
? 'size-16 md:size-20'
: 'flex h-16 min-w-[140px] max-w-[220px] items-center gap-2 px-3 md:h-20 md:min-w-[160px] md:max-w-[240px]'
)}
onClick={(e) => {
if (!isInteractive) return
e.preventDefault()
e.stopPropagation()
handleOpenPreview()
}}
onKeyDown={(e) => {
if (!isInteractive) return
if (e.key === 'Enter' || e.key === ' ') {
e.preventDefault()
e.stopPropagation()
handleOpenPreview()
}
}}
>
{isImage &&
attachment.dataUrl?.trim() &&
attachment.dataUrl.startsWith('data:') ? (
<img
src={attachment.dataUrl}
alt={attachment.name}
className='size-full object-cover'
/>
) : (
<>
<div className='flex size-10 flex-shrink-0 items-center justify-center rounded bg-[var(--surface-3)] md:size-12'>
{getFileIcon(attachment.type)}
</div>
<div className='min-w-0 flex-1'>
<div className='truncate font-medium text-[var(--text-primary)] text-xs md:text-sm'>
{attachment.name}
</div>
{attachment.size && (
<div className='text-[var(--text-muted)] text-micro md:text-xs'>
{formatFileSize(attachment.size)}
</div>
)}
</div>
</>
)}
</div>
)
})}
</div>
</div>
)}
{/* Only render message bubble if there's actual text content (not just file count message) */}
{message.content && !String(message.content).startsWith('Sent') && (
<div className='flex justify-end'>
<div className='max-w-[80%] rounded-3xl bg-[var(--surface-3)] px-4 py-3'>
<div className='whitespace-pre-wrap break-words text-[var(--text-primary)] text-base leading-relaxed'>
{isJsonObject ? (
<pre>{JSON.stringify(message.content, null, 2)}</pre>
) : (
<span>{message.content as string}</span>
)}
</div>
</div>
</div>
)}
</div>
</div>
) : (
<div className='px-4 pt-5 pb-2' data-message-id={message.id}>
<div className='mx-auto max-w-3xl'>
<div className='flex flex-col space-y-3'>
{/* Direct content rendering - tool calls are now handled via SSE events */}
<div>
<div className='break-words text-base'>
{isJsonObject ? (
<pre className='text-[var(--text-primary)]'>
{JSON.stringify(cleanTextContent, null, 2)}
</pre>
) : (
<MarkdownRenderer content={cleanTextContent as string} />
)}
</div>
</div>
{message.files && message.files.length > 0 && (
<div className='flex flex-wrap gap-2'>
{message.files.map((file) => (
<ChatFileDownload key={file.id} file={file} />
))}
</div>
)}
{message.type === 'assistant' && !isJsonObject && !message.isInitialMessage && (
<div className='flex items-center justify-start space-x-2'>
{/* Copy Button - Only show when not streaming */}
{!message.isStreaming && (
<Tooltip.Root delayDuration={300}>
<Tooltip.Trigger asChild>
<Button
variant='ghost-secondary'
className='p-0'
onClick={() => {
const contentToCopy =
typeof cleanTextContent === 'string'
? cleanTextContent
: JSON.stringify(cleanTextContent, null, 2)
navigator.clipboard.writeText(contentToCopy)
setIsCopied(true)
setTimeout(() => setIsCopied(false), 2000)
}}
>
{isCopied ? (
<Check className='size-3' strokeWidth={2} />
) : (
<Duplicate className='size-3' />
)}
</Button>
</Tooltip.Trigger>
<Tooltip.Content side='top' align='center' sideOffset={5}>
{isCopied ? 'Copied!' : 'Copy to clipboard'}
</Tooltip.Content>
</Tooltip.Root>
)}
{/* Download All Button - Only show when there are files */}
{!message.isStreaming && message.files && (
<ChatFileDownloadAll files={message.files} />
)}
</div>
)}
</div>
</div>
</div>
)
return <Tooltip.Provider>{content}</Tooltip.Provider>
},
(prevProps, nextProps) => {
return (
prevProps.message.id === nextProps.message.id &&
prevProps.message.content === nextProps.message.content &&
prevProps.message.isStreaming === nextProps.message.isStreaming &&
prevProps.message.isInitialMessage === nextProps.message.isInitialMessage &&
prevProps.message.files?.length === nextProps.message.files?.length
)
}
)
@@ -0,0 +1,503 @@
'use client'
import { useCallback, useEffect, useRef } from 'react'
import { createLogger } from '@sim/logger'
import * as THREE from 'three'
const logger = createLogger('Particles')
interface ShaderUniforms {
u_time: { type: string; value: number }
u_frequency: { type: string; value: number }
u_red: { type: string; value: number }
u_green: { type: string; value: number }
u_blue: { type: string; value: number }
}
interface ParticlesProps {
audioLevels: number[]
isListening: boolean
isPlayingAudio: boolean
isStreaming: boolean
isMuted: boolean
isProcessingInterruption?: boolean
className?: string
}
class SimpleBloomComposer {
private renderer: THREE.WebGLRenderer
private scene: THREE.Scene
private camera: THREE.Camera
private bloomScene: THREE.Scene
private bloomMaterial: THREE.ShaderMaterial
private renderTarget: THREE.WebGLRenderTarget
private quad: THREE.Mesh
constructor(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera) {
this.renderer = renderer
this.scene = scene
this.camera = camera
this.bloomScene = new THREE.Scene()
this.renderTarget = new THREE.WebGLRenderTarget(
renderer.domElement.width,
renderer.domElement.height,
{
minFilter: THREE.LinearFilter,
magFilter: THREE.LinearFilter,
format: THREE.RGBAFormat,
}
)
this.bloomMaterial = new THREE.ShaderMaterial({
uniforms: {
tDiffuse: { value: null },
strength: { value: 1.5 },
threshold: { value: 0.3 },
radius: { value: 0.8 },
},
vertexShader: `
varying vec2 vUv;
void main() {
vUv = uv;
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
}
`,
fragmentShader: `
uniform sampler2D tDiffuse;
uniform float strength;
uniform float threshold;
uniform float radius;
varying vec2 vUv;
void main() {
vec4 color = texture2D(tDiffuse, vUv);
// Simple bloom effect
float brightness = dot(color.rgb, vec3(0.299, 0.587, 0.114));
if (brightness > threshold) {
color.rgb *= strength;
}
gl_FragColor = color;
}
`,
})
const geometry = new THREE.PlaneGeometry(2, 2)
this.quad = new THREE.Mesh(geometry, this.bloomMaterial)
this.bloomScene.add(this.quad)
}
render() {
this.renderer.setRenderTarget(this.renderTarget)
this.renderer.render(this.scene, this.camera)
this.bloomMaterial.uniforms.tDiffuse.value = this.renderTarget.texture
this.renderer.setRenderTarget(null)
this.renderer.render(this.bloomScene, new THREE.OrthographicCamera(-1, 1, 1, -1, 0, 1))
}
setSize(width: number, height: number) {
this.renderTarget.setSize(width, height)
}
dispose() {
this.renderTarget.dispose()
this.bloomMaterial.dispose()
}
}
const vertexShader = `
vec3 mod289(vec3 x)
{
return x - floor(x * (1.0 / 289.0)) * 289.0;
}
vec4 mod289(vec4 x)
{
return x - floor(x * (1.0 / 289.0)) * 289.0;
}
vec4 permute(vec4 x)
{
return mod289(((x*34.0)+10.0)*x);
}
vec4 taylorInvSqrt(vec4 r)
{
return 1.79284291400159 - 0.85373472095314 * r;
}
vec3 fade(vec3 t) {
return t*t*t*(t*(t*6.0-15.0)+10.0);
}
float pnoise(vec3 P, vec3 rep)
{
vec3 Pi0 = mod(floor(P), rep); // Integer part, modulo period
vec3 Pi1 = mod(Pi0 + vec3(1.0), rep); // Integer part + 1, mod period
Pi0 = mod289(Pi0);
Pi1 = mod289(Pi1);
vec3 Pf0 = fract(P); // Fractional part for interpolation
vec3 Pf1 = Pf0 - vec3(1.0); // Fractional part - 1.0
vec4 ix = vec4(Pi0.x, Pi1.x, Pi0.x, Pi1.x);
vec4 iy = vec4(Pi0.yy, Pi1.yy);
vec4 iz0 = Pi0.zzzz;
vec4 iz1 = Pi1.zzzz;
vec4 ixy = permute(permute(ix) + iy);
vec4 ixy0 = permute(ixy + iz0);
vec4 ixy1 = permute(ixy + iz1);
vec4 gx0 = ixy0 * (1.0 / 7.0);
vec4 gy0 = fract(floor(gx0) * (1.0 / 7.0)) - 0.5;
gx0 = fract(gx0);
vec4 gz0 = vec4(0.5) - abs(gx0) - abs(gy0);
vec4 sz0 = step(gz0, vec4(0.0));
gx0 -= sz0 * (step(0.0, gx0) - 0.5);
gy0 -= sz0 * (step(0.0, gy0) - 0.5);
vec4 gx1 = ixy1 * (1.0 / 7.0);
vec4 gy1 = fract(floor(gx1) * (1.0 / 7.0)) - 0.5;
gx1 = fract(gx1);
vec4 gz1 = vec4(0.5) - abs(gx1) - abs(gy1);
vec4 sz1 = step(gz1, vec4(0.0));
gx1 -= sz1 * (step(0.0, gx1) - 0.5);
gy1 -= sz1 * (step(0.0, gy1) - 0.5);
vec3 g000 = vec3(gx0.x,gy0.x,gz0.x);
vec3 g100 = vec3(gx0.y,gy0.y,gz0.y);
vec3 g010 = vec3(gx0.z,gy0.z,gz0.z);
vec3 g110 = vec3(gx0.w,gy0.w,gz0.w);
vec3 g001 = vec3(gx1.x,gy1.x,gz1.x);
vec3 g101 = vec3(gx1.y,gy1.y,gz1.y);
vec3 g011 = vec3(gx1.z,gy1.z,gz1.z);
vec3 g111 = vec3(gx1.w,gy1.w,gz1.w);
vec4 norm0 = taylorInvSqrt(vec4(dot(g000, g000), dot(g010, g010), dot(g100, g100), dot(g110, g110)));
g000 *= norm0.x;
g010 *= norm0.y;
g100 *= norm0.z;
g110 *= norm0.w;
vec4 norm1 = taylorInvSqrt(vec4(dot(g001, g001), dot(g011, g011), dot(g101, g101), dot(g111, g111)));
g001 *= norm1.x;
g011 *= norm1.y;
g101 *= norm1.z;
g111 *= norm1.w;
float n000 = dot(g000, Pf0);
float n100 = dot(g100, vec3(Pf1.x, Pf0.yz));
float n010 = dot(g010, vec3(Pf0.x, Pf1.y, Pf0.z));
float n110 = dot(g110, vec3(Pf1.xy, Pf0.z));
float n001 = dot(g001, vec3(Pf0.xy, Pf1.z));
float n101 = dot(g101, vec3(Pf1.x, Pf0.y, Pf1.z));
float n011 = dot(g011, vec3(Pf0.x, Pf1.yz));
float n111 = dot(g111, Pf1);
vec3 fade_xyz = fade(Pf0);
vec4 n_z = mix(vec4(n000, n100, n010, n110), vec4(n001, n101, n011, n111), fade_xyz.z);
vec2 n_yz = mix(n_z.xy, n_z.zw, fade_xyz.y);
float n_xyz = mix(n_yz.x, n_yz.y, fade_xyz.x);
return 2.2 * n_xyz;
}
uniform float u_time;
uniform float u_frequency;
void main() {
float noise = 5. * pnoise(position + u_time, vec3(10.));
float displacement = (u_frequency / 30.) * (noise / 10.);
vec3 newPosition = position + normal * displacement;
gl_Position = projectionMatrix * modelViewMatrix * vec4(newPosition, 1.0);
}
`
const fragmentShader = `
uniform float u_red;
uniform float u_blue;
uniform float u_green;
void main() {
gl_FragColor = vec4(vec3(u_red, u_green, u_blue), 1.0);
}
`
export function ParticlesVisualization({
audioLevels,
isListening,
isPlayingAudio,
isStreaming,
isMuted,
isProcessingInterruption,
className,
}: ParticlesProps) {
const containerRef = useRef<HTMLDivElement>(null)
const rendererRef = useRef<THREE.WebGLRenderer | null>(null)
const sceneRef = useRef<THREE.Scene | null>(null)
const cameraRef = useRef<THREE.PerspectiveCamera | null>(null)
const meshRef = useRef<THREE.Mesh | null>(null)
const uniformsRef = useRef<ShaderUniforms | null>(null)
const clockRef = useRef<THREE.Clock | null>(null)
const bloomComposerRef = useRef<SimpleBloomComposer | null>(null)
const animationFrameRef = useRef<number>(0)
const mouseRef = useRef({ x: 0, y: 0 })
const isInitializedRef = useRef(false)
const cleanup = useCallback(() => {
if (animationFrameRef.current) {
cancelAnimationFrame(animationFrameRef.current)
animationFrameRef.current = 0
}
if (bloomComposerRef.current) {
bloomComposerRef.current.dispose()
bloomComposerRef.current = null
}
if (rendererRef.current) {
if (rendererRef.current.domElement?.parentNode) {
rendererRef.current.domElement.parentNode.removeChild(rendererRef.current.domElement)
}
rendererRef.current.dispose()
rendererRef.current = null
}
sceneRef.current = null
cameraRef.current = null
meshRef.current = null
uniformsRef.current = null
clockRef.current = null
isInitializedRef.current = false
}, [])
useEffect(() => {
if (!containerRef.current || isInitializedRef.current) return
const container = containerRef.current
const containerWidth = 400
const containerHeight = 400
isInitializedRef.current = true
while (container.firstChild) {
container.removeChild(container.firstChild)
}
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true })
renderer.setSize(containerWidth, containerHeight)
renderer.setClearColor(0x000000, 0)
renderer.outputColorSpace = THREE.SRGBColorSpace
container.appendChild(renderer.domElement)
rendererRef.current = renderer
const scene = new THREE.Scene()
sceneRef.current = scene
const camera = new THREE.PerspectiveCamera(45, containerWidth / containerHeight, 0.1, 1000)
camera.position.set(0, -2, 14)
camera.lookAt(0, 0, 0)
cameraRef.current = camera
const uniforms = {
u_time: { type: 'f', value: 0.0 },
u_frequency: { type: 'f', value: 0.0 },
u_red: { type: 'f', value: 0.8 },
u_green: { type: 'f', value: 0.6 },
u_blue: { type: 'f', value: 1.0 },
}
uniformsRef.current = uniforms
let mat: THREE.Material
try {
mat = new THREE.ShaderMaterial({
uniforms,
vertexShader,
fragmentShader,
})
} catch (error) {
logger.error('❌ Shader compilation error, using fallback material:', error)
mat = new THREE.MeshBasicMaterial({
color: 0xb794f6, // Light purple color
wireframe: true,
})
}
const geo = new THREE.IcosahedronGeometry(4, 30) // Match tutorial: radius 4, subdivisions 30
const mesh = new THREE.Mesh(geo, mat)
if (mat instanceof THREE.ShaderMaterial || mat instanceof THREE.MeshBasicMaterial) {
mat.wireframe = true
}
scene.add(mesh)
meshRef.current = mesh
const bloomComposer = new SimpleBloomComposer(renderer, scene, camera)
bloomComposerRef.current = bloomComposer
const clock = new THREE.Clock()
clockRef.current = clock
const handleMouseMove = (e: MouseEvent) => {
const rect = container.getBoundingClientRect()
const windowHalfX = containerWidth / 2
const windowHalfY = containerHeight / 2
mouseRef.current.x = (e.clientX - rect.left - windowHalfX) / 100
mouseRef.current.y = (e.clientY - rect.top - windowHalfY) / 100
}
container.addEventListener('mousemove', handleMouseMove)
const updateCameraPosition = () => {
if (!camera || !scene) return
camera.position.x += (mouseRef.current.x - camera.position.x) * 0.05
camera.position.y += (-mouseRef.current.y - camera.position.y) * 0.5
camera.lookAt(scene.position)
}
const calculateAudioIntensity = (elapsedTime: number, avgLevel: number) => {
const baselineIntensity = 8 + Math.sin(elapsedTime * 0.5) * 3
let audioIntensity = baselineIntensity
if (isMuted) {
// When muted, only show minimal baseline animation
audioIntensity = baselineIntensity * 0.2
} else if (isProcessingInterruption) {
// Special pulsing effect during interruption processing
audioIntensity = 35 + Math.sin(elapsedTime * 4) * 10
} else if (isPlayingAudio) {
// Strong animation when AI is speaking - use simulated levels + enhancement
const aiIntensity = 60 + Math.sin(elapsedTime * 3) * 20
audioIntensity = Math.max(avgLevel * 0.8, aiIntensity)
} else if (isStreaming) {
// Pulsing animation when AI is thinking/streaming
audioIntensity = 40 + Math.sin(elapsedTime * 2) * 15
} else if (isListening && avgLevel > 0) {
// Scale user input more dramatically for better visual feedback
const userVoiceIntensity = avgLevel * 2.5 // Amplify user voice significantly
audioIntensity = Math.max(userVoiceIntensity, baselineIntensity * 1.5)
// Add some dynamic variation based on audio levels
const variationFactor = Math.min(avgLevel / 20, 1) // Cap at reasonable level
audioIntensity += Math.sin(elapsedTime * 8) * (10 * variationFactor)
} else {
// Idle state - subtle breathing animation
audioIntensity = baselineIntensity
}
// Clamp to reasonable range
audioIntensity = Math.max(audioIntensity, 3) // Never completely still
audioIntensity = Math.min(audioIntensity, 120) // Prevent excessive animation
return audioIntensity
}
const updateShaderColors = (
uniforms: ShaderUniforms,
elapsedTime: number,
avgLevel: number
) => {
if (isMuted) {
// Muted: dim purple-gray
uniforms.u_red.value = 0.25
uniforms.u_green.value = 0.1
uniforms.u_blue.value = 0.5
} else if (isProcessingInterruption) {
// Interruption: bright purple
uniforms.u_red.value = 0.6
uniforms.u_green.value = 0.2
uniforms.u_blue.value = 0.9
} else if (isPlayingAudio) {
// AI speaking: brand purple (#701FFC)
uniforms.u_red.value = 0.44
uniforms.u_green.value = 0.12
uniforms.u_blue.value = 0.99
} else if (isListening && avgLevel > 10) {
// User speaking: lighter purple with intensity-based variation
const intensity = Math.min(avgLevel / 50, 1)
uniforms.u_red.value = 0.35 + intensity * 0.15
uniforms.u_green.value = 0.1 + intensity * 0.1
uniforms.u_blue.value = 0.8 + intensity * 0.2
} else if (isStreaming) {
// AI thinking: pulsing brand purple
const pulse = (Math.sin(elapsedTime * 2) + 1) / 2
uniforms.u_red.value = 0.35 + pulse * 0.15
uniforms.u_green.value = 0.08 + pulse * 0.08
uniforms.u_blue.value = 0.95 + pulse * 0.05
} else {
// Default idle: soft brand purple
uniforms.u_red.value = 0.4
uniforms.u_green.value = 0.15
uniforms.u_blue.value = 0.9
}
}
const animate = () => {
if (!camera || !clock || !scene || !bloomComposer || !isInitializedRef.current) return
updateCameraPosition()
if (uniforms) {
const elapsedTime = clock.getElapsedTime()
const avgLevel = audioLevels.reduce((sum, level) => sum + level, 0) / audioLevels.length
uniforms.u_time.value = elapsedTime
const audioIntensity = calculateAudioIntensity(elapsedTime, avgLevel)
updateShaderColors(uniforms, elapsedTime, avgLevel)
uniforms.u_frequency.value = audioIntensity
}
bloomComposer.render()
animationFrameRef.current = requestAnimationFrame(animate)
}
animate()
return () => {
container.removeEventListener('mousemove', handleMouseMove)
cleanup()
}
}, [])
useEffect(() => {
const handleResize = () => {
if (
rendererRef.current &&
cameraRef.current &&
bloomComposerRef.current &&
containerRef.current
) {
const containerWidth = 400
const containerHeight = 400
cameraRef.current.aspect = containerWidth / containerHeight
cameraRef.current.updateProjectionMatrix()
rendererRef.current.setSize(containerWidth, containerHeight)
bloomComposerRef.current.setSize(containerWidth, containerHeight)
}
}
window.addEventListener('resize', handleResize)
return () => window.removeEventListener('resize', handleResize)
}, [])
return (
<div
ref={containerRef}
className={className}
style={{
width: '400px',
height: '400px',
display: 'flex',
justifyContent: 'center',
alignItems: 'center',
}}
/>
)
}
@@ -0,0 +1,580 @@
'use client'
import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'
import { Button, cn } from '@sim/emcn'
import { createLogger } from '@sim/logger'
import { Mic, MicOff, Phone } from 'lucide-react'
import dynamic from 'next/dynamic'
import { requestJson } from '@/lib/api/client/request'
import { speechTokenContract } from '@/lib/api/contracts/media/speech'
import { arrayBufferToBase64, floatTo16BitPCM } from '@/lib/speech/audio'
import {
CHUNK_SEND_INTERVAL_MS,
ELEVENLABS_WS_URL,
MAX_CHAT_SESSION_MS,
SAMPLE_RATE,
} from '@/lib/speech/config'
const ParticlesVisualization = dynamic(
() =>
import('@/app/(interfaces)/chat/components/voice-interface/components/particles').then(
(mod) => mod.ParticlesVisualization
),
{ ssr: false }
)
const logger = createLogger('VoiceInterface')
interface VoiceInterfaceProps {
onCallEnd?: () => void
onVoiceTranscript?: (transcript: string) => void
onVoiceStart?: () => void
onVoiceEnd?: () => void
onInterrupt?: () => void
isStreaming?: boolean
isPlayingAudio?: boolean
audioContextRef?: RefObject<AudioContext | null>
messages?: Array<{ content: string; type: 'user' | 'assistant' }>
className?: string
chatId?: string
}
const EMPTY_MESSAGES: Array<{ content: string; type: 'user' | 'assistant' }> = []
export function VoiceInterface({
onCallEnd,
onVoiceTranscript,
onVoiceStart,
onVoiceEnd,
onInterrupt,
isStreaming = false,
isPlayingAudio = false,
audioContextRef: sharedAudioContextRef,
messages = EMPTY_MESSAGES,
className,
chatId,
}: VoiceInterfaceProps) {
const [state, setState] = useState<'idle' | 'listening' | 'agent_speaking'>('idle')
const [isInitialized, setIsInitialized] = useState(false)
const [isMuted, setIsMuted] = useState(false)
const [audioLevels, setAudioLevels] = useState<number[]>(() => new Array(200).fill(0))
const permissionStatusRef = useRef<'prompt' | 'granted' | 'denied'>('prompt')
const [currentTranscript, setCurrentTranscript] = useState('')
const currentStateRef = useRef<'idle' | 'listening' | 'agent_speaking'>('idle')
const isCallEndedRef = useRef(false)
const updateState = useCallback((next: 'idle' | 'listening' | 'agent_speaking') => {
setState(next)
currentStateRef.current = next
}, [])
const mediaStreamRef = useRef<MediaStream | null>(null)
const audioContextRef = useRef<AudioContext | null>(null)
const analyserRef = useRef<AnalyserNode | null>(null)
const animationFrameRef = useRef<number | null>(null)
const isMutedRef = useRef(false)
const wsRef = useRef<WebSocket | null>(null)
const processorRef = useRef<ScriptProcessorNode | null>(null)
const pcmBufferRef = useRef<Float32Array[]>([])
const sendIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
const sessionTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
const committedTextRef = useRef('')
const lastPartialRef = useRef('')
const onVoiceTranscriptRef = useRef(onVoiceTranscript)
onVoiceTranscriptRef.current = onVoiceTranscript
const updateIsMuted = useCallback((next: boolean) => {
setIsMuted(next)
isMutedRef.current = next
}, [])
const stopSendingAudio = useCallback(() => {
if (sessionTimerRef.current) {
clearTimeout(sessionTimerRef.current)
sessionTimerRef.current = null
}
if (sendIntervalRef.current) {
clearInterval(sendIntervalRef.current)
sendIntervalRef.current = null
}
pcmBufferRef.current = []
}, [])
const flushAudioBuffer = useCallback(() => {
const ws = wsRef.current
if (!ws || ws.readyState !== WebSocket.OPEN) return
const chunks = pcmBufferRef.current
if (chunks.length === 0) return
pcmBufferRef.current = []
let totalLength = 0
for (const chunk of chunks) totalLength += chunk.length
const merged = new Float32Array(totalLength)
let offset = 0
for (const chunk of chunks) {
merged.set(chunk, offset)
offset += chunk.length
}
const pcm16 = floatTo16BitPCM(merged)
ws.send(
JSON.stringify({
message_type: 'input_audio_chunk',
audio_base_64: arrayBufferToBase64(pcm16),
sample_rate: SAMPLE_RATE,
commit: false,
})
)
}, [])
const startSendingAudio = useCallback(() => {
if (sendIntervalRef.current) return
pcmBufferRef.current = []
sendIntervalRef.current = setInterval(flushAudioBuffer, CHUNK_SEND_INTERVAL_MS)
}, [flushAudioBuffer])
const closeWebSocket = useCallback(() => {
stopSendingAudio()
if (wsRef.current) {
if (
wsRef.current.readyState === WebSocket.OPEN ||
wsRef.current.readyState === WebSocket.CONNECTING
) {
wsRef.current.close()
}
wsRef.current = null
}
}, [stopSendingAudio])
const connectWebSocket = useCallback(async (): Promise<boolean> => {
try {
let tokenData: Awaited<ReturnType<typeof requestJson<typeof speechTokenContract>>>
try {
tokenData = await requestJson(speechTokenContract, {
body: chatId ? { chatId } : {},
})
} catch (err) {
logger.error('Failed to get STT token', err)
return false
}
const token = typeof tokenData.token === 'string' ? tokenData.token : undefined
if (!token) {
logger.error('STT token missing from response')
return false
}
const params = new URLSearchParams({
token,
model_id: 'scribe_v2_realtime',
audio_format: 'pcm_16000',
commit_strategy: 'vad',
vad_silence_threshold_secs: '1.0',
})
const ws = new WebSocket(`${ELEVENLABS_WS_URL}?${params.toString()}`)
wsRef.current = ws
committedTextRef.current = ''
return new Promise<boolean>((resolve) => {
ws.onopen = () => resolve(true)
ws.onerror = () => {
logger.error('STT WebSocket connection error')
resolve(false)
}
ws.onmessage = (event) => {
if (isCallEndedRef.current) return
try {
const msg = JSON.parse(event.data)
if (msg.message_type === 'partial_transcript') {
if (msg.text) {
lastPartialRef.current = msg.text
setCurrentTranscript(msg.text)
}
} else if (
msg.message_type === 'committed_transcript' ||
msg.message_type === 'committed_transcript_with_timestamps'
) {
const finalText = msg.text || lastPartialRef.current
lastPartialRef.current = ''
if (finalText) {
committedTextRef.current = committedTextRef.current
? `${committedTextRef.current} ${finalText}`
: finalText
setCurrentTranscript('')
onVoiceTranscriptRef.current?.(finalText)
}
} else if (
msg.message_type === 'error' ||
msg.message_type === 'auth_error' ||
msg.message_type === 'quota_exceeded'
) {
logger.error('ElevenLabs STT error', { type: msg.message_type, error: msg.error })
}
} catch {
// Ignore non-JSON messages
}
}
ws.onclose = () => {
wsRef.current = null
if (currentStateRef.current === 'listening' && !isCallEndedRef.current) {
stopSendingAudio()
updateState('idle')
}
}
})
} catch (error) {
logger.error('Failed to connect STT WebSocket', error)
return false
}
}, [chatId])
const setupAudioPipeline = useCallback(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
channelCount: 1,
sampleRate: SAMPLE_RATE,
},
})
permissionStatusRef.current = 'granted'
mediaStreamRef.current = stream
const ac = new AudioContext({ sampleRate: SAMPLE_RATE })
audioContextRef.current = ac
if (ac.state === 'suspended') {
await ac.resume()
}
const source = ac.createMediaStreamSource(stream)
const analyser = ac.createAnalyser()
analyser.fftSize = 256
analyser.smoothingTimeConstant = 0.8
source.connect(analyser)
analyserRef.current = analyser
const processor = ac.createScriptProcessor(4096, 1, 1)
processor.onaudioprocess = (e) => {
if (!isMutedRef.current && currentStateRef.current === 'listening') {
pcmBufferRef.current.push(new Float32Array(e.inputBuffer.getChannelData(0)))
}
}
source.connect(processor)
processor.connect(ac.destination)
processorRef.current = processor
const updateVisualization = () => {
if (!analyserRef.current) return
const bufferLength = analyserRef.current.frequencyBinCount
const dataArray = new Uint8Array(bufferLength)
analyserRef.current.getByteFrequencyData(dataArray)
const levels = []
for (let i = 0; i < 200; i++) {
const dataIndex = Math.floor((i / 200) * bufferLength)
const value = dataArray[dataIndex] || 0
levels.push((value / 255) * 100)
}
setAudioLevels(levels)
animationFrameRef.current = requestAnimationFrame(updateVisualization)
}
updateVisualization()
return true
} catch (error) {
logger.error('Error setting up audio pipeline:', error)
permissionStatusRef.current = 'denied'
return false
}
}, [])
const startListening = useCallback(async () => {
if (currentStateRef.current !== 'idle' || isMutedRef.current || isCallEndedRef.current) return
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
const connected = await connectWebSocket()
if (!connected || isCallEndedRef.current) return
}
updateState('listening')
setCurrentTranscript('')
startSendingAudio()
sessionTimerRef.current = setTimeout(() => {
logger.info('Voice session reached max duration, stopping')
stopSendingAudio()
closeWebSocket()
updateState('idle')
}, MAX_CHAT_SESSION_MS)
}, [connectWebSocket, updateState, startSendingAudio, stopSendingAudio, closeWebSocket])
const stopListening = useCallback(() => {
stopSendingAudio()
updateState('idle')
setCurrentTranscript('')
}, [updateState, stopSendingAudio])
useEffect(() => {
if (isPlayingAudio && state === 'listening') {
stopSendingAudio()
closeWebSocket()
updateState('agent_speaking')
setCurrentTranscript('')
updateIsMuted(true)
if (mediaStreamRef.current) {
mediaStreamRef.current.getAudioTracks().forEach((track) => {
track.enabled = false
})
}
} else if (!isPlayingAudio && state === 'agent_speaking') {
updateState('idle')
setCurrentTranscript('')
updateIsMuted(false)
if (mediaStreamRef.current) {
mediaStreamRef.current.getAudioTracks().forEach((track) => {
track.enabled = true
})
}
}
}, [isPlayingAudio, state, updateState, updateIsMuted, stopSendingAudio, closeWebSocket])
const handleInterrupt = useCallback(() => {
if (state === 'agent_speaking') {
onInterrupt?.()
updateIsMuted(false)
if (mediaStreamRef.current) {
mediaStreamRef.current.getAudioTracks().forEach((track) => {
track.enabled = true
})
}
updateState('idle')
setCurrentTranscript('')
}
}, [state, onInterrupt, updateState, updateIsMuted])
const handleCallEnd = useCallback(() => {
isCallEndedRef.current = true
stopSendingAudio()
closeWebSocket()
updateState('idle')
setCurrentTranscript('')
updateIsMuted(false)
if (processorRef.current) {
processorRef.current.disconnect()
processorRef.current = null
}
if (mediaStreamRef.current) {
mediaStreamRef.current.getTracks().forEach((track) => track.stop())
mediaStreamRef.current = null
}
if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
audioContextRef.current.close().catch(() => {})
audioContextRef.current = null
}
if (animationFrameRef.current) {
cancelAnimationFrame(animationFrameRef.current)
animationFrameRef.current = null
}
onInterrupt?.()
onCallEnd?.()
}, [onCallEnd, onInterrupt, updateState, updateIsMuted, stopSendingAudio, closeWebSocket])
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
if (event.code === 'Space') {
event.preventDefault()
handleInterrupt()
}
}
document.addEventListener('keydown', handleKeyDown)
return () => document.removeEventListener('keydown', handleKeyDown)
}, [handleInterrupt])
const toggleMute = useCallback(() => {
if (state === 'agent_speaking') {
handleInterrupt()
return
}
const newMutedState = !isMuted
updateIsMuted(newMutedState)
if (mediaStreamRef.current) {
mediaStreamRef.current.getAudioTracks().forEach((track) => {
track.enabled = !newMutedState
})
}
if (newMutedState) {
stopListening()
} else if (state === 'idle') {
startListening()
}
}, [isMuted, state, handleInterrupt, stopListening, startListening, updateIsMuted])
useEffect(() => {
isCallEndedRef.current = false
let cancelled = false
async function init() {
const audioOk = await setupAudioPipeline()
if (!audioOk || cancelled) return
const wsOk = await connectWebSocket()
if (!wsOk || cancelled) return
setIsInitialized(true)
}
init()
return () => {
cancelled = true
}
}, [setupAudioPipeline, connectWebSocket])
useEffect(() => {
if (isInitialized && !isMuted && state === 'idle') {
startListening()
}
}, [isInitialized, isMuted, state, startListening])
useEffect(() => {
return () => {
isCallEndedRef.current = true
stopSendingAudio()
if (wsRef.current) {
wsRef.current.close()
wsRef.current = null
}
if (processorRef.current) {
processorRef.current.disconnect()
processorRef.current = null
}
if (mediaStreamRef.current) {
mediaStreamRef.current.getTracks().forEach((track) => track.stop())
mediaStreamRef.current = null
}
if (audioContextRef.current) {
audioContextRef.current.close()
audioContextRef.current = null
}
if (animationFrameRef.current) {
cancelAnimationFrame(animationFrameRef.current)
animationFrameRef.current = null
}
}
}, [stopSendingAudio])
const getStatusText = () => {
switch (state) {
case 'listening':
return 'Listening...'
case 'agent_speaking':
return 'Press Space or tap to interrupt'
default:
return isInitialized ? 'Ready' : 'Initializing...'
}
}
const getButtonContent = () => {
if (state === 'agent_speaking') {
return (
<svg className='size-6' viewBox='0 0 24 24' fill='currentColor'>
<rect x='6' y='6' width='12' height='12' rx='2' />
</svg>
)
}
return isMuted ? <MicOff className='size-6' /> : <Mic className='size-6' />
}
return (
<div
className={cn(
'light fixed inset-0 z-[100] flex flex-col bg-[var(--bg)] text-[var(--text-primary)]',
className
)}
>
<div className='flex flex-1 flex-col items-center justify-center px-8'>
<div className='relative mb-16'>
<ParticlesVisualization
audioLevels={audioLevels}
isListening={state === 'listening'}
isPlayingAudio={state === 'agent_speaking'}
isStreaming={isStreaming}
isMuted={isMuted}
className='h-80 w-80 md:h-96 md:w-96'
/>
</div>
<div className='mb-16 flex h-24 items-center justify-center'>
{currentTranscript && (
<div className='max-w-2xl px-8'>
<p className='overflow-hidden text-center text-[var(--text-primary)] text-xl leading-relaxed'>
{currentTranscript}
</p>
</div>
)}
</div>
<p className='mb-8 text-center text-[var(--text-muted)] text-lg'>
{getStatusText()}
{isMuted && <span className='ml-2 text-[var(--text-muted)] text-sm'>(Muted)</span>}
</p>
</div>
<div className='px-8 pb-12'>
<div className='flex items-center justify-center gap-x-12'>
<Button onClick={handleCallEnd} variant='outline' className='size-14 rounded-full p-0'>
<Phone className='size-6 rotate-[135deg]' />
</Button>
<Button
onClick={toggleMute}
variant='outline'
disabled={!isInitialized}
className={cn(
'size-14 rounded-full p-0',
isMuted ? 'text-[var(--text-muted)]' : 'text-[var(--text-primary)]'
)}
>
{getButtonContent()}
</Button>
</div>
</div>
</div>
)
}