import asyncio import json import logging from collections.abc import AsyncIterator from contextlib import asynccontextmanager from dataclasses import dataclass from actions import ActionController, supports_actions from dotenv import find_dotenv, load_dotenv from hold_music import hold_beats from personas import Persona, compose_instructions, resolve_persona from livekit.agents import ( Agent, AgentServer, AgentSession, AudioConfig, BackgroundAudioPlayer, JobContext, TurnHandlingOptions, cli, inference, llm as agents_llm, ) from livekit.agents.llm import function_tool from livekit.plugins import lemonslice from livekit.rtc import RpcError, RpcInvocationData load_dotenv(find_dotenv(usecwd=False)) logger = logging.getLogger("avatar") server = AgentServer() @dataclass class State: persona: Persona avatar: lemonslice.AvatarSession session_id: str class MotionAgent(Agent): def __init__(self, persona: Persona, actions: ActionController) -> None: super().__init__( instructions=compose_instructions(persona), tts=inference.TTS("cartesia/sonic-3.5", voice=persona.voice_id), chat_ctx=agents_llm.ChatContext.empty(), ) self._actions = actions @function_tool async def wave(self) -> str: """Wave to the user. Only call when they explicitly ask you to wave.""" return await self._actions.play("wave") @function_tool async def dance(self) -> str: """Dance for the user. Only call when they explicitly ask you to dance.""" return await self._actions.play("dance") @function_tool async def turn(self) -> str: """Turn side to side. Only call when they explicitly ask you to turn.""" return await self._actions.play("turn") @server.rtc_session() async def entrypoint(ctx: JobContext) -> None: meta = json.loads(ctx.job.metadata) if ctx.job.metadata else {} initial = resolve_persona(meta.get("set_avatar")) logger.info("starting session with persona %s", initial.id) session = AgentSession( stt=inference.STT("deepgram/nova-3"), llm=inference.LLM("google/gemini-3.5-flash"), turn_handling=TurnHandlingOptions( interruption={"resume_false_interruption": False}, ), ) def make_avatar(p: Persona) -> lemonslice.AvatarSession: return lemonslice.AvatarSession( agent_image_url=p.image_url, agent_prompt=p.speaking_prompt, agent_idle_prompt=p.idle_prompt, idle_timeout=120, response_done_timeout=2, ) actions = ActionController() ctx.add_shutdown_callback(actions.shutdown) def make_agent(p: Persona) -> Agent: if supports_actions(p.id): return MotionAgent(p, actions) return Agent( instructions=compose_instructions(p), tts=inference.TTS("cartesia/sonic-3.5", voice=p.voice_id), chat_ctx=agents_llm.ChatContext.empty(), ) avatar = make_avatar(initial) session_id = await avatar.start(session, room=ctx.room) state = State(persona=initial, avatar=avatar, session_id=session_id) await state.avatar.wait_for_join() if supports_actions(initial.id): actions.set_session(state.session_id, initial.id) await session.start(agent=make_agent(initial), room=ctx.room) if supports_actions(initial.id): await actions.opening_wave() session.generate_reply( instructions=( f"It's your turn to speak first. Open with a single short greeting in " f"character as {initial.name} and then stop." + (" Do not call wave — you already waved." if supports_actions(initial.id) else "") ) ) bg_audio = BackgroundAudioPlayer() await bg_audio.start(room=ctx.room, agent_session=session) @asynccontextmanager async def hold_music() -> AsyncIterator[None]: handle = bg_audio.play(AudioConfig(source=hold_beats(), fade_in=1.0, fade_out=0.4)) try: yield finally: handle.stop() switch_lock = asyncio.Lock() @ctx.room.local_participant.register_rpc_method("set_avatar") async def set_avatar(data: RpcInvocationData) -> str: if switch_lock.locked(): raise RpcError( RpcError.ErrorCode.APPLICATION_ERROR, "Still switching to the previous persona, please try again in a moment.", ) new_persona = resolve_persona(json.loads(data.payload)["value"]) async with switch_lock: if new_persona.id == state.persona.id: return json.dumps({"id": state.persona.id}) logger.info("switching persona: %s -> %s", state.persona.id, new_persona.id) session.interrupt() async with hold_music(): await actions.cancel() await state.avatar.aclose() state.avatar = make_avatar(new_persona) state.session_id = await state.avatar.start(session, room=ctx.room) await state.avatar.wait_for_join() session.update_agent(make_agent(new_persona)) state.persona = new_persona # Lemonslice's video pipeline needs a beat after wait_for_join # before it actually consumes audio + emits frames. await asyncio.sleep(1.2) if supports_actions(new_persona.id): actions.set_session(state.session_id, new_persona.id) await actions.opening_wave() session.generate_reply( instructions=( f"It's your turn to speak first. Open with a single short line in " f"character as {state.persona.name} (acknowledge that you're who they " "just picked) and then stop." + ( " Do not call wave — you already waved." if supports_actions(new_persona.id) else "" ) ) ) return json.dumps({"id": state.persona.id}) if __name__ == "__main__": cli.run_app(server)