wehub-resource-sync
303 lines
10 KiB
HTML
303 lines
10 KiB
HTML
<!--
|
|
Copyright 2025 Google LLC
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
-->
|
|
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Project Livewire</title>
|
|
<link rel="stylesheet" href="styles/style.css">
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200" />
|
|
|
|
<!-- Favicon -->
|
|
<link rel="icon" type="image/x-icon" href="assets/favicon.ico">
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<h1>Project Livewire</h1>
|
|
<p>Speak into your microphone and optionally share your webcam or screen to engage in a multimedia conversation.</p>
|
|
</div>
|
|
|
|
<div class="input-container">
|
|
<button id="micButton" disabled class="action-button">
|
|
<span class="material-symbols-outlined">mic</span>
|
|
</button>
|
|
<button id="webcamButton" class="action-button">
|
|
<span class="material-symbols-outlined">videocam</span>
|
|
</button>
|
|
<button id="screenButton" class="action-button">
|
|
<span class="material-symbols-outlined">present_to_all</span>
|
|
</button>
|
|
<div class="text-input-container">
|
|
<input type="text" id="textInput" placeholder="Type your message..." class="text-input">
|
|
<button id="sendButton" class="action-button">
|
|
<span class="material-symbols-outlined">send</span>
|
|
</button>
|
|
<button id="interruptButton" class="action-button" style="display: none;">
|
|
<span class="material-symbols-outlined">cancel</span>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="video-container">
|
|
<video id="videoPreview" autoplay playsinline class="hidden"></video>
|
|
</div>
|
|
|
|
<div id="output"></div>
|
|
|
|
<!-- Load EventEmitter3 first -->
|
|
<script src="https://cdn.jsdelivr.net/npm/eventemitter3@5.0.1/dist/eventemitter3.umd.min.js"></script>
|
|
|
|
<script type="module">
|
|
import { AudioRecorder } from './src/audio/audio-recorder.js';
|
|
import { AudioStreamer } from './src/audio/audio-streamer.js';
|
|
import { MediaHandler } from './src/media/media-handler.js';
|
|
import { GeminiAPI } from './src/api/gemini-api.js';
|
|
import { base64ToArrayBuffer } from './src/utils/utils.js';
|
|
|
|
// Initialize components
|
|
const output = document.getElementById('output');
|
|
const audioRecorder = new AudioRecorder();
|
|
const audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 24000 });
|
|
const audioStreamer = new AudioStreamer(audioContext);
|
|
const mediaHandler = new MediaHandler();
|
|
const wsEndpoint = 'ws://localhost:8081';
|
|
const api = new GeminiAPI(wsEndpoint);
|
|
|
|
let isRecording = false;
|
|
let hasShownSpeakingMessage = false;
|
|
let currentTurn = 0;
|
|
let lastAudioTurn = -1;
|
|
|
|
// Initialize media handler
|
|
mediaHandler.initialize(document.getElementById('videoPreview'));
|
|
|
|
// Set up API handlers
|
|
api.onReady = () => {
|
|
document.getElementById('micButton').disabled = false;
|
|
};
|
|
|
|
api.onAudioData = async (audioData) => {
|
|
try {
|
|
if (!api.isSpeaking || lastAudioTurn !== currentTurn) {
|
|
logMessage('Gemini: Speaking...');
|
|
api.isSpeaking = true;
|
|
lastAudioTurn = currentTurn;
|
|
document.getElementById('interruptButton').style.display = 'inline-block';
|
|
}
|
|
const arrayBuffer = base64ToArrayBuffer(audioData);
|
|
audioStreamer.addPCM16(new Uint8Array(arrayBuffer));
|
|
audioStreamer.resume();
|
|
} catch (error) {
|
|
console.error('Error playing audio:', error);
|
|
}
|
|
};
|
|
|
|
api.onTextContent = (text) => {
|
|
if (text.trim()) {
|
|
logMessage('Gemini: ' + text);
|
|
}
|
|
};
|
|
|
|
api.onTurnComplete = () => {
|
|
logMessage('Gemini: Finished speaking');
|
|
api.isSpeaking = false; // Reset speaking state
|
|
audioStreamer.complete();
|
|
document.getElementById('interruptButton').style.display = 'none';
|
|
};
|
|
|
|
// Add interruption handler
|
|
api.onInterrupted = (data) => {
|
|
logMessage('Gemini: Response interrupted');
|
|
api.isSpeaking = false;
|
|
audioStreamer.stop(); // Stop current playback and clear queue
|
|
document.getElementById('interruptButton').style.display = 'none';
|
|
|
|
// Show visual feedback for interruption
|
|
const messageElement = document.createElement('p');
|
|
messageElement.className = 'interrupted-message';
|
|
messageElement.textContent = 'Response interrupted by user input';
|
|
output.appendChild(messageElement);
|
|
output.scrollTop = output.scrollHeight;
|
|
};
|
|
|
|
// Add function call and response handlers
|
|
api.onFunctionCall = (data) => {
|
|
logMessage('Function: ' + data.name);
|
|
logMessage('Parameters: ' + JSON.stringify(data.args, null, 2));
|
|
};
|
|
|
|
api.onFunctionResponse = (data) => {
|
|
logMessage('API Response: ' + JSON.stringify(data, null, 2));
|
|
};
|
|
|
|
// UI Event Handlers
|
|
async function startRecording() {
|
|
try {
|
|
// If model is speaking, treat this as an interruption
|
|
if (api.isSpeaking) {
|
|
audioStreamer.stop();
|
|
api.isSpeaking = false;
|
|
}
|
|
|
|
await audioContext.resume();
|
|
await audioRecorder.start();
|
|
hasShownSpeakingMessage = false;
|
|
currentTurn++;
|
|
|
|
audioRecorder.on('data', (base64Data) => {
|
|
if (!hasShownSpeakingMessage) {
|
|
logMessage('You: Speaking...');
|
|
hasShownSpeakingMessage = true;
|
|
}
|
|
api.sendAudioChunk(base64Data);
|
|
});
|
|
|
|
isRecording = true;
|
|
document.getElementById('micButton').innerHTML =
|
|
'<span class="material-symbols-outlined">stop</span>';
|
|
} catch (error) {
|
|
console.error('Error starting recording:', error);
|
|
logMessage('Error: ' + error.message);
|
|
}
|
|
}
|
|
|
|
function stopRecording() {
|
|
audioRecorder.stop();
|
|
isRecording = false;
|
|
hasShownSpeakingMessage = false;
|
|
document.getElementById('micButton').innerHTML =
|
|
'<span class="material-symbols-outlined">mic</span>';
|
|
logMessage('You: Recording stopped.');
|
|
|
|
// Stop video streams
|
|
mediaHandler.stopAll();
|
|
document.getElementById('webcamButton').innerHTML =
|
|
'<span class="material-symbols-outlined">videocam</span>';
|
|
document.getElementById('screenButton').innerHTML =
|
|
'<span class="material-symbols-outlined">present_to_all</span>';
|
|
|
|
api.sendEndMessage();
|
|
api.isSpeaking = false;
|
|
}
|
|
|
|
function logMessage(message) {
|
|
const messageElement = document.createElement('p');
|
|
|
|
// Add specific styling based on message content
|
|
if (message.startsWith('Function:')) {
|
|
messageElement.className = 'function-name';
|
|
} else if (message.startsWith('Parameters:')) {
|
|
messageElement.className = 'function-params';
|
|
} else if (message.startsWith('API Response:')) {
|
|
messageElement.className = 'api-response';
|
|
} else if (message.startsWith('Gemini:')) {
|
|
messageElement.className = 'gemini-message';
|
|
} else if (message.startsWith('You:')) {
|
|
messageElement.className = 'user-message';
|
|
}
|
|
|
|
messageElement.textContent = message;
|
|
output.appendChild(messageElement);
|
|
output.scrollTop = output.scrollHeight;
|
|
}
|
|
|
|
// Add function to send text message
|
|
function sendTextMessage() {
|
|
const textInput = document.getElementById('textInput');
|
|
const text = textInput.value.trim();
|
|
if (!text) return;
|
|
|
|
// Clear input
|
|
textInput.value = '';
|
|
|
|
// Log user message
|
|
logMessage('You: ' + text);
|
|
|
|
// Send text message
|
|
api.sendTextMessage(text);
|
|
}
|
|
|
|
// Set up button click handlers
|
|
document.getElementById('micButton').onclick = () => {
|
|
if (isRecording) {
|
|
stopRecording();
|
|
} else {
|
|
startRecording();
|
|
}
|
|
};
|
|
|
|
// Add send button handler
|
|
document.getElementById('sendButton').onclick = sendTextMessage;
|
|
|
|
// Add keypress handler for text input
|
|
document.getElementById('textInput').addEventListener('keypress', function(e) {
|
|
if (e.key === 'Enter') {
|
|
sendTextMessage();
|
|
}
|
|
});
|
|
|
|
document.getElementById('webcamButton').onclick = async () => {
|
|
if (mediaHandler.isWebcamActive) {
|
|
mediaHandler.stopAll();
|
|
document.getElementById('webcamButton').innerHTML =
|
|
'<span class="material-symbols-outlined">videocam</span>';
|
|
} else {
|
|
const success = await mediaHandler.startWebcam();
|
|
if (success) {
|
|
document.getElementById('webcamButton').innerHTML =
|
|
'<span class="material-symbols-outlined">videocam_off</span>';
|
|
mediaHandler.startFrameCapture((base64Image) => {
|
|
api.sendImage(base64Image);
|
|
});
|
|
}
|
|
}
|
|
};
|
|
|
|
document.getElementById('screenButton').onclick = async () => {
|
|
if (mediaHandler.isScreenActive) {
|
|
mediaHandler.stopAll();
|
|
document.getElementById('screenButton').innerHTML =
|
|
'<span class="material-symbols-outlined">present_to_all</span>';
|
|
} else {
|
|
const success = await mediaHandler.startScreenShare();
|
|
if (success) {
|
|
document.getElementById('screenButton').innerHTML =
|
|
'<span class="material-symbols-outlined">cancel_presentation</span>';
|
|
mediaHandler.startFrameCapture((base64Image) => {
|
|
api.sendImage(base64Image);
|
|
});
|
|
}
|
|
}
|
|
};
|
|
|
|
// Add CSS for interrupted message
|
|
const style = document.createElement('style');
|
|
style.textContent = `
|
|
.interrupted-message {
|
|
color: #ff6b6b;
|
|
font-style: italic;
|
|
margin: 4px 0;
|
|
padding: 4px 8px;
|
|
border-left: 3px solid #ff6b6b;
|
|
background-color: rgba(255, 107, 107, 0.1);
|
|
}
|
|
`;
|
|
document.head.appendChild(style);
|
|
</script>
|
|
</body>
|
|
</html> |