wehub-resource-sync
417 lines
15 KiB
HTML
417 lines
15 KiB
HTML
<!--
|
|
Copyright 2025 Google LLC
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
-->
|
|
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>Project Livewire Mobile</title>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, viewport-fit=cover">
|
|
<meta name="theme-color" content="#000000">
|
|
<link rel="stylesheet" href="styles/mobile-style.css">
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200" />
|
|
<link rel="icon" type="image/x-icon" href="assets/favicon.ico">
|
|
</head>
|
|
<body>
|
|
<div class="header-section">
|
|
<h1>Project Livewire 🍝</h1>
|
|
</div>
|
|
|
|
<div id="functionInfo" class="function-info">
|
|
Waiting for function calls...
|
|
</div>
|
|
|
|
<div class="video-container">
|
|
<video id="videoPreview" autoplay playsinline class="hidden"></video>
|
|
</div>
|
|
|
|
<div id="output" class="chat-output"></div>
|
|
|
|
<div class="controls">
|
|
<div id="playButtonContainer" class="centered-button-container">
|
|
<button id="connectButton" class="action-button">
|
|
<span class="material-symbols-outlined">play_arrow</span>
|
|
</button>
|
|
</div>
|
|
<div id="mediaButtonsContainer" class="media-buttons-container hidden">
|
|
<button id="stopButton" class="action-button">
|
|
<span class="material-symbols-outlined">stop</span>
|
|
</button>
|
|
<button id="micButton" class="action-button" disabled>
|
|
<span class="material-symbols-outlined">mic</span>
|
|
</button>
|
|
<button id="webcamButton" class="action-button" disabled>
|
|
<span class="material-symbols-outlined">videocam</span>
|
|
</button>
|
|
<button id="switchCameraButton" class="action-button hidden">
|
|
<span class="material-symbols-outlined">flip_camera_ios</span>
|
|
</button>
|
|
<button id="screenButton" class="action-button hidden" disabled>
|
|
<span class="material-symbols-outlined">present_to_all</span>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Load EventEmitter3 first -->
|
|
<script src="https://cdn.jsdelivr.net/npm/eventemitter3@5.0.1/dist/eventemitter3.umd.min.js"></script>
|
|
|
|
<script type="module">
|
|
import { AudioRecorder } from './src/audio/audio-recorder.js';
|
|
import { AudioStreamer } from './src/audio/audio-streamer.js';
|
|
import { MediaHandler } from './src/media/media-handler.js';
|
|
import { GeminiAPI } from './src/api/gemini-api.js';
|
|
import { base64ToArrayBuffer } from './src/utils/utils.js';
|
|
|
|
// Check if device is mobile first
|
|
const urlParams = new URLSearchParams(window.location.search);
|
|
const forceMobile = urlParams.get('mobile') === 'true';
|
|
const isMobileDevice = forceMobile || /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent);
|
|
|
|
// Initialize components
|
|
const output = document.getElementById('output');
|
|
const audioRecorder = new AudioRecorder();
|
|
const audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 24000 });
|
|
const audioStreamer = new AudioStreamer(audioContext);
|
|
const mediaHandler = new MediaHandler();
|
|
const wsEndpoint = 'ws://localhost:8081';
|
|
const api = new GeminiAPI(wsEndpoint);
|
|
|
|
let isStarted = false;
|
|
let isRecording = false;
|
|
let isMuted = false;
|
|
let currentTurn = 0;
|
|
let lastAudioTurn = -1;
|
|
|
|
// Function to enable media controls
|
|
function enableMediaControls() {
|
|
document.getElementById('micButton').disabled = false;
|
|
document.getElementById('webcamButton').disabled = false;
|
|
if (!isMobileDevice) {
|
|
document.getElementById('screenButton').disabled = false;
|
|
}
|
|
}
|
|
|
|
// Recording functions
|
|
async function startRecording() {
|
|
try {
|
|
// If model is speaking, treat this as an interruption
|
|
if (api.isSpeaking) {
|
|
audioStreamer.stop(); // Stop current playback and clear queue
|
|
api.isSpeaking = false;
|
|
}
|
|
|
|
await audioContext.resume();
|
|
await audioRecorder.start();
|
|
currentTurn++;
|
|
|
|
audioRecorder.on('data', (base64Data) => {
|
|
api.sendAudioChunk(base64Data);
|
|
});
|
|
|
|
isRecording = true;
|
|
isMuted = false;
|
|
const micButton = document.getElementById('micButton');
|
|
micButton.classList.add('active');
|
|
} catch (error) {
|
|
console.error('Error starting recording:', error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function stopRecording() {
|
|
audioRecorder.stop();
|
|
isRecording = false;
|
|
isMuted = false;
|
|
|
|
if (isStarted) {
|
|
api.sendEndMessage();
|
|
}
|
|
api.isSpeaking = false;
|
|
}
|
|
|
|
// Initialize media handler
|
|
mediaHandler.initialize(document.getElementById('videoPreview'));
|
|
|
|
// Show screen share button on non-mobile devices
|
|
if (!isMobileDevice) {
|
|
document.getElementById('screenButton').classList.remove('hidden');
|
|
}
|
|
|
|
// Connect button handler
|
|
document.getElementById('connectButton').onclick = async () => {
|
|
try {
|
|
document.getElementById('connectButton').disabled = true;
|
|
await audioContext.resume();
|
|
|
|
// Try to ensure WebSocket connection
|
|
try {
|
|
await api.ensureConnected();
|
|
} catch (error) {
|
|
throw new Error('Unable to connect to server');
|
|
}
|
|
|
|
isStarted = true;
|
|
document.getElementById('playButtonContainer').classList.add('hidden');
|
|
document.getElementById('mediaButtonsContainer').classList.remove('hidden');
|
|
enableMediaControls();
|
|
|
|
// Start recording immediately
|
|
await startRecording();
|
|
} catch (error) {
|
|
console.error('Error starting session:', error);
|
|
// Show connection error
|
|
const existingError = document.querySelector('.connection-error');
|
|
if (!existingError) {
|
|
const errorElement = document.createElement('div');
|
|
errorElement.className = 'connection-error';
|
|
errorElement.textContent = 'Cannot connect to server. Please check if the server is running.';
|
|
document.body.appendChild(errorElement);
|
|
}
|
|
resetUIState();
|
|
}
|
|
};
|
|
|
|
// Set up API handlers
|
|
api.onReady = () => {
|
|
document.getElementById('connectButton').disabled = false;
|
|
// Remove any existing connection error message
|
|
const existingError = document.querySelector('.connection-error');
|
|
if (existingError) {
|
|
existingError.remove();
|
|
}
|
|
};
|
|
|
|
api.onError = (error) => {
|
|
console.error('WebSocket error:', error);
|
|
if (error.error_type === 'quota_exceeded') {
|
|
// Show quota error in chat with special styling
|
|
const messageElement = document.createElement('p');
|
|
messageElement.className = 'error-message';
|
|
messageElement.textContent = '⚠️ ' + error.message + ' ' + error.action;
|
|
output.appendChild(messageElement);
|
|
output.scrollTop = output.scrollHeight;
|
|
} else if (error.error_type === 'websocket_error' || error.error_type === 'connection_error') {
|
|
// Show connection error at the top
|
|
const existingError = document.querySelector('.connection-error');
|
|
if (!existingError) {
|
|
const errorElement = document.createElement('div');
|
|
errorElement.className = 'connection-error';
|
|
errorElement.textContent = 'Cannot connect to server. Please check if the server is running.';
|
|
document.body.appendChild(errorElement);
|
|
}
|
|
}
|
|
resetUIState();
|
|
if (isRecording) {
|
|
stopRecording();
|
|
}
|
|
mediaHandler.stopAll();
|
|
};
|
|
|
|
api.onClose = () => {
|
|
if (isStarted) {
|
|
resetUIState();
|
|
if (isRecording) {
|
|
stopRecording();
|
|
}
|
|
mediaHandler.stopAll();
|
|
}
|
|
};
|
|
|
|
api.onAudioData = async (audioData) => {
|
|
try {
|
|
if (!api.isSpeaking || lastAudioTurn !== currentTurn) {
|
|
api.isSpeaking = true;
|
|
lastAudioTurn = currentTurn;
|
|
}
|
|
const arrayBuffer = base64ToArrayBuffer(audioData);
|
|
audioStreamer.addPCM16(new Uint8Array(arrayBuffer));
|
|
audioStreamer.resume();
|
|
} catch (error) {
|
|
console.error('Error playing audio:', error);
|
|
}
|
|
};
|
|
|
|
api.onTurnComplete = () => {
|
|
api.isSpeaking = false;
|
|
audioStreamer.complete();
|
|
};
|
|
|
|
// Add function call and response handlers
|
|
api.onFunctionCall = (data) => {
|
|
const functionInfo = document.getElementById('functionInfo');
|
|
// Use <pre> to preserve formatting and spans for coloring
|
|
functionInfo.innerHTML = `<pre><span class="function-name-display">Function: ${data.name}</span>\n<span class="function-params-display">Parameters: ${JSON.stringify(data.args, null, 2)}</span></pre>`;
|
|
};
|
|
|
|
api.onFunctionResponse = (data) => {
|
|
const functionInfo = document.getElementById('functionInfo');
|
|
// Stringify the response data
|
|
let responseString = JSON.stringify(data, null, 2);
|
|
|
|
// Use regex to find the line with "title": and wrap it in <strong> tags
|
|
// This assumes the title value is a simple string.
|
|
const titleRegex = /(^\s*"title":\s*".*?"(,?)\s*$)/gm;
|
|
responseString = responseString.replace(titleRegex, '<strong>$1</strong>');
|
|
|
|
// Append response with its own color, preserving previous content and adding pre tag
|
|
// Ensure the modified string is treated as HTML
|
|
const responseElement = document.createElement('pre');
|
|
responseElement.innerHTML = `<span class="function-response-display">Response:\n${responseString}</span>`;
|
|
functionInfo.appendChild(responseElement);
|
|
};
|
|
|
|
// Button handlers
|
|
document.getElementById('micButton').onclick = () => {
|
|
isMuted = !isMuted;
|
|
const micButton = document.getElementById('micButton');
|
|
if (isMuted) {
|
|
micButton.classList.remove('active');
|
|
audioRecorder.mute();
|
|
audioRecorder.off('data');
|
|
} else {
|
|
micButton.classList.add('active');
|
|
audioRecorder.unmute();
|
|
audioRecorder.on('data', (base64Data) => {
|
|
api.sendAudioChunk(base64Data);
|
|
});
|
|
}
|
|
};
|
|
|
|
document.getElementById('stopButton').onclick = async () => {
|
|
stopRecording();
|
|
mediaHandler.stopAll();
|
|
audioStreamer.stop(); // Stop current playback and clear queue like interruption
|
|
api.isSpeaking = false;
|
|
|
|
// Reset to initial state - only show play button
|
|
isStarted = false;
|
|
document.getElementById('connectButton').disabled = false;
|
|
document.getElementById('mediaButtonsContainer').classList.add('hidden');
|
|
document.getElementById('playButtonContainer').classList.remove('hidden');
|
|
|
|
api.sendEndMessage();
|
|
};
|
|
|
|
document.getElementById('webcamButton').onclick = async () => {
|
|
if (mediaHandler.isWebcamActive) {
|
|
mediaHandler.stopAll();
|
|
document.getElementById('webcamButton').classList.remove('active');
|
|
document.getElementById('switchCameraButton').classList.add('hidden');
|
|
} else {
|
|
// First stop screen sharing if it's active
|
|
if (mediaHandler.isScreenActive) {
|
|
mediaHandler.stopAll();
|
|
document.getElementById('screenButton').classList.remove('active');
|
|
}
|
|
|
|
const success = await mediaHandler.startWebcam();
|
|
if (success) {
|
|
document.getElementById('webcamButton').classList.add('active');
|
|
if (isMobileDevice) {
|
|
document.getElementById('switchCameraButton').classList.remove('hidden');
|
|
}
|
|
mediaHandler.startFrameCapture((base64Image) => {
|
|
api.sendImage(base64Image);
|
|
});
|
|
}
|
|
}
|
|
};
|
|
|
|
document.getElementById('switchCameraButton').onclick = async () => {
|
|
await mediaHandler.switchCamera();
|
|
};
|
|
|
|
document.getElementById('screenButton').onclick = async () => {
|
|
if (mediaHandler.isScreenActive) {
|
|
mediaHandler.stopAll();
|
|
document.getElementById('screenButton').classList.remove('active');
|
|
} else {
|
|
// First stop webcam if it's active
|
|
if (mediaHandler.isWebcamActive) {
|
|
mediaHandler.stopAll();
|
|
document.getElementById('webcamButton').classList.remove('active');
|
|
document.getElementById('switchCameraButton').classList.add('hidden');
|
|
}
|
|
|
|
const success = await mediaHandler.startScreenShare();
|
|
if (success) {
|
|
document.getElementById('screenButton').classList.add('active');
|
|
mediaHandler.startFrameCapture((base64Image) => {
|
|
api.sendImage(base64Image);
|
|
});
|
|
}
|
|
}
|
|
};
|
|
|
|
function logMessage(message) {
|
|
const messageElement = document.createElement('p');
|
|
|
|
// Add specific styling based on message content
|
|
if (message.startsWith('Function:')) {
|
|
messageElement.className = 'function-name';
|
|
} else if (message.startsWith('Parameters:')) {
|
|
messageElement.className = 'function-params';
|
|
} else if (message.startsWith('API Response:')) {
|
|
messageElement.className = 'api-response';
|
|
} else if (message.startsWith('Gemini:')) {
|
|
messageElement.className = 'gemini-message';
|
|
} else if (message.startsWith('You:')) {
|
|
messageElement.className = 'user-message';
|
|
}
|
|
|
|
messageElement.textContent = message;
|
|
output.appendChild(messageElement);
|
|
output.scrollTop = output.scrollHeight;
|
|
}
|
|
|
|
// Remove references to interruptButton and muteButton
|
|
function resetUIState() {
|
|
isStarted = false;
|
|
document.getElementById('connectButton').disabled = false;
|
|
|
|
document.getElementById('mediaButtonsContainer').classList.add('hidden');
|
|
document.getElementById('playButtonContainer').classList.remove('hidden');
|
|
|
|
// Reset all buttons
|
|
document.getElementById('micButton').disabled = true;
|
|
document.getElementById('micButton').classList.remove('active');
|
|
|
|
document.getElementById('webcamButton').disabled = true;
|
|
document.getElementById('webcamButton').classList.remove('active');
|
|
document.getElementById('webcamButton').innerHTML =
|
|
'<span class="material-symbols-outlined">videocam</span>';
|
|
|
|
document.getElementById('screenButton').disabled = true;
|
|
document.getElementById('screenButton').classList.remove('active');
|
|
document.getElementById('screenButton').innerHTML =
|
|
'<span class="material-symbols-outlined">present_to_all</span>';
|
|
|
|
document.getElementById('switchCameraButton').classList.add('hidden');
|
|
}
|
|
|
|
// Update the text content handler to show messages
|
|
api.onTextContent = (text) => {
|
|
if (text.trim()) {
|
|
logMessage('Gemini: ' + text);
|
|
}
|
|
};
|
|
|
|
api.onInterrupted = () => {
|
|
api.isSpeaking = false;
|
|
audioStreamer.stop(); // Stop current playback and clear queue
|
|
};
|
|
</script>
|
|
</body>
|
|
</html> |