const puppeteer = require('puppeteer'); const { BROWSER_CONFIG } = require('./browser_config'); const AuthHelper = require('./auth_helper'); const { getPuppeteerLaunchOptions } = require('./puppeteer_config'); // Use consistent base URL for all requests const BASE_URL = 'http://localhost:5000'; // Colors for console output const colors = { reset: '\x1b[0m', bright: '\x1b[1m', red: '\x1b[31m', green: '\x1b[32m', yellow: '\x1b[33m', blue: '\x1b[34m', magenta: '\x1b[35m', cyan: '\x1b[36m' }; function log(message, color = colors.reset) { console.log(`${color}[${new Date().toISOString()}] ${message}${colors.reset}`); } function logError(message) { console.error(`${colors.red}[${new Date().toISOString()}] ERROR: ${message}${colors.reset}`); } function logSuccess(message) { log(`✓ ${message}`, colors.green); } function logInfo(message) { log(`ℹ ${message}`, colors.blue); } function logWarning(message) { log(`⚠ ${message}`, colors.yellow); } async function delay(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } async function takeScreenshot(page, name) { const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `/tmp/benchmark_test_${name}_${timestamp}.png`; await page.screenshot({ path: filename, fullPage: true }); logInfo(`Screenshot saved: ${filename}`); } async function waitForTextOnPage(page, text, timeout = 30000) { logInfo(`Waiting for text: "${text}"`); try { await page.waitForFunction( (searchText) => document.body.innerText.includes(searchText), { timeout }, text ); logSuccess(`Found text: "${text}"`); return true; } catch { logError(`Text not found within ${timeout}ms: "${text}"`); return false; } } async function typeWithDelay(page, selector, text, charDelay = 50) { await page.focus(selector); await page.evaluate(sel => { document.querySelector(sel).value = ''; }, selector); for (const char of text) { await page.type(selector, char); await new Promise(resolve => setTimeout(resolve, charDelay)); } } async function keepSessionAlive(page) { // Make a simple API call to keep the session active try { await page.evaluate(async () => { const response = await fetch('/api/settings', { credentials: 'same-origin' }); console.log('Keep-alive response:', response.status); }); } catch { // Ignore errors } } (async () => { let browser; try { log('=== Starting Benchmark Settings Puppeteer Test ===', colors.bright); browser = await puppeteer.launch(getPuppeteerLaunchOptions()); logSuccess('Browser launched'); const page = await browser.newPage(); // Set viewport to ensure consistent rendering await page.setViewport({ width: 1280, height: 800 }); // Log browser console errors page.on('console', msg => { if (msg.type() === 'error') { logError(`Browser console: ${msg.text()}`); } }); // Log network errors page.on('requestfailed', request => { if (!request.url().includes('favicon')) { logError(`Request failed: ${request.url()} - ${request.failure().errorText}`); } }); // Register and login using AuthHelper with consistent base URL const auth = new AuthHelper(page, BASE_URL); const username = 'benchmark_' + Date.now(); const password = 'T3st!Secure#2024$LDR'; logInfo(`Creating user: ${username}`); await auth.register(username, password); // Verify we're logged in const isLoggedIn = await auth.isLoggedIn(); if (!isLoggedIn) { throw new Error('Failed to authenticate user'); } logSuccess('User authenticated successfully'); await takeScreenshot(page, 'after_auth'); // Keep session alive await keepSessionAlive(page); // Configure Ollama provider and model in settings logInfo('Configuring Ollama provider and gemma3:12b model for benchmarks...'); // Keep session alive before navigation await keepSessionAlive(page); // Save cookies before navigation const cookies = await page.cookies(); logInfo(`Saved ${cookies.length} cookies from session`); // Navigate to settings - use consistent base URL await page.goto(`${BASE_URL}/settings`, { waitUntil: 'domcontentloaded' }); await delay(2000); // Check if we're still logged in (not redirected to login) const currentUrl = page.url(); if (currentUrl.includes('/auth/login')) { logError('Session lost - redirected to login page'); await takeScreenshot(page, 'session_lost_settings'); // Restore cookies and try again await page.setCookie(...cookies); await page.goto(`${BASE_URL}/settings`, { waitUntil: 'domcontentloaded' }); await delay(1000); // If still redirected, login again if (page.url().includes('/auth/login')) { await auth.login(username, password); await page.goto(`${BASE_URL}/settings`, { waitUntil: 'domcontentloaded' }); await delay(2000); } } // Set provider to Ollama logInfo('Setting provider to Ollama...'); const providerSet = await page.evaluate(() => { // Try multiple selectors for the provider field const providerSelectors = [ 'select[name="llm.provider"]', '#llm\\.provider', 'select[id*="provider"]', 'select[data-setting="llm.provider"]', '#provider', '.provider-select', 'select.ldr-form-control' // Look for any select with // ldr-form-control class ]; let providerSelect = null; for (const selector of providerSelectors) { try { const element = document.querySelector(selector); if (element && element.tagName === 'SELECT') { // Check if this is likely the provider select by looking at its options const options = Array.from(element.options).map(opt => opt.value.toLowerCase()); if (options.some(opt => opt.includes('ollama') || opt.includes('openai') || opt.includes('anthropic'))) { providerSelect = element; console.log(`Found provider select with selector: ${selector}`); break; } } } catch { // Some selectors might throw errors, continue } } if (providerSelect) { // Find the Ollama option const ollamaOption = Array.from(providerSelect.options).find( opt => opt.value.toLowerCase() === 'ollama' || opt.text.toLowerCase().includes('ollama') ); if (ollamaOption) { providerSelect.value = ollamaOption.value; providerSelect.dispatchEvent(new Event('change', { bubbles: true })); console.log(`Set provider to Ollama (value: ${ollamaOption.value})`); return true; } console.log('Could not find Ollama option in provider select'); console.log('Available options:', Array.from(providerSelect.options).map(opt => opt.value)); } console.log('Could not find provider field'); return false; }); if (providerSet) { logSuccess('Provider set to Ollama'); await delay(1000); // Wait for model list to update // Set model to gemma3:12b logInfo('Setting model to gemma3:12b...'); const modelSet = await page.evaluate(() => { // Try multiple selectors for the model field const modelSelectors = [ 'select[name="llm.model"]', '#llm\\.model', 'select[id*="model"]', 'select[data-setting="llm.model"]', '#model', '.model-select' ]; let modelSelect = null; for (const selector of modelSelectors) { try { const element = document.querySelector(selector); if (element && element.tagName === 'SELECT') { // Check if this is likely the model select const options = element.options ? Array.from(element.options).map(opt => opt.value.toLowerCase()) : []; if (options.some(opt => opt.includes('gemma') || opt.includes('gpt') || opt.includes('claude'))) { modelSelect = element; console.log(`Found model select with selector: ${selector}`); break; } } } catch { // Some selectors might throw errors, continue } } if (modelSelect) { // Look for gemma3:12b option const options = modelSelect.options ? Array.from(modelSelect.options) : []; const gemmaOption = options.find(opt => opt.value === 'gemma3:12b' || opt.value.toLowerCase() === 'gemma3:12b' || opt.text.includes('gemma3:12b') || opt.text.toLowerCase().includes('gemma3:12b') ); if (gemmaOption) { modelSelect.value = gemmaOption.value; modelSelect.dispatchEvent(new Event('change', { bubbles: true })); console.log(`Set model to gemma3:12b (value: ${gemmaOption.value})`); return true; } console.log('Could not find gemma3:12b option'); console.log('Available model options:', options.map(opt => opt.value).slice(0, 10)); // Try to find any Ollama model as fallback const ollamaOption = options.find(opt => opt.value.toLowerCase().includes('gemma') || opt.text.toLowerCase().includes('gemma') ); if (ollamaOption) { modelSelect.value = ollamaOption.value; modelSelect.dispatchEvent(new Event('change', { bubbles: true })); console.log(`Set model to fallback Ollama model: ${ollamaOption.value}`); return true; } } console.log('Could not find model field'); return false; }); if (modelSet) { logSuccess('Model set to gemma3:12b'); } else { logWarning('Could not set model to gemma3:12b'); } // Set minimal iterations for faster testing logInfo('Setting minimal iterations for faster testing...'); await page.evaluate(() => { const iterationsInput = document.querySelector('input[name="search.iterations"]') || document.querySelector('#search\\.iterations'); if (iterationsInput) { iterationsInput.value = '1'; iterationsInput.dispatchEvent(new Event('change', { bubbles: true })); console.log('Set iterations to 1'); } const questionsInput = document.querySelector('input[name="search.questions_per_iteration"]') || document.querySelector('#search\\.questions_per_iteration'); if (questionsInput) { questionsInput.value = '1'; questionsInput.dispatchEvent(new Event('change', { bubbles: true })); console.log('Set questions per iteration to 1'); } }); // Try to save settings const saved = await page.evaluate(() => { // Look for save button const saveButtons = Array.from(document.querySelectorAll('button')).filter( btn => btn.textContent.includes('Save') || btn.textContent.includes('Update') ); if (saveButtons.length > 0) { saveButtons[0].click(); console.log('Clicked save button'); return true; } // Try submitting the form const forms = document.querySelectorAll('form'); if (forms.length > 0) { forms[0].submit(); console.log('Submitted form'); return true; } return false; }); if (saved) { await delay(2000); logSuccess('Settings saved'); } else { logWarning('Could not save settings'); } } else { logWarning('Could not set provider - continuing anyway'); } await takeScreenshot(page, 'after_settings'); // Keep session alive before navigation await keepSessionAlive(page); // Navigate to benchmark page logInfo('Navigating to benchmark page...'); // Save current cookies again const settingsCookies = await page.cookies(); logInfo(`Saved ${settingsCookies.length} cookies before benchmark navigation`); await page.goto(`${BASE_URL}/benchmark`, { waitUntil: 'domcontentloaded' }); await delay(2000); // Check if we were redirected to login const benchmarkUrl = page.url(); if (benchmarkUrl.includes('/auth/login')) { logError('Session lost when navigating to benchmark - redirected to login page'); await takeScreenshot(page, 'session_lost_benchmark'); // Restore cookies and try again await page.setCookie(...settingsCookies); await page.goto(`${BASE_URL}/benchmark`, { waitUntil: 'domcontentloaded' }); await delay(1000); // If still redirected, login again if (page.url().includes('/auth/login')) { await auth.login(username, password); await page.goto(`${BASE_URL}/benchmark`, { waitUntil: 'domcontentloaded' }); await delay(2000); } } await takeScreenshot(page, 'benchmark_page'); // Check if benchmark page loaded const hasBenchmarkTitle = await waitForTextOnPage(page, 'Benchmark Dashboard', 5000); if (!hasBenchmarkTitle) { logError('Benchmark page did not load correctly'); await takeScreenshot(page, 'benchmark_error'); // Try alternative text to verify we're on the benchmark page const hasBenchmarkText = await page.evaluate(() => { const text = document.body.innerText.toLowerCase(); return text.includes('benchmark') || text.includes('simpleqa') || text.includes('browsecomp'); }); if (hasBenchmarkText) { logInfo('Found benchmark-related content on page'); } } // Look for benchmark interface elements logInfo('Analyzing benchmark page...'); try { // Log page content for debugging const pageTitle = await page.title(); logInfo(`Page title: ${pageTitle}`); // Check for dataset configuration section const hasDatasets = await page.evaluate(() => { const text = document.body.innerText; return text.includes('SimpleQA') || text.includes('BrowseComp') || text.includes('Dataset'); }); if (hasDatasets) { logSuccess('Found dataset configuration section'); // Try to set minimal dataset counts try { // Look for SimpleQA input fields const simpleQAInputs = await page.$$('input[type="number"]'); if (simpleQAInputs.length > 0) { logInfo(`Found ${simpleQAInputs.length} number input fields`); // Set first input (likely SimpleQA) to 2 await simpleQAInputs[0].click({ clickCount: 3 }); await simpleQAInputs[0].type('2'); logSuccess('Set first dataset count to 2'); // If there's a second input (BrowseComp), set it to 0 if (simpleQAInputs.length > 1) { await simpleQAInputs[1].click({ clickCount: 3 }); await simpleQAInputs[1].type('0'); logSuccess('Set second dataset count to 0'); } } } catch (configError) { logWarning(`Could not configure datasets: ${configError.message}`); } } // Look for any button that might start the benchmark const buttons = await page.$$('button'); let startButton = null; for (const button of buttons) { const buttonText = await page.evaluate(el => el.textContent || '', button); const buttonClass = await page.evaluate(el => el.className || '', button); logInfo(`Found button: "${buttonText.trim()}" (class: ${buttonClass})`); // Check for various possible button texts if (buttonText.match(/start|run|begin|execute|launch/i) && (buttonText.match(/benchmark/i) || buttonClass.includes('primary'))) { startButton = button; logSuccess(`Selected button: "${buttonText.trim()}"`); break; } } if (!startButton && buttons.length > 0) { // If no specific benchmark button found, try the first primary button for (const button of buttons) { const buttonClass = await page.evaluate(el => el.className || '', button); if (buttonClass.includes('primary') || buttonClass.includes('success')) { startButton = button; const buttonText = await page.evaluate(el => el.textContent || '', button); logWarning(`Using primary button as fallback: "${buttonText.trim()}"`); break; } } } if (startButton) { logInfo('Clicking start button...'); // Keep session alive before starting benchmark await keepSessionAlive(page); await startButton.click(); await delay(2000); // Check for any alerts or errors await page.on('dialog', async dialog => { logInfo(`Dialog appeared: ${dialog.message()}`); await dialog.accept(); }); await takeScreenshot(page, 'after_start_click'); // Wait and check for progress logInfo('Waiting for benchmark to start...'); await delay(5000); // Check if benchmark is running const isRunning = await page.evaluate(() => { const text = document.body.innerText; return text.includes('progress') || text.includes('running') || text.includes('Processing') || text.includes('%'); }); if (isRunning) { logSuccess('Benchmark appears to be running!'); await takeScreenshot(page, 'benchmark_running'); } else { logWarning('Could not confirm benchmark is running'); // Check for error messages const hasError = await page.evaluate(() => { const text = document.body.innerText; return text.includes('error') || text.includes('Error') || text.includes('failed') || text.includes('Failed'); }); if (hasError) { logError('Page contains error messages'); // Try to extract error message const errorText = await page.evaluate(() => { const errorElements = Array.from(document.querySelectorAll('*')).filter(el => { const text = el.textContent || ''; return (text.includes('error') || text.includes('Error')) && el.children.length === 0; // Leaf nodes only }); return errorElements.map(el => el.textContent.trim()).join(' | '); }); logError(`Error details: ${errorText}`); } } await takeScreenshot(page, 'benchmark_final_state'); } else { logError('Could not find any button to start benchmark'); await takeScreenshot(page, 'no_start_button'); } } catch (error) { logError(`Error during benchmark test: ${error.message}`); await takeScreenshot(page, 'benchmark_error'); } // Log page title and URL for debugging const title = await page.title(); const url = page.url(); logInfo(`Final page - Title: ${title}, URL: ${url}`); logSuccess('Benchmark test completed'); } catch (error) { logError(`Test failed: ${error.message}`); console.error(error.stack); process.exit(1); } finally { if (browser) { await browser.close(); logSuccess('Browser closed'); } } })();