const puppeteer = require('puppeteer'); const AuthHelper = require('./auth_helper'); const { getPuppeteerLaunchOptions } = require('./puppeteer_config'); // Color codes for terminal output const colors = { reset: '\x1b[0m', bright: '\x1b[1m', red: '\x1b[31m', green: '\x1b[32m', yellow: '\x1b[33m', blue: '\x1b[34m', magenta: '\x1b[35m' }; function log(message, type = 'info') { const timestamp = new Date().toISOString(); const prefix = { info: `${colors.blue}ℹ`, success: `${colors.green}✓`, error: `${colors.red}✗`, warn: `${colors.yellow}⚠`, test: `${colors.magenta}▶` }[type] || ''; console.log(`${prefix} [${timestamp}] ${message}${colors.reset}`); } // Test configuration const TEST_CONFIG = { baseUrl: 'http://localhost:5000', apiKeys: { initial: 'sk-test-initial-key-123456789', changed: 'sk-test-changed-key-987654321', final: 'sk-test-final-key-555555555' }, timeout: 30000, headless: process.env.SHOW_BROWSER !== '1', slowMo: process.env.SHOW_BROWSER === '1' ? 100 : 0 }; // Main test function async function runComprehensiveApiKeyTest() { let browser; let page; let authHelper; const timestamp = Date.now(); const testUsername = `api_key_test_${timestamp}`; const testPassword = 'TestP@ssw0rd123'; const testResults = { authentication: false, settingsPageAccess: false, apiKeyFieldFound: false, enterKeySave: false, apiSave: false, persistence: false, multipleChanges: false }; const apiCalls = []; try { log(`${colors.bright}=== Starting Comprehensive API Key Settings Test ===${colors.reset}`, 'info'); log(`Test user: ${testUsername}`, 'info'); // Launch browser browser = await puppeteer.launch(getPuppeteerLaunchOptions()); page = await browser.newPage(); await page.setViewport({ width: 1920, height: 1080 }); // Log browser console errors page.on('console', msg => { if (msg.type() === 'error') { log(`Browser console: ${msg.text()}`, 'error'); } }); // Set up request interception to monitor API calls await page.setRequestInterception(true); page.on('request', request => { const url = request.url(); if (url.includes('/api/settings') || url.includes('/auth/')) { apiCalls.push({ method: request.method(), url, timestamp: new Date().toISOString() }); } request.continue(); }); // Initialize auth helper authHelper = new AuthHelper(page, TEST_CONFIG.baseUrl); // Test 1: Authentication log('Test 1: User Registration and Authentication', 'test'); try { await authHelper.register(testUsername, testPassword); const isAuthenticated = await authHelper.isLoggedIn(); if (isAuthenticated) { testResults.authentication = true; log('Authentication successful', 'success'); } else { throw new Error('Failed to verify authentication'); } } catch (error) { log(`Authentication failed: ${error.message}`, 'error'); throw error; } // Test 2: Navigate to Settings Page log('Test 2: Accessing Settings Page', 'test'); try { await page.goto(`${TEST_CONFIG.baseUrl}/settings`, { waitUntil: 'domcontentloaded', timeout: TEST_CONFIG.timeout }); // Verify we're on the settings page const currentUrl = page.url(); if (!currentUrl.includes('/settings')) { throw new Error(`Not on settings page. Current URL: ${currentUrl}`); } // Wait for settings to load await page.waitForSelector('input, select, textarea', { timeout: 10000 }); testResults.settingsPageAccess = true; log('Settings page loaded successfully', 'success'); // Take screenshot await page.screenshot({ path: `/tmp/api_key_test_settings_${timestamp}.png`, fullPage: true }); log(`Screenshot saved: /tmp/api_key_test_settings_${timestamp}.png`, 'info'); } catch (error) { log(`Failed to access settings page: ${error.message}`, 'error'); throw error; } // Test 3: Find API Key Field log('Test 3: Locating OpenAI Endpoint API Key Field', 'test'); let apiKeyInput = null; const apiKeySelectors = [ 'input[name="llm.openai_endpoint.api_key"]', 'input[id="llm.openai_endpoint.api_key"]', 'input[data-setting="llm.openai_endpoint.api_key"]', '#llm\\.openai_endpoint\\.api_key', 'input[type="password"][name*="openai"][name*="api_key"]', 'input[type="text"][name*="openai"][name*="api_key"]' ]; // First, check if we need to navigate to a specific section const sections = await page.evaluate(() => { const links = Array.from(document.querySelectorAll('a[href*="settings"], .nav-link, [role="tab"]')); return links.map(link => ({ text: link.textContent.trim(), href: link.href, isActive: link.classList.contains('active') })); }); log(`Found ${sections.length} settings sections`, 'info'); // Look for LLM or API section const llmSection = sections.find(s => s.text.toLowerCase().includes('llm') || s.text.toLowerCase().includes('api') || s.text.toLowerCase().includes('model') ); if (llmSection && !llmSection.isActive) { log(`Clicking on section: ${llmSection.text}`, 'info'); await page.evaluate((href) => { const link = document.querySelector(`a[href="${href}"]`); if (link) link.click(); }, llmSection.href); await new Promise(resolve => setTimeout(resolve, 1000)); } // Try to find the API key field for (const selector of apiKeySelectors) { try { apiKeyInput = await page.$(selector); if (apiKeyInput) { const isVisible = await apiKeyInput.evaluate(el => { const rect = el.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; }); if (isVisible) { log(`Found API key field with selector: ${selector}`, 'success'); testResults.apiKeyFieldFound = true; break; } } } catch { // Continue with next selector } } if (!apiKeyInput || !testResults.apiKeyFieldFound) { // Try expanding collapsible sections log('API key field not immediately visible, looking for expandable sections', 'info'); const expandables = await page.$$('[data-toggle="collapse"], .accordion-button, details summary'); for (const expandable of expandables) { const text = await expandable.evaluate(el => el.textContent); if (text.toLowerCase().includes('llm') || text.toLowerCase().includes('openai')) { log(`Expanding section: ${text.trim()}`, 'info'); await expandable.click(); await new Promise(resolve => setTimeout(resolve, 500)); } } // Try finding the field again for (const selector of apiKeySelectors) { apiKeyInput = await page.$(selector); if (apiKeyInput) { const isVisible = await apiKeyInput.evaluate(el => el.offsetParent !== null); if (isVisible) { log(`Found API key field after expanding sections: ${selector}`, 'success'); testResults.apiKeyFieldFound = true; break; } } } } if (!testResults.apiKeyFieldFound) { throw new Error('Could not find OpenAI Endpoint API key field'); } // Test 4: Enter Key Save Functionality log('Test 4: Testing Enter Key Save (Individual Field)', 'test'); try { // Wait for event handlers to be attached log('Waiting for event handlers to be attached...', 'info'); await page.waitForFunction(() => { const input = document.querySelector('input[name="llm.openai_endpoint.api_key"]'); if (!input) return false; // Check if the input has data indicating handlers are attached // The settings.js adds event listeners dynamically return true; // For now, just wait a bit }, { timeout: 5000 }).catch(() => { log('Handler wait timed out', 'warn'); }); // Additional wait to ensure handlers are ready await new Promise(resolve => setTimeout(resolve, 2000)); // Focus the input first await apiKeyInput.focus(); // Clear the field completely await page.keyboard.down('Control'); await page.keyboard.press('a'); await page.keyboard.up('Control'); await page.keyboard.press('Backspace'); // Type the new value await apiKeyInput.type(TEST_CONFIG.apiKeys.initial); // Small delay to ensure typing is complete await new Promise(resolve => setTimeout(resolve, 100)); // Get initial value const valueBefore = await apiKeyInput.evaluate(el => el.value); log(`Typed value: ${valueBefore}`, 'info'); // Press Enter to save log('Pressing Enter to save...', 'info'); await page.keyboard.press('Enter'); // Wait for save operation to start and complete // First wait for the saving class to appear await page.waitForFunction( () => { const input = document.querySelector('input[name="llm.openai_endpoint.api_key"]'); const container = input?.closest('.form-group, .settings-item') || input; return container?.classList.contains('saving'); }, { timeout: 5000 } ).catch(() => log('No saving indicator detected', 'warn')); // Then wait for save-success class or toast message const successIndicators = await page.waitForFunction( () => { const indicators = []; // Check for save-success class on input const input = document.querySelector('input[name="llm.openai_endpoint.api_key"]'); if (input && input.classList.contains('save-success')) { indicators.push('Input has save-success class'); } // Check for toast notifications const toastMessage = document.querySelector('.toast-message, .alert-success'); if (toastMessage && toastMessage.textContent.toLowerCase().includes('saved')) { indicators.push(toastMessage.textContent.trim()); } // Check if ui.showMessage was called (look for any success toast) const toasts = document.querySelectorAll('[class*="toast"][class*="success"], .notification.success'); toasts.forEach(toast => { if (toast.offsetParent !== null) { // Is visible indicators.push('Success toast visible'); } }); return indicators.length > 0 ? indicators : null; }, { timeout: 5000 } ).then(result => result.jsonValue()) .catch(() => []); if (successIndicators.length > 0) { log(`Success indicators found: ${successIndicators.join(', ')}`, 'success'); testResults.enterKeySave = true; } else { log('No immediate success indicators found, checking persistence...', 'warn'); } // Verify by reloading the page log('Reloading page to check persistence...', 'info'); await page.reload({ waitUntil: 'domcontentloaded' }); // Find the field again apiKeyInput = null; for (const selector of apiKeySelectors) { apiKeyInput = await page.$(selector); if (apiKeyInput) break; } if (apiKeyInput) { const valueAfterReload = await apiKeyInput.evaluate(el => el.value); if (valueAfterReload && valueAfterReload.includes(TEST_CONFIG.apiKeys.initial.slice(-5))) { log('Value persisted after reload!', 'success'); testResults.enterKeySave = true; testResults.persistence = true; } else { log(`Value after reload: ${valueAfterReload || 'empty'}`, 'warn'); } } } catch (error) { log(`Enter key save test failed: ${error.message}`, 'error'); } // Test 5: API-based Save log('Test 5: Testing API-based Settings Save', 'test'); try { const apiResponse = await page.evaluate(async (apiKey) => { const response = await fetch('/settings/api/llm.openai_endpoint.api_key', { method: 'PUT', headers: { 'Content-Type': 'application/json', }, credentials: 'same-origin', body: JSON.stringify({ value: apiKey }) }); return { ok: response.ok, status: response.status, data: await response.json() }; }, TEST_CONFIG.apiKeys.changed); if (apiResponse.ok) { log(`API save successful: ${apiResponse.data.message || 'OK'}`, 'success'); testResults.apiSave = true; // Verify the change const verifyResponse = await page.evaluate(async () => { const response = await fetch('/settings/api/llm.openai_endpoint.api_key', { credentials: 'same-origin' }); return await response.json(); }); const apiKeySetting = verifyResponse; if (apiKeySetting && apiKeySetting.value) { log(`API key verified via API: ***${apiKeySetting.value.slice(-5)}`, 'success'); } } else { log(`API save failed: ${apiResponse.status} - ${JSON.stringify(apiResponse.data)}`, 'error'); } } catch (error) { log(`API save test failed: ${error.message}`, 'error'); } // Test 6: Multiple Changes and Final Verification log('Test 6: Testing Multiple Changes', 'test'); try { // Reload to get fresh state await page.reload({ waitUntil: 'domcontentloaded' }); // Find field again apiKeyInput = null; for (const selector of apiKeySelectors) { apiKeyInput = await page.$(selector); if (apiKeyInput) break; } if (apiKeyInput) { // Make final change await apiKeyInput.click({ clickCount: 3 }); await page.keyboard.press('Backspace'); await apiKeyInput.type(TEST_CONFIG.apiKeys.final); // Try different save methods log('Trying blur event (Tab)', 'info'); await page.keyboard.press('Tab'); await new Promise(resolve => setTimeout(resolve, 1000)); log('Clicking elsewhere on page', 'info'); await page.click('body'); await new Promise(resolve => setTimeout(resolve, 1000)); // Final verification const finalCheck = await page.evaluate(async () => { const response = await fetch('/settings/api/llm.openai_endpoint.api_key', { credentials: 'same-origin' }); return await response.json(); }); const finalApiKeySetting = finalCheck; if (finalApiKeySetting && finalApiKeySetting.value) { log(`Final API key value: ***${finalApiKeySetting.value.slice(-5)}`, 'info'); if (finalApiKeySetting.value.includes(TEST_CONFIG.apiKeys.final.slice(-5)) || finalApiKeySetting.value.includes(TEST_CONFIG.apiKeys.changed.slice(-5))) { testResults.multipleChanges = true; log('Multiple changes test passed', 'success'); } } } } catch (error) { log(`Multiple changes test failed: ${error.message}`, 'error'); } // Take final screenshot await page.screenshot({ path: `/tmp/api_key_test_final_${timestamp}.png`, fullPage: false }); log(`Final screenshot saved: /tmp/api_key_test_final_${timestamp}.png`, 'info'); } catch (error) { log(`Test suite failed: ${error.message}`, 'error'); console.error(error); // Take error screenshot if (page) { await page.screenshot({ path: `/tmp/api_key_test_error_${timestamp}.png`, fullPage: true }); log(`Error screenshot saved: /tmp/api_key_test_error_${timestamp}.png`, 'info'); } } finally { // Generate test report log('\n' + colors.bright + '=== Test Results Summary ===' + colors.reset, 'info'); const totalTests = Object.keys(testResults).length; const passedTests = Object.values(testResults).filter(v => v).length; const failedTests = totalTests - passedTests; for (const [test, result] of Object.entries(testResults)) { const status = result ? `${colors.green}PASS${colors.reset}` : `${colors.red}FAIL${colors.reset}`; const testName = test.replace(/([A-Z])/g, ' $1').trim(); log(`${testName}: ${status}`, result ? 'success' : 'error'); } log(`\nTotal: ${totalTests} | Passed: ${passedTests} | Failed: ${failedTests}`, 'info'); // Log API calls made during test if (apiCalls.length > 0) { log('\n' + colors.bright + '=== API Calls Made ===' + colors.reset, 'info'); apiCalls.forEach(call => { log(`${call.method} ${call.url}`, 'info'); }); } // Clean up if (browser) { await browser.close(); log('Browser closed', 'info'); } // Exit with appropriate code process.exit(failedTests > 0 ? 1 : 0); } } // Run the test runComprehensiveApiKeyTest().catch(error => { console.error('Unhandled error:', error); process.exit(1); });