#!/usr/bin/env node /** * Error Handling UI Tests * * Tests for 404, 401, 429 error handling, and form validation errors. * * Run: node test_error_handling_ci.js */ const { setupTest, teardownTest, TestResults, log, delay, navigateTo, withTimeout } = require('./test_lib'); /** * Navigate with a single retry on timeout. * * Used for deliberately-bad URLs (404 probes) where a slow CI server can * push the first goto past its 60s timeout. Without the retry, a single * stuck navigation consumes the suite's wall-clock budget and triggers the * SIGTERM that detaches the page for every subsequent sub-test. */ async function navigateToWithRetry(page, url) { try { return await navigateTo(page, url); } catch (firstError) { await delay(2000); return await navigateTo(page, url); } } // ============================================================================ // 404 Error Handling Tests // ============================================================================ const Error404Tests = { async nonExistentPageShows404(page, baseUrl) { const response = await navigateToWithRetry(page, `${baseUrl}/nonexistent-page-12345`); const result = await page.evaluate(() => { const bodyText = document.body.textContent?.toLowerCase() || ''; return { has404Text: bodyText.includes('404') || bodyText.includes('not found'), hasErrorPage: !!document.querySelector('.error-page, .not-found, [class*="404"]'), hasHomeLink: !!document.querySelector('a[href="/"], a[href*="home"]'), pageTitle: document.title }; }); const statusCode = response?.status(); const passed = statusCode === 404 || result.has404Text || result.hasErrorPage; return { passed, message: passed ? `404 handled (status: ${statusCode}, has404Text: ${result.has404Text})` : `Unexpected response for non-existent page (status: ${statusCode})` }; }, async invalidResearchIdHandled(page, baseUrl) { // The /results/ route always renders pages/results.html (status 200); // not-found handling is client-side: results.js fetches /api/report/, // gets a 404, and calls showError() which injects an // `.alert-danger` (with "Error loading research results: HTTP error 404") // into #results-content. Assert the SPECIFIC outcome: // 1. The results page actually rendered (#research-results container) — // proves we did not land on login/a generic error page. // 2. A targeted error element appeared inside #results-content // (not the bare substring "error" anywhere in page chrome). await navigateToWithRetry(page, `${baseUrl}/results/invalid-research-id-12345`); // Confirm the results page chrome rendered before waiting on the async error. await page.waitForSelector('#research-results #results-content', { timeout: 15000 }); // results.js renders the error asynchronously after the failed /api/report // fetch. Wait for the targeted alert rather than reading body text once. try { await page.waitForFunction(() => { const alert = document.querySelector('#results-content .alert-danger'); return !!alert && (alert.textContent || '').trim().length > 0; }, { timeout: 15000 }); } catch { // Targeted alert never appeared within 15s. Don't fail here — the DOM // re-read below makes the final pass/fail decision — but log a hint so a // slow render is distinguishable from "no error element rendered" during // triage. log('invalidResearchIdHandled: timed out waiting for #results-content .alert-danger (slow render?)'); } const result = await page.evaluate(() => { const onResultsPage = !!document.querySelector('#research-results'); const alert = document.querySelector('#results-content .alert-danger'); return { onResultsPage, hasTargetedError: !!alert, errorText: alert?.textContent?.trim().substring(0, 120) || '', currentPath: window.location.pathname }; }); const passed = result.onResultsPage && result.hasTargetedError; return { passed, message: passed ? `Invalid research ID surfaced targeted error on results page (path: ${result.currentPath}, error: "${result.errorText}")` : `Invalid research ID not handled (onResultsPage: ${result.onResultsPage}, targetedError: ${result.hasTargetedError}, path: ${result.currentPath})` }; }, async invalidDocumentIdHandled(page, baseUrl) { // Use fetch instead of page navigation to avoid flaky domcontentloaded timeouts // The Flask route returns a simple text "Document not found" with status 404 const result = await page.evaluate(async (url) => { try { const response = await fetch(`${url}/library/document/invalid-doc-id-12345`); const text = await response.text(); const bodyText = text.toLowerCase(); return { status: response.status, hasErrorText: bodyText.includes('not found') || bodyText.includes('error'), redirected: response.redirected, finalUrl: response.url }; } catch (e) { return { error: e.message }; } }, baseUrl); if (result.error) { return { passed: null, skipped: true, message: `Fetch failed: ${result.error}` }; } const passed = result.status === 404 || result.hasErrorText || result.redirected; return { passed, message: passed ? `Invalid document ID handled (status: ${result.status})` : 'Invalid document ID not handled gracefully' }; } }; // ============================================================================ // 401 Authentication Error Tests // ============================================================================ const Error401Tests = { async unauthenticatedRedirectsToLogin(page, baseUrl) { // Navigate to login page first to avoid pending XHR interference await page.goto(`${baseUrl}/auth/login`, { waitUntil: 'domcontentloaded', timeout: 15000 }); // Clear cookies to simulate unauthenticated state const client = await page.target().createCDPSession(); await client.send('Network.clearBrowserCookies'); await navigateTo(page, `${baseUrl}/settings/`); const result = await page.evaluate(() => { const currentPath = window.location.pathname; const hasLoginForm = !!document.querySelector('form[action*="login"], input[type="password"], .login-form'); const bodyText = document.body.textContent?.toLowerCase() || ''; const hasLoginText = bodyText.includes('login') || bodyText.includes('sign in'); return { currentPath, redirectedToLogin: currentPath.includes('login') || currentPath.includes('auth'), hasLoginForm, hasLoginText }; }); const passed = result.redirectedToLogin || result.hasLoginForm || result.hasLoginText; return { passed, message: passed ? `Unauthenticated user redirected to login (path: ${result.currentPath})` : 'Protected route accessible without authentication' }; }, async apiUnauthorizedReturns401(page, baseUrl) { // Clear cookies const client = await page.target().createCDPSession(); await client.send('Network.clearBrowserCookies'); await navigateTo(page, `${baseUrl}/`); const result = await page.evaluate(async (url) => { try { const response = await fetch(`${url}/api/history`); return { status: response.status, is401or403: response.status === 401 || response.status === 403, redirected: response.redirected, finalUrl: response.url }; } catch (e) { return { error: e.message }; } }, baseUrl); if (result.error) { return { passed: null, skipped: true, message: `API call failed: ${result.error}` }; } const passed = result.is401or403 || result.redirected; return { passed, message: passed ? `Unauthenticated API returns ${result.status} or redirects` : `API accessible without auth (status: ${result.status})` }; } }; // ============================================================================ // API Error Response Tests // ============================================================================ const ApiErrorTests = { async apiMissingParamsReturns400(page, baseUrl) { // Re-authenticate first const ctx = await page.evaluate(() => ({ authenticated: !!document.cookie })); await navigateTo(page, `${baseUrl}/`); const result = await page.evaluate(async (url) => { try { // Try to start research without required params const response = await fetch(`${url}/api/start_research`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({}) // Empty body, missing required 'query' }); const data = await response.json().catch(() => ({})); return { status: response.status, is400: response.status === 400, hasError: 'error' in data || 'message' in data, errorMessage: data.error || data.message }; } catch (e) { return { error: e.message }; } }, baseUrl); if (result.error) { return { passed: null, skipped: true, message: `API call failed: ${result.error}` }; } // 400 or 422 are both acceptable for validation errors const passed = result.status === 400 || result.status === 422 || result.hasError; return { passed, message: passed ? `Missing params returns ${result.status} with error message` : `Unexpected response for missing params (status: ${result.status})` }; }, async apiInvalidIdReturns404(page, baseUrl) { await navigateTo(page, `${baseUrl}/`); const result = await page.evaluate(async (url) => { try { const response = await fetch(`${url}/api/research/nonexistent-id-12345`); return { status: response.status, is404: response.status === 404 }; } catch (e) { return { error: e.message }; } }, baseUrl); if (result.error) { return { passed: null, skipped: true, message: `API call failed: ${result.error}` }; } // Accept 401 as well — auth session may not carry over to the API request const passed = result.is404 || result.status === 401; return { passed, message: passed ? `Invalid research ID returns ${result.status}` : `Invalid ID returns ${result.status} (expected 404 or 401)` }; } }; // ============================================================================ // Rate Limiting Tests // ============================================================================ const RateLimitTests = { async rateLimitingSectionRenders(page, baseUrl) { // The metrics dashboard (pages/metrics.html) renders a server-side // "Rate Limiting Analytics" section that is present regardless of LLM or // collected data — its metric values default to 0/0%. This is the page // that surfaces rate-limit info to the user, so assert the SPECIFIC // server-rendered container + the stable rate-limiting element IDs, // instead of skipping on a loose "rate limit" body-text substring (which // matched the section heading and made the test never really fail). await navigateTo(page, `${baseUrl}/metrics/`); // The metrics route is @login_required: an unauthenticated request 302s // to /auth/login. The 401-auth sub-tests earlier in this suite clear // cookies, and on the shared server the re-auth can fail to fully restore // a navigable session (the sibling rate-limiting-endpoint test skips on // the same 401). Treat a login redirect as an environmental skip — but if // we DID land on the metrics page, assert the section strictly (real fail). const landingPath = page.url(); if (/\/auth\/login/.test(landingPath) || !(await page.$('#metrics'))) { return { passed: null, skipped: true, message: `Metrics dashboard not reachable (session lost / redirected): ${landingPath}` }; } // Page-specific container present — now require the server-rendered // rate-limiting section. A timeout here is a REAL failure. await page.waitForSelector('#metrics #rate-limit-success-rate', { timeout: 15000 }); const result = await page.evaluate(() => { const onMetricsPage = !!document.querySelector('#metrics'); // Stable, server-rendered rate-limiting elements from metrics.html. const successRate = document.querySelector('#rate-limit-success-rate'); const events = document.querySelector('#rate-limit-events'); const enginesTracked = document.querySelector('#engines-tracked'); const engineStatusGrid = document.querySelector('#engine-status-grid'); const chart = document.querySelector('#rate-limiting-chart'); // The section heading text confirms this is the rate-limiting block, // not some other metric card reusing a similar id. const heading = Array.from(document.querySelectorAll('h2')) .find(h => /rate limiting analytics/i.test(h.textContent || '')); return { onMetricsPage, hasSuccessRate: !!successRate, hasEvents: !!events, hasEnginesTracked: !!enginesTracked, hasEngineStatusGrid: !!engineStatusGrid, hasChart: !!chart, hasHeading: !!heading, successRateText: successRate?.textContent?.trim().substring(0, 20) || '' }; }); const passed = result.onMetricsPage && result.hasHeading && result.hasSuccessRate && result.hasEvents && result.hasEnginesTracked && result.hasEngineStatusGrid && result.hasChart; return { passed, message: passed ? `Rate Limiting Analytics section rendered on metrics dashboard (success-rate value: "${result.successRateText}")` : `Rate limiting section incomplete (metricsPage: ${result.onMetricsPage}, heading: ${result.hasHeading}, successRate: ${result.hasSuccessRate}, events: ${result.hasEvents}, engines: ${result.hasEnginesTracked}, grid: ${result.hasEngineStatusGrid}, chart: ${result.hasChart})` }; }, async rateLimitingStatusEndpoint(page, baseUrl) { await navigateTo(page, `${baseUrl}/`); const result = await page.evaluate(async (url) => { try { const response = await fetch(`${url}/metrics/api/rate-limiting/current`); if (!response.ok) return { ok: false, status: response.status }; // Guard against HTML responses (e.g. login page redirects) const contentType = response.headers.get('content-type') || ''; if (!contentType.includes('application/json')) { return { ok: false, status: response.status, error: `Non-JSON content-type: ${contentType}` }; } const data = await response.json(); return { ok: true, status: response.status, hasLimits: Object.keys(data).length > 0 }; } catch (e) { return { ok: false, error: e.message }; } }, baseUrl); if (!result.ok && result.status === 404) { return { passed: null, skipped: true, message: 'Rate limiting status endpoint not found' }; } if (!result.ok && (result.status === 401 || result.status === 403)) { return { passed: null, skipped: true, message: `Rate limiting endpoint requires auth (status ${result.status})` }; } if (!result.ok && result.error && result.error.startsWith('Non-JSON content-type')) { return { passed: null, skipped: true, message: `Rate limiting endpoint returned non-JSON (likely auth redirect)` }; } return { passed: result.ok, message: result.ok ? 'Rate limiting status endpoint responds' : `Rate limiting endpoint failed: ${result.error || 'status ' + result.status}` }; } }; // ============================================================================ // Form Validation Error Tests // ============================================================================ const FormValidationTests = { async emptyQueryShowsError(page, baseUrl) { await navigateTo(page, `${baseUrl}/`); // Try to submit empty research form const result = await page.evaluate(() => { const form = document.querySelector('form.research-form, form[action*="research"], #research-form'); const submitBtn = document.querySelector('button[type="submit"], .submit-btn, .btn-primary'); if (!form && !submitBtn) return { hasForm: false }; // Clear any existing query const queryInput = document.querySelector('input[name*="query"], textarea[name*="query"], #query'); if (queryInput) queryInput.value = ''; // Click submit if (submitBtn) submitBtn.click(); return new Promise(resolve => { let attempts = 0; const check = () => { const hasError = !!document.querySelector( '.error, .invalid-feedback, .form-error, .alert-danger, [class*="error"]' ); const hasInvalidInput = !!document.querySelector('input:invalid, textarea:invalid'); const errorText = document.querySelector('.error, .invalid-feedback')?.textContent?.trim(); if (hasError || hasInvalidInput || ++attempts >= 15) { resolve({ hasForm: true, hasError, hasInvalidInput, errorText }); } else { setTimeout(check, 200); } }; setTimeout(check, 200); }); }); if (!result.hasForm) { return { passed: null, skipped: true, message: 'No research form found' }; } const passed = result.hasError || result.hasInvalidInput; return { passed, message: passed ? `Empty query validation works (error: ${result.errorText || 'shown'})` : 'Empty query did not show validation error' }; }, async invalidSettingsShowsError(page, baseUrl) { await navigateTo(page, `${baseUrl}/settings/`); const result = await page.evaluate(() => { // Look for any numeric input and try to set invalid value const numericInput = document.querySelector( 'input[type="number"], ' + 'input[name*="temperature"], ' + 'input[name*="iterations"]' ); if (!numericInput) return { hasInput: false }; // Set invalid value numericInput.value = '-999'; numericInput.dispatchEvent(new Event('change', { bubbles: true })); numericInput.dispatchEvent(new Event('input', { bubbles: true })); return new Promise(resolve => { let attempts = 0; const check = () => { const hasError = !!document.querySelector( '.error, .invalid-feedback, .form-error, [class*="error"]' ); const hasInvalidInput = !!document.querySelector('input:invalid'); if (hasError || hasInvalidInput || ++attempts >= 15) { resolve({ hasInput: true, hasError, hasInvalidInput }); } else { setTimeout(check, 200); } }; setTimeout(check, 200); }); }); if (!result.hasInput) { return { passed: null, skipped: true, message: 'No numeric input found to test validation' }; } const passed = result.hasError || result.hasInvalidInput; return { passed, message: passed ? 'Invalid settings value shows validation error' : 'Invalid settings value did not trigger validation' }; }, async requiredFieldsMarked(page, baseUrl) { await navigateTo(page, `${baseUrl}/`); const result = await page.evaluate(() => { const requiredInputs = document.querySelectorAll('[required], .required'); const requiredLabels = document.querySelectorAll('label.required, label:has(+ [required])'); // Note: :contains() is not valid CSS - check for asterisks differently const asteriskIndicators = document.querySelectorAll('.required-indicator, .asterisk'); // Check for labels that contain asterisks in their text content const labelsWithAsterisk = Array.from(document.querySelectorAll('label')).filter(label => label.textContent.includes('*')); return { requiredInputCount: requiredInputs.length, requiredLabelCount: requiredLabels.length, hasAsterisks: asteriskIndicators.length > 0 || labelsWithAsterisk.length > 0 || document.body.innerHTML.includes('*') || document.body.innerHTML.includes('required') }; }); if (result.requiredInputCount === 0 && result.requiredLabelCount === 0) { return { passed: null, skipped: true, message: 'No required field indicators found' }; } return { passed: true, message: `Required fields marked (${result.requiredInputCount} inputs, ${result.requiredLabelCount} labels)` }; } }; // ============================================================================ // Main Test Runner // ============================================================================ async function main() { log.section('Error Handling Tests'); const ctx = await setupTest({ authenticate: true }); const results = new TestResults('Error Handling Tests'); const { page } = ctx; const { baseUrl } = ctx.config; // Per-sub-test timeout + about:blank recovery on failure. // // The suite has a wall-clock budget enforced externally (300s in CI). If // any single sub-test hangs past that budget the runner SIGTERMs the // process and every remaining sub-test cascades into "detached frame" // errors. A per-sub-test timeout caps each call well below the suite // budget; resetting to about:blank on failure prevents a half-loaded // page from breaking the next test. const subTestTimeout = ctx.config.isCI ? 60000 : 30000; async function run(category, name, testFn) { try { const result = await withTimeout(testFn(), subTestTimeout, `${category}/${name}`); if (result && result.skipped) { results.skip(category, name, result.message); } else { results.add(category, name, result.passed, result.message || ''); } } catch (error) { results.add(category, name, false, `Error: ${error.message}`); try { await page.goto('about:blank', { timeout: 5000 }); } catch { // Best-effort recovery — don't mask the original failure. } } } try { // 404 Error Tests log.section('404 Errors'); await run('404', 'Non-existent Page Shows 404', () => Error404Tests.nonExistentPageShows404(page, baseUrl)); await run('404', 'Invalid Research ID Handled', () => Error404Tests.invalidResearchIdHandled(page, baseUrl)); await run('404', 'Invalid Document ID Handled', () => Error404Tests.invalidDocumentIdHandled(page, baseUrl)); // 401 Authentication Tests log.section('401 Authentication'); await run('401', 'Unauthenticated Redirects To Login', () => Error401Tests.unauthenticatedRedirectsToLogin(page, baseUrl)); // After 401 test cleared cookies, re-authenticate on a fresh login page. // We avoid waitForNavigation here — if login fails (CSRF, wrong password), // it stays on /auth/login and waitForNavigation hangs for the full timeout. // Instead: click submit, then poll for URL change or session cookie. let reAuthOk = false; try { await page.goto(`${baseUrl}/auth/login`, { waitUntil: 'domcontentloaded', timeout: 15000 }); await page.waitForSelector('input[name="username"]', { timeout: 10000 }); await page.$eval('input[name="username"]', el => { el.value = ''; }); await page.$eval('input[name="password"]', el => { el.value = ''; }); await page.type('input[name="username"]', 'test_admin'); await page.type('input[name="password"]', 'testpass123'); await page.click('button[type="submit"]'); // Poll for up to 15s: either we leave /auth/login or get a session cookie for (let i = 0; i < 30; i++) { await delay(500); const url = page.url(); if (!url.includes('/auth/login')) { reAuthOk = true; break; } const cookies = await page.cookies(); if (cookies.some(c => c.name === 'session')) { // Have session cookie but still on login page — navigate away await page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: 15000 }); reAuthOk = true; break; } } } catch (reAuthError) { log.warning(`Direct re-auth failed: ${reAuthError.message}`); } // Fallback: try ensureAuthenticated if direct login failed if (!reAuthOk) { try { await withTimeout( ctx.authHelper.ensureAuthenticated(), 60000, 'Re-authentication after 401 tests' ); reAuthOk = true; } catch (error) { log.warning(`Re-authentication timed out: ${error.message}`); } } if (reAuthOk) { await run('401', 'API Unauthorized Returns 401', () => Error401Tests.apiUnauthorizedReturns401(page, baseUrl)); } else { results.skip('401', 'API Unauthorized Returns 401', 'Skipped — re-authentication timed out'); } // Re-authenticate again for remaining tests if (!reAuthOk) { try { await withTimeout( ctx.authHelper.ensureAuthenticated(), 60000, 'Re-authentication for API tests' ); reAuthOk = true; } catch (error) { log.warning(`Re-authentication timed out again: ${error.message}`); } } // API Error Tests (require authenticated session) log.section('API Errors'); if (reAuthOk) { await run('API', 'API Missing Params Returns 400', () => ApiErrorTests.apiMissingParamsReturns400(page, baseUrl)); await run('API', 'API Invalid ID Returns 404', () => ApiErrorTests.apiInvalidIdReturns404(page, baseUrl)); } else { results.skip('API', 'API Missing Params Returns 400', 'Skipped — could not re-authenticate after 401 tests'); results.skip('API', 'API Invalid ID Returns 404', 'Skipped — could not re-authenticate after 401 tests'); } // Rate Limiting Tests (require authenticated session) log.section('Rate Limiting'); if (reAuthOk) { await run('RateLimit', 'Rate Limiting Section Renders', () => RateLimitTests.rateLimitingSectionRenders(page, baseUrl)); await run('RateLimit', 'Rate Limiting Status Endpoint', () => RateLimitTests.rateLimitingStatusEndpoint(page, baseUrl)); } else { results.skip('RateLimit', 'Rate Limiting Section Renders', 'Skipped — could not re-authenticate after 401 tests'); results.skip('RateLimit', 'Rate Limiting Status Endpoint', 'Skipped — could not re-authenticate after 401 tests'); } // Form Validation Tests log.section('Form Validation'); await run('Validation', 'Empty Query Shows Error', () => FormValidationTests.emptyQueryShowsError(page, baseUrl)); await run('Validation', 'Invalid Settings Shows Error', () => FormValidationTests.invalidSettingsShowsError(page, baseUrl)); await run('Validation', 'Required Fields Marked', () => FormValidationTests.requiredFieldsMarked(page, baseUrl)); } catch (error) { log.error(`Fatal error: ${error.message}`); console.error(error.stack); } finally { results.print(); results.save(); await teardownTest(ctx); process.exit(results.exitCode()); } } // Run if executed directly if (require.main === module) { main().catch(error => { console.error('Test runner failed:', error); process.exit(1); }); } module.exports = { Error404Tests, Error401Tests, ApiErrorTests, RateLimitTests, FormValidationTests };