/** * UI Test Suite Runner * * Runs all UI tests in sequence and provides a summary report. * This script executes each test individually and tracks pass/fail status. * * Prerequisites: Web server running on http://127.0.0.1:5000 * * Usage: * node tests/ui_tests/run_all_tests.js # run all tests * node tests/ui_tests/run_all_tests.js --shard=auth-login # run one shard * * Valid shards: auth-login, auth-register, auth-pages, research-workflow, * research-form, research-metrics, settings-core, settings-pages, * library, history-news, mobile, api-crud, error-benchmark, accessibility, * chat-core, chat-lifecycle, link-analytics */ // Keep in sync with `strategy.matrix.shard` in .github/workflows/docker-tests.yml. // A mismatch would cause silent test misrouting. // // Shard design (17 shards, ~4 tests each): // Each shard runs in its own Docker container with a dedicated server. // Keeping shards small prevents cascade failures when one test stresses // the server (e.g., encrypted DB creation in auth-register). const VALID_SHARDS = [ 'auth-login', // login/auth flow tests 'auth-register', // registration (isolated — heavy SQLCipher DB creation) 'auth-pages', // page browsing, navigation, comprehensive auth 'research-workflow', // core research lifecycle 'research-form', // research form interactions + results 'research-metrics', // metrics charts, dashboard, progress 'settings-core', // settings page, errors, save, interactions 'settings-pages', // settings tabs, star reviews 'library', // collections, documents 'history-news', // history page, news subscriptions 'mobile', // mobile navigation, interactions, UI functionality 'api-crud', // API endpoints, CRUD operations, rate limiting 'error-benchmark', // error handling/recovery, benchmark, context overflow 'accessibility', // keyboard navigation & ARIA 'chat-core', // chat-mode v2 input + a11y + chips + nav 'chat-lifecycle', // chat-mode v2 session lifecycle + export + persistence 'link-analytics', // /metrics/links full-page render + XSS runtime check ]; const { spawn } = require('child_process'); const http = require('http'); const path = require('path'); /** Format a Date as HH:MM:SS for log timestamps */ function ts(date = new Date()) { return date.toISOString().slice(11, 19); } /** * Wait for the server to be responsive before starting the next test. * Prevents cascade failures when a previous test stressed the server * (e.g., registration creating encrypted databases). */ async function waitForServer(maxWaitMs = 60000) { const startTime = Date.now(); let delay = 1000; let wasDown = false; while (Date.now() - startTime < maxWaitMs) { try { const ok = await new Promise((resolve) => { const req = http.get('http://127.0.0.1:5000/api/v1/health', { timeout: 5000 }, (res) => { resolve(res.statusCode >= 200 && res.statusCode < 400); res.resume(); }); req.on('error', () => resolve(false)); req.on('timeout', () => { req.destroy(); resolve(false); }); }); if (ok) { if (wasDown) console.log('Server recovered after being unresponsive'); return true; } wasDown = true; } catch { wasDown = true; } // Capture into const so the closure isn't flagged as no-loop-func. const sleepFor = delay; await new Promise(r => setTimeout(r, sleepFor)); delay = Math.min(delay * 2, 8000); } console.log(`Server did not respond within ${maxWaitMs/1000}s — subsequent tests may fail`); return false; } const tests = [ // ===================================================================== // Shard: auth-login (2 tests) // ===================================================================== { name: 'Authentication Flow Test', file: 'test_auth_flow.js', shard: 'auth-login', description: 'Tests registration, login, and logout functionality' }, { name: 'Login Validation Test', file: 'test_login_validation.js', shard: 'auth-login', description: 'Tests login form validation' }, // ===================================================================== // Shard: auth-register (2 tests) // Register Full Flow is isolated because it creates an encrypted // SQLCipher database (CPU-intensive key derivation + 58 tables + // 500+ settings) which can block the server for 2+ minutes. // ===================================================================== { name: 'Register Validation Test', file: 'test_register_validation.js', shard: 'auth-register', description: 'Tests registration form validation without auth' }, { name: 'Register Full Flow Test', file: 'test_register_full_flow.js', shard: 'auth-register', description: 'Tests complete registration flow (CPU-heavy SQLCipher DB creation)' }, // ===================================================================== // Shard: auth-pages (3 tests) // ===================================================================== { name: 'All Pages Browser Test', file: 'test_pages_browser.js', shard: 'auth-pages', description: 'Tests all main pages for basic functionality' }, { name: 'Full Navigation Test', file: 'test_full_navigation.js', shard: 'auth-pages', description: 'Tests full app navigation flow' }, { name: 'Auth Comprehensive CI Tests', file: 'test_auth_comprehensive_ci.js', shard: 'auth-pages', description: 'Tests password strength, form validation, remember me, sessions' }, // ===================================================================== // Shard: research-workflow (3 tests) // ===================================================================== { name: 'Research Workflow Test', file: 'test_research_workflow.js', shard: 'research-workflow', description: 'Tests the complete research lifecycle from submission to results' }, { name: 'Research Workflow CI Tests', file: 'test_research_workflow_ci.js', shard: 'research-workflow', description: 'Tests research form, progress page, results, exports' }, { name: 'Follow-up Research CI Tests', file: 'test_followup_research_ci.js', shard: 'research-workflow', description: 'Tests follow-up research flow' }, // ===================================================================== // Shard: research-form (3 tests) // ===================================================================== { name: 'Research Form CI Tests', file: 'test_research_form_ci.js', shard: 'research-form', description: 'Tests advanced options, mode toggle, dropdowns, validation' }, { name: 'Research Results Test', file: 'test_research_results.js', shard: 'research-form', description: 'Tests error handling for non-existent research and history page structure' }, { name: 'Results & Exports CI Tests', file: 'test_results_exports_ci.js', shard: 'research-form', description: 'Tests star ratings, export buttons, download functionality' }, // ===================================================================== // Shard: research-metrics (3 tests) // ===================================================================== { name: 'Metrics Charts Test', file: 'test_metrics_charts.js', shard: 'research-metrics', description: 'Tests Chart.js rendering for token and search charts' }, { name: 'Metrics Dashboard CI Tests', file: 'test_metrics_dashboard_ci.js', shard: 'research-metrics', description: 'Tests metrics dashboard, cost analytics, star reviews, links' }, { name: 'Realtime Progress CI Tests', file: 'test_realtime_progress_ci.js', shard: 'research-metrics', description: 'Tests progress page and real-time elements' }, // ===================================================================== // Shard: link-analytics (1 test — release-pipeline only) // ===================================================================== { name: 'Link Analytics Full Page Test', file: 'test_link_analytics_full.js', shard: 'link-analytics', description: 'Verifies /metrics/links renders, Recent Researches header shows numeric total, no console errors, no script element leaks' }, // ===================================================================== // Shard: settings-core (4 tests) // ===================================================================== { name: 'Settings Page Test', file: 'test_settings_page.js', shard: 'settings-core', description: 'Tests settings page loading and API integration' }, { name: 'Settings Error Detection Test', file: 'test_settings_errors.js', shard: 'settings-core', description: 'Tests error handling when changing settings' }, { name: 'Settings Save Test', file: 'test_settings_save.js', shard: 'settings-core', description: 'Tests settings save workflow and validation' }, { name: 'Settings Interactions CI Tests', file: 'test_settings_interactions_ci.js', shard: 'settings-core', description: 'Tests tabs, search, toggles, save, raw config' }, // ===================================================================== // Shard: settings-pages (3 tests) // ===================================================================== { name: 'Settings Pages CI Tests', file: 'test_settings_pages_ci.js', shard: 'settings-pages', description: 'Tests settings tabs, navigation, provider/engine settings' }, { name: 'Star Reviews Test', file: 'test_star_reviews.js', shard: 'settings-pages', description: 'Tests star reviews analytics page and visualizations' }, { name: 'Journal Quality CI Tests', file: 'test_journal_quality_ci.js', shard: 'settings-pages', description: 'Tests journal quality dashboard: tabs, threshold slider settings round-trip, data-status APIs' }, // ===================================================================== // Shard: library (5 tests) // ===================================================================== { name: 'Library Collections CI Tests', file: 'test_library_collections_ci.js', shard: 'library', description: 'Tests library page, collections, document details' }, { name: 'Library Documents CI Tests', file: 'test_library_documents_ci.js', shard: 'library', description: 'Tests filters, views, PDF/text viewers, bulk actions' }, { name: 'Library Collections Page Test', file: 'library/test_collections_page.js', shard: 'library', description: 'Tests library collections page' }, { name: 'Download Manager CI Tests', file: 'test_download_manager_ci.js', shard: 'library', description: 'Tests download manager: stats, filters, selection cycle, collections API' }, { name: 'Zotero Integration CI Tests', file: 'test_zotero_integration_ci.js', shard: 'library', description: 'Tests Zotero page structure, config/status APIs, API-key non-leak contract, unconfigured negative paths' }, // ===================================================================== // Shard: history-news (3 tests) // ===================================================================== { name: 'History Page CI Tests', file: 'test_history_page_ci.js', shard: 'history-news', description: 'Tests history table, actions, search/filter' }, { name: 'History Page Test', file: 'test_history_page.js', shard: 'history-news', description: 'Tests history page functionality' }, { name: 'News Subscriptions CI Tests', file: 'test_news_subscriptions_ci.js', shard: 'history-news', description: 'Tests news feeds, subscription CRUD, form validation' }, // ===================================================================== // Shard: mobile (4 tests) // ===================================================================== { name: 'Mobile Interactions CI Tests', file: 'test_mobile_interactions_ci.js', shard: 'mobile', description: 'Tests mobile modals, navigation, forms' }, { name: 'Mobile Navigation CI Test', file: 'mobile/test_mobile_navigation_ci.js', shard: 'mobile', description: 'Tests mobile navigation patterns' }, { name: 'UI Functionality CI Tests', file: 'mobile/test_ui_functionality_ci.js', shard: 'mobile', description: 'Tests forms, dropdowns, modals, navigation, buttons' }, { name: 'Loading & Feedback CI Tests', file: 'test_loading_feedback_ci.js', shard: 'mobile', description: 'Tests spinners, toasts, progress bars, hover states' }, // ===================================================================== // Shard: api-crud (3 tests) // ===================================================================== { name: 'API Endpoints CI Tests', file: 'test_api_endpoints_ci.js', shard: 'api-crud', description: 'Tests all major API endpoints' }, { name: 'CRUD Operations CI Tests', file: 'test_crud_operations_ci.js', shard: 'api-crud', description: 'Tests collections, subscriptions, documents CRUD' }, { name: 'Rate Limiting Functionality Test', file: 'test_rate_limiting_settings.js', shard: 'api-crud', description: 'Tests rate limiting works on auth endpoints and static files are exempt' }, // ===================================================================== // Shard: error-benchmark (4 tests) // ===================================================================== { name: 'Error Recovery Test', file: 'test_error_recovery.js', shard: 'error-benchmark', description: 'Tests how the UI handles various error conditions gracefully' }, { name: 'Error Handling CI Tests', file: 'test_error_handling_ci.js', shard: 'error-benchmark', description: 'Tests 404, 401, 429, validation errors' }, { name: 'Benchmark CI Tests', file: 'test_benchmark_ci.js', shard: 'error-benchmark', description: 'Tests benchmark dashboard and results pages' }, { name: 'Context Overflow CI Tests', file: 'test_context_overflow_ci.js', shard: 'error-benchmark', description: 'Tests context overflow analytics page' }, // ===================================================================== // Shard: accessibility (1 test) // ===================================================================== { name: 'Keyboard & Accessibility CI Tests', file: 'test_keyboard_accessibility_ci.js', shard: 'accessibility', description: 'Tests keyboard navigation, shortcuts, ARIA, focus management' }, // ===================================================================== // Shard: chat-core (7 tests) // chat-mode v2 — input, a11y, security, navigation. These tests do // not require an LLM backend; they exercise the chat page's // client-side behavior + the chat HTTP routes. // ===================================================================== { name: 'Chat ARIA Live Region Test', file: 'chat/test_chat_aria_live.js', shard: 'chat-core', description: 'Tests role=log + aria-live on .ldr-chat-messages' }, { name: 'Chat Keyboard & Input Test', file: 'chat/test_chat_keyboard_and_input.js', shard: 'chat-core', description: 'Tests Enter-to-send, Shift+Enter newline, textarea state', // Re-enabled: the "CDP input not delivered" failure (#4430) was // Chrome's password-leak-detection dialog after login, now disabled // via the seeded profile in chrome_profile.js. }, { name: 'Chat CSRF Required Test', file: 'chat/test_chat_csrf_required.js', shard: 'chat-core', description: 'Tests that state-mutating chat endpoints reject missing CSRF tokens' }, { name: 'Chat Suggestion Chips Test', file: 'chat/test_chat_suggestion_chips.js', shard: 'chat-core', description: 'Tests suggestion-chip click dispatches a chat message' }, { name: 'Chat New Chat Button Test', file: 'chat/test_chat_new_chat_button.js', shard: 'chat-core', description: 'Tests "New Chat" button starts a fresh session' }, { name: 'Chat URL ?q= Param Test', file: 'chat/test_chat_url_q_param.js', shard: 'chat-core', description: 'Tests /chat?q=... pre-fills the input' }, { name: 'Chat Page Navigation Test', file: 'chat/test_chat_page_navigation.js', shard: 'chat-core', description: 'Tests sidebar navigation to /chat works' }, // ===================================================================== // Shard: chat-lifecycle (6 tests) // chat-mode v2 — session lifecycle (edit/archive/export), error // surfacing, reload persistence. LLM-dependent tests live in the // skipCI section below. // ===================================================================== { name: 'Chat Archived Session Rejects Send Test', file: 'chat/test_chat_archived_session_rejects.js', shard: 'chat-lifecycle', description: 'Tests an archived session rejects POST /api/chat/sessions//messages with 409' }, { name: 'Chat Edit Title Test', file: 'chat/test_chat_edit_title.js', shard: 'chat-lifecycle', description: 'Tests in-place rename of a chat session via PATCH' }, { name: 'Chat Export Markdown Test', file: 'chat/test_chat_export_markdown.js', shard: 'chat-lifecycle', description: 'Tests export-to-markdown endpoint + UI flow' }, { name: 'Chat Reload Persistence Test', file: 'chat/test_chat_reload_persistence.js', shard: 'chat-lifecycle', description: 'Tests messages persist across a page reload' }, { name: 'Chat Session Management Test', file: 'chat/test_chat_session_management.js', shard: 'chat-lifecycle', description: 'Tests list/archive/reactivate/delete via the chat UI' }, { name: 'Chat Error States Test', file: 'chat/test_chat_error_states.js', shard: 'chat-lifecycle', description: 'Tests 404/429/network-down error rendering' }, // ===================================================================== // Skipped tests (skipCI: true) — still need shard assignments for // local runs. Shard names can be anything valid since they never run // in CI; assigned to the closest active shard. // ===================================================================== { name: 'Research Submit Test', file: 'test_research_submit.js', shard: 'research-form', description: 'Tests research submission', skipCI: true, // Requires LLM backend }, { name: 'Export Functionality Test', file: 'test_export_functionality.js', shard: 'research-form', description: 'Tests export features', skipCI: true, // Auth hangs with "Navigating frame was detached" in Docker }, { name: 'Concurrent Limit Test', file: 'test_concurrent_limit.js', shard: 'research-workflow', description: 'Tests concurrent research limits', skipCI: true, // Requires LLM backend — always fails without model server }, { name: 'News Feed CI Tests', file: 'test_news_feed_ci.js', shard: 'history-news', description: 'Tests feed, filters, templates, subscription management', skipCI: true, // Intermittent 60s navigation timeouts; core coverage in test_news_subscriptions_ci.js }, { name: 'Settings Validation Test', file: 'test_settings_validation.js', shard: 'settings-core', description: 'Tests settings input validation', skipCI: true, // Same frame-detachment issue as Export test in Docker }, { name: 'Research Form Validation Test', file: 'test_research_form_validation.js', shard: 'research-form', description: 'Tests research form field validation', skipCI: true, // Redundant with test_research_form_ci.js; auth frame-detachment in Docker }, { name: 'Form Validation ARIA Tests', file: 'test_form_validation_aria_ci.js', shard: 'accessibility', description: 'Tests inline form validation with ARIA support', skipCI: true, // Auth frame-detachment in Docker causes intermittent 120s timeout }, { name: 'Research Simple Test', file: 'test_research_simple.js', shard: 'research-workflow', description: 'Tests basic research flow', skipCI: true, // Requires LLM backend to complete research submission }, { name: 'Research Form Test', file: 'test_research_form.js', shard: 'research-form', description: 'Tests research form interactions', skipCI: true, // Diagnostic test — requires LLM for form submission }, { name: 'Research API Test', file: 'test_research_api.js', shard: 'research-workflow', description: 'Tests research API endpoints via UI', skipCI: true, // Diagnostic test — requires functioning LLM API }, { name: 'Queue Simple Test', file: 'test_queue_simple.js', shard: 'research-workflow', description: 'Tests research queue functionality', skipCI: true, // Requires LLM backend — always fails without model server }, { name: 'Chat Message Flow E2E Test', file: 'chat/test_chat_message_flow.js', shard: 'chat-lifecycle', description: 'End-to-end: send message, watch research streaming, assert assistant response', skipCI: true, // Requires LDR_TEST_LLM_URL + LDR_TEST_LLM_MODEL backend }, { name: 'Chat report_content Refactor Test', file: 'chat/test_chat_report_content_refactor.js', shard: 'chat-lifecycle', description: 'Verifies report_content shape change: chat shows answer-only, /results assembles full', skipCI: true, // Requires LDR_TEST_LLM_URL + LDR_TEST_LLM_MODEL backend }, ]; async function runTest(test) { return new Promise((resolve) => { const startTime = Date.now(); console.log(`\n[${ts()}] Running: ${test.name}`); const testProcess = spawn('node', [test.file], { cwd: path.join(__dirname), stdio: 'inherit', env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=4096' } }); // Add timeout for individual tests // 300 seconds in CI to handle slow registration/auth tests. // In Docker, new user registration creates encrypted SQLCipher databases // with key derivation + 58 tables + 500+ settings, which can take 60-120s. // Subsequent tests may also be slow while the server recovers. // 60 seconds locally for faster feedback. const isCI = !!process.env.CI; const timeoutMs = isCI ? 300000 : 60000; const timeout = setTimeout(() => { const elapsed = Math.round((Date.now() - startTime) / 1000); console.log(`\n⏱️ Test timeout: ${test.name} exceeded ${timeoutMs/1000} seconds (${elapsed}s elapsed)`); console.log(`🔪 Sending SIGTERM to PID ${testProcess.pid}...`); testProcess.kill('SIGTERM'); setTimeout(() => { if (!testProcess.killed) { console.log(`🔫 Process still alive, sending SIGKILL to PID ${testProcess.pid}...`); testProcess.kill('SIGKILL'); } }, 5000); }, timeoutMs); testProcess.on('close', (code) => { clearTimeout(timeout); const elapsed = Math.round((Date.now() - startTime) / 1000); const success = code === 0; console.log(`[${ts()}] ${success ? '✅' : '❌'} ${test.name}: ${success ? 'PASSED' : 'FAILED'} (${elapsed}s)`); // Grep-friendly line for post-run duration analysis (used to rebalance shards). console.log(`TIMING: ${test.name}: ${elapsed}`); if (code !== 0 && code !== null) { console.log(` Exit code: ${code}`); } resolve({ name: test.name, success, code, duration: elapsed }); }); testProcess.on('error', (error) => { clearTimeout(timeout); const elapsed = Math.round((Date.now() - startTime) / 1000); console.log(`[${ts()}] ❌ ${test.name}: ERROR - ${error.message} (${elapsed}s)`); resolve({ name: test.name, success: false, error: error.message, duration: elapsed }); }); }); } function parseShardArg() { const shardArg = process.argv.find(arg => arg.startsWith('--shard=')); return shardArg ? shardArg.split('=')[1] : null; } function validateBeforeRun(requestedShard, isCI) { // Every test must declare a shard so filtering can't silently skip it. const untagged = tests.filter(t => !t.shard); if (untagged.length > 0) { console.error('FATAL: the following tests are missing a `shard:` property:'); untagged.forEach(t => console.error(` - ${t.name} (${t.file})`)); console.error(`Valid shards: ${VALID_SHARDS.join(', ')}`); process.exit(1); } if (requestedShard && !VALID_SHARDS.includes(requestedShard)) { console.error(`FATAL: unknown shard "${requestedShard}".`); console.error(`Valid shards: ${VALID_SHARDS.join(', ')}`); process.exit(1); } // Matrix misconfiguration guard: if CI forgot to pass --shard, every // matrix cell would otherwise run the full suite. Fail loud. if (isCI && !requestedShard) { console.error('FATAL: CI=true but no --shard flag provided.'); console.error('Matrix is misconfigured — each cell must pass --shard=.'); console.error(`Valid shards: ${VALID_SHARDS.join(', ')}`); process.exit(1); } } async function runAllTests() { const suiteStart = new Date(); const isCI = !!process.env.CI; const requestedShard = parseShardArg(); validateBeforeRun(requestedShard, isCI); const shardLabel = requestedShard ? ` [shard: ${requestedShard}]` : ''; console.log(`[${ts(suiteStart)}] Starting UI Test Suite${shardLabel}\n`); const results = []; for (const test of tests) { // skipCI takes priority: a test marked skipCI stays skipped in CI even // if its shard matches. Keeps existing skip semantics intact. if (test.skipCI && isCI) { // Only log the skip when this shard would have run the test. if (!requestedShard || test.shard === requestedShard) { console.log(`\n[${ts()}] ⏭️ Skipping: ${test.name} (not supported in CI Docker)`); results.push({ name: test.name, success: true, duration: 0, skipped: true }); } continue; } // Shard filter: silently drop tests that don't belong to the requested shard. if (requestedShard && test.shard !== requestedShard) { continue; } // Ensure server is responsive before starting each test. // Prevents cascade failures when a previous test stressed the server. await waitForServer(); const result = await runTest(test); results.push(result); } // Print summary const suiteEnd = new Date(); const wallTime = Math.round((suiteEnd - suiteStart) / 1000); console.log(`\n[${ts(suiteEnd)}] TEST SUMMARY`); const passed = results.filter(r => r.success).length; const failed = results.filter(r => !r.success).length; const totalDuration = results.reduce((sum, r) => sum + (r.duration || 0), 0); // Sort by duration descending so slowest tests are easy to spot const sorted = [...results].sort((a, b) => (b.duration || 0) - (a.duration || 0)); sorted.forEach(result => { const status = result.success ? '✅ PASS' : '❌ FAIL'; const duration = result.duration ? ` (${result.duration}s)` : ''; console.log(`${status} ${result.name}${duration}`); if (result.error) { console.log(` Error: ${result.error}`); } }); const shardSuffix = requestedShard ? ` [shard: ${requestedShard}]` : ''; const rate = results.length === 0 ? 0 : Math.round((passed / results.length) * 100); console.log(`Total: ${results.length} | Passed: ${passed} | Failed: ${failed} | Duration: ${totalDuration}s | Wall: ${wallTime}s | Rate: ${rate}%${shardSuffix}`); if (failed === 0) { console.log('All tests passed!'); } else { console.log(`${failed} test(s) failed.`); } process.exit(failed > 0 ? 1 : 0); } runAllTests().catch(error => { console.error('💥 Test runner error:', error); process.exit(1); });