#!/usr/bin/env node /** * Context Overflow UI Tests * * Tests for the context overflow analytics page and related functionality. * * Run: node test_context_overflow_ci.js */ const { setupTest, teardownTest, TestResults, log, navigateTo, withTimeout } = require('./test_lib'); // ============================================================================ // Context Overflow Page Tests // ============================================================================ const ContextOverflowTests = { async contextOverflowPageLoads(page, baseUrl) { await navigateTo(page, `${baseUrl}/metrics/context-overflow`); // The page-specific scaffold is server-rendered (pages/context_overflow.html): //
...

Token Usage & Context Analytics

// Gate the pass on that container AND the exact header text so the test // fails if a login/error/wrong page loaded instead. The old check passed // on a bare `h1, .page-title` or a body-wide "overflow"/"truncat" substring // (which also matches the static help-panel chrome), so it could not tell // the real page apart from any other page that has a heading. const result = await page.evaluate(() => { const container = document.getElementById('context-overflow'); const header = container?.querySelector('.ldr-page-header h1'); return { hasContainer: !!container, headerText: header?.textContent?.trim() || '', is404: document.body.textContent?.includes('404') || document.body.textContent?.includes('Not Found') }; }); if (result.is404) { return { passed: null, skipped: true, message: 'Context overflow page not found (feature may not be enabled)' }; } const passed = result.hasContainer && result.headerText === 'Token Usage & Context Analytics'; return { passed, message: passed ? `Context overflow page loaded (#context-overflow header: "${result.headerText}")` : `Context overflow page failed to load (container=${result.hasContainer}, header="${result.headerText}")` }; }, async truncationRateDisplay(page, baseUrl) { await navigateTo(page, `${baseUrl}/metrics/context-overflow`); // Poll for the controller to resolve one of two end states: // (1) #truncation-rate injected (data path), or // (2) #empty-no-data shown (no-LLM / fresh-DB path). // Replaces a fixed delay(1000) — the controller resolves quickly when idle, // but polling avoids flakiness if the fetch is slow under CI load. try { await page.waitForFunction( () => { if (document.getElementById('truncation-rate')) return true; const empty = document.getElementById('empty-no-data'); return !!empty && getComputedStyle(empty).display !== 'none'; }, { timeout: 5000 } ); } catch (_) { // Fall through — the assertion below reports what's actually on the page. } // The truncation rate is rendered by context-overflow.js into the // overflow-specific element `#truncation-rate` (inside #context-overflow-section), // and ONLY when the API returns requests_with_context_data > 0. With no LLM / // fresh DB (CI), there is no token data, so the controller shows the // #empty-no-data state and never injects #truncation-rate. // // The old test read a body-wide percentage regex, which matched the static // help-panel text ("Green (<10%) / Orange (10-20%) / Red (>20%)") and so // reported a bogus "10%" truncation rate on a page with zero data. We now // read the rate ONLY from #truncation-rate, and when that element is absent // we assert the empty-state rendered on the correct page instead of skipping. const result = await page.evaluate(() => { const onPage = !!document.getElementById('context-overflow'); const rateElement = document.getElementById('truncation-rate'); const rateText = rateElement?.textContent?.trim() || ''; const ratePct = rateText.match(/(\d+(?:\.\d*)?)\s*%/); const emptyNoData = document.getElementById('empty-no-data'); const emptyVisible = emptyNoData ? getComputedStyle(emptyNoData).display !== 'none' : false; return { onPage, hasRateElement: !!rateElement, rateText, rateIsPercent: !!ratePct, emptyVisible }; }); if (result.hasRateElement) { // Data present: the dedicated rate element must show a percentage. return { passed: result.rateIsPercent, message: result.rateIsPercent ? `Truncation rate displayed in #truncation-rate: "${result.rateText}"` : `#truncation-rate present but not a percentage: "${result.rateText}"` }; } // No data (CI no-LLM path): assert we are on the overflow page and the // zero-data empty state rendered, rather than skipping or matching help text. return { passed: result.onPage && result.emptyVisible, message: result.onPage && result.emptyVisible ? 'No token data: #truncation-rate absent and #empty-no-data shown (expected with no LLM)' : `Truncation-rate state unexpected (onPage=${result.onPage}, emptyVisible=${result.emptyVisible})` }; }, async averageTruncatedTokens(page, baseUrl) { await navigateTo(page, `${baseUrl}/metrics/context-overflow`); const result = await page.evaluate(() => { const avgElement = document.querySelector( '[class*="average-tokens"], ' + '[class*="avg-truncated"], ' + '.token-average' ); // Look for token count patterns const tokenPattern = /(\d[\d,]*)\s*(?:tokens?|truncated)/i; const bodyText = document.body.textContent || ''; const tokenMatch = bodyText.match(tokenPattern); const hasAvgText = bodyText.toLowerCase().includes('average') && bodyText.toLowerCase().includes('token'); return { hasAvgElement: !!avgElement, avgText: avgElement?.textContent?.trim(), hasTokenMatch: !!tokenMatch, tokenValue: tokenMatch ? tokenMatch[1] : null, hasAvgText }; }); if (!result.hasAvgElement && !result.hasTokenMatch && !result.hasAvgText) { return { passed: null, skipped: true, message: 'No average truncated tokens display found' }; } return { passed: true, message: result.hasAvgElement ? `Average tokens: "${result.avgText}"` : (result.tokenValue ? `Token count found: ${result.tokenValue}` : 'Average token info found') }; }, async contextOverflowChart(page, baseUrl) { await navigateTo(page, `${baseUrl}/metrics/context-overflow`); // Poll for the controller to resolve one of two end states: // (1) #context-chart canvas injected (data path), or // (2) #empty-no-data shown (no-LLM / fresh-DB path). try { await page.waitForFunction( () => { if (document.getElementById('context-chart')) return true; const empty = document.getElementById('empty-no-data'); return !!empty && getComputedStyle(empty).display !== 'none'; }, { timeout: 5000 } ); } catch (_) { // Fall through — the assertion below reports what's actually on the page. } // The overflow scatter chart is a single page-specific // that context-overflow.js injects into #context-overflow-section, and ONLY when // the API reports requests_with_context_data > 0. With no LLM / fresh DB (CI) there // is no token data, so no canvas is created. // // The old test matched `canvas, svg, .chart, [class*="chart"]` — the substring // selector matched unrelated sidebar/header FontAwesome icons (), // so it reported "Found 4 charts (types: i, i, i, i)" with zero real charts. We now // assert on the overflow-specific chart scaffold: the #context-overflow-section // injection point must exist on the correct page, and if a real #context-chart // canvas is present (data path) it must be a ; otherwise the empty state // must be shown — never counting icon chrome. const result = await page.evaluate(() => { const onPage = !!document.getElementById('context-overflow'); const section = document.getElementById('context-overflow-section'); const contextChart = document.getElementById('context-chart'); const isCanvas = contextChart?.tagName?.toLowerCase() === 'canvas'; const emptyNoData = document.getElementById('empty-no-data'); const emptyVisible = emptyNoData ? getComputedStyle(emptyNoData).display !== 'none' : false; return { onPage, hasSection: !!section, hasContextChart: !!contextChart, isCanvas, emptyVisible }; }); if (result.hasContextChart) { // Data path: the overflow-specific chart element must be a real canvas. return { passed: result.onPage && result.hasSection && result.isCanvas, message: result.isCanvas ? 'Overflow scatter chart present (#context-chart is a )' : `#context-chart present but not a ` }; } // No data (CI no-LLM path): the chart injection point exists on the correct // page and the empty state rendered. This fails (not skips) if we are on the // wrong page or the overflow scaffold did not load. return { passed: result.onPage && result.hasSection && result.emptyVisible, message: result.onPage && result.hasSection && result.emptyVisible ? 'No token data: #context-overflow-section scaffold present and #empty-no-data shown (no #context-chart canvas without data)' : `Overflow chart scaffold unexpected (onPage=${result.onPage}, section=${result.hasSection}, emptyVisible=${result.emptyVisible})` }; }, async periodFilterWorks(page, baseUrl) { await navigateTo(page, `${baseUrl}/metrics/context-overflow`); const result = await page.evaluate(() => { const periodFilter = document.querySelector( 'select[name*="period"], ' + '#period-filter, ' + '.period-filter, ' + 'select[id*="period"]' ); if (periodFilter) { const options = Array.from(periodFilter.options); return { exists: true, type: 'select', optionCount: options.length, options: options.map(o => o.text).slice(0, 6) }; } // Check for button-based filter const buttons = document.querySelectorAll('.period-btn, .time-filter button, [data-period]'); if (buttons.length > 0) { return { exists: true, type: 'buttons', buttonCount: buttons.length, options: Array.from(buttons).map(b => b.textContent?.trim()).slice(0, 6) }; } return { exists: false }; }); if (!result.exists) { return { passed: null, skipped: true, message: 'No period filter found' }; } return { passed: true, message: result.type === 'select' ? `Period filter (${result.optionCount} options): ${result.options.join(', ')}` : `Period buttons (${result.buttonCount}): ${result.options.join(', ')}` }; }, async overflowDetailsTable(page, baseUrl) { await navigateTo(page, `${baseUrl}/metrics/context-overflow`); const result = await page.evaluate(() => { const table = document.querySelector('table, .overflow-details, .data-table'); if (!table) return { hasTable: false }; const headers = Array.from(table.querySelectorAll('th')).map(th => th.textContent?.toLowerCase().trim()); const rows = table.querySelectorAll('tbody tr'); return { hasTable: true, headerCount: headers.length, rowCount: rows.length, headers: headers.slice(0, 6), hasResearchColumn: headers.some(h => h.includes('research') || h.includes('query')), hasTokenColumn: headers.some(h => h.includes('token')) }; }); if (!result.hasTable) { return { passed: null, skipped: true, message: 'No overflow details table found' }; } return { passed: true, message: `Details table: ${result.rowCount} rows, columns: ${result.headers.join(', ')}` }; } }; // ============================================================================ // Context Overflow API Tests // ============================================================================ const ContextOverflowApiTests = { async contextOverflowApiResponds(page, baseUrl) { await navigateTo(page, `${baseUrl}/metrics/context-overflow`); const result = await page.evaluate(async (url) => { try { const response = await fetch(`${url}/metrics/api/context-overflow`); if (!response.ok) return { ok: false, status: response.status }; const data = await response.json(); return { ok: true, status: response.status, hasData: Object.keys(data).length > 0, keys: Object.keys(data).slice(0, 5) }; } catch (e) { return { ok: false, error: e.message }; } }, baseUrl); if (!result.ok && result.status === 404) { return { passed: null, skipped: true, message: 'Context overflow API not found' }; } return { passed: result.ok, message: result.ok ? `Context overflow API responds (keys: ${result.keys.join(', ')})` : `API failed: ${result.error || 'status ' + result.status}` }; } }; // ============================================================================ // Main Test Runner // ============================================================================ async function main() { log.section('Context Overflow Tests'); const ctx = await setupTest({ authenticate: true }); const results = new TestResults('Context Overflow Tests'); const { page } = ctx; const { baseUrl } = ctx.config; const subTestTimeout = ctx.config.isCI ? 60000 : 30000; async function run(category, name, testFn) { try { const result = await withTimeout( testFn(page, baseUrl), subTestTimeout, `${category}/${name}` ); if (result.skipped) { results.skip(category, name, result.message); } else { results.add(category, name, result.passed, result.message); } } catch (error) { results.add(category, name, false, `Error: ${error.message}`); } } try { // Pre-navigate to context overflow page once to check if it loads // (avoids 7 × 30s timeouts in CI when page is slow) log.section('Context Overflow Page'); let pageAccessible = false; try { await navigateTo(page, `${baseUrl}/metrics/context-overflow`); const is404 = await page.evaluate(() => document.body.textContent?.includes('404') || document.body.textContent?.includes('Not Found') ); pageAccessible = !is404; } catch (navError) { log.warning(`Context overflow page not accessible: ${navError.message}`); } if (!pageAccessible) { const skipMsg = 'Context overflow page not accessible in CI'; results.skip('Page', 'Context Overflow Page Loads', skipMsg); results.skip('Page', 'Truncation Rate Display', skipMsg); results.skip('Page', 'Average Truncated Tokens', skipMsg); results.skip('Page', 'Context Overflow Chart', skipMsg); results.skip('Page', 'Period Filter Works', skipMsg); results.skip('Page', 'Overflow Details Table', skipMsg); results.skip('API', 'Context Overflow API Responds', skipMsg); } else { await run('Page', 'Context Overflow Page Loads', (p, u) => ContextOverflowTests.contextOverflowPageLoads(p, u)); await run('Page', 'Truncation Rate Display', (p, u) => ContextOverflowTests.truncationRateDisplay(p, u)); await run('Page', 'Average Truncated Tokens', (p, u) => ContextOverflowTests.averageTruncatedTokens(p, u)); await run('Page', 'Context Overflow Chart', (p, u) => ContextOverflowTests.contextOverflowChart(p, u)); await run('Page', 'Period Filter Works', (p, u) => ContextOverflowTests.periodFilterWorks(p, u)); await run('Page', 'Overflow Details Table', (p, u) => ContextOverflowTests.overflowDetailsTable(p, u)); // API Tests log.section('Context Overflow API'); await run('API', 'Context Overflow API Responds', (p, u) => ContextOverflowApiTests.contextOverflowApiResponds(p, u)); } } catch (error) { log.error(`Fatal error: ${error.message}`); console.error(error.stack); } finally { results.print(); results.save(); await teardownTest(ctx); process.exit(results.exitCode()); } } // Run if executed directly if (require.main === module) { main().catch(error => { console.error('Test runner failed:', error); process.exit(1); }); } module.exports = { ContextOverflowTests, ContextOverflowApiTests };