learningcircuit--local-deep-research/tests/ui_tests/test_benchmark_ci.js

#!/usr/bin/env node
/**
 * Benchmark UI Tests
 *
 * Tests for the benchmark dashboard and results pages.
 *
 * Run: node test_benchmark_ci.js
 */

const { setupTest, teardownTest, TestResults, log, delay, navigateTo, withTimeout } = require('./test_lib');

// ============================================================================
// Benchmark Dashboard Tests
// ============================================================================
const BenchmarkDashboardTests = {
    async benchmarkPageLoads(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(() => {
            return {
                hasContent: !!document.querySelector('.benchmark-container, .ldr-benchmark, #benchmark, .benchmark'),
                hasHeader: !!document.querySelector('h1, .benchmark-header, .page-title'),
                headerText: document.querySelector('h1, .benchmark-header, .page-title')?.textContent?.trim(),
                hasForm: !!document.querySelector('form, .benchmark-form, .config-form')
            };
        });

        const passed = result.hasContent || result.hasHeader || result.hasForm;
        return {
            passed,
            message: passed
                ? `Benchmark page loaded (header: "${result.headerText}", form: ${result.hasForm})`
                : 'Benchmark page failed to load'
        };
    },

    async benchmarkFormStructure(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(() => {
            const form = document.querySelector('form, .benchmark-form, .config-form');
            if (!form) return { hasForm: false };

            const inputs = form.querySelectorAll('input, select, textarea');
            const selects = form.querySelectorAll('select');
            const buttons = form.querySelectorAll('button[type="submit"], .btn-primary, .start-btn');

            return {
                hasForm: true,
                inputCount: inputs.length,
                selectCount: selects.length,
                hasSubmitButton: buttons.length > 0,
                buttonText: buttons[0]?.textContent?.trim()
            };
        });

        if (!result.hasForm) {
            return { passed: null, skipped: true, message: 'No benchmark form found' };
        }

        return {
            passed: result.inputCount > 0 || result.selectCount > 0,
            message: `Benchmark form: ${result.inputCount} inputs, ${result.selectCount} selects, submit="${result.buttonText}"`
        };
    },

    async configDropdowns(page, baseUrl) {
        // The benchmark form (benchmark.html) has #evaluation_provider (a <select>)
        // and #evaluation_model (a custom-dropdown input). On load,
        // populateEvaluationProviders() (benchmark.html) fills the select with the
        // built-in provider list (ollama/openai/anthropic/...), which renders
        // client-side with no LLM — the async /settings/api/available-models call
        // only *replaces* that list if it returns providers. So this asserts the
        // select is actually populated with non-empty options (a render/contract
        // check that catches the script not running), not that discovery succeeded.
        // The old test scanned all <select>s and passed if any name contained
        // "provider" — true for #evaluation_provider regardless of its options.
        await navigateTo(page, `${baseUrl}/benchmark/`);
        await page.waitForSelector('#evaluation_provider', { timeout: 15000 });
        // Providers populate synchronously on DOMContentLoaded; wait briefly for a
        // real (non-stub) option in case the inline script hasn't finished when we
        // first query. 5s is ample for a synchronous fill, so a genuine "script
        // never ran" failure surfaces fast rather than after a long timeout.
        await page.waitForFunction(
            () => {
                const sel = document.querySelector('#evaluation_provider');
                return sel && Array.from(sel.options).some(o => o.value && o.value.length > 0);
            },
            { timeout: 5000 }
        ).catch(() => {});

        const result = await page.evaluate(() => {
            const provider = document.querySelector('#evaluation_provider');
            const model = document.querySelector('#evaluation_model');
            const opts = provider ? Array.from(provider.options).map(o => o.value).filter(Boolean) : [];
            return {
                hasProvider: !!provider,
                providerOptionCount: opts.length,
                providerOptions: opts.slice(0, 6),
                // #evaluation_model is a custom-dropdown <input> (render_dropdown),
                // not a <select>; assert the type and that it lives in the form.
                modelIsInput: model?.tagName === 'INPUT',
                modelInForm: !!model?.closest('form#benchmark-form'),
            };
        });

        const passed = result.hasProvider && result.providerOptionCount > 0
            && result.modelIsInput && result.modelInForm;
        return {
            passed,
            message: passed
                ? `Benchmark config: #evaluation_provider has ${result.providerOptionCount} options (${result.providerOptions.join(', ')}) + #evaluation_model is an input inside #benchmark-form`
                : `Benchmark config incomplete (provider=${result.hasProvider}, providerOptions=${result.providerOptionCount}, modelIsInput=${result.modelIsInput}, modelInForm=${result.modelInForm})`
        };
    },

    async startBenchmarkButton(page, baseUrl) {
        // The real start control is #start-benchmark-btn: a submit button inside
        // form#benchmark-form labelled "Start Benchmark" (benchmark.html:296). The
        // old test fuzzy-matched any button whose text contained start/run/begin.
        await navigateTo(page, `${baseUrl}/benchmark/`);
        await page.waitForSelector('#start-benchmark-btn', { timeout: 15000 });

        const result = await page.evaluate(() => {
            const btn = document.querySelector('#start-benchmark-btn');
            if (!btn) return { found: false };
            return {
                found: true,
                tag: btn.tagName.toLowerCase(),
                type: btn.type,
                text: btn.textContent?.trim() || '',
                inForm: !!btn.closest('form#benchmark-form'),
            };
        });

        if (!result.found) {
            return { passed: false, message: '#start-benchmark-btn not found' };
        }
        const passed = result.tag === 'button' && result.type === 'submit'
            && /start benchmark/i.test(result.text) && result.inForm;
        return {
            passed,
            message: passed
                ? `Start button is a submit button in #benchmark-form ("${result.text}")`
                : `Start button contract failed (tag=${result.tag}, type=${result.type}, text="${result.text}", inForm=${result.inForm})`
        };
    },

    async runningBenchmarksSection(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(() => {
            const runningSection = document.querySelector(
                '[class*="running"], ' +
                '[id*="running"], ' +
                '.active-benchmarks, ' +
                '.in-progress'
            );

            const hasRunningText = document.body.textContent?.toLowerCase().includes('running') ||
                                   document.body.textContent?.toLowerCase().includes('in progress');

            return {
                hasSection: !!runningSection,
                hasRunningText
            };
        });

        if (!result.hasSection && !result.hasRunningText) {
            return { passed: null, skipped: true, message: 'No running benchmarks section found' };
        }

        return {
            passed: true,
            message: 'Running benchmarks section found'
        };
    },

    async benchmarkHistoryTable(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(() => {
            const table = document.querySelector('table, .benchmark-history, .history-list');
            if (!table) return { hasTable: false };

            const rows = table.querySelectorAll('tbody tr, .history-item');
            const headers = Array.from(table.querySelectorAll('th')).map(th => th.textContent?.toLowerCase().trim());

            return {
                hasTable: true,
                rowCount: rows.length,
                hasDateColumn: headers.some(h => h.includes('date') || h.includes('time') || h.includes('created')),
                hasStatusColumn: headers.some(h => h.includes('status') || h.includes('state')),
                headers: headers.slice(0, 6)
            };
        });

        if (!result.hasTable) {
            return { passed: null, skipped: true, message: 'No benchmark history table found' };
        }

        return {
            passed: true,
            message: `History table: ${result.rowCount} rows (columns: ${result.headers.join(', ')})`
        };
    }
};

// ============================================================================
// Benchmark Results Tests
// ============================================================================
const BenchmarkResultsTests = {
    async benchmarkResultsPageLoads(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/results`);

        const result = await page.evaluate(() => {
            return {
                hasContent: !!document.querySelector('.benchmark-results, .results-container, #results'),
                hasHeader: !!document.querySelector('h1, .page-title'),
                headerText: document.querySelector('h1, .page-title')?.textContent?.trim(),
                hasResults: document.querySelectorAll('.result-card, .benchmark-result, table').length > 0
            };
        });

        const passed = result.hasContent || result.hasHeader || result.hasResults;
        return {
            passed,
            message: passed
                ? `Benchmark results page loaded (header: "${result.headerText}")`
                : 'Benchmark results page failed to load'
        };
    },

    async resultsMetricsCards(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/results`);

        const result = await page.evaluate(() => {
            const cards = document.querySelectorAll('.metric-card, .stats-card, .card, .result-metric');
            const cardTexts = Array.from(cards).map(c => c.textContent?.toLowerCase() || '');

            return {
                cardCount: cards.length,
                hasAccuracyCard: cardTexts.some(t => t.includes('accuracy') || t.includes('correct')),
                hasSpeedCard: cardTexts.some(t => t.includes('speed') || t.includes('time') || t.includes('latency')),
                hasQualityCard: cardTexts.some(t => t.includes('quality') || t.includes('score'))
            };
        });

        if (result.cardCount === 0) {
            return { passed: null, skipped: true, message: 'No metrics cards found on results page' };
        }

        return {
            passed: true,
            message: `Results metrics: ${result.cardCount} cards (accuracy=${result.hasAccuracyCard}, speed=${result.hasSpeedCard}, quality=${result.hasQualityCard})`
        };
    },

    async comparisonCharts(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/results`);
        await delay(1000);

        const result = await page.evaluate(() => {
            const charts = document.querySelectorAll('canvas, svg, .chart, .recharts-wrapper');
            const comparisonSection = document.querySelector('[class*="comparison"], [id*="comparison"]');

            return {
                hasCharts: charts.length > 0,
                chartCount: charts.length,
                hasComparisonSection: !!comparisonSection
            };
        });

        if (!result.hasCharts && !result.hasComparisonSection) {
            return { passed: null, skipped: true, message: 'No comparison charts found' };
        }

        return {
            passed: true,
            message: `Comparison visualization: ${result.chartCount} charts found`
        };
    },

    async qualityScoreDisplay(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/results`);

        const result = await page.evaluate(() => {
            const scoreElement = document.querySelector(
                '[class*="quality-score"], ' +
                '[class*="overall-score"], ' +
                '.score-display'
            );

            // Look for percentage or score pattern
            const scorePattern = /(\d+(?:\.\d*)?)\s*(?:%|\/\s*100|points?|score)/i;
            const bodyText = document.body.textContent || '';
            const scoreMatch = bodyText.match(scorePattern);

            return {
                hasScoreElement: !!scoreElement,
                scoreText: scoreElement?.textContent?.trim(),
                foundScoreInText: !!scoreMatch,
                scoreValue: scoreMatch ? scoreMatch[1] : null
            };
        });

        if (!result.hasScoreElement && !result.foundScoreInText) {
            return { passed: null, skipped: true, message: 'No quality score display found' };
        }

        return {
            passed: true,
            message: result.hasScoreElement
                ? `Quality score displayed: "${result.scoreText}"`
                : `Score found: ${result.scoreValue}`
        };
    },

    async cancelBenchmarkButton(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(() => {
            const buttons = Array.from(document.querySelectorAll('button, a.btn'));
            const cancelBtn = buttons.find(b => {
                const text = b.textContent?.toLowerCase() || '';
                return text.includes('cancel') || text.includes('stop') || text.includes('abort');
            });

            return {
                found: !!cancelBtn,
                text: cancelBtn?.textContent?.trim(),
                disabled: cancelBtn?.disabled
            };
        });

        if (!result.found) {
            return { passed: null, skipped: true, message: 'No cancel button found (may only appear during active benchmark)' };
        }

        return {
            passed: true,
            message: `Cancel button found: "${result.text}"`
        };
    },

    async deleteBenchmarkButton(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(() => {
            const buttons = Array.from(document.querySelectorAll('button, a.btn'));
            const deleteBtn = buttons.find(b => {
                const text = b.textContent?.toLowerCase() || '';
                return text.includes('delete') || text.includes('remove');
            });

            // Also check for trash icons
            const trashIcon = document.querySelector('.fa-trash, .delete-icon, [class*="trash"]');

            return {
                found: !!deleteBtn || !!trashIcon,
                text: deleteBtn?.textContent?.trim(),
                hasIcon: !!trashIcon
            };
        });

        if (!result.found) {
            return { passed: null, skipped: true, message: 'No delete button found (may require existing benchmarks)' };
        }

        return {
            passed: true,
            message: result.text ? `Delete button found: "${result.text}"` : 'Delete icon found'
        };
    },

    async exportDropdown(page, baseUrl) {
        // The per-run Export dropdown only renders for COMPLETED runs.
        await navigateTo(page, `${baseUrl}/benchmark/`);
        await delay(1500);  // let the async history fetch render run cards

        const result = await page.evaluate(() => {
            const toggle = Array.from(document.querySelectorAll('.dropdown-toggle'))
                .find(b => (b.textContent || '').toLowerCase().includes('export'));
            if (!toggle) return { found: false };

            const menu = toggle.closest('.dropdown')?.querySelector('.dropdown-menu');
            const items = menu
                ? Array.from(menu.querySelectorAll('.dropdown-item')).map(i => (i.textContent || '').toLowerCase())
                : [];
            const itemText = items.join(' | ');
            const checkbox = menu?.querySelector('input[type="checkbox"]');
            const checkboxLabel = (checkbox?.closest('label')?.textContent || '').toLowerCase();

            return {
                found: true,
                hasSummary: itemText.includes('summary'),
                hasExamples: itemText.includes('examples'),
                hasSettingsCheckbox: !!checkbox && checkboxLabel.includes('settings'),
                // The opt-in default must be OFF so the default download is lean.
                checkboxDefaultOff: checkbox ? checkbox.checked === false : null,
            };
        });

        if (!result.found) {
            return { passed: null, skipped: true, message: 'No Export dropdown found (requires a completed benchmark run)' };
        }

        const ok = result.hasSummary && result.hasExamples
            && result.hasSettingsCheckbox && result.checkboxDefaultOff === true;
        return {
            passed: ok,
            message: ok
                ? 'Export dropdown has summary/examples items + settings-snapshot checkbox (default off)'
                : `Export dropdown incomplete: summary=${result.hasSummary} examples=${result.hasExamples} settingsCheckbox=${result.hasSettingsCheckbox} checkboxDefaultOff=${result.checkboxDefaultOff}`,
        };
    }
};

// ============================================================================
// Benchmark API Tests
// ============================================================================
const BenchmarkApiTests = {
    async benchmarkHistoryApiResponds(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(async (url) => {
            try {
                const response = await fetch(`${url}/benchmark/api/history`);
                if (!response.ok) return { ok: false, status: response.status };

                const data = await response.json();
                return {
                    ok: true,
                    status: response.status,
                    count: Array.isArray(data) ? data.length : (data.items?.length || 0)
                };
            } catch (e) {
                return { ok: false, error: e.message };
            }
        }, baseUrl);

        if (!result.ok && result.status === 404) {
            return { passed: null, skipped: true, message: 'Benchmark history API not found' };
        }

        return {
            passed: result.ok,
            message: result.ok
                ? `Benchmark history API responds (${result.count} items)`
                : `Benchmark API failed: ${result.error || 'status ' + result.status}`
        };
    },

    async benchmarkConfigsApiResponds(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(async (url) => {
            try {
                const response = await fetch(`${url}/benchmark/api/configs`);
                if (!response.ok) return { ok: false, status: response.status };

                const data = await response.json();
                return {
                    ok: true,
                    status: response.status,
                    count: Array.isArray(data) ? data.length : Object.keys(data).length
                };
            } catch (e) {
                return { ok: false, error: e.message };
            }
        }, baseUrl);

        if (!result.ok && result.status === 404) {
            return { passed: null, skipped: true, message: 'Benchmark configs API not found' };
        }

        return {
            passed: result.ok,
            message: result.ok
                ? `Benchmark configs API responds (${result.count} configs)`
                : `Configs API failed: ${result.error || 'status ' + result.status}`
        };
    },

    async benchmarkRunningApiResponds(page, baseUrl) {
        await navigateTo(page, `${baseUrl}/benchmark/`);

        const result = await page.evaluate(async (url) => {
            try {
                const response = await fetch(`${url}/benchmark/api/running`);
                if (!response.ok) return { ok: false, status: response.status };

                const data = await response.json();
                return {
                    ok: true,
                    status: response.status,
                    hasData: data !== null && data !== undefined
                };
            } catch (e) {
                return { ok: false, error: e.message };
            }
        }, baseUrl);

        if (!result.ok && result.status === 404) {
            return { passed: null, skipped: true, message: 'Benchmark running API not found' };
        }

        return {
            passed: result.ok,
            message: result.ok
                ? `Benchmark running API responds`
                : `Running API failed: ${result.error || 'status ' + result.status}`
        };
    }
};

// ============================================================================
// Main Test Runner
// ============================================================================
async function main() {
    log.section('Benchmark Tests');

    const ctx = await setupTest({ authenticate: true });
    const results = new TestResults('Benchmark Tests');
    const { page } = ctx;
    const { baseUrl } = ctx.config;

    const subTestTimeout = ctx.config.isCI ? 60000 : 30000;
    async function run(category, name, testFn) {
        try {
            const result = await withTimeout(
                testFn(page, baseUrl),
                subTestTimeout,
                `${category}/${name}`
            );
            if (result.skipped) {
                results.skip(category, name, result.message);
            } else {
                results.add(category, name, result.passed, result.message);
            }
        } catch (error) {
            results.add(category, name, false, `Error: ${error.message}`);
        }
    }

    try {
        // Benchmark Dashboard Tests
        log.section('Benchmark Dashboard');
        await run('Dashboard', 'Benchmark Page Loads', (p, u) => BenchmarkDashboardTests.benchmarkPageLoads(p, u));
        await run('Dashboard', 'Benchmark Form Structure', (p, u) => BenchmarkDashboardTests.benchmarkFormStructure(p, u));
        await run('Dashboard', 'Config Dropdowns', (p, u) => BenchmarkDashboardTests.configDropdowns(p, u));
        await run('Dashboard', 'Start Benchmark Button', (p, u) => BenchmarkDashboardTests.startBenchmarkButton(p, u));
        await run('Dashboard', 'Running Benchmarks Section', (p, u) => BenchmarkDashboardTests.runningBenchmarksSection(p, u));
        await run('Dashboard', 'Benchmark History Table', (p, u) => BenchmarkDashboardTests.benchmarkHistoryTable(p, u));

        // Benchmark Results Tests
        log.section('Benchmark Results');
        await run('Results', 'Benchmark Results Page Loads', (p, u) => BenchmarkResultsTests.benchmarkResultsPageLoads(p, u));
        await run('Results', 'Results Metrics Cards', (p, u) => BenchmarkResultsTests.resultsMetricsCards(p, u));
        await run('Results', 'Comparison Charts', (p, u) => BenchmarkResultsTests.comparisonCharts(p, u));
        await run('Results', 'Quality Score Display', (p, u) => BenchmarkResultsTests.qualityScoreDisplay(p, u));
        await run('Results', 'Cancel Benchmark Button', (p, u) => BenchmarkResultsTests.cancelBenchmarkButton(p, u));
        await run('Results', 'Delete Benchmark Button', (p, u) => BenchmarkResultsTests.deleteBenchmarkButton(p, u));
        await run('Results', 'Export Dropdown', (p, u) => BenchmarkResultsTests.exportDropdown(p, u));

        // Benchmark API Tests
        log.section('Benchmark APIs');
        await run('API', 'Benchmark History API', (p, u) => BenchmarkApiTests.benchmarkHistoryApiResponds(p, u));
        await run('API', 'Benchmark Configs API', (p, u) => BenchmarkApiTests.benchmarkConfigsApiResponds(p, u));
        await run('API', 'Benchmark Running API', (p, u) => BenchmarkApiTests.benchmarkRunningApiResponds(p, u));

    } catch (error) {
        log.error(`Fatal error: ${error.message}`);
        console.error(error.stack);
    } finally {
        results.print();
        results.save();
        await teardownTest(ctx);
        process.exit(results.exitCode());
    }
}

// Run if executed directly
if (require.main === module) {
    main().catch(error => {
        console.error('Test runner failed:', error);
        process.exit(1);
    });
}

module.exports = { BenchmarkDashboardTests, BenchmarkResultsTests, BenchmarkApiTests };