jackwener--opencli/clis/indeed/search.js

/**
 * Indeed keyword search.
 *
 * Drives the public `/jobs?q=&l=` listing page through a real browser
 * because Cloudflare returns 403 to bare fetches. Extraction happens
 * inside `page.evaluate` against the rendered DOM (Indeed's JSON-island
 * is heavily obfuscated and changes shape per A/B bucket).
 */

import { cli, Strategy } from '@jackwener/opencli/registry';
import { EmptyResultError, CommandExecutionError } from '@jackwener/opencli/errors';
import {
    SEARCH_COLUMNS,
    requireQuery,
    requireBoundedInt,
    requireNonNegativeInt,
    requireFromage,
    requireSort,
    buildSearchUrl,
    searchCardToRow,
} from './utils.js';

cli({
    site: 'indeed',
    name: 'search',
    access: 'read',
    description: 'Indeed keyword job search (rendered DOM via browser session, US site)',
    domain: 'www.indeed.com',
    strategy: Strategy.COOKIE,
    browser: true,
    navigateBefore: false,
    args: [
        { name: 'query', positional: true, required: true, help: 'Job keyword (title / skill / company)' },
        { name: 'location', type: 'string', default: '', help: 'Location filter (e.g. "remote", "New York, NY", "San Francisco")' },
        { name: 'fromage', type: 'string', default: '', help: 'Recency filter, days back: 1 / 3 / 7 / 14' },
        { name: 'sort', type: 'string', default: 'relevance', help: 'Sort order: relevance | date' },
        { name: 'start', type: 'int', default: 0, help: 'Pagination offset (multiple of 10, 0-based)' },
        { name: 'limit', type: 'int', default: 15, help: 'Max rows to return (1-25, capped at one page)' },
    ],
    columns: SEARCH_COLUMNS,
    func: async (page, kwargs) => {
        const query = requireQuery(kwargs.query);
        const location = String(kwargs.location ?? '').trim();
        const fromage = requireFromage(kwargs.fromage);
        const sort = requireSort(kwargs.sort);
        const start = requireNonNegativeInt(kwargs.start, 0, 'start');
        const limit = requireBoundedInt(kwargs.limit, 15, 25, 'limit');

        const url = buildSearchUrl({ query, location, fromage, sort, start });
        await page.goto(url);
        await page.wait(4);

        let cards;
        try {
            cards = await page.evaluate(`(async () => {
                const hasResults = () => !!document.querySelector('.job_seen_beacon');
                const hasEmptyState = () => {
                    const text = document.body?.innerText || '';
                    return !!document.querySelector('[data-testid="searchCountPages"], [data-testid="searchCount"], [data-testid="noResultsMessage"], [data-testid="empty-serp-result"]')
                        || /did not match any jobs|no jobs found|0 jobs/i.test(text);
                };
                let ready = hasResults() || hasEmptyState();
                for (let i = 0; i < 30; i++) {
                    if (ready) break;
                    await new Promise(r => setTimeout(r, 500));
                    ready = hasResults() || hasEmptyState();
                }
                const blocks = document.querySelectorAll('.job_seen_beacon');
                const seen = new Set();
                const out = [];
                for (const b of blocks) {
                    const titleA = b.querySelector('h2.jobTitle a, [class*="jcs-JobTitle"]');
                    const jk = titleA?.getAttribute('data-jk');
                    if (!jk || seen.has(jk)) continue;
                    seen.add(jk);
                    const tags = Array.from(b.querySelectorAll('.metadataContainer li span'))
                        .map(s => (s.textContent || '').trim())
                        .filter(Boolean);
                    out.push({
                        jk,
                        title: b.querySelector('h2.jobTitle span')?.textContent?.trim() ?? '',
                        company: b.querySelector('[data-testid="company-name"]')?.textContent?.trim() ?? '',
                        location: b.querySelector('[data-testid="text-location"]')?.textContent?.trim() ?? '',
                        salary: b.querySelector('.salary-snippet-container span')?.textContent?.trim() ?? '',
                        tags,
                    });
                }
                const blockedHeadline = document.title || '';
                const challenge = blockedHeadline.includes('Just a moment') || !!document.querySelector('[id^="cf-"]');
                return { cards: out, challenge, ready };
            })()`);
        }
        catch (e) {
            throw new CommandExecutionError(`Failed to scrape Indeed search DOM: ${e?.message ?? e}`, 'The page may not have fully loaded; try again.');
        }

        if (cards?.challenge) {
            throw new CommandExecutionError('Indeed served a Cloudflare challenge page', 'Open https://www.indeed.com in the connected browser and clear the challenge, then retry.');
        }
        if (!cards?.ready) {
            throw new CommandExecutionError('Indeed search page did not expose result or empty-state markers within 15s', 'Indeed may still be loading or the DOM shape may have changed; retry after opening Indeed in the connected browser.');
        }

        const list = Array.isArray(cards?.cards) ? cards.cards : [];
        if (list.length === 0) {
            throw new EmptyResultError('indeed search', `No Indeed jobs matched "${query}"${location ? ` in ${location}` : ''}`);
        }
        return list.slice(0, limit).map((c, i) => searchCardToRow(c, start + i + 1));
    },
});