#!/usr/bin/env node /** * check-liveness.mjs — Playwright job link liveness checker * * Tests whether job posting URLs are still active or have expired. * Uses the same detection logic as scan.md step 7.5. * Zero Claude API tokens. Two rungs: a free ATS API check first * (Greenhouse/Lever — no browser), then Playwright for everything else. * * Usage: * node check-liveness.mjs [url2] ... * node check-liveness.mjs --file urls.txt * * Exit code: 0 if all active, 1 if any expired or uncertain */ import { chromium } from 'playwright'; import { readFile } from 'fs/promises'; import { checkUrlLivenessWithFallback, createHeadedPageProvider, newLivenessPage, jitteredDelayMs, sleep, } from './liveness-browser.mjs'; import { checkLivenessViaApi } from './liveness-api.mjs'; async function main() { const args = process.argv.slice(2); // Portals like pracuj.pl serve a Cloudflare anti-bot wall to headless Chromium. // On a challenge we retry once in a headed browser (which clears it); pass // --no-fallback to stay fully headless (e.g. on a machine with no display). const noFallback = args.includes('--no-fallback'); // --throttle or --throttle=: wait base..2*base ms (jittered) between checks // to stay under rate-based WAF limits. pracuj.pl's Cloudflare flags the session // after ~2 rapid hits, so a bulk run needs spacing. Default base 5000ms. const throttleArg = args.find((a) => a === '--throttle' || a.startsWith('--throttle=')); const throttleBaseMs = throttleArg ? (Number(throttleArg.split('=')[1]) || 5000) : 0; const positional = args.filter((a) => a !== '--no-fallback' && a !== throttleArg); if (positional.length === 0) { console.error('Usage: node check-liveness.mjs [--no-fallback] [--throttle[=ms]] [url2] ...'); console.error(' node check-liveness.mjs [--no-fallback] [--throttle[=ms]] --file urls.txt'); process.exit(1); } let urls; if (positional[0] === '--file') { const text = await readFile(positional[1], 'utf-8'); urls = text.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#')); } else { urls = positional; } const notes = [ noFallback ? null : 'headed fallback on challenge', throttleBaseMs ? `throttle ~${throttleBaseMs / 1000}-${(throttleBaseMs * 2) / 1000}s` : null, ].filter(Boolean); console.log(`Checking ${urls.length} URL(s)...${notes.length ? ` (${notes.join(', ')})` : ''}\n`); // Lazy browser: the API rung resolves ATS postings with no browser at all, so we // only launch Playwright if a URL actually needs the fallback. let browser = null, page = null, headed = null; async function ensureBrowser() { if (browser) return; browser = await chromium.launch({ headless: true }); page = await newLivenessPage(browser); headed = noFallback ? null : createHeadedPageProvider(chromium); } let active = 0, expired = 0, uncertain = 0, viaApi = 0; // Sequential — project rule: never Playwright in parallel for (let i = 0; i < urls.length; i++) { const url = urls[i]; let result, reason, usedBrowser = false; // Rung 1: zero-token ATS API check. A conclusive active/expired wins; otherwise fall through. const api = await checkLivenessViaApi(url); if (api) { ({ result, reason } = api); viaApi++; } else { // Rung 2: Playwright — handles non-ATS pages and inconclusive API results. await ensureBrowser(); const getHeadedPage = headed ? () => headed.get() : undefined; ({ result, reason } = await checkUrlLivenessWithFallback(page, url, { getHeadedPage })); usedBrowser = true; } const icon = { active: '✅', expired: '❌', uncertain: '⚠️' }[result]; console.log(`${icon} ${result.padEnd(10)} ${api ? '(api) ' : ' '}${url}`); if (result !== 'active') console.log(` ${reason}`); if (result === 'active') active++; else if (result === 'expired') expired++; else uncertain++; // Throttle only matters between browser checks (the API is cheap, not WAF-rate-limited). const wait = usedBrowser && i < urls.length - 1 ? jitteredDelayMs(throttleBaseMs) : 0; if (wait) await sleep(wait); } if (headed) await headed.close(); if (browser) await browser.close(); console.log(`\nResults: ${active} active ${expired} expired ${uncertain} uncertain (${viaApi} via API, no browser)`); if (expired > 0 || uncertain > 0) process.exit(1); } main().catch(err => { console.error('Fatal:', err.message); process.exit(1); });