wehub-resource-sync
32 lines
909 B
TypeScript
32 lines
909 B
TypeScript
import assert from 'node:assert';
|
|
|
|
import { Dataset, JSDOMCrawler } from '@crawlee/jsdom';
|
|
import { Actor } from 'apify';
|
|
|
|
if (process.env.STORAGE_IMPLEMENTATION === 'LOCAL') {
|
|
// @ts-ignore
|
|
await Actor.init({ storage: new (await import('@apify/storage-local')).ApifyStorageLocal() });
|
|
} else {
|
|
await Actor.init();
|
|
}
|
|
|
|
const crawler = new JSDOMCrawler();
|
|
|
|
crawler.router.addDefaultHandler(async ({ window, document, enqueueLinks, request, log }) => {
|
|
const { url } = request;
|
|
await enqueueLinks({
|
|
globs: ['https://crawlee.dev/js/docs/**'],
|
|
});
|
|
|
|
const pageTitle = window.document.title;
|
|
const { title } = document;
|
|
assert.strictEqual(pageTitle, title);
|
|
log.info(`URL: ${url} TITLE: ${pageTitle}`);
|
|
|
|
await Dataset.pushData({ url, pageTitle });
|
|
});
|
|
|
|
await crawler.run(['https://crawlee.dev/js/docs/quick-start']);
|
|
|
|
await Actor.exit({ exit: Actor.isAtHome() });
|