wehub-resource-sync
40 lines
1.1 KiB
JavaScript
40 lines
1.1 KiB
JavaScript
import { Actor, LogLevel, log as Logger } from 'apify';
|
|
import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
|
|
|
|
const mainOptions = {
|
|
exit: Actor.isAtHome(),
|
|
storage:
|
|
process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
|
|
? new (await import('@apify/storage-local')).ApifyStorageLocal()
|
|
: undefined,
|
|
};
|
|
|
|
await Actor.main(async () => {
|
|
const crawler = new CheerioCrawler({
|
|
async requestHandler({ $, enqueueLinks, request, log }) {
|
|
const { url } = request;
|
|
await enqueueLinks({
|
|
globs: ['https://crawlee.dev/js/docs/**'],
|
|
});
|
|
|
|
const pageTitle = $('title').first().text();
|
|
log.info(`REQUEST ID: ${request.id} URL: ${url} TITLE: ${pageTitle}`);
|
|
|
|
await Dataset.pushData({ url, pageTitle });
|
|
},
|
|
experiments: {
|
|
requestLocking: true,
|
|
},
|
|
log: Logger.child({
|
|
prefix: 'CheerioCrawler',
|
|
// level: LogLevel.DEBUG,
|
|
}),
|
|
});
|
|
|
|
try {
|
|
await crawler.run(['https://crawlee.dev/js/docs/quick-start']);
|
|
} catch (e) {
|
|
console.error(e);
|
|
}
|
|
}, mainOptions);
|