wehub-resource-sync
54 lines
1.8 KiB
JavaScript
54 lines
1.8 KiB
JavaScript
import { Actor } from 'apify';
|
|
import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
|
|
|
|
const mainOptions = {
|
|
exit: Actor.isAtHome(),
|
|
storage:
|
|
process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
|
|
? new (await import('@apify/storage-local')).ApifyStorageLocal()
|
|
: undefined,
|
|
};
|
|
|
|
await Actor.main(async () => {
|
|
const crawler = new CheerioCrawler({
|
|
maxRequestsPerCrawl: 10,
|
|
autoscaledPoolOptions: { desiredConcurrency: 2 },
|
|
async requestHandler({ $, request }) {
|
|
const {
|
|
url,
|
|
userData: { label },
|
|
} = request;
|
|
|
|
if (label === 'START') {
|
|
const links = $('a.card')
|
|
.toArray()
|
|
.map((item) => $(item).attr('href'));
|
|
for (const link of links) {
|
|
const actorDetailUrl = `https://crawlee.dev${link}`;
|
|
await crawler.addRequests([
|
|
{
|
|
url: actorDetailUrl,
|
|
userData: { label: 'DETAIL' },
|
|
},
|
|
]);
|
|
}
|
|
} else if (label === 'DETAIL') {
|
|
const uniqueIdentifier = url.split('/').slice(-2).join('/');
|
|
const title = $('.markdown h1').text();
|
|
const firstParagraph = $('.markdown > p').text();
|
|
const modifiedDate = $('.theme-last-updated time').attr('datetime');
|
|
|
|
await Dataset.pushData({
|
|
url,
|
|
uniqueIdentifier,
|
|
title,
|
|
firstParagraph,
|
|
modifiedDate,
|
|
});
|
|
}
|
|
},
|
|
});
|
|
|
|
await crawler.run([{ url: 'https://crawlee.dev/js/docs/examples', userData: { label: 'START' } }]);
|
|
}, mainOptions);
|