wehub-resource-sync
53 lines
1.6 KiB
JavaScript
53 lines
1.6 KiB
JavaScript
import { BasicCrawler } from '@crawlee/basic';
|
|
import { RequestQueue } from '@crawlee/core';
|
|
import { log } from '@crawlee/core';
|
|
import { Actor } from 'apify';
|
|
import { setTimeout } from 'node:timers/promises';
|
|
|
|
log.setLevel(log.LEVELS.DEBUG);
|
|
|
|
process.env.CRAWLEE_INTERNAL_TIMEOUT = '30000';
|
|
|
|
const mainOptions = {
|
|
exit: Actor.isAtHome(),
|
|
storage:
|
|
process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
|
|
? new (await import('@apify/storage-local')).ApifyStorageLocal()
|
|
: undefined,
|
|
};
|
|
|
|
await Actor.main(async () => {
|
|
const input = await Actor.getInputOrThrow();
|
|
|
|
log.info('Starting the crawler', input);
|
|
|
|
const requestQueue = await RequestQueue.open(input.queueId);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
async requestHandler({ request, addRequests, pushData }) {
|
|
await setTimeout(Math.random() * 200); // Simulate some work
|
|
|
|
const url = new URL(request.url);
|
|
const index = Number(url.pathname.slice(1));
|
|
const newUrls = [];
|
|
|
|
for (const newIndex of [index + 1, index + 2, 2 * index]) {
|
|
if (newIndex <= input.limit) {
|
|
url.pathname = `/${newIndex + 1}`;
|
|
newUrls.push(url.toString());
|
|
}
|
|
}
|
|
|
|
log.info(`Enqueueing ${newUrls.length} new urls`);
|
|
if (newUrls.length > 0) {
|
|
await addRequests(newUrls);
|
|
}
|
|
|
|
await pushData({ url });
|
|
},
|
|
});
|
|
|
|
await crawler.run(['https://example.tld/1']);
|
|
}, mainOptions);
|