wehub-resource-sync
28 lines
847 B
TypeScript
28 lines
847 B
TypeScript
import { RequestQueue, CheerioCrawler } from 'crawlee';
|
|
|
|
// Open the default request queue associated with the current run
|
|
const requestQueue = await RequestQueue.open();
|
|
|
|
// Enqueue the initial requests
|
|
await requestQueue.addRequests([
|
|
{ url: 'https://example.com/1' },
|
|
{ url: 'https://example.com/2' },
|
|
{ url: 'https://example.com/3' },
|
|
// ...
|
|
]);
|
|
|
|
// The crawler will automatically process requests from the queue.
|
|
// It's used the same way for Puppeteer/Playwright crawlers
|
|
const crawler = new CheerioCrawler({
|
|
requestQueue,
|
|
async requestHandler({ enqueueLinks }) {
|
|
// Add new request to the queue
|
|
await requestQueue.addRequests([{ url: 'https://example.com/new-page' }]);
|
|
// Add links found on page to the queue
|
|
await enqueueLinks();
|
|
},
|
|
});
|
|
|
|
// Run the crawler
|
|
await crawler.run();
|