wehub-resource-sync
26 lines
825 B
TypeScript
26 lines
825 B
TypeScript
import { CheerioCrawler } from 'crawlee';
|
|
|
|
// The crawler will automatically process requests from the queue.
|
|
// It's used the same way for Puppeteer/Playwright crawlers.
|
|
const crawler = new CheerioCrawler({
|
|
// Note that we're not specifying the requestQueue here
|
|
async requestHandler({ crawler, enqueueLinks }) {
|
|
// Add new request to the queue
|
|
await crawler.addRequests([{ url: 'https://example.com/new-page' }]);
|
|
// Add links found on page to the queue
|
|
await enqueueLinks();
|
|
},
|
|
});
|
|
|
|
// Add the initial requests.
|
|
// Note that we are not opening the request queue explicitly before
|
|
await crawler.addRequests([
|
|
{ url: 'https://example.com/1' },
|
|
{ url: 'https://example.com/2' },
|
|
{ url: 'https://example.com/3' },
|
|
// ...
|
|
]);
|
|
|
|
// Run the crawler
|
|
await crawler.run();
|