wehub-resource-sync
43 lines
1.6 KiB
TypeScript
43 lines
1.6 KiB
TypeScript
// This is technically correct, but
|
|
// we need to explicitly open/use both the request queue and the request list.
|
|
// We suggest using the request queue and batch add the requests instead.
|
|
|
|
import { RequestList, RequestQueue, PuppeteerCrawler } from 'crawlee';
|
|
|
|
// Prepare the sources array with URLs to visit (it can contain millions of URLs)
|
|
const sources = [
|
|
{ url: 'http://www.example.com/page-1' },
|
|
{ url: 'http://www.example.com/page-2' },
|
|
{ url: 'http://www.example.com/page-3' },
|
|
// ...
|
|
];
|
|
|
|
// Open the request list with the initial sources array
|
|
const requestList = await RequestList.open('my-list', sources);
|
|
|
|
// Open the default request queue. It's not necessary to add any requests to the queue
|
|
const requestQueue = await RequestQueue.open();
|
|
|
|
// The crawler will automatically process requests from the list and the queue.
|
|
// It's used the same way for Cheerio/Playwright crawlers
|
|
const crawler = new PuppeteerCrawler({
|
|
requestList,
|
|
requestQueue,
|
|
// Each request from the request list is enqueued to the request queue one by one.
|
|
// At this point request with the same URL would exist in the list and the queue
|
|
async requestHandler({ crawler, enqueueLinks }) {
|
|
// Add new request to the queue
|
|
await crawler.addRequests(['http://www.example.com/new-page']);
|
|
|
|
// Add links found on page to the queue
|
|
await enqueueLinks();
|
|
|
|
// The requests above would be added to the queue (but not to the list)
|
|
// and would be processed after the request list is empty.
|
|
// No more requests could be added to the list here
|
|
},
|
|
});
|
|
|
|
// Run the crawler
|
|
await crawler.run();
|