wehub-resource-sync
36 lines
1.2 KiB
TypeScript
36 lines
1.2 KiB
TypeScript
import { BasicCrawler } from 'crawlee';
|
|
|
|
// Create a BasicCrawler - the simplest crawler that enables
|
|
// users to implement the crawling logic themselves.
|
|
const crawler = new BasicCrawler({
|
|
// This function will be called for each URL to crawl.
|
|
async requestHandler({ pushData, request, sendRequest, log }) {
|
|
const { url } = request;
|
|
log.info(`Processing ${url}...`);
|
|
|
|
// Fetch the page HTML via the crawlee sendRequest utility method
|
|
// By default, the method will use the current request that is being handled, so you don't have to
|
|
// provide it yourself. You can also provide a custom request if you want.
|
|
const { body } = await sendRequest();
|
|
|
|
// Store the HTML and URL to the default dataset.
|
|
await pushData({
|
|
url,
|
|
html: body,
|
|
});
|
|
},
|
|
});
|
|
|
|
// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
|
|
await crawler.addRequests([
|
|
'https://www.google.com',
|
|
'https://www.example.com',
|
|
'https://www.bing.com',
|
|
'https://www.wikipedia.com',
|
|
]);
|
|
|
|
// Run the crawler and wait for it to finish.
|
|
await crawler.run();
|
|
|
|
console.log('Crawler finished.');
|