wehub-resource-sync
27 lines
976 B
TypeScript
27 lines
976 B
TypeScript
import { PlaywrightCrawler, Dataset } from 'crawlee';
|
|
|
|
// PlaywrightCrawler crawls the web using a headless
|
|
// browser controlled by the Playwright library.
|
|
const crawler = new PlaywrightCrawler({
|
|
// Use the requestHandler to process each of the crawled pages.
|
|
async requestHandler({ request, page, enqueueLinks, log }) {
|
|
const title = await page.title();
|
|
log.info(`Title of ${request.loadedUrl} is '${title}'`);
|
|
|
|
// Save results as JSON to ./storage/datasets/default
|
|
await Dataset.pushData({ title, url: request.loadedUrl });
|
|
|
|
// Extract links from the current page
|
|
// and add them to the crawling queue.
|
|
await enqueueLinks();
|
|
},
|
|
// Uncomment this option to see the browser window.
|
|
// headless: false,
|
|
|
|
// Let's limit our crawls to make our tests shorter and safer.
|
|
maxRequestsPerCrawl: 50,
|
|
});
|
|
|
|
// Add first URL to the queue and start the crawl.
|
|
await crawler.run(['https://crawlee.dev']);
|