wehub-resource-sync
28 lines
719 B
TypeScript
28 lines
719 B
TypeScript
import { Actor } from 'apify';
|
|
import { CheerioCrawler } from 'crawlee';
|
|
|
|
await Actor.init();
|
|
|
|
const crawler = new CheerioCrawler({
|
|
async requestHandler({ request, $, enqueueLinks }) {
|
|
const { url } = request;
|
|
|
|
// Extract HTML title of the page.
|
|
const title = $('title').text();
|
|
console.log(`Title of ${url}: ${title}`);
|
|
|
|
// Add URLs that match the provided pattern.
|
|
await enqueueLinks({
|
|
globs: ['https://www.iana.org/*'],
|
|
});
|
|
|
|
// Save extracted data to dataset.
|
|
await Actor.pushData({ url, title });
|
|
},
|
|
});
|
|
|
|
// Enqueue the initial request and run the crawler
|
|
await crawler.run(['https://www.iana.org/']);
|
|
|
|
await Actor.exit();
|