wehub-resource-sync
24 lines
864 B
TypeScript
24 lines
864 B
TypeScript
import { FileDownload } from 'crawlee';
|
|
|
|
// Create a FileDownload - a custom crawler instance that will download files from URLs.
|
|
const crawler = new FileDownload({
|
|
async requestHandler({ body, request, contentType, getKeyValueStore }) {
|
|
const url = new URL(request.url);
|
|
const kvs = await getKeyValueStore();
|
|
|
|
await kvs.setValue(url.pathname.replace(/\//g, '_'), body, { contentType: contentType.type });
|
|
},
|
|
});
|
|
|
|
// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
|
|
await crawler.addRequests([
|
|
'https://pdfobject.com/pdf/sample.pdf',
|
|
'https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
|
|
'https://upload.wikimedia.org/wikipedia/commons/c/c8/Example.ogg',
|
|
]);
|
|
|
|
// Run the downloader and wait for it to finish.
|
|
await crawler.run();
|
|
|
|
console.log('Crawler finished.');
|