import http from 'node:http';
import { Actor } from 'apify';
import { PlaywrightCrawler, Dataset } from '@crawlee/playwright';
// Self-contained fixture: a start page that uses so the
// crawler can only discover the linked pages if it honors the base href when
// resolving the otherwise-relative values.
const pages = {
'/start': `
Start
A
B
C
Elsewhere (absolute)
`,
'/sub/a': 'AA',
'/sub/b': 'BB',
'/sub/c': 'CC',
'/elsewhere': 'ElsewhereElsewhere',
};
const server = http.createServer((req, res) => {
const body = pages[req.url];
if (body === undefined) {
res.statusCode = 404;
res.end('Not Found');
return;
}
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
res.end(body);
});
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
const { port } = server.address();
const baseUrl = `http://127.0.0.1:${port}`;
const mainOptions = {
exit: Actor.isAtHome(),
storage:
process.env.STORAGE_IMPLEMENTATION === 'LOCAL'
? new (await import('@apify/storage-local')).ApifyStorageLocal()
: undefined,
};
await Actor.main(async () => {
const crawler = new PlaywrightCrawler({
maxRequestsPerCrawl: 30,
async requestHandler({ parseWithCheerio, enqueueLinks, request, log }) {
const { url, loadedUrl } = request;
const $ = await parseWithCheerio('title', 1_000);
const pageTitle = $('title').first().text();
log.info(`URL: ${url}; LOADED_URL: ${loadedUrl}; TITLE: ${pageTitle}`);
await Dataset.pushData({ url, loadedUrl, pageTitle });
// Only the /sub/ links should match — the absolute /elsewhere link
// and any unresolved relative URLs (which would land at /a, /b, /c
// without base-href handling) are filtered out by the glob.
await enqueueLinks({
globs: [`${baseUrl}/sub/**`],
});
},
});
await crawler.run([`${baseUrl}/start`]);
}, mainOptions);
server.close();