import fs from 'node:fs'; import type { Server } from 'node:http'; import path from 'node:path'; import { setTimeout } from 'node:timers/promises'; import bodyParser from 'body-parser'; import { entries } from 'crawlee'; import type { Application } from 'express'; import express from 'express'; export const startExpressAppPromise = async (app: Application, port: number) => { return new Promise((resolve) => { const server = app.listen(port, () => resolve(server)); }); }; export const responseSamples = { json: { foo: 'bar' }, xml: '\n' + '\n' + '\n' + ' https://apify.com\n' + ' Web Scraping, Data Extraction and Automation · Apify\n' + '\n' + '', complexXml: fs.readFileSync(path.join(__dirname, 'data/complex.xml'), 'utf-8'), image: fs.readFileSync(path.join(__dirname, 'data/apify.png')), html: ` Example Domain

Example Domain

This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.

More information...

`, resources: ` `, cacheable: { html: ` Cacheable example website `, css: ` body { background-color: #f0f0f2; margin: 0; padding: 0; font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; } `, js: ` console.log('Hello world!'); `, }, htmlWithOutsideRedirect: ` Redirecting outside click me `, cloudflareBlocking: ` Just a moment...

dummypage.co

Checking if the site connection is secure

dummypage.co needs to review the security of your connection before proceeding.
`, outsideIframe: ` Outside iframe title

Outside iframe

`, insideIframe: ` In iframe

In iframe

Some content from inside of an iframe.

`, shadowRoots: `
`, }; export async function runExampleComServer(): Promise<[Server, number]> { const app = express(); app.use( bodyParser.urlencoded({ extended: true, }), ); app.use(bodyParser.json()); const special = express.Router(); const cacheable = express.Router(); // "special" pages with debugging info and responses for use in tests (() => { special.get('/getRawHeaders', (req, res) => { res.send(JSON.stringify(req.rawHeaders)); }); special.all('/getDebug', (req, res) => { res.json({ headers: req.headers, method: req.method, bodyLength: +(req.headers['content-length'] ?? 0), }); }); special.post('/mock', (req, res) => { const { headers, statusCode, error = false, body } = req.body; if (error) { throw new Error(error); } entries(headers as Record).forEach(([key, value]) => res.setHeader(key, value)); res.status(statusCode).send(body); }); special.get('/headers', (req, res) => { res.status(200).json({ headers: req.headers }); }); special.get('/invalidContentType', (_req, res) => { res.send({ some: 'json' }); }); special.post('/jsonError', (_req, res) => { res.status(500).json({ message: 'CUSTOM_ERROR' }); }); special.get('/mirror', (_req, res) => { res.send('TitleDATA'); }); special.get('/html-type', (_req, res) => { res.send(responseSamples.html); }); special.get('/json-type', (_req, res) => { res.json(responseSamples.json); }); special.get('/xml-type', (_req, res) => { res.type('application/xml'); res.send(responseSamples.xml); }); special.get('/complex-xml', (_req, res) => { res.type('application/xml'); res.send(responseSamples.complexXml); }); special.get('/image-type', (_req, res) => { res.type('image/png'); res.send(responseSamples.image); }); special.get('/timeout', async (_req, res) => { await setTimeout(32000); res.type('html').send('
TEST
'); }); special.get('/resources', async (_req, res) => { res.type('html').send(responseSamples.resources); }); special.get('/redirect', (_req, res) => { res.type('html').send(responseSamples.htmlWithOutsideRedirect); }); special.get('/redirect-outside', (req, res) => { res.redirect('https://example.com'); }); special.get('/cloudflareBlocking', async (_req, res) => { res.type('html').status(403).send(responseSamples.cloudflareBlocking); }); special.get('/outside-iframe', (_req, res) => { res.type('html').send(responseSamples.outsideIframe); }); special.get('/outside-iframe-csp', (_req, res) => { res.setHeader('Content-Security-Policy', "require-trusted-types-for 'script'"); res.type('html').send(responseSamples.outsideIframe); }); special.get('/inside-iframe', (_req, res) => { res.type('html').send(responseSamples.insideIframe); }); special.get('/shadow-root', (_req, res) => { res.type('html').send(responseSamples.shadowRoots); }); special.get('/html-entities', (_req, res) => { res.type('html').send('"<>"<>'); }); })(); // "cacheable" site with one page, scripts and stylesheets (() => { cacheable.get('/', (req, res) => { res.send(responseSamples.cacheable.html); }); cacheable.get('/style.css', (req, res) => { res.type('text/css').send(responseSamples.cacheable.css); }); cacheable.get('/script.js', (req, res) => { res.type('application/javascript').send(responseSamples.cacheable.js); }); })(); app.use('/special', special); app.use('/cacheable', cacheable); app.get('**/*', async (req, res) => { await setTimeout(50); res.send(responseSamples.html); }); const server = await startExpressAppPromise(app, 0); const { port } = server.address() as any; return [server, port]; }