import { readFile, rm } from 'node:fs/promises';
import type { Server } from 'node:http';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
import type {
CrawlingContext,
EnqueueLinksOptions,
ErrorHandler,
RequestHandler,
RequestOptions,
Source,
} from '@crawlee/basic';
import {
BasicCrawler,
Configuration,
CriticalError,
EventType,
KeyValueStore,
MissingRouteError,
NonRetryableError,
Request,
RequestList,
RequestQueue,
} from '@crawlee/basic';
import { RequestState } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
import { RobotsTxtFile, sleep } from '@crawlee/utils';
import express from 'express';
import type { SetRequired } from 'type-fest';
import type { Mock } from 'vitest';
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';
import log from '@apify/log';
import { startExpressAppPromise } from '../../shared/_helper';
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';
describe('BasicCrawler', () => {
let logLevel: number;
const localStorageEmulator = new MemoryStorageEmulator();
const events = Configuration.getEventManager();
const HOSTNAME = '127.0.0.1';
let port: number;
let server: Server;
beforeAll(async () => {
const app = express();
app.get('/', (req, res) => {
res.send(`Example Domain`);
});
server = await startExpressAppPromise(app, 0);
port = (server.address() as AddressInfo).port;
});
beforeAll(async () => {
logLevel = log.getLevel();
log.setLevel(log.LEVELS.OFF);
});
beforeEach(async () => {
vitest.clearAllMocks();
await localStorageEmulator.init();
});
afterAll(async () => {
await localStorageEmulator.destroy();
});
afterAll(async () => {
log.setLevel(logLevel);
});
afterAll(() => {
server.close();
});
test('does not leak sigint events', async () => {
let count = 0;
const crawler = new BasicCrawler({
requestHandler: () => {
count = process.listenerCount('SIGINT');
},
});
await crawler.run(['https://example.com']);
expect(process.listenerCount('SIGINT')).toBe(count - 1);
});
test('should run in parallel thru all the requests', async () => {
const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index}` }));
const sourcesCopy = JSON.parse(JSON.stringify(sources));
const processed: { url: string }[] = [];
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async ({ request }) => {
await sleep(10);
processed.push({ url: request.url });
};
const basicCrawler = new BasicCrawler({
requestList,
minConcurrency: 25,
maxConcurrency: 25,
requestHandler,
});
await basicCrawler.run();
expect(basicCrawler.autoscaledPool!.minConcurrency).toBe(25);
expect(processed).toEqual(sourcesCopy);
expect(await requestList.isFinished()).toBe(true);
expect(await requestList.isEmpty()).toBe(true);
});
test('should allow using run method multiple times', async () => {
const sources = [...Array(100).keys()].map((index) => `https://example.com/${index}`);
const sourcesCopy = JSON.parse(JSON.stringify(sources));
const processed: { url: string }[] = [];
const requestHandler: RequestHandler = async ({ request }) => {
await sleep(10);
processed.push({ url: request.url });
};
const basicCrawler = new BasicCrawler({
minConcurrency: 25,
maxConcurrency: 25,
requestHandler,
});
await basicCrawler.run(sources);
await basicCrawler.run(sources);
await basicCrawler.run(sources);
expect(processed).toHaveLength(sourcesCopy.length * 3);
});
test('should process 4 requests total when calling run() twice with maxRequestsPerCrawl: 2', async () => {
const processed: { url: string }[] = [];
const requestHandler: RequestHandler = async ({ request }) => {
await sleep(10);
processed.push({ url: request.url });
};
const crawler = new BasicCrawler({
maxRequestsPerCrawl: 2,
minConcurrency: 1,
maxConcurrency: 1,
requestHandler,
});
// First run should process 2 requests
await crawler.run([...Array(5).keys()].map((index) => `https://example.com/first/${index}`));
expect(processed).toHaveLength(2);
// Make sure no extra requests were enqueued
expect(await localStorageEmulator.getRequestQueueItems()).toEqual([]);
// Second run should process 2 more requests
await crawler.run([...Array(5).keys()].map((index) => `https://example.com/second/${index}`));
expect(processed).toHaveLength(4);
// Make sure no extra requests were enqueued
expect(await localStorageEmulator.getRequestQueueItems()).toEqual([]);
const processedUrls = processed.map((p) => p.url);
expect(processedUrls).toEqual([
'https://example.com/first/0',
'https://example.com/first/1',
'https://example.com/second/0',
'https://example.com/second/1',
]);
});
test('addRequests should respect maxCrawlDepth', async () => {
const processedUrls: string[] = [];
const requestHandler: RequestHandler = async ({ request, addRequests }) => {
processedUrls.push(request.url);
const url = new URL(request.url);
url.pathname = `${url.pathname}deep/`;
await addRequests([url.toString()]);
};
const crawler = new BasicCrawler({
maxCrawlDepth: 2,
maxRequestsPerCrawl: 10, // safeguard against infinite loops
requestHandler,
});
await crawler.run(['https://example.com/']);
expect(processedUrls).toEqual([
'https://example.com/',
'https://example.com/deep/',
'https://example.com/deep/deep/',
]);
});
test('enqueueLinks should respect maxCrawlDepth', async () => {
const processedUrls: string[] = [];
const requestHandler: RequestHandler = async ({ request, enqueueLinks }) => {
processedUrls.push(request.url);
const url = new URL(request.url);
url.pathname = `${url.pathname}deep/`;
await enqueueLinks({
urls: [url.toString()],
});
};
const crawler = new BasicCrawler({
maxCrawlDepth: 2,
maxRequestsPerCrawl: 10, // safeguard against infinite loops
requestHandler,
});
await crawler.run(['https://example.com/']);
expect(processedUrls).toEqual([
'https://example.com/',
'https://example.com/deep/',
'https://example.com/deep/deep/',
]);
});
describe('enqueueLinksWithCrawlDepth()', () => {
let onSkippedRequestMock: Mock;
let addRequestsBatchedMock: Mock;
let options: SetRequired;
let request: Request;
let requestQueue: RequestQueue;
type EnqueueLinksWrapperOptions = Parameters;
class TestCrawler extends BasicCrawler {
public exposedEnqueueLinksWithCrawlDepth(...enqueueLinksOptions: EnqueueLinksWrapperOptions) {
return this.enqueueLinksWithCrawlDepth(...enqueueLinksOptions);
}
}
const crawler = new TestCrawler({ maxCrawlDepth: 3 });
beforeEach(() => {
addRequestsBatchedMock = vi.fn().mockImplementation(async () => ({
addedRequests: [],
waitForAllRequestsToBeAdded: Promise.resolve([]),
requestsOverLimit: [],
}));
onSkippedRequestMock = vi.fn();
options = {
urls: ['https://example.com/1/', 'https://example.com/2/'],
onSkippedRequest: onSkippedRequestMock,
};
request = new Request({ url: 'https://example.com/', crawlDepth: 2 });
requestQueue = {
addRequestsBatched: addRequestsBatchedMock as RequestQueue['addRequestsBatched'],
} as RequestQueue;
});
it('should generate requests with maxCrawlDepth', async () => {
await crawler.exposedEnqueueLinksWithCrawlDepth(options, request, requestQueue);
const requests = addRequestsBatchedMock.mock.calls[0][0];
expect(requests).toHaveLength(2);
expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
expect(requests[1]).toMatchObject({ url: 'https://example.com/2/', crawlDepth: 3 });
expect(onSkippedRequestMock).not.toBeCalled();
});
it('should skip requests with crawlDepth exceeding maxCrawlDepth', async () => {
const requestWithMaxDepth = new Request({ url: 'https://example.com/', crawlDepth: 3 });
await crawler.exposedEnqueueLinksWithCrawlDepth(options, requestWithMaxDepth, requestQueue);
const requests = addRequestsBatchedMock.mock.calls[0][0];
expect(requests).toHaveLength(0);
const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);
expect(skippedRequests).toHaveLength(2);
expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'depth' });
expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'depth' });
});
it('should respect user provided transformRequestFunction', async () => {
const transformRequestFunction = vi.fn((req: RequestOptions) => req);
const optionsWithTransform = { ...options, transformRequestFunction };
await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, request, requestQueue);
expect(transformRequestFunction).toHaveBeenCalled();
const requests = addRequestsBatchedMock.mock.calls[0][0];
expect(requests).toHaveLength(2);
expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
});
});
it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => {
type AddCrawlDepthWrapperOptions = Parameters;
class TestCrawler extends BasicCrawler {
public exposedAddCrawlDepthRequestGenerator(...enqueueLinksOptions: AddCrawlDepthWrapperOptions) {
return this.addCrawlDepthRequestGenerator(...enqueueLinksOptions);
}
}
const crawler = new TestCrawler();
const requests = ['https://example.com/1/', { url: 'https://example.com/2/' }];
const newCrawlDepth = 4;
const addRequestsGenerator = crawler.exposedAddCrawlDepthRequestGenerator(requests, newCrawlDepth);
const generatedRequests: Source[] = [];
for await (const generatedRequest of addRequestsGenerator) {
generatedRequests.push(generatedRequest);
}
expect(generatedRequests).toHaveLength(2);
expect(generatedRequests[0].url).toBe('https://example.com/1/');
expect(generatedRequests[0].crawlDepth).toBe(4);
expect(generatedRequests[1].url).toBe('https://example.com/2/');
expect(generatedRequests[1].crawlDepth).toBe(4);
});
test('should correctly combine shorthand and full length options', async () => {
const shorthandOptions = {
minConcurrency: 123,
maxConcurrency: 456,
maxRequestsPerMinute: 789,
};
const autoscaledPoolOptions = {
minConcurrency: 16,
maxConcurrency: 32,
maxTasksPerMinute: 64,
};
const collectResults = (crawler: BasicCrawler): typeof shorthandOptions | typeof autoscaledPoolOptions => {
return {
minConcurrency: crawler.autoscaledPool!.minConcurrency,
maxConcurrency: crawler.autoscaledPool!.maxConcurrency,
// eslint-disable-next-line dot-notation -- accessing a private member
maxRequestsPerMinute: crawler.autoscaledPool!['maxTasksPerMinute'],
// eslint-disable-next-line dot-notation
maxTasksPerMinute: crawler.autoscaledPool!['maxTasksPerMinute'],
};
};
const requestList = await RequestList.open(null, []);
const requestHandler = async () => {};
const results = await Promise.all(
[
new BasicCrawler({
requestList,
requestHandler,
...shorthandOptions,
}),
new BasicCrawler({
requestList,
requestHandler,
autoscaledPoolOptions,
}),
new BasicCrawler({
requestList,
requestHandler,
...shorthandOptions,
autoscaledPoolOptions,
}),
].map(async (c) => {
await c.run();
return collectResults(c);
}),
);
expect(results[0]).toEqual(expect.objectContaining(shorthandOptions));
expect(results[1]).toEqual(expect.objectContaining(autoscaledPoolOptions));
expect(results[2]).toEqual(expect.objectContaining(shorthandOptions));
});
test('auto-saved state object', async () => {
const sources = [...Array(50).keys()].map((index) => ({ url: `https://example.com/${index}` }));
const sourcesCopy = JSON.parse(JSON.stringify(sources));
const processed: { url: string }[] = [];
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async ({ request, crawler }) => {
await sleep(10);
const state = await crawler.useState({ processed });
state.processed.push({ url: request.url });
};
const basicCrawler = new BasicCrawler({
requestList,
requestHandler,
});
await basicCrawler.run();
const state = await basicCrawler.useState();
expect(processed).toEqual(sourcesCopy);
expect(state.processed).toEqual(sourcesCopy);
expect(state.processed).toBe(processed);
expect(state.processed).toEqual(sourcesCopy);
expect(await requestList.isFinished()).toBe(true);
expect(await requestList.isEmpty()).toBe(true);
});
test.each([
EventType.MIGRATING,
EventType.ABORTING,
])('should pause on %s event and persist RequestList state', async (event) => {
const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index + 1}` }));
let persistResolve!: (value?: unknown) => void;
const persistPromise = new Promise((res) => {
persistResolve = res;
});
// Mock the calls to persist sources.
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
getValueSpy.mockResolvedValue(null);
const processed: { url: string }[] = [];
const requestList = await RequestList.open('reqList', sources);
const requestHandler: RequestHandler = async ({ request }) => {
if (request.url.endsWith('200')) events.emit(event);
processed.push({ url: request.url });
};
const basicCrawler = new BasicCrawler({
requestList,
minConcurrency: 25,
maxConcurrency: 25,
requestHandler,
});
let finished = false;
// Mock the call to persist state.
setValueSpy.mockImplementationOnce(persistResolve as any);
// The crawler will pause after 200 requests
const runPromise = basicCrawler.run();
void runPromise.then(() => {
finished = true;
});
// need to monkeypatch the stats class, otherwise it will never finish
basicCrawler.stats.persistState = async () => Promise.resolve();
await persistPromise;
expect(finished).toBe(false);
expect(await requestList.isFinished()).toBe(false);
expect(await requestList.isEmpty()).toBe(false);
expect(processed.length).toBe(200);
expect(getValueSpy).toBeCalled();
expect(setValueSpy).toBeCalled();
// clean up
// @ts-expect-error Accessing private method
await basicCrawler.autoscaledPool!._destroy();
});
test('should retry failed requests', async () => {
const sources = [
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
{ url: 'http://example.com/3' },
];
const processed: Dictionary = {};
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async ({ request }) => {
await sleep(10);
processed[request.url] = request;
if (request.url === 'http://example.com/2') {
throw Error(`This is ${request.retryCount}th error!`);
}
request.userData.foo = 'bar';
};
const basicCrawler = new BasicCrawler({
requestList,
maxRequestRetries: 10,
minConcurrency: 3,
maxConcurrency: 3,
requestHandler,
});
await basicCrawler.run();
expect(processed['http://example.com/1'].userData.foo).toBe('bar');
expect(processed['http://example.com/1'].errorMessages).toEqual([]);
expect(processed['http://example.com/1'].retryCount).toBe(0);
expect(processed['http://example.com/3'].userData.foo).toBe('bar');
expect(processed['http://example.com/3'].errorMessages).toEqual([]);
expect(processed['http://example.com/3'].retryCount).toBe(0);
expect(processed['http://example.com/2'].userData.foo).toBeUndefined();
expect(processed['http://example.com/2'].errorMessages).toHaveLength(11);
expect(processed['http://example.com/2'].retryCount).toBe(10);
expect(await requestList.isFinished()).toBe(true);
expect(await requestList.isEmpty()).toBe(true);
});
test('should retry failed requests based on `request.maxRetries`', async () => {
const sources = [
{ url: 'http://example.com/1', maxRetries: 10 },
{ url: 'http://example.com/2', maxRetries: 5 },
{ url: 'http://example.com/3', maxRetries: 1 },
];
const processed: Dictionary = {};
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async ({ request }) => {
await sleep(10);
processed[request.url] = request;
throw Error(`This is ${request.retryCount}th error!`);
};
const basicCrawler = new BasicCrawler({
requestList,
minConcurrency: 3,
maxConcurrency: 3,
requestHandler,
});
await basicCrawler.run();
expect(processed['http://example.com/1'].errorMessages).toHaveLength(11);
expect(processed['http://example.com/1'].retryCount).toBe(10);
expect(processed['http://example.com/2'].errorMessages).toHaveLength(6);
expect(processed['http://example.com/2'].retryCount).toBe(5);
expect(processed['http://example.com/3'].errorMessages).toHaveLength(2);
expect(processed['http://example.com/3'].retryCount).toBe(1);
expect(await requestList.isFinished()).toBe(true);
expect(await requestList.isEmpty()).toBe(true);
});
test('should not retry requests with noRetry set to true ', async () => {
const noRetryRequest = new Request({ url: 'http://example.com/3' });
noRetryRequest.noRetry = true;
const sources = [
{ url: 'http://example.com/1', noRetry: true },
{ url: 'http://example.com/2' },
noRetryRequest,
];
const processed: Dictionary = {};
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async ({ request }) => {
await sleep(10);
processed[request.url] = request;
request.userData.foo = 'bar';
throw Error(`This is ${request.retryCount}th error!`);
};
let failedRequestHandlerCalls = 0;
const failedRequestHandler = async () => {
failedRequestHandlerCalls++;
};
const basicCrawler = new BasicCrawler({
requestList,
maxRequestRetries: 10,
minConcurrency: 3,
maxConcurrency: 3,
requestHandler,
failedRequestHandler,
});
await basicCrawler.run();
expect(processed['http://example.com/1'].userData.foo).toBe('bar');
expect(processed['http://example.com/1'].errorMessages).toHaveLength(1);
expect(processed['http://example.com/1'].retryCount).toBe(0);
expect(processed['http://example.com/3'].userData.foo).toBe('bar');
expect(processed['http://example.com/3'].errorMessages).toHaveLength(1);
expect(processed['http://example.com/3'].retryCount).toBe(0);
expect(processed['http://example.com/2'].userData.foo).toBe('bar');
expect(processed['http://example.com/2'].errorMessages).toHaveLength(11);
expect(processed['http://example.com/2'].retryCount).toBe(10);
expect(failedRequestHandlerCalls).toBe(3);
expect(await requestList.isFinished()).toBe(true);
expect(await requestList.isEmpty()).toBe(true);
});
test('should correctly track request.state', async () => {
const sources = [{ url: 'http://example.com/1' }];
const requestList = await RequestList.open(null, sources);
const requestStates: RequestState[] = [];
const requestHandler: RequestHandler = async ({ request }) => {
requestStates.push(request.state);
throw new Error('Error');
};
const errorHandler: RequestHandler = async ({ request }) => {
requestStates.push(request.state);
};
const basicCrawler = new BasicCrawler({
requestList,
maxConcurrency: 1,
maxRequestRetries: 1,
requestHandler,
errorHandler,
});
await basicCrawler.run();
expect(requestStates).toEqual([
RequestState.REQUEST_HANDLER,
RequestState.ERROR_HANDLER,
RequestState.REQUEST_HANDLER,
]);
});
test('should use errorHandler', async () => {
const sources = [{ url: 'http://example.com/', label: 'start' }];
let errorHandlerCalls = 0;
let failedRequestHandlerCalls = 0;
const failed: Dictionary<{ request: Request; error: Error }> = {};
const requestList = await RequestList.open({ sources });
const requestHandler: RequestHandler = async ({ request }) => {
expect(request.label).toBe(errorHandlerCalls === 0 ? 'start' : `error_${errorHandlerCalls}`);
throw new Error(`This is an error ${errorHandlerCalls}`);
};
const errorHandler: ErrorHandler = async ({ request }, error) => {
expect(error.message).toBe(`This is an error ${errorHandlerCalls}`);
errorHandlerCalls++;
request.label = `error_${errorHandlerCalls}`;
};
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
failed[request.url] = { request, error };
failedRequestHandlerCalls++;
};
const basicCrawler = new BasicCrawler({
requestList,
requestHandler,
errorHandler,
failedRequestHandler,
});
await basicCrawler.run();
expect(errorHandlerCalls).toBe(3);
expect(failedRequestHandlerCalls).toBe(1);
expect(Object.values(failed)).toHaveLength(1);
expect(failed['http://example.com/'].request.label).not.toBe('start');
expect(failed['http://example.com/'].request.label).toBe('error_3');
expect(failed['http://example.com/'].error.message).toEqual('This is an error 3');
});
test('should allow to handle failed requests', async () => {
const sources = [
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
{ url: 'http://example.com/3' },
];
const processed: Dictionary = {};
const failed: Dictionary = {};
const errors: Error[] = [];
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async ({ request }) => {
await Promise.reject(new Error('some-error'));
processed[request.url] = request;
};
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
failed[request.url] = request;
errors.push(error);
};
const basicCrawler = new BasicCrawler({
requestList,
requestHandler,
failedRequestHandler,
});
await basicCrawler.run();
expect(failed['http://example.com/1'].errorMessages).toHaveLength(4);
expect(failed['http://example.com/1'].retryCount).toBe(3);
expect(failed['http://example.com/2'].errorMessages).toHaveLength(4);
expect(failed['http://example.com/2'].retryCount).toBe(3);
expect(failed['http://example.com/3'].errorMessages).toHaveLength(4);
expect(failed['http://example.com/3'].retryCount).toBe(3);
expect(Object.values(failed)).toHaveLength(3);
expect(Object.values(processed)).toHaveLength(0);
expect(await requestList.isFinished()).toBe(true);
expect(await requestList.isEmpty()).toBe(true);
errors.forEach((error) => expect(error).toBeInstanceOf(Error));
});
test('should not retry on NonRetryableError', async () => {
const sources = [
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
{ url: 'http://example.com/3' },
];
const failed: Dictionary = {};
const errors: Error[] = [];
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async () => {
throw new NonRetryableError('some-error');
};
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
failed[request.url] = request;
errors.push(error);
};
const basicCrawler = new BasicCrawler({
requestList,
requestHandler,
failedRequestHandler,
});
await basicCrawler.run();
expect(failed['http://example.com/1'].errorMessages).toHaveLength(1);
expect(failed['http://example.com/1'].retryCount).toBe(0);
expect(failed['http://example.com/2'].errorMessages).toHaveLength(1);
expect(failed['http://example.com/2'].retryCount).toBe(0);
expect(failed['http://example.com/3'].errorMessages).toHaveLength(1);
expect(failed['http://example.com/3'].retryCount).toBe(0);
expect(Object.values(failed)).toHaveLength(3);
expect(await requestList.isFinished()).toBe(true);
expect(await requestList.isEmpty()).toBe(true);
errors.forEach((error) => expect(error).toBeInstanceOf(NonRetryableError));
});
test('noRetry after calling errorHandler', async () => {
const sources = [{ url: `http://example.com` }];
const requestList = await RequestList.open(null, sources);
let request: Request>;
const crawler = new BasicCrawler({
requestList,
errorHandler: (context, error) => {
request = context.request;
context.request.noRetry = true;
},
maxRequestRetries: 3,
requestHandler: () => {
throw new Error('Failure');
},
});
await crawler.run();
expect(request!.retryCount).toBe(0);
});
test('should crash on CriticalError', async () => {
const sources = [
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
{ url: 'http://example.com/3' },
];
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async () => {
throw new CriticalError('some-error');
};
const failedRequestHandler = vitest.fn() as ErrorHandler;
const basicCrawler = new BasicCrawler({
requestList,
requestHandler,
failedRequestHandler,
});
await expect(basicCrawler.run()).rejects.toThrow(CriticalError);
expect(failedRequestHandler).not.toBeCalled();
expect(await requestList.isFinished()).toBe(false);
});
test('should crash on MissingRouteError', async () => {
const sources = [
{ url: 'http://example.com/1', label: 'TEST' }, // will match
{ url: 'http://example.com/2', label: 'FOO' }, // will fail as no FOO route or default route exists
{ url: 'http://example.com/3' }, // will fail as no default route exists
];
const requestList = await RequestList.open(null, sources);
const failedRequestHandler = vitest.fn() as ErrorHandler;
const basicCrawler = new BasicCrawler({
requestList,
failedRequestHandler,
});
const testRoute = vitest.fn();
basicCrawler.router.addHandler('TEST', testRoute);
await expect(basicCrawler.run()).rejects.toThrow(MissingRouteError);
expect(failedRequestHandler).not.toBeCalled();
expect(testRoute).toBeCalled();
expect(await requestList.isFinished()).toBe(false);
});
test('should correctly combine RequestList and RequestQueue', async () => {
const sources = [
{ url: 'http://example.com/0' },
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
];
const processed: Dictionary = {};
const requestList = await RequestList.open(null, sources);
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
const requestHandler: RequestHandler = async ({ request }) => {
await sleep(10);
processed[request.url] = request;
if (request.url === 'http://example.com/1') {
throw Error(`This is ${request.retryCount}th error!`);
}
request.userData.foo = 'bar';
};
const basicCrawler = new BasicCrawler({
requestList,
requestQueue,
maxRequestRetries: 3,
minConcurrency: 1,
maxConcurrency: 1,
requestHandler,
});
vitest.spyOn(requestQueue, 'handledCount').mockResolvedValueOnce(0);
vitest
.spyOn(requestQueue, 'addRequest')
.mockResolvedValueOnce({ requestId: 'id-0' } as any)
.mockResolvedValueOnce({ requestId: 'id-1' } as any)
.mockResolvedValueOnce({ requestId: 'id-2' } as any);
const request0 = new Request({ id: 'id-0', ...sources[0] });
const request1 = new Request({ id: 'id-1', ...sources[1] });
const request2 = new Request({ id: 'id-2', ...sources[2] });
const queueContent = [request0, request1, request2, request1, request1, request1];
vitest.spyOn(requestQueue, 'fetchNextRequest').mockImplementation(async () => queueContent.shift() ?? null);
const markReqHandled = vitest
.spyOn(requestQueue, 'markRequestHandled')
.mockReturnValue(Promise.resolve() as any);
const reclaimReq = vitest.spyOn(requestQueue, 'reclaimRequest').mockReturnValue(Promise.resolve() as any);
vitest.spyOn(requestQueue, 'isEmpty').mockImplementation(async () => queueContent.length <= 0);
vitest.spyOn(requestQueue, 'isFinished').mockResolvedValueOnce(true);
await basicCrawler.run();
// 1st try
expect(reclaimReq).toBeCalledWith(request1, expect.objectContaining({}));
expect(reclaimReq).toBeCalledTimes(3);
expect(processed['http://example.com/0'].userData.foo).toBe('bar');
expect(processed['http://example.com/0'].errorMessages).toEqual([]);
expect(processed['http://example.com/0'].retryCount).toBe(0);
expect(processed['http://example.com/2'].userData.foo).toBe('bar');
expect(processed['http://example.com/2'].errorMessages).toEqual([]);
expect(processed['http://example.com/2'].retryCount).toBe(0);
expect(processed['http://example.com/1'].userData.foo).toBeUndefined();
expect(processed['http://example.com/1'].errorMessages).toHaveLength(4);
expect(processed['http://example.com/1'].retryCount).toBe(3);
expect(await requestList.isFinished()).toBe(true);
expect(await requestList.isEmpty()).toBe(true);
vitest.restoreAllMocks();
});
test('should say that task is not ready requestList is not set and requestQueue is empty', async () => {
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
requestQueue.isEmpty = async () => Promise.resolve(true);
const crawler = new BasicCrawler({
requestQueue,
requestHandler: async () => {},
});
// @ts-expect-error Accessing private prop
expect(await crawler._isTaskReadyFunction()).toBe(false);
});
test('should be possible to override isFinishedFunction and isTaskReadyFunction of underlying AutoscaledPool', async () => {
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
const processed: Request[] = [];
const queue: Request[] = [];
let isFinished = false;
let isFinishedFunctionCalled = false;
let isTaskReadyFunctionCalled = false;
const basicCrawler = new BasicCrawler({
requestQueue,
autoscaledPoolOptions: {
minConcurrency: 1,
maxConcurrency: 1,
isFinishedFunction: async () => {
isFinishedFunctionCalled = true;
return Promise.resolve(isFinished);
},
isTaskReadyFunction: async () => {
isTaskReadyFunctionCalled = true;
return Promise.resolve(!isFinished);
},
},
requestHandler: async ({ request }) => {
await sleep(10);
processed.push(request);
},
});
// Speed up the test
// @ts-expect-error Accessing private prop
basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;
const request0 = new Request({ url: 'http://example.com/0' });
const request1 = new Request({ url: 'http://example.com/1' });
vitest.spyOn(requestQueue, 'handledCount').mockReturnValue(Promise.resolve() as any);
let handledCount = 0;
const markRequestHandled = vitest.spyOn(requestQueue, 'markRequestHandled').mockImplementation(async () => {
handledCount++;
// Only set isFinished after both requests have been handled
if (handledCount >= 2) {
// Small delay to ensure the test can verify everything
setTimeout(() => {
isFinished = true;
}, 50);
}
return Promise.resolve() as any;
});
const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished');
requestQueue.fetchNextRequest = async () => queue.pop()!;
requestQueue.isEmpty = async () => Promise.resolve(!queue.length);
// Add requests with buffer time for crawler startup.
// Use longer delays to avoid flakiness under CPU load from parallel tests.
setTimeout(() => queue.push(request0), 500);
setTimeout(() => queue.push(request1), 1000);
await basicCrawler.run();
expect(markRequestHandled).toBeCalledWith(request0);
expect(markRequestHandled).toBeCalledWith(request1);
expect(isFinishedOrig).not.toBeCalled();
expect(isFinishedFunctionCalled).toBe(true);
expect(isTaskReadyFunctionCalled).toBe(true);
// TODO: see why the request1 was passed as a second parameter to includes
expect(processed.includes(request0)).toBe(true);
vitest.restoreAllMocks();
});
test('keepAlive', async () => {
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
const processed: Request[] = [];
const queue: Request[] = [];
const basicCrawler = new BasicCrawler({
requestQueue,
keepAlive: true,
requestHandler: async ({ request }) => {
await sleep(10);
processed.push(request);
},
});
// Speed up the test
// @ts-expect-error Accessing private prop
basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;
const request0 = new Request({ url: 'http://example.com/0' });
const request1 = new Request({ url: 'http://example.com/1' });
vitest.spyOn(requestQueue, 'handledCount').mockReturnValue(Promise.resolve() as any);
const markRequestHandled = vitest
.spyOn(requestQueue, 'markRequestHandled')
.mockReturnValue(Promise.resolve() as any);
const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished');
requestQueue.fetchNextRequest = async () => Promise.resolve(queue.pop()!);
requestQueue.isEmpty = async () => Promise.resolve(!queue.length);
// Use longer delays to avoid flakiness under CPU load from parallel tests.
setTimeout(() => queue.push(request0), 500);
setTimeout(() => queue.push(request1), 1000);
setTimeout(() => {
void basicCrawler.teardown();
}, 3000);
await basicCrawler.run();
expect(markRequestHandled).toBeCalledWith(request0);
expect(markRequestHandled).toBeCalledWith(request1);
expect(isFinishedOrig).not.toBeCalled();
// TODO: see why the request1 was passed as a second parameter to includes
expect(processed.includes(request0)).toBe(true);
vitest.restoreAllMocks();
});
test('should support maxRequestsPerCrawl parameter', async () => {
const sources = [
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
{ url: 'http://example.com/3' },
{ url: 'http://example.com/4' },
{ url: 'http://example.com/5' },
];
const processed: Dictionary = {};
const requestList = await RequestList.open(null, sources);
const requestHandler: RequestHandler = async ({ request }) => {
await sleep(10);
processed[request.url] = request;
if (request.url === 'http://example.com/2') throw Error();
request.userData.foo = 'bar';
};
let failedRequestHandlerCalls = 0;
const failedRequestHandler = async () => {
failedRequestHandlerCalls++;
};
const basicCrawler = new BasicCrawler({
requestList,
maxRequestRetries: 3,
maxRequestsPerCrawl: 3,
maxConcurrency: 1,
requestHandler,
failedRequestHandler,
});
await basicCrawler.run();
expect(processed['http://example.com/1'].userData.foo).toBe('bar');
expect(processed['http://example.com/1'].errorMessages).toEqual([]);
expect(processed['http://example.com/1'].retryCount).toBe(0);
expect(processed['http://example.com/3'].userData.foo).toBe('bar');
expect(processed['http://example.com/3'].errorMessages).toEqual([]);
expect(processed['http://example.com/3'].retryCount).toBe(0);
expect(processed['http://example.com/2'].userData.foo).toEqual(undefined);
expect(processed['http://example.com/2'].errorMessages).toHaveLength(4);
expect(processed['http://example.com/2'].retryCount).toBe(3);
expect(failedRequestHandlerCalls).toBe(1);
expect(await requestList.isFinished()).toBe(false);
expect(await requestList.isEmpty()).toBe(false);
});
test('should derive handledRequestCount from Statistics', async () => {
const requestQueue = new RequestQueue({ id: 'id', client: Configuration.getStorageClient() });
requestQueue.isEmpty = async () => false;
requestQueue.isFinished = async () => false;
requestQueue.fetchNextRequest = async () => new Request({ id: 'id', url: 'http://example.com' });
// @ts-expect-error Overriding the method for testing purposes
requestQueue.markRequestHandled = async () => {};
// Even though the request queue reports 33 handled requests (e.g. from a previous crawler run),
// the crawler should use its own Statistics to track the count and process all 40 requests.
vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);
let count = 0;
const crawler = new BasicCrawler({
requestQueue,
maxConcurrency: 1,
requestHandler: async () => {
await sleep(1);
count++;
},
maxRequestsPerCrawl: 40,
});
await crawler.run();
// The crawler should have processed 40 requests (its own limit), not 7 (40 - 33).
expect(count).toBe(40);
vitest.restoreAllMocks();
});
test('should timeout after handleRequestTimeoutSecs', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });
const results: Request[] = [];
const crawler = new BasicCrawler({
requestList,
handleRequestTimeoutSecs: 0.01,
maxRequestRetries: 1,
requestHandler: async () => sleep(1000),
failedRequestHandler: async ({ request }) => {
results.push(request);
},
});
await crawler.run();
expect(results).toHaveLength(1);
expect(results[0].url).toEqual(url);
results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out'));
});
test('limits handleRequestTimeoutSecs and derived vars to a valid value', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });
const results = [];
const crawler = new BasicCrawler({
requestList,
requestHandlerTimeoutSecs: Infinity,
maxRequestRetries: 1,
requestHandler: async () => sleep(1000),
failedRequestHandler: async ({ request }) => {
results.push(request);
},
});
const maxSignedInteger = 2 ** 31 - 1;
// @ts-expect-error Accessing private prop
expect(crawler.requestHandlerTimeoutMillis).toBe(maxSignedInteger);
// @ts-expect-error Accessing private prop
expect(crawler.internalTimeoutMillis).toBe(maxSignedInteger);
});
test('should not log stack trace for timeout errors by default', async () => {
const sources = [{ url: `http://${HOSTNAME}:${port}` }];
const requestList = await RequestList.open(null, sources);
const crawler = new BasicCrawler({
requestList,
requestHandlerTimeoutSecs: 0.1,
maxRequestRetries: 3,
requestHandler: async () => sleep(1e3),
});
const warningSpy = vitest.spyOn(crawler.log, 'warning');
const errorSpy = vitest.spyOn(crawler.log, 'error');
await crawler.run();
expect(warningSpy.mock.calls.length).toBe(3);
for (const args of warningSpy.mock.calls) {
expect(args.length).toBe(2);
expect(typeof args[0]).toBe('string');
expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
expect(/requestHandler timed out after/.test(args[0])).toBe(true);
expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false);
expect(args[1]).toBeDefined();
}
expect(errorSpy.mock.calls.length).toBe(1);
for (const args of errorSpy.mock.calls) {
expect(args.length).toBe(2);
expect(typeof args[0]).toBe('string');
expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
expect(/requestHandler timed out after/.test(args[0])).toBe(true);
expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false);
expect(args[1]).toBeDefined();
}
});
test('should log stack trace for non-timeout errors only when request will no longer be retried by default', async () => {
const sources = [{ url: `http://${HOSTNAME}:${port}` }];
const requestList = await RequestList.open(null, sources);
const crawler = new BasicCrawler({
requestList,
maxRequestRetries: 3,
requestHandler: () => {
throw new Error('Other non-timeout error');
},
});
const warningSpy = vitest.spyOn(crawler.log, 'warning');
const errorSpy = vitest.spyOn(crawler.log, 'error');
await crawler.run();
expect(warningSpy.mock.calls.length).toBe(3);
for (const args of warningSpy.mock.calls) {
expect(args.length).toBe(2);
expect(typeof args[0]).toBe('string');
expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
expect(/Other non-timeout error/.test(args[0])).toBe(true);
expect(args[0].split('\n').length).toBeLessThanOrEqual(2);
expect(args[1]).toBeDefined();
}
expect(errorSpy.mock.calls.length).toBe(1);
for (const args of errorSpy.mock.calls) {
expect(args.length).toBe(2);
expect(typeof args[0]).toBe('string');
expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
expect(/Other non-timeout error/.test(args[0])).toBe(true);
expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
expect(args[1]).toBeDefined();
}
});
test('should log stack trace for timeout errors when verbose log is enabled', async () => {
log.setLevel(log.LEVELS.INFO);
process.env.CRAWLEE_VERBOSE_LOG = 'true';
const sources = [{ url: `http://${HOSTNAME}:${port}` }];
const requestList = await RequestList.open(null, sources);
const crawler = new BasicCrawler({
requestList,
requestHandlerTimeoutSecs: 0.1,
maxRequestRetries: 3,
requestHandler: async () => sleep(1e3),
});
const warningSpy = vitest.spyOn(crawler.log, 'warning');
const errorSpy = vitest.spyOn(crawler.log, 'error');
await crawler.run();
expect(warningSpy.mock.calls.length).toBe(3);
for (const args of warningSpy.mock.calls) {
expect(args.length).toBe(2);
expect(typeof args[0]).toBe('string');
expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
expect(/requestHandler timed out after/.test(args[0])).toBe(true);
expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true);
expect(args[1]).toBeDefined();
}
expect(errorSpy.mock.calls.length).toBe(1);
for (const args of errorSpy.mock.calls) {
expect(args.length).toBe(2);
expect(typeof args[0]).toBe('string');
expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
expect(/requestHandler timed out after/.test(args[0])).toBe(true);
expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true);
expect(args[1]).toBeDefined();
}
log.setLevel(log.LEVELS.OFF);
process.env.CRAWLEE_VERBOSE_LOG = undefined;
});
test('should log stack trace for non-timeout errors when verbose log is enabled', async () => {
log.setLevel(log.LEVELS.INFO);
process.env.CRAWLEE_VERBOSE_LOG = 'true';
const sources = [{ url: `http://${HOSTNAME}:${port}` }];
const requestList = await RequestList.open(null, sources);
const crawler = new BasicCrawler({
requestList,
maxRequestRetries: 3,
requestHandler: () => {
throw new Error('Other non-timeout error');
},
});
const warningSpy = vitest.spyOn(crawler.log, 'warning');
const errorSpy = vitest.spyOn(crawler.log, 'error');
await crawler.run();
expect(warningSpy.mock.calls.length).toBe(3);
for (const args of warningSpy.mock.calls) {
expect(args.length).toBe(2);
expect(typeof args[0]).toBe('string');
expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
expect(/Other non-timeout error/.test(args[0])).toBe(true);
expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
expect(args[1]).toBeDefined();
}
expect(errorSpy.mock.calls.length).toBe(1);
for (const args of errorSpy.mock.calls) {
expect(args.length).toBe(2);
expect(typeof args[0]).toBe('string');
expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
expect(/Other non-timeout error/.test(args[0])).toBe(true);
expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
expect(args[1]).toBeDefined();
}
log.setLevel(log.LEVELS.OFF);
process.env.CRAWLEE_VERBOSE_LOG = undefined;
});
describe('Uses SessionPool', () => {
it('should use SessionPool when useSessionPool is true ', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });
const results: Request[] = [];
const crawler = new BasicCrawler({
requestList,
handleRequestTimeoutSecs: 0.01,
maxRequestRetries: 1,
useSessionPool: true,
sessionPoolOptions: {
maxPoolSize: 10,
persistStateKey: 'POOL',
},
requestHandler: async ({ session }) => {
expect(session!.constructor.name).toEqual('Session');
expect(session!.id).toBeDefined();
},
failedRequestHandler: async ({ request }) => {
results.push(request);
},
});
await crawler.run();
expect(crawler.sessionPool).toBeDefined();
expect(results).toHaveLength(0);
});
it('should use pass options to sessionPool', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });
const crawler = new BasicCrawler({
requestList,
handleRequestTimeoutSecs: 0.01,
maxRequestRetries: 1,
useSessionPool: true,
sessionPoolOptions: {
maxPoolSize: 10,
persistStateKey: 'POOL',
},
requestHandler: async () => {},
failedRequestHandler: async () => {},
});
await crawler.run();
// @ts-expect-error private symbol
expect(crawler.sessionPool.maxPoolSize).toEqual(10);
});
it('should destroy Session pool after it is finished', async () => {
const url = 'https://example.com';
const requestList = await RequestList.open({ sources: [{ url }] });
events.off(EventType.PERSIST_STATE);
const crawler = new BasicCrawler({
requestList,
handleRequestTimeoutSecs: 0.01,
maxRequestRetries: 1,
useSessionPool: true,
sessionPoolOptions: {
maxPoolSize: 10,
},
requestHandler: async () => {},
failedRequestHandler: async () => {},
});
await crawler.run();
expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);
// @ts-expect-error private symbol
expect(crawler.sessionPool.maxPoolSize).toEqual(10);
});
});
describe('CrawlingContext', () => {
test('should be kept and later deleted', async () => {
const urls = [
'https://example.com/0',
'https://example.com/1',
'https://example.com/2',
'https://example.com/3',
];
const requestList = await RequestList.open(null, urls);
let counter = 0;
let finish: (value?: unknown) => void;
const allFinishedPromise = new Promise((resolve) => {
finish = resolve;
});
const mainContexts: CrawlingContext[] = [];
const otherContexts: CrawlingContext[][] = [];
const crawler = new BasicCrawler({
requestList,
minConcurrency: 4,
async requestHandler(crawlingContext) {
// @ts-expect-error Accessing private prop
mainContexts[counter] = crawler.crawlingContexts.get(crawlingContext.id);
// @ts-expect-error Accessing private prop
otherContexts[counter] = Array.from(crawler.crawlingContexts).map(([, v]) => v);
counter++;
if (counter === 4) finish();
await allFinishedPromise;
},
});
await crawler.run();
expect(counter).toBe(4);
expect(mainContexts).toHaveLength(4);
expect(otherContexts).toHaveLength(4);
// @ts-expect-error Accessing private prop
expect(crawler.crawlingContexts.size).toBe(0);
mainContexts.forEach((ctx, idx) => {
expect(typeof ctx.id).toBe('string');
expect(otherContexts[idx]).toContain(ctx);
});
otherContexts.forEach((list, idx) => {
expect(list).toHaveLength(idx + 1);
});
});
});
describe('sendRequest', () => {
const html = `foobarHello, world!
`;
const httpServer = http.createServer((request, response) => {
response.setHeader('content-type', 'text/html');
response.end(html);
});
let url: string;
beforeAll(async () => {
await new Promise((resolve) => {
httpServer.listen(0, () => {
url = `http://127.0.0.1:${(httpServer.address() as AddressInfo).port}/`;
resolve();
});
});
});
afterAll(async () => {
await new Promise((resolve) => httpServer.close(resolve));
});
test('works', async () => {
const responses: { statusCode: number; body: string }[] = [];
const requestList = await RequestList.open(null, [url]);
const crawler = new BasicCrawler({
useSessionPool: true,
requestList,
async requestHandler({ sendRequest }) {
const response = await sendRequest();
responses.push({
statusCode: response.statusCode,
body: response.body,
});
},
});
await crawler.run();
expect(responses).toStrictEqual([
{
statusCode: 200,
body: html,
},
]);
});
test('works without session', async () => {
const requestList = await RequestList.open(null, [url]);
const responses: { statusCode: number; body: string }[] = [];
const crawler = new BasicCrawler({
useSessionPool: false,
requestList,
async requestHandler({ sendRequest }) {
const response = await sendRequest();
responses.push({
statusCode: response.statusCode,
body: response.body,
});
},
});
await crawler.run();
expect(responses).toStrictEqual([
{
statusCode: 200,
body: html,
},
]);
});
test('proxyUrl TypeScript support', async () => {
const crawler = new BasicCrawler({
useSessionPool: true,
async requestHandler({ sendRequest }) {
await sendRequest({
proxyUrl: 'http://example.com',
});
},
});
expect(crawler).toBeTruthy();
});
});
describe('Request enqueuing limits with maxRequestsPerCrawl', () => {
test('should not enqueue more requests than maxRequestsPerCrawl allows', async () => {
const requestQueue = await RequestQueue.open();
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
const crawler = new BasicCrawler({
requestQueue,
maxRequestsPerCrawl: 5,
requestHandler: async () => {},
});
crawler.stats.state.requestsFinished = 2;
// Try to add 6 requests - should only add 3 due to limit
const requestsToAdd = [
'http://example.com/1',
'http://example.com/2',
'http://example.com/3',
'http://example.com/4',
'http://example.com/5',
'http://example.com/6',
];
await crawler.addRequests(requestsToAdd);
// Should only have added the first 3 requests (since 2 were already processed, limit allows 3 more)
expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
{ url: 'http://example.com/3' },
]);
});
test('should not enqueue more requests than maxRequestsPerCrawl allows across multiple addRequests calls', async () => {
const requestQueue = await RequestQueue.open();
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
const crawler = new BasicCrawler({
requestQueue,
maxRequestsPerCrawl: 5,
requestHandler: async () => {},
});
crawler.stats.state.requestsFinished = 1;
// First call - should add 2 requests (2 more slots to go)
await crawler.addRequests(['http://example.com/1', 'http://example.com/2']);
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
]);
// Second call - should add only 2 more requests
await crawler.addRequests([
'http://example.com/3',
'http://example.com/4',
'http://example.com/5', // This should be ignored
'http://example.com/6', // This should be ignored
]);
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
{ url: 'http://example.com/3' },
{ url: 'http://example.com/4' },
]);
// Third call - should add no requests (limit already reached)
await crawler.addRequests(['http://example.com/7', 'http://example.com/8']);
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
{ url: 'http://example.com/1' },
{ url: 'http://example.com/2' },
{ url: 'http://example.com/3' },
{ url: 'http://example.com/4' },
]);
expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(3);
});
test('should respect robots.txt when limiting requests', async () => {
const requestQueue = await RequestQueue.open();
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
const crawler = new BasicCrawler({
requestQueue,
maxRequestsPerCrawl: 2,
respectRobotsTxtFile: true,
requestHandler: async () => {},
});
crawler.stats.state.requestsFinished = 0;
// Mock robots.txt checking to disallow some URLs
vitest.spyOn(crawler as any, 'isAllowedBasedOnRobotsTxtFile').mockImplementation(async (url) => {
return url !== 'http://example.com/2';
});
await crawler.addRequests([
'http://example.com/1', // Allowed by robots.txt
'http://example.com/2', // Blocked by robots.txt
'http://example.com/3', // Allowed by robots.txt && within limit
'http://example.com/4', // Would exceed limit
]);
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
{ url: 'http://example.com/1' },
{ url: 'http://example.com/3' },
]);
// Should only have added the first request (allowed by robots.txt and within limit)
expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
});
test.each([
{
testName: 'custom user-agent robots.txt rules',
userAgent: 'MyCrawler',
visitedUrls: [
{
url: 'http://example.com/yes',
},
{
url: 'http://example.com/my-crawler/anything',
},
],
},
{
testName: 'catch-all robots.txt rules with custom user-agent',
userAgent: 'RandomCrawler',
visitedUrls: [
{
url: 'http://example.com/yes',
},
],
},
])('should respect $testName', async ({ userAgent, visitedUrls }) => {
const requestQueue = await RequestQueue.open();
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
override async getRobotsTxtFileForUrl(_: string) {
return RobotsTxtFile.from(
'http://example.com/robots.txt',
`User-agent: *
Disallow: /
Allow: /yes
User-agent: MyCrawler
Disallow: /no
Allow: /my-crawler
`,
);
}
})({
requestQueue,
respectRobotsTxtFile: { userAgent },
requestHandler: async () => {},
});
await crawler.addRequests([
'http://example.com/yes', // Allowed by robots.txt
'http://example.com/no', // Blocked by robots.txt for "MyCrawler"
'http://example.com/no-globally', // Blocked by robots.txt, "*" rule
'http://example.com/my-crawler/anything', // Blocked by robots.txt for all user-agents, but allowed for "MyCrawler"
]);
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject(visitedUrls);
// Should only have added the first request (allowed by robots.txt and within limit)
expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
});
test('enqueueLinks should respect custom user-agent robots.txt rules', async () => {
const requestQueue = await RequestQueue.open();
const visitedUrls: string[] = [];
const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
override async getRobotsTxtFileForUrl(_: string) {
return RobotsTxtFile.from(
'http://example.com/robots.txt',
`User-agent: *
Disallow: /
Allow: /yes
User-agent: MyCrawler
Disallow: /no
Allow: /my-crawler
`,
);
}
})({
requestQueue,
maxConcurrency: 1,
respectRobotsTxtFile: { userAgent: 'MyCrawler' },
requestHandler: async (context) => {
visitedUrls.push(context.request.url);
if (context.request.label) {
return;
}
await context.enqueueLinks({
urls: [
'http://example.com/yes',
'http://example.com/no',
'http://example.com/no-globally',
'http://example.com/my-crawler/anything',
],
label: 'child',
});
},
});
await crawler.run(['http://example.com/start']);
expect(visitedUrls).toEqual([
'http://example.com/start',
'http://example.com/yes',
'http://example.com/my-crawler/anything',
]);
});
test('enqueueLinks forwards respectRobotsTxtFile.userAgent to the robots.txt check', async () => {
const requestQueue = await RequestQueue.open();
const isAllowedSpy = vitest.fn(() => true);
const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
override async getRobotsTxtFileForUrl(_: string) {
return { isAllowed: isAllowedSpy } as unknown as RobotsTxtFile;
}
})({
requestQueue,
maxConcurrency: 1,
respectRobotsTxtFile: { userAgent: 'MyCrawler' },
requestHandler: async (context) => {
if (context.request.label) return;
await context.enqueueLinks({ urls: ['http://example.com/child'], label: 'child' });
},
});
await crawler.run(['http://example.com/start']);
expect(isAllowedSpy).toHaveBeenCalledWith('http://example.com/child', 'MyCrawler');
});
test('enqueueLinks should respect maxRequestsPerCrawl', async () => {
const requestQueue = await RequestQueue.open();
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
// Will try to add 6 requests - should only add 3 due to limit
const requestsToAdd = [
'http://example.com/1',
'http://example.com/2',
'http://example.com/3',
'http://example.com/4',
'http://example.com/5',
'http://example.com/6',
];
const visitedUrls: string[] = [];
const crawler = new BasicCrawler({
requestQueue,
maxRequestsPerCrawl: 5,
requestHandler: async (context) => {
visitedUrls.push(context.request.url);
if (context.request.label) {
return;
}
crawler.stats.state.requestsFinished = 2;
await context.enqueueLinks({ urls: requestsToAdd, label: 'not-undefined' });
},
});
await crawler.run(['http://example.com']);
expect(visitedUrls).toEqual([
'http://example.com', // added by crawler.run()
'http://example.com/1',
'http://example.com/2',
]);
// Should only have added the first 2 requests (since 2 were already processed and 1 is in progress, limit allows 2 more)
expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(2);
});
test('enqueueLinks limit log message should only be logged once', async () => {
const requestQueue = await RequestQueue.open();
// Will try to add 10 requests with a limit of 2
const requestsToAdd = Array.from({ length: 10 }, (_, i) => `http://example.com/${i}`);
const crawler = new BasicCrawler({
requestQueue,
requestHandler: async (context) => {
if (context.request.label) {
return;
}
await context.enqueueLinks({ urls: requestsToAdd, limit: 2, label: 'child' });
},
});
const infoSpy = vitest.spyOn(crawler.log, 'info');
await crawler.run(['http://example.com']);
// The enqueueLinks limit message should only appear once, not 8 times (for each skipped request)
const enqueueLimitMessages = infoSpy.mock.calls.filter(
(call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
);
expect(enqueueLimitMessages).toHaveLength(1);
});
test('enqueueLinks limit log message should be logged again on subsequent runs', async () => {
const requestQueue = await RequestQueue.open();
const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/${i}`);
const crawler = new BasicCrawler({
requestQueue,
requestHandler: async (context) => {
if (context.request.label) {
return;
}
await context.enqueueLinks({ urls: requestsToAdd, limit: 1, label: 'child' });
},
});
const infoSpy = vitest.spyOn(crawler.log, 'info');
// First run
await crawler.run(['http://example.com/first']);
// Second run with a new start URL
await crawler.run(['http://example.com/second']);
// The enqueueLinks limit message should appear twice (once per run)
const enqueueLimitMessages = infoSpy.mock.calls.filter(
(call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
);
expect(enqueueLimitMessages).toHaveLength(2);
});
test('enqueueLinks limit log message should be logged once per request handler, not once per run', async () => {
const requestQueue = await RequestQueue.open();
// Each handler will try to add 5 URLs with a limit of 1
const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/child${i}`);
const crawler = new BasicCrawler({
requestQueue,
requestHandler: async (context) => {
if (context.request.label === 'child') {
return;
}
await context.enqueueLinks({ urls: requestsToAdd, limit: 1, label: 'child' });
},
});
const infoSpy = vitest.spyOn(crawler.log, 'info');
// Single run with two initial requests - both will trigger the limit
await crawler.run(['http://example.com/first', 'http://example.com/second']);
// The enqueueLinks limit message should appear twice (once per request handler that triggered the limit)
const enqueueLimitMessages = infoSpy.mock.calls.filter(
(call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
);
expect(enqueueLimitMessages).toHaveLength(2);
});
test('maxCrawlDepth limit log message should only be logged once per run', async () => {
const requestQueue = await RequestQueue.open();
// Each handler will try to add URLs that exceed maxCrawlDepth
const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/child${i}`);
const crawler = new BasicCrawler({
requestQueue,
maxCrawlDepth: 1, // Only allow depth 0 (initial) and depth 1 (first level children)
requestHandler: async (context) => {
// Stop processing at depth 2 to avoid infinite loops
if (context.request.crawlDepth >= 2) {
return;
}
// This will add requests at depth+1, so initial requests add at depth 1 (allowed)
// and depth 1 requests add at depth 2 (blocked by maxCrawlDepth)
await context.enqueueLinks({
urls: requestsToAdd,
label: `depth-${context.request.crawlDepth + 1}`,
});
},
});
const infoSpy = vitest.spyOn(crawler.log, 'info');
// Run with two initial requests
// Each will enqueue children at depth 1, then those children will try to enqueue at depth 2 (blocked)
await crawler.run(['http://example.com/first', 'http://example.com/second']);
// The maxCrawlDepth limit message should only appear once per run, even though multiple requests triggered it
const maxCrawlDepthMessages = infoSpy.mock.calls.filter(
(call) => typeof call[0] === 'string' && call[0].includes('maxCrawlDepth'),
);
expect(maxCrawlDepthMessages).toHaveLength(1);
});
test('should not count duplicate URLs toward maxRequestsPerCrawl limit (addRequests)', async () => {
const requestQueue = await RequestQueue.open();
const crawler = new BasicCrawler({
requestQueue,
maxRequestsPerCrawl: 5,
requestHandler: async () => {},
});
// 10 duplicate links to the same URL + 1 unique link at the end
const requestsToAdd = [
...Array.from({ length: 10 }, () => 'http://example.com/same'),
'http://example.com/new',
];
await crawler.addRequests(requestsToAdd);
// Both unique URLs should have been enqueued — duplicates should not consume the budget
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
{ url: 'http://example.com/same' },
{ url: 'http://example.com/new' },
]);
});
test('addRequestsBatched with maxNewRequests should correctly report requestsOverLimit for array input', async () => {
const queue = await RequestQueue.open();
const result = await queue.addRequestsBatched(
[
{ url: 'http://example.com/a' },
{ url: 'http://example.com/b' },
{ url: 'http://example.com/c' },
{ url: 'http://example.com/d' },
{ url: 'http://example.com/e' },
],
{ maxNewRequests: 2 },
);
const addedUrls = result.addedRequests.filter((r) => !r.wasAlreadyPresent).map((r) => r.uniqueKey);
const overLimitUrls = (result.requestsOverLimit ?? []).map((r) => (typeof r === 'string' ? r : r.url));
expect(addedUrls).toHaveLength(2);
expect(overLimitUrls).toHaveLength(3);
});
test('addRequestsBatched with maxNewRequests should correctly report requestsOverLimit for generator input', async () => {
const queue = await RequestQueue.open();
async function* urls() {
yield { url: 'http://example.com/a' };
yield { url: 'http://example.com/b' };
yield { url: 'http://example.com/c' };
yield { url: 'http://example.com/d' };
yield { url: 'http://example.com/e' };
}
const result = await queue.addRequestsBatched(urls(), { maxNewRequests: 2 });
const addedUrls = result.addedRequests.filter((r) => !r.wasAlreadyPresent).map((r) => r.uniqueKey);
const overLimitUrls = (result.requestsOverLimit ?? []).map((r) => (typeof r === 'string' ? r : r.url));
expect(addedUrls).toHaveLength(2);
expect(overLimitUrls).toHaveLength(3);
});
test('should not count duplicate URLs toward maxRequestsPerCrawl limit (enqueueLinks)', async () => {
const requestQueue = await RequestQueue.open();
const visitedUrls: string[] = [];
const crawler = new BasicCrawler({
requestQueue,
maxRequestsPerCrawl: 5,
requestHandler: async (context) => {
visitedUrls.push(context.request.url);
if (context.request.label) {
return;
}
// Enqueue 10 duplicate links + 1 new unique link
const urls = [...Array.from({ length: 10 }, () => 'http://example.com/'), 'http://example.com/new'];
await context.enqueueLinks({ urls, label: 'child' });
},
});
await crawler.run(['http://example.com/']);
// Both the start URL and the new URL should have been visited
expect(visitedUrls).toContain('http://example.com/');
expect(visitedUrls).toContain('http://example.com/new');
});
});
describe('addRequests input validation', () => {
test('should throw error when passed a non-iterable value', async () => {
const crawler = new BasicCrawler({
requestHandler: async () => {},
});
await expect(crawler.addRequests(new WeakSet() as any)).rejects.toThrow(
'Expected an iterable or async iterable, got weakset',
);
});
});
describe('Dataset helpers, crawler parallelism', () => {
const payload: Dictionary[] = [{ foo: 'bar', baz: 123 }];
const getPayload: (id: string) => Dictionary[] = (id) => [{ foo: id }];
const tmpDir = `${__dirname}/tmp/foo/bar`;
beforeAll(async () => {
await rm(tmpDir, { recursive: true, force: true });
});
test('should expose default Dataset methods', async () => {
const crawler = new BasicCrawler();
await crawler.pushData(payload);
expect((await crawler.getData()).items).toEqual(payload);
});
test('export data', async () => {
const row: Dictionary = { foo: 'bar', baz: 123 };
const crawler = new BasicCrawler();
await crawler.pushData(row);
await crawler.pushData(row);
await crawler.pushData(row);
await crawler.exportData(`${tmpDir}/result.csv`);
await crawler.exportData(`${tmpDir}/result.json`);
const csv = await readFile(`${tmpDir}/result.csv`);
expect(csv.toString()).toBe('foo,baz\nbar,123\nbar,123\nbar,123\n');
const json = await readFile(`${tmpDir}/result.json`);
expect(json.toString()).toBe(
'[\n' +
' {\n' +
' "foo": "bar",\n' +
' "baz": 123\n' +
' },\n' +
' {\n' +
' "foo": "bar",\n' +
' "baz": 123\n' +
' },\n' +
' {\n' +
' "foo": "bar",\n' +
' "baz": 123\n' +
' }\n' +
']\n',
);
await rm(`${tmpDir}/result.csv`);
await rm(`${tmpDir}/result.json`);
});
test('exports do not fail on empty dataset', async () => {
const crawler = new BasicCrawler();
await crawler.exportData(`${tmpDir}/result.csv`);
await crawler.exportData(`${tmpDir}/result.json`);
const csv = await readFile(`${tmpDir}/result.csv`);
expect(csv.toString()).toBe('');
const json = await readFile(`${tmpDir}/result.json`);
expect(json.toString()).toBe('[]\n');
await rm(`${tmpDir}/result.csv`);
await rm(`${tmpDir}/result.json`);
});
test('should expose pushData helper', async () => {
const crawler = new BasicCrawler({
requestHandler: async ({ pushData }) => pushData(payload),
});
await crawler.run([
{
url: `http://${HOSTNAME}:${port}`,
},
]);
expect((await crawler.getData()).items).toEqual(payload);
});
test('crawler.exportData works with `collectAllKeys`', async () => {
const crawler = new BasicCrawler();
await crawler.pushData([{ foo: 'bar', baz: 123 }]);
await crawler.pushData([{ foo: 'baz', qux: 456 }]);
await crawler.exportData(`${tmpDir}/result.csv`, 'csv', { collectAllKeys: true });
const csv = await readFile(`${tmpDir}/result.csv`);
expect(csv.toString()).toBe('foo,baz,qux\nbar,123,\nbaz,,456\n');
await rm(`${tmpDir}/result.csv`);
});
test("Crawlers with different Configurations don't share Datasets", async () => {
const crawlerA = new BasicCrawler({}, new Configuration({ persistStorage: false }));
const crawlerB = new BasicCrawler({}, new Configuration({ persistStorage: false }));
await crawlerA.pushData(getPayload('A'));
await crawlerB.pushData(getPayload('B'));
expect((await crawlerA.getData()).items).toEqual(getPayload('A'));
expect((await crawlerB.getData()).items).toEqual(getPayload('B'));
});
test('Crawlers with different Configurations run separately', async () => {
const crawlerA = new BasicCrawler(
{ requestHandler: () => {} },
new Configuration({ persistStorage: false }),
);
const crawlerB = new BasicCrawler(
{ requestHandler: () => {} },
new Configuration({ persistStorage: false }),
);
await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);
expect(crawlerA.stats.state.requestsFinished).toBe(1);
expect(crawlerB.stats.state.requestsFinished).toBe(1);
});
test('Crawlers with different Configurations does not use global Configuration', async () => {
const getGlobalConfigSpy = vitest.spyOn(Configuration, 'getGlobalConfig');
const configA = new Configuration({ persistStorage: false });
const crawlerA = new BasicCrawler({ requestHandler: () => {} }, configA);
const configB = new Configuration({ persistStorage: false });
const crawlerB = new BasicCrawler({ requestHandler: () => {} }, configB);
await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);
expect(getGlobalConfigSpy.mock.calls.length).toBe(0);
expect(crawlerA.requestQueue?.config).toBe(configA);
expect(crawlerB.requestQueue?.config).toBe(configB);
});
});
});