wehub-resource-sync
2216 lines
85 KiB
TypeScript
2216 lines
85 KiB
TypeScript
import { readFile, rm } from 'node:fs/promises';
|
|
import type { Server } from 'node:http';
|
|
import http from 'node:http';
|
|
import type { AddressInfo } from 'node:net';
|
|
|
|
import type {
|
|
CrawlingContext,
|
|
EnqueueLinksOptions,
|
|
ErrorHandler,
|
|
RequestHandler,
|
|
RequestOptions,
|
|
Source,
|
|
} from '@crawlee/basic';
|
|
import {
|
|
BasicCrawler,
|
|
Configuration,
|
|
CriticalError,
|
|
EventType,
|
|
KeyValueStore,
|
|
MissingRouteError,
|
|
NonRetryableError,
|
|
Request,
|
|
RequestList,
|
|
RequestQueue,
|
|
} from '@crawlee/basic';
|
|
import { RequestState } from '@crawlee/core';
|
|
import type { Dictionary } from '@crawlee/utils';
|
|
import { RobotsTxtFile, sleep } from '@crawlee/utils';
|
|
import express from 'express';
|
|
import type { SetRequired } from 'type-fest';
|
|
import type { Mock } from 'vitest';
|
|
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';
|
|
|
|
import log from '@apify/log';
|
|
|
|
import { startExpressAppPromise } from '../../shared/_helper';
|
|
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';
|
|
|
|
describe('BasicCrawler', () => {
|
|
let logLevel: number;
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
const events = Configuration.getEventManager();
|
|
|
|
const HOSTNAME = '127.0.0.1';
|
|
let port: number;
|
|
let server: Server;
|
|
beforeAll(async () => {
|
|
const app = express();
|
|
|
|
app.get('/', (req, res) => {
|
|
res.send(`<html><head><title>Example Domain</title></head></html>`);
|
|
});
|
|
|
|
server = await startExpressAppPromise(app, 0);
|
|
port = (server.address() as AddressInfo).port;
|
|
});
|
|
|
|
beforeAll(async () => {
|
|
logLevel = log.getLevel();
|
|
log.setLevel(log.LEVELS.OFF);
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
vitest.clearAllMocks();
|
|
await localStorageEmulator.init();
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await localStorageEmulator.destroy();
|
|
});
|
|
|
|
afterAll(async () => {
|
|
log.setLevel(logLevel);
|
|
});
|
|
|
|
afterAll(() => {
|
|
server.close();
|
|
});
|
|
|
|
test('does not leak sigint events', async () => {
|
|
let count = 0;
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestHandler: () => {
|
|
count = process.listenerCount('SIGINT');
|
|
},
|
|
});
|
|
|
|
await crawler.run(['https://example.com']);
|
|
|
|
expect(process.listenerCount('SIGINT')).toBe(count - 1);
|
|
});
|
|
|
|
test('should run in parallel thru all the requests', async () => {
|
|
const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index}` }));
|
|
const sourcesCopy = JSON.parse(JSON.stringify(sources));
|
|
|
|
const processed: { url: string }[] = [];
|
|
const requestList = await RequestList.open(null, sources);
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await sleep(10);
|
|
processed.push({ url: request.url });
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
minConcurrency: 25,
|
|
maxConcurrency: 25,
|
|
requestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(basicCrawler.autoscaledPool!.minConcurrency).toBe(25);
|
|
expect(processed).toEqual(sourcesCopy);
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
});
|
|
|
|
test('should allow using run method multiple times', async () => {
|
|
const sources = [...Array(100).keys()].map((index) => `https://example.com/${index}`);
|
|
const sourcesCopy = JSON.parse(JSON.stringify(sources));
|
|
|
|
const processed: { url: string }[] = [];
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await sleep(10);
|
|
processed.push({ url: request.url });
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
minConcurrency: 25,
|
|
maxConcurrency: 25,
|
|
requestHandler,
|
|
});
|
|
|
|
await basicCrawler.run(sources);
|
|
await basicCrawler.run(sources);
|
|
await basicCrawler.run(sources);
|
|
|
|
expect(processed).toHaveLength(sourcesCopy.length * 3);
|
|
});
|
|
|
|
test('should process 4 requests total when calling run() twice with maxRequestsPerCrawl: 2', async () => {
|
|
const processed: { url: string }[] = [];
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await sleep(10);
|
|
processed.push({ url: request.url });
|
|
};
|
|
|
|
const crawler = new BasicCrawler({
|
|
maxRequestsPerCrawl: 2,
|
|
minConcurrency: 1,
|
|
maxConcurrency: 1,
|
|
requestHandler,
|
|
});
|
|
|
|
// First run should process 2 requests
|
|
await crawler.run([...Array(5).keys()].map((index) => `https://example.com/first/${index}`));
|
|
expect(processed).toHaveLength(2);
|
|
|
|
// Make sure no extra requests were enqueued
|
|
expect(await localStorageEmulator.getRequestQueueItems()).toEqual([]);
|
|
|
|
// Second run should process 2 more requests
|
|
await crawler.run([...Array(5).keys()].map((index) => `https://example.com/second/${index}`));
|
|
expect(processed).toHaveLength(4);
|
|
|
|
// Make sure no extra requests were enqueued
|
|
expect(await localStorageEmulator.getRequestQueueItems()).toEqual([]);
|
|
|
|
const processedUrls = processed.map((p) => p.url);
|
|
|
|
expect(processedUrls).toEqual([
|
|
'https://example.com/first/0',
|
|
'https://example.com/first/1',
|
|
'https://example.com/second/0',
|
|
'https://example.com/second/1',
|
|
]);
|
|
});
|
|
|
|
test('addRequests should respect maxCrawlDepth', async () => {
|
|
const processedUrls: string[] = [];
|
|
|
|
const requestHandler: RequestHandler = async ({ request, addRequests }) => {
|
|
processedUrls.push(request.url);
|
|
const url = new URL(request.url);
|
|
url.pathname = `${url.pathname}deep/`;
|
|
|
|
await addRequests([url.toString()]);
|
|
};
|
|
|
|
const crawler = new BasicCrawler({
|
|
maxCrawlDepth: 2,
|
|
maxRequestsPerCrawl: 10, // safeguard against infinite loops
|
|
requestHandler,
|
|
});
|
|
|
|
await crawler.run(['https://example.com/']);
|
|
|
|
expect(processedUrls).toEqual([
|
|
'https://example.com/',
|
|
'https://example.com/deep/',
|
|
'https://example.com/deep/deep/',
|
|
]);
|
|
});
|
|
|
|
test('enqueueLinks should respect maxCrawlDepth', async () => {
|
|
const processedUrls: string[] = [];
|
|
|
|
const requestHandler: RequestHandler = async ({ request, enqueueLinks }) => {
|
|
processedUrls.push(request.url);
|
|
const url = new URL(request.url);
|
|
url.pathname = `${url.pathname}deep/`;
|
|
|
|
await enqueueLinks({
|
|
urls: [url.toString()],
|
|
});
|
|
};
|
|
|
|
const crawler = new BasicCrawler({
|
|
maxCrawlDepth: 2,
|
|
maxRequestsPerCrawl: 10, // safeguard against infinite loops
|
|
requestHandler,
|
|
});
|
|
|
|
await crawler.run(['https://example.com/']);
|
|
|
|
expect(processedUrls).toEqual([
|
|
'https://example.com/',
|
|
'https://example.com/deep/',
|
|
'https://example.com/deep/deep/',
|
|
]);
|
|
});
|
|
|
|
describe('enqueueLinksWithCrawlDepth()', () => {
|
|
let onSkippedRequestMock: Mock;
|
|
let addRequestsBatchedMock: Mock;
|
|
let options: SetRequired<EnqueueLinksOptions, 'urls'>;
|
|
let request: Request;
|
|
let requestQueue: RequestQueue;
|
|
|
|
type EnqueueLinksWrapperOptions = Parameters<BasicCrawler['enqueueLinksWithCrawlDepth']>;
|
|
class TestCrawler extends BasicCrawler {
|
|
public exposedEnqueueLinksWithCrawlDepth(...enqueueLinksOptions: EnqueueLinksWrapperOptions) {
|
|
return this.enqueueLinksWithCrawlDepth(...enqueueLinksOptions);
|
|
}
|
|
}
|
|
|
|
const crawler = new TestCrawler({ maxCrawlDepth: 3 });
|
|
|
|
beforeEach(() => {
|
|
addRequestsBatchedMock = vi.fn().mockImplementation(async () => ({
|
|
addedRequests: [],
|
|
waitForAllRequestsToBeAdded: Promise.resolve([]),
|
|
requestsOverLimit: [],
|
|
}));
|
|
onSkippedRequestMock = vi.fn();
|
|
|
|
options = {
|
|
urls: ['https://example.com/1/', 'https://example.com/2/'],
|
|
onSkippedRequest: onSkippedRequestMock,
|
|
};
|
|
request = new Request({ url: 'https://example.com/', crawlDepth: 2 });
|
|
requestQueue = {
|
|
addRequestsBatched: addRequestsBatchedMock as RequestQueue['addRequestsBatched'],
|
|
} as RequestQueue;
|
|
});
|
|
|
|
it('should generate requests with maxCrawlDepth', async () => {
|
|
await crawler.exposedEnqueueLinksWithCrawlDepth(options, request, requestQueue);
|
|
|
|
const requests = addRequestsBatchedMock.mock.calls[0][0];
|
|
expect(requests).toHaveLength(2);
|
|
expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
|
|
expect(requests[1]).toMatchObject({ url: 'https://example.com/2/', crawlDepth: 3 });
|
|
|
|
expect(onSkippedRequestMock).not.toBeCalled();
|
|
});
|
|
|
|
it('should skip requests with crawlDepth exceeding maxCrawlDepth', async () => {
|
|
const requestWithMaxDepth = new Request({ url: 'https://example.com/', crawlDepth: 3 });
|
|
await crawler.exposedEnqueueLinksWithCrawlDepth(options, requestWithMaxDepth, requestQueue);
|
|
|
|
const requests = addRequestsBatchedMock.mock.calls[0][0];
|
|
expect(requests).toHaveLength(0);
|
|
|
|
const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);
|
|
expect(skippedRequests).toHaveLength(2);
|
|
expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'depth' });
|
|
expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'depth' });
|
|
});
|
|
|
|
it('should respect user provided transformRequestFunction', async () => {
|
|
const transformRequestFunction = vi.fn((req: RequestOptions) => req);
|
|
const optionsWithTransform = { ...options, transformRequestFunction };
|
|
|
|
await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, request, requestQueue);
|
|
|
|
expect(transformRequestFunction).toHaveBeenCalled();
|
|
|
|
const requests = addRequestsBatchedMock.mock.calls[0][0];
|
|
expect(requests).toHaveLength(2);
|
|
expect(requests[0]).toMatchObject({ url: 'https://example.com/1/', crawlDepth: 3 });
|
|
});
|
|
});
|
|
|
|
it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => {
|
|
type AddCrawlDepthWrapperOptions = Parameters<BasicCrawler['addCrawlDepthRequestGenerator']>;
|
|
class TestCrawler extends BasicCrawler {
|
|
public exposedAddCrawlDepthRequestGenerator(...enqueueLinksOptions: AddCrawlDepthWrapperOptions) {
|
|
return this.addCrawlDepthRequestGenerator(...enqueueLinksOptions);
|
|
}
|
|
}
|
|
|
|
const crawler = new TestCrawler();
|
|
|
|
const requests = ['https://example.com/1/', { url: 'https://example.com/2/' }];
|
|
const newCrawlDepth = 4;
|
|
const addRequestsGenerator = crawler.exposedAddCrawlDepthRequestGenerator(requests, newCrawlDepth);
|
|
|
|
const generatedRequests: Source[] = [];
|
|
for await (const generatedRequest of addRequestsGenerator) {
|
|
generatedRequests.push(generatedRequest);
|
|
}
|
|
|
|
expect(generatedRequests).toHaveLength(2);
|
|
expect(generatedRequests[0].url).toBe('https://example.com/1/');
|
|
expect(generatedRequests[0].crawlDepth).toBe(4);
|
|
|
|
expect(generatedRequests[1].url).toBe('https://example.com/2/');
|
|
expect(generatedRequests[1].crawlDepth).toBe(4);
|
|
});
|
|
|
|
test('should correctly combine shorthand and full length options', async () => {
|
|
const shorthandOptions = {
|
|
minConcurrency: 123,
|
|
maxConcurrency: 456,
|
|
maxRequestsPerMinute: 789,
|
|
};
|
|
|
|
const autoscaledPoolOptions = {
|
|
minConcurrency: 16,
|
|
maxConcurrency: 32,
|
|
maxTasksPerMinute: 64,
|
|
};
|
|
|
|
const collectResults = (crawler: BasicCrawler): typeof shorthandOptions | typeof autoscaledPoolOptions => {
|
|
return {
|
|
minConcurrency: crawler.autoscaledPool!.minConcurrency,
|
|
maxConcurrency: crawler.autoscaledPool!.maxConcurrency,
|
|
// eslint-disable-next-line dot-notation -- accessing a private member
|
|
maxRequestsPerMinute: crawler.autoscaledPool!['maxTasksPerMinute'],
|
|
// eslint-disable-next-line dot-notation
|
|
maxTasksPerMinute: crawler.autoscaledPool!['maxTasksPerMinute'],
|
|
};
|
|
};
|
|
|
|
const requestList = await RequestList.open(null, []);
|
|
const requestHandler = async () => {};
|
|
|
|
const results = await Promise.all(
|
|
[
|
|
new BasicCrawler({
|
|
requestList,
|
|
requestHandler,
|
|
...shorthandOptions,
|
|
}),
|
|
new BasicCrawler({
|
|
requestList,
|
|
requestHandler,
|
|
autoscaledPoolOptions,
|
|
}),
|
|
new BasicCrawler({
|
|
requestList,
|
|
requestHandler,
|
|
...shorthandOptions,
|
|
autoscaledPoolOptions,
|
|
}),
|
|
].map(async (c) => {
|
|
await c.run();
|
|
return collectResults(c);
|
|
}),
|
|
);
|
|
|
|
expect(results[0]).toEqual(expect.objectContaining(shorthandOptions));
|
|
|
|
expect(results[1]).toEqual(expect.objectContaining(autoscaledPoolOptions));
|
|
|
|
expect(results[2]).toEqual(expect.objectContaining(shorthandOptions));
|
|
});
|
|
|
|
test('auto-saved state object', async () => {
|
|
const sources = [...Array(50).keys()].map((index) => ({ url: `https://example.com/${index}` }));
|
|
const sourcesCopy = JSON.parse(JSON.stringify(sources));
|
|
|
|
const processed: { url: string }[] = [];
|
|
const requestList = await RequestList.open(null, sources);
|
|
const requestHandler: RequestHandler = async ({ request, crawler }) => {
|
|
await sleep(10);
|
|
const state = await crawler.useState({ processed });
|
|
state.processed.push({ url: request.url });
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
requestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
const state = await basicCrawler.useState();
|
|
|
|
expect(processed).toEqual(sourcesCopy);
|
|
expect(state.processed).toEqual(sourcesCopy);
|
|
expect(state.processed).toBe(processed);
|
|
expect(state.processed).toEqual(sourcesCopy);
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
});
|
|
|
|
test.each([
|
|
EventType.MIGRATING,
|
|
EventType.ABORTING,
|
|
])('should pause on %s event and persist RequestList state', async (event) => {
|
|
const sources = [...Array(500).keys()].map((index) => ({ url: `https://example.com/${index + 1}` }));
|
|
|
|
let persistResolve!: (value?: unknown) => void;
|
|
const persistPromise = new Promise((res) => {
|
|
persistResolve = res;
|
|
});
|
|
|
|
// Mock the calls to persist sources.
|
|
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
|
|
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
|
|
getValueSpy.mockResolvedValue(null);
|
|
|
|
const processed: { url: string }[] = [];
|
|
const requestList = await RequestList.open('reqList', sources);
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
if (request.url.endsWith('200')) events.emit(event);
|
|
processed.push({ url: request.url });
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
minConcurrency: 25,
|
|
maxConcurrency: 25,
|
|
requestHandler,
|
|
});
|
|
|
|
let finished = false;
|
|
// Mock the call to persist state.
|
|
setValueSpy.mockImplementationOnce(persistResolve as any);
|
|
// The crawler will pause after 200 requests
|
|
const runPromise = basicCrawler.run();
|
|
void runPromise.then(() => {
|
|
finished = true;
|
|
});
|
|
|
|
// need to monkeypatch the stats class, otherwise it will never finish
|
|
basicCrawler.stats.persistState = async () => Promise.resolve();
|
|
await persistPromise;
|
|
|
|
expect(finished).toBe(false);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
expect(processed.length).toBe(200);
|
|
|
|
expect(getValueSpy).toBeCalled();
|
|
expect(setValueSpy).toBeCalled();
|
|
|
|
// clean up
|
|
// @ts-expect-error Accessing private method
|
|
await basicCrawler.autoscaledPool!._destroy();
|
|
});
|
|
|
|
test('should retry failed requests', async () => {
|
|
const sources = [
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
{ url: 'http://example.com/3' },
|
|
];
|
|
const processed: Dictionary<Request> = {};
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await sleep(10);
|
|
processed[request.url] = request;
|
|
|
|
if (request.url === 'http://example.com/2') {
|
|
throw Error(`This is ${request.retryCount}th error!`);
|
|
}
|
|
|
|
request.userData.foo = 'bar';
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
maxRequestRetries: 10,
|
|
minConcurrency: 3,
|
|
maxConcurrency: 3,
|
|
requestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(processed['http://example.com/1'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/1'].errorMessages).toEqual([]);
|
|
expect(processed['http://example.com/1'].retryCount).toBe(0);
|
|
expect(processed['http://example.com/3'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/3'].errorMessages).toEqual([]);
|
|
expect(processed['http://example.com/3'].retryCount).toBe(0);
|
|
|
|
expect(processed['http://example.com/2'].userData.foo).toBeUndefined();
|
|
expect(processed['http://example.com/2'].errorMessages).toHaveLength(11);
|
|
expect(processed['http://example.com/2'].retryCount).toBe(10);
|
|
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
});
|
|
|
|
test('should retry failed requests based on `request.maxRetries`', async () => {
|
|
const sources = [
|
|
{ url: 'http://example.com/1', maxRetries: 10 },
|
|
{ url: 'http://example.com/2', maxRetries: 5 },
|
|
{ url: 'http://example.com/3', maxRetries: 1 },
|
|
];
|
|
const processed: Dictionary<Request> = {};
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await sleep(10);
|
|
processed[request.url] = request;
|
|
throw Error(`This is ${request.retryCount}th error!`);
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
minConcurrency: 3,
|
|
maxConcurrency: 3,
|
|
requestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(processed['http://example.com/1'].errorMessages).toHaveLength(11);
|
|
expect(processed['http://example.com/1'].retryCount).toBe(10);
|
|
expect(processed['http://example.com/2'].errorMessages).toHaveLength(6);
|
|
expect(processed['http://example.com/2'].retryCount).toBe(5);
|
|
expect(processed['http://example.com/3'].errorMessages).toHaveLength(2);
|
|
expect(processed['http://example.com/3'].retryCount).toBe(1);
|
|
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
});
|
|
|
|
test('should not retry requests with noRetry set to true ', async () => {
|
|
const noRetryRequest = new Request({ url: 'http://example.com/3' });
|
|
noRetryRequest.noRetry = true;
|
|
|
|
const sources = [
|
|
{ url: 'http://example.com/1', noRetry: true },
|
|
{ url: 'http://example.com/2' },
|
|
noRetryRequest,
|
|
];
|
|
const processed: Dictionary<Request> = {};
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await sleep(10);
|
|
processed[request.url] = request;
|
|
request.userData.foo = 'bar';
|
|
throw Error(`This is ${request.retryCount}th error!`);
|
|
};
|
|
|
|
let failedRequestHandlerCalls = 0;
|
|
const failedRequestHandler = async () => {
|
|
failedRequestHandlerCalls++;
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
maxRequestRetries: 10,
|
|
minConcurrency: 3,
|
|
maxConcurrency: 3,
|
|
requestHandler,
|
|
failedRequestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(processed['http://example.com/1'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/1'].errorMessages).toHaveLength(1);
|
|
expect(processed['http://example.com/1'].retryCount).toBe(0);
|
|
expect(processed['http://example.com/3'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/3'].errorMessages).toHaveLength(1);
|
|
expect(processed['http://example.com/3'].retryCount).toBe(0);
|
|
|
|
expect(processed['http://example.com/2'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/2'].errorMessages).toHaveLength(11);
|
|
expect(processed['http://example.com/2'].retryCount).toBe(10);
|
|
|
|
expect(failedRequestHandlerCalls).toBe(3);
|
|
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
});
|
|
|
|
test('should correctly track request.state', async () => {
|
|
const sources = [{ url: 'http://example.com/1' }];
|
|
const requestList = await RequestList.open(null, sources);
|
|
const requestStates: RequestState[] = [];
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
requestStates.push(request.state);
|
|
throw new Error('Error');
|
|
};
|
|
|
|
const errorHandler: RequestHandler = async ({ request }) => {
|
|
requestStates.push(request.state);
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
maxConcurrency: 1,
|
|
maxRequestRetries: 1,
|
|
requestHandler,
|
|
errorHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(requestStates).toEqual([
|
|
RequestState.REQUEST_HANDLER,
|
|
RequestState.ERROR_HANDLER,
|
|
RequestState.REQUEST_HANDLER,
|
|
]);
|
|
});
|
|
|
|
test('should use errorHandler', async () => {
|
|
const sources = [{ url: 'http://example.com/', label: 'start' }];
|
|
|
|
let errorHandlerCalls = 0;
|
|
let failedRequestHandlerCalls = 0;
|
|
|
|
const failed: Dictionary<{ request: Request; error: Error }> = {};
|
|
const requestList = await RequestList.open({ sources });
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
expect(request.label).toBe(errorHandlerCalls === 0 ? 'start' : `error_${errorHandlerCalls}`);
|
|
throw new Error(`This is an error ${errorHandlerCalls}`);
|
|
};
|
|
|
|
const errorHandler: ErrorHandler = async ({ request }, error) => {
|
|
expect(error.message).toBe(`This is an error ${errorHandlerCalls}`);
|
|
errorHandlerCalls++;
|
|
request.label = `error_${errorHandlerCalls}`;
|
|
};
|
|
|
|
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
|
|
failed[request.url] = { request, error };
|
|
failedRequestHandlerCalls++;
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
requestHandler,
|
|
errorHandler,
|
|
failedRequestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(errorHandlerCalls).toBe(3);
|
|
expect(failedRequestHandlerCalls).toBe(1);
|
|
expect(Object.values(failed)).toHaveLength(1);
|
|
expect(failed['http://example.com/'].request.label).not.toBe('start');
|
|
expect(failed['http://example.com/'].request.label).toBe('error_3');
|
|
expect(failed['http://example.com/'].error.message).toEqual('This is an error 3');
|
|
});
|
|
|
|
test('should allow to handle failed requests', async () => {
|
|
const sources = [
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
{ url: 'http://example.com/3' },
|
|
];
|
|
const processed: Dictionary<Request> = {};
|
|
const failed: Dictionary<Request> = {};
|
|
const errors: Error[] = [];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await Promise.reject(new Error('some-error'));
|
|
processed[request.url] = request;
|
|
};
|
|
|
|
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
|
|
failed[request.url] = request;
|
|
errors.push(error);
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
requestHandler,
|
|
failedRequestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(failed['http://example.com/1'].errorMessages).toHaveLength(4);
|
|
expect(failed['http://example.com/1'].retryCount).toBe(3);
|
|
expect(failed['http://example.com/2'].errorMessages).toHaveLength(4);
|
|
expect(failed['http://example.com/2'].retryCount).toBe(3);
|
|
expect(failed['http://example.com/3'].errorMessages).toHaveLength(4);
|
|
expect(failed['http://example.com/3'].retryCount).toBe(3);
|
|
expect(Object.values(failed)).toHaveLength(3);
|
|
expect(Object.values(processed)).toHaveLength(0);
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
errors.forEach((error) => expect(error).toBeInstanceOf(Error));
|
|
});
|
|
|
|
test('should not retry on NonRetryableError', async () => {
|
|
const sources = [
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
{ url: 'http://example.com/3' },
|
|
];
|
|
const failed: Dictionary<Request> = {};
|
|
const errors: Error[] = [];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const requestHandler: RequestHandler = async () => {
|
|
throw new NonRetryableError('some-error');
|
|
};
|
|
|
|
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
|
|
failed[request.url] = request;
|
|
errors.push(error);
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
requestHandler,
|
|
failedRequestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(failed['http://example.com/1'].errorMessages).toHaveLength(1);
|
|
expect(failed['http://example.com/1'].retryCount).toBe(0);
|
|
expect(failed['http://example.com/2'].errorMessages).toHaveLength(1);
|
|
expect(failed['http://example.com/2'].retryCount).toBe(0);
|
|
expect(failed['http://example.com/3'].errorMessages).toHaveLength(1);
|
|
expect(failed['http://example.com/3'].retryCount).toBe(0);
|
|
expect(Object.values(failed)).toHaveLength(3);
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
errors.forEach((error) => expect(error).toBeInstanceOf(NonRetryableError));
|
|
});
|
|
|
|
test('noRetry after calling errorHandler', async () => {
|
|
const sources = [{ url: `http://example.com` }];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
let request: Request<Dictionary<any>>;
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
errorHandler: (context, error) => {
|
|
request = context.request;
|
|
context.request.noRetry = true;
|
|
},
|
|
maxRequestRetries: 3,
|
|
requestHandler: () => {
|
|
throw new Error('Failure');
|
|
},
|
|
});
|
|
|
|
await crawler.run();
|
|
|
|
expect(request!.retryCount).toBe(0);
|
|
});
|
|
|
|
test('should crash on CriticalError', async () => {
|
|
const sources = [
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
{ url: 'http://example.com/3' },
|
|
];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const requestHandler: RequestHandler = async () => {
|
|
throw new CriticalError('some-error');
|
|
};
|
|
|
|
const failedRequestHandler = vitest.fn() as ErrorHandler;
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
requestHandler,
|
|
failedRequestHandler,
|
|
});
|
|
|
|
await expect(basicCrawler.run()).rejects.toThrow(CriticalError);
|
|
|
|
expect(failedRequestHandler).not.toBeCalled();
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
});
|
|
|
|
test('should crash on MissingRouteError', async () => {
|
|
const sources = [
|
|
{ url: 'http://example.com/1', label: 'TEST' }, // will match
|
|
{ url: 'http://example.com/2', label: 'FOO' }, // will fail as no FOO route or default route exists
|
|
{ url: 'http://example.com/3' }, // will fail as no default route exists
|
|
];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const failedRequestHandler = vitest.fn() as ErrorHandler;
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
failedRequestHandler,
|
|
});
|
|
const testRoute = vitest.fn();
|
|
basicCrawler.router.addHandler('TEST', testRoute);
|
|
|
|
await expect(basicCrawler.run()).rejects.toThrow(MissingRouteError);
|
|
|
|
expect(failedRequestHandler).not.toBeCalled();
|
|
expect(testRoute).toBeCalled();
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
});
|
|
|
|
test('should correctly combine RequestList and RequestQueue', async () => {
|
|
const sources = [
|
|
{ url: 'http://example.com/0' },
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
];
|
|
const processed: Dictionary<Request> = {};
|
|
const requestList = await RequestList.open(null, sources);
|
|
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await sleep(10);
|
|
processed[request.url] = request;
|
|
|
|
if (request.url === 'http://example.com/1') {
|
|
throw Error(`This is ${request.retryCount}th error!`);
|
|
}
|
|
|
|
request.userData.foo = 'bar';
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
requestQueue,
|
|
maxRequestRetries: 3,
|
|
minConcurrency: 1,
|
|
maxConcurrency: 1,
|
|
requestHandler,
|
|
});
|
|
|
|
vitest.spyOn(requestQueue, 'handledCount').mockResolvedValueOnce(0);
|
|
|
|
vitest
|
|
.spyOn(requestQueue, 'addRequest')
|
|
.mockResolvedValueOnce({ requestId: 'id-0' } as any)
|
|
.mockResolvedValueOnce({ requestId: 'id-1' } as any)
|
|
.mockResolvedValueOnce({ requestId: 'id-2' } as any);
|
|
|
|
const request0 = new Request({ id: 'id-0', ...sources[0] });
|
|
const request1 = new Request({ id: 'id-1', ...sources[1] });
|
|
const request2 = new Request({ id: 'id-2', ...sources[2] });
|
|
|
|
const queueContent = [request0, request1, request2, request1, request1, request1];
|
|
|
|
vitest.spyOn(requestQueue, 'fetchNextRequest').mockImplementation(async () => queueContent.shift() ?? null);
|
|
|
|
const markReqHandled = vitest
|
|
.spyOn(requestQueue, 'markRequestHandled')
|
|
.mockReturnValue(Promise.resolve() as any);
|
|
const reclaimReq = vitest.spyOn(requestQueue, 'reclaimRequest').mockReturnValue(Promise.resolve() as any);
|
|
|
|
vitest.spyOn(requestQueue, 'isEmpty').mockImplementation(async () => queueContent.length <= 0);
|
|
vitest.spyOn(requestQueue, 'isFinished').mockResolvedValueOnce(true);
|
|
|
|
await basicCrawler.run();
|
|
|
|
// 1st try
|
|
|
|
expect(reclaimReq).toBeCalledWith(request1, expect.objectContaining({}));
|
|
expect(reclaimReq).toBeCalledTimes(3);
|
|
|
|
expect(processed['http://example.com/0'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/0'].errorMessages).toEqual([]);
|
|
expect(processed['http://example.com/0'].retryCount).toBe(0);
|
|
expect(processed['http://example.com/2'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/2'].errorMessages).toEqual([]);
|
|
expect(processed['http://example.com/2'].retryCount).toBe(0);
|
|
|
|
expect(processed['http://example.com/1'].userData.foo).toBeUndefined();
|
|
expect(processed['http://example.com/1'].errorMessages).toHaveLength(4);
|
|
expect(processed['http://example.com/1'].retryCount).toBe(3);
|
|
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
|
|
vitest.restoreAllMocks();
|
|
});
|
|
|
|
test('should say that task is not ready requestList is not set and requestQueue is empty', async () => {
|
|
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
|
|
requestQueue.isEmpty = async () => Promise.resolve(true);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
// @ts-expect-error Accessing private prop
|
|
expect(await crawler._isTaskReadyFunction()).toBe(false);
|
|
});
|
|
|
|
test('should be possible to override isFinishedFunction and isTaskReadyFunction of underlying AutoscaledPool', async () => {
|
|
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
|
|
const processed: Request[] = [];
|
|
const queue: Request[] = [];
|
|
let isFinished = false;
|
|
let isFinishedFunctionCalled = false;
|
|
let isTaskReadyFunctionCalled = false;
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestQueue,
|
|
autoscaledPoolOptions: {
|
|
minConcurrency: 1,
|
|
maxConcurrency: 1,
|
|
isFinishedFunction: async () => {
|
|
isFinishedFunctionCalled = true;
|
|
return Promise.resolve(isFinished);
|
|
},
|
|
isTaskReadyFunction: async () => {
|
|
isTaskReadyFunctionCalled = true;
|
|
return Promise.resolve(!isFinished);
|
|
},
|
|
},
|
|
requestHandler: async ({ request }) => {
|
|
await sleep(10);
|
|
processed.push(request);
|
|
},
|
|
});
|
|
|
|
// Speed up the test
|
|
// @ts-expect-error Accessing private prop
|
|
basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;
|
|
|
|
const request0 = new Request({ url: 'http://example.com/0' });
|
|
const request1 = new Request({ url: 'http://example.com/1' });
|
|
|
|
vitest.spyOn(requestQueue, 'handledCount').mockReturnValue(Promise.resolve() as any);
|
|
|
|
let handledCount = 0;
|
|
const markRequestHandled = vitest.spyOn(requestQueue, 'markRequestHandled').mockImplementation(async () => {
|
|
handledCount++;
|
|
// Only set isFinished after both requests have been handled
|
|
if (handledCount >= 2) {
|
|
// Small delay to ensure the test can verify everything
|
|
setTimeout(() => {
|
|
isFinished = true;
|
|
}, 50);
|
|
}
|
|
return Promise.resolve() as any;
|
|
});
|
|
|
|
const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished');
|
|
|
|
requestQueue.fetchNextRequest = async () => queue.pop()!;
|
|
requestQueue.isEmpty = async () => Promise.resolve(!queue.length);
|
|
|
|
// Add requests with buffer time for crawler startup.
|
|
// Use longer delays to avoid flakiness under CPU load from parallel tests.
|
|
setTimeout(() => queue.push(request0), 500);
|
|
setTimeout(() => queue.push(request1), 1000);
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(markRequestHandled).toBeCalledWith(request0);
|
|
expect(markRequestHandled).toBeCalledWith(request1);
|
|
expect(isFinishedOrig).not.toBeCalled();
|
|
expect(isFinishedFunctionCalled).toBe(true);
|
|
expect(isTaskReadyFunctionCalled).toBe(true);
|
|
|
|
// TODO: see why the request1 was passed as a second parameter to includes
|
|
expect(processed.includes(request0)).toBe(true);
|
|
|
|
vitest.restoreAllMocks();
|
|
});
|
|
|
|
test('keepAlive', async () => {
|
|
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
|
|
const processed: Request[] = [];
|
|
const queue: Request[] = [];
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestQueue,
|
|
keepAlive: true,
|
|
requestHandler: async ({ request }) => {
|
|
await sleep(10);
|
|
processed.push(request);
|
|
},
|
|
});
|
|
|
|
// Speed up the test
|
|
// @ts-expect-error Accessing private prop
|
|
basicCrawler.autoscaledPoolOptions.maybeRunIntervalSecs = 0.05;
|
|
|
|
const request0 = new Request({ url: 'http://example.com/0' });
|
|
const request1 = new Request({ url: 'http://example.com/1' });
|
|
|
|
vitest.spyOn(requestQueue, 'handledCount').mockReturnValue(Promise.resolve() as any);
|
|
const markRequestHandled = vitest
|
|
.spyOn(requestQueue, 'markRequestHandled')
|
|
.mockReturnValue(Promise.resolve() as any);
|
|
|
|
const isFinishedOrig = vitest.spyOn(requestQueue, 'isFinished');
|
|
|
|
requestQueue.fetchNextRequest = async () => Promise.resolve(queue.pop()!);
|
|
requestQueue.isEmpty = async () => Promise.resolve(!queue.length);
|
|
|
|
// Use longer delays to avoid flakiness under CPU load from parallel tests.
|
|
setTimeout(() => queue.push(request0), 500);
|
|
setTimeout(() => queue.push(request1), 1000);
|
|
setTimeout(() => {
|
|
void basicCrawler.teardown();
|
|
}, 3000);
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(markRequestHandled).toBeCalledWith(request0);
|
|
expect(markRequestHandled).toBeCalledWith(request1);
|
|
expect(isFinishedOrig).not.toBeCalled();
|
|
|
|
// TODO: see why the request1 was passed as a second parameter to includes
|
|
expect(processed.includes(request0)).toBe(true);
|
|
|
|
vitest.restoreAllMocks();
|
|
});
|
|
|
|
test('should support maxRequestsPerCrawl parameter', async () => {
|
|
const sources = [
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
{ url: 'http://example.com/3' },
|
|
{ url: 'http://example.com/4' },
|
|
{ url: 'http://example.com/5' },
|
|
];
|
|
const processed: Dictionary<Request> = {};
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const requestHandler: RequestHandler = async ({ request }) => {
|
|
await sleep(10);
|
|
processed[request.url] = request;
|
|
if (request.url === 'http://example.com/2') throw Error();
|
|
request.userData.foo = 'bar';
|
|
};
|
|
|
|
let failedRequestHandlerCalls = 0;
|
|
const failedRequestHandler = async () => {
|
|
failedRequestHandlerCalls++;
|
|
};
|
|
|
|
const basicCrawler = new BasicCrawler({
|
|
requestList,
|
|
maxRequestRetries: 3,
|
|
maxRequestsPerCrawl: 3,
|
|
maxConcurrency: 1,
|
|
requestHandler,
|
|
failedRequestHandler,
|
|
});
|
|
|
|
await basicCrawler.run();
|
|
|
|
expect(processed['http://example.com/1'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/1'].errorMessages).toEqual([]);
|
|
expect(processed['http://example.com/1'].retryCount).toBe(0);
|
|
expect(processed['http://example.com/3'].userData.foo).toBe('bar');
|
|
expect(processed['http://example.com/3'].errorMessages).toEqual([]);
|
|
expect(processed['http://example.com/3'].retryCount).toBe(0);
|
|
|
|
expect(processed['http://example.com/2'].userData.foo).toEqual(undefined);
|
|
expect(processed['http://example.com/2'].errorMessages).toHaveLength(4);
|
|
expect(processed['http://example.com/2'].retryCount).toBe(3);
|
|
|
|
expect(failedRequestHandlerCalls).toBe(1);
|
|
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
});
|
|
|
|
test('should derive handledRequestCount from Statistics', async () => {
|
|
const requestQueue = new RequestQueue({ id: 'id', client: Configuration.getStorageClient() });
|
|
requestQueue.isEmpty = async () => false;
|
|
requestQueue.isFinished = async () => false;
|
|
|
|
requestQueue.fetchNextRequest = async () => new Request({ id: 'id', url: 'http://example.com' });
|
|
// @ts-expect-error Overriding the method for testing purposes
|
|
requestQueue.markRequestHandled = async () => {};
|
|
|
|
// Even though the request queue reports 33 handled requests (e.g. from a previous crawler run),
|
|
// the crawler should use its own Statistics to track the count and process all 40 requests.
|
|
vitest.spyOn(requestQueue, 'handledCount').mockResolvedValue(33);
|
|
|
|
let count = 0;
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
maxConcurrency: 1,
|
|
requestHandler: async () => {
|
|
await sleep(1);
|
|
count++;
|
|
},
|
|
maxRequestsPerCrawl: 40,
|
|
});
|
|
|
|
await crawler.run();
|
|
// The crawler should have processed 40 requests (its own limit), not 7 (40 - 33).
|
|
expect(count).toBe(40);
|
|
vitest.restoreAllMocks();
|
|
});
|
|
|
|
test('should timeout after handleRequestTimeoutSecs', async () => {
|
|
const url = 'https://example.com';
|
|
const requestList = await RequestList.open({ sources: [{ url }] });
|
|
|
|
const results: Request[] = [];
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
handleRequestTimeoutSecs: 0.01,
|
|
maxRequestRetries: 1,
|
|
requestHandler: async () => sleep(1000),
|
|
failedRequestHandler: async ({ request }) => {
|
|
results.push(request);
|
|
},
|
|
});
|
|
|
|
await crawler.run();
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0].url).toEqual(url);
|
|
results[0].errorMessages.forEach((msg) => expect(msg).toMatch('requestHandler timed out'));
|
|
});
|
|
|
|
test('limits handleRequestTimeoutSecs and derived vars to a valid value', async () => {
|
|
const url = 'https://example.com';
|
|
const requestList = await RequestList.open({ sources: [{ url }] });
|
|
|
|
const results = [];
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
requestHandlerTimeoutSecs: Infinity,
|
|
maxRequestRetries: 1,
|
|
requestHandler: async () => sleep(1000),
|
|
failedRequestHandler: async ({ request }) => {
|
|
results.push(request);
|
|
},
|
|
});
|
|
|
|
const maxSignedInteger = 2 ** 31 - 1;
|
|
// @ts-expect-error Accessing private prop
|
|
expect(crawler.requestHandlerTimeoutMillis).toBe(maxSignedInteger);
|
|
// @ts-expect-error Accessing private prop
|
|
expect(crawler.internalTimeoutMillis).toBe(maxSignedInteger);
|
|
});
|
|
|
|
test('should not log stack trace for timeout errors by default', async () => {
|
|
const sources = [{ url: `http://${HOSTNAME}:${port}` }];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
requestHandlerTimeoutSecs: 0.1,
|
|
maxRequestRetries: 3,
|
|
requestHandler: async () => sleep(1e3),
|
|
});
|
|
|
|
const warningSpy = vitest.spyOn(crawler.log, 'warning');
|
|
const errorSpy = vitest.spyOn(crawler.log, 'error');
|
|
|
|
await crawler.run();
|
|
|
|
expect(warningSpy.mock.calls.length).toBe(3);
|
|
for (const args of warningSpy.mock.calls) {
|
|
expect(args.length).toBe(2);
|
|
expect(typeof args[0]).toBe('string');
|
|
expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
|
|
expect(/requestHandler timed out after/.test(args[0])).toBe(true);
|
|
expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false);
|
|
expect(args[1]).toBeDefined();
|
|
}
|
|
|
|
expect(errorSpy.mock.calls.length).toBe(1);
|
|
for (const args of errorSpy.mock.calls) {
|
|
expect(args.length).toBe(2);
|
|
expect(typeof args[0]).toBe('string');
|
|
expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
|
|
expect(/requestHandler timed out after/.test(args[0])).toBe(true);
|
|
expect(/at Timeout\._onTimeout/.test(args[0])).toBe(false);
|
|
expect(args[1]).toBeDefined();
|
|
}
|
|
});
|
|
|
|
test('should log stack trace for non-timeout errors only when request will no longer be retried by default', async () => {
|
|
const sources = [{ url: `http://${HOSTNAME}:${port}` }];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
maxRequestRetries: 3,
|
|
requestHandler: () => {
|
|
throw new Error('Other non-timeout error');
|
|
},
|
|
});
|
|
|
|
const warningSpy = vitest.spyOn(crawler.log, 'warning');
|
|
const errorSpy = vitest.spyOn(crawler.log, 'error');
|
|
|
|
await crawler.run();
|
|
|
|
expect(warningSpy.mock.calls.length).toBe(3);
|
|
for (const args of warningSpy.mock.calls) {
|
|
expect(args.length).toBe(2);
|
|
expect(typeof args[0]).toBe('string');
|
|
expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
|
|
expect(/Other non-timeout error/.test(args[0])).toBe(true);
|
|
expect(args[0].split('\n').length).toBeLessThanOrEqual(2);
|
|
expect(args[1]).toBeDefined();
|
|
}
|
|
|
|
expect(errorSpy.mock.calls.length).toBe(1);
|
|
for (const args of errorSpy.mock.calls) {
|
|
expect(args.length).toBe(2);
|
|
expect(typeof args[0]).toBe('string');
|
|
expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
|
|
expect(/Other non-timeout error/.test(args[0])).toBe(true);
|
|
expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
|
|
expect(args[1]).toBeDefined();
|
|
}
|
|
});
|
|
|
|
test('should log stack trace for timeout errors when verbose log is enabled', async () => {
|
|
log.setLevel(log.LEVELS.INFO);
|
|
process.env.CRAWLEE_VERBOSE_LOG = 'true';
|
|
const sources = [{ url: `http://${HOSTNAME}:${port}` }];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
requestHandlerTimeoutSecs: 0.1,
|
|
maxRequestRetries: 3,
|
|
requestHandler: async () => sleep(1e3),
|
|
});
|
|
|
|
const warningSpy = vitest.spyOn(crawler.log, 'warning');
|
|
const errorSpy = vitest.spyOn(crawler.log, 'error');
|
|
|
|
await crawler.run();
|
|
|
|
expect(warningSpy.mock.calls.length).toBe(3);
|
|
for (const args of warningSpy.mock.calls) {
|
|
expect(args.length).toBe(2);
|
|
expect(typeof args[0]).toBe('string');
|
|
expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
|
|
expect(/requestHandler timed out after/.test(args[0])).toBe(true);
|
|
expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true);
|
|
expect(args[1]).toBeDefined();
|
|
}
|
|
|
|
expect(errorSpy.mock.calls.length).toBe(1);
|
|
for (const args of errorSpy.mock.calls) {
|
|
expect(args.length).toBe(2);
|
|
expect(typeof args[0]).toBe('string');
|
|
expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
|
|
expect(/requestHandler timed out after/.test(args[0])).toBe(true);
|
|
expect(/at Timeout\._onTimeout/.test(args[0])).toBe(true);
|
|
expect(args[1]).toBeDefined();
|
|
}
|
|
|
|
log.setLevel(log.LEVELS.OFF);
|
|
process.env.CRAWLEE_VERBOSE_LOG = undefined;
|
|
});
|
|
|
|
test('should log stack trace for non-timeout errors when verbose log is enabled', async () => {
|
|
log.setLevel(log.LEVELS.INFO);
|
|
process.env.CRAWLEE_VERBOSE_LOG = 'true';
|
|
const sources = [{ url: `http://${HOSTNAME}:${port}` }];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
maxRequestRetries: 3,
|
|
requestHandler: () => {
|
|
throw new Error('Other non-timeout error');
|
|
},
|
|
});
|
|
|
|
const warningSpy = vitest.spyOn(crawler.log, 'warning');
|
|
const errorSpy = vitest.spyOn(crawler.log, 'error');
|
|
|
|
await crawler.run();
|
|
|
|
expect(warningSpy.mock.calls.length).toBe(3);
|
|
for (const args of warningSpy.mock.calls) {
|
|
expect(args.length).toBe(2);
|
|
expect(typeof args[0]).toBe('string');
|
|
expect(/Reclaiming failed request back to the list or queue/.test(args[0])).toBe(true);
|
|
expect(/Other non-timeout error/.test(args[0])).toBe(true);
|
|
expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
|
|
expect(args[1]).toBeDefined();
|
|
}
|
|
|
|
expect(errorSpy.mock.calls.length).toBe(1);
|
|
for (const args of errorSpy.mock.calls) {
|
|
expect(args.length).toBe(2);
|
|
expect(typeof args[0]).toBe('string');
|
|
expect(/Request failed and reached maximum retries/.test(args[0])).toBe(true);
|
|
expect(/Other non-timeout error/.test(args[0])).toBe(true);
|
|
expect(/at _?BasicCrawler\.requestHandler/.test(args[0])).toBe(true);
|
|
expect(args[1]).toBeDefined();
|
|
}
|
|
|
|
log.setLevel(log.LEVELS.OFF);
|
|
process.env.CRAWLEE_VERBOSE_LOG = undefined;
|
|
});
|
|
|
|
describe('Uses SessionPool', () => {
|
|
it('should use SessionPool when useSessionPool is true ', async () => {
|
|
const url = 'https://example.com';
|
|
const requestList = await RequestList.open({ sources: [{ url }] });
|
|
const results: Request[] = [];
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
handleRequestTimeoutSecs: 0.01,
|
|
maxRequestRetries: 1,
|
|
useSessionPool: true,
|
|
sessionPoolOptions: {
|
|
maxPoolSize: 10,
|
|
persistStateKey: 'POOL',
|
|
},
|
|
requestHandler: async ({ session }) => {
|
|
expect(session!.constructor.name).toEqual('Session');
|
|
expect(session!.id).toBeDefined();
|
|
},
|
|
failedRequestHandler: async ({ request }) => {
|
|
results.push(request);
|
|
},
|
|
});
|
|
|
|
await crawler.run();
|
|
expect(crawler.sessionPool).toBeDefined();
|
|
expect(results).toHaveLength(0);
|
|
});
|
|
|
|
it('should use pass options to sessionPool', async () => {
|
|
const url = 'https://example.com';
|
|
const requestList = await RequestList.open({ sources: [{ url }] });
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
handleRequestTimeoutSecs: 0.01,
|
|
maxRequestRetries: 1,
|
|
useSessionPool: true,
|
|
sessionPoolOptions: {
|
|
maxPoolSize: 10,
|
|
persistStateKey: 'POOL',
|
|
},
|
|
requestHandler: async () => {},
|
|
failedRequestHandler: async () => {},
|
|
});
|
|
await crawler.run();
|
|
|
|
// @ts-expect-error private symbol
|
|
expect(crawler.sessionPool.maxPoolSize).toEqual(10);
|
|
});
|
|
|
|
it('should destroy Session pool after it is finished', async () => {
|
|
const url = 'https://example.com';
|
|
const requestList = await RequestList.open({ sources: [{ url }] });
|
|
events.off(EventType.PERSIST_STATE);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
handleRequestTimeoutSecs: 0.01,
|
|
maxRequestRetries: 1,
|
|
useSessionPool: true,
|
|
sessionPoolOptions: {
|
|
maxPoolSize: 10,
|
|
},
|
|
requestHandler: async () => {},
|
|
failedRequestHandler: async () => {},
|
|
});
|
|
|
|
await crawler.run();
|
|
expect(events.listenerCount(EventType.PERSIST_STATE)).toEqual(0);
|
|
// @ts-expect-error private symbol
|
|
expect(crawler.sessionPool.maxPoolSize).toEqual(10);
|
|
});
|
|
});
|
|
|
|
describe('CrawlingContext', () => {
|
|
test('should be kept and later deleted', async () => {
|
|
const urls = [
|
|
'https://example.com/0',
|
|
'https://example.com/1',
|
|
'https://example.com/2',
|
|
'https://example.com/3',
|
|
];
|
|
const requestList = await RequestList.open(null, urls);
|
|
let counter = 0;
|
|
let finish: (value?: unknown) => void;
|
|
const allFinishedPromise = new Promise((resolve) => {
|
|
finish = resolve;
|
|
});
|
|
const mainContexts: CrawlingContext[] = [];
|
|
const otherContexts: CrawlingContext[][] = [];
|
|
const crawler = new BasicCrawler({
|
|
requestList,
|
|
minConcurrency: 4,
|
|
async requestHandler(crawlingContext) {
|
|
// @ts-expect-error Accessing private prop
|
|
mainContexts[counter] = crawler.crawlingContexts.get(crawlingContext.id);
|
|
// @ts-expect-error Accessing private prop
|
|
otherContexts[counter] = Array.from(crawler.crawlingContexts).map(([, v]) => v);
|
|
counter++;
|
|
if (counter === 4) finish();
|
|
await allFinishedPromise;
|
|
},
|
|
});
|
|
await crawler.run();
|
|
|
|
expect(counter).toBe(4);
|
|
expect(mainContexts).toHaveLength(4);
|
|
expect(otherContexts).toHaveLength(4);
|
|
// @ts-expect-error Accessing private prop
|
|
expect(crawler.crawlingContexts.size).toBe(0);
|
|
mainContexts.forEach((ctx, idx) => {
|
|
expect(typeof ctx.id).toBe('string');
|
|
expect(otherContexts[idx]).toContain(ctx);
|
|
});
|
|
otherContexts.forEach((list, idx) => {
|
|
expect(list).toHaveLength(idx + 1);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('sendRequest', () => {
|
|
const html = `<!DOCTYPE html><html><head><title>foobar</title></head><body><p>Hello, world!</p></body></html>`;
|
|
|
|
const httpServer = http.createServer((request, response) => {
|
|
response.setHeader('content-type', 'text/html');
|
|
response.end(html);
|
|
});
|
|
|
|
let url: string;
|
|
|
|
beforeAll(async () => {
|
|
await new Promise<void>((resolve) => {
|
|
httpServer.listen(0, () => {
|
|
url = `http://127.0.0.1:${(httpServer.address() as AddressInfo).port}/`;
|
|
|
|
resolve();
|
|
});
|
|
});
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await new Promise((resolve) => httpServer.close(resolve));
|
|
});
|
|
|
|
test('works', async () => {
|
|
const responses: { statusCode: number; body: string }[] = [];
|
|
|
|
const requestList = await RequestList.open(null, [url]);
|
|
|
|
const crawler = new BasicCrawler({
|
|
useSessionPool: true,
|
|
requestList,
|
|
async requestHandler({ sendRequest }) {
|
|
const response = await sendRequest();
|
|
|
|
responses.push({
|
|
statusCode: response.statusCode,
|
|
body: response.body,
|
|
});
|
|
},
|
|
});
|
|
|
|
await crawler.run();
|
|
|
|
expect(responses).toStrictEqual([
|
|
{
|
|
statusCode: 200,
|
|
body: html,
|
|
},
|
|
]);
|
|
});
|
|
|
|
test('works without session', async () => {
|
|
const requestList = await RequestList.open(null, [url]);
|
|
|
|
const responses: { statusCode: number; body: string }[] = [];
|
|
|
|
const crawler = new BasicCrawler({
|
|
useSessionPool: false,
|
|
requestList,
|
|
async requestHandler({ sendRequest }) {
|
|
const response = await sendRequest();
|
|
|
|
responses.push({
|
|
statusCode: response.statusCode,
|
|
body: response.body,
|
|
});
|
|
},
|
|
});
|
|
|
|
await crawler.run();
|
|
|
|
expect(responses).toStrictEqual([
|
|
{
|
|
statusCode: 200,
|
|
body: html,
|
|
},
|
|
]);
|
|
});
|
|
|
|
test('proxyUrl TypeScript support', async () => {
|
|
const crawler = new BasicCrawler({
|
|
useSessionPool: true,
|
|
async requestHandler({ sendRequest }) {
|
|
await sendRequest({
|
|
proxyUrl: 'http://example.com',
|
|
});
|
|
},
|
|
});
|
|
|
|
expect(crawler).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
describe('Request enqueuing limits with maxRequestsPerCrawl', () => {
|
|
test('should not enqueue more requests than maxRequestsPerCrawl allows', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
maxRequestsPerCrawl: 5,
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
crawler.stats.state.requestsFinished = 2;
|
|
|
|
// Try to add 6 requests - should only add 3 due to limit
|
|
const requestsToAdd = [
|
|
'http://example.com/1',
|
|
'http://example.com/2',
|
|
'http://example.com/3',
|
|
'http://example.com/4',
|
|
'http://example.com/5',
|
|
'http://example.com/6',
|
|
];
|
|
|
|
await crawler.addRequests(requestsToAdd);
|
|
|
|
// Should only have added the first 3 requests (since 2 were already processed, limit allows 3 more)
|
|
expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
|
|
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
{ url: 'http://example.com/3' },
|
|
]);
|
|
});
|
|
|
|
test('should not enqueue more requests than maxRequestsPerCrawl allows across multiple addRequests calls', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
maxRequestsPerCrawl: 5,
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
crawler.stats.state.requestsFinished = 1;
|
|
|
|
// First call - should add 2 requests (2 more slots to go)
|
|
await crawler.addRequests(['http://example.com/1', 'http://example.com/2']);
|
|
|
|
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
]);
|
|
|
|
// Second call - should add only 2 more requests
|
|
await crawler.addRequests([
|
|
'http://example.com/3',
|
|
'http://example.com/4',
|
|
'http://example.com/5', // This should be ignored
|
|
'http://example.com/6', // This should be ignored
|
|
]);
|
|
|
|
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
{ url: 'http://example.com/3' },
|
|
{ url: 'http://example.com/4' },
|
|
]);
|
|
|
|
// Third call - should add no requests (limit already reached)
|
|
await crawler.addRequests(['http://example.com/7', 'http://example.com/8']);
|
|
|
|
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/2' },
|
|
{ url: 'http://example.com/3' },
|
|
{ url: 'http://example.com/4' },
|
|
]);
|
|
|
|
expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(3);
|
|
});
|
|
|
|
test('should respect robots.txt when limiting requests', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
maxRequestsPerCrawl: 2,
|
|
respectRobotsTxtFile: true,
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
crawler.stats.state.requestsFinished = 0;
|
|
|
|
// Mock robots.txt checking to disallow some URLs
|
|
vitest.spyOn(crawler as any, 'isAllowedBasedOnRobotsTxtFile').mockImplementation(async (url) => {
|
|
return url !== 'http://example.com/2';
|
|
});
|
|
|
|
await crawler.addRequests([
|
|
'http://example.com/1', // Allowed by robots.txt
|
|
'http://example.com/2', // Blocked by robots.txt
|
|
'http://example.com/3', // Allowed by robots.txt && within limit
|
|
'http://example.com/4', // Would exceed limit
|
|
]);
|
|
|
|
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
|
|
{ url: 'http://example.com/1' },
|
|
{ url: 'http://example.com/3' },
|
|
]);
|
|
|
|
// Should only have added the first request (allowed by robots.txt and within limit)
|
|
expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
test.each([
|
|
{
|
|
testName: 'custom user-agent robots.txt rules',
|
|
userAgent: 'MyCrawler',
|
|
visitedUrls: [
|
|
{
|
|
url: 'http://example.com/yes',
|
|
},
|
|
{
|
|
url: 'http://example.com/my-crawler/anything',
|
|
},
|
|
],
|
|
},
|
|
{
|
|
testName: 'catch-all robots.txt rules with custom user-agent',
|
|
userAgent: 'RandomCrawler',
|
|
visitedUrls: [
|
|
{
|
|
url: 'http://example.com/yes',
|
|
},
|
|
],
|
|
},
|
|
])('should respect $testName', async ({ userAgent, visitedUrls }) => {
|
|
const requestQueue = await RequestQueue.open();
|
|
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
|
|
|
|
const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
|
|
override async getRobotsTxtFileForUrl(_: string) {
|
|
return RobotsTxtFile.from(
|
|
'http://example.com/robots.txt',
|
|
`User-agent: *
|
|
Disallow: /
|
|
Allow: /yes
|
|
|
|
User-agent: MyCrawler
|
|
Disallow: /no
|
|
Allow: /my-crawler
|
|
`,
|
|
);
|
|
}
|
|
})({
|
|
requestQueue,
|
|
respectRobotsTxtFile: { userAgent },
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
await crawler.addRequests([
|
|
'http://example.com/yes', // Allowed by robots.txt
|
|
'http://example.com/no', // Blocked by robots.txt for "MyCrawler"
|
|
'http://example.com/no-globally', // Blocked by robots.txt, "*" rule
|
|
'http://example.com/my-crawler/anything', // Blocked by robots.txt for all user-agents, but allowed for "MyCrawler"
|
|
]);
|
|
|
|
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject(visitedUrls);
|
|
|
|
// Should only have added the first request (allowed by robots.txt and within limit)
|
|
expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
|
|
});
|
|
|
|
test('enqueueLinks should respect custom user-agent robots.txt rules', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
const visitedUrls: string[] = [];
|
|
|
|
const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
|
|
override async getRobotsTxtFileForUrl(_: string) {
|
|
return RobotsTxtFile.from(
|
|
'http://example.com/robots.txt',
|
|
`User-agent: *
|
|
Disallow: /
|
|
Allow: /yes
|
|
|
|
User-agent: MyCrawler
|
|
Disallow: /no
|
|
Allow: /my-crawler
|
|
`,
|
|
);
|
|
}
|
|
})({
|
|
requestQueue,
|
|
maxConcurrency: 1,
|
|
respectRobotsTxtFile: { userAgent: 'MyCrawler' },
|
|
requestHandler: async (context) => {
|
|
visitedUrls.push(context.request.url);
|
|
|
|
if (context.request.label) {
|
|
return;
|
|
}
|
|
|
|
await context.enqueueLinks({
|
|
urls: [
|
|
'http://example.com/yes',
|
|
'http://example.com/no',
|
|
'http://example.com/no-globally',
|
|
'http://example.com/my-crawler/anything',
|
|
],
|
|
label: 'child',
|
|
});
|
|
},
|
|
});
|
|
|
|
await crawler.run(['http://example.com/start']);
|
|
|
|
expect(visitedUrls).toEqual([
|
|
'http://example.com/start',
|
|
'http://example.com/yes',
|
|
'http://example.com/my-crawler/anything',
|
|
]);
|
|
});
|
|
|
|
test('enqueueLinks forwards respectRobotsTxtFile.userAgent to the robots.txt check', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
const isAllowedSpy = vitest.fn(() => true);
|
|
|
|
const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
|
|
override async getRobotsTxtFileForUrl(_: string) {
|
|
return { isAllowed: isAllowedSpy } as unknown as RobotsTxtFile;
|
|
}
|
|
})({
|
|
requestQueue,
|
|
maxConcurrency: 1,
|
|
respectRobotsTxtFile: { userAgent: 'MyCrawler' },
|
|
requestHandler: async (context) => {
|
|
if (context.request.label) return;
|
|
await context.enqueueLinks({ urls: ['http://example.com/child'], label: 'child' });
|
|
},
|
|
});
|
|
|
|
await crawler.run(['http://example.com/start']);
|
|
|
|
expect(isAllowedSpy).toHaveBeenCalledWith('http://example.com/child', 'MyCrawler');
|
|
});
|
|
|
|
test('enqueueLinks should respect maxRequestsPerCrawl', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
|
|
|
|
// Will try to add 6 requests - should only add 3 due to limit
|
|
const requestsToAdd = [
|
|
'http://example.com/1',
|
|
'http://example.com/2',
|
|
'http://example.com/3',
|
|
'http://example.com/4',
|
|
'http://example.com/5',
|
|
'http://example.com/6',
|
|
];
|
|
const visitedUrls: string[] = [];
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
maxRequestsPerCrawl: 5,
|
|
requestHandler: async (context) => {
|
|
visitedUrls.push(context.request.url);
|
|
|
|
if (context.request.label) {
|
|
return;
|
|
}
|
|
|
|
crawler.stats.state.requestsFinished = 2;
|
|
|
|
await context.enqueueLinks({ urls: requestsToAdd, label: 'not-undefined' });
|
|
},
|
|
});
|
|
|
|
await crawler.run(['http://example.com']);
|
|
|
|
expect(visitedUrls).toEqual([
|
|
'http://example.com', // added by crawler.run()
|
|
'http://example.com/1',
|
|
'http://example.com/2',
|
|
]);
|
|
|
|
// Should only have added the first 2 requests (since 2 were already processed and 1 is in progress, limit allows 2 more)
|
|
expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
test('enqueueLinks limit log message should only be logged once', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
|
|
// Will try to add 10 requests with a limit of 2
|
|
const requestsToAdd = Array.from({ length: 10 }, (_, i) => `http://example.com/${i}`);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
requestHandler: async (context) => {
|
|
if (context.request.label) {
|
|
return;
|
|
}
|
|
|
|
await context.enqueueLinks({ urls: requestsToAdd, limit: 2, label: 'child' });
|
|
},
|
|
});
|
|
|
|
const infoSpy = vitest.spyOn(crawler.log, 'info');
|
|
|
|
await crawler.run(['http://example.com']);
|
|
|
|
// The enqueueLinks limit message should only appear once, not 8 times (for each skipped request)
|
|
const enqueueLimitMessages = infoSpy.mock.calls.filter(
|
|
(call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
|
|
);
|
|
expect(enqueueLimitMessages).toHaveLength(1);
|
|
});
|
|
|
|
test('enqueueLinks limit log message should be logged again on subsequent runs', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
|
|
const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/${i}`);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
requestHandler: async (context) => {
|
|
if (context.request.label) {
|
|
return;
|
|
}
|
|
|
|
await context.enqueueLinks({ urls: requestsToAdd, limit: 1, label: 'child' });
|
|
},
|
|
});
|
|
|
|
const infoSpy = vitest.spyOn(crawler.log, 'info');
|
|
|
|
// First run
|
|
await crawler.run(['http://example.com/first']);
|
|
|
|
// Second run with a new start URL
|
|
await crawler.run(['http://example.com/second']);
|
|
|
|
// The enqueueLinks limit message should appear twice (once per run)
|
|
const enqueueLimitMessages = infoSpy.mock.calls.filter(
|
|
(call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
|
|
);
|
|
expect(enqueueLimitMessages).toHaveLength(2);
|
|
});
|
|
|
|
test('enqueueLinks limit log message should be logged once per request handler, not once per run', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
|
|
// Each handler will try to add 5 URLs with a limit of 1
|
|
const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/child${i}`);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
requestHandler: async (context) => {
|
|
if (context.request.label === 'child') {
|
|
return;
|
|
}
|
|
|
|
await context.enqueueLinks({ urls: requestsToAdd, limit: 1, label: 'child' });
|
|
},
|
|
});
|
|
|
|
const infoSpy = vitest.spyOn(crawler.log, 'info');
|
|
|
|
// Single run with two initial requests - both will trigger the limit
|
|
await crawler.run(['http://example.com/first', 'http://example.com/second']);
|
|
|
|
// The enqueueLinks limit message should appear twice (once per request handler that triggered the limit)
|
|
const enqueueLimitMessages = infoSpy.mock.calls.filter(
|
|
(call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
|
|
);
|
|
expect(enqueueLimitMessages).toHaveLength(2);
|
|
});
|
|
|
|
test('maxCrawlDepth limit log message should only be logged once per run', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
|
|
// Each handler will try to add URLs that exceed maxCrawlDepth
|
|
const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/child${i}`);
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
maxCrawlDepth: 1, // Only allow depth 0 (initial) and depth 1 (first level children)
|
|
requestHandler: async (context) => {
|
|
// Stop processing at depth 2 to avoid infinite loops
|
|
if (context.request.crawlDepth >= 2) {
|
|
return;
|
|
}
|
|
|
|
// This will add requests at depth+1, so initial requests add at depth 1 (allowed)
|
|
// and depth 1 requests add at depth 2 (blocked by maxCrawlDepth)
|
|
await context.enqueueLinks({
|
|
urls: requestsToAdd,
|
|
label: `depth-${context.request.crawlDepth + 1}`,
|
|
});
|
|
},
|
|
});
|
|
|
|
const infoSpy = vitest.spyOn(crawler.log, 'info');
|
|
|
|
// Run with two initial requests
|
|
// Each will enqueue children at depth 1, then those children will try to enqueue at depth 2 (blocked)
|
|
await crawler.run(['http://example.com/first', 'http://example.com/second']);
|
|
|
|
// The maxCrawlDepth limit message should only appear once per run, even though multiple requests triggered it
|
|
const maxCrawlDepthMessages = infoSpy.mock.calls.filter(
|
|
(call) => typeof call[0] === 'string' && call[0].includes('maxCrawlDepth'),
|
|
);
|
|
expect(maxCrawlDepthMessages).toHaveLength(1);
|
|
});
|
|
|
|
test('should not count duplicate URLs toward maxRequestsPerCrawl limit (addRequests)', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
maxRequestsPerCrawl: 5,
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
// 10 duplicate links to the same URL + 1 unique link at the end
|
|
const requestsToAdd = [
|
|
...Array.from({ length: 10 }, () => 'http://example.com/same'),
|
|
'http://example.com/new',
|
|
];
|
|
|
|
await crawler.addRequests(requestsToAdd);
|
|
|
|
// Both unique URLs should have been enqueued — duplicates should not consume the budget
|
|
await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject([
|
|
{ url: 'http://example.com/same' },
|
|
{ url: 'http://example.com/new' },
|
|
]);
|
|
});
|
|
|
|
test('addRequestsBatched with maxNewRequests should correctly report requestsOverLimit for array input', async () => {
|
|
const queue = await RequestQueue.open();
|
|
|
|
const result = await queue.addRequestsBatched(
|
|
[
|
|
{ url: 'http://example.com/a' },
|
|
{ url: 'http://example.com/b' },
|
|
{ url: 'http://example.com/c' },
|
|
{ url: 'http://example.com/d' },
|
|
{ url: 'http://example.com/e' },
|
|
],
|
|
{ maxNewRequests: 2 },
|
|
);
|
|
|
|
const addedUrls = result.addedRequests.filter((r) => !r.wasAlreadyPresent).map((r) => r.uniqueKey);
|
|
|
|
const overLimitUrls = (result.requestsOverLimit ?? []).map((r) => (typeof r === 'string' ? r : r.url));
|
|
|
|
expect(addedUrls).toHaveLength(2);
|
|
expect(overLimitUrls).toHaveLength(3);
|
|
});
|
|
|
|
test('addRequestsBatched with maxNewRequests should correctly report requestsOverLimit for generator input', async () => {
|
|
const queue = await RequestQueue.open();
|
|
|
|
async function* urls() {
|
|
yield { url: 'http://example.com/a' };
|
|
yield { url: 'http://example.com/b' };
|
|
yield { url: 'http://example.com/c' };
|
|
yield { url: 'http://example.com/d' };
|
|
yield { url: 'http://example.com/e' };
|
|
}
|
|
|
|
const result = await queue.addRequestsBatched(urls(), { maxNewRequests: 2 });
|
|
|
|
const addedUrls = result.addedRequests.filter((r) => !r.wasAlreadyPresent).map((r) => r.uniqueKey);
|
|
|
|
const overLimitUrls = (result.requestsOverLimit ?? []).map((r) => (typeof r === 'string' ? r : r.url));
|
|
|
|
expect(addedUrls).toHaveLength(2);
|
|
expect(overLimitUrls).toHaveLength(3);
|
|
});
|
|
|
|
test('should not count duplicate URLs toward maxRequestsPerCrawl limit (enqueueLinks)', async () => {
|
|
const requestQueue = await RequestQueue.open();
|
|
|
|
const visitedUrls: string[] = [];
|
|
|
|
const crawler = new BasicCrawler({
|
|
requestQueue,
|
|
maxRequestsPerCrawl: 5,
|
|
requestHandler: async (context) => {
|
|
visitedUrls.push(context.request.url);
|
|
|
|
if (context.request.label) {
|
|
return;
|
|
}
|
|
|
|
// Enqueue 10 duplicate links + 1 new unique link
|
|
const urls = [...Array.from({ length: 10 }, () => 'http://example.com/'), 'http://example.com/new'];
|
|
|
|
await context.enqueueLinks({ urls, label: 'child' });
|
|
},
|
|
});
|
|
|
|
await crawler.run(['http://example.com/']);
|
|
|
|
// Both the start URL and the new URL should have been visited
|
|
expect(visitedUrls).toContain('http://example.com/');
|
|
expect(visitedUrls).toContain('http://example.com/new');
|
|
});
|
|
});
|
|
|
|
describe('addRequests input validation', () => {
|
|
test('should throw error when passed a non-iterable value', async () => {
|
|
const crawler = new BasicCrawler({
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
await expect(crawler.addRequests(new WeakSet() as any)).rejects.toThrow(
|
|
'Expected an iterable or async iterable, got weakset',
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('Dataset helpers, crawler parallelism', () => {
|
|
const payload: Dictionary[] = [{ foo: 'bar', baz: 123 }];
|
|
const getPayload: (id: string) => Dictionary[] = (id) => [{ foo: id }];
|
|
|
|
const tmpDir = `${__dirname}/tmp/foo/bar`;
|
|
|
|
beforeAll(async () => {
|
|
await rm(tmpDir, { recursive: true, force: true });
|
|
});
|
|
|
|
test('should expose default Dataset methods', async () => {
|
|
const crawler = new BasicCrawler();
|
|
|
|
await crawler.pushData(payload);
|
|
|
|
expect((await crawler.getData()).items).toEqual(payload);
|
|
});
|
|
|
|
test('export data', async () => {
|
|
const row: Dictionary = { foo: 'bar', baz: 123 };
|
|
const crawler = new BasicCrawler();
|
|
|
|
await crawler.pushData(row);
|
|
await crawler.pushData(row);
|
|
await crawler.pushData(row);
|
|
|
|
await crawler.exportData(`${tmpDir}/result.csv`);
|
|
await crawler.exportData(`${tmpDir}/result.json`);
|
|
|
|
const csv = await readFile(`${tmpDir}/result.csv`);
|
|
expect(csv.toString()).toBe('foo,baz\nbar,123\nbar,123\nbar,123\n');
|
|
const json = await readFile(`${tmpDir}/result.json`);
|
|
expect(json.toString()).toBe(
|
|
'[\n' +
|
|
' {\n' +
|
|
' "foo": "bar",\n' +
|
|
' "baz": 123\n' +
|
|
' },\n' +
|
|
' {\n' +
|
|
' "foo": "bar",\n' +
|
|
' "baz": 123\n' +
|
|
' },\n' +
|
|
' {\n' +
|
|
' "foo": "bar",\n' +
|
|
' "baz": 123\n' +
|
|
' }\n' +
|
|
']\n',
|
|
);
|
|
|
|
await rm(`${tmpDir}/result.csv`);
|
|
await rm(`${tmpDir}/result.json`);
|
|
});
|
|
|
|
test('exports do not fail on empty dataset', async () => {
|
|
const crawler = new BasicCrawler();
|
|
|
|
await crawler.exportData(`${tmpDir}/result.csv`);
|
|
await crawler.exportData(`${tmpDir}/result.json`);
|
|
|
|
const csv = await readFile(`${tmpDir}/result.csv`);
|
|
expect(csv.toString()).toBe('');
|
|
const json = await readFile(`${tmpDir}/result.json`);
|
|
expect(json.toString()).toBe('[]\n');
|
|
|
|
await rm(`${tmpDir}/result.csv`);
|
|
await rm(`${tmpDir}/result.json`);
|
|
});
|
|
|
|
test('should expose pushData helper', async () => {
|
|
const crawler = new BasicCrawler({
|
|
requestHandler: async ({ pushData }) => pushData(payload),
|
|
});
|
|
|
|
await crawler.run([
|
|
{
|
|
url: `http://${HOSTNAME}:${port}`,
|
|
},
|
|
]);
|
|
|
|
expect((await crawler.getData()).items).toEqual(payload);
|
|
});
|
|
|
|
test('crawler.exportData works with `collectAllKeys`', async () => {
|
|
const crawler = new BasicCrawler();
|
|
await crawler.pushData([{ foo: 'bar', baz: 123 }]);
|
|
await crawler.pushData([{ foo: 'baz', qux: 456 }]);
|
|
|
|
await crawler.exportData(`${tmpDir}/result.csv`, 'csv', { collectAllKeys: true });
|
|
|
|
const csv = await readFile(`${tmpDir}/result.csv`);
|
|
expect(csv.toString()).toBe('foo,baz,qux\nbar,123,\nbaz,,456\n');
|
|
|
|
await rm(`${tmpDir}/result.csv`);
|
|
});
|
|
|
|
test("Crawlers with different Configurations don't share Datasets", async () => {
|
|
const crawlerA = new BasicCrawler({}, new Configuration({ persistStorage: false }));
|
|
const crawlerB = new BasicCrawler({}, new Configuration({ persistStorage: false }));
|
|
|
|
await crawlerA.pushData(getPayload('A'));
|
|
await crawlerB.pushData(getPayload('B'));
|
|
|
|
expect((await crawlerA.getData()).items).toEqual(getPayload('A'));
|
|
|
|
expect((await crawlerB.getData()).items).toEqual(getPayload('B'));
|
|
});
|
|
|
|
test('Crawlers with different Configurations run separately', async () => {
|
|
const crawlerA = new BasicCrawler(
|
|
{ requestHandler: () => {} },
|
|
new Configuration({ persistStorage: false }),
|
|
);
|
|
const crawlerB = new BasicCrawler(
|
|
{ requestHandler: () => {} },
|
|
new Configuration({ persistStorage: false }),
|
|
);
|
|
|
|
await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
|
|
await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);
|
|
|
|
expect(crawlerA.stats.state.requestsFinished).toBe(1);
|
|
expect(crawlerB.stats.state.requestsFinished).toBe(1);
|
|
});
|
|
|
|
test('Crawlers with different Configurations does not use global Configuration', async () => {
|
|
const getGlobalConfigSpy = vitest.spyOn(Configuration, 'getGlobalConfig');
|
|
|
|
const configA = new Configuration({ persistStorage: false });
|
|
const crawlerA = new BasicCrawler({ requestHandler: () => {} }, configA);
|
|
const configB = new Configuration({ persistStorage: false });
|
|
const crawlerB = new BasicCrawler({ requestHandler: () => {} }, configB);
|
|
|
|
await crawlerA.run([{ url: `http://${HOSTNAME}:${port}` }]);
|
|
await crawlerB.run([{ url: `http://${HOSTNAME}:${port}` }]);
|
|
|
|
expect(getGlobalConfigSpy.mock.calls.length).toBe(0);
|
|
expect(crawlerA.requestQueue?.config).toBe(configA);
|
|
expect(crawlerB.requestQueue?.config).toBe(configB);
|
|
});
|
|
});
|
|
});
|