wehub-resource-sync
1358 lines
52 KiB
TypeScript
1358 lines
52 KiB
TypeScript
import type { Server } from 'node:http';
|
|
|
|
import { BROWSER_POOL_EVENTS, BrowserPool, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool';
|
|
import { BLOCKED_STATUS_CODES } from '@crawlee/core';
|
|
import type { PuppeteerCrawlingContext, PuppeteerGoToOptions, PuppeteerRequestHandler } from '@crawlee/puppeteer';
|
|
import {
|
|
AutoscaledPool,
|
|
EnqueueStrategy,
|
|
ProxyConfiguration,
|
|
Request,
|
|
RequestList,
|
|
RequestState,
|
|
Session,
|
|
} from '@crawlee/puppeteer';
|
|
import { sleep } from '@crawlee/utils';
|
|
import type { HTTPResponse } from 'puppeteer';
|
|
import puppeteer from 'puppeteer';
|
|
|
|
import { ENV_VARS } from '@apify/consts';
|
|
import log from '@apify/log';
|
|
|
|
import { runExampleComServer } from '../../shared/_helper';
|
|
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator';
|
|
import { BrowserCrawlerTest } from './basic_browser_crawler';
|
|
|
|
describe('BrowserCrawler', () => {
|
|
let prevEnvHeadless: string;
|
|
let logLevel: number;
|
|
|
|
let serverAddress = 'http://localhost:';
|
|
let port: number;
|
|
let server: Server;
|
|
|
|
beforeAll(async () => {
|
|
prevEnvHeadless = process.env.CRAWLEE_HEADLESS!;
|
|
process.env.CRAWLEE_HEADLESS = '1';
|
|
logLevel = log.getLevel();
|
|
log.setLevel(log.LEVELS.ERROR);
|
|
|
|
[server, port] = await runExampleComServer();
|
|
serverAddress += port;
|
|
});
|
|
|
|
afterAll(async () => {
|
|
log.setLevel(logLevel);
|
|
process.env.CRAWLEE_HEADLESS = prevEnvHeadless;
|
|
server.close();
|
|
});
|
|
|
|
test.concurrent('should work', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const sources = [
|
|
{ url: `${serverAddress}/?q=1` },
|
|
{ url: `${serverAddress}/?q=2` },
|
|
{ url: `${serverAddress}/?q=3` },
|
|
{ url: `${serverAddress}/?q=4` },
|
|
{ url: `${serverAddress}/?q=5` },
|
|
{ url: `${serverAddress}/?q=6` },
|
|
];
|
|
const sourcesCopy = JSON.parse(JSON.stringify(sources));
|
|
const processed: Request[] = [];
|
|
const failed: Request[] = [];
|
|
const requestList = await RequestList.open(null, sources);
|
|
const requestHandler: PuppeteerRequestHandler = async ({ page, request, response }) => {
|
|
await page.waitForSelector('title');
|
|
|
|
expect(response!.status()).toBe(200);
|
|
request.userData.title = await page.title();
|
|
processed.push(request);
|
|
};
|
|
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
minConcurrency: 1,
|
|
maxConcurrency: 1,
|
|
requestHandler,
|
|
failedRequestHandler: async ({ request }) => {
|
|
failed.push(request);
|
|
},
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
expect(browserCrawler.autoscaledPool!.minConcurrency).toBe(1);
|
|
expect(processed).toHaveLength(6);
|
|
expect(failed).toHaveLength(0);
|
|
|
|
processed.forEach((request, id) => {
|
|
expect(request.url).toEqual(sourcesCopy[id].url);
|
|
expect(request.userData.title).toBe('Example Domain');
|
|
});
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should teardown browser pool', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: 'http://example.com/?q=1' }],
|
|
});
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
requestHandler: async () => {},
|
|
maxRequestRetries: 1,
|
|
});
|
|
|
|
// Spy on destroy and track if it was called
|
|
let destroyCalled = false;
|
|
const originalDestroy = browserCrawler.browserPool.destroy.bind(browserCrawler.browserPool);
|
|
browserCrawler.browserPool.destroy = async () => {
|
|
destroyCalled = true;
|
|
return originalDestroy();
|
|
};
|
|
|
|
await browserCrawler.run();
|
|
expect(destroyCalled).toBe(true);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should retire session after TimeoutError', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: 'http://example.com/?q=1' }],
|
|
});
|
|
class TimeoutError extends Error {}
|
|
let markBadCalled = false;
|
|
let sessionGoto!: Session;
|
|
const browserCrawler = new (class extends BrowserCrawlerTest {
|
|
protected override async _navigationHandler(
|
|
ctx: PuppeteerCrawlingContext,
|
|
): Promise<HTTPResponse | null | undefined> {
|
|
sessionGoto = ctx.session!;
|
|
const originalMarkBad = sessionGoto.markBad.bind(sessionGoto);
|
|
sessionGoto.markBad = () => {
|
|
markBadCalled = true;
|
|
return originalMarkBad();
|
|
};
|
|
throw new TimeoutError();
|
|
}
|
|
})({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
requestHandler: async () => {},
|
|
maxRequestRetries: 1,
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
expect(markBadCalled).toBe(true);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should evaluate preNavigationHooks', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: 'http://example.com/?q=1' }],
|
|
});
|
|
let isEvaluated = false;
|
|
|
|
const browserCrawler = new (class extends BrowserCrawlerTest {
|
|
protected override async _navigationHandler(
|
|
ctx: PuppeteerCrawlingContext,
|
|
gotoOptions: PuppeteerGoToOptions,
|
|
): Promise<HTTPResponse | null | undefined> {
|
|
isEvaluated = ctx.hookFinished as boolean;
|
|
return ctx.page.goto(ctx.request.url, gotoOptions);
|
|
}
|
|
})({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
requestHandler: async () => {},
|
|
maxRequestRetries: 0,
|
|
preNavigationHooks: [
|
|
async (crawlingContext) => {
|
|
await sleep(10);
|
|
crawlingContext.hookFinished = true;
|
|
},
|
|
],
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
expect(isEvaluated).toBeTruthy();
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should evaluate postNavigationHooks', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: `${serverAddress}/?q=1` }],
|
|
});
|
|
let isEvaluated = false;
|
|
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
requestHandler: async ({ hookFinished }) => {
|
|
isEvaluated = hookFinished as boolean;
|
|
},
|
|
maxRequestRetries: 0,
|
|
postNavigationHooks: [
|
|
async (crawlingContext) => {
|
|
await sleep(10);
|
|
crawlingContext.hookFinished = true;
|
|
},
|
|
],
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
expect(isEvaluated).toBeTruthy();
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('errorHandler has open page', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: `${serverAddress}/?q=1` }],
|
|
});
|
|
|
|
const result: string[] = [];
|
|
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
requestHandler: async (ctx) => {
|
|
throw new Error('Test error');
|
|
},
|
|
maxRequestRetries: 1,
|
|
errorHandler: async (ctx, error) => {
|
|
result.push(await ctx.page.evaluate(() => window.location.origin));
|
|
},
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
expect(result.length).toBe(1);
|
|
expect(result[0]).toBe(serverAddress);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('errorHandler has open page after non-timeout navigation error', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: `${serverAddress}/?q=1` }],
|
|
});
|
|
|
|
const pageClosedStates: boolean[] = [];
|
|
|
|
const browserCrawler = new (class extends BrowserCrawlerTest {
|
|
protected override async _navigationHandler(): Promise<HTTPResponse | null | undefined> {
|
|
throw new Error('net::ERR_NAME_NOT_RESOLVED');
|
|
}
|
|
})({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
requestHandler: async () => {},
|
|
maxRequestRetries: 1,
|
|
errorHandler: async (ctx) => {
|
|
pageClosedStates.push(ctx.page.isClosed());
|
|
},
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
expect(pageClosedStates).toHaveLength(1);
|
|
expect(pageClosedStates[0]).toBe(false);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should correctly track request.state', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const sources = [{ url: `${serverAddress}/?q=1` }];
|
|
const requestList = await RequestList.open(null, sources);
|
|
const requestStates: RequestState[] = [];
|
|
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
preNavigationHooks: [
|
|
async ({ request }) => {
|
|
requestStates.push(request.state);
|
|
},
|
|
],
|
|
postNavigationHooks: [
|
|
async ({ request }) => {
|
|
requestStates.push(request.state);
|
|
},
|
|
],
|
|
requestHandler: async ({ request }) => {
|
|
requestStates.push(request.state);
|
|
throw new Error('Error');
|
|
},
|
|
maxRequestRetries: 1,
|
|
errorHandler: async ({ request }) => {
|
|
requestStates.push(request.state);
|
|
},
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
expect(requestStates).toEqual([
|
|
RequestState.BEFORE_NAV,
|
|
RequestState.AFTER_NAV,
|
|
RequestState.REQUEST_HANDLER,
|
|
RequestState.ERROR_HANDLER,
|
|
RequestState.BEFORE_NAV,
|
|
RequestState.AFTER_NAV,
|
|
RequestState.REQUEST_HANDLER,
|
|
]);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should allow modifying gotoOptions by pre navigation hooks', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: `${serverAddress}/?q=1` }],
|
|
});
|
|
let optionsGoto: PuppeteerGoToOptions;
|
|
const browserCrawler = new (class extends BrowserCrawlerTest {
|
|
protected override async _navigationHandler(
|
|
ctx: PuppeteerCrawlingContext,
|
|
gotoOptions: PuppeteerGoToOptions,
|
|
): Promise<HTTPResponse | null | undefined> {
|
|
optionsGoto = gotoOptions;
|
|
return ctx.page.goto(ctx.request.url, gotoOptions);
|
|
}
|
|
})({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
requestHandler: async () => {},
|
|
maxRequestRetries: 0,
|
|
preNavigationHooks: [
|
|
async (_crawlingContext, gotoOptions) => {
|
|
gotoOptions!.timeout = 60000;
|
|
},
|
|
],
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
expect(optionsGoto!.timeout).toEqual(60000);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should ignore errors in Page.close()', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
for (let i = 0; i < 2; i++) {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: `${serverAddress}/?q=1` }],
|
|
});
|
|
let failedCalled = false;
|
|
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
requestHandler: async ({ page }) => {
|
|
page.close = async () => {
|
|
if (i === 0) {
|
|
throw new Error();
|
|
} else {
|
|
return Promise.reject(new Error());
|
|
}
|
|
};
|
|
return Promise.resolve();
|
|
},
|
|
failedRequestHandler: async () => {
|
|
failedCalled = true;
|
|
},
|
|
});
|
|
await browserCrawler.run();
|
|
expect(failedCalled).toBe(false);
|
|
}
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should respect the requestHandlerTimeoutSecs option', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: `${serverAddress}/?q=1` }],
|
|
});
|
|
|
|
const callSpy = vitest.fn();
|
|
|
|
// Use a very long delay for "bad" so it can never fire during test execution.
|
|
// The test verifies that the 500ms timeout aborts the handler before "bad" would fire.
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
requestHandler: async () => {
|
|
setTimeout(() => callSpy('good'), 300);
|
|
setTimeout(() => callSpy('bad'), 60_000);
|
|
await new Promise(() => {});
|
|
},
|
|
requestHandlerTimeoutSecs: 0.5,
|
|
maxRequestRetries: 0,
|
|
});
|
|
await browserCrawler.run();
|
|
|
|
expect(callSpy).toBeCalledTimes(1);
|
|
expect(callSpy).toBeCalledWith('good');
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should not throw without SessionPool', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: 'http://example.com/?q=1' }],
|
|
});
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: false,
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
expect(browserCrawler).toBeDefined();
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should correctly set session pool options', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: 'http://example.com/?q=1' }],
|
|
});
|
|
|
|
const crawler = new BrowserCrawlerTest({
|
|
requestList,
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
useSessionPool: true,
|
|
persistCookiesPerSession: false,
|
|
sessionPoolOptions: {
|
|
sessionOptions: {
|
|
maxUsageCount: 1,
|
|
},
|
|
persistStateKeyValueStoreId: 'abc',
|
|
},
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
// @ts-expect-error Accessing private prop
|
|
expect(crawler.sessionPoolOptions.sessionOptions.maxUsageCount).toBe(1);
|
|
// @ts-expect-error Accessing private prop
|
|
expect(crawler.sessionPoolOptions.persistStateKeyValueStoreId).toBe('abc');
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.skip('should persist cookies per session', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
const name = `list-${Math.random()}`;
|
|
const requestList = await RequestList.open({
|
|
persistStateKey: name,
|
|
persistRequestsKey: name,
|
|
sources: [
|
|
{ url: 'http://example.com/?q=1' },
|
|
{ url: 'http://example.com/?q=2' },
|
|
{ url: 'http://example.com/?q=3' },
|
|
{ url: 'http://example.com/?q=4' },
|
|
],
|
|
});
|
|
const goToPageSessions = [];
|
|
const loadedCookies: string[] = [];
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
persistCookiesPerSession: true,
|
|
requestHandler: async ({ session, request }) => {
|
|
loadedCookies.push(session!.getCookieString(request.url));
|
|
return Promise.resolve();
|
|
},
|
|
preNavigationHooks: [
|
|
async ({ session, page }) => {
|
|
await page.setCookie({
|
|
name: 'TEST',
|
|
value: '12321312312',
|
|
domain: 'example.com',
|
|
expires: Date.now() + 100000,
|
|
});
|
|
goToPageSessions.push(session);
|
|
},
|
|
],
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
expect(loadedCookies).toHaveLength(4);
|
|
|
|
loadedCookies.forEach((cookie) => {
|
|
// TODO this test is flaky in CI and we need some more info to debug why.
|
|
if (cookie !== 'TEST=12321312312') {
|
|
// for some reason, the CI failures report the first cookie to be just empty string
|
|
|
|
console.log('loadedCookies:');
|
|
|
|
console.dir(loadedCookies);
|
|
}
|
|
|
|
expect(cookie).toEqual('TEST=12321312312');
|
|
});
|
|
});
|
|
|
|
test.concurrent('should throw on "blocked" status codes', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const baseUrl = 'https://example.com/';
|
|
const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
|
|
return {
|
|
url: baseUrl + statusCode,
|
|
userData: { statusCode },
|
|
};
|
|
});
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
let called = false;
|
|
const failedRequests: Request[] = [];
|
|
const crawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
persistCookiesPerSession: false,
|
|
maxRequestRetries: 0,
|
|
requestHandler: async () => {
|
|
called = true;
|
|
},
|
|
failedRequestHandler: async ({ request }) => {
|
|
failedRequests.push(request);
|
|
},
|
|
});
|
|
|
|
// @ts-expect-error Overriding protected method
|
|
crawler._navigationHandler = async ({ request }) => {
|
|
return { status: () => request.userData.statusCode };
|
|
};
|
|
|
|
await crawler.run();
|
|
|
|
expect(failedRequests.length).toBe(3);
|
|
failedRequests.forEach((fr) => {
|
|
const [msg] = fr.errorMessages;
|
|
expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
|
|
});
|
|
expect(called).toBe(false);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('retryOnBlocked should retry on Cloudflare challenge', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const urls = [new URL('/special/cloudflareBlocking', serverAddress).href];
|
|
const maxSessionRotations = 1;
|
|
|
|
let processed = false;
|
|
const errorMessages: string[] = [];
|
|
|
|
const crawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
retryOnBlocked: true,
|
|
maxSessionRotations,
|
|
requestHandler: async ({ page, response }) => {
|
|
processed = true;
|
|
},
|
|
failedRequestHandler: async ({ request }) => {
|
|
errorMessages.push(...request.errorMessages);
|
|
},
|
|
});
|
|
|
|
await crawler.run(urls);
|
|
|
|
expect(errorMessages).toHaveLength(urls.length * (maxSessionRotations + 1));
|
|
expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
|
|
expect(processed).toBe(false);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('retryOnBlocked throws on "blocked" status codes', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const baseUrl = 'https://example.com/';
|
|
const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
|
|
return {
|
|
url: baseUrl + statusCode,
|
|
userData: { statusCode },
|
|
};
|
|
});
|
|
const requestList = await RequestList.open(null, sources);
|
|
const maxSessionRotations = 1;
|
|
const errorMessages: string[] = [];
|
|
|
|
let processed = false;
|
|
const crawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
retryOnBlocked: true,
|
|
maxSessionRotations,
|
|
requestHandler: async () => {
|
|
processed = true;
|
|
},
|
|
failedRequestHandler: async ({ request }) => {
|
|
errorMessages.push(...request.errorMessages);
|
|
},
|
|
});
|
|
|
|
// @ts-expect-error Overriding protected method
|
|
crawler._navigationHandler = async ({ request }) => {
|
|
return { status: () => request.userData.statusCode };
|
|
};
|
|
|
|
await crawler.run();
|
|
|
|
expect(errorMessages.length).toBe(sources.length * (maxSessionRotations + 1));
|
|
expect(errorMessages.every((x) => x.includes('Detected a session error, rotating session...'))).toBe(true);
|
|
expect(processed).toBe(false);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should throw on "blocked" status codes (retire session)', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const baseUrl = 'https://example.com/';
|
|
const sources = BLOCKED_STATUS_CODES.map((statusCode) => {
|
|
return {
|
|
url: baseUrl + statusCode,
|
|
userData: { statusCode },
|
|
};
|
|
});
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
let called = false;
|
|
const failedRequests: Request[] = [];
|
|
const crawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
persistCookiesPerSession: false,
|
|
maxRequestRetries: 0,
|
|
requestHandler: async () => {
|
|
called = true;
|
|
},
|
|
failedRequestHandler: async ({ request }) => {
|
|
failedRequests.push(request);
|
|
},
|
|
});
|
|
|
|
// @ts-expect-error Overriding protected method
|
|
crawler._navigationHandler = async ({ request }) => {
|
|
return { status: () => request.userData.statusCode };
|
|
};
|
|
|
|
await crawler.run();
|
|
|
|
expect(failedRequests.length).toBe(3);
|
|
failedRequests.forEach((fr) => {
|
|
const [msg] = fr.errorMessages;
|
|
expect(msg).toContain(`Request blocked - received ${fr.userData.statusCode} status code.`);
|
|
});
|
|
expect(called).toBe(false);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should retire browser with session', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: 'http://example.com/?q=1' }],
|
|
});
|
|
let resolve: (value?: unknown) => void;
|
|
|
|
const retirementPromise = new Promise((r) => {
|
|
resolve = r;
|
|
});
|
|
let called = false;
|
|
const browserCrawler = new (class extends BrowserCrawlerTest {
|
|
protected override async _navigationHandler(
|
|
ctx: PuppeteerCrawlingContext,
|
|
): Promise<HTTPResponse | null | undefined> {
|
|
ctx.crawler.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, () => {
|
|
resolve();
|
|
called = true;
|
|
});
|
|
ctx.session!.retire();
|
|
return ctx.page.goto(ctx.request.url);
|
|
}
|
|
})({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
useSessionPool: true,
|
|
requestHandler: async () => {
|
|
await retirementPromise;
|
|
},
|
|
maxRequestRetries: 1,
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
expect(called).toBeTruthy();
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should increment session usage correctly', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const sessionUsageHistory: number[] = [];
|
|
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
useSessionPool: true,
|
|
sessionPoolOptions: {
|
|
maxPoolSize: 1,
|
|
},
|
|
requestHandler: async ({ session }) => {
|
|
sessionUsageHistory.push(session!.usageCount);
|
|
},
|
|
});
|
|
|
|
await browserCrawler.run([
|
|
{ url: `${serverAddress}/?q=1` },
|
|
{ url: `${serverAddress}/?q=2` },
|
|
{ url: `${serverAddress}/?q=3` },
|
|
{ url: `${serverAddress}/?q=4` },
|
|
{ url: `${serverAddress}/?q=5` },
|
|
{ url: `${serverAddress}/?q=6` },
|
|
]);
|
|
|
|
expect(sessionUsageHistory).toEqual([0, 1, 2, 3, 4, 5]);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('should allow using fingerprints from browser pool', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [{ url: `${serverAddress}/?q=1` }],
|
|
});
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
useFingerprints: true,
|
|
fingerprintOptions: {
|
|
fingerprintGeneratorOptions: {
|
|
operatingSystems: [OperatingSystemsName.windows],
|
|
},
|
|
},
|
|
},
|
|
requestList,
|
|
useSessionPool: false,
|
|
requestHandler: async ({ browserController }) => {
|
|
expect(browserController.launchContext.fingerprint).toBeDefined();
|
|
},
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
expect.hasAssertions();
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
describe('proxy', () => {
|
|
// TODO move to actor sdk tests before splitting the repos
|
|
// test('browser should launch with correct proxyUrl', async () => {
|
|
// process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
|
|
// const status = { connected: true };
|
|
// const fakeCall = async () => {
|
|
// return { body: status } as never;
|
|
// };
|
|
//
|
|
// // @ts-expect-error FIXME
|
|
// const stub = gotScrapingSpy.mockImplementation(fakeCall);
|
|
// const proxyConfiguration = await Actor.createProxyConfiguration();
|
|
// const generatedProxyUrl = new URL(await proxyConfiguration.newUrl()).href.slice(0, -1);
|
|
// let browserProxy;
|
|
//
|
|
// const browserCrawler = new BrowserCrawlerTest({
|
|
// browserPoolOptions: {
|
|
// browserPlugins: [puppeteerPlugin],
|
|
// postLaunchHooks: [(pageId, browserController) => {
|
|
// browserProxy = browserController.launchContext.proxyUrl;
|
|
// }],
|
|
// },
|
|
// useSessionPool: false,
|
|
// persistCookiesPerSession: false,
|
|
// navigationTimeoutSecs: 1,
|
|
// requestList,
|
|
// maxRequestsPerCrawl: 1,
|
|
// maxRequestRetries: 0,
|
|
// requestHandler: async () => {},
|
|
// proxyConfiguration,
|
|
// });
|
|
// await browserCrawler.run();
|
|
// delete process.env[ENV_VARS.PROXY_PASSWORD];
|
|
//
|
|
// expect(browserProxy).toEqual(generatedProxyUrl);
|
|
//
|
|
// stub.mockClear();
|
|
// });
|
|
|
|
// TODO move to actor sdk tests before splitting the repos
|
|
// test('requestHandler should expose the proxyInfo object with sessions correctly', async () => {
|
|
// process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
|
|
// const status = { connected: true };
|
|
// const fakeCall = async () => {
|
|
// return { body: status } as never;
|
|
// };
|
|
//
|
|
// // @ts-expect-error FIXME
|
|
// const stub = gotScrapingSpy.mockImplementation(fakeCall);
|
|
//
|
|
// const proxyConfiguration = await Actor.createProxyConfiguration();
|
|
// const proxies: ProxyInfo[] = [];
|
|
// const sessions: Session[] = [];
|
|
// const requestHandler = async ({ session, proxyInfo }: BrowserCrawlingContext) => {
|
|
// proxies.push(proxyInfo);
|
|
// sessions.push(session);
|
|
// };
|
|
//
|
|
// const browserCrawler = new BrowserCrawlerTest({
|
|
// browserPoolOptions: {
|
|
// browserPlugins: [puppeteerPlugin],
|
|
// },
|
|
// requestList,
|
|
// requestHandler,
|
|
//
|
|
// proxyConfiguration,
|
|
// useSessionPool: true,
|
|
// sessionPoolOptions: {
|
|
// maxPoolSize: 1,
|
|
// },
|
|
// });
|
|
//
|
|
// await browserCrawler.run();
|
|
//
|
|
// expect(proxies[0].sessionId).toEqual(sessions[0].id);
|
|
// expect(proxies[1].sessionId).toEqual(sessions[1].id);
|
|
// expect(proxies[2].sessionId).toEqual(sessions[2].id);
|
|
// expect(proxies[3].sessionId).toEqual(sessions[3].id);
|
|
//
|
|
// delete process.env[ENV_VARS.PROXY_PASSWORD];
|
|
// stub.mockClear();
|
|
// });
|
|
|
|
// This test manipulates environment variables, so it must NOT be run concurrently
|
|
test('browser should launch with rotated custom proxy', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';
|
|
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{ url: `${serverAddress}/?q=1` },
|
|
{ url: `${serverAddress}/?q=2` },
|
|
{ url: `${serverAddress}/?q=3` },
|
|
],
|
|
});
|
|
|
|
const proxyConfiguration = new ProxyConfiguration({
|
|
proxyUrls: ['http://proxy.com:1111', 'http://proxy.com:2222', 'http://proxy.com:3333'],
|
|
});
|
|
|
|
const browserProxies: string[] = [];
|
|
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
maxOpenPagesPerBrowser: 1,
|
|
retireBrowserAfterPageCount: 1,
|
|
},
|
|
requestList,
|
|
requestHandler: async () => {},
|
|
proxyConfiguration,
|
|
maxRequestRetries: 0,
|
|
maxConcurrency: 1,
|
|
});
|
|
|
|
browserCrawler.browserPool.postLaunchHooks.push((_pageId, browserController) => {
|
|
browserProxies.push(browserController.launchContext.proxyUrl!);
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
|
|
// @ts-expect-error Accessing private property
|
|
const proxiesToUse = proxyConfiguration.proxyUrls!;
|
|
for (const proxyUrl of proxiesToUse) {
|
|
expect(browserProxies.includes(new URL(proxyUrl!).href.slice(0, -1))).toBeTruthy();
|
|
}
|
|
|
|
delete process.env[ENV_VARS.PROXY_PASSWORD];
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('proxy rotation on error works as expected', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{ url: 'http://example.com/?q=1' },
|
|
{ url: 'http://example.com/?q=2' },
|
|
{ url: 'http://example.com/?q=3' },
|
|
{ url: 'http://example.com/?q=4' },
|
|
],
|
|
});
|
|
|
|
const goodProxyUrl = 'http://good.proxy';
|
|
const proxyConfiguration = new ProxyConfiguration({
|
|
proxyUrls: ['http://localhost', 'http://localhost:1234', goodProxyUrl],
|
|
});
|
|
const requestHandler = vitest.fn();
|
|
|
|
const browserCrawler = new (class extends BrowserCrawlerTest {
|
|
protected override async _navigationHandler(
|
|
ctx: PuppeteerCrawlingContext,
|
|
): Promise<HTTPResponse | null | undefined> {
|
|
const { session } = ctx;
|
|
const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
|
|
|
|
if (proxyInfo!.url !== goodProxyUrl) {
|
|
throw new Error('ERR_PROXY_CONNECTION_FAILED');
|
|
}
|
|
|
|
return null;
|
|
}
|
|
})({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
maxRequestRetries: 0,
|
|
maxConcurrency: 1,
|
|
useSessionPool: true,
|
|
proxyConfiguration,
|
|
requestHandler,
|
|
});
|
|
|
|
await expect(browserCrawler.run()).resolves.not.toThrow();
|
|
expect(requestHandler).toHaveBeenCalledTimes(requestList!.length());
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('proxy rotation on error respects maxSessionRotations, calls failedRequestHandler', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{ url: 'http://example.com/?q=1' },
|
|
{ url: 'http://example.com/?q=2' },
|
|
{ url: 'http://example.com/?q=3' },
|
|
{ url: 'http://example.com/?q=4' },
|
|
],
|
|
});
|
|
|
|
const proxyConfiguration = new ProxyConfiguration({
|
|
proxyUrls: ['http://localhost', 'http://localhost:1234'],
|
|
});
|
|
const failedRequestHandler = vitest.fn();
|
|
|
|
/**
|
|
* The first increment is the base case when the proxy is retrieved for the first time.
|
|
*/
|
|
let numberOfRotations = -requestList!.length();
|
|
const browserCrawler = new (class extends BrowserCrawlerTest {
|
|
protected override async _navigationHandler(
|
|
ctx: PuppeteerCrawlingContext,
|
|
): Promise<HTTPResponse | null | undefined> {
|
|
const { session } = ctx;
|
|
const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
|
|
|
|
numberOfRotations++;
|
|
|
|
if (proxyInfo!.url.includes('localhost')) {
|
|
throw new Error('ERR_PROXY_CONNECTION_FAILED');
|
|
}
|
|
|
|
return null;
|
|
}
|
|
})({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
maxSessionRotations: 5,
|
|
maxConcurrency: 1,
|
|
proxyConfiguration,
|
|
requestHandler: async () => {},
|
|
failedRequestHandler,
|
|
});
|
|
|
|
await browserCrawler.run();
|
|
expect(failedRequestHandler).toBeCalledTimes(requestList!.length());
|
|
expect(numberOfRotations).toBe(requestList!.length() * 5);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test.concurrent('proxy rotation logs the original proxy error', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{ url: 'http://example.com/?q=1' },
|
|
{ url: 'http://example.com/?q=2' },
|
|
{ url: 'http://example.com/?q=3' },
|
|
{ url: 'http://example.com/?q=4' },
|
|
],
|
|
});
|
|
|
|
const proxyConfiguration = new ProxyConfiguration({ proxyUrls: ['http://localhost:1234'] });
|
|
|
|
const proxyError =
|
|
'Proxy responded with 400 - Bad request. Also, this error message contains some useful payload.';
|
|
|
|
const crawler = new (class extends BrowserCrawlerTest {
|
|
protected override async _navigationHandler(
|
|
ctx: PuppeteerCrawlingContext,
|
|
): Promise<HTTPResponse | null | undefined> {
|
|
const { session } = ctx;
|
|
const proxyInfo = await this.proxyConfiguration!.newProxyInfo(session?.id);
|
|
|
|
if (proxyInfo!.url.includes('localhost')) {
|
|
throw new Error(proxyError);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
})({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
maxSessionRotations: 1,
|
|
maxConcurrency: 1,
|
|
proxyConfiguration,
|
|
requestHandler: async () => {},
|
|
});
|
|
|
|
const spy = vitest.spyOn((crawler as any).log, 'warning' as any).mockImplementation(() => {});
|
|
|
|
await crawler.run([serverAddress]);
|
|
|
|
expect(spy).toBeCalled();
|
|
expect(spy.mock.calls[0][0]).toEqual(
|
|
'When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.',
|
|
);
|
|
expect(spy.mock.calls[1][0]).toEqual(expect.stringContaining(proxyError));
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('Crawling context', () => {
|
|
// This describe block manipulates log levels (global state), so tests must NOT be concurrent
|
|
|
|
test('uses correct crawling context', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
const actualLogLevel = log.getLevel();
|
|
log.setLevel(log.LEVELS.OFF);
|
|
|
|
try {
|
|
const sources = ['http://example.com/'];
|
|
const requestList = await RequestList.open(null, sources.slice());
|
|
|
|
let prepareCrawlingContext: PuppeteerCrawlingContext;
|
|
|
|
const gotoFunction = async (crawlingContext: PuppeteerCrawlingContext) => {
|
|
prepareCrawlingContext = crawlingContext;
|
|
expect(crawlingContext.request).toBeInstanceOf(Request);
|
|
expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
|
|
expect(crawlingContext.session).toBeInstanceOf(Session);
|
|
expect(typeof crawlingContext.page).toBe('object');
|
|
};
|
|
|
|
const requestHandler = async (crawlingContext: PuppeteerCrawlingContext) => {
|
|
expect(crawlingContext === prepareCrawlingContext).toEqual(true);
|
|
expect(crawlingContext.request).toBeInstanceOf(Request);
|
|
expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
|
|
expect(crawlingContext.session).toBeInstanceOf(Session);
|
|
expect(typeof crawlingContext.page).toBe('object');
|
|
expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);
|
|
expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);
|
|
|
|
throw new Error('some error');
|
|
};
|
|
|
|
const failedRequestHandler = async (crawlingContext: PuppeteerCrawlingContext, error: Error) => {
|
|
expect(crawlingContext).toBe(prepareCrawlingContext);
|
|
expect(crawlingContext.request).toBeInstanceOf(Request);
|
|
expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);
|
|
expect(crawlingContext.session).toBeInstanceOf(Session);
|
|
expect(typeof crawlingContext.page).toBe('object');
|
|
expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);
|
|
expect(crawlingContext.crawler.browserPool).toBeInstanceOf(BrowserPool);
|
|
expect(Object.hasOwn(crawlingContext, 'response')).toBe(true);
|
|
|
|
expect(crawlingContext.error).toBeInstanceOf(Error);
|
|
expect(error).toBeInstanceOf(Error);
|
|
expect(error.message).toEqual('some error');
|
|
};
|
|
|
|
const browserCrawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
requestList,
|
|
maxRequestRetries: 0,
|
|
maxConcurrency: 1,
|
|
useSessionPool: true,
|
|
requestHandler,
|
|
failedRequestHandler,
|
|
});
|
|
// @ts-expect-error Overriding protected method
|
|
browserCrawler._navigationHandler = gotoFunction;
|
|
|
|
await browserCrawler.run();
|
|
} finally {
|
|
log.setLevel(actualLogLevel);
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
});
|
|
|
|
// These tests cannot run concurrently because they use crawler.run([urls])
|
|
// which creates internal request queues that can conflict
|
|
test("enqueueLinks() should skip links that don't match the strategy post redirect", async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const succeeded: string[] = [];
|
|
|
|
const crawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
maxConcurrency: 1,
|
|
maxRequestRetries: 0,
|
|
requestHandler: async ({ page, enqueueLinks }) => {
|
|
succeeded.push(await page.title());
|
|
await enqueueLinks({ strategy: EnqueueStrategy.SameOrigin });
|
|
},
|
|
});
|
|
|
|
await crawler.run([`${serverAddress}/special/redirect`]);
|
|
|
|
expect(succeeded).toHaveLength(1);
|
|
expect(succeeded[0]).toEqual('Redirecting outside');
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
|
|
test('enqueueLinks should respect maxCrawlDepth', async () => {
|
|
const localStorageEmulator = new MemoryStorageEmulator();
|
|
await localStorageEmulator.init();
|
|
const puppeteerPlugin = new PuppeteerPlugin(puppeteer);
|
|
|
|
try {
|
|
const succeeded: string[] = [];
|
|
|
|
const crawler = new BrowserCrawlerTest({
|
|
browserPoolOptions: {
|
|
browserPlugins: [puppeteerPlugin],
|
|
},
|
|
maxCrawlDepth: 1,
|
|
maxRequestsPerCrawl: 10, // avoiding accidental runaway
|
|
requestHandler: async ({ page, enqueueLinks }) => {
|
|
succeeded.push(await page.title());
|
|
await enqueueLinks({ strategy: EnqueueStrategy.All });
|
|
},
|
|
});
|
|
|
|
await crawler.run([`${serverAddress}/special/html-type`]);
|
|
|
|
expect(succeeded).toHaveLength(2);
|
|
expect(succeeded).toEqual(['Example Domain', 'Example Domains']);
|
|
} finally {
|
|
await localStorageEmulator.destroy();
|
|
}
|
|
});
|
|
});
|