/* eslint-disable dot-notation -- Accessing private properties */ import http from 'node:http'; import { promisify } from 'node:util'; import { sleep } from 'crawlee'; import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator'; import playwright from 'playwright'; import type { Server as ProxyChainServer } from 'proxy-chain'; import type { Page } from 'puppeteer'; import puppeteer from 'puppeteer'; import { addTimeoutToPromise } from '@apify/timeout'; import type { BrowserController } from '../../packages/browser-pool/src/abstract-classes/browser-controller'; import { BrowserPool } from '../../packages/browser-pool/src/browser-pool'; import { BROWSER_POOL_EVENTS } from '../../packages/browser-pool/src/events'; import { BrowserName, OperatingSystemsName } from '../../packages/browser-pool/src/fingerprinting/types'; import { PlaywrightPlugin } from '../../packages/browser-pool/src/playwright/playwright-plugin'; import { PuppeteerPlugin } from '../../packages/browser-pool/src/puppeteer/puppeteer-plugin'; import { createProxyServer } from './browser-plugins/create-proxy-server'; const fingerprintingMatrix: [string, PlaywrightPlugin | PuppeteerPlugin][] = [ [ 'Playwright - persistent', new PlaywrightPlugin(playwright.chromium, { useIncognitoPages: false, }), ], [ 'Playwright - Incognito', new PlaywrightPlugin(playwright.chromium, { useIncognitoPages: true, }), ], [ 'Puppeteer - Persistent', new PuppeteerPlugin(puppeteer, { useIncognitoPages: false, }), ], [ 'Puppeteer - Incognito', new PuppeteerPlugin(puppeteer, { useIncognitoPages: true, }), ], ]; // Tests could be generated from this blueprint for each plugin describe.each([ ['Puppeteer', new PuppeteerPlugin(puppeteer)], ['Playwright', new PlaywrightPlugin(playwright.chromium)], // Chromium is faster than firefox and webkit ])('BrowserPool - %s', (_, plugin) => { let browserPool: BrowserPool<{ browserPlugins: [typeof plugin]; closeInactiveBrowserAfterSecs: 2 }>; beforeEach(async () => { vitest.clearAllMocks(); browserPool = new BrowserPool({ browserPlugins: [plugin], closeInactiveBrowserAfterSecs: 2, retireInactiveBrowserAfterSecs: 2, }); }); afterEach(async () => { await browserPool?.destroy(); }); let target: http.Server; let unprotectedProxy: ProxyChainServer; let protectedProxy: ProxyChainServer; beforeAll(async () => { target = http.createServer((request, response) => { response.end(request.socket.remoteAddress); }); await promisify(target.listen.bind(target) as any)(0, '127.0.0.1'); unprotectedProxy = createProxyServer('127.0.0.2', '', ''); await unprotectedProxy.listen(); protectedProxy = createProxyServer('127.0.0.3', 'foo', 'bar'); await protectedProxy.listen(); }); afterAll(async () => { await promisify(target.close.bind(target))(); await unprotectedProxy.close(false); await protectedProxy.close(false); }); describe('Initialization & retirement', () => { test('should retire browsers', async () => { await browserPool.newPage(); browserPool.retireAllBrowsers(); expect(browserPool.startingBrowserControllers.size).toBe(0); expect(browserPool.activeBrowserControllers.size).toBe(0); expect(browserPool.retiredBrowserControllers.size).toBe(1); }); test('should destroy pool', async () => { const page = await browserPool.newPage(); const browserController = browserPool.getBrowserControllerByPage(page)!; vitest.spyOn(browserController, 'close'); await browserPool.destroy(); expect(browserController.close).toHaveBeenCalled(); expect(browserPool.startingBrowserControllers.size).toBe(0); expect(browserPool.activeBrowserControllers.size).toBe(0); expect(browserPool.retiredBrowserControllers.size).toBe(0); expect(browserPool['browserKillerInterval']).toBeUndefined(); }); }); describe('Basic user functionality', () => { // Basic user facing functionality test('should open new page', async () => { const page = await browserPool.newPage(); expect(page.goto).toBeDefined(); expect(page.close).toBeDefined(); }); // https://github.com/apify/crawlee/issues/3670 test('should not leak aborted cancelTask between concurrent newPage calls', async () => { const previousTimeout = browserPool.operationTimeoutMillis; browserPool.operationTimeoutMillis = 1; // Each newPage call is wrapped in its own outer addTimeoutToPromise, // matching how BasicCrawler wraps _runRequestHandler. Without the fix, // queued limiter callbacks inherit the previous task's aborted // cancelTask via AsyncLocalStorage propagation through p-limit, causing // their first tryCancel() to throw InternalTimeoutError pre-emptively; // that error then gets silently swallowed by the outer wrap. const results = await Promise.allSettled( Array.from({ length: 5 }, () => addTimeoutToPromise(async () => browserPool.newPage(), 60_000, 'outer timed out'), ), ); browserPool.operationTimeoutMillis = previousTimeout; browserPool.retireAllBrowsers(); // All calls must reject — none should silently resolve with undefined. expect(results.every((r) => r.status === 'rejected')).toBe(true); // Each rejection should reflect this call's own newPage timeout, not // a leaked "canceled due to a timeout" from a sibling's aborted context. for (const r of results) { expect(r.status === 'rejected' && (r.reason as Error).message).toMatch( /browserController\.newPage\(\) (failed|timed out)/, ); } }); // TODO: this test is very flaky in the CI test.skip('should allow early aborting in case of outer timeout', async () => { const timeout = browserPool.operationTimeoutMillis; browserPool.operationTimeoutMillis = 500; // @ts-expect-error mocking private method const spy = vitest.spyOn(BrowserPool.prototype, '_executeHooks'); await browserPool.newPage(); expect(spy).toBeCalledTimes(4); spy.mockReset(); await expect( addTimeoutToPromise(async () => browserPool.newPage(), 10, 'opening new page timed out'), ).rejects.toThrowError('opening new page timed out'); // We terminated early enough so only preLaunchHooks were not executed, // thanks to `tryCancel()` calls after each await. If we did not run // inside `addTimeoutToPromise()`, this would not work and we would get // 4 calls instead of just one. expect(spy).toBeCalledTimes(1); browserPool.operationTimeoutMillis = timeout; browserPool.retireAllBrowsers(); }); test('should open new page in incognito context', async () => { const browserPoolIncognito = new BrowserPool({ browserPlugins: [new PlaywrightPlugin(playwright.chromium, { useIncognitoPages: true })], closeInactiveBrowserAfterSecs: 2, }); const page = await browserPoolIncognito.newPage(); await browserPoolIncognito.newPage(); await browserPoolIncognito.newPage(); expect(page.context().pages()).toHaveLength(1); }); test('should open new page in new browser', async () => { vitest.spyOn(plugin, 'launch'); await browserPool.newPage(); await browserPool.newPageInNewBrowser(); await browserPool.newPageInNewBrowser(); expect(browserPool.startingBrowserControllers.size).toBe(0); expect(browserPool.activeBrowserControllers.size).toBe(3); expect(plugin.launch).toHaveBeenCalledTimes(3); }); test('should correctly override page close', async () => { // @ts-expect-error Private function vitest.spyOn(browserPool!, '_overridePageClose'); const page = await browserPool.newPage(); expect(browserPool['_overridePageClose']).toBeCalled(); const controller = browserPool.getBrowserControllerByPage(page)!; expect(controller.activePages).toEqual(1); expect(controller.totalPages).toEqual(1); await page.close(); expect(controller.activePages).toEqual(0); expect(controller.totalPages).toEqual(1); }); test('should retire browser after page count', async () => { browserPool.retireBrowserAfterPageCount = 2; vitest.spyOn(browserPool, 'retireBrowserController'); expect(browserPool.activeBrowserControllers.size).toBe(0); await browserPool.newPage(); await browserPool.newPage(); await browserPool.newPage(); expect(browserPool.activeBrowserControllers.size).toBe(1); expect(browserPool.retiredBrowserControllers.size).toBe(1); expect(browserPool.retireBrowserController).toBeCalledTimes(1); }); test('should allow max pages per browser', async () => { browserPool.maxOpenPagesPerBrowser = 1; // @ts-expect-error Private function vitest.spyOn(browserPool!, '_launchBrowser'); await browserPool.newPage(); expect(browserPool.activeBrowserControllers.size).toBe(1); await browserPool.newPage(); expect(browserPool.activeBrowserControllers.size).toBe(2); await browserPool.newPage(); expect(browserPool.activeBrowserControllers.size).toBe(3); expect(browserPool['_launchBrowser']).toBeCalledTimes(3); }); test('should allow max pages per browser - no race condition', async () => { browserPool.maxOpenPagesPerBrowser = 1; // @ts-expect-error Private function vitest.spyOn(browserPool, '_launchBrowser'); const usePlugin = { browserPlugin: plugin, }; await Promise.all([browserPool.newPage(usePlugin), browserPool.newPage(usePlugin)]); expect(browserPool.activeBrowserControllers.size).toBe(2); expect(browserPool['_launchBrowser']).toBeCalledTimes(2); }); test('should close retired browsers', async () => { browserPool.retireBrowserAfterPageCount = 1; clearInterval(browserPool['browserKillerInterval']!); browserPool['browserKillerInterval'] = setInterval( async () => browserPool['_closeInactiveRetiredBrowsers'](), 100, ); // @ts-expect-error Private function vitest.spyOn(browserPool!, '_closeRetiredBrowserWithNoPages'); expect(browserPool.retiredBrowserControllers.size).toBe(0); const page = await browserPool.newPage(); const controller = browserPool.getBrowserControllerByPage(page)!; vitest.spyOn(controller, 'close'); expect(browserPool.retiredBrowserControllers.size).toBe(1); await page.close(); await new Promise((resolve) => setTimeout(() => { resolve(); }, 1000), ); expect(browserPool['_closeRetiredBrowserWithNoPages']).toHaveBeenCalled(); expect(controller.close).toHaveBeenCalled(); expect(browserPool.retiredBrowserControllers.size).toBe(0); }); describe('hooks', () => { test('should run hooks in series with custom args', async () => { const indexArray: number[] = []; const createAsyncHookReturningIndex = (i: number) => async () => { const index = await new Promise((resolve) => setTimeout(() => resolve(i), 100)); indexArray.push(index); }; const hooks = new Array(10); for (let i = 0; i < hooks.length; i++) { hooks[i] = createAsyncHookReturningIndex(i); } await browserPool['_executeHooks'](hooks); expect(indexArray).toHaveLength(10); indexArray.forEach((v, index) => expect(v).toEqual(index)); }); test('browser lifecycle works correctly', async () => { let resolvePreLaunchHook: (() => void) | null = null; let resolvePostLaunchHook: (() => void) | null = null; const preLaunchPromise = new Promise((resolve) => { resolvePreLaunchHook = resolve; }); const postLaunchPromise = new Promise((resolve) => { resolvePostLaunchHook = resolve; }); browserPool.preLaunchHooks = [...browserPool.preLaunchHooks, async () => preLaunchPromise]; browserPool.postLaunchHooks = [...browserPool.postLaunchHooks, async () => postLaunchPromise]; const newPagePromise = browserPool.newPage(); await sleep(200); expect(browserPool.startingBrowserControllers.size).toBe(1); expect(browserPool.activeBrowserControllers.size).toBe(0); expect(browserPool.retiredBrowserControllers.size).toBe(0); await sleep(5e3); // Make the wait longer than the browser pool's retireInactiveBrowserAfterSecs + closeInactiveBrowserAfterSecs resolvePreLaunchHook!(); resolvePostLaunchHook!(); const page = await newPagePromise; expect(browserPool.startingBrowserControllers.size).toBe(0); expect(browserPool.activeBrowserControllers.size).toBe(1); expect(browserPool.retiredBrowserControllers.size).toBe(0); await (page.evaluate as any)('() => {}'); // Make sure the page is usable await page.close(); }); describe('preLaunchHooks', () => { test('should evaluate hook before launching browser with correct args', async () => { const myAsyncHook = async () => Promise.resolve(); browserPool.preLaunchHooks.push(myAsyncHook); // @ts-expect-error Private function vitest.spyOn(browserPool!, '_executeHooks'); const page = await browserPool.newPage(); const pageId = browserPool.getPageId(page)!; const { launchContext } = browserPool.getBrowserControllerByPage(page)!; expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith( 1, browserPool.preLaunchHooks, pageId, launchContext, ); }); // We had a problem where if the first newPage() call, which launches // a browser failed in hooks, then the browserController would get stuck // in limbo and subsequent newPage() calls would never resolve. test('error in hook does not leave browser stuck in limbo', async () => { const errorMessage = 'pre-launch failed'; browserPool.preLaunchHooks = [ async () => { throw new Error(errorMessage); }, ]; const attempts = 5; for (let i = 0; i < attempts; i++) { try { await browserPool.newPage(); } catch (err) { expect((err as Error).message).toBe(errorMessage); } } expect(browserPool.activeBrowserControllers.size).toBe(0); expect.assertions(attempts + 1); }); }); describe('postLaunchHooks', () => { test('should evaluate hook after launching browser with correct args', async () => { const myAsyncHook = async () => Promise.resolve(); browserPool.postLaunchHooks = [myAsyncHook]; // @ts-expect-error Private function vitest.spyOn(browserPool, '_executeHooks'); const page = await browserPool.newPage(); const pageId = browserPool.getPageId(page)!; const browserController = browserPool.getBrowserControllerByPage(page)!; expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith( 2, browserPool.postLaunchHooks, pageId, browserController, ); }); // We had a problem where if the first newPage() call, which launches // a browser failed in hooks, then the browserController would get stuck // in limbo and subsequent newPage() calls would never resolve. test('error in hook does not leave browser stuck in limbo', async () => { const errorMessage = 'post-launch failed'; const controllers: BrowserController[] = []; browserPool.postLaunchHooks = [ async (_pageId, browserController) => { controllers.push(browserController); throw new Error(errorMessage); }, ]; const attempts = 5; for (let i = 0; i < attempts; i++) { try { await browserPool.newPage(); } catch (err) { expect((err as Error).message).toBe(errorMessage); } } // Wait until all browsers are closed. This will only resolve if all close, // if it does not resolve, the test will timeout and fail. await new Promise((resolve) => { const int = setInterval(() => { const stillWaiting = controllers.some((c) => c.isActive === true); if (!stillWaiting) { clearInterval(int); resolve(); } }, 10); }); expect(browserPool.activeBrowserControllers.size).toBe(0); expect.assertions(attempts + 1); }); }); describe('prePageCreateHooks', () => { test('should evaluate hook after launching browser with correct args', async () => { const myAsyncHook = async () => Promise.resolve(); browserPool.prePageCreateHooks = [myAsyncHook]; // @ts-expect-error Private function vitest.spyOn(browserPool, '_executeHooks'); const page = await browserPool.newPage(); const pageId = browserPool.getPageId(page)!; const browserController = browserPool.getBrowserControllerByPage(page)!; expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith( 3, browserPool.prePageCreateHooks, pageId, browserController, browserController.launchContext.useIncognitoPages ? {} : undefined, ); }); }); describe('postPageCreateHooks', () => { test('should evaluate hook after launching browser with correct args', async () => { const myAsyncHook = async () => Promise.resolve(); browserPool.postPageCreateHooks = [myAsyncHook]; // @ts-expect-error Private function vitest.spyOn(browserPool, '_executeHooks'); const page = await browserPool.newPage(); const browserController = browserPool.getBrowserControllerByPage(page); expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith( 4, browserPool.postPageCreateHooks, page, browserController, ); }); }); describe('prePageCloseHooks', () => { test('should evaluate hook after launching browser with correct args', async () => { const myAsyncHook = async () => Promise.resolve(); browserPool.prePageCloseHooks = [myAsyncHook]; // @ts-expect-error Private function vitest.spyOn(browserPool, '_executeHooks'); const page = await browserPool.newPage(); await page.close(); const browserController = browserPool.getBrowserControllerByPage(page); expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith( 5, browserPool.prePageCloseHooks, page, browserController, ); }); }); describe('postPageCloseHooks', () => { test('should evaluate hook after launching browser with correct args', async () => { const myAsyncHook = async () => Promise.resolve(); browserPool.postPageCloseHooks = [myAsyncHook]; // @ts-expect-error Private function vitest.spyOn(browserPool, '_executeHooks'); const page = await browserPool.newPage(); const pageId = browserPool.getPageId(page); await page.close(); const browserController = browserPool.getBrowserControllerByPage(page); expect(browserPool['_executeHooks']).toHaveBeenNthCalledWith( 6, browserPool.postPageCloseHooks, pageId, browserController, ); }); }); describe('default browser automation masking', () => { describe.each(fingerprintingMatrix)('%s', (_name, fingerprintPlugin) => { let browserPoolWithDefaults: BrowserPool; let page: any; beforeEach(async () => { browserPoolWithDefaults = new BrowserPool({ browserPlugins: [fingerprintPlugin], closeInactiveBrowserAfterSecs: 2, }); page = await browserPoolWithDefaults.newPage(); }); afterEach(async () => { if (page) await page.close(); await browserPoolWithDefaults.destroy(); }); test('should hide webdriver', async () => { await page.goto(`file://${__dirname}/test.html`); const webdriver = await page.evaluate(() => { return navigator.webdriver; }); // Can be undefined or false, depending on the chrome version. expect(webdriver).toBeFalsy(); }); }); }); describe('fingerprinting', () => { describe.each(fingerprintingMatrix)('%s', (_name, fingerprintPlugin) => { let browserPoolWithFP: BrowserPool; let page: any; beforeEach(async () => { browserPoolWithFP = new BrowserPool({ browserPlugins: [fingerprintPlugin], closeInactiveBrowserAfterSecs: 2, useFingerprints: true, }); page = await browserPoolWithFP.newPage(); }); afterEach(async () => { if (page) await page.close(); await browserPoolWithFP.destroy(); }); test('should override fingerprint', async () => { await page.goto(`file://${__dirname}/test.html`); // @ts-expect-error mistypings const browserController = browserPoolWithFP.getBrowserControllerByPage(page); const data: { hardwareConcurrency: number; userAgent: string } = await page.evaluate(() => { return { hardwareConcurrency: navigator.hardwareConcurrency, userAgent: navigator.userAgent, }; }); // @ts-expect-error mistypings const { fingerprint } = browserController!.launchContext! .fingerprint as BrowserFingerprintWithHeaders; expect(data.hardwareConcurrency).toBe(fingerprint?.navigator.hardwareConcurrency); expect(data.userAgent).toBe(fingerprint?.navigator.userAgent); }); test('should hide webdriver', async () => { await page.goto(`file://${__dirname}/test.html`); const webdriver = await page.evaluate(() => { return navigator.webdriver; }); // Can be undefined or false, depending on the chrome version. expect(webdriver).toBeFalsy(); }); }); describe('caching', () => { const commonOptions = { browserPlugins: [ new PlaywrightPlugin(playwright.chromium, { useIncognitoPages: true, }), ], }; let browserPoolCache: BrowserPool; afterEach(async () => { await browserPoolCache.destroy(); }); test('should use fingerprint cache by default', async () => { browserPoolCache = new BrowserPool({ ...commonOptions, useFingerprints: true, }); expect(browserPoolCache.fingerprintCache).toBeDefined(); }); test('should turn off cache', async () => { browserPoolCache = new BrowserPool({ ...commonOptions, useFingerprints: true, fingerprintOptions: { useFingerprintCache: false, }, }); expect(browserPoolCache.fingerprintCache).toBeUndefined(); }); test('should limit cache size', async () => { browserPoolCache = new BrowserPool({ ...commonOptions, useFingerprints: true, fingerprintOptions: { fingerprintCacheSize: 1, }, }); // cast to any type in order to access the maxSize property for testing purposes. const cache: any = browserPoolCache!.fingerprintCache!; expect(cache.maxSize).toBe(1); }); test('should cache fingerprints', async () => { browserPoolCache = new BrowserPool({ ...commonOptions, useFingerprints: true, preLaunchHooks: [ (_pageId, launchContext) => { // @ts-expect-error issue caused by generics launchContext.extend({ session: { id: '123' } }); }, ], }); const mock = vitest.fn(); browserPoolCache.fingerprintInjector!.attachFingerprintToPlaywright = mock; const page: Page = await browserPoolCache.newPageInNewBrowser(); expect(mock.mock.calls[0][1]).toBeDefined(); const page2: Page = await browserPoolCache.newPageInNewBrowser(); await page.close(); await page2.close(); // expect fingerprint parameter of the first call to equal fingerprint parameter of the second call expect(mock.mock.calls[0][1]).toBe(mock.mock.calls[1][1]); }); }); }); describe('generator configuration', () => { const commonOptions = { browserPlugins: [ new PlaywrightPlugin(playwright.firefox, { useIncognitoPages: true, }), ], }; let browserPoolConfig: BrowserPool; afterEach(async () => { await browserPoolConfig.destroy(); }); test('should use native os and browser', async () => { browserPoolConfig = new BrowserPool({ ...commonOptions, useFingerprints: true, }); const oldGet = browserPoolConfig.fingerprintGenerator!.getFingerprint; const mock = vitest.fn((options) => { return oldGet.bind(browserPoolConfig.fingerprintGenerator)(options); }); browserPoolConfig.fingerprintGenerator!.getFingerprint = mock; const page: Page = await browserPoolConfig.newPage(); await page.close(); const defaultOptions = mock.mock.calls[0][0]; expect(defaultOptions.browsers.includes('firefox')).toBe(true); let os: string; switch (process.platform) { case 'darwin': os = 'macos'; break; case 'win32': os = 'windows'; break; default: os = 'linux'; } expect(defaultOptions.operatingSystems.includes(os)).toBe(true); }); test('should allow changing options', async () => { browserPoolConfig = new BrowserPool({ ...commonOptions, useFingerprints: true, fingerprintOptions: { fingerprintGeneratorOptions: { operatingSystems: [OperatingSystemsName.windows], browsers: [BrowserName.chrome], }, }, }); const oldGet = browserPoolConfig.fingerprintGenerator!.getFingerprint; const mock = vitest.fn((options) => { return oldGet.bind(browserPoolConfig.fingerprintGenerator)(options); }); browserPoolConfig.fingerprintGenerator!.getFingerprint = mock; const page: Page = await browserPoolConfig.newPageInNewBrowser(); await page.close(); const [options] = mock.mock.calls[0]; expect(options.operatingSystems.includes('windows')).toBe(true); expect(options.browsers.includes('chrome')).toBe(true); }); }); }); describe('events', () => { test(`should emit ${BROWSER_POOL_EVENTS.BROWSER_LAUNCHED} event`, async () => { browserPool.maxOpenPagesPerBrowser = 1; let calls = 0; let argument; browserPool.on(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, (arg) => { argument = arg; calls++; }); await browserPool.newPage(); const page = await browserPool.newPage(); expect(calls).toEqual(2); expect(argument).toEqual(browserPool.getBrowserControllerByPage(page)); }); test(`should emit ${BROWSER_POOL_EVENTS.BROWSER_RETIRED} event`, async () => { browserPool.retireBrowserAfterPageCount = 1; let calls = 0; let argument; browserPool.on(BROWSER_POOL_EVENTS.BROWSER_RETIRED, (arg) => { argument = arg; calls++; }); await browserPool.newPage(); const page = await browserPool.newPage(); expect(calls).toEqual(2); expect(argument).toEqual(browserPool.getBrowserControllerByPage(page)); }); test(`should emit ${BROWSER_POOL_EVENTS.PAGE_CREATED} event`, async () => { let calls = 0; let argument; browserPool.on(BROWSER_POOL_EVENTS.PAGE_CREATED, (arg) => { argument = arg; calls++; }); const page = await browserPool.newPage(); expect(argument).toEqual(page); const page2 = await browserPool.newPage(); expect(calls).toEqual(2); expect(argument).toEqual(page2); }); test(`should emit ${BROWSER_POOL_EVENTS.PAGE_CLOSED} event`, async () => { let calls = 0; let argument; browserPool.on(BROWSER_POOL_EVENTS.PAGE_CLOSED, (arg) => { argument = arg; calls++; }); const page = await browserPool.newPage(); await page.close(); expect(argument).toEqual(page); const page2 = await browserPool.newPage(); await page2.close(); expect(calls).toEqual(2); expect(argument).toEqual(page2); }); }); }); });