import type { Server } from 'node:http'; import http from 'node:http'; import type { AddressInfo } from 'node:net'; import { promisify } from 'node:util'; import type { CommonLibrary, UnwrapPromise } from '@crawlee/browser-pool'; import { LaunchContext, PlaywrightBrowser, PlaywrightController, PlaywrightPlugin, PuppeteerController, PuppeteerPlugin, } from '@crawlee/browser-pool'; import playwright from 'playwright'; import type { Server as ProxyChainServer } from 'proxy-chain'; import type { Browser } from 'puppeteer'; import puppeteer from 'puppeteer'; import { runExampleComServer } from '../../shared/_helper'; import { createProxyServer } from './create-proxy-server'; // Firefox browser launch is significantly slower than Chromium/WebKit (~12s vs <1s). // Under CPU load from parallel tests, it can exceed 2 minutes. Use 5 minute timeout. vitest.setConfig({ testTimeout: 300_000 }); let port: number; let server: Server; let serverAddress = 'http://localhost:'; beforeAll(async () => { [server, port] = await runExampleComServer(); serverAddress += port; }); afterAll(() => { server.close(); }); const runPluginTest = < P extends typeof PlaywrightPlugin | typeof PuppeteerPlugin, C extends typeof PuppeteerController | typeof PlaywrightController, L extends CommonLibrary, >( Plugin: P, Controller: C, library: L, ) => { describe(`${library.constructor.name} - ${'name' in library ? library.name!() : ''} general`, () => { test.concurrent('should launch browser', async () => { const plugin = new Plugin(library as never); const browser = await plugin.launch(); try { expect(typeof browser.newPage).toBe('function'); expect(typeof browser.close).toBe('function'); } finally { await browser.close(); } }); test.concurrent('should create launch context', () => { const plugin = new Plugin(library as never); const id = 'abc'; const launchOptions = { foo: 'bar' }; const proxyUrl = 'http://proxy.com/'; const context = plugin.createLaunchContext({ id, // @ts-expect-error Testing options launchOptions, }); expect(context).toBeInstanceOf(LaunchContext); context.proxyUrl = proxyUrl; context.extend({ one: 1, }); const desiredObject = { id, launchOptions, browserPlugin: plugin, _proxyUrl: proxyUrl.slice(0, -1), one: 1, useIncognitoPages: false, }; // expect(context).toMatchObject(desiredObject) // Switch to this after the issue with `TypeError: prop.startsWith is not a function` is solved. expect(context.id).toEqual(desiredObject.id); expect(context.launchOptions).toEqual(desiredObject.launchOptions); expect(context.browserPlugin).toEqual(desiredObject.browserPlugin); expect(context['_proxyUrl']).toEqual(desiredObject._proxyUrl); // eslint-disable-line expect(context.one).toEqual(desiredObject.one); expect(context.useIncognitoPages).toEqual(desiredObject.useIncognitoPages); }); test.concurrent('should get default launchContext values from plugin options', async () => { const proxyUrl = 'http://apify1234@10.10.10.0:8080/'; const plugin = new Plugin(library as never, { proxyUrl, userDataDir: 'test', useIncognitoPages: true, }); const context = plugin.createLaunchContext(); expect(context.proxyUrl).toEqual(proxyUrl.slice(0, -1)); expect(context.useIncognitoPages).toBeTruthy(); expect(context.userDataDir).toEqual('test'); }); test.concurrent('should allow unsetting launchContext proxyUrl', () => { const plugin = new Plugin(library as never); const context = plugin.createLaunchContext(); context.proxyUrl = 'http://proxy.com/'; expect(context.proxyUrl).toEqual('http://proxy.com'); context.proxyUrl = undefined; expect(context.proxyUrl).toBeUndefined(); }); test.concurrent('should create browser controller', () => { const plugin = new Plugin(library as never); const browserController = plugin.createController(); expect(browserController).toBeInstanceOf(Controller); }); test.concurrent('should work with cookies', async () => { const plugin = new Plugin(library as never); const browserController = plugin.createController(); const context = plugin.createLaunchContext(); const browser = await plugin.launch(context as never); try { browserController.assignBrowser(browser as never, context as never); browserController.activate(); const page = await browserController.newPage(); await browserController.setCookies(page as never, [ { name: 'TEST', value: 'TESTER-COOKIE', url: serverAddress }, ]); await page.goto(serverAddress, { waitUntil: 'domcontentloaded' }); const cookies = await browserController.getCookies(page as never); expect(cookies[0].name).toBe('TEST'); expect(cookies[0].value).toBe('TESTER-COOKIE'); } finally { await browser.close(); } }); test.concurrent('newPage options cannot be used with persistent context', async () => { const plugin = new Plugin(library as never); const browserController = plugin.createController(); const context = plugin.createLaunchContext({ useIncognitoPages: false, }); const browser = await plugin.launch(context as never); try { browserController.assignBrowser(browser as never, context as never); browserController.activate(); try { const page = await browserController.newPage({}); await page.close(); expect(false).toBe(true); } catch (error: any) { expect(error.message).toBe( 'A new page can be created with provided context only when using incognito pages or experimental containers.', ); } } finally { await browser.close(); } }); }); }; describe('Plugins', () => { let target: http.Server; let unprotectedProxy: ProxyChainServer; let protectedProxy: ProxyChainServer; beforeAll(async () => { target = http.createServer((request, response) => { response.end(request.socket.remoteAddress); }); await promisify(target.listen.bind(target) as any)(0, '127.0.0.1'); unprotectedProxy = createProxyServer('127.0.0.2', '', ''); await unprotectedProxy.listen(); protectedProxy = createProxyServer('127.0.0.3', 'foo', 'bar'); await protectedProxy.listen(); }); afterAll(async () => { await promisify(target.close.bind(target))(); await unprotectedProxy.close(false); await protectedProxy.close(false); }); describe('Puppeteer specifics', () => { test.concurrent('should work with non authenticated proxyUrl', async () => { const proxyUrl = `http://127.0.0.2:${unprotectedProxy.port}`; const plugin = new PuppeteerPlugin(puppeteer); const context = plugin.createLaunchContext({ proxyUrl, launchOptions: { args: [ // Exclude loopback interface from proxy bypass list, // so the request to localhost goes through proxy. // This way there's no need for a 3rd party server. '--proxy-bypass-list=<-loopback>', ], }, }); const browser = await plugin.launch(context); try { const page = await browser.newPage(); const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`); const text = await response!.text(); expect(text).toBe('127.0.0.2'); await page.close(); } finally { await browser.close(); } }); test.concurrent('should work with authenticated proxyUrl', async () => { const proxyUrl = `http://foo:bar@127.0.0.3:${protectedProxy.port}`; const plugin = new PuppeteerPlugin(puppeteer); const context = plugin.createLaunchContext({ proxyUrl, launchOptions: { args: [ // Exclude loopback interface from proxy bypass list, // so the request to localhost goes through proxy. // This way there's no need for a 3rd party server. '--proxy-bypass-list=<-loopback>', ], }, }); const browser = await plugin.launch(context); try { const page = await browser.newPage(); const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`); const text = await response!.text(); expect(text).toBe('127.0.0.3'); await page.close(); } finally { await browser.close(); } }); test.concurrent('should use persistent context by default', async () => { const plugin = new PuppeteerPlugin(puppeteer); const browserController = plugin.createController(); const launchContext = plugin.createLaunchContext(); const browser = await plugin.launch(launchContext); try { browserController.assignBrowser(browser, launchContext); browserController.activate(); const page = await browserController.newPage(); const browserContext = page.browserContext(); expect(browserContext.id).toBeFalsy(); } finally { await browser.close(); } }); test.concurrent('should use incognito pages by option', async () => { const plugin = new PuppeteerPlugin(puppeteer); const browserController = plugin.createController(); const launchContext = plugin.createLaunchContext({ useIncognitoPages: true }); const browser = await plugin.launch(launchContext); try { browserController.assignBrowser(browser, launchContext); browserController.activate(); const page = await browserController.newPage(); const browserContext = page.browserContext(); expect(browserContext.id).toBeTruthy(); } finally { await browser.close(); } }); test.concurrent('should allow calling functions on the proxied browser', async () => { const plugin = new PuppeteerPlugin(puppeteer); const launchContext = plugin.createLaunchContext(); const browser = await plugin.launch(launchContext); try { await expect(browser.pages()).resolves.toBeDefined(); } finally { await browser.close(); } }); test.concurrent('should pass launch options to browser', async () => { const plugin = new PuppeteerPlugin(puppeteer); const userAgent = 'HelloWorld'; const launchOptions = { args: [`--user-agent=${userAgent}`], }; const launchContext = plugin.createLaunchContext({ launchOptions }); const browser = await plugin.launch(launchContext); try { expect(await browser.userAgent()).toBe(userAgent); } finally { await browser.close(); } }); test.concurrent('proxyUsername and proxyPassword as newPage options', async () => { const plugin = new PuppeteerPlugin(puppeteer); const browserController = new PuppeteerController(plugin); const launchContext = plugin.createLaunchContext({ useIncognitoPages: true, }); const browser = await plugin.launch(launchContext); try { browserController.assignBrowser(browser, launchContext); browserController.activate(); const page = await browserController.newPage({ proxyServer: `http://127.0.0.3:${protectedProxy.port}`, proxyUsername: 'foo', proxyPassword: 'bar', proxyBypassList: ['<-loopback>'], }); const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`); const text = await response!.text(); expect(text).toBe('127.0.0.3'); await page.close(); } finally { await browser.close(); } }); }); runPluginTest(PuppeteerPlugin, PuppeteerController, puppeteer); describe('normalizeProxyOptions', () => { test.each([ ['socks5://user:pass@proxy.example.com:1080', 'socks5://proxy.example.com:1080'], ['http://user:pass@proxy.example.com:8080', 'http://proxy.example.com:8080'], ])('PlaywrightController should handle %s', (proxyUrl, expectedServer) => { const plugin = new PlaywrightPlugin(playwright.chromium); const browserController = new PlaywrightController(plugin); const result = browserController.normalizeProxyOptions(proxyUrl, {}); expect(result).toMatchObject({ proxy: { server: expectedServer, username: 'user', password: 'pass' }, }); }); test.each([ ['socks5://user:pass@proxy.example.com:1080', 'socks5://proxy.example.com:1080'], ['http://user:pass@proxy.example.com:8080', 'http://proxy.example.com:8080'], ])('PuppeteerController should handle %s', (proxyUrl, expectedServer) => { const plugin = new PuppeteerPlugin(puppeteer); const browserController = new PuppeteerController(plugin); const result = browserController.normalizeProxyOptions(proxyUrl, {}); expect(result).toMatchObject({ proxyServer: expectedServer, proxyUsername: 'user', proxyPassword: 'pass', }); }); }); describe('Playwright specifics', () => { let browser: playwright.Browser; afterEach(async () => { await browser.close(); }); describe.each(['chromium', 'firefox', 'webkit'] as const)('with %s', (browserName) => { test('should work with non authenticated proxyUrl', async () => { const proxyUrl = `http://127.0.0.2:${unprotectedProxy.port}`; const plugin = new PlaywrightPlugin(playwright[browserName]); const launchOptions = browserName === 'chromium' ? { args: [ // Exclude loopback interface from proxy bypass list, // so the request to localhost goes through proxy. // This way there's no need for a 3rd party server. '--proxy-bypass-list=<-loopback>', ], } : undefined; const context = plugin.createLaunchContext({ proxyUrl, launchOptions, }); browser = await plugin.launch(context); expect(context.launchOptions!.proxy!.server).toEqual(proxyUrl); const page = await browser.newPage(); const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`); const text = await response!.text(); expect(text).toBe('127.0.0.2'); await page.close(); }); test('should work with authenticated proxyUrl', async () => { const proxyUrl = `http://foo:bar@127.0.0.3:${protectedProxy.port}`; const plugin = new PlaywrightPlugin(playwright[browserName]); const launchOptions = browserName === 'chromium' ? { args: [ // Exclude loopback interface from proxy bypass list, // so the request to localhost goes through proxy. // This way there's no need for a 3rd party server. '--proxy-bypass-list=<-loopback>', ], } : undefined; const context = plugin.createLaunchContext({ proxyUrl, launchOptions, }); browser = await plugin.launch(context); const page = await browser.newPage(); const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`); const text = await response!.text(); expect(text).toBe('127.0.0.3'); await page.close(); }); test('proxy as newPage option', async () => { const plugin = new PlaywrightPlugin(playwright.chromium); const browserController = new PlaywrightController(plugin); const launchContext = plugin.createLaunchContext({ useIncognitoPages: true, }); browser = await plugin.launch(launchContext); browserController.assignBrowser(browser, launchContext); browserController.activate(); const page = await browserController.newPage({ proxy: { server: `http://127.0.0.3:${protectedProxy.port}`, username: 'foo', password: 'bar', bypass: '<-loopback>', }, }); const response = await page.goto(`http://127.0.0.1:${(target.address() as AddressInfo).port}`); const text = await response!.text(); expect(text).toBe('127.0.0.3'); await page.close(); }); test('should use incognito context by option', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const browserController = plugin.createController(); const launchContext = plugin.createLaunchContext({ useIncognitoPages: true }); browser = await plugin.launch(launchContext); browserController.assignBrowser(browser, launchContext); browserController.activate(); const page = await browserController.newPage(); const browserContext = page.context(); await browserController.newPage(); expect(browserContext.pages()).toHaveLength(1); }); test('should use persistent context by default', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const browserController = plugin.createController(); const launchContext = plugin.createLaunchContext(); browser = await plugin.launch(launchContext); browserController.assignBrowser(browser, launchContext); browserController.activate(); const page = await browserController.newPage(); const context = page.context(); await browserController.newPage(); expect(context.pages()).toHaveLength(3); // 3 pages because of the about:blank. }); test('should pass launch options to browser', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); let ran = false; const launchOptions = { logger: { isEnabled: () => { ran = true; return false; }, log: () => {}, }, }; const launchContext = plugin.createLaunchContext({ launchOptions }); browser = await plugin.launch(launchContext); expect(ran).toBe(true); }); describe('PlaywrightBrowser', () => { test('should create new page', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const launchContext = plugin.createLaunchContext(); browser = await plugin.launch(launchContext); const page = await browser.newPage(); expect(typeof page.close).toBe('function'); expect(typeof page.evaluate).toBe('function'); }); test('should emit disconnected event on close', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const launchContext = plugin.createLaunchContext(); browser = await plugin.launch(launchContext); let called = false; browser.on('disconnected', () => { called = true; }); await browser.close(); expect(called).toBe(true); }); test('should be used only with incognito pages context', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const launchContext = plugin.createLaunchContext({ useIncognitoPages: false }); browser = await plugin.launch(launchContext); expect(browser).toBeInstanceOf(PlaywrightBrowser); await browser.close(); const launchContext2 = plugin.createLaunchContext({ useIncognitoPages: true }); browser = await plugin.launch(launchContext2); expect(browser).not.toBeInstanceOf(PlaywrightBrowser); }); test('should return correct version', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const launchContext = plugin.createLaunchContext({ useIncognitoPages: false }); browser = await plugin.launch(launchContext); const version1 = browser.version(); await browser.close(); const launchContext2 = plugin.createLaunchContext({ useIncognitoPages: true }); browser = await plugin.launch(launchContext2); expect(version1).toEqual(browser.version()); }); test('should return all contexts', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const launchContext = plugin.createLaunchContext(); browser = await plugin.launch(launchContext); const contexts = browser.contexts(); expect(contexts).toHaveLength(1); // Cast to any to access private property expect(contexts[0]).toEqual((browser as any)._browserContext); }); test('should return correct connected status', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const launchContext = plugin.createLaunchContext(); browser = await plugin.launch(launchContext); expect(browser.isConnected()).toBe(true); await browser.close(); expect(browser.isConnected()).toBe(false); }); test('should throw on newContext call', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const launchContext = plugin.createLaunchContext(); browser = await plugin.launch(launchContext); await expect(browser.newContext()).rejects.toThrow( 'Function `newContext()` is not available in incognito mode', ); }); test('should have same public interface as playwright browserType', async () => { const plugin = new PlaywrightPlugin(playwright[browserName]); const originalFunctionNames = [ 'close', 'contexts', 'isConnected', 'newContext', 'newPage', 'version', ] as const; const launchContext = plugin.createLaunchContext({ useIncognitoPages: true }); browser = await plugin.launch(launchContext); for (const originalFunctionName of originalFunctionNames) { expect(typeof browser[originalFunctionName]).toBe('function'); } expect.hasAssertions(); }); }); }); }); runPluginTest(PlaywrightPlugin, PlaywrightController, playwright.chromium); runPluginTest(PlaywrightPlugin, PlaywrightController, playwright.firefox); runPluginTest(PlaywrightPlugin, PlaywrightController, playwright.webkit); });