import type { Server } from 'node:http'; import path from 'node:path'; import { KeyValueStore, launchPlaywright, playwrightUtils, Request } from '@crawlee/playwright'; import type { Browser, Page } from 'playwright'; import { chromium } from 'playwright'; import log from '@apify/log'; import { runExampleComServer } from '../shared/_helper'; import { MemoryStorageEmulator } from '../shared/MemoryStorageEmulator'; let serverAddress = 'http://localhost:'; let port: number; let server: Server; const launchContext = { launchOptions: { headless: true } }; beforeAll(async () => { [server, port] = await runExampleComServer(); serverAddress += port; }); afterAll(() => { server.close(); }); describe('playwrightUtils', () => { let ll: number; const localStorageEmulator = new MemoryStorageEmulator(); beforeAll(async () => { ll = log.getLevel(); log.setLevel(log.LEVELS.ERROR); }); beforeEach(async () => { await localStorageEmulator.init(); }); afterAll(async () => { log.setLevel(ll); await localStorageEmulator.destroy(); }); test('injectFile()', async () => { const browser2 = await launchPlaywright(launchContext); const survive = async (browser: Browser) => { // Survive navigations const page = await browser.newPage(); // @ts-expect-error let result = await page.evaluate(() => window.injectedVariable === 42); expect(result).toBe(false); await playwrightUtils.injectFile(page, path.join(__dirname, '..', 'shared', 'data', 'inject_file.txt'), { surviveNavigations: true, }); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable); expect(result).toBe(42); await page.goto('about:chrome'); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable); expect(result).toBe(42); await page.goto(serverAddress); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable); expect(result).toBe(42); }; const remove = async (browser: Browser) => { // Remove with navigations const page = await browser.newPage(); // @ts-expect-error let result = await page.evaluate(() => window.injectedVariable === 42); expect(result).toBe(false); await page.goto('about:chrome'); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable === 42); expect(result).toBe(false); await playwrightUtils.injectFile(page, path.join(__dirname, '..', 'shared', 'data', 'inject_file.txt')); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable); expect(result).toBe(42); await page.goto(serverAddress); // @ts-expect-error result = await page.evaluate(() => window.injectedVariable === 42); expect(result).toBe(false); }; try { await Promise.all([survive(browser2), remove(browser2)]); } finally { await browser2.close(); } }); test('injectJQuery()', async () => { const browser = await launchPlaywright(launchContext); try { const page = await browser.newPage(); await page.goto('about:blank'); // NOTE: Chrome already defines window.$ as alias to document.querySelector(), // (https://developers.google.com/web/tools/chrome-devtools/console/command-line-reference#queryselector) const result1 = await page.evaluate(() => { return { // @ts-expect-error isDefined: window.jQuery !== undefined, }; }); expect(result1).toEqual({ isDefined: false, }); await playwrightUtils.injectJQuery(page); const result2 = await page.evaluate(() => { /* global $ */ return { // @ts-expect-error isDefined: window.jQuery === window.$, // @ts-expect-error text: $('h1').text(), }; }); expect(result2).toEqual({ isDefined: true, text: '', }); await page.reload(); const result3 = await page.evaluate(() => { return { // @ts-expect-error isDefined: window.jQuery === window.$, // @ts-expect-error text: $('h1').text(), }; }); expect(result3).toEqual({ isDefined: true, text: '', }); } finally { await browser.close(); } }); test('parseWithCheerio() works', async () => { const browser = await launchPlaywright(launchContext); try { const page = await browser.newPage(); await page.goto(serverAddress); const $ = await playwrightUtils.parseWithCheerio(page); const title = $('h1').text().trim(); expect(title).toBe('Example Domain'); } finally { await browser.close(); } }); test('parseWithCheerio() iframe expansion works', async () => { const browser = await launchPlaywright(launchContext); try { const page = await browser.newPage(); await page.goto(new URL('/special/outside-iframe', serverAddress).toString()); const $ = await playwrightUtils.parseWithCheerio(page); const headings = $('h1') .map((_, el) => $(el).text()) .get(); const titles = $('title') .map((_, el) => $(el).text()) .get(); expect(titles).toEqual(['Outside iframe title']); expect(headings).toEqual(['Outside iframe', 'In iframe']); } finally { await browser.close(); } }); test('parseWithCheerio() iframe expansion works with Trusted Types CSP', async () => { const browser = await launchPlaywright(launchContext); try { const page = await browser.newPage(); await page.goto(new URL('/special/outside-iframe-csp', serverAddress).toString()); const $ = await playwrightUtils.parseWithCheerio(page); const headings = $('h1') .map((_, el) => $(el).text()) .get(); expect(headings).toEqual(['Outside iframe', 'In iframe']); } finally { await browser.close(); } }); describe('blockRequests()', () => { let browser: Browser = null as any; beforeAll(async () => { browser = await launchPlaywright(launchContext); }); afterAll(async () => { await browser.close(); }); test('works with default values', async () => { const loadedUrls: string[] = []; const page = await browser.newPage(); await playwrightUtils.blockRequests(page); page.on('response', (response) => { if (response.url() !== `${serverAddress}/special/resources`) { loadedUrls.push(response.url()); } }); await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle' }); expect(loadedUrls).toEqual([`${serverAddress}/script.js`]); }); test('works with overridden values', async () => { const loadedUrls: string[] = []; const page = await browser.newPage(); await playwrightUtils.blockRequests(page, { urlPatterns: ['.css'], }); page.on('response', (response) => loadedUrls.push(response.url())); await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle' }); expect(loadedUrls).toEqual( expect.arrayContaining([ `${serverAddress}/image.png`, `${serverAddress}/script.js`, `${serverAddress}/image.gif`, ]), ); }); }); test('gotoExtended() works', async () => { const browser = await chromium.launch({ headless: true }); try { const page = await browser.newPage(); const request = new Request({ url: `${serverAddress}/special/getDebug`, method: 'POST', headers: { 'Content-Type': 'application/json; charset=utf-8', }, payload: '{ "foo": "bar" }', }); const response = await playwrightUtils.gotoExtended(page, request); const { method, headers, bodyLength } = JSON.parse(await response!.text()); expect(method).toBe('POST'); expect(bodyLength).toBe(16); expect(headers['content-type']).toBe('application/json; charset=utf-8'); } finally { await browser.close(); } }, 60_000); describe('shadow root expansion', () => { let browser: Browser; beforeAll(async () => { browser = await launchPlaywright(launchContext); }); afterAll(async () => { await browser.close(); }); test('no expansion with ignoreShadowRoots: true', async () => { const page = await browser.newPage(); await page.goto(`${serverAddress}/special/shadow-root`); const result = await playwrightUtils.parseWithCheerio(page, true); const text = result('body').text().trim(); expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(0); expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0); }); test('expansion works', async () => { const page = await browser.newPage(); await page.goto(`${serverAddress}/special/shadow-root`); const result = await playwrightUtils.parseWithCheerio(page); const text = result('body').text().trim(); expect([...text.matchAll(/\[GOOD\]/g)]).toHaveLength(2); expect([...text.matchAll(/\[BAD\]/g)]).toHaveLength(0); }); }); describe('infiniteScroll()', () => { function isAtBottom() { return window.innerHeight + window.pageYOffset >= document.body.offsetHeight; } let browser: Browser; beforeAll(async () => { browser = await chromium.launch({ headless: true }); }); afterAll(async () => { await browser.close(); }); let page: Page; beforeEach(async () => { page = await browser.newPage(); let count = 0; const content = Array(1000) .fill(null) .map(() => { return `