import type { Server } from 'node:http';
import type { RequestQueueOperationOptions, Source } from 'crawlee';
import {
Configuration,
launchPlaywright,
launchPuppeteer,
playwrightClickElements,
playwrightUtils,
puppeteerClickElements,
puppeteerUtils,
RequestQueue,
} from 'crawlee';
import type { Browser as PWBrowser, Page as PWPage } from 'playwright';
import type { Browser as PPBrowser, Target } from 'puppeteer';
import { runExampleComServer } from '../../shared/_helper';
function isPuppeteerBrowser(browser: PPBrowser | PWBrowser): browser is PPBrowser {
return (browser as PPBrowser).targets !== undefined;
}
function isPlaywrightBrowser(browser: PPBrowser | PWBrowser): browser is PWBrowser {
return (browser as PWBrowser).browserType !== undefined;
}
const apifyClient = Configuration.getStorageClient();
function createRequestQueueMock() {
const enqueued: Source[] = [];
const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
// @ts-expect-error Override method for testing
requestQueue.addRequests = async function (requests) {
const processedRequests: Source[] = [];
for await (const request of requests) {
processedRequests.push(typeof request === 'string' ? { url: request } : request);
}
enqueued.push(...processedRequests);
return { processedRequests, unprocessedRequests: [] as never[] };
};
return { enqueued, requestQueue };
}
const testCases = [
{
caseName: 'Puppeteer',
launchBrowser: launchPuppeteer,
clickElements: puppeteerClickElements,
utils: puppeteerUtils,
},
{
caseName: 'Playwright',
launchBrowser: launchPlaywright,
clickElements: playwrightClickElements,
utils: playwrightUtils,
},
];
testCases.forEach(({ caseName, launchBrowser, clickElements, utils }) => {
describe(`${caseName}: enqueueLinksByClickingElements()`, () => {
let browser: PPBrowser | PWBrowser;
let server: Server;
let serverAddress = 'http://localhost:';
let serverPort: number;
let page: any;
beforeAll(async () => {
[server, serverPort] = await runExampleComServer();
serverAddress += serverPort;
browser = await launchBrowser({ launchOptions: { headless: true } });
});
afterAll(async () => {
await browser.close();
server.close();
});
beforeEach(async () => {
page = await browser.newPage();
});
afterEach(async () => {
await page.close();
});
test('should work', async () => {
const { enqueued, requestQueue } = createRequestQueueMock();
const html = `
link
`;
await page.setContent(html);
await utils.enqueueLinksByClickingElements({
page,
requestQueue,
selector: 'a',
transformRequestFunction: (request) => {
request.uniqueKey = 'key';
return request;
},
waitForPageIdleSecs: 0.025,
maxWaitForPageIdleSecs: 0.25,
});
expect(enqueued).toHaveLength(1);
expect(enqueued[0].url).toMatch(`${serverAddress}/`);
expect(enqueued[0].uniqueKey).toBe('key');
expect(page.url()).toBe('about:blank');
});
test('accepts forefront option', async () => {
const addedRequests: { request: Source; options?: RequestQueueOperationOptions }[] = [];
const requestQueue = new RequestQueue({ id: 'xxx', client: Configuration.getStorageClient() });
requestQueue.addRequests = async (requests, options) => {
for await (const request of requests) {
addedRequests.push({ request: typeof request === 'string' ? { url: request } : request, options });
}
return { processedRequests: [], unprocessedRequests: [] };
};
const html = `
link1
link2
`;
await page.setContent(html);
await utils.enqueueLinksByClickingElements({
page,
requestQueue,
selector: 'a',
waitForPageIdleSecs: 0.025,
maxWaitForPageIdleSecs: 0.25,
forefront: true,
});
expect(addedRequests).toHaveLength(2);
expect(addedRequests[0].options!.forefront).toBe(true);
expect(addedRequests[1].options!.forefront).toBe(true);
});
describe('clickElements()', () => {
test('should click an element', async () => {
const html = `
div
`;
await page.setContent(html);
await clickElements.clickElements(page, 'div');
// @ts-expect-error Custom property
const wasClicked = await page.evaluate(() => window.clicked);
expect(wasClicked).toBe(true);
});
test('should click an empty element', async () => {
const html = `
`;
await page.setContent(html);
await clickElements.clickElements(page, 'div');
// @ts-expect-error Custom property
const wasClicked = await page.evaluate(() => window.clicked);
expect(wasClicked).toBe(true);
});
test('should click multiple elements', async () => {
const html = `
`;
await page.setContent(html);
await clickElements.clickElements(page, 'header, div, p, footer');
// @ts-expect-error Custom property
const clickedElements = await page.evaluate(() => window.clickedElements);
expect(clickedElements).toEqual(['HEADER', 'DIV', 'P', 'FOOTER']);
});
test('should click hidden elements', async () => {
const html = `
`;
await page.setContent(html);
await clickElements.clickElements(page, 'div');
// @ts-expect-error Custom property
const wasClicked = await page.evaluate(() => window.clicked);
expect(wasClicked).toBe(true);
});
test('should click elements stacked on top of each other', async () => {
const html = `
div
main
`;
await page.setContent(html);
await clickElements.clickElements(page, 'header, div, main, footer');
// @ts-expect-error Custom property
const clickedElements = await page.evaluate(() => window.clickedElements);
expect(clickedElements).toEqual(['HEADER', 'DIV', 'MAIN', 'FOOTER']);
});
});
describe('double clickElements()', () => {
test('should double click an element', async () => {
const html = `
div
`;
await page.setContent(html);
await clickElements.clickElements(page, 'div', { clickCount: 2, delay: 100 });
const wasClicked = await page.evaluate(() => (window as any).clicked);
expect(wasClicked).toBe(true);
});
test('should double click an empty element', async () => {
const html = `
`;
await page.setContent(html);
await clickElements.clickElements(page, 'div', { clickCount: 2, delay: 100 });
const wasClicked = await page.evaluate(() => (window as any).clicked);
expect(wasClicked).toBe(true);
});
test('should double click multiple elements', async () => {
const html = `
`;
await page.setContent(html);
await clickElements.clickElements(page, 'header, div, p, footer', { clickCount: 2, delay: 100 });
const clickedElements = await page.evaluate(() => (window as any).clickedElements);
expect(clickedElements).toEqual(['HEADER', 'DIV', 'P', 'FOOTER']);
});
test('should double click hidden elements', async () => {
const html = `
`;
await page.setContent(html);
await clickElements.clickElements(page, 'div', { clickCount: 2, delay: 100 });
const wasClicked = await page.evaluate(() => (window as any).clicked);
expect(wasClicked).toBe(true);
});
test('should click elements stacked on top of each other', async () => {
const html = `
div
main
`;
await page.setContent(html);
await clickElements.clickElements(page, 'header, div, main, footer', { clickCount: 2, delay: 100 });
const clickedElements = await page.evaluate(() => (window as any).clickedElements);
expect(clickedElements).toEqual(['HEADER', 'DIV', 'MAIN', 'FOOTER']);
});
});
describe('select line with triple clickElements()', () => {
test('should select the text by triple clicking', async () => {
const html = `
`;
await page.setContent(html);
await page.focus('textarea');
const text = "This is the text that we are going to try to select. Let's see how it goes.";
await page.keyboard.type(text);
await clickElements.clickElements(page, 'textarea', { clickCount: 3, delay: 100 });
expect(
await page.evaluate(() => {
const textarea = document.querySelector('textarea')!;
return textarea.value.substring(textarea.selectionStart, textarea.selectionEnd);
}),
).toBe(text);
});
});
describe('clickElementsAndInterceptNavigationRequests()', () => {
function getOpts(overrides = {}) {
return {
page,
selector: 'div',
waitForPageIdleMillis: 25,
maxWaitForPageIdleMillis: 250,
...overrides,
};
}
test('should intercept navigation by clicking a link', async () => {
const html = `
link
`;
await page.setContent(html);
const interceptedRequests = await clickElements.clickElementsAndInterceptNavigationRequests(
getOpts({
selector: 'a',
}),
);
expect(interceptedRequests).toHaveLength(1);
expect(interceptedRequests[0].url).toMatch(`${serverAddress}/`);
expect(page.url()).toBe('about:blank');
});
test('should intercept navigation with window.location', async () => {
const html = `
div
`;
await page.goto(serverAddress);
await page.setContent(html);
const interceptedRequests = await clickElements.clickElementsAndInterceptNavigationRequests(getOpts());
expect(interceptedRequests).toHaveLength(1);
expect(interceptedRequests[0].url).toMatch(`${serverAddress}/`);
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return window.location = ');
});
test('should save the hash when changing it with window.location', async () => {
const html = `
div
`;
await page.goto(serverAddress);
await page.setContent(html);
const interceptedRequests = await clickElements.clickElementsAndInterceptNavigationRequests(getOpts());
expect(interceptedRequests).toHaveLength(1);
expect(interceptedRequests[0].url).toBe(`${serverAddress}/#foo`);
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return window.location = ');
});
test('should prevent reload from cache with window.reload()', async () => {
const html = `
div
`;
await page.goto(serverAddress);
await page.setContent(html);
const interceptedRequests = await clickElements.clickElementsAndInterceptNavigationRequests(getOpts());
expect(interceptedRequests).toHaveLength(1);
expect(interceptedRequests[0].url).toMatch(`${serverAddress}/`);
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return window.location.reload()');
});
test('should prevent forced reload with window.reload(true)', async () => {
const html = `
div
`;
await page.goto(serverAddress);
await page.setContent(html);
const interceptedRequests = await clickElements.clickElementsAndInterceptNavigationRequests(getOpts());
expect(interceptedRequests).toHaveLength(1);
expect(interceptedRequests[0].url).toMatch(`${serverAddress}/`);
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return window.location.reload(true)');
});
test('should prevent manipulation with window.history', async () => {
const html = `
div
`;
await page.goto(serverAddress);
await page.setContent(html);
const interceptedRequests = await clickElements.clickElementsAndInterceptNavigationRequests(getOpts());
expect(interceptedRequests).toHaveLength(0);
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return handleClick();');
});
test('should save urls pushed to window.history', async () => {
const html = `
div
`;
await page.goto(`${serverAddress}/bar/`);
await page.setContent(html);
const interceptedRequests = await clickElements.clickElementsAndInterceptNavigationRequests(getOpts());
expect(interceptedRequests).toHaveLength(1);
expect(interceptedRequests[0].url).toBe(`${serverAddress}/bar/foo`);
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return window.history.pushState');
});
test.skip('should close newly opened tabs', async () => {
const html = `
div
`;
await page.setContent(html);
const callCounts = await new Promise<{ create: number; destroy: number }>((resolve) => {
if (isPuppeteerBrowser(browser)) {
let spawnedTarget: Target;
const counts = {
create: 0,
destroy: 0,
};
(browser as PPBrowser).on('targetcreated', (target) => {
counts.create++;
if ((clickElements as typeof puppeteerClickElements).isTargetRelevant(page, target))
spawnedTarget = target;
});
browser.on('targetdestroyed', (target) => {
counts.destroy++;
if (spawnedTarget === target) resolve(counts);
});
}
if (isPlaywrightBrowser(browser)) {
const counts = {
create: 0,
destroy: 0,
};
page.on('popup', (target: PWPage) => {
counts.create++;
target.on('close', () => {
counts.destroy++;
resolve(counts);
});
});
}
clickElements
.clickElementsAndInterceptNavigationRequests(
getOpts({
waitForPageIdleMillis: 1000,
maxWaitForPageIdleMillis: 5000,
}),
)
.catch(() => {
/* will throw because afterEach will close the page */
});
});
expect(callCounts.create).toBe(1);
expect(callCounts.destroy).toBe(1);
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return window.open(');
});
test.skip('should save urls from newly opened tabs', async () => {
const html = `
div
`;
await page.setContent(html);
const interceptedRequests = await clickElements.clickElementsAndInterceptNavigationRequests(
getOpts({
waitForPageIdleMillis: 1000,
maxWaitForPageIdleMillis: 5000,
}),
);
await new Promise((r) => setTimeout(r, 1000));
expect(interceptedRequests).toHaveLength(1);
expect(interceptedRequests[0].url).toBe(`${serverAddress}/`);
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return window.open(');
});
});
});
});