wehub-resource-sync
850 lines
34 KiB
TypeScript
850 lines
34 KiB
TypeScript
import {
|
|
Configuration,
|
|
deserializeArray,
|
|
EventType,
|
|
KeyValueStore,
|
|
ProxyConfiguration,
|
|
Request,
|
|
RequestList,
|
|
} from '@crawlee/core';
|
|
import type { gotScraping } from '@crawlee/utils';
|
|
import { sleep } from '@crawlee/utils';
|
|
import { beforeAll, type MockedFunction } from 'vitest';
|
|
|
|
import log from '@apify/log';
|
|
|
|
import { MemoryStorageEmulator } from '../shared/MemoryStorageEmulator';
|
|
|
|
/**
|
|
* Stand-in for underscore.js shuffle (weird, but how else?)
|
|
*/
|
|
function shuffle(array: unknown[]): unknown[] {
|
|
const out = [...array];
|
|
for (let i = out.length - 1; i > 0; i--) {
|
|
const j = Math.floor(Math.random() * (i + 1));
|
|
[out[i], out[j]] = [out[j], out[i]];
|
|
}
|
|
return out;
|
|
}
|
|
|
|
vitest.mock('@crawlee/utils/src/internals/gotScraping', async () => {
|
|
return {
|
|
gotScraping: vitest.fn(),
|
|
};
|
|
});
|
|
|
|
let gotScrapingSpy: MockedFunction<typeof gotScraping>;
|
|
|
|
beforeAll(async () => {
|
|
// @ts-ignore for some reason, this fails when the project is not built :/
|
|
const { gotScraping } = await import('@crawlee/utils');
|
|
gotScrapingSpy = vitest.mocked(gotScraping);
|
|
});
|
|
|
|
describe('RequestList', () => {
|
|
let ll: number;
|
|
const emulator = new MemoryStorageEmulator();
|
|
const events = Configuration.getEventManager();
|
|
|
|
beforeAll(() => {
|
|
ll = log.getLevel();
|
|
log.setLevel(log.LEVELS.ERROR);
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
await emulator.init();
|
|
vitest.restoreAllMocks();
|
|
});
|
|
|
|
afterAll(async () => {
|
|
log.setLevel(ll);
|
|
await emulator.destroy();
|
|
});
|
|
|
|
test('should not accept to pages with same uniqueKey', async () => {
|
|
const requestList = await RequestList.open(null, [
|
|
{ url: 'https://example.com/1' },
|
|
{ url: 'https://example.com/1#same' },
|
|
]);
|
|
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
|
|
const req = await requestList.fetchNextRequest();
|
|
|
|
expect(req!.url).toBe('https://example.com/1');
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(await requestList.fetchNextRequest()).toBe(null);
|
|
|
|
await requestList.markRequestHandled(req!);
|
|
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
});
|
|
|
|
test('must be initialized before using any of the methods', async () => {
|
|
// @ts-expect-error private constructor
|
|
const requestList = new RequestList({ sources: [{ url: 'https://example.com' }] });
|
|
const requestObj = new Request({ url: 'https://example.com' });
|
|
|
|
await expect(requestList.isEmpty()).rejects.toThrow();
|
|
await expect(requestList.isFinished()).rejects.toThrow();
|
|
expect(() => requestList.getState()).toThrowError();
|
|
await expect(requestList.markRequestHandled(requestObj)).rejects.toThrow();
|
|
await expect(requestList.reclaimRequest(requestObj)).rejects.toThrow();
|
|
await expect(requestList.fetchNextRequest()).rejects.toThrow();
|
|
|
|
await requestList.initialize();
|
|
|
|
await expect(requestList.isEmpty()).resolves.not.toThrow();
|
|
await expect(requestList.isFinished()).resolves.not.toThrow();
|
|
expect(() => requestList.getState()).not.toThrowError();
|
|
await expect(requestList.fetchNextRequest()).resolves.not.toThrow();
|
|
await expect(requestList.reclaimRequest(requestObj)).resolves.not.toThrow();
|
|
await expect(requestList.fetchNextRequest()).resolves.not.toThrow();
|
|
await expect(requestList.markRequestHandled(requestObj)).resolves.not.toThrow();
|
|
});
|
|
|
|
test('should correctly initialize itself', async () => {
|
|
const sources = [
|
|
{ url: 'https://example.com/1' },
|
|
{ url: 'https://example.com/2' },
|
|
{ url: 'https://example.com/3' },
|
|
{ url: 'https://example.com/4' },
|
|
{ url: 'https://example.com/5' },
|
|
{ url: 'https://example.com/6' },
|
|
{ url: 'https://example.com/7' },
|
|
{ url: 'https://example.com/8' },
|
|
];
|
|
const sourcesCopy = JSON.parse(JSON.stringify(sources));
|
|
|
|
const originalList = await RequestList.open(null, sources);
|
|
|
|
const r1 = await originalList.fetchNextRequest(); // 1
|
|
const r2 = await originalList.fetchNextRequest(); // 2
|
|
await originalList.fetchNextRequest(); // 3
|
|
const r4 = await originalList.fetchNextRequest(); // 4
|
|
const r5 = await originalList.fetchNextRequest(); // 5
|
|
await originalList.fetchNextRequest(); // 6
|
|
|
|
await originalList.markRequestHandled(r1!);
|
|
await originalList.markRequestHandled(r2!);
|
|
await originalList.markRequestHandled(r4!);
|
|
await originalList.reclaimRequest(r5!);
|
|
|
|
const newList = await RequestList.open({
|
|
sources: sourcesCopy,
|
|
state: originalList.getState(),
|
|
});
|
|
|
|
expect(await newList.isEmpty()).toBe(false);
|
|
expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/3');
|
|
expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/5');
|
|
expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/6');
|
|
expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/7');
|
|
expect((await newList.fetchNextRequest())!.url).toBe('https://example.com/8');
|
|
expect(await newList.isEmpty()).toBe(true);
|
|
});
|
|
|
|
test('`RequestList` is `for .. await` iterable', async () => {
|
|
const sources = [
|
|
'https://example.com/1',
|
|
'https://example.com/2',
|
|
'https://example.com/3',
|
|
'https://example.com/4',
|
|
'https://example.com/5',
|
|
'https://example.com/6',
|
|
'https://example.com/7',
|
|
'https://example.com/8',
|
|
];
|
|
const requestList = await RequestList.open(null, sources);
|
|
|
|
for await (const request of requestList) {
|
|
expect(request?.url).toBe(sources.shift());
|
|
}
|
|
});
|
|
|
|
test('should correctly load list from hosted files in correct order', async () => {
|
|
const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');
|
|
const list1 = ['https://example.com', 'https://google.com', 'https://wired.com'];
|
|
const list2 = ['https://another.com', 'https://page.com'];
|
|
spy.mockImplementationOnce(() => new Promise((resolve) => setTimeout(() => resolve(list1) as any, 100)) as any);
|
|
spy.mockResolvedValueOnce(list2);
|
|
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{ method: 'GET', requestsFromUrl: 'http://example.com/list-1' },
|
|
{ method: 'POST', requestsFromUrl: 'http://example.com/list-2' },
|
|
],
|
|
});
|
|
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[0] });
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[1] });
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[2] });
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[0] });
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[1] });
|
|
|
|
expect(spy).toBeCalledTimes(2);
|
|
expect(spy).toBeCalledWith({ url: 'http://example.com/list-1', urlRegExp: undefined });
|
|
expect(spy).toBeCalledWith({ url: 'http://example.com/list-2', urlRegExp: undefined });
|
|
});
|
|
|
|
test('should use regex parameter to parse urls', async () => {
|
|
const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd';
|
|
const listArr = ['https://example.com', 'HTTP://google.com'];
|
|
gotScrapingSpy.mockResolvedValue({ body: listStr } as any);
|
|
|
|
const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g;
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{
|
|
method: 'GET',
|
|
requestsFromUrl: 'http://example.com/list-1',
|
|
regex,
|
|
},
|
|
],
|
|
});
|
|
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[0] });
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[1] });
|
|
|
|
expect(gotScrapingSpy).toBeCalledWith({ url: 'http://example.com/list-1', encoding: 'utf8' });
|
|
});
|
|
|
|
test('should fix gdoc sharing url in `requestsFromUrl` automatically (GH issue #639)', async () => {
|
|
const list = ['https://example.com', 'https://google.com', 'https://wired.com'];
|
|
const wrongUrls = [
|
|
'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU',
|
|
'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/',
|
|
'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit?usp=sharing',
|
|
'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/123123132',
|
|
'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/?q=blablabla',
|
|
'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit#gid=0',
|
|
];
|
|
const correctUrl =
|
|
'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/gviz/tq?tqx=out:csv';
|
|
|
|
gotScrapingSpy.mockResolvedValue({ body: JSON.stringify(list) } as any);
|
|
|
|
const requestList = await RequestList.open({
|
|
sources: wrongUrls.map((requestsFromUrl) => ({ requestsFromUrl })),
|
|
});
|
|
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[0] });
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[1] });
|
|
expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[2] });
|
|
|
|
expect(gotScrapingSpy).toBeCalledWith({ url: correctUrl, encoding: 'utf8' });
|
|
});
|
|
|
|
test('should handle requestsFromUrl with no URLs', async () => {
|
|
const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');
|
|
spy.mockResolvedValueOnce([]);
|
|
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{
|
|
method: 'GET',
|
|
requestsFromUrl: 'http://example.com/list-1',
|
|
},
|
|
],
|
|
});
|
|
|
|
expect(await requestList.fetchNextRequest()).toBe(null);
|
|
|
|
expect(spy).toBeCalledTimes(1);
|
|
expect(spy).toBeCalledWith({ url: 'http://example.com/list-1', urlRegExp: undefined });
|
|
});
|
|
|
|
test('should use the defined proxy server when using `requestsFromUrl`', async () => {
|
|
const proxyUrls = ['http://proxyurl.usedforthe.download', 'http://another.proxy.url'];
|
|
|
|
const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');
|
|
spy.mockResolvedValue([]);
|
|
|
|
const proxyConfiguration = new ProxyConfiguration({
|
|
proxyUrls,
|
|
});
|
|
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{ requestsFromUrl: 'http://example.com/list-1' },
|
|
{ requestsFromUrl: 'http://example.com/list-2' },
|
|
{ requestsFromUrl: 'http://example.com/list-3' },
|
|
],
|
|
proxyConfiguration,
|
|
});
|
|
|
|
expect(spy).not.toBeCalledWith(expect.not.objectContaining({ proxyUrl: expect.any(String) }));
|
|
});
|
|
|
|
test('should correctly handle reclaimed pages', async () => {
|
|
const requestList = await RequestList.open({
|
|
sources: [
|
|
{ url: 'https://example.com/1' },
|
|
{ url: 'https://example.com/2' },
|
|
{ url: 'https://example.com/3' },
|
|
{ url: 'https://example.com/4' },
|
|
{ url: 'https://example.com/5' },
|
|
{ url: 'https://example.com/6' },
|
|
],
|
|
});
|
|
|
|
//
|
|
// Fetch first 5 urls
|
|
//
|
|
|
|
const request1 = await requestList.fetchNextRequest();
|
|
const request2 = await requestList.fetchNextRequest();
|
|
const request3 = await requestList.fetchNextRequest();
|
|
const request4 = await requestList.fetchNextRequest();
|
|
const request5 = await requestList.fetchNextRequest();
|
|
|
|
expect(request1!.url).toBe('https://example.com/1');
|
|
expect(request2!.url).toBe('https://example.com/2');
|
|
expect(request3!.url).toBe('https://example.com/3');
|
|
expect(request4!.url).toBe('https://example.com/4');
|
|
expect(request5!.url).toBe('https://example.com/5');
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: [
|
|
'https://example.com/1',
|
|
'https://example.com/2',
|
|
'https://example.com/3',
|
|
'https://example.com/4',
|
|
'https://example.com/5',
|
|
],
|
|
nextIndex: 5,
|
|
nextUniqueKey: 'https://example.com/6',
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(requestList.inProgress.size).toBe(5);
|
|
expect(requestList.reclaimed.size).toBe(0);
|
|
|
|
//
|
|
// Mark 1st, 2nd handled
|
|
// Reclaim 3rd 4th
|
|
//
|
|
|
|
await requestList.markRequestHandled(request1!);
|
|
await requestList.markRequestHandled(request2!);
|
|
await requestList.reclaimRequest(request3!);
|
|
await requestList.reclaimRequest(request4!);
|
|
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: ['https://example.com/3', 'https://example.com/4', 'https://example.com/5'],
|
|
nextIndex: 5,
|
|
nextUniqueKey: 'https://example.com/6',
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(requestList.inProgress).toEqual(expect.objectContaining(requestList.reclaimed));
|
|
|
|
//
|
|
// Mark 5th handled
|
|
//
|
|
|
|
await requestList.markRequestHandled(request5!);
|
|
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: ['https://example.com/3', 'https://example.com/4'],
|
|
nextIndex: 5,
|
|
nextUniqueKey: 'https://example.com/6',
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(requestList.inProgress).toEqual(expect.objectContaining(requestList.reclaimed));
|
|
|
|
//
|
|
// Fetch 3rd and 4th
|
|
// Mark 4th handled
|
|
//
|
|
|
|
const reclaimed3 = await requestList.fetchNextRequest();
|
|
expect(reclaimed3!.url).toBe('https://example.com/3');
|
|
const reclaimed4 = await requestList.fetchNextRequest();
|
|
expect(reclaimed4!.url).toBe('https://example.com/4');
|
|
await requestList.markRequestHandled(request4!);
|
|
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: ['https://example.com/3'],
|
|
nextIndex: 5,
|
|
nextUniqueKey: 'https://example.com/6',
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(requestList.inProgress).toEqual(expect.objectContaining(requestList.reclaimed));
|
|
|
|
//
|
|
// Mark 3rd handled
|
|
//
|
|
|
|
await requestList.markRequestHandled(request3!);
|
|
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: [],
|
|
nextIndex: 5,
|
|
nextUniqueKey: 'https://example.com/6',
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(requestList.inProgress).toEqual(expect.objectContaining(requestList.reclaimed));
|
|
|
|
//
|
|
// Fetch 6th
|
|
//
|
|
|
|
const request6 = await requestList.fetchNextRequest();
|
|
|
|
expect(request6!.url).toBe('https://example.com/6');
|
|
expect(await requestList.fetchNextRequest()).toBe(null);
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: ['https://example.com/6'],
|
|
nextIndex: 6,
|
|
nextUniqueKey: null,
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(requestList.inProgress).toEqual(expect.objectContaining(requestList.reclaimed));
|
|
|
|
//
|
|
// Reclaim 6th
|
|
//
|
|
|
|
await requestList.reclaimRequest(request6!);
|
|
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: ['https://example.com/6'],
|
|
nextIndex: 6,
|
|
nextUniqueKey: null,
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(false);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(requestList.inProgress).toEqual(expect.objectContaining(requestList.reclaimed));
|
|
|
|
//
|
|
// Fetch 6th
|
|
//
|
|
|
|
const reclaimed6 = await requestList.fetchNextRequest();
|
|
|
|
expect(reclaimed6!.url).toBe('https://example.com/6');
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: ['https://example.com/6'],
|
|
nextIndex: 6,
|
|
nextUniqueKey: null,
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
expect(await requestList.isFinished()).toBe(false);
|
|
expect(requestList.inProgress).toEqual(expect.objectContaining(requestList.reclaimed));
|
|
|
|
//
|
|
// Mark 6th handled
|
|
//
|
|
|
|
await requestList.markRequestHandled(reclaimed6!);
|
|
|
|
expect(requestList.getState()).toEqual({
|
|
inProgress: [],
|
|
nextIndex: 6,
|
|
nextUniqueKey: null,
|
|
});
|
|
expect(await requestList.isEmpty()).toBe(true);
|
|
expect(await requestList.isFinished()).toBe(true);
|
|
expect(requestList.inProgress).toEqual(expect.objectContaining(requestList.reclaimed));
|
|
});
|
|
|
|
test('should correctly persist its state when persistStateKey is set', async () => {
|
|
const PERSIST_STATE_KEY = 'some-key';
|
|
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
|
|
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
|
|
|
|
getValueSpy.mockResolvedValueOnce(null);
|
|
|
|
const opts = {
|
|
sources: [
|
|
{ url: 'https://example.com/1' },
|
|
{ url: 'https://example.com/2' },
|
|
{ url: 'https://example.com/3' },
|
|
],
|
|
persistStateKey: PERSIST_STATE_KEY,
|
|
};
|
|
const optsCopy = JSON.parse(JSON.stringify(opts));
|
|
|
|
const requestList = await RequestList.open(opts);
|
|
expect(requestList.isStatePersisted).toBe(true);
|
|
|
|
// Fetch one request and check that state is not persisted.
|
|
const request1 = await requestList.fetchNextRequest();
|
|
expect(requestList.isStatePersisted).toBe(false);
|
|
|
|
// Persist state.
|
|
setValueSpy.mockResolvedValueOnce();
|
|
events.emit(EventType.PERSIST_STATE);
|
|
await sleep(20);
|
|
expect(requestList.isStatePersisted).toBe(true);
|
|
|
|
// Do some other changes and persist it again.
|
|
const request2 = await requestList.fetchNextRequest();
|
|
expect(requestList.isStatePersisted).toBe(false);
|
|
await requestList.markRequestHandled(request2!);
|
|
expect(requestList.isStatePersisted).toBe(false);
|
|
setValueSpy.mockResolvedValueOnce();
|
|
events.emit(EventType.PERSIST_STATE);
|
|
await sleep(20);
|
|
expect(requestList.isStatePersisted).toBe(true);
|
|
|
|
// Reclaim event doesn't change the state.
|
|
await requestList.reclaimRequest(request1!);
|
|
expect(requestList.isStatePersisted).toBe(true);
|
|
|
|
// Now initiate new request list from saved state and check that it's same as state
|
|
// of original request list.
|
|
getValueSpy.mockResolvedValueOnce(requestList.getState());
|
|
const requestList2 = await RequestList.open(optsCopy);
|
|
expect(requestList2.getState()).toEqual(requestList.getState());
|
|
});
|
|
|
|
test('should correctly persist its sources when persistRequestsKey is set', async () => {
|
|
const PERSIST_REQUESTS_KEY = 'some-key';
|
|
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
|
|
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
|
|
|
|
let persistedRequests;
|
|
|
|
const opts = {
|
|
sources: [
|
|
{ url: 'https://example.com/1' },
|
|
{ url: 'https://example.com/2' },
|
|
{ url: 'https://example.com/3' },
|
|
],
|
|
persistRequestsKey: PERSIST_REQUESTS_KEY,
|
|
};
|
|
|
|
// Expect an attempt to load sources.
|
|
getValueSpy.mockResolvedValueOnce(null);
|
|
|
|
// Expect persist sources.
|
|
setValueSpy.mockImplementationOnce(async (_key, value) => {
|
|
persistedRequests = value;
|
|
});
|
|
|
|
const requestList = await RequestList.open(opts);
|
|
expect(requestList.areRequestsPersisted).toBe(true);
|
|
|
|
const opts2 = {
|
|
sources: [{ url: 'https://test.com/1' }, { url: 'https://test.com/2' }, { url: 'https://test.com/3' }],
|
|
persistRequestsKey: PERSIST_REQUESTS_KEY,
|
|
};
|
|
|
|
getValueSpy.mockResolvedValueOnce(persistedRequests);
|
|
|
|
const requestList2 = await RequestList.open(opts2);
|
|
expect(requestList2.areRequestsPersisted).toBe(true);
|
|
expect(requestList2.requests).toEqual(requestList.requests);
|
|
});
|
|
|
|
test('should correctly persist sources from requestsFromUrl if persistRequestsKey is set', async () => {
|
|
const PERSIST_REQUESTS_KEY = 'some-key';
|
|
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
|
|
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
|
|
const spy = vitest.spyOn(RequestList.prototype as any, '_downloadListOfUrls');
|
|
let persistedRequests: any;
|
|
|
|
const opts = {
|
|
sources: [
|
|
{ url: 'https://example.com/1' },
|
|
{ url: 'https://example.com/2' },
|
|
{ requestsFromUrl: 'http://example.com/list-urls.txt', userData: { isFromUrl: true } },
|
|
{ url: 'https://example.com/5' },
|
|
],
|
|
persistRequestsKey: PERSIST_REQUESTS_KEY,
|
|
};
|
|
|
|
const urlsFromTxt = ['http://example.com/3', 'http://example.com/4'];
|
|
spy.mockResolvedValueOnce(urlsFromTxt);
|
|
|
|
getValueSpy.mockResolvedValueOnce(null);
|
|
setValueSpy.mockImplementationOnce(async (_key, value) => {
|
|
persistedRequests = value;
|
|
});
|
|
|
|
const requestList = await RequestList.open(opts);
|
|
expect(requestList.areRequestsPersisted).toBe(true);
|
|
const requests = await deserializeArray(persistedRequests);
|
|
expect(requestList.requests).toHaveLength(5);
|
|
expect(requests).toEqual(requestList.requests);
|
|
|
|
expect(spy).toBeCalledTimes(1);
|
|
expect(spy).toBeCalledWith({ url: 'http://example.com/list-urls.txt', urlRegExp: undefined });
|
|
});
|
|
|
|
test('handles correctly inconsistent inProgress fields in state', async () => {
|
|
// NOTE: This is a test for the deleteFromInProgress hotfix - see RequestList.initialize()
|
|
|
|
const sources = [
|
|
{ url: 'https://www.ams360.com' },
|
|
{ url: 'https://www.anybus.com' },
|
|
{ url: 'https://www.anychart.com' },
|
|
{ url: 'https://www.example.com' },
|
|
];
|
|
|
|
const state = {
|
|
nextIndex: 2,
|
|
nextUniqueKey: 'https://www.anychart.com',
|
|
inProgress: ['https://www.ams360.com', 'https://www.anybus.com', 'https://www.anychart.com'],
|
|
};
|
|
|
|
const requestList = await RequestList.open({
|
|
sources,
|
|
state,
|
|
});
|
|
|
|
// Get requests from list
|
|
let reqs: Request[] = [];
|
|
for (let i = 0; i < 5; i++) {
|
|
const request = await requestList.fetchNextRequest();
|
|
if (!request) break;
|
|
reqs.push(request);
|
|
}
|
|
|
|
reqs = shuffle(reqs) as typeof reqs;
|
|
|
|
for (let i = 0; i < reqs.length; i++) {
|
|
await requestList.reclaimRequest(reqs[i]);
|
|
}
|
|
});
|
|
|
|
test('it gets correct length()', async () => {
|
|
const sources = [
|
|
{ url: 'https://www.example.com' },
|
|
{ url: 'https://www.ams360.com' },
|
|
{ url: 'https://www.anybus.com' },
|
|
{ url: 'https://www.anychart.com' },
|
|
{ url: 'https://www.example.com' },
|
|
];
|
|
|
|
const requestList = await RequestList.open({
|
|
sources,
|
|
});
|
|
|
|
expect(requestList.length()).toBe(4);
|
|
});
|
|
|
|
test('it gets correct handledCount()', async () => {
|
|
const sources = [
|
|
{ url: 'https://www.example.com' },
|
|
{ url: 'https://www.ams360.com' },
|
|
{ url: 'https://www.anybus.com' },
|
|
{ url: 'https://www.anychart.com' },
|
|
{ url: 'https://www.example.com' },
|
|
];
|
|
|
|
const requestList = await RequestList.open({
|
|
sources,
|
|
});
|
|
|
|
const req1 = await requestList.fetchNextRequest();
|
|
const req2 = await requestList.fetchNextRequest();
|
|
const req3 = await requestList.fetchNextRequest();
|
|
expect(requestList.handledCount()).toBe(0);
|
|
|
|
await requestList.markRequestHandled(req2!);
|
|
expect(requestList.handledCount()).toBe(1);
|
|
|
|
await requestList.markRequestHandled(req3!);
|
|
expect(requestList.handledCount()).toBe(2);
|
|
|
|
await requestList.reclaimRequest(req1!);
|
|
expect(requestList.handledCount()).toBe(2);
|
|
});
|
|
|
|
test('should correctly keep duplicate URLs while keepDuplicateUrls is set', async () => {
|
|
const sources = [
|
|
{ url: 'https://www.example.com' },
|
|
{ url: 'https://www.example.com' },
|
|
{ url: 'https://www.example.com' },
|
|
{ url: 'https://www.ex2mple.com' },
|
|
];
|
|
const sourcesCopy = JSON.parse(JSON.stringify(sources));
|
|
|
|
let requestList = await RequestList.open({
|
|
sources,
|
|
keepDuplicateUrls: true,
|
|
});
|
|
|
|
expect(requestList.length()).toBe(4);
|
|
|
|
log.setLevel(log.LEVELS.INFO);
|
|
const warnSpy = vitest.spyOn(console, 'warn').mockImplementation(() => {});
|
|
|
|
requestList = await RequestList.open({
|
|
sources: sourcesCopy.concat([
|
|
{ url: 'https://www.example.com', uniqueKey: '123' },
|
|
{ url: 'https://www.example.com', uniqueKey: '123' },
|
|
{ url: 'https://www.example.com', uniqueKey: '456' },
|
|
{ url: 'https://www.ex2mple.com', uniqueKey: '456' },
|
|
]),
|
|
keepDuplicateUrls: true,
|
|
});
|
|
|
|
expect(requestList.length()).toBe(6);
|
|
expect(warnSpy).toBeCalled();
|
|
expect(warnSpy.mock.calls[0][0]).toMatch(`Check your sources' unique keys.`);
|
|
|
|
log.setLevel(log.LEVELS.ERROR);
|
|
});
|
|
|
|
describe('Apify.RequestList.open()', () => {
|
|
test('should work', async () => {
|
|
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
|
|
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
|
|
|
|
const name = 'xxx';
|
|
const SDK_KEY = `SDK_${name}`;
|
|
const sources = [{ url: 'https://example.com' }];
|
|
|
|
const rl = await RequestList.open(name, sources);
|
|
expect(rl).toBeInstanceOf(RequestList);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.persistStateKey.startsWith(SDK_KEY)).toBe(true);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.persistRequestsKey.startsWith(SDK_KEY)).toBe(true);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.sources).toEqual([]);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.isInitialized).toBe(true);
|
|
|
|
expect(getValueSpy).toBeCalledTimes(2);
|
|
expect(setValueSpy).toBeCalledTimes(1);
|
|
});
|
|
|
|
test('should work with string sources', async () => {
|
|
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
|
|
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
|
|
|
|
const name = 'xxx';
|
|
const SDK_KEY = `SDK_${name}`;
|
|
const sources = ['https://example.com'];
|
|
const requests = sources.map((url) => ({ url, uniqueKey: url }));
|
|
|
|
const rl = await RequestList.open(name, sources);
|
|
expect(rl).toBeInstanceOf(RequestList);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.persistStateKey.startsWith(SDK_KEY)).toBe(true);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.persistRequestsKey.startsWith(SDK_KEY)).toBe(true);
|
|
expect(rl.requests).toEqual(requests);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.isInitialized).toBe(true);
|
|
|
|
expect(getValueSpy).toBeCalledTimes(2);
|
|
expect(setValueSpy).toBeCalledTimes(1);
|
|
});
|
|
|
|
test('should correctly pass options', async () => {
|
|
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
|
|
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
|
|
|
|
const name = 'xxx';
|
|
const SDK_KEY = `SDK_${name}`;
|
|
let counter = 0;
|
|
const sources = [{ url: 'https://example.com' }];
|
|
const requests = sources.map(({ url }) => ({ url, uniqueKey: `${url}-${counter++}` }));
|
|
const options = {
|
|
keepDuplicateUrls: true,
|
|
persistStateKey: 'yyy',
|
|
};
|
|
|
|
const rl = await RequestList.open(name, sources, options);
|
|
expect(rl).toBeInstanceOf(RequestList);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.persistStateKey.startsWith(SDK_KEY)).toBe(true);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.persistRequestsKey.startsWith(SDK_KEY)).toBe(true);
|
|
expect(rl.requests).toEqual(requests);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.isInitialized).toBe(true);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.keepDuplicateUrls).toBe(true);
|
|
|
|
expect(getValueSpy).toBeCalledTimes(2);
|
|
expect(setValueSpy).toBeCalledTimes(1);
|
|
});
|
|
|
|
test('should work with null name', async () => {
|
|
const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue');
|
|
const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue');
|
|
|
|
const name: string | null = null;
|
|
const sources = [{ url: 'https://example.com' }];
|
|
const requests = sources.map(({ url }) => ({ url, uniqueKey: url }));
|
|
|
|
const rl = await RequestList.open(name, sources);
|
|
expect(rl).toBeInstanceOf(RequestList);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.persistStateKey == null).toBe(true);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.persistRequestsKey == null).toBe(true);
|
|
expect(rl.requests).toEqual(requests);
|
|
// @ts-expect-error accessing private var
|
|
expect(rl.isInitialized).toBe(true);
|
|
|
|
expect(getValueSpy).not.toBeCalled();
|
|
expect(setValueSpy).not.toBeCalled();
|
|
});
|
|
|
|
test('should throw on invalid parameters', async () => {
|
|
const args = [[], ['x', {}], ['x', 6, {}], ['x', [], []]] as const;
|
|
for (const arg of args) {
|
|
try {
|
|
// @ts-ignore
|
|
await RequestList.open(...arg);
|
|
throw new Error('wrong error');
|
|
} catch (err) {
|
|
const e = err as Error;
|
|
expect(e.message).not.toBe('wrong error');
|
|
if (e.message.match('argument to be of type `string`')) {
|
|
expect(e.message).toMatch('received type `undefined`');
|
|
} else if (e.message.match('argument to be of type `array`')) {
|
|
const isMatched =
|
|
e.message.match('received type `Object`') ||
|
|
e.message.match('received type `number`') ||
|
|
e.message.match('received type `undefined`');
|
|
expect(isMatched).toBeTruthy();
|
|
} else if (e.message.match('argument to be of type `null`')) {
|
|
expect(e.message).toMatch('received type `undefined`');
|
|
}
|
|
}
|
|
}
|
|
});
|
|
});
|
|
|
|
// This test is here to run locally. It would take too long
|
|
// when running a test suite and in CI with large source arrays
|
|
// and would be flaky with small source arrays, so manual inspection
|
|
// looks like the best idea, since multiple runs with various values
|
|
// need to be tested and compared (read: I'm too lazy to automate this)
|
|
|
|
// test('memory consumption does not spike', async () => {
|
|
// function getMemoryInMbytes() {
|
|
// const memory = process.memoryUsage();
|
|
// return (memory.heapUsed + memory.external) / 1024 / 1024;
|
|
// }
|
|
// const sources = [];
|
|
// for (let i = 0; i < 1e6; i++) {
|
|
// sources.push({ url: `https://example.com?page=${i}` });
|
|
// }
|
|
// const startingMemory = getMemoryInMbytes();
|
|
// console.log(startingMemory, 'MB');
|
|
//
|
|
// process.env.APIFY_LOCAL_STORAGE_DIR = 'tmp';
|
|
// const rl = await RequestList.open({ sources, persistRequestsKey: null });
|
|
// const instanceMemory = getMemoryInMbytes();
|
|
// console.log(instanceMemory, 'MB');
|
|
//
|
|
// const initMemory = getMemoryInMbytes();
|
|
// console.log(initMemory, 'MB');
|
|
// });
|
|
});
|