apify--crawlee/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts

import { MemoryStorage } from '@crawlee/memory-storage';
import type { RequestQueueInfo } from '@crawlee/types';
import { Configuration, RequestQueue } from 'crawlee';

const originalClient = Configuration.getStorageClient();
Configuration.useStorageClient(new MemoryStorage({ persistStorage: false, writeMetadata: false }));

afterAll(() => {
    Configuration.useStorageClient(originalClient);
});

let requestQueueInfo: RequestQueueInfo;

beforeAll(async () => {
    requestQueueInfo = await Configuration.getStorageClient()
        .requestQueues()
        .getOrCreate('test-request-queue-not-called-on-cached-request');
});

describe('RequestQueue#addRequest should not call the API if the request is already in the queue', () => {
    test('should not call the API if the request is already in the queue', async () => {
        const requestQueue = new RequestQueue({ id: requestQueueInfo.id, client: Configuration.getStorageClient() });

        const clientSpy = vitest.spyOn(requestQueue.client, 'addRequest');

        const requestData = await requestQueue.addRequest({ url: 'https://example.com' });

        expect(clientSpy).toHaveBeenCalledTimes(1);

        await requestQueue.markRequestHandled({
            id: requestData.requestId,
            url: 'https://example.com',
            uniqueKey: requestData.uniqueKey,
        } as any);

        await requestQueue.addRequest({ url: 'https://example.com' });

        expect(clientSpy).toHaveBeenCalledTimes(1);
    });
});

describe('RequestQueue#addRequests should not call the API if the request is already in the queue', () => {
    test('should not call the API if the request is already in the queue', async () => {
        const requestQueue = new RequestQueue({ id: requestQueueInfo.id, client: Configuration.getStorageClient() });

        const clientSpy = vitest.spyOn(requestQueue.client, 'batchAddRequests');

        const requestData = await requestQueue.addRequests([{ url: 'https://example2.com' }]);

        expect(clientSpy).toHaveBeenCalledTimes(1);

        await requestQueue.markRequestHandled({
            id: requestData.processedRequests[0].requestId,
            uniqueKey: requestData.processedRequests[0].uniqueKey,
            url: 'https://example2.com',
        } as any);

        await requestQueue.addRequests([{ url: 'https://example2.com' }]);

        expect(clientSpy).toHaveBeenCalledTimes(1);
    });
});