wehub-resource-sync
115 lines
4.2 KiB
TypeScript
115 lines
4.2 KiB
TypeScript
import nock from 'nock';
|
|
import { beforeEach, describe, expect, it } from 'vitest';
|
|
|
|
import { RobotsTxtFile } from '../src/internals/robots';
|
|
|
|
describe('RobotsTxtFile', () => {
|
|
beforeEach(() => {
|
|
nock.disableNetConnect();
|
|
nock('http://not-exists.com')
|
|
.persist()
|
|
.get('/robots.txt')
|
|
.delay(500)
|
|
.reply(
|
|
200,
|
|
[
|
|
'User-agent: *',
|
|
'Disallow: *deny_all/',
|
|
'crawl-delay: 10',
|
|
|
|
'User-agent: Googlebot',
|
|
'Disallow: *deny_googlebot/',
|
|
'crawl-delay: 1',
|
|
|
|
'user-agent: Mozilla',
|
|
'crawl-delay: 2',
|
|
|
|
'sitemap: http://not-exists.com/sitemap_1.xml',
|
|
'sitemap: http://not-exists.com/sitemap_2.xml',
|
|
].join('\n'),
|
|
)
|
|
.get('*')
|
|
.reply(404);
|
|
});
|
|
|
|
afterEach(() => {
|
|
nock.cleanAll();
|
|
nock.enableNetConnect();
|
|
});
|
|
|
|
it('generates the correct robots.txt URL', async () => {
|
|
const robots = await RobotsTxtFile.find('http://not-exists.com/nested/index.html');
|
|
expect(robots.getSitemaps()).not.toHaveLength(0);
|
|
});
|
|
|
|
it('parses allow/deny directives from robots.txt', async () => {
|
|
const robots = await RobotsTxtFile.find('http://not-exists.com/robots.txt');
|
|
console.log(robots.isAllowed('https://crawlee.dev'));
|
|
expect(robots.isAllowed('http://not-exists.com/something/page.html')).toBe(true);
|
|
expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html')).toBe(true);
|
|
expect(robots.isAllowed('http://not-exists.com/deny_all/page.html')).toBe(false);
|
|
});
|
|
|
|
it('extracts sitemap urls', async () => {
|
|
const robots = await RobotsTxtFile.find('http://not-exists.com/robots.txt');
|
|
expect(robots.getSitemaps()).toEqual([
|
|
'http://not-exists.com/sitemap_1.xml',
|
|
'http://not-exists.com/sitemap_2.xml',
|
|
]);
|
|
});
|
|
|
|
it('respects user-set timeout', async () => {
|
|
const start = +Date.now();
|
|
const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', undefined, { timeoutMillis: 200 });
|
|
|
|
await expect(robots).rejects.toThrow(/timeout/i);
|
|
const end = +Date.now();
|
|
|
|
expect(end - start).toBeGreaterThanOrEqual(200);
|
|
expect(end - start).toBeLessThanOrEqual(500);
|
|
});
|
|
|
|
it('respects AbortSignal parameter', async () => {
|
|
const controller = new AbortController();
|
|
setTimeout(() => controller.abort(), 200);
|
|
|
|
const start = +Date.now();
|
|
const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', undefined, { signal: controller.signal });
|
|
|
|
await expect(robots).rejects.toThrow(/aborted/i);
|
|
const end = +Date.now();
|
|
|
|
expect(end - start).toBeGreaterThanOrEqual(200);
|
|
expect(end - start).toBeLessThanOrEqual(500);
|
|
});
|
|
|
|
it('respects AbortSignal parameter and timeout together', async () => {
|
|
const controller = new AbortController();
|
|
|
|
const start = +Date.now();
|
|
const robots = RobotsTxtFile.find('http://not-exists.com/robots.txt', undefined, {
|
|
signal: controller.signal,
|
|
timeoutMillis: 200,
|
|
});
|
|
|
|
await expect(robots).rejects.toThrow(/timeout/i);
|
|
const end = +Date.now();
|
|
|
|
expect(end - start).toBeGreaterThanOrEqual(200);
|
|
expect(end - start).toBeLessThanOrEqual(500);
|
|
});
|
|
|
|
it('parses allow/deny directives from explicitly provided robots.txt contents', async () => {
|
|
const contents = `User-agent: *',
|
|
Disallow: *deny_all/
|
|
crawl-delay: 10
|
|
User-agent: Googlebot
|
|
Disallow: *deny_googlebot/`;
|
|
const robots = RobotsTxtFile.from('http://not-exists.com/robots.txt', contents);
|
|
expect(robots.isAllowed('http://not-exists.com/something/page.html')).toBe(true);
|
|
expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html')).toBe(true);
|
|
expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html', 'Googlebot')).toBe(false);
|
|
expect(robots.isAllowed('http://not-exists.com/deny_all/page.html')).toBe(true);
|
|
});
|
|
});
|