wehub-resource-sync
110 lines
3.5 KiB
TypeScript
110 lines
3.5 KiB
TypeScript
/**
|
|
* E2E tests for request-level crawler options:
|
|
* userAgent / locale / referer / setCookies / proxyUrl / proxy / customHeader / robotsTxt
|
|
*/
|
|
import { describe, it } from 'node:test';
|
|
import assert from 'node:assert';
|
|
import { crawl, crawlWithHeaders } from '../helpers/client';
|
|
|
|
describe('userAgent', () => {
|
|
it('accepts custom userAgent in body', async () => {
|
|
const res = await crawl({ userAgent: 'CustomBot/1.0' });
|
|
assert.strictEqual(res.status, 200);
|
|
assert.notStrictEqual(res.body.data.content, undefined);
|
|
});
|
|
|
|
it('X-User-Agent header is accepted', async () => {
|
|
const res = await crawlWithHeaders({ 'X-User-Agent': 'CustomBot/1.0' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
});
|
|
|
|
describe('locale', () => {
|
|
it('accepts locale in body', async () => {
|
|
const res = await crawl({ locale: 'en-US' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
|
|
it('X-Locale header is accepted', async () => {
|
|
const res = await crawlWithHeaders({ 'X-Locale': 'ja-JP' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
});
|
|
|
|
describe('referer', () => {
|
|
it('accepts referer in body', async () => {
|
|
const res = await crawl({ referer: 'https://google.com' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
|
|
it('X-Referer header is accepted', async () => {
|
|
const res = await crawlWithHeaders({ 'X-Referer': 'https://google.com' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
});
|
|
|
|
describe('setCookies', () => {
|
|
it('accepts cookies array in body', async () => {
|
|
const res = await crawl({
|
|
setCookies: ['session=abc123; Path=/; Domain=example.com'],
|
|
});
|
|
assert.strictEqual(res.status, 200);
|
|
assert.notStrictEqual(res.body.data.content, undefined);
|
|
});
|
|
|
|
it('X-Set-Cookie header is accepted', async () => {
|
|
const res = await crawlWithHeaders({
|
|
'X-Set-Cookie': 'session=abc123; Path=/; Domain=example.com',
|
|
});
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
});
|
|
|
|
describe('proxyUrl', () => {
|
|
it('accepts proxyUrl in body', async () => {
|
|
const res = await crawl({ proxyUrl: 'http://proxy.example.com:8080' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
|
|
it('X-Proxy-Url header is accepted', async () => {
|
|
const res = await crawlWithHeaders({
|
|
'X-Proxy-Url': 'socks5://proxy.example.com:1080',
|
|
});
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
});
|
|
|
|
describe('proxy', () => {
|
|
it('accepts proxy flag in body', async () => {
|
|
const res = await crawl({ proxy: 'us' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
|
|
it('X-Proxy header is accepted', async () => {
|
|
const res = await crawlWithHeaders({ 'X-Proxy': 'de' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
});
|
|
|
|
describe('customHeader', () => {
|
|
it('accepts customHeader dict in body', async () => {
|
|
const res = await crawl({
|
|
customHeader: { 'X-Custom-Test': 'hello' },
|
|
});
|
|
assert.strictEqual(res.status, 200);
|
|
assert.notStrictEqual(res.body.data.content, undefined);
|
|
});
|
|
});
|
|
|
|
describe('robotsTxt', () => {
|
|
it('accepts robotsTxt in body', async () => {
|
|
const res = await crawl({ robotsTxt: 'Googlebot' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
|
|
it('X-Robots-Txt header is accepted', async () => {
|
|
const res = await crawlWithHeaders({ 'X-Robots-Txt': 'Googlebot' });
|
|
assert.strictEqual(res.status, 200);
|
|
});
|
|
});
|