wehub-resource-sync
117 lines
5.3 KiB
TypeScript
117 lines
5.3 KiB
TypeScript
import type { Request } from '@crawlee/core';
|
|
import {
|
|
constructGlobObjectsFromGlobs,
|
|
constructRegExpObjectsFromPseudoUrls,
|
|
constructRegExpObjectsFromRegExps,
|
|
createRequestOptions,
|
|
createRequests,
|
|
validateGlobPattern,
|
|
} from '@crawlee/core';
|
|
|
|
describe('Enqueue links shared functions', () => {
|
|
describe('constructGlobObjectsFromGlobs()', () => {
|
|
test('should work', () => {
|
|
const globs = [
|
|
'https://example.com/**/*',
|
|
{ glob: '?(http|https)://cool.com/', userData: { foo: 'bar' }, label: 'foobar' },
|
|
];
|
|
const globObjects = constructGlobObjectsFromGlobs(globs);
|
|
expect(globObjects).toHaveLength(2);
|
|
expect(globObjects[0].glob).toEqual('https://example.com/**/*');
|
|
expect(globObjects[0].userData).toBe(undefined);
|
|
expect(globObjects[1].glob).toEqual('?(http|https)://cool.com/');
|
|
expect(globObjects[1].userData).toStrictEqual({ foo: 'bar' });
|
|
expect(globObjects[1].label).toBe('foobar');
|
|
});
|
|
});
|
|
|
|
describe('constructRegExpObjectsFromRegExps()', () => {
|
|
test('should work', () => {
|
|
const regexps = [
|
|
/^https:\/\/example\.com\/(\w|\/)+/,
|
|
{ regexp: /^(http|https):\/\/cool\.com\//, userData: { foo: 'bar' } },
|
|
];
|
|
const regexpObjects = constructRegExpObjectsFromRegExps(regexps);
|
|
expect(regexpObjects).toHaveLength(2);
|
|
expect(regexpObjects[0].regexp.test('https://example.com/')).toBe(false);
|
|
expect(regexpObjects[0].userData).toBe(undefined);
|
|
expect(regexpObjects[1].regexp.test('https://cool.com/')).toBe(true);
|
|
expect(regexpObjects[1].userData).toStrictEqual({ foo: 'bar' });
|
|
});
|
|
});
|
|
|
|
describe('constructRegExpObjectsFromPseudoUrls()', () => {
|
|
test('should work', () => {
|
|
const pseudoUrls = [
|
|
'http[s?]://example.com/',
|
|
{ purl: 'http[s?]://example.com[.*]', userData: { foo: 'bar' } },
|
|
];
|
|
const urlPatternObjects = constructRegExpObjectsFromPseudoUrls(pseudoUrls);
|
|
expect(urlPatternObjects).toHaveLength(2);
|
|
urlPatternObjects.forEach((urlPatternObject) => {
|
|
expect(urlPatternObject.regexp.test('https://example.com/')).toBe(true);
|
|
});
|
|
expect(urlPatternObjects[0].regexp.test('https://example.com/foo')).toBe(false);
|
|
expect(urlPatternObjects[0].userData).toBe(undefined);
|
|
expect(urlPatternObjects[1].regexp.test('https://example.com/foo')).toBe(true);
|
|
expect(urlPatternObjects[1].userData).toStrictEqual({ foo: 'bar' });
|
|
});
|
|
|
|
test('should cache items', () => {
|
|
const pseudoUrls0 = constructRegExpObjectsFromPseudoUrls(['http[s?]://example.com/[.*]']);
|
|
const pseudoUrls1 = constructRegExpObjectsFromPseudoUrls(['http[s?]://example.com/[.*]']);
|
|
expect(pseudoUrls0[0]).toEqual(pseudoUrls1[0]);
|
|
|
|
const globs0 = constructGlobObjectsFromGlobs(['https://example.com/**/*']);
|
|
const globs1 = constructGlobObjectsFromGlobs(['https://example.com/**/*']);
|
|
expect(globs0[0]).toEqual(globs1[0]);
|
|
|
|
const regexps0 = constructRegExpObjectsFromRegExps([/^https:\/\/example\.com\/(\w|\/)+/]);
|
|
const regexps1 = constructRegExpObjectsFromRegExps([/^https:\/\/example\.com\/(\w|\/)+/]);
|
|
expect(regexps0[0]).toEqual(regexps1[0]);
|
|
});
|
|
});
|
|
|
|
describe('createRequests()', () => {
|
|
test('should work', () => {
|
|
const sources = [
|
|
'http://example.com/foo',
|
|
{ url: 'https://example.com/bar', method: 'POST', label: 'POST-REQUEST' },
|
|
'https://apify.com',
|
|
];
|
|
const pseudoUrls = [{ purl: 'http[s?]://example.com/[.*]', userData: { one: 1 } }];
|
|
const urlPatternObjects = constructRegExpObjectsFromPseudoUrls(pseudoUrls);
|
|
|
|
const transformRequestFunction = (request: Request) => {
|
|
request.userData.foo = 'bar';
|
|
return request;
|
|
};
|
|
|
|
const requestOptions = createRequestOptions(sources);
|
|
const requests = createRequests(requestOptions, urlPatternObjects)
|
|
.map(transformRequestFunction)
|
|
.filter((r) => !!r);
|
|
|
|
expect(requests).toHaveLength(2);
|
|
requests.forEach((r) => {
|
|
expect(r.url).toMatch(/^https?:\/\/example\.com\//);
|
|
expect(r.userData).toMatchObject({ foo: 'bar', one: 1 });
|
|
});
|
|
expect(requests[0].method).toBe('GET');
|
|
expect(requests[1].method).toBe('POST');
|
|
expect(requests[1].userData).toEqual({ foo: 'bar', one: 1, label: 'POST-REQUEST' });
|
|
});
|
|
});
|
|
|
|
describe('validateGlobPattern()', () => {
|
|
test('should throw for empty glob patterns', () => {
|
|
const globPattern = 'https://example.com/**/*';
|
|
expect(() => validateGlobPattern(globPattern)).not.toThrow();
|
|
const emptyGlobPattern = '';
|
|
expect(() => validateGlobPattern(emptyGlobPattern)).toThrow(
|
|
/Cannot parse Glob pattern '': it must be an non-empty string/,
|
|
);
|
|
});
|
|
});
|
|
});
|