import { social } from '@crawlee/utils'; const { EMAIL_REGEX, EMAIL_REGEX_GLOBAL, FACEBOOK_REGEX, FACEBOOK_REGEX_GLOBAL, INSTAGRAM_REGEX, INSTAGRAM_REGEX_GLOBAL, LINKEDIN_REGEX, LINKEDIN_REGEX_GLOBAL, TWITTER_REGEX, TWITTER_REGEX_GLOBAL, YOUTUBE_REGEX, YOUTUBE_REGEX_GLOBAL, TIKTOK_REGEX, TIKTOK_REGEX_GLOBAL, PINTEREST_REGEX, PINTEREST_REGEX_GLOBAL, DISCORD_REGEX, DISCORD_REGEX_GLOBAL, emailsFromText, emailsFromUrls, parseHandlesFromHtml, phonesFromText, phonesFromUrls, } = social; describe('utils.social', () => { describe('emailsFromText()', () => { const testEmailsFromText = (text: string, phones: string[]) => { expect(emailsFromText(text)).toEqual(phones); }; test('works with arg with no emails or invalid', () => { // @ts-expect-error invalid input type expect(emailsFromText()).toEqual([]); testEmailsFromText('', []); // @ts-expect-error invalid input type testEmailsFromText(null, []); // @ts-expect-error invalid input type testEmailsFromText(undefined, []); // @ts-expect-error invalid input type testEmailsFromText({}, []); testEmailsFromText(' ', []); testEmailsFromText(' \n\n\r\t\n ', []); }); test('extracts emails correctly', () => { testEmailsFromText(' info@example.com ', ['info@example.com']); testEmailsFromText( ` info@example.com info+something@example.NET john.bob.dole@some-domain.co.uk `, ['info@example.com', 'info+something@example.NET', 'john.bob.dole@some-domain.co.uk'], ); testEmailsFromText( ` this'is'also'valid'email@EXAMPLE.travel easy-address@some-domain.co.uk \n\n easy-address@some-domain.co.uk not @ an.email.com @also.not.an.email `, [ "this'is'also'valid'email@EXAMPLE.travel", 'easy-address@some-domain.co.uk', 'easy-address@some-domain.co.uk', ], ); testEmailsFromText(' some.super.long.email.address@some.super.long.domain.name.co.br ', [ 'some.super.long.email.address@some.super.long.domain.name.co.br', ]); }); }); describe('emailsFromUrls()', () => { test('throws on invalid arg', () => { expect(() => { // @ts-expect-error invalid input type emailsFromUrls(); }).toThrowError(/must be an array/); expect(() => { // @ts-expect-error invalid input type emailsFromUrls({}); }).toThrowError(/must be an array/); expect(() => { // @ts-expect-error invalid input type emailsFromUrls('fwefwef'); }).toThrowError(/must be an array/); expect(() => { // @ts-expect-error invalid input type emailsFromUrls(12345); }).toThrowError(/must be an array/); }); test('extracts emails correctly', () => { expect(emailsFromUrls([])).toEqual([]); // @ts-expect-error invalid input type expect(emailsFromUrls([1, 2, {}, 'fwef', null, undefined])).toEqual([]); expect(emailsFromUrls(['mailto:info@example.com'])).toEqual(['info@example.com']); expect( emailsFromUrls([ 'http://www.example.com', 'mailto:info@example.com', 'mailto:info@example.com', 'email.without.mailto.prefix@example.com', '', '\n\n\n', ]), ).toEqual(['info@example.com', 'info@example.com']); expect( emailsFromUrls([ 'http://www.example.com', 'mailto:info@example.com', 'mailto:info@example.com', 'email.without.mailto.prefix@example.com', '', '\n\n\n', ]), ).toEqual(['info@example.com', 'info@example.com']); }); }); describe('phonesFromText()', () => { const testPhonesFromText = (text: string, phones: string[]) => { expect(phonesFromText(text)).toEqual(phones); }; test('works with arg with no phones or invalid', () => { // @ts-expect-error invalid input type expect(phonesFromText()).toEqual([]); testPhonesFromText('', []); // @ts-expect-error invalid input type testPhonesFromText(null, []); // @ts-expect-error invalid input type testPhonesFromText(undefined, []); // @ts-expect-error testPhonesFromText({}, []); testPhonesFromText(' ', []); testPhonesFromText(' \n\n\r\t\n ', []); }); test('extracts phones correctly', () => { testPhonesFromText( ` +420775123456 +420775123456 +420 775 123 456 775123456 775123456 \n\n 00420775123456 1234567 1234567890 +44 7911 123456 `, [ '+420775123456', '+420775123456', '+420 775 123 456', '775123456', '775123456', '00420775123456', '1234567', '1234567890', '+44 7911 123456', ], ); testPhonesFromText( ` 413-577-1234 00413-577-1234 981-413-777-8888 413.233.2343 +413.233.2343 or 413 233 2343 562-3113 123456789 401 311 7898 123456789 `, [ '413-577-1234', '00413-577-1234', '981-413-777-8888', '413.233.2343', '+413.233.2343', '413 233 2343', '562-3113', '123456789', '401 311 7898', '123456789', ], ); testPhonesFromText( ` 1 (413) 555-2378 +1 (413) 555-2378 1(413)555-2378 001 (413) 555-2378 1 (413) 555 2378 1(413)555-2378 or 1(413)555.2378 or 1 (413) 555-2378 or 1 (413) 555 2378 or (303) 494-2320 `, [ '1 (413) 555-2378', '+1 (413) 555-2378', '1(413)555-2378', '001 (413) 555-2378', '1 (413) 555 2378', '1(413)555-2378', '1(413)555.2378', '1 (413) 555-2378', '1 (413) 555 2378', '(303) 494-2320', ], ); testPhonesFromText( ` 123-456-789 123 456 789 123.456.789 123.456.789.123 +123.456.789.123 `, ['123-456-789', '123 456 789', '123.456.789', '123.456.789.123', '+123.456.789.123'], ); testPhonesFromText( ` (000)000-0000 (000)000 0000 (000)000.0000 (000) 000-0000 (000) 000 0000 (000) 000.0000 `, [ '(000)000-0000', '(000)000 0000', '(000)000.0000', '(000) 000-0000', '(000) 000 0000', '(000) 000.0000', ], ); testPhonesFromText( ` 000-0000 000 0000 000.0000 0000000 0000000000 (000)0000000 `, ['000-0000', '000 0000', '000.0000', '0000000', '0000000000', '(000)0000000'], ); }); test('skips invalid phones', () => { testPhonesFromText( ` 2018-10-11 123 456789 345 1 2 3 4 5 6 7 8 `, [], ); }); }); describe('phonesFromUrls()', () => { test('throws on invalid arg', () => { expect(() => { // @ts-expect-error invalid input type phonesFromUrls(); }).toThrowError(/must be an array/); expect(() => { // @ts-expect-error invalid input type phonesFromUrls({}); }).toThrowError(/must be an array/); expect(() => { // @ts-expect-error invalid input type phonesFromUrls('fwefwef'); }).toThrowError(/must be an array/); expect(() => { // @ts-expect-error invalid input type phonesFromUrls(12345); }).toThrowError(/must be an array/); }); test('extracts phones correctly', () => { expect(phonesFromUrls([])).toEqual([]); // @ts-expect-error invalid input type expect(phonesFromUrls([1, 2, {}, 'fwef', null, undefined])).toEqual([]); expect( phonesFromUrls([ 'tel:12345678', 'tel:/22345678', // 'tel://32345678', 'PHONE:42345678', // 'phone:/52345678', 'phone://62345678', 'telephone:72345678', 'telephone:/82345678', 'telephone://92345678', 'callto:97345678', 'CALLTO:/+98345678', 'callto://9992345678', ]), ).toEqual([ '12345678', '22345678', '32345678', '42345678', '52345678', '62345678', '72345678', '82345678', '92345678', '97345678', '+98345678', '9992345678', ]); expect( phonesFromUrls([ 'https://www.example.com', 'ftp://www.example.com', '1234567', '+42055555567', 'tel://+42012345678', 'tel://+420.123.456', 'http://www.example.com', ]), ).toEqual(['+42012345678', '+420.123.456']); }); }); describe('parseHandlesFromHtml()', () => { const EMPTY_RESULT: social.SocialHandles = { emails: [], phones: [], phonesUncertain: [], linkedIns: [], twitters: [], instagrams: [], facebooks: [], youtubes: [], tiktoks: [], pinterests: [], discords: [], }; test('handles invalid arg', () => { // @ts-expect-error invalid input type expect(parseHandlesFromHtml()).toEqual(EMPTY_RESULT); // @ts-expect-error invalid input type expect(parseHandlesFromHtml(undefined)).toEqual(EMPTY_RESULT); // @ts-expect-error invalid input type expect(parseHandlesFromHtml(null)).toEqual(EMPTY_RESULT); // @ts-expect-error invalid input type expect(parseHandlesFromHtml({})).toEqual(EMPTY_RESULT); // @ts-expect-error invalid input type expect(parseHandlesFromHtml(1234)).toEqual(EMPTY_RESULT); }); test('works', () => { expect(parseHandlesFromHtml('')).toEqual(EMPTY_RESULT); expect(parseHandlesFromHtml(' ')).toEqual(EMPTY_RESULT); const html = 'use the data in this [YouTube Video](https://www.youtube.com/watch?v=BsidLZKdYWQ).\\n\\n## Sample result\\n' + 'use the data in this [YouTube Video](https://www.youtube.com/watch?v=BsidLZKd123).\\\\n\\\\n## Sample result\\\\n'; expect(parseHandlesFromHtml(html)).toMatchObject({ youtubes: [ 'https://www.youtube.com/watch?v=BsidLZKd123', 'https://www.youtube.com/watch?v=BsidLZKdYWQ', ], }); expect( parseHandlesFromHtml(`
bob@example.com
bob@example.com testing skipping duplicates xxx@blablacarl@example.com