import type { CheerioRoot } from '@crawlee/utils';
import { htmlToText } from '@crawlee/utils';
import * as cheerio from 'cheerio';
import * as htmlToTextData from '../shared/data/html_to_text_test_data';
const checkHtmlToText = (html: string | CheerioRoot, expectedText: string, hasBody = false) => {
const text1 = htmlToText(html);
expect(text1).toEqual(expectedText);
// Test embedding into gives the same result
if (typeof html === 'string' && !hasBody) {
const html2 = `
Title should be ignored
${html}
`;
const text2 = htmlToText(html2);
expect(text2).toEqual(expectedText);
}
};
describe('htmlToText()', () => {
test('handles invalid args', () => {
// @ts-expect-error invalid input type
checkHtmlToText(null, '');
checkHtmlToText('', '');
// @ts-expect-error invalid input type
checkHtmlToText(0, '');
// @ts-expect-error invalid input type
checkHtmlToText(undefined, '');
});
test('handles basic HTML elements correctly', () => {
checkHtmlToText('Plain text node', 'Plain text node');
checkHtmlToText(' Plain text node ', 'Plain text node');
checkHtmlToText(' \nPlain text node \n ', 'Plain text node');
checkHtmlToText('Header 1
Header 2
', 'Header 1\nHeader 2');
checkHtmlToText('Header 1
Header 2
', 'Header 1\nHeader 2');
checkHtmlToText('Header 1
Header 2
', 'Header 1\nHeader 2');
checkHtmlToText('Header 1
Header 2
', 'Header 1\nHeader 2');
checkHtmlToText('Header 1
Header 2
', 'Header 1\n\nHeader 2');
checkHtmlToText('Header 1
Header 2
', 'Header 1\n\nHeader 2');
checkHtmlToText('Header 1
\n
\nHeader 2
', 'Header 1\n\nHeader 2');
checkHtmlToText('Header 1
\n
\n
Header 2
', 'Header 1\n\n\nHeader 2');
checkHtmlToText(
'Header 1
\n
\n
Header 2
',
'Header 1\n\n\n\nHeader 2',
);
checkHtmlToText('', 'Div\nParagraph');
checkHtmlToText('Div1
Div2
', 'Div1\nDiv2');
checkHtmlToText('Div1
', 'Div1');
checkHtmlToText('Div1
', 'Div1');
checkHtmlToText('Div1
', 'Div1');
checkHtmlToText('Div1
', 'Div1');
checkHtmlToText('Div1
', 'Div1');
checkHtmlToText('A B C D E\n\nF G', 'A B C D E F G');
checkHtmlToText('A B C D E\n\nF G
', 'A B C D E\n\nF G');
checkHtmlToText(
'Heading 1
Heading 2
',
'Heading 1\nDeep Div\nHeading 2',
);
checkHtmlToText('this_word_should_be_one', 'this_word_should_be_one');
checkHtmlToText('some text', 'some text');
checkHtmlToText(
`
| Cell A1 | Cell A2 |
Cell A3 |
| Cell B1 | Cell B2 |
`,
'Cell A1\tCell A2\tCell A3 \t\nCell B1\tCell B2',
);
});
test('handles HTML entities correctly', () => {
checkHtmlToText('á é', 'á é');
});
test('handles larger HTML documents', () => {
const { html, text } = htmlToTextData;
// Careful here - don't change any whitespace in the text below or the test will break, even trailing!
checkHtmlToText(html, text, true);
});
test('works with Cheerio object', () => {
const html1 = 'Some text';
checkHtmlToText(cheerio.load(html1, { decodeEntities: true }), 'Some text');
const html2 = 'Text outside of body
';
checkHtmlToText(cheerio.load(html2, { decodeEntities: true }), 'Text outside of body');
});
});