import { describe, expect, it } from 'vitest'; import { handleContainsHtml, handleIsHtml } from '../../src/assertions/html'; import { createMockProvider, createProviderResponse } from '../factories/provider'; import type { AssertionParams, AtomicTestCase } from '../../src/types/index'; const mockProvider = createMockProvider({ id: 'mock', response: createProviderResponse({ output: 'mock' }), }); const defaultParams = { baseType: 'contains-html' as const, assertionValueContext: { vars: {}, test: {} as AtomicTestCase, prompt: 'test prompt', logProbs: undefined, provider: mockProvider, providerResponse: { output: 'test' }, }, output: 'test', providerResponse: { output: 'test' }, test: {} as AtomicTestCase, }; describe('handleContainsHtml', () => { it('should pass when output contains HTML tags', () => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'contains-html' }, outputString: '
Hello World
', inverse: false, }; const result = handleContainsHtml(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Assertion passed', assertion: params.assertion, }); }); it('should fail when output does not contain HTML tags', () => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'contains-html' }, outputString: 'Just plain text without any HTML', inverse: false, }; const result = handleContainsHtml(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Expected output to contain HTML content', assertion: params.assertion, }); }); it('should detect various HTML tags', () => { const htmlExamples = [ '

paragraph

', 'text', '', '

heading

', '
nested
', 'TestContent', 'Some text with emphasis in it', '', '
', '
cell
', '
Text with & entity
', '

Multiple   HTML < entities >

', 'Web component', '
Modern HTML
', ]; htmlExamples.forEach((html) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'contains-html' }, outputString: html, inverse: false, }; const result = handleContainsHtml(params); expect(result.pass).toBe(true); expect(result.score).toBe(1); }); }); it('should not detect false positives', () => { const nonHtmlExamples = [ 'Just plain text', '2 < 3 and 4 > 1', 'email@example.com', 'Math: a < b > c', '< div >', // space after < means it's not a valid HTML tag 'Generic text', // Single unpaired tag without HTML indicators 'if (ad; }', // Code with comparison operators 'The price is <$50', 'Email me at ', ]; nonHtmlExamples.forEach((text) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'contains-html' }, outputString: text, inverse: false, }; const result = handleContainsHtml(params); expect(result.pass).toBe(false); expect(result.score).toBe(0); }); }); it('should handle inverse assertion correctly', () => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'not-contains-html' }, outputString: 'Just plain text', inverse: true, }; const result = handleContainsHtml(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Assertion passed', assertion: params.assertion, }); }); it('should handle inverse assertion when HTML is present', () => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'not-contains-html' }, outputString: '
HTML content
', inverse: true, }; const result = handleContainsHtml(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Expected output to not contain HTML content', assertion: params.assertion, }); }); it('should detect HTML even with attributes and complex structures', () => { const complexHtml = `

Title

Paragraph with link

description
`; const params: AssertionParams = { ...defaultParams, assertion: { type: 'contains-html' }, outputString: complexHtml, inverse: false, }; const result = handleContainsHtml(params); expect(result.pass).toBe(true); expect(result.score).toBe(1); }); it('should handle edge cases with minimal HTML indicators', () => { // These should pass - have multiple HTML indicators const minimalHtmlPasses = [ '
 
', // tag + entity 'text', // comment + tag '
', // multiple self-closing tags 'Text with bold and italic', // multiple paired tags ]; minimalHtmlPasses.forEach((html) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'contains-html' }, outputString: html, inverse: false, }; const result = handleContainsHtml(params); expect(result.pass).toBe(true); }); // These should fail - only one HTML indicator const minimalHtmlFails = [ '', // single unpaired non-standard tag '&', // just an entity '', // just a comment ]; minimalHtmlFails.forEach((text) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'contains-html' }, outputString: text, inverse: false, }; const result = handleContainsHtml(params); expect(result.pass).toBe(false); }); }); }); describe('handleIsHtml', () => { it('should pass when output is valid HTML', () => { const validHtmlExamples = [ '
Hello World
', 'TestContent', '

Simple paragraph

', '
Nested
', '', '
', '
Content
', '
With whitespace
', ]; validHtmlExamples.forEach((html) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: html, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(true); expect(result.score).toBe(1); }); }); it('should fail when output is not valid HTML', () => { const invalidHtmlExamples = [ { output: 'Just plain text', reason: 'Output must be wrapped in HTML tags' }, { output: 'Some text with HTML inside', reason: 'Output must be wrapped in HTML tags', }, { output: 'Text before
HTML
', reason: 'Output must be wrapped in HTML tags', }, { output: '
HTML
Text after', reason: 'Output must be wrapped in HTML tags' }, { output: 'XML', reason: 'Output appears to be XML, not HTML', }, { output: '', reason: 'Output is empty' }, { output: ' ', reason: 'Output is empty' }, { output: '', reason: 'Output does not contain recognized HTML elements' }, { output: '2 < 3 and 4 > 1', reason: 'Output must be wrapped in HTML tags' }, ]; invalidHtmlExamples.forEach(({ output, reason }) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: output, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(false); expect(result.score).toBe(0); expect(result.reason).toBe(reason); }); }); it('should handle inverse assertion correctly', () => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'not-is-html' }, outputString: 'Just plain text', inverse: true, }; const result = handleIsHtml(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Assertion passed', assertion: params.assertion, }); }); it('should handle inverse assertion when HTML is present', () => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'not-is-html' }, outputString: '
Valid HTML
', inverse: true, }; const result = handleIsHtml(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Output is valid HTML', assertion: params.assertion, }); }); it('should accept HTML fragments', () => { const fragments = [ '
Fragment without doctype
', '

Title

Paragraph

', '
  • Item 1
  • Item 2
', '
', ]; fragments.forEach((html) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: html, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(true); }); }); it('should reject mixed content', () => { const mixedContent = [ 'Here is some HTML:
test
', '
HTML
and some text', 'Text
more text', ]; mixedContent.forEach((content) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: content, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(false); }); }); it('should accept bare elements that parse5 hoists into head', () => { // parse5 places top-level ', '', 'Page', ]; headOnlyExamples.forEach((html) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: html, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(true); }); }); it('should accept SVG and template wrappers', () => { const fragments = ['', '']; fragments.forEach((html) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: html, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(true); }); }); it('should accept custom elements', () => { const fragments = [ 'Web component', 'Uppercase custom element', ]; fragments.forEach((html) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: html, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(true); }); }); it('should reject doctype-only input', () => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: '', inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(false); expect(result.reason).toBe('Output does not contain recognized HTML elements'); }); it('should reject table foster-parented text', () => { // parse5 hoists stray text inside out to per HTML5 spec, // so the wrapper check must still catch it. const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: '
stray text
x
', inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(false); expect(result.reason).toBe('Output must be wrapped in HTML tags'); }); it('should reject unknown tags that merely share a prefix with html/head/body', () => { // Regression: a substring check like input.includes('`, causing the auto-injected to be treated as // user-provided and the assertion to incorrectly pass. const prefixCollisions = [ { input: 'plain text', tag: 'headphones' }, { input: 'x', tag: 'bodyguard' }, { input: 'x', tag: 'htmlworld' }, { input: 'x', tag: 'headd' }, ]; prefixCollisions.forEach(({ input }) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: input, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(false); expect(result.reason).toBe('Output does not contain recognized HTML elements'); }); }); it('should still accept body variants regardless of trailing chars', () => { // Control cases for the wrapper detection fix — body followed by `>`, // whitespace, or `/` must continue to count as user-declared body. const bodyVariants = ['x', 'x', '']; bodyVariants.forEach((input) => { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: input, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(true); }); }); it('should reject incomplete or ignored wrapper tags', () => { const invalidWrappers = [ { input: '', reason: 'Output must be wrapped in HTML tags' }, { input: 'plain { const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: input, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(false); expect(result.reason).toBe(reason); }); }); it('should handle pathologically deep nesting without stack overflow', () => { // Nesting 15k unknown elements forces the iterative walker to actually // traverse the full depth: unknown tags fail `isUserProvidedElement`, so // `findFirstElement` cannot short-circuit until it reaches the
at // the leaf. A recursive walker blows V8's ~10-15k stack-frame limit here. const depth = 15_000; const deeplyNested = `${''.repeat(depth)}
leaf
${''.repeat(depth)}`; const params: AssertionParams = { ...defaultParams, assertion: { type: 'is-html' }, outputString: deeplyNested, inverse: false, }; const result = handleIsHtml(params); expect(result.pass).toBe(true); }); });