JavaScript powers interactive websites and runs in every modern browser environment.
", "Rust provides memory safety without garbage collection through its ownership model.
", "Go was designed at Google for building scalable network services and cloud infrastructure.
", "Ruby on Rails popularized convention over configuration in web application frameworks.
", "TypeScript adds static type checking to JavaScript for large-scale application development.
", "Java remains dominant in enterprise software and Android mobile application development.
", "C++ is essential for game engines, operating systems, and high-performance computing.
", ] FILLER_BLOCK = "\n".join(FILLER_PARAGRAPHS) def _make_test_html(target_copies: int = 1, extra_body: str = "") -> str: """Build HTML with `target_copies` of TARGET_TEXT + filler for BM25 variety.""" target_block = f"{TARGET_TEXT}
\n" * target_copies return _wrap_html(target_block + FILLER_BLOCK + extra_body) # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- class TestBM25Deduplication: """Core deduplication tests for issue #1213.""" def test_exact_duplicates_collapsed(self): """Identical paragraphs should appear only once in output.""" html = _make_test_html(target_copies=5) filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01) results = filt.filter_content(html) matching = [r for r in results if "versatile" in r] assert len(matching) == 1 def test_no_duplicates_unchanged(self): """Unique paragraphs should all be preserved (none removed by dedup).""" html = _make_test_html(target_copies=1) filt = BM25ContentFilter(user_query="programming languages", bm25_threshold=0.01) results = filt.filter_content(html) # At least some pass threshold; key point is no false dedup assert len(results) >= 1 # All results should be unique strings assert len(results) == len(set(results)) def test_mixed_unique_and_duplicate(self): """Duplicates removed, uniques kept.""" extra = f"{TARGET_TEXT}
{TARGET_TEXT}
" html = _make_test_html(target_copies=1, extra_body=extra) filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01) results = filt.filter_content(html) matching = [r for r in results if "versatile" in r] assert len(matching) == 1 def test_document_order_preserved(self): """First occurrence should be kept, not the last.""" first_text = "Python was created by Guido van Rossum in the early nineties as a successor to the ABC language." second_text = "Python supports multiple programming paradigms including procedural and functional styles." body = ( f"{first_text}
" f"{second_text}
" f"{first_text}
" # duplicate + FILLER_BLOCK ) html = _wrap_html(body) filt = BM25ContentFilter(user_query="Python history", bm25_threshold=0.01) results = filt.filter_content(html) guido_results = [r for r in results if "Guido" in r] assert len(guido_results) == 1 def test_empty_html(self): """Empty input should return empty list.""" filt = BM25ContentFilter() assert filt.filter_content("") == [] def test_none_html(self): """None input should return empty list.""" filt = BM25ContentFilter() assert filt.filter_content(None) == [] def test_no_body(self): """HTML without explicit body should still work.""" body = f"{TARGET_TEXT}
" * 3 + FILLER_BLOCK filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01) results = filt.filter_content(body) matching = [r for r in results if "versatile" in r] assert len(matching) == 1 def test_single_paragraph_with_filler(self): """Single target paragraph — nothing to deduplicate.""" html = _make_test_html(target_copies=1) filt = BM25ContentFilter(user_query="Python programming", bm25_threshold=0.01) results = filt.filter_content(html) matching = [r for r in results if "versatile" in r] assert len(matching) == 1 def test_high_threshold_filters_all(self): """Very high threshold should return nothing.""" html = _make_test_html(target_copies=3) filt = BM25ContentFilter(bm25_threshold=9999.0) results = filt.filter_content(html) assert results == [] def test_duplicate_with_different_tags(self): """Same text in different tag types should still be deduplicated.""" text = "Python enables rapid prototyping and development of complex software systems." extra = f"{text}