chore: import upstream snapshot with attribution

tests/text_processing/test_text_cleaner_behavior.py

@@ -0,0 +1,369 @@
+"""
+Behavioral tests for text_cleaner module.
+
+Tests the remove_surrogates function for handling encoding issues.
+"""
+
+
+class TestRemoveSurrogatesBasic:
+    """Tests for basic remove_surrogates functionality."""
+
+    def test_returns_same_for_normal_text(self):
+        """Returns same text for normal ASCII."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello World"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_returns_same_for_utf8_text(self):
+        """Returns same text for valid UTF-8."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello 世界 🌍"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_empty_string(self):
+        """Handles empty string."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        result = remove_surrogates("")
+        assert result == ""
+
+    def test_handles_none(self):
+        """Handles None input."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        result = remove_surrogates(None)
+        assert result is None
+
+    def test_returns_string(self):
+        """Returns a string type."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        result = remove_surrogates("test")
+        assert isinstance(result, str)
+
+
+class TestRemoveSurrogatesUnicode:
+    """Tests for Unicode handling in remove_surrogates."""
+
+    def test_handles_chinese_characters(self):
+        """Handles Chinese characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "这是中文文本"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_japanese_characters(self):
+        """Handles Japanese characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "これは日本語です"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_korean_characters(self):
+        """Handles Korean characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "한국어 텍스트"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_arabic_characters(self):
+        """Handles Arabic characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "مرحبا بالعالم"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_emoji(self):
+        """Handles emoji characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello 👋 World 🌍"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_mixed_scripts(self):
+        """Handles mixed script text."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "English, 中文, 日本語, 한국어"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_special_unicode_symbols(self):
+        """Handles special Unicode symbols."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "€ £ ¥ © ® ™ § ¶"
+        result = remove_surrogates(text)
+        assert result == text
+
+
+class TestRemoveSurrogatesSurrogateHandling:
+    """Tests for actual surrogate character handling."""
+
+    def test_removes_lone_high_surrogate(self):
+        """Removes lone high surrogate character."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # \ud800 is a high surrogate
+        text = "Hello\ud800World"
+        result = remove_surrogates(text)
+        # Should not contain the surrogate, and should be valid UTF-8
+        assert result.encode("utf-8") is not None
+        assert "\ud800" not in result
+
+    def test_removes_lone_low_surrogate(self):
+        """Removes lone low surrogate character."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # \udc00 is a low surrogate
+        text = "Hello\udc00World"
+        result = remove_surrogates(text)
+        # Should not contain the surrogate
+        assert result.encode("utf-8") is not None
+        assert "\udc00" not in result
+
+    def test_removes_multiple_surrogates(self):
+        """Removes multiple surrogate characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "\ud800text\ud801more\udc00text"
+        result = remove_surrogates(text)
+        assert result.encode("utf-8") is not None
+
+    def test_preserves_text_around_surrogates(self):
+        """Preserves text around surrogates."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "before\ud800after"
+        result = remove_surrogates(text)
+        assert "before" in result
+        assert "after" in result
+
+
+class TestRemoveSurrogatesEdgeCases:
+    """Tests for edge cases in remove_surrogates."""
+
+    def test_handles_very_long_text(self):
+        """Handles very long text."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "a" * 100000
+        result = remove_surrogates(text)
+        assert len(result) == 100000
+
+    def test_handles_newlines(self):
+        """Handles text with newlines."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Line 1\nLine 2\nLine 3"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_tabs(self):
+        """Handles text with tabs."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Col1\tCol2\tCol3"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_carriage_returns(self):
+        """Handles text with carriage returns."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Line 1\r\nLine 2"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_null_characters(self):
+        """Handles text with null characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "before\x00after"
+        result = remove_surrogates(text)
+        # Null character should be preserved as it's valid UTF-8
+        assert result == text
+
+    def test_handles_whitespace_only(self):
+        """Handles whitespace-only text."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "   \t\n  "
+        result = remove_surrogates(text)
+        assert result == text
+
+
+class TestRemoveSurrogatesBoundaryCodepoints:
+    """Tests for boundary Unicode codepoints."""
+
+    def test_handles_bmp_boundary(self):
+        """Handles characters at BMP boundary."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # U+FFFF is last character in BMP
+        text = "test\ufffftest"
+        result = remove_surrogates(text)
+        assert isinstance(result, str)
+        # Should be encodable as UTF-8
+        result.encode("utf-8")
+
+    def test_handles_supplementary_planes(self):
+        """Handles characters from supplementary planes."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # 𐀀 is U+10000, first supplementary character
+        text = "test\U00010000test"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_high_codepoints(self):
+        """Handles high Unicode codepoints."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # U+1F600 is 😀
+        text = "test\U0001f600test"
+        result = remove_surrogates(text)
+        assert result == text
+
+
+class TestRemoveSurrogatesRealWorldScenarios:
+    """Tests for real-world scenarios that produce surrogates."""
+
+    def test_handles_pdf_extraction_style_text(self):
+        """Handles text that might come from PDF extraction."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # PDF extraction sometimes produces ligatures and special chars
+        text = "final flow ffect"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_mathematical_symbols(self):
+        """Handles mathematical symbols."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "∑ ∏ ∫ √ ∞ ≈ ≠ ≤ ≥"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_greek_letters(self):
+        """Handles Greek letters (common in scientific text)."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "α β γ δ ε ζ η θ"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_result_is_utf8_encodable(self):
+        """Result is always UTF-8 encodable."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # Mix of normal text and surrogates
+        text = "normal\ud800text\udc00more"
+        result = remove_surrogates(text)
+        # Should not raise
+        encoded = result.encode("utf-8")
+        assert isinstance(encoded, bytes)
+
+
+class TestRemoveSurrogatesConsistency:
+    """Tests for consistency of remove_surrogates behavior."""
+
+    def test_idempotent(self):
+        """Applying twice gives same result."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello\ud800World"
+        result1 = remove_surrogates(text)
+        result2 = remove_surrogates(result1)
+        assert result1 == result2
+
+    def test_preserves_length_for_clean_text(self):
+        """Preserves length for text without surrogates."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello World"
+        result = remove_surrogates(text)
+        assert len(result) == len(text)
+
+    def test_deterministic(self):
+        """Same input always produces same output."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "test\ud800text"
+        result1 = remove_surrogates(text)
+        result2 = remove_surrogates(text)
+        assert result1 == result2