chore: import upstream snapshot with attribution

2026-07-13 13:08:55 +08:00
commit 7a0da7932b
2985 changed files with 1049377 additions and 0 deletions
@@ -0,0 +1 @@
+"""Tests for text_processing module."""
@@ -0,0 +1,102 @@
+"""Tests for text_cleaner module."""
+
+from local_deep_research.text_processing.text_cleaner import remove_surrogates
+
+
+class TestRemoveSurrogates:
+    """Tests for remove_surrogates function."""
+
+    def test_returns_none_for_none_input(self):
+        """Should return None for None input."""
+        result = remove_surrogates(None)
+        assert result is None
+
+    def test_returns_empty_string_for_empty_input(self):
+        """Should return empty string for empty input."""
+        result = remove_surrogates("")
+        assert result == ""
+
+    def test_returns_normal_text_unchanged(self):
+        """Should return normal text unchanged."""
+        text = "Hello, World! This is normal text."
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_unicode_text(self):
+        """Should handle valid Unicode text."""
+        text = "Hello, World! Test text"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_removes_surrogate_characters(self):
+        """Should remove/replace surrogate characters."""
+        # Surrogate characters are in range U+D800 to U+DFFF
+        text_with_surrogate = "Hello\ud800World"
+        result = remove_surrogates(text_with_surrogate)
+        # Result should be encodable as UTF-8
+        result.encode("utf-8")  # Should not raise
+        # Surrogate should be replaced
+        assert "\ud800" not in result
+
+    def test_handles_mixed_valid_and_invalid(self):
+        """Should handle text with both valid and invalid characters."""
+        text = "Valid text\ud800more valid"
+        result = remove_surrogates(text)
+        # Should preserve valid parts
+        assert "Valid text" in result
+        assert "more valid" in result
+        # Should be encodable
+        result.encode("utf-8")
+
+    def test_preserves_newlines(self):
+        """Should preserve newlines and whitespace."""
+        text = "Line 1\nLine 2\r\nLine 3\tTabbed"
+        result = remove_surrogates(text)
+        assert "\n" in result
+        assert "\t" in result
+
+    def test_handles_special_characters(self):
+        """Should handle special characters correctly."""
+        text = "<html>&nbsp;test'\"quote</html>"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_long_text(self):
+        """Should handle long text efficiently."""
+        text = "A" * 10000
+        result = remove_surrogates(text)
+        assert len(result) == 10000
+
+    def test_handles_consecutive_surrogates(self):
+        """Should handle multiple consecutive surrogate characters."""
+        text = "Start\ud800\ud801\ud802End"
+        result = remove_surrogates(text)
+        # Should be encodable
+        result.encode("utf-8")
+        assert "Start" in result
+        assert "End" in result
+
+    def test_exception_handler_fallback(self):
+        """Should use ignore-based fallback when encode raises an exception.
+
+        Uses a str subclass whose encode method raises on surrogatepass,
+        forcing the except branch (lines 32-38) to execute.
+        """
+
+        class BadString(str):
+            _call_count = 0
+
+            def encode(self, encoding="utf-8", errors="strict"):
+                BadString._call_count += 1
+                if errors == "surrogatepass":
+                    raise RuntimeError("Simulated encode failure")
+                return super().encode(encoding, errors=errors)
+
+        BadString._call_count = 0
+        text = BadString("hello world")
+        result = remove_surrogates(text)
+        assert isinstance(result, str)
+        assert "hello" in result
+        # surrogatepass was attempted (triggering exception),
+        # then ignore was used as fallback
+        assert BadString._call_count >= 2
@@ -0,0 +1,369 @@
+"""
+Behavioral tests for text_cleaner module.
+
+Tests the remove_surrogates function for handling encoding issues.
+"""
+
+
+class TestRemoveSurrogatesBasic:
+    """Tests for basic remove_surrogates functionality."""
+
+    def test_returns_same_for_normal_text(self):
+        """Returns same text for normal ASCII."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello World"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_returns_same_for_utf8_text(self):
+        """Returns same text for valid UTF-8."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello 世界 🌍"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_empty_string(self):
+        """Handles empty string."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        result = remove_surrogates("")
+        assert result == ""
+
+    def test_handles_none(self):
+        """Handles None input."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        result = remove_surrogates(None)
+        assert result is None
+
+    def test_returns_string(self):
+        """Returns a string type."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        result = remove_surrogates("test")
+        assert isinstance(result, str)
+
+
+class TestRemoveSurrogatesUnicode:
+    """Tests for Unicode handling in remove_surrogates."""
+
+    def test_handles_chinese_characters(self):
+        """Handles Chinese characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "这是中文文本"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_japanese_characters(self):
+        """Handles Japanese characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "これは日本語です"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_korean_characters(self):
+        """Handles Korean characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "한국어 텍스트"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_arabic_characters(self):
+        """Handles Arabic characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "مرحبا بالعالم"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_emoji(self):
+        """Handles emoji characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello 👋 World 🌍"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_mixed_scripts(self):
+        """Handles mixed script text."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "English, 中文, 日本語, 한국어"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_special_unicode_symbols(self):
+        """Handles special Unicode symbols."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "€ £ ¥ © ® ™ § ¶"
+        result = remove_surrogates(text)
+        assert result == text
+
+
+class TestRemoveSurrogatesSurrogateHandling:
+    """Tests for actual surrogate character handling."""
+
+    def test_removes_lone_high_surrogate(self):
+        """Removes lone high surrogate character."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # \ud800 is a high surrogate
+        text = "Hello\ud800World"
+        result = remove_surrogates(text)
+        # Should not contain the surrogate, and should be valid UTF-8
+        assert result.encode("utf-8") is not None
+        assert "\ud800" not in result
+
+    def test_removes_lone_low_surrogate(self):
+        """Removes lone low surrogate character."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # \udc00 is a low surrogate
+        text = "Hello\udc00World"
+        result = remove_surrogates(text)
+        # Should not contain the surrogate
+        assert result.encode("utf-8") is not None
+        assert "\udc00" not in result
+
+    def test_removes_multiple_surrogates(self):
+        """Removes multiple surrogate characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "\ud800text\ud801more\udc00text"
+        result = remove_surrogates(text)
+        assert result.encode("utf-8") is not None
+
+    def test_preserves_text_around_surrogates(self):
+        """Preserves text around surrogates."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "before\ud800after"
+        result = remove_surrogates(text)
+        assert "before" in result
+        assert "after" in result
+
+
+class TestRemoveSurrogatesEdgeCases:
+    """Tests for edge cases in remove_surrogates."""
+
+    def test_handles_very_long_text(self):
+        """Handles very long text."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "a" * 100000
+        result = remove_surrogates(text)
+        assert len(result) == 100000
+
+    def test_handles_newlines(self):
+        """Handles text with newlines."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Line 1\nLine 2\nLine 3"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_tabs(self):
+        """Handles text with tabs."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Col1\tCol2\tCol3"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_carriage_returns(self):
+        """Handles text with carriage returns."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Line 1\r\nLine 2"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_null_characters(self):
+        """Handles text with null characters."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "before\x00after"
+        result = remove_surrogates(text)
+        # Null character should be preserved as it's valid UTF-8
+        assert result == text
+
+    def test_handles_whitespace_only(self):
+        """Handles whitespace-only text."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "   \t\n  "
+        result = remove_surrogates(text)
+        assert result == text
+
+
+class TestRemoveSurrogatesBoundaryCodepoints:
+    """Tests for boundary Unicode codepoints."""
+
+    def test_handles_bmp_boundary(self):
+        """Handles characters at BMP boundary."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # U+FFFF is last character in BMP
+        text = "test\ufffftest"
+        result = remove_surrogates(text)
+        assert isinstance(result, str)
+        # Should be encodable as UTF-8
+        result.encode("utf-8")
+
+    def test_handles_supplementary_planes(self):
+        """Handles characters from supplementary planes."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # 𐀀 is U+10000, first supplementary character
+        text = "test\U00010000test"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_high_codepoints(self):
+        """Handles high Unicode codepoints."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # U+1F600 is 😀
+        text = "test\U0001f600test"
+        result = remove_surrogates(text)
+        assert result == text
+
+
+class TestRemoveSurrogatesRealWorldScenarios:
+    """Tests for real-world scenarios that produce surrogates."""
+
+    def test_handles_pdf_extraction_style_text(self):
+        """Handles text that might come from PDF extraction."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # PDF extraction sometimes produces ligatures and special chars
+        text = "ﬁnal ﬂow ﬀect"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_mathematical_symbols(self):
+        """Handles mathematical symbols."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "∑ ∏ ∫ √ ∞ ≈ ≠ ≤ ≥"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_handles_greek_letters(self):
+        """Handles Greek letters (common in scientific text)."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "α β γ δ ε ζ η θ"
+        result = remove_surrogates(text)
+        assert result == text
+
+    def test_result_is_utf8_encodable(self):
+        """Result is always UTF-8 encodable."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        # Mix of normal text and surrogates
+        text = "normal\ud800text\udc00more"
+        result = remove_surrogates(text)
+        # Should not raise
+        encoded = result.encode("utf-8")
+        assert isinstance(encoded, bytes)
+
+
+class TestRemoveSurrogatesConsistency:
+    """Tests for consistency of remove_surrogates behavior."""
+
+    def test_idempotent(self):
+        """Applying twice gives same result."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello\ud800World"
+        result1 = remove_surrogates(text)
+        result2 = remove_surrogates(result1)
+        assert result1 == result2
+
+    def test_preserves_length_for_clean_text(self):
+        """Preserves length for text without surrogates."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "Hello World"
+        result = remove_surrogates(text)
+        assert len(result) == len(text)
+
+    def test_deterministic(self):
+        """Same input always produces same output."""
+        from local_deep_research.text_processing.text_cleaner import (
+            remove_surrogates,
+        )
+
+        text = "test\ud800text"
+        result1 = remove_surrogates(text)
+        result2 = remove_surrogates(text)
+        assert result1 == result2
@@ -0,0 +1,124 @@
+"""Extended tests for text_cleaner.remove_surrogates.
+
+Adds edge cases for surrogate pair handling, emoji preservation, and
+encoding scenarios not covered by the existing test_text_cleaner.py.
+"""
+
+from local_deep_research.text_processing.text_cleaner import remove_surrogates
+
+
+class TestRemoveSurrogatesEmojiAndSymbols:
+    """Emoji and special symbol preservation."""
+
+    def test_common_emoji_preserved(self):
+        text = "Hello 🌍 World 🚀"
+        assert remove_surrogates(text) == text
+
+    def test_math_symbols_preserved(self):
+        text = "∑ ∫ ∏ √ ∞ ≈"
+        assert remove_surrogates(text) == text
+
+    def test_currency_symbols_preserved(self):
+        text = "Price: €100 ¥200 £50"
+        assert remove_surrogates(text) == text
+
+    def test_box_drawing_preserved(self):
+        text = "┌──┐\n│  │\n└──┘"
+        assert remove_surrogates(text) == text
+
+
+class TestRemoveSurrogatesMixedScripts:
+    """Mixed script handling."""
+
+    def test_cjk_characters_preserved(self):
+        text = "日本語テスト Chinese中文 Korean한국어"
+        assert remove_surrogates(text) == text
+
+    def test_arabic_preserved(self):
+        text = "مرحبا بالعالم"
+        assert remove_surrogates(text) == text
+
+    def test_cyrillic_preserved(self):
+        text = "Привет мир"
+        assert remove_surrogates(text) == text
+
+    def test_devanagari_preserved(self):
+        text = "नमस्ते दुनिया"
+        assert remove_surrogates(text) == text
+
+
+class TestRemoveSurrogatesSurrogatePairs:
+    """Surrogate pair specific tests."""
+
+    def test_lone_high_surrogate_handled(self):
+        text = "before\ud800after"
+        result = remove_surrogates(text)
+        assert "before" in result
+        assert "after" in result
+        result.encode("utf-8")  # Must not raise
+
+    def test_lone_low_surrogate_handled(self):
+        text = "text\udfffend"
+        result = remove_surrogates(text)
+        assert "text" in result
+        assert "end" in result
+        result.encode("utf-8")
+
+    def test_consecutive_high_surrogates(self):
+        text = "\ud800\ud801\ud802"
+        result = remove_surrogates(text)
+        result.encode("utf-8")
+
+    def test_surrogate_at_start(self):
+        text = "\ud800hello"
+        result = remove_surrogates(text)
+        assert "hello" in result
+        result.encode("utf-8")
+
+    def test_surrogate_at_end(self):
+        text = "hello\ud800"
+        result = remove_surrogates(text)
+        assert "hello" in result
+        result.encode("utf-8")
+
+    def test_only_surrogates(self):
+        text = "\ud800\udc00\ud801"
+        result = remove_surrogates(text)
+        result.encode("utf-8")  # Must not raise
+
+
+class TestRemoveSurrogatesLargeInput:
+    """Performance and large input tests."""
+
+    def test_large_clean_text(self):
+        text = "x" * 100000
+        result = remove_surrogates(text)
+        assert len(result) == 100000
+
+    def test_large_text_with_scattered_surrogates(self):
+        parts = []
+        for i in range(100):
+            parts.append("normal text " * 10)
+            parts.append("\ud800")
+        text = "".join(parts)
+        result = remove_surrogates(text)
+        result.encode("utf-8")
+        assert "normal text" in result
+
+
+class TestRemoveSurrogatesSpecialCases:
+    """Special and boundary cases."""
+
+    def test_null_byte_handled(self):
+        text = "before\x00after"
+        result = remove_surrogates(text)
+        assert "before" in result
+        assert "after" in result
+
+    def test_replacement_char_preserved(self):
+        text = "test\ufffdtext"
+        assert remove_surrogates(text) == text
+
+    def test_bom_preserved(self):
+        text = "\ufeffHello"
+        assert remove_surrogates(text) == text