learningcircuit--local-deep-research/tests/text_processing/test_text_cleaner_behavior.py

"""
Behavioral tests for text_cleaner module.

Tests the remove_surrogates function for handling encoding issues.
"""


class TestRemoveSurrogatesBasic:
    """Tests for basic remove_surrogates functionality."""

    def test_returns_same_for_normal_text(self):
        """Returns same text for normal ASCII."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "Hello World"
        result = remove_surrogates(text)
        assert result == text

    def test_returns_same_for_utf8_text(self):
        """Returns same text for valid UTF-8."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "Hello 世界 🌍"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_empty_string(self):
        """Handles empty string."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        result = remove_surrogates("")
        assert result == ""

    def test_handles_none(self):
        """Handles None input."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        result = remove_surrogates(None)
        assert result is None

    def test_returns_string(self):
        """Returns a string type."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        result = remove_surrogates("test")
        assert isinstance(result, str)


class TestRemoveSurrogatesUnicode:
    """Tests for Unicode handling in remove_surrogates."""

    def test_handles_chinese_characters(self):
        """Handles Chinese characters."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "这是中文文本"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_japanese_characters(self):
        """Handles Japanese characters."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "これは日本語です"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_korean_characters(self):
        """Handles Korean characters."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "한국어 텍스트"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_arabic_characters(self):
        """Handles Arabic characters."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "مرحبا بالعالم"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_emoji(self):
        """Handles emoji characters."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "Hello 👋 World 🌍"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_mixed_scripts(self):
        """Handles mixed script text."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "English, 中文, 日本語, 한국어"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_special_unicode_symbols(self):
        """Handles special Unicode symbols."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "€ £ ¥ © ® ™ § ¶"
        result = remove_surrogates(text)
        assert result == text


class TestRemoveSurrogatesSurrogateHandling:
    """Tests for actual surrogate character handling."""

    def test_removes_lone_high_surrogate(self):
        """Removes lone high surrogate character."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        # \ud800 is a high surrogate
        text = "Hello\ud800World"
        result = remove_surrogates(text)
        # Should not contain the surrogate, and should be valid UTF-8
        assert result.encode("utf-8") is not None
        assert "\ud800" not in result

    def test_removes_lone_low_surrogate(self):
        """Removes lone low surrogate character."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        # \udc00 is a low surrogate
        text = "Hello\udc00World"
        result = remove_surrogates(text)
        # Should not contain the surrogate
        assert result.encode("utf-8") is not None
        assert "\udc00" not in result

    def test_removes_multiple_surrogates(self):
        """Removes multiple surrogate characters."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "\ud800text\ud801more\udc00text"
        result = remove_surrogates(text)
        assert result.encode("utf-8") is not None

    def test_preserves_text_around_surrogates(self):
        """Preserves text around surrogates."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "before\ud800after"
        result = remove_surrogates(text)
        assert "before" in result
        assert "after" in result


class TestRemoveSurrogatesEdgeCases:
    """Tests for edge cases in remove_surrogates."""

    def test_handles_very_long_text(self):
        """Handles very long text."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "a" * 100000
        result = remove_surrogates(text)
        assert len(result) == 100000

    def test_handles_newlines(self):
        """Handles text with newlines."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "Line 1\nLine 2\nLine 3"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_tabs(self):
        """Handles text with tabs."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "Col1\tCol2\tCol3"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_carriage_returns(self):
        """Handles text with carriage returns."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "Line 1\r\nLine 2"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_null_characters(self):
        """Handles text with null characters."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "before\x00after"
        result = remove_surrogates(text)
        # Null character should be preserved as it's valid UTF-8
        assert result == text

    def test_handles_whitespace_only(self):
        """Handles whitespace-only text."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "   \t\n  "
        result = remove_surrogates(text)
        assert result == text


class TestRemoveSurrogatesBoundaryCodepoints:
    """Tests for boundary Unicode codepoints."""

    def test_handles_bmp_boundary(self):
        """Handles characters at BMP boundary."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        # U+FFFF is last character in BMP
        text = "test\ufffftest"
        result = remove_surrogates(text)
        assert isinstance(result, str)
        # Should be encodable as UTF-8
        result.encode("utf-8")

    def test_handles_supplementary_planes(self):
        """Handles characters from supplementary planes."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        # 𐀀 is U+10000, first supplementary character
        text = "test\U00010000test"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_high_codepoints(self):
        """Handles high Unicode codepoints."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        # U+1F600 is 😀
        text = "test\U0001f600test"
        result = remove_surrogates(text)
        assert result == text


class TestRemoveSurrogatesRealWorldScenarios:
    """Tests for real-world scenarios that produce surrogates."""

    def test_handles_pdf_extraction_style_text(self):
        """Handles text that might come from PDF extraction."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        # PDF extraction sometimes produces ligatures and special chars
        text = "final flow ffect"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_mathematical_symbols(self):
        """Handles mathematical symbols."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "∑ ∏ ∫ √ ∞ ≈ ≠ ≤ ≥"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_greek_letters(self):
        """Handles Greek letters (common in scientific text)."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "α β γ δ ε ζ η θ"
        result = remove_surrogates(text)
        assert result == text

    def test_result_is_utf8_encodable(self):
        """Result is always UTF-8 encodable."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        # Mix of normal text and surrogates
        text = "normal\ud800text\udc00more"
        result = remove_surrogates(text)
        # Should not raise
        encoded = result.encode("utf-8")
        assert isinstance(encoded, bytes)


class TestRemoveSurrogatesConsistency:
    """Tests for consistency of remove_surrogates behavior."""

    def test_idempotent(self):
        """Applying twice gives same result."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "Hello\ud800World"
        result1 = remove_surrogates(text)
        result2 = remove_surrogates(result1)
        assert result1 == result2

    def test_preserves_length_for_clean_text(self):
        """Preserves length for text without surrogates."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "Hello World"
        result = remove_surrogates(text)
        assert len(result) == len(text)

    def test_deterministic(self):
        """Same input always produces same output."""
        from local_deep_research.text_processing.text_cleaner import (
            remove_surrogates,
        )

        text = "test\ud800text"
        result1 = remove_surrogates(text)
        result2 = remove_surrogates(text)
        assert result1 == result2