learningcircuit--local-deep-research/tests/text_processing/test_text_cleaner_extended.py

"""Extended tests for text_cleaner.remove_surrogates.

Adds edge cases for surrogate pair handling, emoji preservation, and
encoding scenarios not covered by the existing test_text_cleaner.py.
"""

from local_deep_research.text_processing.text_cleaner import remove_surrogates


class TestRemoveSurrogatesEmojiAndSymbols:
    """Emoji and special symbol preservation."""

    def test_common_emoji_preserved(self):
        text = "Hello 🌍 World 🚀"
        assert remove_surrogates(text) == text

    def test_math_symbols_preserved(self):
        text = "∑ ∫ ∏ √ ∞ ≈"
        assert remove_surrogates(text) == text

    def test_currency_symbols_preserved(self):
        text = "Price: €100 ¥200 £50"
        assert remove_surrogates(text) == text

    def test_box_drawing_preserved(self):
        text = "┌──┐\n│  │\n└──┘"
        assert remove_surrogates(text) == text


class TestRemoveSurrogatesMixedScripts:
    """Mixed script handling."""

    def test_cjk_characters_preserved(self):
        text = "日本語テスト Chinese中文 Korean한국어"
        assert remove_surrogates(text) == text

    def test_arabic_preserved(self):
        text = "مرحبا بالعالم"
        assert remove_surrogates(text) == text

    def test_cyrillic_preserved(self):
        text = "Привет мир"
        assert remove_surrogates(text) == text

    def test_devanagari_preserved(self):
        text = "नमस्ते दुनिया"
        assert remove_surrogates(text) == text


class TestRemoveSurrogatesSurrogatePairs:
    """Surrogate pair specific tests."""

    def test_lone_high_surrogate_handled(self):
        text = "before\ud800after"
        result = remove_surrogates(text)
        assert "before" in result
        assert "after" in result
        result.encode("utf-8")  # Must not raise

    def test_lone_low_surrogate_handled(self):
        text = "text\udfffend"
        result = remove_surrogates(text)
        assert "text" in result
        assert "end" in result
        result.encode("utf-8")

    def test_consecutive_high_surrogates(self):
        text = "\ud800\ud801\ud802"
        result = remove_surrogates(text)
        result.encode("utf-8")

    def test_surrogate_at_start(self):
        text = "\ud800hello"
        result = remove_surrogates(text)
        assert "hello" in result
        result.encode("utf-8")

    def test_surrogate_at_end(self):
        text = "hello\ud800"
        result = remove_surrogates(text)
        assert "hello" in result
        result.encode("utf-8")

    def test_only_surrogates(self):
        text = "\ud800\udc00\ud801"
        result = remove_surrogates(text)
        result.encode("utf-8")  # Must not raise


class TestRemoveSurrogatesLargeInput:
    """Performance and large input tests."""

    def test_large_clean_text(self):
        text = "x" * 100000
        result = remove_surrogates(text)
        assert len(result) == 100000

    def test_large_text_with_scattered_surrogates(self):
        parts = []
        for i in range(100):
            parts.append("normal text " * 10)
            parts.append("\ud800")
        text = "".join(parts)
        result = remove_surrogates(text)
        result.encode("utf-8")
        assert "normal text" in result


class TestRemoveSurrogatesSpecialCases:
    """Special and boundary cases."""

    def test_null_byte_handled(self):
        text = "before\x00after"
        result = remove_surrogates(text)
        assert "before" in result
        assert "after" in result

    def test_replacement_char_preserved(self):
        text = "test\ufffdtext"
        assert remove_surrogates(text) == text

    def test_bom_preserved(self):
        text = "\ufeffHello"
        assert remove_surrogates(text) == text