learningcircuit--local-deep…/tests/text_processing/test_text_cleaner.py

"""Tests for text_cleaner module."""

from local_deep_research.text_processing.text_cleaner import remove_surrogates


class TestRemoveSurrogates:
    """Tests for remove_surrogates function."""

    def test_returns_none_for_none_input(self):
        """Should return None for None input."""
        result = remove_surrogates(None)
        assert result is None

    def test_returns_empty_string_for_empty_input(self):
        """Should return empty string for empty input."""
        result = remove_surrogates("")
        assert result == ""

    def test_returns_normal_text_unchanged(self):
        """Should return normal text unchanged."""
        text = "Hello, World! This is normal text."
        result = remove_surrogates(text)
        assert result == text

    def test_handles_unicode_text(self):
        """Should handle valid Unicode text."""
        text = "Hello, World! Test text"
        result = remove_surrogates(text)
        assert result == text

    def test_removes_surrogate_characters(self):
        """Should remove/replace surrogate characters."""
        # Surrogate characters are in range U+D800 to U+DFFF
        text_with_surrogate = "Hello\ud800World"
        result = remove_surrogates(text_with_surrogate)
        # Result should be encodable as UTF-8
        result.encode("utf-8")  # Should not raise
        # Surrogate should be replaced
        assert "\ud800" not in result

    def test_handles_mixed_valid_and_invalid(self):
        """Should handle text with both valid and invalid characters."""
        text = "Valid text\ud800more valid"
        result = remove_surrogates(text)
        # Should preserve valid parts
        assert "Valid text" in result
        assert "more valid" in result
        # Should be encodable
        result.encode("utf-8")

    def test_preserves_newlines(self):
        """Should preserve newlines and whitespace."""
        text = "Line 1\nLine 2\r\nLine 3\tTabbed"
        result = remove_surrogates(text)
        assert "\n" in result
        assert "\t" in result

    def test_handles_special_characters(self):
        """Should handle special characters correctly."""
        text = "<html>&nbsp;test'\"quote</html>"
        result = remove_surrogates(text)
        assert result == text

    def test_handles_long_text(self):
        """Should handle long text efficiently."""
        text = "A" * 10000
        result = remove_surrogates(text)
        assert len(result) == 10000

    def test_handles_consecutive_surrogates(self):
        """Should handle multiple consecutive surrogate characters."""
        text = "Start\ud800\ud801\ud802End"
        result = remove_surrogates(text)
        # Should be encodable
        result.encode("utf-8")
        assert "Start" in result
        assert "End" in result

    def test_exception_handler_fallback(self):
        """Should use ignore-based fallback when encode raises an exception.

        Uses a str subclass whose encode method raises on surrogatepass,
        forcing the except branch (lines 32-38) to execute.
        """

        class BadString(str):
            _call_count = 0

            def encode(self, encoding="utf-8", errors="strict"):
                BadString._call_count += 1
                if errors == "surrogatepass":
                    raise RuntimeError("Simulated encode failure")
                return super().encode(encoding, errors=errors)

        BadString._call_count = 0
        text = BadString("hello world")
        result = remove_surrogates(text)
        assert isinstance(result, str)
        assert "hello" in result
        # surrogatepass was attempted (triggering exception),
        # then ignore was used as fallback
        assert BadString._call_count >= 2