"""Tests for text_cleaner module.""" from local_deep_research.text_processing.text_cleaner import remove_surrogates class TestRemoveSurrogates: """Tests for remove_surrogates function.""" def test_returns_none_for_none_input(self): """Should return None for None input.""" result = remove_surrogates(None) assert result is None def test_returns_empty_string_for_empty_input(self): """Should return empty string for empty input.""" result = remove_surrogates("") assert result == "" def test_returns_normal_text_unchanged(self): """Should return normal text unchanged.""" text = "Hello, World! This is normal text." result = remove_surrogates(text) assert result == text def test_handles_unicode_text(self): """Should handle valid Unicode text.""" text = "Hello, World! Test text" result = remove_surrogates(text) assert result == text def test_removes_surrogate_characters(self): """Should remove/replace surrogate characters.""" # Surrogate characters are in range U+D800 to U+DFFF text_with_surrogate = "Hello\ud800World" result = remove_surrogates(text_with_surrogate) # Result should be encodable as UTF-8 result.encode("utf-8") # Should not raise # Surrogate should be replaced assert "\ud800" not in result def test_handles_mixed_valid_and_invalid(self): """Should handle text with both valid and invalid characters.""" text = "Valid text\ud800more valid" result = remove_surrogates(text) # Should preserve valid parts assert "Valid text" in result assert "more valid" in result # Should be encodable result.encode("utf-8") def test_preserves_newlines(self): """Should preserve newlines and whitespace.""" text = "Line 1\nLine 2\r\nLine 3\tTabbed" result = remove_surrogates(text) assert "\n" in result assert "\t" in result def test_handles_special_characters(self): """Should handle special characters correctly.""" text = " test'\"quote" result = remove_surrogates(text) assert result == text def test_handles_long_text(self): """Should handle long text efficiently.""" text = "A" * 10000 result = remove_surrogates(text) assert len(result) == 10000 def test_handles_consecutive_surrogates(self): """Should handle multiple consecutive surrogate characters.""" text = "Start\ud800\ud801\ud802End" result = remove_surrogates(text) # Should be encodable result.encode("utf-8") assert "Start" in result assert "End" in result def test_exception_handler_fallback(self): """Should use ignore-based fallback when encode raises an exception. Uses a str subclass whose encode method raises on surrogatepass, forcing the except branch (lines 32-38) to execute. """ class BadString(str): _call_count = 0 def encode(self, encoding="utf-8", errors="strict"): BadString._call_count += 1 if errors == "surrogatepass": raise RuntimeError("Simulated encode failure") return super().encode(encoding, errors=errors) BadString._call_count = 0 text = BadString("hello world") result = remove_surrogates(text) assert isinstance(result, str) assert "hello" in result # surrogatepass was attempted (triggering exception), # then ignore was used as fallback assert BadString._call_count >= 2