wehub-resource-sync
60 lines
1.9 KiB
Python
60 lines
1.9 KiB
Python
"""Unit tests for :func:`lightrag.utils.strip_control_characters`.
|
|
|
|
The helper removes C0 control / separator characters (notably the
|
|
``\\x1c``-``\\x1f`` FS/GS/RS/US separators) and Unicode surrogates while
|
|
preserving ``\\t``/``\\n``/``\\r`` and *not* touching markup or surrounding
|
|
whitespace. It is the cleaner injected at the parse-result persist points so
|
|
parser-extracted body text cannot carry these chars into chunks/storage.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from lightrag.utils import strip_control_characters
|
|
|
|
pytestmark = pytest.mark.offline
|
|
|
|
|
|
def test_removes_fs_gs_rs_us_separators():
|
|
raw = "a\x1cb\x1dc\x1ed\x1fe"
|
|
assert strip_control_characters(raw) == "abcde"
|
|
|
|
|
|
def test_removes_other_c0_and_del():
|
|
# NUL, BS, VT, FF, SO..US range members, and DEL.
|
|
raw = "x\x00\x08\x0b\x0c\x0e\x1f\x7fy"
|
|
assert strip_control_characters(raw) == "xy"
|
|
|
|
|
|
def test_preserves_tab_newline_carriage_return():
|
|
raw = "line1\tcol\nline2\r\nline3"
|
|
# The whitespace control chars survive verbatim.
|
|
assert strip_control_characters(raw) == raw
|
|
|
|
|
|
def test_noop_for_clean_text_returns_unchanged():
|
|
clean = " 普通文本 with <table id='t'>markup</table> <kept> "
|
|
# No html.unescape, no .strip(): markup, entities and surrounding
|
|
# whitespace are all preserved exactly.
|
|
assert strip_control_characters(clean) == clean
|
|
|
|
|
|
def test_empty_and_falsey_input():
|
|
assert strip_control_characters("") == ""
|
|
|
|
|
|
def test_removes_surrogates():
|
|
raw = "ok\ud800tail"
|
|
assert strip_control_characters(raw) == "oktail"
|
|
|
|
|
|
def test_replacement_char_can_substitute():
|
|
raw = "a\x1fb"
|
|
assert strip_control_characters(raw, replacement_char="_") == "a_b"
|
|
|
|
|
|
def test_cjk_separators_removed_without_inserting_spaces():
|
|
# Critical: deletion (not space-substitution) so CJK text is not split by
|
|
# spurious whitespace that would break the V chunker downstream.
|
|
raw = "中\x1f文\x1c内\x1d容"
|
|
assert strip_control_characters(raw) == "中文内容"
|