import re # rule based segmentation based on https://stackoverflow.com/a/31505798, works surprisingly well def split_sentences( text: str, min_sentence_len: int = 20, retain_format: bool = False ) -> list[tuple[str, int, int]]: """ the text may not contain substrings "" or "" """ alphabets = r"([A-Za-z])" prefixes = r"(Mr|St|Mrs|Ms|Dr)[.]" suffixes = r"(Inc|Ltd|Jr|Sr|Co)" starters = r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" # noqa: E501 acronyms = r"([A-Z][.][A-Z][.](?:[A-Z][.])?)" websites = r"[.](com|net|org|io|gov|edu|me)" digits = r"([0-9])" multiple_dots = r"\.{2,}" # fmt: off if retain_format: text = text.replace("\n","") else: text = text.replace("\n"," ") text = re.sub(prefixes,"\\1", text) text = re.sub(websites,"\\1", text) text = re.sub(digits + "[.]" + digits,"\\1\\2",text) # text = re.sub(multiple_dots, lambda match: "" * len(match.group(0)) + "", text) # TODO(theomonnom): need improvement for ""..." dots", check capital + next sentence should not be # noqa: E501 # small text = re.sub(multiple_dots, lambda match: "" * len(match.group(0)), text) if "Ph.D" in text: text = text.replace("Ph.D.","PhD") text = re.sub(r"\s" + alphabets + "[.] "," \\1 ",text) text = re.sub(acronyms+" "+starters,"\\1 \\2",text) text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]","\\1\\2\\3",text) # noqa: E501 text = re.sub(alphabets + "[.]" + alphabets + "[.]","\\1\\2",text) text = re.sub(r" "+suffixes+"[.] "+starters," \\1 \\2",text) text = re.sub(r" "+suffixes+"[.]"," \\1",text) text = re.sub(r" " + alphabets + "[.]"," \\1",text) # mark end of sentence punctuations with text = re.sub(r"([.!?。!?])([\"”])", "\\1\\2", text) text = re.sub(r"([.!?。!?])(?![\"”])", "\\1", text) text = text.replace("",".") # fmt: on if retain_format: text = text.replace("", "\n") splitted_sentences = text.split("") text = text.replace("", "") sentences: list[tuple[str, int, int]] = [] buff = "" start_pos = 0 end_pos = 0 pre_pad = "" if retain_format else " " for match in splitted_sentences: if retain_format: sentence = match else: sentence = match.strip() if not sentence: continue buff += pre_pad + sentence end_pos += len(match) if len(buff) > min_sentence_len: sentences.append((buff[len(pre_pad) :], start_pos, end_pos)) start_pos = end_pos buff = "" if buff: sentences.append((buff[len(pre_pad) :], start_pos, len(text) - 1)) return sentences