s > 9 correctly.
parent_list = None
for list in self.list:
self.o(
" " if parent_list == "ol" and list.name == "ul" else " "
)
parent_list = list.name
if li.name == "ul":
self.o(self.ul_item_mark + " ")
elif li.name == "ol":
li.num += 1
self.o(str(li.num) + ". ")
self.start = True
if tag == "caption":
if not start:
# Ensure caption text ends on its own line before table rows
self.soft_br()
if tag in ["table", "tr", "td", "th"]:
if self.ignore_tables:
if tag == "tr":
if start:
pass
else:
self.soft_br()
else:
pass
elif self.bypass_tables:
if start:
self.soft_br()
attr_str = ""
if attrs:
attr_str = "".join(
' {}="{}"'.format(k, v) if v is not None else " {}".format(k)
for k, v in attrs.items()
)
if tag in ["td", "th"]:
self.o("<{}{}>\n\n".format(tag, attr_str))
else:
self.o("<{}{}>".format(tag, attr_str))
else:
if tag in ["td", "th"]:
self.o("\n".format(tag))
else:
self.o("".format(tag))
else:
if tag == "table":
if start:
self.table_start = True
if self.pad_tables:
self.o("<" + config.TABLE_MARKER_FOR_PAD + ">")
self.o(" \n")
else:
# Ensure table starts on its own line (GFM requirement)
self.soft_br()
else:
if self.pad_tables:
# add break in case the table is empty or its 1 row table
self.soft_br()
self.o("")
self.o(" \n")
if tag in ["td", "th"] and start:
if self.pad_tables:
# pad_tables mode: keep upstream inter-cell delimiter only
# (pad post-processor adds leading/trailing pipes and alignment)
if self.split_next_td:
self.o("| ")
else:
# GFM mode: leading pipe on first cell, spaced pipes between cells
if self.split_next_td:
self.o(" | ")
else:
self.o("| ")
self.split_next_td = True
if tag == "tr" and start:
self.td_count = 0
if tag == "tr" and not start:
if not self.pad_tables:
# Add trailing pipe for GFM compliance
self.o(" |")
self.split_next_td = False
self.soft_br()
if tag == "tr" and not start and self.table_start:
if self.pad_tables:
# pad_tables: plain separator (post-processor reformats)
self.o("|".join(["---"] * self.td_count))
else:
# GFM: separator with leading/trailing pipes
self.o("| " + " | ".join(["---"] * self.td_count) + " |")
self.soft_br()
self.table_start = False
if tag in ["td", "th"] and start:
self.td_count += 1
if tag == "pre":
if start:
self.startpre = True
self.pre = True
else:
self.pre = False
if self.mark_code:
self.out("\n[/code]")
self.p()
if tag in ["sup", "sub"] and self.include_sup_sub:
if start:
self.o("<{}>".format(tag))
else:
self.o("".format(tag))
# TODO: Add docstring for these one letter functions
def pbr(self) -> None:
"Pretty print has a line break"
if self.p_p == 0:
self.p_p = 1
def p(self) -> None:
"Set pretty print to 1 or 2 lines"
self.p_p = 1 if self.single_line_break else 2
def soft_br(self) -> None:
"Soft breaks"
self.pbr()
self.br_toggle = " "
def o(
self, data: str, puredata: bool = False, force: Union[bool, str] = False
) -> None:
"""
Deal with indentation and whitespace
"""
if self.abbr_data is not None:
self.abbr_data += data
if not self.quiet:
if self.google_doc:
# prevent white space immediately after 'begin emphasis'
# marks ('**' and '_')
lstripped_data = data.lstrip()
if self.drop_white_space and not (self.pre or self.code):
data = lstripped_data
if lstripped_data != "":
self.drop_white_space = 0
if puredata and not self.pre:
# This is a very dangerous call ... it could mess up
# all handling of when not handled properly
# (see entityref)
data = re.sub(r"\s+", r" ", data)
if data and data[0] == " ":
self.space = True
data = data[1:]
if not data and not force:
return
if self.startpre:
# self.out(" :") #TODO: not output when already one there
if not data.startswith("\n") and not data.startswith("\r\n"):
# stuff...
data = "\n" + data
if self.mark_code:
self.out("\n[code]")
self.p_p = 0
bq = ">" * self.blockquote
if not (force and data and data[0] == ">") and self.blockquote:
bq += " "
if self.pre:
if not self.list:
bq += " "
# else: list content is already partially indented
bq += " " * len(self.list)
data = data.replace("\n", "\n" + bq)
if self.startpre:
self.startpre = False
if self.list:
# use existing initial indentation
data = data.lstrip("\n")
if self.start:
self.space = False
self.p_p = 0
self.start = False
if force == "end":
# It's the end.
self.p_p = 0
self.out("\n")
self.space = False
if self.p_p:
self.out((self.br_toggle + "\n" + bq) * self.p_p)
self.space = False
self.br_toggle = ""
if self.space:
if not self.lastWasNL:
self.out(" ")
self.space = False
if self.a and (
(self.p_p == 2 and self.links_each_paragraph) or force == "end"
):
if force == "end":
self.out("\n")
newa = []
for link in self.a:
if self.outcount > link.outcount:
self.out(
" ["
+ str(link.count)
+ "]: "
+ urlparse.urljoin(self.baseurl, link.attrs["href"])
)
if "title" in link.attrs and link.attrs["title"] is not None:
self.out(" (" + link.attrs["title"] + ")")
self.out("\n")
else:
newa.append(link)
# Don't need an extra line when nothing was done.
if self.a != newa:
self.out("\n")
self.a = newa
if self.abbr_list and force == "end":
for abbr, definition in self.abbr_list.items():
self.out(" *[" + abbr + "]: " + definition + "\n")
self.p_p = 0
self.out(data)
self.outcount += 1
def handle_data(self, data: str, entity_char: bool = False) -> None:
if not data:
# Data may be empty for some HTML entities. For example,
# LEFT-TO-RIGHT MARK.
return
if self.stressed:
data = data.strip()
self.stressed = False
self.preceding_stressed = True
elif self.preceding_stressed:
if (
re.match(r"[^][(){}\s.!?]", data[0])
and not hn(self.current_tag)
and self.current_tag not in ["a", "code", "pre"]
):
# should match a letter or common punctuation
data = " " + data
self.preceding_stressed = False
if self.style:
self.style_def.update(dumb_css_parser(data))
if self.maybe_automatic_link is not None:
href = self.maybe_automatic_link
if (
href == data
and self.absolute_url_matcher.match(href)
and self.use_automatic_links
):
self.o("<" + data + ">")
self.empty_link = False
return
else:
self.o("[")
self.maybe_automatic_link = None
self.empty_link = False
if not self.code and not self.pre and not entity_char:
data = escape_md_section(
data,
snob=self.escape_snob,
escape_dot=self.escape_dot,
escape_plus=self.escape_plus,
escape_dash=self.escape_dash,
)
self.preceding_data = data
self.o(data, puredata=True)
def charref(self, name: str) -> str:
if name[0] in ["x", "X"]:
c = int(name[1:], 16)
else:
c = int(name)
if not self.unicode_snob and c in unifiable_n:
return unifiable_n[c]
else:
try:
return chr(c)
except ValueError: # invalid unicode
return ""
def entityref(self, c: str) -> str:
if not self.unicode_snob and c in config.UNIFIABLE:
return config.UNIFIABLE[c]
try:
ch = html.entities.html5[c + ";"]
except KeyError:
return "&" + c + ";"
return config.UNIFIABLE[c] if c == "nbsp" else ch
def google_nest_count(self, style: Dict[str, str]) -> int:
"""
Calculate the nesting count of google doc lists
:type style: dict
:rtype: int
"""
nest_count = 0
if "margin-left" in style:
nest_count = int(style["margin-left"][:-2]) // self.google_list_indent
return nest_count
def optwrap(self, text: str) -> str:
"""
Wrap all paragraphs in the provided text.
:type text: str
:rtype: str
"""
if not self.body_width:
return text
result = ""
newlines = 0
# I cannot think of a better solution for now.
# To avoid the non-wrap behaviour for entire paras
# because of the presence of a link in it
if not self.wrap_links:
self.inline_links = False
for para in text.split("\n"):
if len(para) > 0:
if not skipwrap(
para, self.wrap_links, self.wrap_list_items, self.wrap_tables
):
indent = ""
if para.startswith(" " + self.ul_item_mark):
# list item continuation: add a double indent to the
# new lines
indent = " "
elif para.startswith("> "):
# blockquote continuation: add the greater than symbol
# to the new lines
indent = "> "
wrapped = wrap(
para,
self.body_width,
break_long_words=False,
subsequent_indent=indent,
)
result += "\n".join(wrapped)
if para.endswith(" "):
result += " \n"
newlines = 1
elif indent:
result += "\n"
newlines = 1
else:
result += "\n\n"
newlines = 2
else:
# Warning for the tempted!!!
# Be aware that obvious replacement of this with
# line.isspace()
# DOES NOT work! Explanations are welcome.
if not config.RE_SPACE.match(para):
result += para + "\n"
newlines = 1
else:
if newlines < 2:
result += "\n"
newlines += 1
return result
def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = None) -> str:
if bodywidth is None:
bodywidth = config.BODY_WIDTH
h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
return h.handle(html)
class CustomHTML2Text(HTML2Text):
def __init__(self, *args, handle_code_in_pre=False, **kwargs):
super().__init__(*args, **kwargs)
self.inside_pre = False
self.inside_code = False
self.inside_link = False
self.preserve_tags = set() # Set of tags to preserve
self.current_preserved_tag = None
self.preserved_content = []
self.preserve_depth = 0
self.handle_code_in_pre = handle_code_in_pre
# Configuration options
self.skip_internal_links = False
self.single_line_break = False
self.mark_code = False
self.include_sup_sub = False
self.body_width = 0
self.ignore_mailto_links = True
self.ignore_links = False
self.escape_backslash = False
self.escape_dot = False
self.escape_plus = False
self.escape_dash = False
self.escape_snob = False
def update_params(self, **kwargs):
"""Update parameters and set preserved tags."""
for key, value in kwargs.items():
if key == "preserve_tags":
self.preserve_tags = set(value)
elif key == "handle_code_in_pre":
self.handle_code_in_pre = value
else:
setattr(self, key, value)
def handle_tag(self, tag, attrs, start):
# Handle tag to update base URL for relative links
# Must be handled before preserved tags since is in
if tag == "base" and start:
href = attrs.get("href") if attrs else None
if href:
self.baseurl = href
# Also let parent class handle it
return super().handle_tag(tag, attrs, start)
# Handle preserved tags
if tag in self.preserve_tags:
if start:
if self.preserve_depth == 0:
self.current_preserved_tag = tag
self.preserved_content = []
# Format opening tag with attributes
attr_str = "".join(
f' {k}="{v}"' for k, v in attrs.items() if v is not None
)
self.preserved_content.append(f"<{tag}{attr_str}>")
self.preserve_depth += 1
return
else:
self.preserve_depth -= 1
if self.preserve_depth == 0:
self.preserved_content.append(f"")
# Output the preserved HTML block with proper spacing
preserved_html = "".join(self.preserved_content)
self.o("\n" + preserved_html + "\n")
self.current_preserved_tag = None
return
# If we're inside a preserved tag, collect all content
if self.preserve_depth > 0:
if start:
# Format nested tags with attributes
attr_str = "".join(
f' {k}="{v}"' for k, v in attrs.items() if v is not None
)
self.preserved_content.append(f"<{tag}{attr_str}>")
else:
self.preserved_content.append(f"")
return
# Handle pre tags
if tag == "pre":
if start:
lang = attrs.get("data-language", "")
self.o(f"\n```{lang}\n") # Markdown code block start
self.inside_pre = True
else:
self.o("\n```\n") # Markdown code block end
self.inside_pre = False
elif tag == "code":
if self.inside_pre and not self.handle_code_in_pre:
# Ignore code tags inside pre blocks if handle_code_in_pre is False
return
if start:
if not self.inside_link:
self.o("`") # Only output backtick if not inside a link
self.inside_code = True
else:
if not self.inside_link:
self.o("`") # Only output backtick if not inside a link
self.inside_code = False
# If inside a link, let the parent class handle the content
if self.inside_link:
super().handle_tag(tag, attrs, start)
else:
super().handle_tag(tag, attrs, start)
def handle_data(self, data, entity_char=False):
"""Override handle_data to capture content within preserved tags."""
if self.preserve_depth > 0:
self.preserved_content.append(data)
return
if self.inside_pre:
# Output the raw content for pre blocks, including content inside code tags
self.o(data) # Directly output the data as-is (preserve newlines)
return
if self.inside_code:
# Inline code: no newlines allowed
self.o(data.replace("\n", " "))
return
# Default behavior for other tags
super().handle_data(data, entity_char)
# # Handle pre tags
# if tag == 'pre':
# if start:
# self.o('```\n')
# self.inside_pre = True
# else:
# self.o('\n```')
# self.inside_pre = False
# # elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
# # pass
# else:
# super().handle_tag(tag, attrs, start)
# def handle_data(self, data, entity_char=False):
# """Override handle_data to capture content within preserved tags."""
# if self.preserve_depth > 0:
# self.preserved_content.append(data)
# return
# super().handle_data(data, entity_char)