chore: import upstream snapshot with attribution

unstructured/documents/coordinates.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, Sequence, Tuple, Union
+
+
+class Orientation(Enum):
+    SCREEN = (1, -1)  # Origin in top left, y increases in the down direction
+    CARTESIAN = (1, 1)  # Origin in bottom left, y increases in upward direction
+
+
+def convert_coordinate(old_t, old_t_max, new_t_max, t_orientation):
+    """Convert a coordinate into another system along an axis using a linear transformation"""
+    return (
+        (1 - old_t / old_t_max) * (1 - t_orientation) / 2
+        + old_t / old_t_max * (1 + t_orientation) / 2
+    ) * new_t_max
+
+
+class CoordinateSystem:
+    """A finite coordinate plane with given width and height."""
+
+    orientation: Orientation
+
+    def __init__(self, width: Union[int, float], height: Union[int, float]):
+        self.width = width
+        self.height = height
+
+    def __eq__(self, other: object):
+        if not isinstance(other, CoordinateSystem):
+            return False
+        return (
+            str(self.__class__.__name__) == str(other.__class__.__name__)
+            and self.width == other.width
+            and self.height == other.height
+            and self.orientation == other.orientation
+        )
+
+    def convert_from_relative(
+        self,
+        x: Union[float, int],
+        y: Union[float, int],
+    ) -> Tuple[Union[float, int], Union[float, int]]:
+        """Convert to this coordinate system from a relative coordinate system."""
+        x_orientation, y_orientation = self.orientation.value
+        new_x = convert_coordinate(x, 1, self.width, x_orientation)
+        new_y = convert_coordinate(y, 1, self.height, y_orientation)
+        return new_x, new_y
+
+    def convert_to_relative(
+        self,
+        x: Union[float, int],
+        y: Union[float, int],
+    ) -> Tuple[Union[float, int], Union[float, int]]:
+        """Convert from this coordinate system to a relative coordinate system."""
+        x_orientation, y_orientation = self.orientation.value
+        new_x = convert_coordinate(x, self.width, 1, x_orientation)
+        new_y = convert_coordinate(y, self.height, 1, y_orientation)
+        return new_x, new_y
+
+    def convert_coordinates_to_new_system(
+        self,
+        new_system: CoordinateSystem,
+        x: Union[float, int],
+        y: Union[float, int],
+    ) -> Tuple[Union[float, int], Union[float, int]]:
+        """Convert from this coordinate system to another given coordinate system."""
+        rel_x, rel_y = self.convert_to_relative(x, y)
+        return new_system.convert_from_relative(rel_x, rel_y)
+
+    def convert_multiple_coordinates_to_new_system(
+        self,
+        new_system: CoordinateSystem,
+        coordinates: Sequence[Tuple[Union[float, int], Union[float, int]]],
+    ) -> Tuple[Tuple[Union[float, int], Union[float, int]], ...]:
+        """Convert (x, y) coordinates from current system to another coordinate system."""
+        new_system_coordinates = []
+        for x, y in coordinates:
+            new_system_coordinates.append(
+                self.convert_coordinates_to_new_system(new_system=new_system, x=x, y=y),
+            )
+        return tuple(new_system_coordinates)
+
+
+class RelativeCoordinateSystem(CoordinateSystem):
+    """Relative coordinate system where x and y are on a scale from 0 to 1."""
+
+    orientation = Orientation.CARTESIAN
+
+    def __init__(self):
+        self.width = 1
+        self.height = 1
+
+
+class PixelSpace(CoordinateSystem):
+    """Coordinate system representing a pixel space, such as an image. The origin is at the top
+    left."""
+
+    orientation = Orientation.SCREEN
+
+
+class PointSpace(CoordinateSystem):
+    """Coordinate system representing a point space, such as a pdf. The origin is at the bottom
+    left."""
+
+    orientation = Orientation.CARTESIAN
+
+
+TYPE_TO_COORDINATE_SYSTEM_MAP: Dict[str, Any] = {
+    "PixelSpace": PixelSpace,
+    "PointSpace": PointSpace,
+    "CoordinateSystem": CoordinateSystem,
+}

unstructured/documents/html_sanitization.py

@@ -0,0 +1,427 @@
+"""Central HTML output-sanitization policy for the ontology (v2) HTML path.
+
+`unstructured` renders untrusted document content into HTML in two places:
+
+* ``OntologyElement.to_html`` (``documents/ontology.py``), which fills
+  ``ElementMetadata.text_as_html``. Some callers return this value to clients
+  verbatim, so it must be safe on its own.
+* ``elements_to_html`` (``partition/html/convert.py``), which assembles a full
+  HTML document from a list of elements.
+
+Both used to interpolate attacker-controlled text, attribute names, attribute
+values, and URL schemes with no output encoding, allowing stored XSS
+(GHSA-v5mq-3xhg-98m9). This module is the single source of truth for the
+sanitization policy shared by both paths:
+
+* an allowlist of HTML tags we ever legitimately emit,
+* an allowlist of attribute names (event-handler ``on*`` attributes are never
+  allowed, killing ``onerror``/``onload``/``onmouseover``),
+* a URL-scheme allowlist for URL-bearing attributes (``href``/``src``/...),
+  which drops ``javascript:`` / ``vbscript:`` and permits ``data:`` only for
+  raster image MIME types on ``img[src]``.
+
+The emitter (``ontology.py``) uses the lightweight filters here plus
+``html.escape`` to make ``text_as_html`` safe on its own; ``elements_to_html``
+additionally runs the assembled document through :func:`sanitize_html_fragment`
+(``nh3``) as defense-in-depth that also covers attributes it injects itself
+(e.g. ``href`` from ``metadata.url``).
+"""
+
+from __future__ import annotations
+
+import re
+
+import nh3
+
+# -- Tags the ontology / convert paths legitimately emit. Anything outside this
+# -- set (``<script>``, ``<iframe>``, ...) is dropped/neutralized. --
+ALLOWED_TAGS: frozenset[str] = frozenset(
+    {
+        # layout / structural
+        "body",
+        "div",
+        "section",
+        "header",
+        "footer",
+        "aside",
+        "nav",
+        "figure",
+        "figcaption",
+        "hr",
+        "br",
+        # text
+        "span",
+        "p",
+        "blockquote",
+        "pre",
+        "address",
+        "time",
+        "mark",
+        "ins",
+        "del",
+        "cite",
+        "sub",
+        "sup",
+        "b",
+        "i",
+        "s",
+        "code",
+        # headings
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        # lists
+        "ul",
+        "ol",
+        "li",
+        "dl",
+        # tables
+        "table",
+        "thead",
+        "tbody",
+        "tr",
+        "td",
+        "th",
+        # links / media
+        "a",
+        "img",
+        "svg",
+        "audio",
+        "video",
+        # forms
+        "form",
+        "input",
+        "label",
+        "button",
+        # misc content
+        "math",
+        "meta",
+    }
+)
+
+# -- Attribute names carrying a URL; their values are scheme-filtered. Keep this
+# -- list broader than the attributes we currently allow so newly-allowed URL
+# -- attributes are scheme-checked by default. --
+URL_ATTRIBUTES: frozenset[str] = frozenset(
+    {
+        "action",
+        "cite",
+        "data-src",
+        "formaction",
+        "href",
+        "poster",
+        "src",
+        "srcset",
+        "xlink:href",
+    }
+)
+
+# -- Data URLs are only needed for embedded image bytes. SVG is intentionally
+# -- excluded because SVG documents can carry active content in some render
+# -- contexts. --
+ALLOWED_DATA_IMAGE_MIME_TYPES: frozenset[str] = frozenset(
+    {
+        "image/avif",
+        "image/bmp",
+        "image/gif",
+        "image/jpeg",
+        "image/jpg",
+        "image/png",
+        "image/webp",
+        "image/x-icon",
+    }
+)
+
+# -- Inline CSS is untrusted too. Keep only inert presentation properties that
+# -- preserve existing table/background formatting and reject layout/overlay
+# -- controls like position/inset/z-index.
+ALLOWED_CSS_PROPERTIES: frozenset[str] = frozenset(
+    {
+        "background-color",
+        "border",
+        "border-bottom",
+        "border-collapse",
+        "border-color",
+        "border-left",
+        "border-right",
+        "border-style",
+        "border-top",
+        "border-width",
+        "color",
+        "font-style",
+        "font-weight",
+        "text-align",
+        "text-decoration",
+        "vertical-align",
+        "white-space",
+    }
+)
+
+# -- Attributes allowed on every tag. --
+_GLOBAL_ATTRIBUTES: frozenset[str] = frozenset(
+    {"class", "id", "style", "title", "dir", "lang", "role", "name", "align"}
+)
+
+# -- Per-tag attributes in addition to the global set. --
+_TAG_ATTRIBUTES: dict[str, frozenset[str]] = {
+    "a": frozenset({"href", "target", "rel"}),
+    "img": frozenset({"src", "alt", "width", "height"}),
+    "svg": frozenset({"src", "alt", "width", "height", "xlink:href"}),
+    "audio": frozenset({"src", "controls"}),
+    "video": frozenset({"src", "controls", "poster", "width", "height"}),
+    "input": frozenset({"type", "checked", "value", "placeholder"}),
+    "td": frozenset({"colspan", "rowspan", "headers", "scope"}),
+    "th": frozenset({"colspan", "rowspan", "headers", "scope"}),
+    "ol": frozenset({"start", "type"}),
+    "label": frozenset({"for"}),
+    "meta": frozenset({"charset", "content"}),
+    "time": frozenset({"datetime"}),
+}
+
+# -- Attribute-name prefixes allowed on any tag (data-page-number, aria-*, ...). --
+_GENERIC_ATTRIBUTE_PREFIXES: frozenset[str] = frozenset({"data-", "aria-"})
+
+# -- URL schemes permitted on URL-bearing attributes. ``data`` is permitted here
+# -- but further restricted to raster image MIME types on ``img[src]`` by
+# -- :func:`is_safe_url` / the nh3 attribute filter; relative URLs (no scheme)
+# -- are always allowed. --
+ALLOWED_URL_SCHEMES: frozenset[str] = frozenset({"http", "https", "mailto", "tel", "data"})
+
+# -- Valid HTML/XML attribute name (prevents attribute-name breakout on emit). --
+_ATTRIBUTE_NAME_RE = re.compile(r"^[a-zA-Z_:][-a-zA-Z0-9_:.]*$")
+
+# -- Matches a leading ``scheme:`` ignoring surrounding whitespace and embedded
+# -- control chars that browsers strip (e.g. ``java\tscript:``). --
+_SCHEME_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9+.\-]*):")
+
+_REQUIRED_BLANK_TARGET_REL_VALUES: frozenset[str] = frozenset({"noopener", "noreferrer"})
+
+# -- Match an anchor start tag and detect `target="_blank"` / an existing `rel`
+# -- so we can add reverse-tabnabbing protection to new-tab links only. nh3's
+# -- output is well-formed (attribute values are escaped, so `>` never appears
+# -- inside one), which makes matching whole start tags with a regex safe. --
+_ANCHOR_START_TAG_RE = re.compile(r"<a\b[^>]*>", flags=re.IGNORECASE)
+_BLANK_TARGET_RE = re.compile(r"""\btarget\s*=\s*["']?_blank\b""", flags=re.IGNORECASE)
+_REL_ATTRIBUTE_RE = re.compile(r"""\brel\s*=""", flags=re.IGNORECASE)
+
+_CSS_COMMENT_RE = re.compile(r"/\*.*?\*/", flags=re.DOTALL)
+_CSS_UNSAFE_VALUE_RE = re.compile(
+    r"(@import|expression\s*\(|url\s*\(|javascript:|vbscript:|data:|-moz-binding|behavior\s*:)",
+    flags=re.IGNORECASE,
+)
+
+
+def _normalize_url(value: str) -> str:
+    """Lower-case and strip whitespace/control chars a browser would ignore."""
+    return re.sub(r"[\x00-\x20]+", "", value).lower()
+
+
+def _normalized_tag_and_attribute(
+    tag_name: str | None = None,
+    attribute_name: str | None = None,
+) -> tuple[str | None, str | None]:
+    tag = tag_name.strip().lower() if tag_name else None
+    attribute = attribute_name.strip().lower() if attribute_name else None
+    return tag, attribute
+
+
+def _is_allowed_data_image_url(
+    normalized_value: str,
+    *,
+    tag_name: str | None = None,
+    attribute_name: str | None = None,
+) -> bool:
+    tag, attribute = _normalized_tag_and_attribute(tag_name, attribute_name)
+    if tag != "img" or attribute != "src":
+        return False
+    if not normalized_value.startswith("data:") or "," not in normalized_value:
+        return False
+    media_type = normalized_value[5:].split(",", 1)[0].split(";", 1)[0]
+    return media_type in ALLOWED_DATA_IMAGE_MIME_TYPES
+
+
+def is_safe_url(
+    value: str,
+    *,
+    tag_name: str | None = None,
+    attribute_name: str | None = None,
+) -> bool:
+    """True if ``value`` is safe to keep in a URL-bearing attribute.
+
+    Relative URLs (no scheme) are allowed. Absolute URLs are allowed only for
+    :data:`ALLOWED_URL_SCHEMES`, and ``data:`` is further narrowed to raster
+    image MIME types on ``img[src]`` so embedded base64 images survive while
+    SVG / HTML / JavaScript data documents are rejected.
+    """
+    normalized = _normalize_url(value)
+    match = _SCHEME_RE.match(normalized)
+    if match is None:
+        # -- no scheme -> relative URL (or a fragment/anchor); safe --
+        return True
+    scheme = match.group(1)
+    if scheme not in ALLOWED_URL_SCHEMES:
+        return False
+    if scheme == "data":
+        return _is_allowed_data_image_url(
+            normalized,
+            tag_name=tag_name,
+            attribute_name=attribute_name,
+        )
+    return True
+
+
+def is_event_handler_attribute(name: str) -> bool:
+    """True for ``on*`` event-handler attribute names (onerror, onload, ...)."""
+    return name.strip().lower().startswith("on")
+
+
+def _is_allowed_attribute_for_tag(name: str, tag_name: str | None) -> bool:
+    lowered = name.lower()
+    if any(lowered.startswith(prefix) for prefix in _GENERIC_ATTRIBUTE_PREFIXES):
+        return True
+
+    allowed_attributes = set(_GLOBAL_ATTRIBUTES)
+    if tag_name:
+        allowed_attributes.update(_TAG_ATTRIBUTES.get(tag_name.strip().lower(), frozenset()))
+    return lowered in allowed_attributes
+
+
+def _link_rel_with_required_values(value: object | None) -> str:
+    existing_values = str(value or "").split()
+    existing_lowered = {rel.lower() for rel in existing_values}
+    missing_values = [
+        rel for rel in sorted(_REQUIRED_BLANK_TARGET_REL_VALUES) if rel not in existing_lowered
+    ]
+    return " ".join([*existing_values, *missing_values]).strip()
+
+
+def sanitize_style_attribute(value: object) -> str:
+    """Return a style attribute containing only safe presentation declarations."""
+    style = _CSS_COMMENT_RE.sub("", str(value))
+    safe_declarations: list[str] = []
+    for declaration in style.split(";"):
+        if ":" not in declaration:
+            continue
+        raw_property, raw_value = declaration.split(":", 1)
+        property_name = raw_property.strip().lower()
+        property_value = " ".join(raw_value.strip().split())
+        if property_name not in ALLOWED_CSS_PROPERTIES:
+            continue
+        if not property_value or _CSS_UNSAFE_VALUE_RE.search(property_value):
+            continue
+        safe_declarations.append(f"{property_name}: {property_value}")
+    return "; ".join(safe_declarations)
+
+
+def sanitize_attributes(
+    attributes: dict[str, object],
+    tag_name: str | None = None,
+) -> dict[str, object]:
+    """Filter an attribute mapping for safe emission (does NOT html-escape).
+
+    Drops event-handler (``on*``) attributes, attribute names that aren't valid
+    HTML attribute names, attributes not allowed for ``tag_name``, and URL-bearing
+    attributes whose value uses an unsafe scheme. Values are returned unchanged;
+    the emitter is responsible for ``html.escape``-ing them so escaping happens
+    exactly once.
+    """
+    safe: dict[str, object] = {}
+    for key, value in attributes.items():
+        name = str(key).strip()
+        lowered = name.lower()
+        if is_event_handler_attribute(lowered):
+            continue
+        if not _ATTRIBUTE_NAME_RE.match(name):
+            continue
+        if not _is_allowed_attribute_for_tag(name, tag_name):
+            continue
+        if lowered == "style":
+            sanitized_style = sanitize_style_attribute(value)
+            if not sanitized_style:
+                continue
+            safe[name] = sanitized_style
+            continue
+        if lowered in URL_ATTRIBUTES and value is not None:
+            candidate = value[0] if isinstance(value, list) and value else value
+            if isinstance(candidate, str) and not is_safe_url(
+                candidate,
+                tag_name=tag_name,
+                attribute_name=lowered,
+            ):
+                continue
+        safe[name] = value
+
+    if (tag_name or "").strip().lower() == "a" and str(safe.get("target", "")).lower() == "_blank":
+        safe["rel"] = _link_rel_with_required_values(safe.get("rel"))
+
+    return safe
+
+
+def is_safe_tag(tag_name: str | None) -> bool:
+    """True if ``tag_name`` is in the emit allowlist."""
+    return bool(tag_name) and tag_name.strip().lower() in ALLOWED_TAGS
+
+
+def _nh3_attribute_filter(tag: str, attribute: str, value: str) -> str | None:
+    """nh3 per-attribute hook: drop event handlers and unsafe URL values."""
+    if is_event_handler_attribute(attribute):
+        return None
+    if attribute.lower() == "style":
+        return sanitize_style_attribute(value) or None
+    if attribute.lower() in URL_ATTRIBUTES and not is_safe_url(
+        value,
+        tag_name=tag,
+        attribute_name=attribute,
+    ):
+        return None
+    return value
+
+
+def _add_rel_to_blank_target_links(html_fragment: str) -> str:
+    """Add reverse-tabnabbing ``rel`` tokens to ``target="_blank"`` anchors only.
+
+    nh3's ``link_rel`` would stamp ``rel`` onto *every* link, stripping the
+    ``Referer`` header from ordinary same-tab navigation. Reverse tabnabbing is
+    only a concern for new-tab links, so we scope the tokens to anchors that
+    actually open a new browsing context and leave same-tab links untouched.
+    """
+
+    def add_rel(match: re.Match[str]) -> str:
+        tag = match.group(0)
+        if not _BLANK_TARGET_RE.search(tag) or _REL_ATTRIBUTE_RE.search(tag):
+            return tag
+        rel = " ".join(sorted(_REQUIRED_BLANK_TARGET_REL_VALUES))
+        return f'{tag[:-1].rstrip()} rel="{rel}">'
+
+    return _ANCHOR_START_TAG_RE.sub(add_rel, html_fragment)
+
+
+def sanitize_html_fragment(html_fragment: str) -> str:
+    """Sanitize an assembled HTML fragment with ``nh3`` (defense-in-depth).
+
+    Applies the shared tag/attribute/URL-scheme allowlists. Used on the final
+    output of ``elements_to_html`` so that attributes injected outside the
+    ontology emitter (e.g. an ``href`` built from ``metadata.url``) are also
+    neutralized.
+    """
+    attributes: dict[str, set[str]] = {"*": set(_GLOBAL_ATTRIBUTES)}
+    for tag, attrs in _TAG_ATTRIBUTES.items():
+        attributes[tag] = set(attrs)
+    cleaned = nh3.clean(
+        html_fragment,
+        tags=set(ALLOWED_TAGS),
+        attributes=attributes,
+        url_schemes=set(ALLOWED_URL_SCHEMES),
+        generic_attribute_prefixes=set(_GENERIC_ATTRIBUTE_PREFIXES),
+        filter_style_properties=set(ALLOWED_CSS_PROPERTIES),
+        attribute_filter=_nh3_attribute_filter,
+        # -- Scope reverse-tabnabbing `rel` tokens to `_blank` anchors ourselves
+        # -- rather than letting nh3 add them to every link (which would strip
+        # -- the Referer header from same-tab navigation). --
+        link_rel=None,
+        strip_comments=True,
+    )
+    return _add_rel_to_blank_target_links(cleaned)

unstructured/documents/mappings.py

@@ -0,0 +1,195 @@
+"""
+This module contains mapping between:
+HTML Tags <-> Elements Ontology <-> Unstructured Element classes
+They are used to simplify transformations between different representations
+of parsed documents
+"""
+
+from typing import Dict, Type
+
+from unstructured.documents import elements, ontology
+from unstructured.documents.elements import Element
+
+
+def get_all_subclasses(cls: type) -> list[type]:
+    """
+    Recursively find all subclasses of a given class.
+
+    Parameters:
+    cls (type): The class for which to find all subclasses.
+
+    Returns:
+    list[type]: A list of all subclasses of the given class.
+    """
+    subclasses = cls.__subclasses__()
+    all_subclasses = subclasses.copy()
+
+    for subclass in subclasses:
+        all_subclasses.extend(get_all_subclasses(subclass))
+
+    return all_subclasses
+
+
+def get_ontology_to_unstructured_type_mapping() -> dict[
+    Type[ontology.OntologyElement], Type[Element]
+]:
+    """
+    Get a mapping of ontology element to unstructured type.
+
+    The dictionary here was created base on ontology mapping json
+    Can be generated via the following code:
+    ```
+        ontology_elements_list = json.loads(
+            Path("unstructured_element_ontology.json").read_text()
+        )
+        ontology_to_unstructured_class_mapping = {
+            ontology_element["name"]: ontology_element["ontologyV1Mapping"]
+            for ontology_element in ontology_elements_list
+    }
+    ```
+
+    Returns:
+    dict: A dictionary where keys are ontology element classes
+          and values are unstructured types.
+    """
+    ontology_to_unstructured_class_mapping: Dict[Type[ontology.OntologyElement], Type[Element]] = {
+        ontology.Document: elements.Text,
+        ontology.Section: elements.Text,
+        ontology.Page: elements.Text,
+        ontology.Column: elements.Text,
+        ontology.Paragraph: elements.NarrativeText,
+        ontology.Header: elements.Header,
+        ontology.Footer: elements.Footer,
+        ontology.Sidebar: elements.Text,
+        ontology.PageBreak: elements.PageBreak,
+        ontology.Title: elements.Title,
+        ontology.Subtitle: elements.Title,
+        ontology.Heading: elements.Title,
+        ontology.NarrativeText: elements.NarrativeText,
+        ontology.Quote: elements.NarrativeText,
+        ontology.Footnote: elements.Text,
+        ontology.Caption: elements.FigureCaption,
+        ontology.PageNumber: elements.PageNumber,
+        ontology.UncategorizedText: elements.Text,
+        ontology.OrderedList: elements.Text,
+        ontology.UnorderedList: elements.Text,
+        ontology.DefinitionList: elements.Text,
+        ontology.ListItem: elements.ListItem,
+        ontology.Table: elements.Table,
+        ontology.TableRow: elements.Table,
+        ontology.TableCell: elements.Table,
+        ontology.TableCellHeader: elements.Table,
+        ontology.TableBody: elements.Table,
+        ontology.TableHeader: elements.Table,
+        ontology.Image: elements.Image,
+        ontology.Figure: elements.Image,
+        ontology.Video: elements.Text,
+        ontology.Audio: elements.Text,
+        ontology.Barcode: elements.Image,
+        ontology.QRCode: elements.Image,
+        ontology.Logo: elements.Image,
+        ontology.CodeBlock: elements.CodeSnippet,
+        ontology.InlineCode: elements.CodeSnippet,
+        ontology.Formula: elements.Formula,
+        ontology.Equation: elements.Formula,
+        ontology.FootnoteReference: elements.Text,
+        ontology.Citation: elements.Text,
+        ontology.Bibliography: elements.Text,
+        ontology.Glossary: elements.Text,
+        ontology.Author: elements.Text,
+        ontology.MetaDate: elements.Text,
+        ontology.Keywords: elements.Text,
+        ontology.Abstract: elements.NarrativeText,
+        ontology.Hyperlink: elements.Text,
+        ontology.TableOfContents: elements.Table,
+        ontology.Index: elements.Text,
+        ontology.Form: elements.Text,
+        ontology.FormField: elements.Text,
+        ontology.FormFieldValue: elements.Text,
+        ontology.Checkbox: elements.Text,
+        ontology.RadioButton: elements.Text,
+        ontology.Button: elements.Text,
+        ontology.Comment: elements.Text,
+        ontology.Highlight: elements.Text,
+        ontology.RevisionInsertion: elements.Text,
+        ontology.RevisionDeletion: elements.Text,
+        ontology.Address: elements.Address,
+        ontology.EmailAddress: elements.EmailAddress,
+        ontology.PhoneNumber: elements.Text,
+        ontology.CalendarDate: elements.Text,
+        ontology.Time: elements.Text,
+        ontology.Currency: elements.Text,
+        ontology.Measurement: elements.Text,
+        ontology.Letterhead: elements.Header,
+        ontology.Signature: elements.Text,
+        ontology.Watermark: elements.Text,
+        ontology.Stamp: elements.Text,
+    }
+
+    return ontology_to_unstructured_class_mapping
+
+
+ALL_ONTOLOGY_ELEMENT_TYPES = get_all_subclasses(ontology.OntologyElement)
+HTML_TAG_AND_CSS_NAME_TO_ELEMENT_TYPE_MAP: Dict[tuple[str, str], Type[ontology.OntologyElement]] = {
+    (tag, element_type().css_class_name): element_type
+    for element_type in ALL_ONTOLOGY_ELEMENT_TYPES
+    for tag in element_type().allowed_tags
+}
+CSS_CLASS_TO_ELEMENT_TYPE_MAP: Dict[str, Type[ontology.OntologyElement]] = {
+    element_type().css_class_name: element_type for element_type in ALL_ONTOLOGY_ELEMENT_TYPES
+}
+
+HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP: Dict[str, Type[ontology.OntologyElement]] = {
+    "a": ontology.Hyperlink,
+    "address": ontology.Address,
+    "aside": ontology.Sidebar,
+    "audio": ontology.Audio,
+    "blockquote": ontology.Quote,
+    "body": ontology.Document,
+    "button": ontology.Button,
+    "cite": ontology.Citation,
+    "code": ontology.CodeBlock,
+    "del": ontology.RevisionDeletion,
+    "div": ontology.UncategorizedText,
+    "dl": ontology.DefinitionList,
+    "figcaption": ontology.Caption,
+    "figure": ontology.Figure,
+    "footer": ontology.Footer,
+    "form": ontology.Form,
+    "h1": ontology.Title,
+    "h2": ontology.Subtitle,
+    "h3": ontology.Heading,
+    "h4": ontology.Heading,
+    "h5": ontology.Heading,
+    "h6": ontology.Heading,
+    "header": ontology.Header,
+    "hr": ontology.PageBreak,
+    "img": ontology.Image,
+    "input": ontology.Checkbox,
+    "ins": ontology.RevisionInsertion,
+    "label": ontology.FormField,
+    "li": ontology.ListItem,
+    "mark": ontology.Highlight,
+    "math": ontology.Equation,
+    "meta": ontology.Keywords,
+    "nav": ontology.Index,
+    "ol": ontology.OrderedList,
+    "p": ontology.Paragraph,
+    "pre": ontology.CodeBlock,
+    "section": ontology.Section,
+    "span": ontology.UncategorizedText,
+    "sub": ontology.FootnoteReference,
+    "svg": ontology.Signature,
+    "table": ontology.Table,
+    "tbody": ontology.TableBody,
+    "td": ontology.TableCell,
+    "th": ontology.TableCellHeader,
+    "thead": ontology.TableHeader,
+    "time": ontology.Time,
+    "tr": ontology.TableRow,
+    "ul": ontology.UnorderedList,
+    "video": ontology.Video,
+}
+
+
+ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE = get_ontology_to_unstructured_type_mapping()

unstructured/documents/ontology.py

@@ -0,0 +1,647 @@
+"""
+This file contains all classes allowed in the ontology V2.
+This Type is used as intermediate representation between HTML
+and Unstructured Elements.
+All the processing could be done without the intermediate representation,
+but it simplifies the process.
+It needs to be decide whether we keep it or not.
+
+The classes are represented as pydantic models to mimic Unstructured Elements V1 solutions.
+However it results in lots of code that could be strongly simplified.
+
+TODO (Pluto): OntologyElement is the only needed class. It could contains data about
+ allowed html tags, css classes and descriptions as metadata.
+"""
+
+from __future__ import annotations
+
+import html
+import uuid
+from copy import copy
+from enum import Enum
+from typing import Any, List, Optional
+
+from bs4 import BeautifulSoup
+from pydantic import BaseModel, Field
+
+from unstructured.documents.html_sanitization import is_safe_tag, sanitize_attributes
+
+
+class ElementTypeEnum(str, Enum):
+    layout = "Layout"
+    text = "Text"
+    list = "List"
+    table = "Table"
+    media = "Media"
+    code = "Code"
+    mathematical = "Mathematical"
+    reference = "Reference"
+    metadata = "Metadata"
+    navigation = "Navigation"
+    form = "Form"
+    annotation = "Annotation"
+    specialized_text = "Specialized Text"
+    document_specific = "Document-Specific"
+
+
+class OntologyElement(BaseModel):
+    text: Optional[str] = Field("", description="Text content of the element")
+    css_class_name: Optional[str] = Field(
+        default_factory=lambda: "", description="CSS class associated with the element"
+    )
+    html_tag_name: Optional[str] = Field(
+        default_factory=lambda: "", description="HTML Tag name associated with the element"
+    )
+    elementType: ElementTypeEnum = Field(description="Type of the element")
+    children: list["OntologyElement"] = Field(
+        default_factory=list, description="List of child elements"
+    )
+
+    description: str = Field(description="Description of the element")
+    allowed_tags: list[str] = Field(description="HTML tags associated with the element")
+
+    additional_attributes: dict[str, Any] = Field(
+        default_factory=dict, description="Optional HTML attributes or CSS properties"
+    )
+
+    def __init__(self, **kwargs: dict[str, Any]):
+        super().__init__(**kwargs)
+        if self.css_class_name == "":  # if None, then do not set
+            self.css_class_name = self.__class__.__name__
+        if self.html_tag_name == "":
+            self.html_tag_name = self.allowed_tags[0]
+        if "id" not in self.additional_attributes:
+            self.additional_attributes["id"] = self.generate_unique_id()
+
+    @staticmethod
+    def generate_unique_id() -> str:
+        return str(uuid.uuid4()).replace("-", "")
+
+    def to_html(self, add_children: bool = True) -> str:
+        additional_attrs = copy(self.additional_attributes)
+        additional_attrs.pop("class", None)
+        additional_attrs.pop("id", None)
+
+        tag_name = self.html_tag_name if is_safe_tag(self.html_tag_name) else "span"
+        attr_str = self._construct_attribute_string(additional_attrs, tag_name)
+        class_attr = (
+            f'class="{html.escape(self.css_class_name, quote=True)}"' if self.css_class_name else ""
+        )
+
+        combined_attr_str = f"{class_attr} {attr_str}".strip()
+
+        children_html = self._generate_children_html(add_children)
+
+        result_html = self._generate_final_html(combined_attr_str, children_html, tag_name)
+
+        return result_html
+
+    def to_text(self, add_children: bool = True, add_img_alt_text: bool = True) -> str:
+        """
+        Returns the text representation of the element.
+
+        Args:
+            add_children: If True, the text of the children will be included.
+                            Otherwise, element is represented as single self-closing tag.
+            add_img_alt_text: If True, the alt text of the image will be included.
+        """
+        if self.children and add_children:
+            children_text = " ".join(
+                child.to_text(add_children, add_img_alt_text).strip() for child in self.children
+            )
+            return children_text
+
+        # -- Derive the plain text from the raw `text`. When it holds markup
+        # -- (e.g. the collapsed subtree stored when the recursion limit is hit)
+        # -- strip the tags with BeautifulSoup; otherwise use it as-is (which also
+        # -- avoids BeautifulSoup's MarkupResemblesLocatorWarning on plain strings).
+        # -- This is deliberately NOT taken from `to_html()`, whose output is now
+        # -- HTML-escaped for safety -- reparsing that escaped output would surface
+        # -- the markup as literal text instead of stripping it. --
+        raw_text = self.text or ""
+        if "<" in raw_text:
+            text = BeautifulSoup(raw_text, "html.parser").get_text().strip()
+        else:
+            text = raw_text.strip()
+
+        if add_img_alt_text and self.html_tag_name == "img" and "alt" in self.additional_attributes:
+            text += f" {self.additional_attributes.get('alt', '')}"
+
+        return text.strip()
+
+    def _construct_attribute_string(self, attributes: dict[str, str], tag_name: str) -> str:
+        # -- Drop event-handler (on*) and unsafe-scheme attributes, then HTML-escape
+        # -- each value (quote=True) so an attacker cannot break out of the quoted
+        # -- value with `">`. Escaping happens here, at emit time, so callers must
+        # -- pass raw (un-escaped) values -- see html_sanitization.sanitize_attributes. --
+        safe_attributes = sanitize_attributes(attributes, tag_name=tag_name)
+        return " ".join(
+            f'{key}="{html.escape(str(value), quote=True)}"' if value else f"{key}"
+            for key, value in safe_attributes.items()
+        )
+
+    def _generate_children_html(self, add_children: bool) -> str:
+        if not add_children or not self.children:
+            return ""
+        return "".join(child.to_html() for child in self.children)
+
+    def _generate_final_html(self, attr_str: str, children_html: str, tag_name: str) -> str:
+        # -- Escape element text; `children_html` is already-sanitized HTML from
+        # -- recursive `to_html` calls, so it must NOT be re-escaped. --
+        text = html.escape(self.text) if self.text else ""
+
+        if text or children_html:
+            inside_tag_text = f"{text} {children_html}".strip()
+            return f"<{tag_name} {attr_str}>{inside_tag_text}</{tag_name}>"
+        else:
+            return f"<{tag_name} {attr_str} />"
+
+    @property
+    def id(self) -> str | None:
+        return self.additional_attributes.get("id", None)
+
+    @property
+    def page_number(self) -> int | None:
+        if "data-page-number" in self.additional_attributes:
+            try:
+                page_attr = self.additional_attributes.get("data-page-number")
+                if page_attr is not None:
+                    return int(page_attr)
+            except ValueError:
+                return None
+        return None
+
+
+def remove_ids_and_class_from_table(
+    soup: BeautifulSoup, class_attr_to_keep: list[str] = ["img", "input"]
+) -> BeautifulSoup:
+    """
+    Remove id and class attributes from tags inside tables,
+    except preserve class attributes for selected tags.
+
+    Args:
+        soup: BeautifulSoup object containing the HTML
+        class_attr_to_keep: a list of tag names whose class attr will be kept
+
+    Returns:
+        BeautifulSoup: Modified soup with attributes removed
+    """
+    for tag in soup.find_all(True):
+        if tag.name.lower() == "table":  # type: ignore
+            continue  # We keep table tag
+        tag.attrs.pop("id", None)  # type: ignore
+        if tag.name.lower() not in class_attr_to_keep:  # type: ignore
+            tag.attrs.pop("class", None)  # type: ignore
+    return soup
+
+
+# Define specific elements
+class Document(OntologyElement):
+    description: str = Field("Root element of the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["body"], frozen=True)
+
+
+class Section(OntologyElement):
+    description: str = Field("A distinct part or subdivision of a document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["section"], frozen=True)
+
+
+class Page(OntologyElement):
+    description: str = Field("A single side of a paper in a document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["div"], frozen=True)
+
+
+class Column(OntologyElement):
+    description: str = Field("A vertical section of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["div"], frozen=True)
+
+
+class Paragraph(OntologyElement):
+    description: str = Field("A self-contained unit of discourse in writing", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["p"], frozen=True)
+
+
+class Header(OntologyElement):
+    description: str = Field("The top section of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["header"], frozen=True)
+
+
+class Footer(OntologyElement):
+    description: str = Field("The bottom section of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["footer"], frozen=True)
+
+
+class Sidebar(OntologyElement):
+    description: str = Field("A side section of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["aside"], frozen=True)
+
+
+class PageBreak(OntologyElement):
+    description: str = Field("A break between pages", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["hr"], frozen=True)
+
+
+class Title(OntologyElement):
+    description: str = Field("Main heading of a document or section", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["h1"], frozen=True)
+
+
+class Subtitle(OntologyElement):
+    description: str = Field("Secondary title of a document or section", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["h2"], frozen=True)
+
+
+class Heading(OntologyElement):
+    description: str = Field("Section headings (levels 1-6)", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["h1", "h2", "h3", "h4", "h5", "h6"], frozen=True)
+
+
+class NarrativeText(OntologyElement):
+    description: str = Field("Main content text", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["p"], frozen=True)
+
+
+class Quote(OntologyElement):
+    description: str = Field("A repetition of someone else's statement", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["blockquote"], frozen=True)
+
+
+class Footnote(OntologyElement):
+    description: str = Field("A note at the bottom of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["div"], frozen=True)
+
+
+class Caption(OntologyElement):
+    description: str = Field("Text describing a figure or image", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["figcaption"], frozen=True)
+
+
+class PageNumber(OntologyElement):
+    description: str = Field("The number of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class UncategorizedText(OntologyElement):
+    description: str = Field("Miscellaneous text", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class OrderedList(OntologyElement):
+    description: str = Field("A list with a specific sequence", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.list, frozen=True)
+    allowed_tags: List[str] = Field(["ol"], frozen=True)
+
+
+class UnorderedList(OntologyElement):
+    description: str = Field("A list without a specific sequence", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.list, frozen=True)
+    allowed_tags: List[str] = Field(["ul"], frozen=True)
+
+
+class DefinitionList(OntologyElement):
+    description: str = Field("A list of terms and their definitions", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.list, frozen=True)
+    allowed_tags: List[str] = Field(["dl"], frozen=True)
+
+
+class ListItem(OntologyElement):
+    description: str = Field("An item in a list", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.list, frozen=True)
+    allowed_tags: List[str] = Field(["li"], frozen=True)
+
+
+class Table(OntologyElement):
+    description: str = Field("A structured set of data", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["table"], frozen=True)
+
+    def to_html(self, add_children: bool = True) -> str:
+        soup = BeautifulSoup(super().to_html(add_children), "html.parser")
+        soup = remove_ids_and_class_from_table(soup)
+        return str(soup)
+
+
+class TableBody(OntologyElement):
+    description: str = Field("A body of the table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["tbody"], frozen=True)
+
+
+class TableHeader(OntologyElement):
+    description: str = Field("A header of the table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["thead"], frozen=True)
+
+
+class TableRow(OntologyElement):
+    description: str = Field("A row in a table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["tr"], frozen=True)
+
+
+class TableCell(OntologyElement):
+    description: str = Field("A cell in a table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["td"], frozen=True)
+
+
+# Note(Pluto): Renamed from TableCellHeader to TableHeaderCell to be consistent with TableCell
+class TableCellHeader(OntologyElement):
+    description: str = Field("A header cell in a table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["th"], frozen=True)
+
+
+class Image(OntologyElement):
+    description: str = Field("A visual representation", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["img"], frozen=True)
+
+
+class Figure(OntologyElement):
+    description: str = Field("An illustration or diagram in a document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["figure"], frozen=True)
+
+
+class Video(OntologyElement):
+    description: str = Field("A moving visual media element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["video"], frozen=True)
+
+
+class Audio(OntologyElement):
+    description: str = Field("A sound or music element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["audio"], frozen=True)
+
+
+class Barcode(OntologyElement):
+    description: str = Field("A machine-readable representation of data", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["img"], frozen=True)
+
+
+class QRCode(OntologyElement):
+    description: str = Field("A two-dimensional barcode", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["img"], frozen=True)
+
+
+class Logo(OntologyElement):
+    description: str = Field("A graphical representation of a company or brand", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["img"], frozen=True)
+
+
+class CodeBlock(OntologyElement):
+    description: str = Field("A block of programming code", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.code, frozen=True)
+    allowed_tags: List[str] = Field(["pre", "code"], frozen=True)
+
+
+class InlineCode(OntologyElement):
+    description: str = Field("Code within a line of text", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.code, frozen=True)
+    allowed_tags: List[str] = Field(["code"], frozen=True)
+
+
+class Formula(OntologyElement):
+    description: str = Field("A mathematical formula", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.mathematical, frozen=True)
+    allowed_tags: List[str] = Field(["math"], frozen=True)
+
+
+class Equation(OntologyElement):
+    description: str = Field("A mathematical equation", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.mathematical, frozen=True)
+    allowed_tags: List[str] = Field(["math"], frozen=True)
+
+
+class FootnoteReference(OntologyElement):
+    description: str = Field(
+        "A subscripted reference to a note at the bottom of a page", frozen=True
+    )
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.reference, frozen=True)
+    allowed_tags: List[str] = Field(["sub"], frozen=True)
+
+
+class Citation(OntologyElement):
+    description: str = Field("A reference to a source", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.reference, frozen=True)
+    allowed_tags: List[str] = Field(["cite"], frozen=True)
+
+
+class Bibliography(OntologyElement):
+    description: str = Field("A list of sources", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.reference, frozen=True)
+    allowed_tags: List[str] = Field(["ul"], frozen=True)
+
+
+class Glossary(OntologyElement):
+    description: str = Field("A list of terms and their definitions", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.reference, frozen=True)
+    allowed_tags: List[str] = Field(["dl"], frozen=True)
+
+
+class Author(OntologyElement):
+    description: str = Field("The creator of the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.metadata, frozen=True)
+    allowed_tags: List[str] = Field(["meta"], frozen=True)
+
+
+class MetaDate(OntologyElement):
+    description: str = Field("The date associated with the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.metadata, frozen=True)
+    allowed_tags: List[str] = Field(["meta"], frozen=True)
+
+
+class Keywords(OntologyElement):
+    description: str = Field("Key terms associated with the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.metadata, frozen=True)
+    allowed_tags: List[str] = Field(["meta"], frozen=True)
+
+
+class Abstract(OntologyElement):
+    description: str = Field("A summary of the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.metadata, frozen=True)
+    allowed_tags: List[str] = Field(["section"], frozen=True)
+
+
+class Hyperlink(OntologyElement):
+    description: str = Field("A reference to data that can be directly followed", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.navigation, frozen=True)
+    allowed_tags: List[str] = Field(["a"], frozen=True)
+
+
+class TableOfContents(OntologyElement):
+    description: str = Field(
+        "A list of the document's contents. Total table columns will be "
+        "equal to the degree of hierarchy (n) plus 1 for the target value. "
+        "Header Row: L1,L2,...Ln,Value",
+        frozen=True,
+    )
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["table"], frozen=True)
+
+    def to_html(self, add_children: bool = True) -> str:
+        soup = BeautifulSoup(super().to_html(add_children), "html.parser")
+        soup = remove_ids_and_class_from_table(soup)
+        return str(soup)
+
+
+class Index(OntologyElement):
+    description: str = Field("An alphabetical list of terms and their page numbers", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.navigation, frozen=True)
+    allowed_tags: List[str] = Field(["nav"], frozen=True)
+
+
+class Form(OntologyElement):
+    description: str = Field("A document section with interactive controls", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["form"], frozen=True)
+
+
+class FormField(OntologyElement):
+    description: str = Field("A property value of a form", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["label"], frozen=True)
+
+
+class FormFieldValue(OntologyElement):
+    description: str = Field("A field for user input", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["input"], frozen=True)
+
+    def to_text(self, add_children: bool = True, add_img_alt_text: bool = True) -> str:
+        text = super().to_text(add_children, add_img_alt_text)
+        value = self.additional_attributes.get("value", "")
+        if not value:
+            return text
+        return f"{text} {value}".strip()
+
+
+class Checkbox(OntologyElement):
+    description: str = Field("A small box that can be checked or unchecked", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["input"], frozen=True)
+
+
+class RadioButton(OntologyElement):
+    description: str = Field("A circular button that can be selected", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["input"], frozen=True)
+
+
+class Button(OntologyElement):
+    description: str = Field("An interactive button element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["button"], frozen=True)
+
+
+class Comment(OntologyElement):
+    description: str = Field("A note or remark", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.annotation, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class Highlight(OntologyElement):
+    description: str = Field("Emphasized text or section", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.annotation, frozen=True)
+    allowed_tags: List[str] = Field(["mark"], frozen=True)
+
+
+class RevisionInsertion(OntologyElement):
+    description: str = Field("A changed or edited element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.annotation, frozen=True)
+    allowed_tags: List[str] = Field(["ins"], frozen=True)
+
+
+class RevisionDeletion(OntologyElement):
+    description: str = Field("A changed or edited element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.annotation, frozen=True)
+    allowed_tags: List[str] = Field(["del"], frozen=True)
+
+
+class Address(OntologyElement):
+    description: str = Field("A physical location", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["address"], frozen=True)
+
+
+class EmailAddress(OntologyElement):
+    description: str = Field("An email address", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["a"], frozen=True)
+
+
+class PhoneNumber(OntologyElement):
+    description: str = Field("A telephone number", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class CalendarDate(OntologyElement):
+    description: str = Field("A calendar date", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["time"], frozen=True)
+
+
+class Time(OntologyElement):
+    description: str = Field("A specific time", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["time"], frozen=True)
+
+
+class Currency(OntologyElement):
+    description: str = Field("A monetary value", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class Measurement(OntologyElement):
+    description: str = Field("A quantitative value with units", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class Letterhead(OntologyElement):
+    description: str = Field("The heading at the top of a letter", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.document_specific, frozen=True)
+    allowed_tags: List[str] = Field(["header"], frozen=True)
+
+
+class Signature(OntologyElement):
+    description: str = Field("A person's name written in a distinctive way", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.document_specific, frozen=True)
+    allowed_tags: List[str] = Field(["img", "svg"], frozen=True)
+
+
+class Watermark(OntologyElement):
+    description: str = Field("A faint design made in paper during manufacture", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.document_specific, frozen=True)
+    allowed_tags: List[str] = Field(["div"], frozen=True)
+
+
+class Stamp(OntologyElement):
+    description: str = Field("An official mark or seal", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.document_specific, frozen=True)
+    allowed_tags: List[str] = Field(["img", "svg"], frozen=True)