chore: import upstream snapshot with attribution

test_unstructured/partition/common/test_common.py

@@ -0,0 +1,468 @@
+import pathlib
+from multiprocessing import Pool
+
+import numpy as np
+import pytest
+from PIL import Image
+from unstructured_inference.constants import IsExtracted
+from unstructured_inference.inference import layout
+from unstructured_inference.inference.elements import TextRegion
+from unstructured_inference.inference.layoutelement import LayoutElement
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.documents.coordinates import PixelSpace
+from unstructured.documents.elements import (
+    TYPE_TO_TEXT_ELEMENT_MAP,
+    CheckBox,
+    CoordinatesMetadata,
+    ElementType,
+    FigureCaption,
+    Header,
+    ListItem,
+    NarrativeText,
+    Text,
+    Title,
+)
+from unstructured.documents.elements import (
+    Image as ImageElement,
+)
+from unstructured.partition.common import common
+
+
+class MockPageLayout(layout.PageLayout):
+    def __init__(self, number: int, image: Image.Image):
+        self.number = number
+        self.image = image
+
+    @property
+    def elements(self):
+        return [
+            LayoutElement(
+                type="Headline",
+                text="Charlie Brown and the Great Pumpkin",
+                bbox=None,
+            ),
+            LayoutElement(
+                type="Subheadline",
+                text="The Beginning",
+                bbox=None,
+            ),
+            LayoutElement(
+                type="Text",
+                text="This time Charlie Brown had it really tricky...",
+                bbox=None,
+            ),
+            LayoutElement(
+                type="Title",
+                text="Another book title in the same page",
+                bbox=None,
+            ),
+        ]
+
+
+class MockDocumentLayout(layout.DocumentLayout):
+    @property
+    def pages(self):
+        return [
+            MockPageLayout(number=1, image=Image.new("1", (1, 1))),
+        ]
+
+
+def test_normalize_layout_element_dict():
+    layout_element = {
+        "type": "Title",
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "coordinate_system": None,
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == Title(
+        text="Some lovely text",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_dict_caption():
+    layout_element = {
+        "type": "Figure",
+        "coordinates": ((1, 2), (3, 4), (5, 6), (7, 8)),
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == ImageElement(
+        text="Some lovely text",
+        coordinates=((1, 2), (3, 4), (5, 6), (7, 8)),
+        coordinate_system=coordinate_system,
+    )
+
+
+@pytest.mark.parametrize(
+    ("element_type", "expected_type", "expected_depth"),
+    [
+        ("Title", Title, None),
+        ("Headline", Title, 1),
+        ("Subheadline", Title, 2),
+        ("Header", Header, None),
+    ],
+)
+def test_normalize_layout_element_headline(element_type, expected_type, expected_depth):
+    layout_element = {
+        "type": element_type,
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(layout_element, coordinate_system=coordinate_system)
+    assert element.metadata.category_depth == expected_depth
+    assert isinstance(element, expected_type)
+
+
+def test_normalize_layout_element_dict_figure_caption():
+    layout_element = {
+        "type": "FigureCaption",
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == FigureCaption(
+        text="Some lovely text",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_dict_misc():
+    layout_element = {
+        "type": "Misc",
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == Text(
+        text="Some lovely text",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_layout_element():
+    layout_element = LayoutElement.from_coords(
+        type="Text",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == NarrativeText(
+        text="Some lovely text",
+        coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_layout_element_narrative_text():
+    layout_element = LayoutElement.from_coords(
+        type="NarrativeText",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == NarrativeText(
+        text="Some lovely text",
+        coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+        coordinate_system=coordinate_system,
+    )
+
+
+@pytest.mark.parametrize(
+    ("element_type", "expected_element_class"),
+    TYPE_TO_TEXT_ELEMENT_MAP.items(),
+)
+def test_normalize_layout_element_layout_element_maps_to_appropriate_text_element(
+    element_type: str,
+    expected_element_class: type[Text],
+):
+    layout_element = LayoutElement.from_coords(
+        type=element_type,
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == expected_element_class(
+        text="Some lovely text",
+        coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+        coordinate_system=coordinate_system,
+    )
+
+
+@pytest.mark.parametrize(
+    ("element_type", "expected_checked"),
+    [
+        (ElementType.CHECK_BOX_UNCHECKED, False),
+        (ElementType.CHECK_BOX_CHECKED, True),
+        (ElementType.RADIO_BUTTON_UNCHECKED, False),
+        (ElementType.RADIO_BUTTON_CHECKED, True),
+        (ElementType.CHECKED, True),
+        (ElementType.UNCHECKED, False),
+    ],
+)
+def test_normalize_layout_element_checkable(element_type: str, expected_checked: bool):
+    layout_element = LayoutElement.from_coords(
+        type=element_type,
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert isinstance(element, CheckBox)
+    assert element == CheckBox(
+        checked=expected_checked,
+        coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_enumerated_list():
+    layout_element = LayoutElement.from_coords(
+        type="List",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="1. I'm so cool! 2. You're cool too. 3. We're all cool!",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    elements = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert elements == [
+        ListItem(
+            text="I'm so cool!",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+        ListItem(
+            text="You're cool too.",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+        ListItem(
+            text="We're all cool!",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+    ]
+
+
+def test_normalize_layout_element_bulleted_list():
+    layout_element = LayoutElement.from_coords(
+        type="List",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="* I'm so cool! * You're cool too. * We're all cool!",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    elements = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert elements == [
+        ListItem(
+            text="I'm so cool!",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+        ListItem(
+            text="You're cool too.",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+        ListItem(
+            text="We're all cool!",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+    ]
+
+
+class MockRunOutput:
+    def __init__(self, returncode, stdout, stderr):
+        self.returncode = returncode
+        self.stdout = stdout
+        self.stderr = stderr
+
+
+def test_convert_office_doc_captures_errors(monkeypatch, caplog):
+    from unstructured.partition.common.common import subprocess
+
+    def mock_run(*args, **kwargs):
+        return MockRunOutput(1, "an error occurred".encode(), "error details".encode())
+
+    monkeypatch.setattr(subprocess, "run", mock_run)
+    common.convert_office_doc("no-real.docx", "fake-directory", target_format="docx")
+    assert "soffice failed to convert to format docx with code 1" in caplog.text
+
+
+def test_convert_office_docs_avoids_concurrent_call_to_soffice():
+    paths_to_save = [pathlib.Path(path) for path in ("/tmp/proc1", "/tmp/proc2", "/tmp/proc3")]
+    for path in paths_to_save:
+        path.mkdir(exist_ok=True)
+        (path / "simple.docx").unlink(missing_ok=True)
+    file_to_convert = example_doc_path("simple.doc")
+
+    with Pool(3) as pool:
+        pool.starmap(common.convert_office_doc, [(file_to_convert, path) for path in paths_to_save])
+
+    assert np.sum([(path / "simple.docx").is_file() for path in paths_to_save]) == 3
+
+
+def test_convert_office_docs_respects_wait_timeout():
+    paths_to_save = [
+        pathlib.Path(path) for path in ("/tmp/wait/proc1", "/tmp/wait/proc2", "/tmp/wait/proc3")
+    ]
+    for path in paths_to_save:
+        path.mkdir(parents=True, exist_ok=True)
+        (path / "simple.docx").unlink(missing_ok=True)
+    file_to_convert = example_doc_path("simple.doc")
+
+    with Pool(3) as pool:
+        pool.starmap(
+            common.convert_office_doc,
+            # set timeout to wait for soffice to be available to 0 so only one process can convert
+            # the doc file on the first try; then the catch all
+            [(file_to_convert, path, "docx", None, 0) for path in paths_to_save],
+        )
+
+    # because this test file is very small we could have occasions where two files are converted
+    # when one of the processes spawned just a little
+    assert np.sum([(path / "simple.docx").is_file() for path in paths_to_save]) < 3
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("<table><tbody><tr><td>👨\\U+1F3FB🔧</td></tr></tbody></table>", True),
+        ("<table><tbody><tr><td>Hello!</td></tr></tbody></table>", False),
+    ],
+)
+def test_contains_emoji(text, expected):
+    assert common.contains_emoji(text) is expected
+
+
+def test_get_page_image_metadata_and_coordinate_system():
+    doc = MockDocumentLayout()
+    metadata = common.get_page_image_metadata(doc.pages[0])
+    assert isinstance(metadata, dict)
+
+
+def test_ocr_data_to_elements():
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    text_regions = [
+        TextRegion.from_coords(
+            163.0,
+            115.0,
+            452.0,
+            129.0,
+            text="LayoutParser: A Unified Toolkit for Deep",
+        ),
+        TextRegion.from_coords(
+            156.0,
+            132.0,
+            457.0,
+            147.0,
+            text="Learning Based Document Image Analysis",
+        ),
+    ]
+    ocr_data = [
+        LayoutElement(
+            bbox=r.bbox,
+            text=r.text,
+            source=r.source,
+            type=ElementType.UNCATEGORIZED_TEXT,
+        )
+        for r in text_regions
+    ]
+    image = Image.open(filename)
+
+    elements = common.ocr_data_to_elements(
+        ocr_data=ocr_data,
+        image_size=image.size,
+    )
+
+    assert len(ocr_data) == len(elements)
+    assert {el.category for el in elements} == {ElementType.UNCATEGORIZED_TEXT}
+
+    # check coordinates metadata
+    image_width, image_height = image.size
+    coordinate_system = PixelSpace(width=image_width, height=image_height)
+    for el, layout_el in zip(elements, ocr_data):
+        assert el.metadata.coordinates == CoordinatesMetadata(
+            points=layout_el.bbox.coordinates,
+            system=coordinate_system,
+        )
+
+
+def test_normalize_layout_element_layout_element_text_source_metadata():
+    layout_element = LayoutElement.from_coords(
+        type="NarrativeText",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+        is_extracted=IsExtracted.TRUE,
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert hasattr(element, "metadata")
+    assert hasattr(element.metadata, "is_extracted")
+    assert element.metadata.is_extracted == "true"

test_unstructured/partition/common/test_lang.py

@@ -0,0 +1,321 @@
+# pyright: reportPrivateUsage=false
+
+"""Unit-test suite for the `unstructured.partition.lang` module."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+
+import pytest
+
+from test_unstructured.unit_utils import LogCaptureFixture
+from unstructured.documents.elements import (
+    NarrativeText,
+    PageBreak,
+)
+from unstructured.partition.common.lang import (
+    _clean_ocr_languages_arg,
+    _convert_language_code_to_pytesseract_lang_code,
+    apply_lang_metadata,
+    check_language_args,
+    detect_languages,
+    prepare_languages_for_tesseract,
+    tesseract_to_paddle_language,
+)
+
+DIRECTORY = pathlib.Path(__file__).parent.resolve()
+EXAMPLE_DOCS_DIRECTORY = os.path.join(DIRECTORY, "..", "..", "example-docs")
+
+
+def test_prepare_languages_for_tesseract_with_one_language():
+    languages = ["en"]
+    assert prepare_languages_for_tesseract(languages) == "eng"
+
+
+def test_prepare_languages_for_tesseract_with_duplicated_languages():
+    languages = ["en", "eng"]
+    assert prepare_languages_for_tesseract(languages) == "eng"
+
+
+def test_prepare_languages_for_tesseract_special_case():
+    languages = ["osd"]
+    assert prepare_languages_for_tesseract(languages) == "osd"
+
+    languages = ["equ"]
+    assert prepare_languages_for_tesseract(languages) == "equ"
+
+
+def test_prepare_languages_for_tesseract_removes_empty_inputs():
+    languages = ["kbd", "es"]
+    assert prepare_languages_for_tesseract(languages) == "spa+spa_old"
+
+
+def test_prepare_languages_for_tesseract_includes_variants():
+    languages = ["chi"]
+    assert prepare_languages_for_tesseract(languages) == "chi_sim+chi_sim_vert+chi_tra+chi_tra_vert"
+
+
+def test_prepare_languages_for_tesseract_with_multiple_languages():
+    languages = ["ja", "afr", "en", "equ"]
+    assert prepare_languages_for_tesseract(languages) == "jpn+jpn_vert+afr+eng+equ"
+
+
+def test_prepare_languages_for_tesseract_warns_nonstandard_language(caplog: LogCaptureFixture):
+    languages = ["zzz", "chi"]
+    assert prepare_languages_for_tesseract(languages) == "chi_sim+chi_sim_vert+chi_tra+chi_tra_vert"
+    assert "not a valid standard language code" in caplog.text
+
+
+def test_prepare_languages_for_tesseract_warns_non_tesseract_language(caplog: LogCaptureFixture):
+    languages = ["kbd", "eng"]
+    assert prepare_languages_for_tesseract(languages) == "eng"
+    assert "not a language supported by Tesseract" in caplog.text
+
+
+def test_prepare_languages_for_tesseract_None_languages():
+    with pytest.raises(ValueError, match="`languages` can not be `None`"):
+        languages = None
+        prepare_languages_for_tesseract(languages)
+
+
+def test_prepare_languages_for_tesseract_no_valid_languages(caplog: LogCaptureFixture):
+    languages = [""]
+    assert prepare_languages_for_tesseract(languages) == "eng"
+    assert "Failed to find any valid standard language code from languages" in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("tesseract_lang", "expected_lang"),
+    [
+        ("eng", "en"),
+        ("chi_sim", "ch"),
+        ("chi_tra", "chinese_cht"),
+        ("deu", "german"),
+        ("jpn", "japan"),
+        ("kor", "korean"),
+    ],
+)
+def test_tesseract_to_paddle_language_valid_codes(tesseract_lang: str, expected_lang: str):
+    assert expected_lang == tesseract_to_paddle_language(tesseract_lang)
+
+
+def test_tesseract_to_paddle_language_invalid_codes(caplog: LogCaptureFixture):
+    tesseract_lang = "unsupported_lang"
+    assert tesseract_to_paddle_language(tesseract_lang) == "en"
+    assert "unsupported_lang is not a language code supported by PaddleOCR," in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("tesseract_lang", "expected_lang"),
+    [
+        ("ENG", "en"),
+        ("Fra", "fr"),
+        ("DEU", "german"),
+    ],
+)
+def test_tesseract_to_paddle_language_case_sensitivity(tesseract_lang: str, expected_lang: str):
+    assert expected_lang == tesseract_to_paddle_language(tesseract_lang)
+
+
+def test_detect_languages_english_auto():
+    text = "This is a short sentence."
+    assert detect_languages(text) == ["eng"]
+
+
+def test_detect_languages_english_provided():
+    text = "This is another short sentence."
+    languages = ["en"]
+    assert detect_languages(text, languages) == ["eng"]
+
+
+def test_detect_languages_korean_auto():
+    text = "안녕하세요"
+    assert detect_languages(text) == ["kor"]
+
+
+def test_detect_languages_gets_multiple_languages():
+    text = "My lubimy mleko i chleb."
+    assert detect_languages(text) == ["ces", "pol", "slk"]
+
+
+def test_detect_languages_warns_for_auto_and_other_input(caplog: LogCaptureFixture):
+    text = "This is another short sentence."
+    languages = ["en", "auto", "rus"]
+    assert detect_languages(text, languages) == ["eng"]
+    assert "rest of the inputted languages will be ignored" in caplog.text
+
+
+def test_detect_languages_raises_TypeError_for_invalid_languages():
+    with pytest.raises(TypeError):
+        text = "This is a short sentence."
+        detect_languages(text, languages="eng") == ["eng"]  # type: ignore
+
+
+def test_apply_lang_metadata_has_no_warning_for_PageBreak(caplog: LogCaptureFixture):
+    elements = [NarrativeText("Sample text."), PageBreak("")]
+    elements = list(
+        apply_lang_metadata(
+            elements=elements,
+            languages=["auto"],
+            detect_language_per_element=True,
+        ),
+    )
+    assert "No features in text." not in [rec.message for rec in caplog.records]
+
+
+@pytest.mark.parametrize(
+    ("lang_in", "expected_lang"),
+    [
+        ("en", "eng"),
+        ("fr", "fra"),
+    ],
+)
+def test_convert_language_code_to_pytesseract_lang_code(lang_in: str, expected_lang: str):
+    assert expected_lang == _convert_language_code_to_pytesseract_lang_code(lang_in)
+
+
+@pytest.mark.parametrize(
+    ("input_ocr_langs", "expected"),
+    [
+        (["eng"], "eng"),  # list
+        ('"deu"', "deu"),  # extra quotation marks
+        ("[deu]", "deu"),  # brackets
+        ("['deu']", "deu"),  # brackets and quotation marks
+        (["[deu]"], "deu"),  # list, brackets and quotation marks
+        (['"deu"'], "deu"),  # list and quotation marks
+        ("deu+spa", "deu+spa"),  # correct input
+    ],
+)
+def test_clean_ocr_languages_arg(input_ocr_langs: str, expected: str):
+    assert _clean_ocr_languages_arg(input_ocr_langs) == expected
+
+
+def test_detect_languages_handles_spelled_out_languages():
+    languages = detect_languages(text="Sample text longer than 5 words.", languages=["Spanish"])
+    assert languages == ["spa"]
+
+
+def test_detect_languages_short_text_fallback_returns_none():
+    """Short ASCII text with language_fallback returning None leaves language unspecified."""
+    result = detect_languages(
+        text="Hi there.",
+        language_fallback=lambda t: None,
+    )
+    assert result is None
+
+
+def test_detect_languages_short_text_fallback_returns_custom():
+    """Short ASCII text triggers fallback; we assert the fallback's return is used as-is."""
+    # Any short (<5 word) ASCII text would hit the fallback; content is irrelevant.
+    result = detect_languages(
+        text="Hi there.",
+        language_fallback=lambda t: ["fra"],
+    )
+    assert result == ["fra"]
+
+
+def test_detect_languages_short_text_default_eng_without_fallback():
+    """Short ASCII text without fallback still defaults to English (backward compat)."""
+    result = detect_languages(text="Hi there.")
+    assert result == ["eng"]
+
+
+def test_apply_lang_metadata_with_language_fallback():
+    """apply_lang_metadata passes language_fallback so short text can return None."""
+    elements = [NarrativeText("Hi.")]
+    result = list(
+        apply_lang_metadata(
+            elements=elements,
+            languages=["auto"],
+            language_fallback=lambda t: None,
+        )
+    )
+    assert len(result) == 1
+    assert result[0].metadata.languages is None
+
+
+@pytest.mark.parametrize(
+    ("languages", "ocr_languages", "expected_langs"),
+    [
+        (["spa"], "deu", ["spa"]),
+        (["spanish"], "english", ["spa"]),
+        (["spa"], "[deu]", ["spa"]),
+        (["spa"], '"deu"', ["spa"]),
+        (["spa"], ["deu"], ["spa"]),
+        (["spa"], ["[deu]"], ["spa"]),
+        (["spa+deu"], "eng+deu", ["spa", "deu"]),
+    ],
+)
+def test_check_language_args_uses_languages_when_ocr_languages_and_languages_are_both_defined(
+    languages: list[str],
+    ocr_languages: list[str] | str,
+    expected_langs: list[str],
+    caplog: LogCaptureFixture,
+):
+    returned_langs = check_language_args(
+        languages=languages,
+        ocr_languages=ocr_languages,
+    )
+    for lang in returned_langs:  # type: ignore
+        assert lang in expected_langs
+        assert "ocr_languages" in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("languages", "ocr_languages", "expected_langs"),
+    [
+        # raise warning and use `ocr_languages` when `languages` is empty or None
+        ([], "deu", ["deu"]),
+        ([""], '"deu"', ["deu"]),
+        ([""], "deu", ["deu"]),
+        ([""], "[deu]", ["deu"]),
+    ],
+)
+def test_check_language_args_uses_ocr_languages_when_languages_is_empty_or_None(
+    languages: list[str],
+    ocr_languages: str,
+    expected_langs: list[str],
+    caplog: LogCaptureFixture,
+):
+    returned_langs = check_language_args(languages=languages, ocr_languages=ocr_languages)
+    for lang in returned_langs:  # type: ignore
+        assert lang in expected_langs
+        assert "ocr_languages" in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("languages", "ocr_languages"),
+    [
+        ([], None),  # how check_language_args is called from auto.partition()
+        ([""], None),
+    ],
+)
+def test_check_language_args_returns_None(
+    languages: list[str],
+    ocr_languages: None,
+):
+    returned_langs = check_language_args(languages=languages, ocr_languages=ocr_languages)
+    assert returned_langs is None
+
+
+def test_check_language_args_returns_auto():
+    assert check_language_args(languages=["eng", "spa", "auto"], ocr_languages=None) == ["auto"]
+
+
+@pytest.mark.parametrize(
+    ("languages", "ocr_languages"),
+    [
+        ([], ["auto"]),
+        ([""], "eng+auto"),
+    ],
+)
+def test_check_language_args_raises_error_when_ocr_languages_contains_auto(
+    languages: list[str],
+    ocr_languages: str | list[str],
+):
+    with pytest.raises(ValueError):
+        check_language_args(
+            languages=languages,
+            ocr_languages=ocr_languages,
+        )

test_unstructured/partition/common/test_metadata.py

@@ -0,0 +1,540 @@
+"""Test-suite for `unstructured.partition.common.metadata` module."""
+
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import copy
+import datetime as dt
+import os
+import pathlib
+from typing import Any, Callable
+
+import pytest
+
+from unstructured.documents.elements import (
+    CheckBox,
+    Element,
+    ElementMetadata,
+    FigureCaption,
+    Header,
+    ListItem,
+    NarrativeText,
+    Text,
+    Title,
+)
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.metadata import (
+    _assign_hash_ids,
+    apply_metadata,
+    get_last_modified_date,
+    set_element_hierarchy,
+)
+
+# ================================================================================================
+# LAST-MODIFIED
+# ================================================================================================
+
+
+class Describe_get_last_modified_date:
+    def it_gets_the_modified_time_of_a_file_identified_by_a_path(self, tmp_path: pathlib.Path):
+        modified_timestamp = dt.datetime(
+            year=2024, month=3, day=5, hour=17, minute=43, second=40
+        ).timestamp()
+        file_path = tmp_path / "some_file.txt"
+        file_path.write_text("abcdefg")
+        os.utime(file_path, (modified_timestamp, modified_timestamp))
+
+        last_modified_date = get_last_modified_date(str(file_path))
+
+        assert last_modified_date == "2024-03-05T17:43:40"
+
+    def but_it_returns_None_when_there_is_no_file_at_that_path(self, tmp_path: pathlib.Path):
+        file_path = tmp_path / "some_file_that_does_not_exist.txt"
+
+        last_modified_date = get_last_modified_date(str(file_path))
+
+        assert last_modified_date is None
+
+
+# ================================================================================================
+# ELEMENT HIERARCHY
+# ================================================================================================
+
+
+class Describe_set_element_hierarchy:
+    def it_applies_default_ruleset(self):
+        elements = [
+            Title(element_id="0", text="Title0"),
+            Text(element_id="1", text="Text0"),
+            Header(element_id="2", text="Header0"),
+            Text(element_id="3", text="Text1"),
+            Title(element_id="4", text="Title1"),
+            Text(element_id="5", text="Text2"),
+        ]
+
+        result = set_element_hierarchy(elements)
+
+        assert result[0].metadata.parent_id is None
+        assert result[1].metadata.parent_id == "0"  # Text0 is under Title0
+        assert result[2].metadata.parent_id is None  # Header0 is higher than Title0
+        assert result[3].metadata.parent_id == "2"  # Text1 is under Header0
+        assert result[4].metadata.parent_id == "2"  # Title1 is under Header0
+        assert result[5].metadata.parent_id == "4"  # Text2 is under Title1, which is under Header0
+
+    def it_applies_category_depth_when_element_category_is_the_same(self):
+        elements = [
+            Title(element_id="0", text="Title0", metadata=ElementMetadata(category_depth=1)),
+            ListItem(element_id="1", text="ListItem0", metadata=ElementMetadata(category_depth=0)),
+            ListItem(element_id="2", text="ListItem1", metadata=ElementMetadata(category_depth=1)),
+            ListItem(element_id="3", text="ListItem2", metadata=ElementMetadata(category_depth=0)),
+        ]
+
+        result = set_element_hierarchy(elements)
+
+        assert result[0].metadata.parent_id is None
+        assert result[1].metadata.parent_id == "0"  # category_depth=0
+        assert result[2].metadata.parent_id == "1"  # category_depth=1, so it is under ListItem0
+        assert result[3].metadata.parent_id == "0"  # category_depth=0
+
+    def but_it_ignores_category_depth_when_elements_are_of_different_categories(self):
+        elements = [
+            Title(element_id="0", text="Title", metadata=ElementMetadata(category_depth=2)),
+            Text(element_id="1", text="Text", metadata=ElementMetadata(category_depth=0)),
+            Header(element_id="2", text="Header", metadata=ElementMetadata(category_depth=2)),
+            Text(element_id="3", text="Text", metadata=ElementMetadata(category_depth=0)),
+            ListItem(element_id="4", text="ListItem", metadata=ElementMetadata(category_depth=1)),
+            NarrativeText(element_id="5", text="", metadata=ElementMetadata(category_depth=0)),
+        ]
+
+        result = set_element_hierarchy(elements)
+
+        assert result[0].metadata.parent_id is None
+        assert result[1].metadata.parent_id == "0"  # Text is under Title despite category_depth=0
+        assert result[2].metadata.parent_id is None
+        assert result[3].metadata.parent_id == "2"  # These are under Header despite category_depth
+        assert result[4].metadata.parent_id == "2"
+        assert result[5].metadata.parent_id == "2"
+
+    def it_skips_elements_with_pre_existing_parent_id(self):
+        elements = [
+            Title(element_id="0", text="Title", metadata=ElementMetadata(parent_id="10")),
+            Title(element_id="1", text="Title"),
+            Text(element_id="2", text="Text"),
+        ]
+
+        result = set_element_hierarchy(elements)
+
+        # Parent ID should not change and element is skipped in figuring out other elements' parents
+        assert result[0].metadata.parent_id == "10"
+        assert result[1].metadata.parent_id is None
+        assert result[2].metadata.parent_id == "1"
+
+    def it_sets_parent_id_for_each_element_in_elements(self):
+        elements_to_set = [
+            Title(text="Title"),  # 0
+            NarrativeText(text="NarrativeText"),  # 1
+            FigureCaption(text="FigureCaption"),  # 2
+            ListItem(text="ListItem"),  # 3
+            ListItem(text="ListItem", metadata=ElementMetadata(category_depth=1)),  # 4
+            ListItem(text="ListItem", metadata=ElementMetadata(category_depth=1)),  # 5
+            ListItem(text="ListItem"),  # 6
+            CheckBox(element_id="some-id-1", checked=True),  # 7
+            Title(text="Title 2"),  # 8
+            ListItem(text="ListItem"),  # 9
+            ListItem(text="ListItem"),  # 10
+            Text(text="Text"),  # 11
+        ]
+        elements = set_element_hierarchy(elements_to_set)
+
+        assert elements[1].metadata.parent_id == elements[0].id, (
+            "NarrativeText should be child of Title"
+        )
+        assert elements[2].metadata.parent_id == elements[0].id, (
+            "FigureCaption should be child of Title"
+        )
+        assert elements[3].metadata.parent_id == elements[0].id, "ListItem should be child of Title"
+        assert elements[4].metadata.parent_id == elements[3].id, "ListItem should be child of Title"
+        assert elements[5].metadata.parent_id == elements[3].id, "ListItem should be child of Title"
+        assert elements[6].metadata.parent_id == elements[0].id, "ListItem should be child of Title"
+        # NOTE(Hubert): moving the category field to Element, caused this to fail.
+        # Checkboxes will soon be deprecated, then we can remove the test.
+        # assert (
+        #         elements[7].metadata.parent_id is None
+        # ), "CheckBox should be None, as it's not a Text based element"
+        assert elements[8].metadata.parent_id is None, "Title 2 should be child of None"
+        assert elements[9].metadata.parent_id == elements[8].id, (
+            "ListItem should be child of Title 2"
+        )
+        assert elements[10].metadata.parent_id == elements[8].id, (
+            "ListItem should be child of Title 2"
+        )
+        assert elements[11].metadata.parent_id == elements[8].id, "Text should be child of Title 2"
+
+    def it_applies_custom_rule_set(self):
+        elements_to_set = [
+            Header(text="Header"),  # 0
+            Title(text="Title"),  # 1
+            NarrativeText(text="NarrativeText"),  # 2
+            Text(text="Text"),  # 3
+            Title(text="Title 2"),  # 4
+            FigureCaption(text="FigureCaption"),  # 5
+        ]
+
+        custom_rule_set = {
+            "Header": ["Title", "Text"],
+            "Title": ["NarrativeText", "UncategorizedText", "FigureCaption"],
+        }
+
+        elements = set_element_hierarchy(
+            elements=elements_to_set,
+            ruleset=custom_rule_set,
+        )
+
+        assert elements[1].metadata.parent_id == elements[0].id, "Title should be child of Header"
+        assert elements[2].metadata.parent_id == elements[1].id, (
+            "NarrativeText should be child of Title"
+        )
+        assert elements[3].metadata.parent_id == elements[1].id, "Text should be child of Title"
+        assert elements[4].metadata.parent_id == elements[0].id, "Title 2 should be child of Header"
+        assert elements[5].metadata.parent_id == elements[4].id, (
+            "FigureCaption should be child of Title 2"
+        )
+
+
+# ================================================================================================
+# APPLY METADATA DECORATOR
+# ================================================================================================
+
+
+class Describe_apply_metadata:
+    """Unit-test suite for `unstructured.partition.common.metadata.apply_metadata()` decorator."""
+
+    # -- unique-ify elements and metadata ---------------------------------
+
+    def it_produces_unique_elements_and_metadata_when_input_reuses_element_instances(self):
+        element = Text(text="Element", metadata=ElementMetadata(filename="foo.bar", page_number=1))
+
+        def fake_partitioner(**kwargs: Any) -> list[Element]:
+            return [element, element, element]
+
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        # -- all elements are unique instances --
+        assert len({id(e) for e in elements}) == len(elements)
+        # -- all metadatas are unique instances --
+        assert len({id(e.metadata) for e in elements}) == len(elements)
+
+    def and_it_produces_unique_elements_and_metadata_when_input_reuses_metadata_instances(self):
+        metadata = ElementMetadata(filename="foo.bar", page_number=1)
+
+        def fake_partitioner(**kwargs: Any) -> list[Element]:
+            return [
+                Text(text="foo", metadata=metadata),
+                Text(text="bar", metadata=metadata),
+                Text(text="baz", metadata=metadata),
+            ]
+
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        # -- all elements are unique instances --
+        assert len({id(e) for e in elements}) == len(elements)
+        # -- all metadatas are unique instances --
+        assert len({id(e.metadata) for e in elements}) == len(elements)
+
+    # -- unique-ids -------------------------------------------------------
+
+    def it_assigns_hash_element_ids_when_unique_ids_arg_is_not_specified(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+        elements_2 = partition()
+
+        # -- SHA1 hash is 32 characters long, no hyphens --
+        assert all(len(e.id) == 32 for e in elements)
+        assert all("-" not in e.id for e in elements)
+        # -- SHA1 hashes are deterministic --
+        assert all(e.id == e2.id for e, e2 in zip(elements, elements_2))
+
+    def it_assigns_hash_element_ids_when_unique_ids_arg_is_False(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(unique_element_ids=False)
+        elements_2 = partition(unique_element_ids=False)
+
+        # -- SHA1 hash is 32 characters long, no hyphens --
+        assert all(len(e.id) == 32 for e in elements)
+        assert all("-" not in e.id for e in elements)
+        # -- SHA1 hashes are deterministic --
+        assert all(e.id == e2.id for e, e2 in zip(elements, elements_2))
+
+    def it_leaves_UUID_element_ids_when_unique_ids_arg_is_True(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(unique_element_ids=True)
+        elements_2 = partition(unique_element_ids=True)
+
+        # -- UUID is 36 characters long with four hyphens --
+        assert all(len(e.id) == 36 for e in elements)
+        assert all(e.id.count("-") == 4 for e in elements)
+        # -- UUIDs are non-deterministic, different every time --
+        assert all(e.id != e2.id for e, e2 in zip(elements, elements_2))
+
+    # -- parent-id --------------------------------------------------------
+
+    def it_computes_and_assigns_parent_id(self, fake_partitioner: Callable[..., list[Element]]):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        title = elements[0]
+        assert title.metadata.category_depth == 1
+        narr_text = elements[1]
+        assert narr_text.metadata.parent_id == title.id
+
+    # -- languages --------------------------------------------------------
+
+    def it_applies_language_metadata(self, fake_partitioner: Callable[..., list[Element]]):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(languages=["auto"], detect_language_per_element=True)
+
+        assert all(e.metadata.languages == ["eng"] for e in elements)
+
+    # -- filetype (MIME-type) ---------------------------------------------
+
+    def it_assigns_the_value_of_a_metadata_file_type_arg_when_there_is_one(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """A `metadata_file_type` arg overrides the file-type specified in the decorator.
+
+        This is used for example by a delegating partitioner to preserve the original file-type in
+        the metadata, like EPUB instead of the HTML that partitioner converts the .epub file to.
+        """
+        partition = apply_metadata(file_type=FileType.DOCX)(fake_partitioner)
+
+        elements = partition(metadata_file_type=FileType.ODT)
+
+        assert all(
+            e.metadata.filetype == "application/vnd.oasis.opendocument.text" for e in elements
+        )
+
+    def and_it_assigns_the_decorator_file_type_when_the_metadata_file_type_arg_is_omitted(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """The `file_type=...` decorator arg is the "normal" way to specify the file-type.
+
+        This is used for principal (non-delegating) partitioners.
+        """
+        partition = apply_metadata(file_type=FileType.DOCX)(fake_partitioner)
+
+        elements = partition()
+
+        DOCX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        assert all(e.metadata.filetype == DOCX_MIME_TYPE for e in elements)
+
+    def and_it_does_not_assign_file_type_metadata_when_both_are_omitted(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """A partitioner can elect to assign `.metadata.filetype` for itself.
+
+        This is done in `partition_image()` for example where the same partitioner is used for
+        multiple file-types.
+        """
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        assert all(e.metadata.filetype == "image/jpeg" for e in elements)
+
+    # -- filename ---------------------------------------------------------
+
+    def it_uses_metadata_filename_arg_value_when_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """A `metadata_filename` arg overrides all other sources."""
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(metadata_filename="a/b/c.xyz")
+
+        assert all(e.metadata.filename == "c.xyz" for e in elements)
+        assert all(e.metadata.file_directory == "a/b" for e in elements)
+
+    def and_it_uses_filename_arg_value_when_metadata_filename_arg_not_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(filename="a/b/c.xyz")
+
+        assert all(e.metadata.filename == "c.xyz" for e in elements)
+        assert all(e.metadata.file_directory == "a/b" for e in elements)
+
+    def and_it_does_not_assign_filename_metadata_when_neither_are_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        assert all(e.metadata.filename == "image.jpeg" for e in elements)
+        assert all(e.metadata.file_directory == "x/y/images" for e in elements)
+
+    # -- last_modified ----------------------------------------------------
+
+    def it_uses_metadata_last_modified_arg_value_when_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """A `metadata_last_modified` arg overrides all other sources."""
+        partition = apply_metadata()(fake_partitioner)
+        metadata_last_modified = "2024-09-26T15:17:53"
+
+        elements = partition(metadata_last_modified=metadata_last_modified)
+
+        assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+    @pytest.mark.parametrize("kwargs", [{}, {"metadata_last_modified": None}])
+    def but_it_does_not_update_last_modified_when_metadata_last_modified_arg_absent_or_None(
+        self, kwargs: dict[str, Any], fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(**kwargs)
+
+        assert all(e.metadata.last_modified == "2020-01-06T05:07:03" for e in elements)
+
+    # -- url --------------------------------------------------------------
+
+    def it_assigns_url_metadata_field_when_url_arg_is_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(url="https://adobe.com/stock/54321")
+
+        assert all(e.metadata.url == "https://adobe.com/stock/54321" for e in elements)
+
+    def and_it_does_not_assign_url_metadata_when_url_arg_is_not_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        assert all(e.metadata.url == "http://images.com" for e in elements)
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture
+    def fake_partitioner(self) -> Callable[..., list[Element]]:
+        def fake_partitioner(**kwargs: Any) -> list[Element]:
+            title = Title("Introduction")
+            title.metadata.category_depth = 1
+            title.metadata.file_directory = "x/y/images"
+            title.metadata.filename = "image.jpeg"
+            title.metadata.filetype = "image/jpeg"
+            title.metadata.last_modified = "2020-01-06T05:07:03"
+            title.metadata.url = "http://images.com"
+
+            narr_text = NarrativeText("To understand bar you must first understand foo.")
+            narr_text.metadata.file_directory = "x/y/images"
+            narr_text.metadata.filename = "image.jpeg"
+            narr_text.metadata.filetype = "image/jpeg"
+            narr_text.metadata.last_modified = "2020-01-06T05:07:03"
+            narr_text.metadata.url = "http://images.com"
+
+            return [title, narr_text]
+
+        return fake_partitioner
+
+
+# ================================================================================================
+# HASH IDS
+# ================================================================================================
+
+
+def test_assign_hash_ids_produces_unique_and_deterministic_SHA1_ids_even_for_duplicate_elements():
+    elements: list[Element] = [
+        Text(text="Element", metadata=ElementMetadata(filename="foo.bar", page_number=1)),
+        Text(text="Element", metadata=ElementMetadata(filename="foo.bar", page_number=1)),
+        Text(text="Element", metadata=ElementMetadata(filename="foo.bar", page_number=1)),
+    ]
+    # -- default ids are UUIDs --
+    assert all(len(e.id) == 36 for e in elements)
+
+    elements = _assign_hash_ids(copy.deepcopy(elements))
+    elements_2 = _assign_hash_ids(copy.deepcopy(elements))
+
+    ids = [e.id for e in elements]
+    # -- ids are now SHA1 --
+    assert all(len(e.id) == 32 for e in elements)
+    # -- each id is unique --
+    assert len(ids) == len(set(ids))
+    # -- ids are deterministic, same value is computed each time --
+    assert all(e.id == e2.id for e, e2 in zip(elements, elements_2))
+
+
+def test_assign_hash_ids_remaps_parent_id_to_new_hash_id():
+    """parent_id values (originally UUIDs) are updated to the corresponding hash IDs."""
+    title = Title(text="Title", metadata=ElementMetadata(filename="foo.bar", page_number=1))
+    child = Text(
+        text="Child",
+        metadata=ElementMetadata(filename="foo.bar", page_number=1, parent_id=title.id),
+    )
+    # -- sanity-check: ids are UUIDs before hashing --
+    assert len(title.id) == 36
+    assert child.metadata.parent_id == title.id
+
+    _assign_hash_ids([title, child])
+
+    # -- ids are now SHA1 hashes --
+    assert len(title.id) == 32
+    # -- parent_id has been updated to the new hash id, not the old UUID --
+    assert child.metadata.parent_id == title.id
+
+
+def test_assign_hash_ids_leaves_unknown_parent_id_unchanged():
+    """A parent_id that has no matching element (e.g. filtered out) is left as-is, not KeyError."""
+    external_parent_id = "some-external-or-filtered-id"
+    orphan = Text(
+        text="Orphan",
+        metadata=ElementMetadata(filename="foo.bar", page_number=1, parent_id=external_parent_id),
+    )
+
+    # -- should not raise KeyError even though external_parent_id is not in id_mapping --
+    _assign_hash_ids([orphan])
+
+    # -- parent_id is left unchanged because it wasn't in the mapping --
+    assert orphan.metadata.parent_id == external_parent_id
+
+
+def test_partition_html_parent_child_relationships_preserved_with_hash_ids():
+    """Integration: partition_html with unique_element_ids=False preserves parent-child links."""
+    from unstructured.partition.html import partition_html
+
+    html = "<html><body><h1>My Title</h1><p>My paragraph</p></body></html>"
+    elements = partition_html(text=html, unique_element_ids=False)
+
+    # -- all element ids should be SHA1 hashes (32 hex chars), not UUIDs (36 chars) --
+    assert all(len(e.id) == 32 for e in elements), "Expected SHA1 hash IDs"
+
+    # -- find the title and its child paragraph --
+    title = next((e for e in elements if isinstance(e, Title)), None)
+    child = next(
+        (e for e in elements if e.metadata.parent_id is not None),
+        None,
+    )
+    assert title is not None, "Expected a Title element"
+    assert child is not None, "Expected at least one element with a parent_id"
+    # -- parent_id must point to the hashed title id, not an old UUID --
+    assert child.metadata.parent_id == title.id