Files
wehub-resource-sync 461bf6fd40
CI / lint (3.11) (push) Has been cancelled
CI / lint (3.12) (push) Has been cancelled
CI / lint (3.13) (push) Has been cancelled
CI / shellcheck (push) Has been cancelled
CI / shfmt (push) Has been cancelled
CI / setup (3.11) (push) Has been cancelled
CI / setup (3.12) (push) Has been cancelled
CI / setup (3.13) (push) Has been cancelled
CI / check-licenses (3.12) (push) Has been cancelled
CI / test_unit (3.11) (push) Has been cancelled
CI / test_unit (3.12) (push) Has been cancelled
CI / test_unit (3.13) (push) Has been cancelled
CI / test_unit_no_extras (3.11) (push) Has been cancelled
CI / test_unit_no_extras (3.12) (push) Has been cancelled
CI / test_json_to_html (3.12) (push) Has been cancelled
CI / test_unit_no_extras (3.13) (push) Has been cancelled
CI / test_unit_dependency_extras (csv, 3.12, --extra csv) (push) Has been cancelled
CI / test_unit_dependency_extras (xlsx, 3.11, --extra xlsx) (push) Has been cancelled
CI / test_unit_dependency_extras (xlsx, 3.12, --extra xlsx) (push) Has been cancelled
CI / test_unit_dependency_extras (csv, 3.11, --extra csv) (push) Has been cancelled
CI / test_unit_dependency_extras (csv, 3.13, --extra csv) (push) Has been cancelled
CI / test_unit_dependency_extras (docx, 3.11, --extra docx) (push) Has been cancelled
CI / test_unit_dependency_extras (docx, 3.12, --extra docx) (push) Has been cancelled
CI / test_unit_dependency_extras (docx, 3.13, --extra docx) (push) Has been cancelled
CI / test_unit_dependency_extras (markdown, 3.11, --extra md) (push) Has been cancelled
CI / test_unit_dependency_extras (markdown, 3.12, --extra md) (push) Has been cancelled
CI / test_unit_dependency_extras (markdown, 3.13, --extra md) (push) Has been cancelled
CI / test_unit_dependency_extras (odt, 3.11, --extra odt) (push) Has been cancelled
CI / test_unit_dependency_extras (odt, 3.12, --extra odt) (push) Has been cancelled
CI / test_unit_dependency_extras (odt, 3.13, --extra odt) (push) Has been cancelled
CI / test_unit_dependency_extras (pdf-image, 3.11, --extra pdf --extra image --extra paddleocr) (push) Has been cancelled
CI / test_unit_dependency_extras (pdf-image, 3.12, --extra pdf --extra image --extra paddleocr) (push) Has been cancelled
CI / test_unit_dependency_extras (pdf-image, 3.13, --extra pdf --extra image --extra paddleocr) (push) Has been cancelled
CI / test_unit_dependency_extras (pptx, 3.11, --extra pptx) (push) Has been cancelled
CI / test_unit_dependency_extras (pptx, 3.12, --extra pptx) (push) Has been cancelled
CI / test_unit_dependency_extras (pptx, 3.13, --extra pptx) (push) Has been cancelled
CI / test_unit_dependency_extras (pypandoc, 3.11, --extra epub --extra org --extra rtf --extra rst) (push) Has been cancelled
CI / test_unit_dependency_extras (pypandoc, 3.12, --extra epub --extra org --extra rtf --extra rst) (push) Has been cancelled
CI / test_unit_dependency_extras (pypandoc, 3.13, --extra epub --extra org --extra rtf --extra rst) (push) Has been cancelled
Build And Push Docker Image / set-short-sha (push) Has been cancelled
Partition Benchmark / setup (push) Has been cancelled
Partition Benchmark / Measure and compare partition() runtime (push) Has been cancelled
CI / test_unit_dependency_extras (xlsx, 3.13, --extra xlsx) (push) Has been cancelled
CI / test_ingest_src (3.12) (push) Has been cancelled
CI / test_json_to_markdown (3.12) (push) Has been cancelled
CI / changelog (push) Has been cancelled
CI / test_dockerfile (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Build And Push Docker Image / build-images (linux/amd64, opensource-linux-8core) (push) Has been cancelled
Build And Push Docker Image / build-images (linux/arm64, ubuntu-24.04-arm) (push) Has been cancelled
Build And Push Docker Image / publish-images (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 13:33:56 +08:00

792 lines
27 KiB
Python

from importlib import reload
from unittest.mock import MagicMock, patch
from pdfminer.layout import LTChar, LTContainer, LTFigure, LTLayoutContainer, LTTextLine
from pdfminer.pdftypes import PDFStream
from test_unstructured.unit_utils import example_doc_path
from unstructured.partition.pdf import partition_pdf
from unstructured.partition.pdf_image.pdfminer_utils import (
CustomPDFPageInterpreter,
_is_duplicate_char,
deduplicate_chars_in_text_line,
extract_text_objects,
get_text_with_deduplication,
)
from unstructured.partition.utils import config as partition_config
def _make_char():
return LTChar(
matrix=(1, 0, 0, 1, 0, 0),
font=MagicMock(),
fontsize=12,
scaling=1,
rise=0,
text="x",
textwidth=10,
textdisp=(0, 1),
ncs=MagicMock(),
graphicstate=MagicMock(),
)
def _make_interpreter(cur_item):
interp = object.__new__(CustomPDFPageInterpreter)
interp.device = MagicMock()
interp.device.cur_item = cur_item
interp.textstate = MagicMock()
interp.graphicstate = MagicMock()
return interp
def test_patch_render_mode_only_new_chars():
"""Only chars added after the snapshot index should be patched."""
page = LTLayoutContainer(bbox=(0, 0, 100, 100))
interp = _make_interpreter(page)
old_char = _make_char()
page.add(old_char)
interp.textstate.render = 3
interp._patch_current_chars_with_render_mode(start=1)
assert not hasattr(old_char, "rendermode")
def test_patch_render_mode_correct_value():
"""Chars in the patched range should get the current render mode."""
page = LTLayoutContainer(bbox=(0, 0, 100, 100))
interp = _make_interpreter(page)
char = _make_char()
page.add(char)
interp.textstate.render = 3
interp._patch_current_chars_with_render_mode(start=0)
assert char.rendermode == 3
def test_patch_render_mode_preserved_after_figure_with_text():
"""When cur_item reverts to the page after a figure that contained text ops,
previously patched chars must keep their original render mode."""
page = LTLayoutContainer(bbox=(0, 0, 100, 100))
interp = _make_interpreter(page)
# Text op on page with render_mode=0
char_a = _make_char()
page.add(char_a)
interp.textstate.render = 0
interp._patch_current_chars_with_render_mode(start=0)
# begin_figure — cur_item switches to figure
figure = LTFigure("test", (0, 0, 50, 50), (1, 0, 0, 1, 0, 0))
interp.device.cur_item = figure
# Text op inside figure
fig_char = _make_char()
figure.add(fig_char)
interp._patch_current_chars_with_render_mode(start=0)
# end_figure — cur_item reverts to page
interp.device.cur_item = page
# Render mode changes, new text op on page
interp.textstate.render = 3
char_b = _make_char()
page.add(char_b)
interp._patch_current_chars_with_render_mode(start=1)
assert char_a.rendermode == 0
assert char_b.rendermode == 3
def test_do_TJ_snapshots_before_super():
"""do_TJ should snapshot len(objs) before super() adds chars,
so only the newly added chars are patched."""
page = LTLayoutContainer(bbox=(0, 0, 100, 100))
interp = _make_interpreter(page)
old_char = _make_char()
page.add(old_char)
old_char.rendermode = 0
new_char = _make_char()
def fake_super_do_TJ(self, seq):
page.add(new_char)
interp.textstate.render = 3
interp.textstate.font = MagicMock()
with patch.object(CustomPDFPageInterpreter.__bases__[0], "do_TJ", fake_super_do_TJ):
interp.do_TJ([b"test"])
assert old_char.rendermode == 0
assert new_char.rendermode == 3
def test_extract_text_objects_nested_containers():
"""Test extract_text_objects with nested LTContainers."""
# Mock LTTextLine objects
mock_text_line1 = MagicMock(spec=LTTextLine)
mock_text_line2 = MagicMock(spec=LTTextLine)
# Mock inner container containing one LTTextLine
mock_inner_container = MagicMock(spec=LTContainer)
mock_inner_container.__iter__.return_value = [mock_text_line2]
# Mock outer container containing another LTTextLine and the inner container
mock_outer_container = MagicMock(spec=LTContainer)
mock_outer_container.__iter__.return_value = [mock_text_line1, mock_inner_container]
# Call the function with the outer container
result = extract_text_objects(mock_outer_container)
# Assert both text line objects are extracted, even from nested containers
assert len(result) == 2
assert mock_text_line1 in result
assert mock_text_line2 in result
# -- Tests for character deduplication (fake bold fix) --
def _create_mock_ltchar(
text: str, x0: float, y0: float, width: float = 6.0, height: float = 2.0
) -> MagicMock:
"""Helper to create a mock LTChar with specified text and position.
Includes x1, y1 so _is_duplicate_char overlap logic works (fake-bold detection
uses bounding box overlap). Default width/height give overlap ratio > 0.5 for
chars within threshold distance.
"""
mock_char = MagicMock(spec=LTChar)
mock_char.get_text.return_value = text
mock_char.x0 = x0
mock_char.y0 = y0
mock_char.x1 = x0 + width
mock_char.y1 = y0 + height
return mock_char
class TestIsDuplicateChar:
"""Tests for _is_duplicate_char function."""
def test_same_char_same_position_is_duplicate(self):
"""Two identical characters at the same position should be duplicates."""
char1 = _create_mock_ltchar("A", 10.0, 20.0)
char2 = _create_mock_ltchar("A", 10.0, 20.0)
assert _is_duplicate_char(char1, char2, threshold=3.0) is True
def test_same_char_close_position_is_duplicate(self):
"""Two identical characters at close positions should be duplicates."""
char1 = _create_mock_ltchar("B", 10.0, 20.0)
char2 = _create_mock_ltchar("B", 11.5, 21.0) # Within 3.0 threshold
assert _is_duplicate_char(char1, char2, threshold=3.0) is True
def test_same_char_far_position_not_duplicate(self):
"""Two identical characters at far positions should not be duplicates."""
char1 = _create_mock_ltchar("C", 10.0, 20.0)
char2 = _create_mock_ltchar("C", 15.0, 20.0) # 5.0 > 3.0 threshold
assert _is_duplicate_char(char1, char2, threshold=3.0) is False
def test_different_chars_same_position_not_duplicate(self):
"""Two different characters at the same position should not be duplicates."""
char1 = _create_mock_ltchar("A", 10.0, 20.0)
char2 = _create_mock_ltchar("B", 10.0, 20.0)
assert _is_duplicate_char(char1, char2, threshold=3.0) is False
def test_threshold_boundary(self):
"""Test behavior at exact threshold boundary."""
char1 = _create_mock_ltchar("X", 10.0, 20.0)
char2 = _create_mock_ltchar("X", 13.0, 20.0) # Exactly at threshold
# At threshold means NOT within threshold (uses < not <=)
assert _is_duplicate_char(char1, char2, threshold=3.0) is False
char3 = _create_mock_ltchar("X", 12.9, 20.0) # Just under threshold
assert _is_duplicate_char(char1, char3, threshold=3.0) is True
class TestDeduplicateCharsInTextLine:
"""Tests for deduplicate_chars_in_text_line function."""
def test_no_duplicates_returns_original(self):
"""Text line without duplicates should return original text."""
chars = [
_create_mock_ltchar("H", 10.0, 20.0),
_create_mock_ltchar("i", 15.0, 20.0),
]
mock_text_line = MagicMock(spec=LTTextLine)
mock_text_line.__iter__ = lambda self: iter(chars)
mock_text_line.get_text.return_value = "Hi"
result = deduplicate_chars_in_text_line(mock_text_line, threshold=3.0)
assert result == "Hi"
def test_fake_bold_duplicates_removed(self):
"""Fake bold text (each char doubled) should be deduplicated."""
# Simulates "BOLD" rendered as "BBOOLLDD" with duplicate positions
chars = [
_create_mock_ltchar("B", 10.0, 20.0),
_create_mock_ltchar("B", 10.5, 20.0), # Duplicate
_create_mock_ltchar("O", 20.0, 20.0),
_create_mock_ltchar("O", 20.5, 20.0), # Duplicate
_create_mock_ltchar("L", 30.0, 20.0),
_create_mock_ltchar("L", 30.5, 20.0), # Duplicate
_create_mock_ltchar("D", 40.0, 20.0),
_create_mock_ltchar("D", 40.5, 20.0), # Duplicate
]
mock_text_line = MagicMock(spec=LTTextLine)
mock_text_line.__iter__ = lambda self: iter(chars)
result = deduplicate_chars_in_text_line(mock_text_line, threshold=3.0)
assert result == "BOLD"
def test_threshold_zero_disables_deduplication(self):
"""Setting threshold to 0 should disable deduplication."""
mock_text_line = MagicMock(spec=LTTextLine)
mock_text_line.get_text.return_value = "BBOOLLDD"
result = deduplicate_chars_in_text_line(mock_text_line, threshold=0)
assert result == "BBOOLLDD"
def test_negative_threshold_disables_deduplication(self):
"""Setting negative threshold should disable deduplication."""
mock_text_line = MagicMock(spec=LTTextLine)
mock_text_line.get_text.return_value = "BBOOLLDD"
result = deduplicate_chars_in_text_line(mock_text_line, threshold=-1.0)
assert result == "BBOOLLDD"
def test_empty_text_line(self):
"""Empty text line should return original text."""
mock_text_line = MagicMock(spec=LTTextLine)
mock_text_line.__iter__ = lambda self: iter([])
mock_text_line.get_text.return_value = ""
result = deduplicate_chars_in_text_line(mock_text_line, threshold=3.0)
assert result == ""
def test_legitimate_repeated_chars_preserved(self):
"""Legitimate repeated characters (different positions) should be preserved."""
# "AA" where both A's are at different positions
chars = [
_create_mock_ltchar("A", 10.0, 20.0),
_create_mock_ltchar("A", 20.0, 20.0), # Different position, not duplicate
]
mock_text_line = MagicMock(spec=LTTextLine)
mock_text_line.__iter__ = lambda self: iter(chars)
result = deduplicate_chars_in_text_line(mock_text_line, threshold=3.0)
assert result == "AA"
class TestGetTextWithDeduplication:
"""Tests for get_text_with_deduplication function."""
def test_with_text_line(self):
"""Should properly deduplicate text from LTTextLine."""
chars = [
_create_mock_ltchar("H", 10.0, 20.0),
_create_mock_ltchar("H", 10.5, 20.0), # Duplicate
_create_mock_ltchar("i", 20.0, 20.0),
]
mock_text_line = MagicMock(spec=LTTextLine)
mock_text_line.__iter__ = lambda self: iter(chars)
result = get_text_with_deduplication(mock_text_line, threshold=3.0)
assert result == "Hi"
def test_with_container(self):
"""Should handle LTContainer with nested LTTextLine."""
chars = [
_create_mock_ltchar("T", 10.0, 20.0),
_create_mock_ltchar("T", 10.5, 20.0), # Duplicate
]
mock_text_line = MagicMock(spec=LTTextLine)
mock_text_line.__iter__ = lambda self: iter(chars)
mock_container = MagicMock(spec=LTContainer)
mock_container.__iter__ = lambda self: iter([mock_text_line])
result = get_text_with_deduplication(mock_container, threshold=3.0)
assert result == "T"
def test_with_generic_object(self):
"""Should fall back to get_text() for non-standard objects."""
mock_obj = MagicMock()
mock_obj.get_text.return_value = "fallback text"
result = get_text_with_deduplication(mock_obj, threshold=3.0)
assert result == "fallback text"
def test_without_get_text(self):
"""Should return empty string for objects without get_text."""
mock_obj = MagicMock(spec=[]) # No get_text method
result = get_text_with_deduplication(mock_obj, threshold=3.0)
assert result == ""
# -- Integration tests for fake-bold PDF deduplication --
class TestFakeBoldPdfIntegration:
"""Integration tests for fake-bold PDF deduplication using real PDF files.
The test PDF (fake-bold-sample.pdf) contains text rendered with the "fake bold"
technique where each character is drawn twice at slightly offset positions.
This causes text extraction to show doubled characters (e.g., "BBOOLLDD" instead
of "BOLD") unless deduplication is applied.
"""
def test_fake_bold_pdf_without_deduplication_shows_doubled_chars(self, monkeypatch):
"""Test that extraction WITHOUT deduplication shows doubled characters.
When PDF_CHAR_DUPLICATE_THRESHOLD is set to 0, deduplication is disabled
and the raw text shows the fake-bold doubled characters.
"""
monkeypatch.setenv("PDF_CHAR_DUPLICATE_THRESHOLD", "0")
reload(partition_config)
filename = example_doc_path("pdf/fake-bold-sample.pdf")
elements = partition_pdf(filename=filename, strategy="fast")
extracted_text = " ".join([el.text for el in elements])
# Without deduplication, fake-bold text appears with doubled characters
assert "BBOOLLDD" in extracted_text, (
"Without deduplication, fake-bold text should show doubled characters "
"like 'BBOOLLDD' instead of 'BOLD'"
)
def test_fake_bold_pdf_with_deduplication_shows_clean_text(self, monkeypatch):
"""Test that extraction WITH deduplication shows clean text.
When PDF_CHAR_DUPLICATE_THRESHOLD is set to default (2.0), deduplication
removes the duplicate characters and produces clean, readable text.
"""
monkeypatch.setenv("PDF_CHAR_DUPLICATE_THRESHOLD", "2.0")
reload(partition_config)
filename = example_doc_path("pdf/fake-bold-sample.pdf")
elements = partition_pdf(filename=filename, strategy="fast")
extracted_text = " ".join([el.text for el in elements])
# With deduplication, fake-bold text should be clean (no doubled chars)
assert "BOLD" in extracted_text, (
"With deduplication, text should contain clean 'BOLD' not 'BBOOLLDD'"
)
# Verify the doubled pattern is NOT present in the deduplicated fake-bold section
# Note: The PDF contains 'BBOOLLDD' as explanatory text, so we check for
# the specific pattern that would appear if deduplication failed on the
# fake-bold rendered text (e.g., "TTEEXXTT" from "TEXT")
assert "TTEEXXTT" not in extracted_text, (
"With deduplication, fake-bold 'TEXT' should not appear as 'TTEEXXTT'"
)
def test_fake_bold_deduplication_reduces_text_length(self, monkeypatch):
"""Test that deduplication reduces text length for fake-bold PDFs.
Compares extraction with and without deduplication to verify that
the deduplicated text is shorter due to removal of duplicate characters.
"""
filename = example_doc_path("pdf/fake-bold-sample.pdf")
# Extract WITHOUT deduplication (threshold=0)
monkeypatch.setenv("PDF_CHAR_DUPLICATE_THRESHOLD", "0")
reload(partition_config)
elements_no_dedup = partition_pdf(filename=filename, strategy="fast")
text_no_dedup = " ".join([el.text for el in elements_no_dedup])
# Extract WITH deduplication (threshold=2.0)
monkeypatch.setenv("PDF_CHAR_DUPLICATE_THRESHOLD", "2.0")
reload(partition_config)
elements_with_dedup = partition_pdf(filename=filename, strategy="fast")
text_with_dedup = " ".join([el.text for el in elements_with_dedup])
# Deduplicated text should be shorter than non-deduplicated text
assert len(text_with_dedup) < len(text_no_dedup), (
f"Deduplicated text ({len(text_with_dedup)} chars) should be shorter "
f"than non-deduplicated text ({len(text_no_dedup)} chars)"
)
# -- Tests for embedded CMap stream parsing --
class TestParseEmbeddedCmapStream:
"""Unit tests for _parse_embedded_cmap_stream.
pdfminer.six does not parse embedded Encoding CMap streams for CIDFonts.
_parse_embedded_cmap_stream is our workaround that extracts code-to-CID
mappings and writing mode from the raw CMap stream bytes.
"""
@staticmethod
def _parse(data: bytes):
from unstructured.partition.pdf_image.pdfminer_utils import _parse_embedded_cmap_stream
return _parse_embedded_cmap_stream(data)
def test_vertical_wmode_is_preserved(self):
"""Embedded CMaps with WMode=1 (vertical) should produce a vertical CMap."""
data = b"""
/WMode 1
1 begincodespacerange
<00> <0A>
endcodespacerange
1 begincidrange
<00> <0A> 0
endcidrange
"""
cmap = self._parse(data)
assert cmap.is_vertical() is True
assert len(cmap.code2cid) == 11
def test_horizontal_wmode_is_default(self):
"""CMaps without WMode or with WMode=0 should produce a horizontal CMap."""
data = b"""
1 begincodespacerange
<00> <05>
endcodespacerange
1 begincidrange
<00> <05> 0
endcidrange
"""
cmap = self._parse(data)
assert cmap.is_vertical() is False
def test_oversized_stream_returns_empty_cmap(self):
"""Streams exceeding the size cap should be rejected before any parsing."""
from unstructured.partition.pdf_image.pdfminer_utils import _MAX_CMAP_STREAM_BYTES
data = b"x" * (_MAX_CMAP_STREAM_BYTES + 1)
cmap = self._parse(data)
assert not cmap.code2cid
def test_reversed_range_is_skipped(self):
"""A begincidrange where end < start should be silently skipped."""
data = b"""
1 begincodespacerange
<00> <FF>
endcodespacerange
1 begincidrange
<80> <10> 0
endcidrange
"""
cmap = self._parse(data)
assert not cmap.code2cid
def test_mapping_cap_bounds_total_entries(self):
"""The total mapping count should be bounded across both cidrange and cidchar."""
from unittest.mock import patch
data = b"""
1 begincodespacerange
<0000> <FFFF>
endcodespacerange
1 begincidrange
<0000> <FFFF> 0
endcidrange
"""
# Use a small cap to verify bounding without allocating huge dicts
with patch("unstructured.partition.pdf_image.pdfminer_utils._MAX_CODE2CID_MAPPINGS", 100):
cmap = self._parse(data)
assert not cmap.code2cid # 65536 > 100: entire CMap discarded, not partial
def test_mapping_budget_second_range_discards_entire_cmap(self):
"""If a later cidrange would exceed the cap, reject the whole map (no holes)."""
from unittest.mock import patch
data = b"""
1 begincodespacerange
<00> <19>
endcodespacerange
2 begincidrange
<00> <09> 0
<0A> <19> 10
endcidrange
"""
with patch("unstructured.partition.pdf_image.pdfminer_utils._MAX_CODE2CID_MAPPINGS", 15):
cmap = self._parse(data)
assert not cmap.code2cid
class TestBoundedStreamDecode:
"""Tests for _decode_pdfstream_with_limit.
Verifies that oversized embedded CMap streams are rejected *before* full
materialization, and that the shared PDFStream object is never mutated.
"""
def test_oversized_flate_stream_rejected_before_materialization(self):
"""A small compressed payload that expands past the limit should be rejected
without fully materializing the output, and the stream should not be mutated."""
import zlib
from pdfminer.psparser import LIT
from unstructured.partition.pdf_image.pdfminer_utils import (
_decode_pdfstream_with_limit,
)
payload = zlib.compress(b"x" * 200)
stream = PDFStream({"Filter": LIT("FlateDecode")}, payload)
result = _decode_pdfstream_with_limit(stream, max_decoded_bytes=100)
assert result is None
# Stream object must not be mutated
assert stream.get_rawdata() is not None
assert stream.data is None
def test_normal_stream_decodes_within_limit(self):
"""A stream that fits within the limit should decode successfully."""
import zlib
from pdfminer.psparser import LIT
from unstructured.partition.pdf_image.pdfminer_utils import (
_decode_pdfstream_with_limit,
)
content = b"begincidrange <00> <05> 0 endcidrange"
payload = zlib.compress(content)
stream = PDFStream({"Filter": LIT("FlateDecode")}, payload)
result = _decode_pdfstream_with_limit(stream, max_decoded_bytes=1000)
assert result == content
def test_encrypted_stream_without_objid_returns_none(self):
"""Decipher requires objid/genno; missing values skip decode (no crash)."""
from unstructured.partition.pdf_image.pdfminer_utils import (
_decode_pdfstream_with_limit,
)
stream = PDFStream({}, b"encrypted-bytes", decipher=lambda *a: b"x")
stream.objid = None
stream.genno = 0
assert _decode_pdfstream_with_limit(stream, max_decoded_bytes=1000) is None
stream.objid = 1
stream.genno = None
assert _decode_pdfstream_with_limit(stream, max_decoded_bytes=1000) is None
def test_uncompressed_stream_returns_raw(self):
"""A stream with no filters should return the raw data directly."""
from unstructured.partition.pdf_image.pdfminer_utils import (
_decode_pdfstream_with_limit,
)
content = b"begincidrange <00> <05> 0 endcidrange"
stream = PDFStream({}, content)
result = _decode_pdfstream_with_limit(stream, max_decoded_bytes=1000)
assert result == content
class TestCustomPDFCIDFont:
"""Tests for CustomPDFCIDFont constructor-time CMap resolution."""
def test_vertical_wmode_sets_font_vertical_flag(self):
"""A vertical embedded CMap (WMode=1) should set font.vertical=True and
font.default_disp != 0 at construction time, not just on the CMap."""
import zlib
from pdfminer.pdfinterp import PDFResourceManager
from pdfminer.psparser import LIT
from unstructured.partition.pdf_image.pdfminer_utils import CustomPDFCIDFont
cmap_data = b"""/CIDInit /ProcSet findresource begin
12 dict begin
begincmap
/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> def
/CMapName /Test-Vertical-H def
/CMapType 1 def
/WMode 1 def
1 begincodespacerange
<00> <0A>
endcodespacerange
1 begincidrange
<00> <0A> 0
endcidrange
endcmap
end
end"""
encoding_stream = PDFStream(
{
"Type": LIT("CMap"),
"CMapName": LIT("Test-Vertical-H"),
"CIDSystemInfo": {
"Registry": b"Adobe",
"Ordering": b"Identity",
"Supplement": 0,
},
"Filter": LIT("FlateDecode"),
},
zlib.compress(cmap_data),
)
tounicode_data = b"""/CIDInit /ProcSet findresource begin
12 dict begin
begincmap
/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def
/CMapName /Adobe-Identity-UCS def
/CMapType 2 def
1 begincodespacerange
<00> <FF>
endcodespacerange
1 beginbfrange
<00> <0A> <0041>
endbfrange
endcmap
end
end"""
tounicode_stream = PDFStream({}, tounicode_data)
spec = {
"Type": LIT("Font"),
"Subtype": LIT("CIDFontType2"),
"BaseFont": LIT("TestSans"),
"CIDSystemInfo": {
"Registry": b"Adobe",
"Ordering": b"Identity",
"Supplement": 0,
},
"FontDescriptor": {
"Type": LIT("FontDescriptor"),
"FontName": LIT("TestSans"),
"Flags": 32,
"FontBBox": [0, -200, 1000, 800],
"ItalicAngle": 0,
"Ascent": 800,
"Descent": -200,
"CapHeight": 700,
"StemV": 80,
},
"DW": 500,
"Encoding": encoding_stream,
"ToUnicode": tounicode_stream,
}
rsrcmgr = PDFResourceManager()
font = CustomPDFCIDFont(rsrcmgr, spec, strict=False)
assert font.cmap.is_vertical() is True
assert font.vertical is True
assert font.default_disp != 0
class TestCustomPDFResourceManager:
"""Tests for CustomPDFResourceManager font construction routing."""
def test_returns_custom_cidfont_for_cid_subtypes(self):
"""CIDFontType2 fonts should be constructed as CustomPDFCIDFont."""
from pdfminer.psparser import LIT
from unstructured.partition.pdf_image.pdfminer_utils import (
CustomPDFCIDFont,
CustomPDFResourceManager,
)
cmap_data = b"""begincmap
1 begincodespacerange <00> <05> endcodespacerange
1 begincidrange <00> <05> 0 endcidrange
endcmap"""
spec = {
"Type": LIT("Font"),
"Subtype": LIT("CIDFontType2"),
"BaseFont": LIT("TestSans"),
"CIDSystemInfo": {
"Registry": b"Adobe",
"Ordering": b"Identity",
"Supplement": 0,
},
"FontDescriptor": {
"Type": LIT("FontDescriptor"),
"FontName": LIT("TestSans"),
"Flags": 32,
"FontBBox": [0, -200, 1000, 800],
"ItalicAngle": 0,
"Ascent": 800,
"Descent": -200,
"CapHeight": 700,
"StemV": 80,
},
"DW": 500,
"Encoding": PDFStream(
{
"Type": LIT("CMap"),
"CMapName": LIT("Test-Custom-H"),
"CIDSystemInfo": {
"Registry": b"Adobe",
"Ordering": b"Identity",
"Supplement": 0,
},
},
cmap_data,
),
}
rsrcmgr = CustomPDFResourceManager()
font = rsrcmgr.get_font(1, spec)
assert isinstance(font, CustomPDFCIDFont)
def test_caches_font_on_repeated_calls(self):
"""Repeated get_font calls with the same objid should return the cached instance."""
from pdfminer.psparser import LIT
from unstructured.partition.pdf_image.pdfminer_utils import (
CustomPDFResourceManager,
)
cmap_data = b"""begincmap
1 begincodespacerange <00> <05> endcodespacerange
1 begincidrange <00> <05> 0 endcidrange
endcmap"""
spec = {
"Type": LIT("Font"),
"Subtype": LIT("CIDFontType2"),
"BaseFont": LIT("TestSans"),
"CIDSystemInfo": {
"Registry": b"Adobe",
"Ordering": b"Identity",
"Supplement": 0,
},
"FontDescriptor": {
"Type": LIT("FontDescriptor"),
"FontName": LIT("TestSans"),
"Flags": 32,
"FontBBox": [0, -200, 1000, 800],
"ItalicAngle": 0,
"Ascent": 800,
"Descent": -200,
"CapHeight": 700,
"StemV": 80,
},
"DW": 500,
"Encoding": PDFStream(
{
"Type": LIT("CMap"),
"CMapName": LIT("Test-Custom-H"),
"CIDSystemInfo": {
"Registry": b"Adobe",
"Ordering": b"Identity",
"Supplement": 0,
},
},
cmap_data,
),
}
rsrcmgr = CustomPDFResourceManager()
font1 = rsrcmgr.get_font(42, spec)
font2 = rsrcmgr.get_font(42, spec)
assert font1 is font2