chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,3 @@
|
||||
# SPDX-FileCopyrightText: 2024-present Adam Fourney <adamfo@microsoft.com>
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
@@ -0,0 +1,279 @@
|
||||
import dataclasses
|
||||
from typing import List
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, kw_only=True)
|
||||
class FileTestVector(object):
|
||||
filename: str
|
||||
mimetype: str | None
|
||||
charset: str | None
|
||||
url: str | None
|
||||
must_include: List[str]
|
||||
must_not_include: List[str]
|
||||
|
||||
|
||||
GENERAL_TEST_VECTORS = [
|
||||
FileTestVector(
|
||||
filename="test.docx",
|
||||
mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"314b0a30-5b04-470b-b9f7-eed2c2bec74a",
|
||||
"49e168b7-d2ae-407f-a055-2167576f39a1",
|
||||
"## d666f1f7-46cb-42bd-9a39-9a39cf2a509f",
|
||||
"# Abstract",
|
||||
"# Introduction",
|
||||
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||
"data:image/png;base64...",
|
||||
],
|
||||
must_not_include=[
|
||||
"data:image/png;base64,iVBORw0KGgoAAAANSU",
|
||||
],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test.xlsx",
|
||||
mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"## 09060124-b5e7-4717-9d07-3c046eb",
|
||||
"6ff4173b-42a5-4784-9b19-f49caff4d93d",
|
||||
"affc7dad-52dc-4b98-9b5d-51e65d8a8ad0",
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test.xls",
|
||||
mimetype="application/vnd.ms-excel",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"## 09060124-b5e7-4717-9d07-3c046eb",
|
||||
"6ff4173b-42a5-4784-9b19-f49caff4d93d",
|
||||
"affc7dad-52dc-4b98-9b5d-51e65d8a8ad0",
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test.pptx",
|
||||
mimetype="application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"2cdda5c8-e50e-4db4-b5f0-9722a649f455",
|
||||
"04191ea8-5c73-4215-a1d3-1cfb43aaaf12",
|
||||
"44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a",
|
||||
"1b92870d-e3b5-4e65-8153-919f4ff45592",
|
||||
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||
"a3f6004b-6f4f-4ea8-bee3-3741f4dc385f", # chart title
|
||||
"2003", # chart value
|
||||
"",
|
||||
],
|
||||
must_not_include=["data:image/jpeg;base64,/9j/4AAQSkZJRgABAQE"],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test_outlook_msg.msg",
|
||||
mimetype="application/vnd.ms-outlook",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"# Email Message",
|
||||
"**From:** test.sender@example.com",
|
||||
"**To:** test.recipient@example.com",
|
||||
"**Subject:** Test Email Message",
|
||||
"## Content",
|
||||
"This is the body of the test email message",
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test.pdf",
|
||||
mimetype="application/pdf",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"While there is contemporaneous exploration of multi-agent approaches"
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test_blog.html",
|
||||
mimetype="text/html",
|
||||
charset="utf-8",
|
||||
url="https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math",
|
||||
must_include=[
|
||||
"Large language models (LLMs) are powerful tools that can generate natural language texts for various applications, such as chatbots, summarization, translation, and more. GPT-4 is currently the state of the art LLM in the world. Is model selection irrelevant? What about inference parameters?",
|
||||
"an example where high cost can easily prevent a generic complex",
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test_wikipedia.html",
|
||||
mimetype="text/html",
|
||||
charset="utf-8",
|
||||
url="https://en.wikipedia.org/wiki/Microsoft",
|
||||
must_include=[
|
||||
"Microsoft entered the operating system (OS) business in 1980 with its own version of [Unix]",
|
||||
'Microsoft was founded by [Bill Gates](/wiki/Bill_Gates "Bill Gates")',
|
||||
],
|
||||
must_not_include=[
|
||||
"You are encouraged to create an account and log in",
|
||||
"154 languages",
|
||||
"move to sidebar",
|
||||
],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test_serp.html",
|
||||
mimetype="text/html",
|
||||
charset="utf-8",
|
||||
url="https://www.bing.com/search?q=microsoft+wikipedia",
|
||||
must_include=[
|
||||
"](https://en.wikipedia.org/wiki/Microsoft",
|
||||
"Microsoft Corporation is **an American multinational corporation and technology company headquartered** in Redmond",
|
||||
"1995–2007: Foray into the Web, Windows 95, Windows XP, and Xbox",
|
||||
],
|
||||
must_not_include=[
|
||||
"https://www.bing.com/ck/a?!&&p=",
|
||||
"data:image/svg+xml,%3Csvg%20width%3D",
|
||||
],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test_mskanji.csv",
|
||||
mimetype="text/csv",
|
||||
charset="cp932",
|
||||
url=None,
|
||||
must_include=[
|
||||
"| 名前 | 年齢 | 住所 |",
|
||||
"| --- | --- | --- |",
|
||||
"| 佐藤太郎 | 30 | 東京 |",
|
||||
"| 三木英子 | 25 | 大阪 |",
|
||||
"| 髙橋淳 | 35 | 名古屋 |",
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test.json",
|
||||
mimetype="application/json",
|
||||
charset="ascii",
|
||||
url=None,
|
||||
must_include=[
|
||||
"5b64c88c-b3c3-4510-bcb8-da0b200602d8",
|
||||
"9700dc99-6685-40b4-9a3a-5e406dcb37f3",
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test_rss.xml",
|
||||
mimetype="text/xml",
|
||||
charset="utf-8",
|
||||
url=None,
|
||||
must_include=[
|
||||
"# The Official Microsoft Blog",
|
||||
"## Ignite 2024: Why nearly 70% of the Fortune 500 now use Microsoft 365 Copilot",
|
||||
"In the case of AI, it is absolutely true that the industry is moving incredibly fast",
|
||||
],
|
||||
must_not_include=["<rss", "<feed"],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test_notebook.ipynb",
|
||||
mimetype="application/json",
|
||||
charset="ascii",
|
||||
url=None,
|
||||
must_include=[
|
||||
"# Test Notebook",
|
||||
"```python",
|
||||
'print("markitdown")',
|
||||
"```",
|
||||
"## Code Cell Below",
|
||||
],
|
||||
must_not_include=[
|
||||
"nbformat",
|
||||
"nbformat_minor",
|
||||
],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test_files.zip",
|
||||
mimetype="application/zip",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"314b0a30-5b04-470b-b9f7-eed2c2bec74a",
|
||||
"49e168b7-d2ae-407f-a055-2167576f39a1",
|
||||
"## d666f1f7-46cb-42bd-9a39-9a39cf2a509f",
|
||||
"# Abstract",
|
||||
"# Introduction",
|
||||
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||
"2cdda5c8-e50e-4db4-b5f0-9722a649f455",
|
||||
"04191ea8-5c73-4215-a1d3-1cfb43aaaf12",
|
||||
"44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a",
|
||||
"1b92870d-e3b5-4e65-8153-919f4ff45592",
|
||||
"## 09060124-b5e7-4717-9d07-3c046eb",
|
||||
"6ff4173b-42a5-4784-9b19-f49caff4d93d",
|
||||
"affc7dad-52dc-4b98-9b5d-51e65d8a8ad0",
|
||||
"Microsoft entered the operating system (OS) business in 1980 with its own version of [Unix]",
|
||||
'Microsoft was founded by [Bill Gates](/wiki/Bill_Gates "Bill Gates")',
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test.epub",
|
||||
mimetype="application/epub+zip",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"**Authors:** Test Author",
|
||||
"A test EPUB document for MarkItDown testing",
|
||||
"# Chapter 1: Test Content",
|
||||
"This is a **test** paragraph with some formatting",
|
||||
"* A bullet point",
|
||||
"* Another point",
|
||||
"# Chapter 2: More Content",
|
||||
"*different* style",
|
||||
"> This is a blockquote for testing",
|
||||
],
|
||||
must_not_include=[],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
DATA_URI_TEST_VECTORS = [
|
||||
FileTestVector(
|
||||
filename="test.docx",
|
||||
mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"314b0a30-5b04-470b-b9f7-eed2c2bec74a",
|
||||
"49e168b7-d2ae-407f-a055-2167576f39a1",
|
||||
"## d666f1f7-46cb-42bd-9a39-9a39cf2a509f",
|
||||
"# Abstract",
|
||||
"# Introduction",
|
||||
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||
"data:image/png;base64,iVBORw0KGgoAAAANSU",
|
||||
],
|
||||
must_not_include=[
|
||||
"data:image/png;base64...",
|
||||
],
|
||||
),
|
||||
FileTestVector(
|
||||
filename="test.pptx",
|
||||
mimetype="application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
charset=None,
|
||||
url=None,
|
||||
must_include=[
|
||||
"2cdda5c8-e50e-4db4-b5f0-9722a649f455",
|
||||
"04191ea8-5c73-4215-a1d3-1cfb43aaaf12",
|
||||
"44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a",
|
||||
"1b92870d-e3b5-4e65-8153-919f4ff45592",
|
||||
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||
"a3f6004b-6f4f-4ea8-bee3-3741f4dc385f", # chart title
|
||||
"2003", # chart value
|
||||
"![This phrase of the caption is Human-written.]", # image caption
|
||||
"data:image/jpeg;base64,/9j/4AAQSkZJRgABAQE",
|
||||
],
|
||||
must_not_include=[
|
||||
"",
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3 -m pytest
|
||||
import subprocess
|
||||
from markitdown import __version__
|
||||
|
||||
# This file contains CLI tests that are not directly tested by the FileTestVectors.
|
||||
# This includes things like help messages, version numbers, and invalid flags.
|
||||
|
||||
|
||||
def test_version() -> None:
|
||||
result = subprocess.run(
|
||||
["python", "-m", "markitdown", "--version"], capture_output=True, text=True
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"CLI exited with error: {result.stderr}"
|
||||
assert __version__ in result.stdout, f"Version not found in output: {result.stdout}"
|
||||
|
||||
|
||||
def test_invalid_flag() -> None:
|
||||
result = subprocess.run(
|
||||
["python", "-m", "markitdown", "--foobar"], capture_output=True, text=True
|
||||
)
|
||||
|
||||
assert result.returncode != 0, f"CLI exited with error: {result.stderr}"
|
||||
assert (
|
||||
"unrecognized arguments" in result.stderr
|
||||
), "Expected 'unrecognized arguments' to appear in STDERR"
|
||||
assert "SYNTAX" in result.stderr, "Expected 'SYNTAX' to appear in STDERR"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""Runs this file's tests from the command line."""
|
||||
test_version()
|
||||
test_invalid_flag()
|
||||
print("All tests passed!")
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3 -m pytest
|
||||
import os
|
||||
import time
|
||||
import pytest
|
||||
import subprocess
|
||||
import locale
|
||||
from typing import List
|
||||
|
||||
if __name__ == "__main__":
|
||||
from _test_vectors import (
|
||||
GENERAL_TEST_VECTORS,
|
||||
DATA_URI_TEST_VECTORS,
|
||||
FileTestVector,
|
||||
)
|
||||
else:
|
||||
from ._test_vectors import (
|
||||
GENERAL_TEST_VECTORS,
|
||||
DATA_URI_TEST_VECTORS,
|
||||
FileTestVector,
|
||||
)
|
||||
|
||||
skip_remote = (
|
||||
True if os.environ.get("GITHUB_ACTIONS") else False
|
||||
) # Don't run these tests in CI
|
||||
|
||||
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")
|
||||
TEST_FILES_URL = "https://raw.githubusercontent.com/microsoft/markitdown/refs/heads/main/packages/markitdown/tests/test_files"
|
||||
|
||||
|
||||
# Prepare CLI test vectors (remove vectors that require mockig the url)
|
||||
CLI_TEST_VECTORS: List[FileTestVector] = []
|
||||
for test_vector in GENERAL_TEST_VECTORS:
|
||||
if test_vector.url is not None:
|
||||
continue
|
||||
CLI_TEST_VECTORS.append(test_vector)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def shared_tmp_dir(tmp_path_factory):
|
||||
return tmp_path_factory.mktemp("pytest_tmp")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", CLI_TEST_VECTORS)
|
||||
def test_output_to_stdout(shared_tmp_dir, test_vector) -> None:
|
||||
"""Test that the CLI outputs to stdout correctly."""
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"python",
|
||||
"-m",
|
||||
"markitdown",
|
||||
os.path.join(TEST_FILES_DIR, test_vector.filename),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"CLI exited with error: {result.stderr}"
|
||||
for test_string in test_vector.must_include:
|
||||
assert test_string in result.stdout
|
||||
for test_string in test_vector.must_not_include:
|
||||
assert test_string not in result.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", CLI_TEST_VECTORS)
|
||||
def test_output_to_file(shared_tmp_dir, test_vector) -> None:
|
||||
"""Test that the CLI outputs to a file correctly."""
|
||||
|
||||
output_file = os.path.join(shared_tmp_dir, test_vector.filename + ".output")
|
||||
result = subprocess.run(
|
||||
[
|
||||
"python",
|
||||
"-m",
|
||||
"markitdown",
|
||||
"-o",
|
||||
output_file,
|
||||
os.path.join(TEST_FILES_DIR, test_vector.filename),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"CLI exited with error: {result.stderr}"
|
||||
assert os.path.exists(output_file), f"Output file not created: {output_file}"
|
||||
|
||||
with open(output_file, "r") as f:
|
||||
output_data = f.read()
|
||||
for test_string in test_vector.must_include:
|
||||
assert test_string in output_data
|
||||
for test_string in test_vector.must_not_include:
|
||||
assert test_string not in output_data
|
||||
|
||||
os.remove(output_file)
|
||||
assert not os.path.exists(output_file), f"Output file not deleted: {output_file}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", CLI_TEST_VECTORS)
|
||||
def test_input_from_stdin_without_hints(shared_tmp_dir, test_vector) -> None:
|
||||
"""Test that the CLI readds from stdin correctly."""
|
||||
|
||||
test_input = b""
|
||||
with open(os.path.join(TEST_FILES_DIR, test_vector.filename), "rb") as stream:
|
||||
test_input = stream.read()
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"python",
|
||||
"-m",
|
||||
"markitdown",
|
||||
os.path.join(TEST_FILES_DIR, test_vector.filename),
|
||||
],
|
||||
input=test_input,
|
||||
capture_output=True,
|
||||
text=False,
|
||||
)
|
||||
|
||||
stdout = result.stdout.decode(locale.getpreferredencoding())
|
||||
assert (
|
||||
result.returncode == 0
|
||||
), f"CLI exited with error: {result.stderr.decode('utf-8')}"
|
||||
for test_string in test_vector.must_include:
|
||||
assert test_string in stdout
|
||||
for test_string in test_vector.must_not_include:
|
||||
assert test_string not in stdout
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
skip_remote,
|
||||
reason="do not run tests that query external urls",
|
||||
)
|
||||
@pytest.mark.parametrize("test_vector", CLI_TEST_VECTORS)
|
||||
def test_convert_url(shared_tmp_dir, test_vector):
|
||||
"""Test the conversion of a stream with no stream info."""
|
||||
# Note: tmp_dir is not used here, but is needed to match the signature
|
||||
|
||||
time.sleep(1) # Ensure we don't hit rate limits
|
||||
result = subprocess.run(
|
||||
["python", "-m", "markitdown", TEST_FILES_URL + "/" + test_vector.filename],
|
||||
capture_output=True,
|
||||
text=False,
|
||||
)
|
||||
|
||||
stdout = result.stdout.decode(locale.getpreferredencoding())
|
||||
assert result.returncode == 0, f"CLI exited with error: {result.stderr}"
|
||||
for test_string in test_vector.must_include:
|
||||
assert test_string in stdout
|
||||
for test_string in test_vector.must_not_include:
|
||||
assert test_string not in stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", DATA_URI_TEST_VECTORS)
|
||||
def test_output_to_file_with_data_uris(shared_tmp_dir, test_vector) -> None:
|
||||
"""Test CLI functionality when keep_data_uris is enabled"""
|
||||
|
||||
output_file = os.path.join(shared_tmp_dir, test_vector.filename + ".output")
|
||||
result = subprocess.run(
|
||||
[
|
||||
"python",
|
||||
"-m",
|
||||
"markitdown",
|
||||
"--keep-data-uris",
|
||||
"-o",
|
||||
output_file,
|
||||
os.path.join(TEST_FILES_DIR, test_vector.filename),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"CLI exited with error: {result.stderr}"
|
||||
assert os.path.exists(output_file), f"Output file not created: {output_file}"
|
||||
|
||||
with open(output_file, "r") as f:
|
||||
output_data = f.read()
|
||||
for test_string in test_vector.must_include:
|
||||
assert test_string in output_data
|
||||
for test_string in test_vector.must_not_include:
|
||||
assert test_string not in output_data
|
||||
|
||||
os.remove(output_file)
|
||||
assert not os.path.exists(output_file), f"Output file not deleted: {output_file}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import tempfile
|
||||
|
||||
"""Runs this file's tests from the command line."""
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
# General tests
|
||||
for test_function in [
|
||||
test_output_to_stdout,
|
||||
test_output_to_file,
|
||||
test_input_from_stdin_without_hints,
|
||||
test_convert_url,
|
||||
]:
|
||||
for test_vector in CLI_TEST_VECTORS:
|
||||
print(
|
||||
f"Running {test_function.__name__} on {test_vector.filename}...",
|
||||
end="",
|
||||
)
|
||||
test_function(tmp_dir, test_vector)
|
||||
print("OK")
|
||||
|
||||
# Data URI tests
|
||||
for test_function in [
|
||||
test_output_to_file_with_data_uris,
|
||||
]:
|
||||
for test_vector in DATA_URI_TEST_VECTORS:
|
||||
print(
|
||||
f"Running {test_function.__name__} on {test_vector.filename}...",
|
||||
end="",
|
||||
)
|
||||
test_function(tmp_dir, test_vector)
|
||||
print("OK")
|
||||
|
||||
print("All tests passed!")
|
||||
@@ -0,0 +1,928 @@
|
||||
"""Tests for ContentUnderstandingConverter.
|
||||
|
||||
Tests accepts() routing, smart routing modality logic, and convert() via mocks.
|
||||
Follows the same pattern as test_docintel_html.py.
|
||||
"""
|
||||
|
||||
import io
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from markitdown.converters._cu_converter import (
|
||||
ContentUnderstandingConverter,
|
||||
ContentUnderstandingFileType,
|
||||
_resolve_analyzer_modality,
|
||||
_get_modality,
|
||||
_detect_file_type,
|
||||
_canonical_mime_type,
|
||||
_content_type_for,
|
||||
_EXTENSION_MAP,
|
||||
)
|
||||
from markitdown._stream_info import StreamInfo
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: create a converter with accepts() working but no SDK init
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_converter(file_types=None, analyzer_id=None, analyzer_modality=None):
|
||||
"""Create a converter bypassing __init__ (no SDK deps needed)."""
|
||||
conv = ContentUnderstandingConverter.__new__(ContentUnderstandingConverter)
|
||||
conv._analyzer_id = analyzer_id
|
||||
conv._analyzer_modality = analyzer_modality
|
||||
|
||||
# Set accepted file types without running SDK-dependent initialization.
|
||||
from markitdown.converters._cu_converter import (
|
||||
_ALL_FILE_TYPES,
|
||||
)
|
||||
|
||||
types = file_types if file_types is not None else _ALL_FILE_TYPES
|
||||
conv._file_types = types
|
||||
|
||||
return conv
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# accepts() tests — extension-based
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAcceptsExtension:
|
||||
"""Test accepts() for supported and unsupported file extensions."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ext",
|
||||
[
|
||||
".pdf",
|
||||
".docx",
|
||||
".pptx",
|
||||
".xlsx",
|
||||
".html",
|
||||
".txt",
|
||||
".md",
|
||||
".rtf",
|
||||
".xml",
|
||||
".eml",
|
||||
".msg",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".jpe",
|
||||
".png",
|
||||
".bmp",
|
||||
".tiff",
|
||||
".heif",
|
||||
".heic",
|
||||
".mp4",
|
||||
".m4v",
|
||||
".mov",
|
||||
".avi",
|
||||
".mkv",
|
||||
".webm",
|
||||
".flv",
|
||||
".wmv",
|
||||
".wav",
|
||||
".mp3",
|
||||
".m4a",
|
||||
".flac",
|
||||
".ogg",
|
||||
".aac",
|
||||
".wma",
|
||||
],
|
||||
)
|
||||
def test_accepts_supported_extensions(self, ext):
|
||||
conv = _make_converter()
|
||||
assert conv.accepts(io.BytesIO(b""), StreamInfo(extension=ext))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ext",
|
||||
[
|
||||
".csv",
|
||||
".json",
|
||||
".zip",
|
||||
".epub",
|
||||
".py",
|
||||
".rs",
|
||||
],
|
||||
)
|
||||
def test_rejects_unsupported_extensions(self, ext):
|
||||
conv = _make_converter()
|
||||
assert not conv.accepts(io.BytesIO(b""), StreamInfo(extension=ext))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# accepts() tests — MIME-based
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAcceptsMime:
|
||||
"""Test accepts() for MIME type matching."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mime",
|
||||
[
|
||||
"application/pdf",
|
||||
"image/jpeg",
|
||||
"video/mp4",
|
||||
"audio/wav",
|
||||
"audio/x-wav",
|
||||
"text/html",
|
||||
"audio/mpeg",
|
||||
"audio/x-m4a",
|
||||
"audio/x-flac",
|
||||
"video/quicktime",
|
||||
"video/webm",
|
||||
"video/x-m4v",
|
||||
"video/x-flv",
|
||||
"video/x-ms-wmv",
|
||||
"audio/aac",
|
||||
"audio/x-ms-wma",
|
||||
],
|
||||
)
|
||||
def test_accepts_supported_mimetypes(self, mime):
|
||||
conv = _make_converter()
|
||||
assert conv.accepts(io.BytesIO(b""), StreamInfo(mimetype=mime))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mime",
|
||||
[
|
||||
"text/csv",
|
||||
"application/json",
|
||||
"application/zip",
|
||||
],
|
||||
)
|
||||
def test_rejects_unsupported_mimetypes(self, mime):
|
||||
conv = _make_converter()
|
||||
assert not conv.accepts(io.BytesIO(b""), StreamInfo(mimetype=mime))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# accepts() tests — cu_file_types restriction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAcceptsFileTypeRestriction:
|
||||
"""Test that cu_file_types restricts which formats are accepted."""
|
||||
|
||||
def test_restricted_to_pdf_only(self):
|
||||
conv = _make_converter(file_types=[ContentUnderstandingFileType.PDF])
|
||||
assert conv.accepts(io.BytesIO(b""), StreamInfo(extension=".pdf"))
|
||||
assert not conv.accepts(io.BytesIO(b""), StreamInfo(extension=".mp4"))
|
||||
assert not conv.accepts(io.BytesIO(b""), StreamInfo(extension=".wav"))
|
||||
assert not conv.accepts(io.BytesIO(b""), StreamInfo(extension=".jpg"))
|
||||
|
||||
def test_restricted_to_audio(self):
|
||||
conv = _make_converter(
|
||||
file_types=[
|
||||
ContentUnderstandingFileType.WAV,
|
||||
ContentUnderstandingFileType.MP3,
|
||||
]
|
||||
)
|
||||
assert conv.accepts(io.BytesIO(b""), StreamInfo(extension=".wav"))
|
||||
assert conv.accepts(io.BytesIO(b""), StreamInfo(extension=".mp3"))
|
||||
assert not conv.accepts(io.BytesIO(b""), StreamInfo(extension=".pdf"))
|
||||
|
||||
def test_webm_value_matches_cli_input(self):
|
||||
assert ContentUnderstandingFileType("webm") == ContentUnderstandingFileType.WEBM
|
||||
|
||||
def test_m4v_value_matches_cli_input(self):
|
||||
assert ContentUnderstandingFileType("m4v") == ContentUnderstandingFileType.M4V
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# file type detection tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDetectFileType:
|
||||
"""Test extension and MIME based file type detection."""
|
||||
|
||||
def test_detects_video_from_mime_without_extension(self):
|
||||
assert (
|
||||
_detect_file_type(StreamInfo(mimetype="video/mp4"))
|
||||
== ContentUnderstandingFileType.MP4
|
||||
)
|
||||
|
||||
def test_detects_audio_from_mime_without_extension(self):
|
||||
assert (
|
||||
_detect_file_type(StreamInfo(mimetype="audio/mpeg"))
|
||||
== ContentUnderstandingFileType.MP3
|
||||
)
|
||||
|
||||
def test_detects_audio_alias_from_mime_without_extension(self):
|
||||
assert (
|
||||
_detect_file_type(StreamInfo(mimetype="audio/x-wav"))
|
||||
== ContentUnderstandingFileType.WAV
|
||||
)
|
||||
|
||||
def test_detects_video_alias_from_mime_without_extension(self):
|
||||
assert (
|
||||
_detect_file_type(StreamInfo(mimetype="video/x-m4v"))
|
||||
== ContentUnderstandingFileType.M4V
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mimetype", "expected"),
|
||||
[
|
||||
("audio/x-wav", "audio/wav"),
|
||||
("audio/x-flac", "audio/flac"),
|
||||
("audio/x-m4a", "audio/mp4"),
|
||||
("video/x-m4v", "video/mp4"),
|
||||
("video/mp4", "video/mp4"),
|
||||
(None, "application/octet-stream"),
|
||||
],
|
||||
)
|
||||
def test_canonical_mime_type(self, mimetype, expected):
|
||||
assert _canonical_mime_type(mimetype) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("file_type", "mimetype", "expected"),
|
||||
[
|
||||
(ContentUnderstandingFileType.PDF, None, "application/pdf"),
|
||||
(ContentUnderstandingFileType.M4V, None, "video/mp4"),
|
||||
(ContentUnderstandingFileType.FLAC, "audio/x-flac", "audio/flac"),
|
||||
],
|
||||
)
|
||||
def test_content_type_for(self, file_type, mimetype, expected):
|
||||
assert _content_type_for(file_type, mimetype) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("file_type", "mimetype", "expected"),
|
||||
[
|
||||
# Extension/file_type wins when mimetype disagrees — the
|
||||
# resolved file_type is the single source of truth so that
|
||||
# analyzer routing and payload metadata stay consistent.
|
||||
(ContentUnderstandingFileType.PDF, "audio/mpeg", "application/pdf"),
|
||||
(ContentUnderstandingFileType.MP3, "application/pdf", "audio/mpeg"),
|
||||
(ContentUnderstandingFileType.MP4, "image/jpeg", "video/mp4"),
|
||||
(ContentUnderstandingFileType.JPEG, "video/mp4", "image/jpeg"),
|
||||
# Subtype distinctions are preserved when consistent
|
||||
# (e.g., HEIC vs HEIF both map to file_type HEIF; if the
|
||||
# caller passed image/heic explicitly, keep it).
|
||||
(ContentUnderstandingFileType.HEIF, "image/heic", "image/heic"),
|
||||
(ContentUnderstandingFileType.HEIF, "image/heif", "image/heif"),
|
||||
],
|
||||
)
|
||||
def test_content_type_for_resolves_conflicts_to_file_type(
|
||||
self, file_type, mimetype, expected
|
||||
):
|
||||
"""When extension and mimetype disagree, file_type wins."""
|
||||
assert _content_type_for(file_type, mimetype) == expected
|
||||
|
||||
def test_conflicting_extension_and_mimetype_in_convert(self):
|
||||
"""End-to-end: conflicting StreamInfo routes by extension and
|
||||
sends a content_type consistent with the resolved file_type."""
|
||||
conv = _make_converter()
|
||||
conv._client = MagicMock()
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = MagicMock(contents=[])
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch(
|
||||
"markitdown.converters._cu_converter.to_llm_input",
|
||||
return_value="ok",
|
||||
):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake"),
|
||||
# .pdf extension but bogus audio mimetype
|
||||
StreamInfo(extension=".pdf", mimetype="audio/mpeg"),
|
||||
)
|
||||
|
||||
call_kwargs = conv._client.begin_analyze_binary.call_args.kwargs
|
||||
# Routed by extension: document modality → prebuilt-documentSearch
|
||||
assert call_kwargs["analyzer_id"] == "prebuilt-documentSearch"
|
||||
# content_type derived from file_type (PDF), not the conflicting mime
|
||||
assert call_kwargs["content_type"] == "application/pdf"
|
||||
|
||||
def test_file_type_restriction_applies_to_mime(self):
|
||||
assert (
|
||||
_detect_file_type(
|
||||
StreamInfo(mimetype="video/mp4"),
|
||||
[ContentUnderstandingFileType.PDF],
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Smart routing tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSmartRouting:
|
||||
"""Test modality-aware analyzer routing."""
|
||||
|
||||
def test_document_analyzer_routes_pdf_to_custom(self):
|
||||
"""Document-based analyzer should be used for PDF."""
|
||||
conv = _make_converter(
|
||||
analyzer_id="my-doc-analyzer",
|
||||
analyzer_modality="document",
|
||||
)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake pdf"),
|
||||
StreamInfo(extension=".pdf", mimetype="application/pdf"),
|
||||
)
|
||||
|
||||
# Should use the custom analyzer for PDF (document modality)
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "my-doc-analyzer"
|
||||
|
||||
def test_document_analyzer_routes_mp3_to_prebuilt(self):
|
||||
"""Document-based analyzer should auto-route MP3 to prebuilt-audioSearch."""
|
||||
conv = _make_converter(
|
||||
analyzer_id="my-doc-analyzer",
|
||||
analyzer_modality="document",
|
||||
)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake audio"),
|
||||
StreamInfo(extension=".mp3", mimetype="audio/mpeg"),
|
||||
)
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "prebuilt-audioSearch"
|
||||
|
||||
def test_document_analyzer_routes_mp4_to_prebuilt(self):
|
||||
"""Document-based analyzer should auto-route MP4 to prebuilt-videoSearch."""
|
||||
conv = _make_converter(
|
||||
analyzer_id="my-doc-analyzer",
|
||||
analyzer_modality="document",
|
||||
)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake video"),
|
||||
StreamInfo(extension=".mp4", mimetype="video/mp4"),
|
||||
)
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "prebuilt-videoSearch"
|
||||
|
||||
def test_no_analyzer_id_uses_auto_routing(self):
|
||||
"""Without analyzer_id, PDF should auto-route to prebuilt-documentSearch."""
|
||||
conv = _make_converter(analyzer_id=None, analyzer_modality=None)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake pdf"),
|
||||
StreamInfo(extension=".pdf", mimetype="application/pdf"),
|
||||
)
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "prebuilt-documentSearch"
|
||||
|
||||
def test_no_analyzer_id_routes_image_to_document_search(self):
|
||||
"""Default image routing should still use prebuilt-documentSearch."""
|
||||
conv = _make_converter(analyzer_id=None, analyzer_modality=None)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake image"),
|
||||
StreamInfo(extension=".jpg", mimetype="image/jpeg"),
|
||||
)
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "prebuilt-documentSearch"
|
||||
|
||||
def test_document_analyzer_routes_image_to_custom(self):
|
||||
"""Document-based analyzers should still handle image documents."""
|
||||
conv = _make_converter(
|
||||
analyzer_id="my-doc-analyzer",
|
||||
analyzer_modality="document",
|
||||
)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake image"),
|
||||
StreamInfo(extension=".jpg", mimetype="image/jpeg"),
|
||||
)
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "my-doc-analyzer"
|
||||
|
||||
def test_image_analyzer_routes_jpeg_to_custom(self):
|
||||
"""Image-based analyzers should be used for image files."""
|
||||
conv = _make_converter(
|
||||
analyzer_id="my-image-analyzer",
|
||||
analyzer_modality="image",
|
||||
)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake image"),
|
||||
StreamInfo(extension=".jpg", mimetype="image/jpeg"),
|
||||
)
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "my-image-analyzer"
|
||||
|
||||
def test_image_analyzer_routes_pdf_to_document_prebuilt(self):
|
||||
"""Image-based analyzers should not claim non-image document files."""
|
||||
conv = _make_converter(
|
||||
analyzer_id="my-image-analyzer",
|
||||
analyzer_modality="image",
|
||||
)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(
|
||||
io.BytesIO(b"fake pdf"),
|
||||
StreamInfo(extension=".pdf", mimetype="application/pdf"),
|
||||
)
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "prebuilt-documentSearch"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mimetype", "expected_analyzer"),
|
||||
[
|
||||
("video/mp4", "prebuilt-videoSearch"),
|
||||
("video/x-m4v", "prebuilt-videoSearch"),
|
||||
("audio/mpeg", "prebuilt-audioSearch"),
|
||||
("audio/x-wav", "prebuilt-audioSearch"),
|
||||
],
|
||||
)
|
||||
def test_mime_only_input_uses_auto_routing(self, mimetype, expected_analyzer):
|
||||
"""MIME-only streams should route to the matching modality analyzer."""
|
||||
conv = _make_converter(analyzer_id=None, analyzer_modality=None)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(io.BytesIO(b"fake content"), StreamInfo(mimetype=mimetype))
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == expected_analyzer
|
||||
|
||||
def test_mime_alias_input_uses_canonical_content_type(self):
|
||||
"""Alias MIME types should be sent to CU as canonical content types."""
|
||||
conv = _make_converter(analyzer_id=None, analyzer_modality=None)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(io.BytesIO(b"fake video"), StreamInfo(mimetype="video/x-m4v"))
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "prebuilt-videoSearch"
|
||||
assert call_args.kwargs["content_type"] == "video/mp4"
|
||||
|
||||
def test_extension_only_input_uses_file_type_content_type(self):
|
||||
"""Extension-only inputs should send CU a matching content type."""
|
||||
conv = _make_converter(analyzer_id=None, analyzer_modality=None)
|
||||
conv._client = MagicMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch("markitdown.converters._cu_converter.to_llm_input", return_value=""):
|
||||
conv.convert(io.BytesIO(b"fake pdf"), StreamInfo(extension=".pdf"))
|
||||
|
||||
call_args = conv._client.begin_analyze_binary.call_args
|
||||
assert call_args.kwargs["analyzer_id"] == "prebuilt-documentSearch"
|
||||
assert call_args.kwargs["content_type"] == "application/pdf"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _infer_prebuilt_modality tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResolveAnalyzerModality:
|
||||
"""Test modality resolution from analyzer IDs."""
|
||||
|
||||
def test_known_document_prebuilts(self):
|
||||
client = MagicMock()
|
||||
assert (
|
||||
_resolve_analyzer_modality(client, "prebuilt-documentSearch") == "document"
|
||||
)
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-invoice") == "document"
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-layout") == "document"
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-receipt") == "document"
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-tax.us.w2") == "document"
|
||||
# Known prebuilts should never call get_analyzer()
|
||||
client.get_analyzer.assert_not_called()
|
||||
|
||||
def test_known_audio_prebuilts(self):
|
||||
client = MagicMock()
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-audioSearch") == "audio"
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-callCenter") == "audio"
|
||||
client.get_analyzer.assert_not_called()
|
||||
|
||||
def test_known_video_prebuilts(self):
|
||||
client = MagicMock()
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-videoSearch") == "video"
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-videoSynopsis") == "video"
|
||||
client.get_analyzer.assert_not_called()
|
||||
|
||||
def test_known_image_prebuilts(self):
|
||||
client = MagicMock()
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-imageSearch") == "image"
|
||||
assert _resolve_analyzer_modality(client, "prebuilt-image") == "image"
|
||||
client.get_analyzer.assert_not_called()
|
||||
|
||||
def test_unknown_prebuilt_falls_back_to_get_analyzer(self):
|
||||
"""Unknown prebuilt-* names should call get_analyzer() for resolution."""
|
||||
client = MagicMock()
|
||||
mock_analyzer = MagicMock()
|
||||
mock_analyzer.base_analyzer_id = "prebuilt-audio"
|
||||
client.get_analyzer.return_value = mock_analyzer
|
||||
|
||||
result = _resolve_analyzer_modality(client, "prebuilt-newAnalyzer")
|
||||
assert result == "audio"
|
||||
client.get_analyzer.assert_called_once_with("prebuilt-newAnalyzer")
|
||||
|
||||
def test_custom_analyzer_calls_get_analyzer(self):
|
||||
"""Custom analyzers should call get_analyzer() to resolve modality."""
|
||||
client = MagicMock()
|
||||
mock_analyzer = MagicMock()
|
||||
mock_analyzer.base_analyzer_id = "prebuilt-document"
|
||||
client.get_analyzer.return_value = mock_analyzer
|
||||
|
||||
result = _resolve_analyzer_modality(client, "my-custom-doc-analyzer")
|
||||
assert result == "document"
|
||||
client.get_analyzer.assert_called_once_with("my-custom-doc-analyzer")
|
||||
|
||||
def test_custom_analyzer_no_base_defaults_to_document(self):
|
||||
"""Analyzer with no base_analyzer_id defaults to document."""
|
||||
client = MagicMock()
|
||||
mock_analyzer = MagicMock()
|
||||
mock_analyzer.base_analyzer_id = None
|
||||
client.get_analyzer.return_value = mock_analyzer
|
||||
|
||||
result = _resolve_analyzer_modality(client, "my-custom-analyzer")
|
||||
assert result == "document"
|
||||
|
||||
def test_get_analyzer_failure_raises_value_error(self):
|
||||
"""Failed get_analyzer() should raise ValueError."""
|
||||
client = MagicMock()
|
||||
client.get_analyzer.side_effect = Exception("not found")
|
||||
|
||||
with pytest.raises(ValueError, match="Failed to resolve analyzer 'bad-id'"):
|
||||
_resolve_analyzer_modality(client, "bad-id")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _get_modality tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetModality:
|
||||
"""Test file type → modality mapping."""
|
||||
|
||||
def test_document_types(self):
|
||||
assert _get_modality(ContentUnderstandingFileType.PDF) == "document"
|
||||
assert _get_modality(ContentUnderstandingFileType.DOCX) == "document"
|
||||
|
||||
def test_image_types(self):
|
||||
assert _get_modality(ContentUnderstandingFileType.JPEG) == "image"
|
||||
assert _get_modality(ContentUnderstandingFileType.PNG) == "image"
|
||||
|
||||
def test_video_types(self):
|
||||
assert _get_modality(ContentUnderstandingFileType.MP4) == "video"
|
||||
assert _get_modality(ContentUnderstandingFileType.MOV) == "video"
|
||||
|
||||
def test_audio_types(self):
|
||||
assert _get_modality(ContentUnderstandingFileType.WAV) == "audio"
|
||||
assert _get_modality(ContentUnderstandingFileType.MP3) == "audio"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# convert() mock tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConvertMock:
|
||||
"""Test convert() with mocked CU SDK."""
|
||||
|
||||
def _run_convert(self, extension, mimetype, expected_output="mock output"):
|
||||
conv = _make_converter()
|
||||
conv._client = MagicMock()
|
||||
|
||||
mock_result = MagicMock()
|
||||
mock_result.contents = []
|
||||
mock_poller = MagicMock()
|
||||
mock_poller.result.return_value = mock_result
|
||||
conv._client.begin_analyze_binary.return_value = mock_poller
|
||||
|
||||
with patch(
|
||||
"markitdown.converters._cu_converter.to_llm_input",
|
||||
return_value=expected_output,
|
||||
):
|
||||
result = conv.convert(
|
||||
io.BytesIO(b"fake content"),
|
||||
StreamInfo(extension=extension, mimetype=mimetype),
|
||||
)
|
||||
return result
|
||||
|
||||
def test_pdf_returns_markdown(self):
|
||||
result = self._run_convert(
|
||||
".pdf", "application/pdf", "---\ncontentType: document\n---\n# Test"
|
||||
)
|
||||
assert "contentType: document" in result.markdown
|
||||
|
||||
def test_mp4_returns_markdown(self):
|
||||
result = self._run_convert(
|
||||
".mp4", "video/mp4", "---\ncontentType: audioVisual\n---\nSpeaker 1: Hello"
|
||||
)
|
||||
assert "contentType: audioVisual" in result.markdown
|
||||
|
||||
def test_wav_returns_markdown(self):
|
||||
result = self._run_convert(
|
||||
".wav", "audio/wav", "---\ncontentType: audioVisual\n---\nSpeaker 1: Hi"
|
||||
)
|
||||
assert "audioVisual" in result.markdown
|
||||
|
||||
def test_empty_result(self):
|
||||
result = self._run_convert(".pdf", "application/pdf", "")
|
||||
assert result.markdown == ""
|
||||
|
||||
def test_jpeg_returns_markdown(self):
|
||||
result = self._run_convert(
|
||||
".jpg", "image/jpeg", "---\ncontentType: document\n---\n# Photo"
|
||||
)
|
||||
assert "contentType: document" in result.markdown
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Init-time get_analyzer() error wrapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetAnalyzerError:
|
||||
"""Test that get_analyzer() failures at init produce a clear error."""
|
||||
|
||||
def test_nonexistent_analyzer_raises_value_error(self):
|
||||
"""A failed get_analyzer() should raise ValueError with analyzer name."""
|
||||
with patch(
|
||||
"markitdown.converters._cu_converter._dependency_exc_info", None
|
||||
), patch(
|
||||
"markitdown.converters._cu_converter.ContentUnderstandingClient"
|
||||
) as MockClient, patch(
|
||||
"markitdown.converters._cu_converter.DefaultAzureCredential"
|
||||
):
|
||||
mock_client = MagicMock()
|
||||
mock_client.get_analyzer.side_effect = Exception("not found")
|
||||
MockClient.return_value = mock_client
|
||||
|
||||
with pytest.raises(ValueError, match="Failed to resolve analyzer 'bad-id'"):
|
||||
ContentUnderstandingConverter(
|
||||
endpoint="https://fake", analyzer_id="bad-id"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registration priority test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistrationPriority:
|
||||
"""Test that CU converter is registered with higher priority than Doc Intel."""
|
||||
|
||||
def test_cu_registered_before_docintel(self):
|
||||
"""When both endpoints are provided, CU should appear before Doc Intel."""
|
||||
with patch(
|
||||
"markitdown.converters._cu_converter._dependency_exc_info", None
|
||||
), patch(
|
||||
"markitdown.converters._cu_converter.ContentUnderstandingClient"
|
||||
), patch(
|
||||
"markitdown.converters._cu_converter.DefaultAzureCredential"
|
||||
), patch(
|
||||
"markitdown.converters._doc_intel_converter._dependency_exc_info", None
|
||||
), patch(
|
||||
"markitdown.converters._doc_intel_converter.DocumentIntelligenceClient"
|
||||
), patch(
|
||||
"markitdown.converters._doc_intel_converter.DefaultAzureCredential"
|
||||
):
|
||||
from markitdown import MarkItDown
|
||||
from markitdown.converters import (
|
||||
ContentUnderstandingConverter,
|
||||
DocumentIntelligenceConverter,
|
||||
)
|
||||
|
||||
md = MarkItDown(
|
||||
cu_endpoint="https://fake-cu",
|
||||
docintel_endpoint="https://fake-di",
|
||||
)
|
||||
|
||||
converter_types = [type(reg.converter) for reg in md._converters]
|
||||
cu_idx = converter_types.index(ContentUnderstandingConverter)
|
||||
di_idx = converter_types.index(DocumentIntelligenceConverter)
|
||||
assert (
|
||||
cu_idx < di_idx
|
||||
), "CU should have higher priority (lower index) than Doc Intel"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI argument tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCLIArgs:
|
||||
"""Test CLI argument parsing for CU flags."""
|
||||
|
||||
def test_use_cu_without_endpoint_exits(self):
|
||||
"""--use-cu without --cu-endpoint should exit with error."""
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "markitdown", "--use-cu", "fake.pdf"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode != 0
|
||||
assert (
|
||||
"cu-endpoint" in result.stderr.lower()
|
||||
or "cu-endpoint" in (result.stdout or "").lower()
|
||||
)
|
||||
|
||||
def test_use_cu_and_use_docintel_mutually_exclusive(self):
|
||||
"""--use-cu and --use-docintel cannot be used together."""
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"markitdown",
|
||||
"--use-cu",
|
||||
"--cu-endpoint",
|
||||
"https://fake",
|
||||
"--use-docintel",
|
||||
"-e",
|
||||
"https://fake-di",
|
||||
"fake.pdf",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode != 0
|
||||
|
||||
def test_cu_file_types_parsing(self):
|
||||
"""--cu-file-types should parse comma-separated values into enum list."""
|
||||
from markitdown.converters import ContentUnderstandingFileType
|
||||
|
||||
raw = "pdf,jpeg,mp4"
|
||||
type_names = [t.strip().lower() for t in raw.split(",") if t.strip()]
|
||||
cu_types = [ContentUnderstandingFileType(name) for name in type_names]
|
||||
|
||||
assert cu_types == [
|
||||
ContentUnderstandingFileType.PDF,
|
||||
ContentUnderstandingFileType.JPEG,
|
||||
ContentUnderstandingFileType.MP4,
|
||||
]
|
||||
|
||||
def test_cu_file_types_invalid_value(self):
|
||||
"""Unknown file type name should raise ValueError."""
|
||||
from markitdown.converters import ContentUnderstandingFileType
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
ContentUnderstandingFileType("nonsense")
|
||||
|
||||
def test_cu_file_types_single_value(self):
|
||||
"""Single file type (no comma) should parse correctly."""
|
||||
from markitdown.converters import ContentUnderstandingFileType
|
||||
|
||||
cu_types = [
|
||||
ContentUnderstandingFileType(t.strip().lower())
|
||||
for t in "wav".split(",")
|
||||
if t.strip()
|
||||
]
|
||||
assert cu_types == [ContentUnderstandingFileType.WAV]
|
||||
|
||||
def test_use_cu_wires_kwargs_to_markitdown(self, capsys):
|
||||
"""--use-cu should pass CU options through to MarkItDown."""
|
||||
import markitdown.__main__ as markitdown_cli
|
||||
|
||||
markitdown_instance = MagicMock()
|
||||
markitdown_instance.convert.return_value.markdown = "converted"
|
||||
markitdown_cls = MagicMock(return_value=markitdown_instance)
|
||||
|
||||
with patch.object(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
"markitdown",
|
||||
"--use-cu",
|
||||
"--cu-endpoint",
|
||||
"https://fake-cu",
|
||||
"--cu-analyzer",
|
||||
"custom-analyzer",
|
||||
"--cu-file-types",
|
||||
"pdf,jpeg,mp4",
|
||||
"fake.pdf",
|
||||
],
|
||||
), patch.object(markitdown_cli, "MarkItDown", markitdown_cls):
|
||||
markitdown_cli.main()
|
||||
|
||||
markitdown_cls.assert_called_once_with(
|
||||
enable_plugins=False,
|
||||
cu_endpoint="https://fake-cu",
|
||||
cu_analyzer_id="custom-analyzer",
|
||||
cu_file_types=[
|
||||
ContentUnderstandingFileType.PDF,
|
||||
ContentUnderstandingFileType.JPEG,
|
||||
ContentUnderstandingFileType.MP4,
|
||||
],
|
||||
)
|
||||
markitdown_instance.convert.assert_called_once_with(
|
||||
"fake.pdf", stream_info=None, keep_data_uris=False
|
||||
)
|
||||
assert capsys.readouterr().out == "converted\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MissingDependencyException test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMissingDependency:
|
||||
"""Test that MissingDependencyException is raised when CU SDK is not installed."""
|
||||
|
||||
def test_missing_deps_message(self):
|
||||
"""Converter construction should surface the optional install hint."""
|
||||
import markitdown.converters._cu_converter as cu_converter_module
|
||||
from markitdown._exceptions import MissingDependencyException
|
||||
|
||||
import_error = ImportError("No module named 'azure.ai.contentunderstanding'")
|
||||
dependency_exc_info = (ImportError, import_error, None)
|
||||
|
||||
with patch.object(
|
||||
cu_converter_module, "_dependency_exc_info", dependency_exc_info
|
||||
), pytest.raises(MissingDependencyException) as exc_info:
|
||||
ContentUnderstandingConverter(endpoint="https://fake-cu")
|
||||
|
||||
assert "az-content-understanding" in str(exc_info.value)
|
||||
assert exc_info.value.__cause__ is import_error
|
||||
@@ -0,0 +1,26 @@
|
||||
import io
|
||||
from markitdown.converters._doc_intel_converter import (
|
||||
DocumentIntelligenceConverter,
|
||||
DocumentIntelligenceFileType,
|
||||
)
|
||||
from markitdown._stream_info import StreamInfo
|
||||
|
||||
|
||||
def _make_converter(file_types):
|
||||
conv = DocumentIntelligenceConverter.__new__(DocumentIntelligenceConverter)
|
||||
conv._file_types = file_types
|
||||
return conv
|
||||
|
||||
|
||||
def test_docintel_accepts_html_extension():
|
||||
conv = _make_converter([DocumentIntelligenceFileType.HTML])
|
||||
stream_info = StreamInfo(mimetype=None, extension=".html")
|
||||
assert conv.accepts(io.BytesIO(b""), stream_info)
|
||||
|
||||
|
||||
def test_docintel_accepts_html_mimetype():
|
||||
conv = _make_converter([DocumentIntelligenceFileType.HTML])
|
||||
stream_info = StreamInfo(mimetype="text/html", extension=None)
|
||||
assert conv.accepts(io.BytesIO(b""), stream_info)
|
||||
stream_info = StreamInfo(mimetype="application/xhtml+xml", extension=None)
|
||||
assert conv.accepts(io.BytesIO(b""), stream_info)
|
||||
BIN
Binary file not shown.
+97
@@ -0,0 +1,97 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document http://www.reportlab.com
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R /F3 4 0 R /F4 5 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/BaseFont /Courier /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/BaseFont /Courier-Bold /Encoding /WinAnsiEncoding /Name /F4 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/BitsPerComponent 8 /ColorSpace /DeviceRGB /Filter [ /ASCII85Decode /FlateDecode ] /Height 70 /Length 4491 /Subtype /Image
|
||||
/Type /XObject /Width 200
|
||||
>>
|
||||
stream
|
||||
Gb"/lq,^Nc)M\9OkX:DBZ5YT>'!op&`0lHCEL`PXM2DFT$QuCdPsfSJ4#%$gW49i\1e&eZ\Acg:2bhaPc+^Q$/Shs#,1&Qu>83CBh729[%A$M]]Z8KL]Cu-OpO.1`\pCPboa2!3#sCC+4Yg#.W)"\K&i)doY5WCH.J-G0E$]%J"BRoZ88okcKEP@C7S%JEA:t(e6:OLb-"MZ3=$fAIE$]%J"BRoZ88okcKEP@C7S%JEA:t(e6:OKV=+u3F\<8ml>&.H@]EH:@TAeD]);ilGV3,!4G2kP0XRN"#r/]G!Kqo1c9Redk1d%Hh[t82=;lMkgMFr4J2#lTY[siCuNDGT]h^te4%1fj10r$&D--;(2UPbQ(Ze:VUL"2G=%qPZVOlc,tegG*BO,:$mI%mCAJ^8q2gWSn>)Ui.KQ!A5_(Z;l?_%(Xb28AGIU0SY?bVp%B6=$P2*I!1?W8WL>aAVWc$%-nk=D8ZjE$stW]LJ-LqIj9qZoFV/lj$Uf)=b`nfl*ANkt_qpb2t'P`;D#\h6o+g'M+"j4Sf:d#_jjZdTS>mnQJm^7>(S3Bq"m(kH5i3;0`YK<5e.0$k"4XA4UX)rfXH+2OamR360'cX$&"Dp"DdSkh^Q;?+H)fBc@YMp?\]UZuQ*lgt@kDS'dARs/]`Roa+]eXm%&TJ4e[RCKq6kQ:5Zpa[hLTEE,M/UR\C]SS.K'HJL7F)F5Ts63hWCKs+aTqi3TN!,P7#o$@'a?^`M.9&=d,$WJ*\b^*N1fR$JFV.s`.W*WfCgS\9h@C2/uC<b(1#bB73rR3UmcP"%)_DZ#=)TA1KkfUBT8F;=Yoc[BR6[ZkS\Y$n/.@mf!9WK1Hj]o3[f>DiFrdD&a`iRQ\df2(Xc53$=i@@upbP,MJ@.sDgm%`^.8+5u//"Hhn%^kIb$5o\2B7%r>DD\NZ:L;otY0]:)'6l[M<&ctoM"($Q_XW1'!4OB>g/3dF]mD],eF*&&'itQ;2e$/VWZ/QmdogQ0&d7ePkDGP[PZkk8TtUWkaJYa$Q)c6I+l<preqG)K\U>pY5H])D-lHdp52<d:Isd8)X0&b+pKUugDNb2NIW.aD_PLN/i(r9N&<3?,2br'%?gT'_i;n9VUeeM#>ko&@JS]_pP&PR0@L`i*pbXB"rgcI`#'#>-Njfe@8+ZC7hHU>Qm2oCj$^ATs<7[sZ@5*,@qslQ\p1m1#p6XrGL'F^?ok?+\fDe1,#<0n78&1'&KK/85E7IuiRklZ$<tM_`dQfdI@$2&Sj]k*=&V0n1RVEA-6,(U`EJ9674P)af%Z>l=<9-YKka`e!9&oUk]3u7Y);o;!X[W(:<D3M@"gDG)[;",CR0eT^r/fRnB+ob8JM\tp9\JoC\=uqFPX9nU`?:Y,eJf!6E95r@KI_iekuY/-+j6DIrXFQW>i@m+VqQff?4r\fn@@4QXN6[dWdtV8:B`3X2:bgH!rR8-r^sf)#EN%"`F/q0Heh_C7H6l'.@I3l<Jr.Q!as3DB-9V*+/'h,_<T8?^2u*t.p$h8d%"Dd\P<5M`MEg>7W_M8qB0Sd$'o&pWH!XFNS.JRZ%[WY$N:rl5tLIb;#&1u\'nOCIB]161$bC,Uuf;ZK6dl()epY<39X_LaOAXU:WAZiYqZk5Tq+hN`O"QdZ7[jLdf^cf`?9i4T#=]JO$'0fC4#Y=^M%VOouL.PZ6/V;r+XoF1Ls*YXu`6'4?,j47_u_U=.T*IX0ed;@5JN=Qlc\gS!W?r;#%jA)NSUh\`='l:HWsF<K@<`EqO<,Ht[H.@PGU,p6$s&YbEgb;cfG9YK,6Fh]@t(EUD@78Ob6ui6[#pIBoZn<UH)N"PrIeP3Y!qa-k8bP>_rQ_q'7l3]f,=As5FN;rm6/&IWa@[9HFr3YtU'N%=ZPr)s`!!&o!IbLLsBH7VnG&"_&hSn3;Nr^mpSZk^2i_aD8<g*:f)-)1j6@3KSHbb_c1PDAXpnkGE:H3Fs0m?uXff0>H]^Oi^Wq(2*ak3>^mA^!FkG4$-Vq(BH"U+YSR;%(5j(bnT,&RrR1d]\O5_42^f/Xa:4msf,Oms&5F6()XE"p6mS/Yc\Ga&`hC/3XdsM;'cTMl(uV@DiFY5AA_VWS4T'&^D<.7.S,`B?:^&!Q[ZVaCi0^$[E#=Xt_;^\;l;M#]`$4;sLf^6$u5)gpB'7TO-@*HXbXF]H[ID>=n%&8-T:f)&:]?hm\RN5/B5sdW)PM[X@>2Jq/jQ%m%0Pk%J`<],L8cn3_`B)dE8ng`*C-";2ro.7o.B2:3d$4r#LEqr#8((eIunkG+V@25V=%+_!:Z/,UQca-*F<WEc]8T!rP(2A>g9*GW$LPXTF:ER(d"o8oX31"!B8VtcoXnY$9m&'30Um_8lI0aO.LY_m,l[3VfKlBY*[!$I#3=:"_\lGs6U"<,-kF4HFN$6[_SQpG)7_H3mnKRB'C9#q8EY(Vaqi(D&r$*Jr?OPiaP#RRYeN0)sia9W*TKT)#N9#q8EY(Vaqi(D&r$*Jr?OPiaP#k^2Zeu*-9hp_.0@)f6Em^OKjQe!N6R'K"tCBm#IB,_F0s/@WUCJX)`/6>DF6M_)Kk.s>$UZPX\r/#8I8q/RiHL2liUAA0Yf`%Ld=068s:8FSq=\;#qPO.M6gNJ<fcJ_B7NfVHbGb(Z#%=9b&C:^$<Xe!_fU:_#gd*5iB@GeEYKud7*YgG3W,jj8\q'/4@l'Y-g\)rX-m;H77/R]Zb%A!YN(tcDfkrJ&o7($5o=b[,R0&h>j\UuBER`krd-YJajg!X[eDSKG#rM875C*#IQWm+O@aOf*TU2U('gKn4DA1c#r\17+;`V`#Kgpm8RC[Ee$Ac&[r[O5Rd>4]?HeBRku2;R,jekQW/J-QCRW&^t>C\3e,:[W(TT!T:2JEND71MtHndISe*l)E$(\gO^@6Z"5Q<4O)uM0mY8/2q_>,e#m"SaP0HaXDih_2UgJZH0.io.<EHam+)Ba'CJ3$,Ve\2W\TOCDf>!]XNj5RmJj5Qb:1EKuS*^?b+;s)^?6l>JE0QrGSUpouIlZac9kX41>cja=/SDr<cA`ZGg.-dcfEr]UahTX\[1!?g.Q+Bc7gR:Jn+!PFgH?<j_Cphp%2cK2o3EmuaRlLL,1SFd.EYSt<=je7H>"[/Y4/S?T:Ij;Z5Vm!N#O9oGbTmC/.8"?)WA+NO#l!d__>E^^P^^5SPK3f#a-$jDf&=>7p3h\GbV.%e"aa`Nr8:-((uUWjP?5h<QpA5fPW)4^9]4ZVP3n=aOl2n\TH8,2g(<%d(S%%7D#]P?GegI?jC3uo^tm=sN6Bp.;hI!R:5>*[4qX1g^KT=J\eG*=-r!s:Xj;^b_=6X9pT)tmOPN+SP\Y9d?!CS[k5Z"1@+N1#RUIImf:>$etSOp!A$%9Npa7^LW;"'>&DLN*LT#=""p0WOpjXfI:C##@j@lDSU=Xe\&,@B>QE*Z4EfD\=1"h4F8$&QPkBC:BBC"p$N?^/:S9o*P:eF4j>`WT<EG,f\ln6T<>&V*-UR?1+=iZ&Y:YJ)Y_O1Q!(3MOc:&,lEK0KT>gBrM!Oo7g$R0Z=@n</A>l[op[I#4)k.C3f6pb.hq#9_d636^F1(A%Us@pB(WNXIQ#TKD-`)0%k[fj0?XEDjbhd[m6#LpLW.'sd9_sqo4)9,(HjMXDMbKZE5`!!P4XRB@/4TISu+m2&%RiFj^;=JGE6IZZZ3Iq9u;tP9Ze)spB!TH+!k0kjm?9TaEqM'"N]I#K68.sENFpG-:BpL"k5<Kf;Cll]p\0.VBgJXYV7bkGELTah(>RWGsjL14<<sE:X3JW]):L"_a3kcBH&.(2Ui6d<sTVSq6Hb2oGX:N_j+E^^?#]Brd-YC/q4&j48+1kK<)&.;T[(aqqDC]Z:"7NhLIO5&BeU-SXc>m&CBpaKs!LDjG%ZThfU"?o'^kNBIDSK`iDtti"$XI@8UFlc6'SU-*bNc0g0qZU.!3k5:g&J5%_GC<>>?gMnQALpYLh_[CK2YgZMfbcG=f9!FE+GUto#YS.Ms0oqhppBVFIgKikS73+[l77Q]oKP@6W$g?K-&@CV\IU2l9TjknpF,mi!B#31b$WjNK[_SbrBad@)rY(?tsa<O6[4Wd_rrCQDK9Fn>?noi;W9O[OS+IIT/JfXc1#&UNmnp:7_c6/aW-".*cBVjc)"DgbY/sJI7I:YTo4n^?6D*J='=Z.q\rms\C-WjI*igXS;Vfo_`Y\`_c8K6r=SE5*WPmG+p6'k&&0eDn]e<kl$//^-6n?[M0Wg:@`F!W*m'jM%_,JfY,&JA=T)'Qh]O:`+1#oOo&Q&lRj>R;8k_3L)o&mP_\+Z6EUKR,;fQ&lT(!/=AJY_cg)ZFd_iV[)$d`W(]Q<7PapAWM]!Y4qoZ_Z9e(Tts:W\F3=&Fc,Mp(`=cJUG]7+gkp\4)_AlnCa6gYn$_U<4&\@+ERR^ZIFpqbN%;:=3k=P0?fC_:0&Ug9TYUE[kI#Bq;+mc:C\/7H:]h-hq^[P:`jZ\paVNd3BCYF4eruZ)J"F0tYQ"^`hf0;~>endstream
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Contents 11 0 R /MediaBox [ 0 0 216 792 ] /Parent 10 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /XObject <<
|
||||
/FormXob.2a351979d8c75d073b2ea4bfb74718f9 6 0 R
|
||||
>>
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 10 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20251205104951+01'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20251205104951+01'00') /Producer (ReportLab PDF Library - www.reportlab.com)
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 7 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
11 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1981
|
||||
>>
|
||||
stream
|
||||
GauI8D/\/e&:hOi=55KJ8;^%$X%4*F0ZR4[TQ-.JlIl=8,25emeWsH3DOUP#SK,Tt>!%\QTf-@>5)IUB4PgI,!l6hOHqp".7kqY?VEc$;f0G$C+?kE+IduY+D6B[cY<`?1&+Bf$SSWa.DI28'F?:CpG_mY"TO]hkXiFku&",h"4G/8GlCSK3UDT`3W#q'Qc;4>t6\tbspa(/l?]"D>nboQo,(,[\*-A;J=Ru^j=[Nu\:iMk7q<+PGo*gWpT4_C)j)t7Oc[5MlffWrhj!99;,0?]r3R(ns^B^I*KQ#f[([aS1g);Q.rrBep&6)sVJs=\.1^pkCa(tBfECI75_;C0LC.)*n<3;@eFZT<Brd%CYO%*fRCl_%R2PLtnF>0lg_SFKN$O.X\o%U7_58YJ,X[`p,%PUL^1]EgT]T\4*B3hOrEZA:[=ui88pZGht%klE^OC$2=@$GiMoTO<eR\C2;10r\O.%_7=c.`)*@0N>,CLh>2ZDq?"(LrLS)ajJ<DG(:N]5MPuT)E6J8.)!Ud7D>Je7M&(V1i'>Z@qg3/WJ@PpL4nr1qU$V_#jqpM+M<[H(LE:pW6uTQK`^P%Q'T&[*Y1\T_7.O:+n^Y)3+d56\hGsIrnqB85q[4L1WG#"Uo.d]Zr"jG`qiT=AU8b5]p3(N2IfnWHVO&$rnNe6[_$[o(m=2Sq-C[bbNOS,qIb?:TGYHhQjjcCe!9%*cuscgU*Eea^B?#^HtoE9p(jd_GR#E1\LTm:MVS5e]+<LZRQ]0^iZNnTs\Iq5l6H+?80%j?IX^UR28jY=Vr!:#Jf=D$QdR#4X6Q%Z^E6hq4[p#rHu/mN!PgeQCn_hEI9M3(_b.pAid<e0?KUakkhL+Sq1A+!S(V0h+OF8nn#&[1+6p+D5^<OP@s\HS\itN&+apX>;a8<=<fVm-cM(u=Q31UTuXZiRNk/X^o=e`8?ha=(l,J:AYGq&k'81mIs68U9).dPb@tBY#5s6I1(;=p-FNV8JLO6-b%6]BUEO#*P:YXQ,+2T8!GeA%OFD*^H'I<qo6Q\KGcc$9<-q;oCHo2eF[F/t'nG3p8bj]<0qUd^A*Un<D3^J]5-EeYaHZL0]dZ(aldZ?U]EL]o1@j;L=l_$._u&B5KRKtY900d3EO'mY6&5WB,D7o]o+7,#h.[N58L+)*ks!_/dIq7L<$Q/>:Ym/3(NJmP3]c2J81f'[9A229?.>nW.Y"uioK$/X(RLnTFa0nhiu#_V(M%6pL-[3&IEZO^iW'pcgSC4%cs*UfWL8=h@<SF-6Ml.SK#\%/6pL;XKVP08]+YR4.1^h^3g+iL6p2jNFKi##9N\7TS:EE')be-k57a"IMZoUV#>]Eoq_3uS@i.]*ai31P3"'$;S,Q%'"=$Vq"-_!pX<>bA]?nd=dOpZa\$"k!cpI9L2SO3gBd7]TKi)s)3F/ADnb^3N)iF')M[\Fq1\^PAl):!YpJp!B\s/2LkZr*`(o%fTO.qa?N[7\P_Dj!-3=OAO3]DtNKn-R1Hc#$?$h;RW[;B(k%DrOQ4lA(kZ`8[,.5E\H/%&9RE1k-pKk^'?Wseh?':/9RD3&&Xf\j=9;Sdd#l!b5li$Q.?@#FtJ9r"D*THt>o=+h,ei<3VCI\e<F.YjMklHmQ252@7%.$dl?FX[.5Ru_<cOnWObGU$sud3sn0Nm?VSip=&P_8=3<b5l"NFchqcT66k!jof;<?"kHRj[>Q+FB(V85-;*H\(QoM*>m2@9WKA`dV0$2F]lQ!^cKY?-F<<RYBD9:P`&#<:DJpSA0L]L_Q`8=5'6'r`p_54_;lcH+H=)4\l8B7YE#pX>K&Mf4jEn:L@C'pmu(T(NAo?onFtPTH*Mah:OJII8OF6<oMipM;1-5S+stnT,o"n]+UmpI>_OX,SeHS'86i`]nN=oW_Hjm1lb%agT!)1^3rJWom\/,?BYNjThVR,cQ'opa8Q#<G9<qeSN'GRRO*(AC(K$'<9uMACIm=MV?Mk2Q*3P"~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 12
|
||||
0000000000 65535 f
|
||||
0000000073 00000 n
|
||||
0000000134 00000 n
|
||||
0000000241 00000 n
|
||||
0000000353 00000 n
|
||||
0000000458 00000 n
|
||||
0000000568 00000 n
|
||||
0000005249 00000 n
|
||||
0000005507 00000 n
|
||||
0000005576 00000 n
|
||||
0000005859 00000 n
|
||||
0000005919 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<4800d64fefba4dd902e51197c7da4e88><4800d64fefba4dd902e51197c7da4e88>]
|
||||
% ReportLab generated PDF document -- digest (http://www.reportlab.com)
|
||||
|
||||
/Info 9 0 R
|
||||
/Root 8 0 R
|
||||
/Size 12
|
||||
>>
|
||||
startxref
|
||||
7992
|
||||
%%EOF
|
||||
Binary file not shown.
+115
File diff suppressed because one or more lines are too long
Binary file not shown.
Vendored
Vendored
+81
@@ -0,0 +1,81 @@
|
||||
TECHMART ELECTRONICS
|
||||
4567 Innovation Blvd
|
||||
San Francisco, CA 94103
|
||||
(415) 555-0199
|
||||
|
||||
===================================
|
||||
|
||||
Store #0342 - Downtown SF
|
||||
11/23/2024 14:32:18 PST
|
||||
TXN: TXN-98765-2024
|
||||
Cashier: Emily Rodriguez
|
||||
Register: POS-07
|
||||
|
||||
-----------------------------------
|
||||
|
||||
Wireless Noise-Cancelling
|
||||
Headphones - Premium Black
|
||||
AUDIO-5521 1 @ $349.99
|
||||
Member Discount $-50.00
|
||||
$299.99
|
||||
USB-C Hub 7-in-1 Adapter
|
||||
with HDMI & Ethernet
|
||||
ACC-8834 2 @ $79.99
|
||||
$159.98
|
||||
Portable SSD 2TB
|
||||
Thunderbolt 3 Compatible
|
||||
STOR-2241 1 @ $289.00
|
||||
Member Discount $-29.00
|
||||
$260.00
|
||||
Ergonomic Wireless Mouse
|
||||
Rechargeable Battery
|
||||
ACC-9012 1 @ $59.99
|
||||
$59.99
|
||||
Screen Cleaning Kit
|
||||
Professional Grade
|
||||
CARE-1156 3 @ $12.99
|
||||
$38.97
|
||||
HDMI 2.1 Cable 6ft
|
||||
8K Resolution Support
|
||||
CABLE-7789 2 @ $24.99
|
||||
Member Discount $-5.00
|
||||
$44.98
|
||||
-----------------------------------
|
||||
|
||||
SUBTOTAL $863.91
|
||||
Member Discount (15%)-$84.00
|
||||
Sales Tax (8.5%) $66.23
|
||||
Rewards Applied -$25.00
|
||||
===================================
|
||||
TOTAL $821.14
|
||||
===================================
|
||||
|
||||
PAYMENT METHOD
|
||||
Visa Card ending in 4782
|
||||
Auth: 847392
|
||||
Ref: REF-20241123-98765
|
||||
|
||||
-----------------------------------
|
||||
|
||||
REWARDS MEMBER
|
||||
Sarah Mitchell
|
||||
ID: TM-447821
|
||||
Points Earned: 821
|
||||
Total Points: 3,247
|
||||
Next Reward: $50 gift card
|
||||
at 5,000 pts (1,753 to go)
|
||||
|
||||
-----------------------------------
|
||||
|
||||
RETURN POLICY
|
||||
Returns within 30 days
|
||||
Receipt required
|
||||
Electronics must be unopened
|
||||
|
||||
*TXN98765202411231432*
|
||||
|
||||
Thank you for shopping!
|
||||
www.techmart.example.com
|
||||
|
||||
===================================
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
ZAVA AUTO REPAIR
|
||||
Certified Collision Repair
|
||||
123 Main Street, Redmond, WA 98052
|
||||
Phone: (425) 000-0000
|
||||
Preliminary Estimate (ID: EST-1008)
|
||||
| Customer Information | | | Vehicle Information | |
|
||||
| -------------------- | ------------------- | --- | ------------------- | ----------------- |
|
||||
| Insured name | Gabriel Diaz | | Year | 2022 |
|
||||
| Claim # | SF-1008 | | Make | Jeep |
|
||||
| Policy # | POL-2022-555 | | Model | Grand Cherokee |
|
||||
| Phone | (425) 111-1111 | | Trim | Limited |
|
||||
| Email | gabriel@contoso.com | | VIN | 1C4RJFBG2NC123456 |
|
||||
| | | | Color | White |
|
||||
| | | | Odometer | 9,800 |
|
||||
| Repair Order # | RO-20221108 | | Estimator | Ellis Turner |
|
||||
Estimate Totals
|
||||
| | | Hours | Rate | Cost |
|
||||
| ---------------- | --- | ----- | ---- | ----- |
|
||||
| Parts | | | | 2,100 |
|
||||
| Body Labor | | 2 | 150 | 300 |
|
||||
| Paint Labor | | 1.5 | 150 | 225 |
|
||||
| Mechanical Labor | | - | - | - |
|
||||
Supplies
|
||||
| | Paint Supplies | | | 60 |
|
||||
| ------------- | ------------------------ | --- | ------ | ------ |
|
||||
| | Body Supplies | | | 30 |
|
||||
| Other Charges | | | | 15 |
|
||||
| Subtotal | | | | 2,730 |
|
||||
| Sales Tax | | | 10.20% | 278.46 |
|
||||
| GRAND TOTAL | | | | 5,738 |
|
||||
| Note | Minor rear bumper repair | | | |
|
||||
This is a preliminary estimate for the visible damage of the vehicle. Additional damage / repairs / parts may be found
|
||||
after the vehicle has been disassembled and damaged parts have been removed. Suspension damages may be
|
||||
present, but can not be determined until an alignment on the vehicle has been done. Parts Prices may vary due to
|
||||
models and vehicle maker price updates. Please be advised if vehicle owner elects to have vehicle sent to service for
|
||||
any mechanical concerns, ALL service departments charge a vehicle diagnostic charge. If the mechanical concern is
|
||||
deemed not related to an insurance claim, vehicle owner will be reponsible for charges.
|
||||
|
||||
ZAVA AUTO REPAIR
|
||||
Certified Collision Repair
|
||||
123 Main Street, Redmond, WA 98052
|
||||
Phone: (425) 000-0000
|
||||
Preliminary Estimate (ID: EST-1008)
|
||||
Customer Information Vehicle Information
|
||||
| Insured name | Bruce Wayne | | Year | 2025 |
|
||||
| -------------- | -------------------------- | --- | --------- | ------------ |
|
||||
| Claim # | | 999 | Make | Batman |
|
||||
| Policy # | IM-BATMAN | | Model | Batmobile |
|
||||
| Phone | (416) 555-1234 | | Trim | Limited |
|
||||
| Email | batman@wayneindustries.com | | VIN | XXX |
|
||||
| | | | Color | Black |
|
||||
| | | | Odometer | 1 |
|
||||
| Repair Order # | RO-20221108 | | Estimator | Ellis Turner |
|
||||
Estimate Totals
|
||||
| | | Hours | Rate | Cost |
|
||||
| ---------------- | --- | ----- | ---- | ------ |
|
||||
| Parts | | | | 99,999 |
|
||||
| Body Labor | | 2 | 150 | 300 |
|
||||
| Paint Labor | | 1.5 | 150 | 225 |
|
||||
| Mechanical Labor | | - | - | - |
|
||||
Supplies
|
||||
| | Paint Supplies | | | 60 |
|
||||
| ------------- | ------------------------ | --- | ------ | --------- |
|
||||
| | Body Supplies | | | 30 |
|
||||
| Other Charges | | | | 15 |
|
||||
| Subtotal | | | | 100,629 |
|
||||
| Sales Tax | | | 10.20% | 10264.158 |
|
||||
| GRAND TOTAL | | | | 211,522 |
|
||||
| Note | Minor rear bumper repair | | | |
|
||||
|
||||
This is a preliminary estimate for the visible damage of the vehicle. Additional damage / repairs / parts may be found
|
||||
after the vehicle has been disassembled and damaged parts have been removed. Suspension damages may be
|
||||
present, but can not be determined until an alignment on the vehicle has been done. Parts Prices may vary due to
|
||||
models and vehicle maker price updates. Please be advised if vehicle owner elects to have vehicle sent to service for
|
||||
any mechanical concerns, ALL service departments charge a vehicle diagnostic charge. If the mechanical concern is
|
||||
deemed not related to an insurance claim, vehicle owner will be reponsible for charges.
|
||||
Vendored
+44
@@ -0,0 +1,44 @@
|
||||
INVENTORY RECONCILIATION REPORT
|
||||
Report ID: SPARSE-2024-INV-1234
|
||||
Warehouse: Distribution Center East
|
||||
Report Date: 2024-11-15
|
||||
Prepared By: Sarah Martinez
|
||||
| Product Code | Location | Expected | Actual | Variance | Status |
|
||||
| ------------ | -------- | -------- | ------ | -------- | -------- |
|
||||
| SKU-8847 | A-12 | 450 | | | |
|
||||
| | B-07 | | 289 | -23 | |
|
||||
| SKU-9201 | | 780 | 778 | | OK |
|
||||
| | C-15 | | | +15 | |
|
||||
| SKU-4563 | D-22 | | 156 | | CRITICAL |
|
||||
| | | 180 | | -24 | |
|
||||
| SKU-7728 | A-08 | 920 | | | |
|
||||
| | | | 935 | +15 | OK |
|
||||
Variance Analysis:
|
||||
Summary Statistics:
|
||||
Total Variance Cost: $4,287.50
|
||||
Critical Items: 1
|
||||
Overall Accuracy: 97.2%
|
||||
Detailed Analysis by Category:
|
||||
The inventory reconciliation reveals several key findings. The primary variance driver is SKU-4563,
|
||||
which shows a -24 unit discrepancy requiring immediate investigation. Location B-07 handling of
|
||||
SKU-8847 also demonstrates significant variance. Cross-location verification protocols should be
|
||||
|
||||
reviewed to prevent future discrepancies. The overall accuracy rate of 97.2% meets our target
|
||||
threshold, but critical items require expedited resolution to maintain operational efficiency.
|
||||
Extended Inventory Review:
|
||||
| Product Code | Category | Unit Cost | Total Value | Last Audit | Notes |
|
||||
| ------------ | ----------- | --------- | ----------- | ---------- | ---------- |
|
||||
| SKU-8847 | Electronics | $45.00 | $13,005.00 | 2024-10-15 | |
|
||||
| SKU-9201 | Hardware | $32.50 | $25,285.00 | 2024-10-22 | Verified |
|
||||
| SKU-4563 | Software | $120.00 | $18,720.00 | | Critical |
|
||||
| SKU-7728 | Accessories | $15.75 | $14,726.25 | 2024-11-01 | |
|
||||
| SKU-3345 | Electronics | $67.00 | $22,445.00 | 2024-10-18 | |
|
||||
| SKU-5512 | Hardware | $89.00 | $31,150.00 | | Pending |
|
||||
| SKU-6678 | Software | $200.00 | $42,000.00 | 2024-10-25 | High Value |
|
||||
| SKU-7789 | Accessories | $8.50 | $5,950.00 | 2024-11-05 | |
|
||||
| SKU-2234 | Electronics | $125.00 | $35,000.00 | | |
|
||||
| SKU-1123 | Hardware | $55.00 | $27,500.00 | 2024-10-30 | Verified |
|
||||
Recommendations:
|
||||
1. Immediate review of SKU-4563 handling procedures. 2. Implement additional verification for critical
|
||||
items. 3. Schedule follow-up audit for high-value products (SKU-6678, SKU-2234).
|
||||
Approval:
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
BOOKING ORDER
|
||||
Print Date 12/15/2024 14:30:22
|
||||
Page 1 of 1
|
||||
STARLIGHT CINEMAS
|
||||
Orders
|
||||
| Order / Rev: | 2024-12-5678 | | | Cinema: | | Downtown Multiplex |
|
||||
| ------------ | -------------- | --- | --- | ---------------- | --- | ------------------ |
|
||||
| Alt Order #: | SC-WINTER-2024 | | | Primary Contact: | | Sarah Johnson |
|
||||
Product Desc: Holiday Movie Marathon Package Location: NYC-01
|
||||
| Estimate: | EST-456 | | | Region: | | NORTHEAST |
|
||||
| -------------------- | ----------------------- | --- | --- | ------- | --- | --------- |
|
||||
| Booking Dates: | 12/20/2024 - 12/31/2024 | | | | | |
|
||||
| Original Date / Rev: | 12/01/24 / 12/10/24 | | | | | |
|
||||
| Order Type: | Premium Package | | | | | |
|
||||
Booking Agency
|
||||
| Name: | Premier Entertainment Group | | | | | |
|
||||
| ---------------- | --------------------------- | --- | --- | -------------- | --- | --------- |
|
||||
| | | | | Billing Type: | | Net 30 |
|
||||
| Contact: | Michael Chen | | | | | |
|
||||
| | | | | Payment Terms: | | Corporate |
|
||||
| Billing Contact: | accounting@premierent.com | | | | | |
|
||||
| | | | | Commission: | | 10% |
|
||||
555 Broadway Suite 1200
|
||||
New York, NY 10012
|
||||
Customer
|
||||
| Name: | Universal Studios Distribution | | | | | |
|
||||
| -------------- | ------------------------------ | --- | --- | --- | --- | --- |
|
||||
| Category: | Film Distributor | | | | | |
|
||||
| Contact Email: | bookings@universalstudios.com | | | | | |
|
||||
| Customer ID: | CUST-98765 | | | | | |
|
||||
| Revenue Code: | FILM-PREMIUM | | | | | |
|
||||
Booking Summary
|
||||
| Start Date | End Date | # Shows | Gross Amount | Net Amount | | |
|
||||
| ---------- | -------- | ------- | ------------ | ---------- | --- | --- |
|
||||
| 12/20/24 | 12/31/24 | 48 | $12,500.00 | $11,250.00 | | |
|
||||
Totals
|
||||
| Month | # Shows | Gross Amount | | Net Amount | | Occupancy |
|
||||
| ------------- | ------- | ------------ | --- | ---------- | --- | --------- |
|
||||
| December 2024 | 48 | $12,500.00 | | $11,250.00 | | 85% |
|
||||
| Totals | 48 | $12,500.00 | | $11,250.00 | | 85% |
|
||||
Account Representatives
|
||||
Representative Territory Region Start Date / End Date Commission %
|
||||
| Sarah Johnson | NYC Metro | NORTHEAST | 12/20/24 - 12/31/24 | | 100% | |
|
||||
| ------------- | --------- | --------- | ------------------- | --- | ---- | --- |
|
||||
Show Schedule Details
|
||||
Ln Screen Start End Movie Title Format Showtime Days Shows Rate Type Total
|
||||
1 SCR-1 12/20/24 12/25/24 Holiday Spectacular IMAX 3D 7:00 PM Daily 12 $250 PM $3,000
|
||||
(Runtime: 142 min); Holiday Season Premium
|
||||
2 SCR-2 12/20/24 12/31/24 Winter Wonderland Standard 4:30 PM Daily 24 $150 MT $3,600
|
||||
(Runtime: 98 min); Matinee Special
|
||||
3 SCR-1 12/26/24 12/31/24 New Year Mystery 4DX 9:30 PM Daily 12 $300 PM $3,600
|
||||
(Runtime: 116 min); Premium Experience
|
||||
Show Details
|
||||
| Show Screen | Date Range | Title | Showtime | Days Type | Rate | Revenue |
|
||||
| ----------- | ---------- | ----- | -------- | --------- | ---- | ------- |
|
||||
1 SCR-1 12/20-12/25 Holiday Spectacular 7:00 PM Daily PM $250 $3,000
|
||||
This booking order is subject to cinema availability and standard terms.
|
||||
2 SCR-2 12/20-12/31 Winter Wonderland 4:30 PM Daily MT $150 $3,600
|
||||
All showtimes are approximate and subject to change.
|
||||
3 SCR-1 12/26-12/31 New Year Mystery 9:30 PM Daily PM $300 $3,600
|
||||
| Total Revenue: | | | | | | $12,500.00 |
|
||||
| -------------- | --- | --- | --- | --- | --- | ---------- |
|
||||
@@ -0,0 +1,65 @@
|
||||
1
|
||||
|
||||
Introduction
|
||||
|
||||
Large language models (LLMs) are becoming a crucial building block in developing powerful agents
|
||||
that utilize LLMs for reasoning, tool usage, and adapting to new observations (Yao et al., 2022; Xi
|
||||
et al., 2023; Wang et al., 2023b) in many real-world tasks. Given the expanding tasks that could
|
||||
benefit from LLMs and the growing task complexity, an intuitive approach to scale up the power of
|
||||
agents is to use multiple agents that cooperate. Prior work suggests that multiple agents can help
|
||||
encourage divergent thinking (Liang et al., 2023), improve factuality and reasoning (Du et al., 2023),
|
||||
and provide validation (Wu et al., 2023). In light of the intuition and early evidence of promise, it is
|
||||
intriguing to ask the following question: how can we facilitate the development of LLM applications
|
||||
that could span a broad spectrum of domains and complexities based on the multi-agent approach?
|
||||
|
||||
Our insight is to use multi-agent conversations to achieve it. There are at least three reasons con-
|
||||
firming its general feasibility and utility thanks to recent advances in LLMs: First, because chat-
|
||||
optimized LLMs (e.g., GPT-4) show the ability to incorporate feedback, LLM agents can cooperate
|
||||
through conversations with each other or human(s), e.g., a dialog where agents provide and seek rea-
|
||||
soning, observations, critiques, and validation. Second, because a single LLM can exhibit a broad
|
||||
range of capabilities (especially when configured with the correct prompt and inference settings),
|
||||
conversations between differently configured agents can help combine these broad LLM capabilities
|
||||
in a modular and complementary manner. Third, LLMs have demonstrated ability to solve complex
|
||||
tasks when the tasks are broken into simpler subtasks. Multi-agent conversations can enable this
|
||||
partitioning and integration in an intuitive manner. How can we leverage the above insights and
|
||||
support different applications with the common requirement of coordinating multiple agents, poten-
|
||||
tially backed by LLMs, humans, or tools exhibiting different capacities? We desire a multi-agent
|
||||
conversation framework with generic abstraction and effective implementation that has the flexibil-
|
||||
ity to satisfy different application needs. Achieving this requires addressing two critical questions:
|
||||
(1) How can we design individual agents that are capable, reusable, customizable, and effective in
|
||||
multi-agent collaboration? (2) How can we develop a straightforward, unified interface that can
|
||||
accommodate a wide range of agent conversation patterns? In practice, applications of varying
|
||||
complexities may need distinct sets of agents with specific capabilities, and may require different
|
||||
conversation patterns, such as single- or multi-turn dialogs, different human involvement modes, and
|
||||
static vs. dynamic conversation. Moreover, developers may prefer the flexibility to program agent
|
||||
interactions in natural language or code. Failing to adequately address these two questions would
|
||||
limit the framework’s scope of applicability and generality.
|
||||
While there is contemporaneous exploration of multi-agent approaches,3 we present AutoGen, a
|
||||
generalized multi-agent conversation framework (Figure 1), based on the following new concepts.
|
||||
1 Customizable and conversable agents. AutoGen uses a generic design of agents that can lever-
|
||||
age LLMs, human inputs, tools, or a combination of them. The result is that developers can
|
||||
easily and quickly create agents with different roles (e.g., agents to write code, execute code,
|
||||
wire in human feedback, validate outputs, etc.) by selecting and configuring a subset of built-in
|
||||
capabilities. The agent’s backend can also be readily extended to allow more custom behaviors.
|
||||
To make these agents suitable for multi-agent conversation, every agent is made conversable –
|
||||
they can receive, react, and respond to messages. When configured properly, an agent can hold
|
||||
multiple turns of conversations with other agents autonomously or solicit human inputs at cer-
|
||||
tain rounds, enabling human agency and automation. The conversable agent design leverages the
|
||||
strong capability of the most advanced LLMs in taking feedback and making progress via chat
|
||||
and also allows combining capabilities of LLMs in a modular fashion. (Section 2.1)
|
||||
|
||||
2 Conversation programming. A fundamental insight of AutoGen is to simplify and unify com-
|
||||
plex LLM application workflows as multi-agent conversations. So AutoGen adopts a program-
|
||||
ming paradigm centered around these inter-agent conversations. We refer to this paradigm as
|
||||
conversation programming, which streamlines the development of intricate applications via two
|
||||
primary steps: (1) defining a set of conversable agents with specific capabilities and roles (as
|
||||
described above); (2) programming the interaction behavior between agents via conversation-
|
||||
centric computation and control. Both steps can be achieved via a fusion of natural and pro-
|
||||
gramming languages to build applications with a wide range of conversation patterns and agent
|
||||
behaviors. AutoGen provides ready-to-use implementations and also allows easy extension and
|
||||
experimentation for both steps. (Section 2.2)
|
||||
|
||||
3We refer to Appendix A for a detailed discussion.
|
||||
|
||||
2
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
%PDF-1.3
|
||||
%“Œ‹ž ReportLab Generated PDF document http://www.reportlab.com
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Author (anonymous) /CreationDate (D:20260108192537+01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20260108192537+01'00') /Producer (ReportLab PDF Library - www.reportlab.com)
|
||||
/Subject (unspecified) /Title (untitled) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 670
|
||||
>>
|
||||
stream
|
||||
Gat$td;IYl'Rf-pcJpsZ/27V[H_WEoW#\5sVS2I3Jt]?;R+`$Ms*f.>6<=3APUNhTmQL<9F,pFup'KGk=TR,7^>/u!#kAE+l;?UQ8Fg(+-O>;^54HWJ*kXdl'VdsI]Y^$-G(GWPR)iGMeWbg3)F'+jfWpCb"rU?d?8?q_r!E2N'0sM)J>=XD.jgunBuga\Wi4MX$WV/b)1F@bC8Nj8(0*)"ZK06BSqlu1$[^37A;/aK=mfgqg$&i),2OH&%^\"B1%B\dd_V>$5OtPri4rcEe3LoBUeL6QAPnpQr+R-t0f]ZSYc?BTAKQ?A&+J#J*N*=6;'?@Cp*>auj0",hDS3bH4[hVs3O="&bk&U@>+8c1&c2iDg6R*%q%iEZq'-!FNSB8#C*'po69R8$S(:.=-$N6'!_[1/jV<$@V3Z_"gd!g!MJMT)mTUN4cWjUQQj]HT_m]0*R=YgTmcl@k>*b/SBce9?.m,bEi#?PI:=r_6G.auM&FtP,>O7T%Z<$f#=g6(2+d@;8?"$8cdI38ZZ>hq5b2_pQY:M\.Kod,pl)ZX7a7Gc'Mf_'SB1X3*L[-51a8`h4)KjJQjLfm/3TIeQY?2+?^.r^HNafjHp<5,1M=W'N>8sb=dB#FC5M`7L91"BC@CfEckPe`M5O:#!Fj$K]s(Gs8rW$>H7gK~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 9
|
||||
0000000000 65535 f
|
||||
0000000073 00000 n
|
||||
0000000114 00000 n
|
||||
0000000221 00000 n
|
||||
0000000333 00000 n
|
||||
0000000526 00000 n
|
||||
0000000594 00000 n
|
||||
0000000890 00000 n
|
||||
0000000949 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<5467fcd5093f18002be6af3fb13ce6c3><5467fcd5093f18002be6af3fb13ce6c3>]
|
||||
% ReportLab generated PDF document -- digest (http://www.reportlab.com)
|
||||
|
||||
/Info 6 0 R
|
||||
/Root 5 0 R
|
||||
/Size 9
|
||||
>>
|
||||
startxref
|
||||
1709
|
||||
%%EOF
|
||||
@@ -0,0 +1,74 @@
|
||||
%PDF-1.3
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Author (anonymous) /CreationDate (D:20260210121342+01'00') /Creator (anonymous) /Keywords () /ModDate (D:20260210121342+01'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (unspecified) /Title (untitled) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 2414
|
||||
>>
|
||||
stream
|
||||
Gat=m?#SK-&r,lL<tO"8con<J;5Cq;2s]18RrdR7Y[)>Ym#<31@rCsJ89.W.qa3u?82hU4/rD6bm_^2.o5\G6@H<5/\G85.&:2)\f,l]`/mA:-0HF*!^.%Yd0?rr<_LD*'1j8Q\=IJXu'N"=HL>KSX^]339h+)S%SB[D8U\2B8rL_pR7\MXONW%HeW99+,0hH$AU#^KYAoZ)6P-2'6m5cj7lZu'kGHQ:/\R1,Ma%hEl2eYq(:LZ"-`3OktM:dm<m,u<)W99/X#l.?0OO\Z_]Y4.9BoSuKGOrdaFbq^/)*_g%gm8s\<gU<a%e]re(gWm_H[^0bn(=;0X%(_^H%$;+Se<aM)L.FrW&=UUc*X3He'XMO]CgP3P*<]$#uOPN2Z#n\O]@7_]#$ZH.Gr&KZm_M+6]8I5lYVcZLH,)V@L:BCib%tuWd,p*A"0Gb=6gIkI+Y5.[<aH_D`iMaKNQpo.UHf=F]to-Ui6XS[Q;Qh\cD=LT#YQGpn9rsUVm:qR"1%pnrSZa/Mi*P+f4j?B6-uV5Em_Zqog)^@RtF[F-adqASQb%i[(eIIqZ)_CVEHpGDgIpX[[uQ4J6DNf5X^CB'JA+,d^#?/[fq)jH^+:rbdW>Y'H/a/1^A\lZD2qMb,5%-$pOaW5-%BjndGRZ<CV&?T^r@PWF)!H#gDKcZj?[/gATBZ;=XJ$_a;??F-qtH(HaQX?W#iIL#17<Y25AC[ePo/pO[]=c0(\#/j9R%W/]$do:5b%.e4%S0Z';YJm/!GO9jt-H8W>JTK5I,b-cnrpc!2H0BZZ`1%R*aB!ZE'JRRYNJ<J4B`!j/maqpD>*Rq$U:[Tq%Lr[m+DHGg*dP\Ee>\#VYo43^R>kA9W2b/WU:k/M#%^2;nC+,e'dAcEOp?t5Kk;4+.f4MU@-mf7iCT_29s_%g,%K_gB8!kWS28T%T6'u_$GK'qX*VP>7>5?dW_<?$QPg!n")cT(<-[c/-kEbS'`*BYR5SB9TPY<1jq1#Q/EWpCJrY=s;bQfH^=uT:DTR3.8/N>W)r8_SF*7+f;4415n3,ECi2P6&bjmn17t+qU8;D])\Qt.8QLi)?kJ`.t+lkW'Y4e876l-2di)Y?.3\K1<(0IrEfm1<:Oc^u?7B::q;On$J5_C7T<u%071ASb!ZD1u7Yd"g`I'`PJ>**>tRZrdD6q3W@5QfbW8242uIHro=(eV*P1KjY,oj4tW&obb>^q-Iur%F#A)mgu8+V*?E<bdEC6V0+Z7OS^l.$W4hmuq:sMdJ=Sk+94D3QtUBZ:AoIiBA%s3#GJdRDFCpZ)7\MZmitKhMID(%ic%oW#tD%ERrqpk,dD3ll!E6m)e):26BLNV!WiRV*d(+Ppl'p$%?J&MqeV<=uNJ_5,4P_NC:lWf`Iu3\u+^>Y]dUOk&c=m2^<YVV2cUoq[`<<W-]MTIC50Klu6rO5RUVZ"h`#"4adtt2qjs2b12hQi!@JBp4Jln>:1Dtc(*!NBU*DeAtLhuWu&JLWFQi:;ka#?AD6V.A_[>n$T,.]8d=tffJ,?'DbCKQ-BnKqTn_:1LGc865V]FFi=AAF`DGhW(F]2^o?>VbGN:;=!-s;ea7]Ll\f+eiZ8XZb0*mZp%8*K_pf+1"2fKuO1pNK%7f_(mPTD@0&ljSV?o$5BpUmleYs^Faq_SM'jX.o\d*6%j(EtY.N"m2B'E@[.Y_8Be+m(58m$\dcqm$?,0it)/=9@9kRfJB;N7D9t\'F<:#c$P82`UKqgN]$kU]5eLPZMR=0bO[rPk"\?hu>sT^KFg`B>!pml-a[ImSeWp!_l3s!E>gFKq4ng:"n=N:m57rHjN)GML<=a1ktQpUT8:?[D:c7+Gm@2q;uN1Q3)hpeThe-&[#`KYZ4e_=o]kk1KH/^jo:"<0_nRJingk\[1Jltc<,.Jq2\*]=AVcIiY#?iMASrc$Bp)4m=NdIOJ&,H=+<MC=^7]?Tb>M"H6ZdXTX2Ba;Gp=J-m]$,8ZCU/77rHJ,%1.[/DlnkH:pIIV$Oh.;:t?5e3.cs^[G:H=e;i>c+>B=)C&l7T)S<Bld"_W)BtgI(/F`Le;ULQ,!FM!^<8Kk?L6b_>G8Jp-TG;!V1144#29r2%;n-RmNHrGdR!76&H"R_D-]`c"1FCgZl*",7SUVuqc0oapDQ=^`nj#FFk@2%[K[V45$!KQIH[=;SUpTE8T!QLliC=5-9]nkQpBVdHM6-g)tYBAPuOqr^qkn[Wh4C;6L89J;D>5@cYM$2Y/24scnNiWp4jWhfJAF^ck!@I(VPV*s,pdkPKn<Zg-T3I%d.sSl"^f-Gm=*riV,>(\770jbu^lf\h1+IH>c;Bo;Pdg;!fA)'kmg$"\P3oX=/N5/rUltb3K-BdRTR;-W)J1bDbE?g<MKG;cK`l?D4l>.,O@6id::q]JXBH\Ws#0[#'8-5JQL>/c~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 9
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000514 00000 n
|
||||
0000000582 00000 n
|
||||
0000000843 00000 n
|
||||
0000000902 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<e319d5c305edb8c0fb6be9e44c6178fa><e319d5c305edb8c0fb6be9e44c6178fa>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 6 0 R
|
||||
/Root 5 0 R
|
||||
/Size 9
|
||||
>>
|
||||
startxref
|
||||
3407
|
||||
%%EOF
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
|
After Width: | Height: | Size: 463 KiB |
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"key1": "string_value",
|
||||
"key2": 1234,
|
||||
"key3": [
|
||||
"list_value1",
|
||||
"list_value2"
|
||||
],
|
||||
"5b64c88c-b3c3-4510-bcb8-da0b200602d8": "uuid_key",
|
||||
"uuid_value": "9700dc99-6685-40b4-9a3a-5e406dcb37f3"
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 145 KiB |
@@ -0,0 +1,4 @@
|
||||
–¼‘O,”N—î,�Z�Š
|
||||
�²“¡‘¾˜Y,30,“Œ‹ž
|
||||
ŽO–؉pŽq,25,‘å�ã
|
||||
îà‹´�~,35,–¼ŒÃ‰®
|
||||
|
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "0f61db80",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Test Notebook"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "3f2a5bbd",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"markitdown\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"markitdown\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9b9c0468",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Code Cell Below"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "37d8088a",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"42\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# comment in code\n",
|
||||
"print(42)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2e3177bd",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"End\n",
|
||||
"\n",
|
||||
"---"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.8"
|
||||
},
|
||||
"title": "Test Notebook Title"
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
+147
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
@@ -0,0 +1,554 @@
|
||||
#!/usr/bin/env python3 -m pytest
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from markitdown._uri_utils import parse_data_uri, file_uri_to_path
|
||||
|
||||
from markitdown import (
|
||||
MarkItDown,
|
||||
UnsupportedFormatException,
|
||||
FileConversionException,
|
||||
StreamInfo,
|
||||
)
|
||||
|
||||
# This file contains module tests that are not directly tested by the FileTestVectors.
|
||||
# This includes things like helper functions and runtime conversion options
|
||||
# (e.g., LLM clients, exiftool path, transcription services, etc.)
|
||||
|
||||
skip_remote = (
|
||||
True if os.environ.get("GITHUB_ACTIONS") else False
|
||||
) # Don't run these tests in CI
|
||||
|
||||
|
||||
# Don't run the llm tests without a key and the client library
|
||||
skip_llm = False if os.environ.get("OPENAI_API_KEY") else True
|
||||
try:
|
||||
import openai
|
||||
except ModuleNotFoundError:
|
||||
skip_llm = True
|
||||
|
||||
# Skip exiftool tests if not installed
|
||||
skip_exiftool = shutil.which("exiftool") is None
|
||||
|
||||
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")
|
||||
|
||||
JPG_TEST_EXIFTOOL = {
|
||||
"Author": "AutoGen Authors",
|
||||
"Title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||
"Description": "AutoGen enables diverse LLM-based applications",
|
||||
"ImageSize": "1615x1967",
|
||||
"DateTimeOriginal": "2024:03:14 22:10:00",
|
||||
}
|
||||
|
||||
MP3_TEST_EXIFTOOL = {
|
||||
"Title": "f67a499e-a7d0-4ca3-a49b-358bd934ae3e",
|
||||
"Artist": "Artist Name Test String",
|
||||
"Album": "Album Name Test String",
|
||||
"SampleRate": "48000",
|
||||
}
|
||||
|
||||
PDF_TEST_URL = "https://arxiv.org/pdf/2308.08155v2.pdf"
|
||||
PDF_TEST_STRINGS = [
|
||||
"While there is contemporaneous exploration of multi-agent approaches"
|
||||
]
|
||||
|
||||
YOUTUBE_TEST_URL = "https://www.youtube.com/watch?v=V2qZ_lgxTzg"
|
||||
YOUTUBE_TEST_STRINGS = [
|
||||
"## AutoGen FULL Tutorial with Python (Step-By-Step)",
|
||||
"This is an intermediate tutorial for installing and using AutoGen locally",
|
||||
"PT15M4S",
|
||||
"the model we're going to be using today is GPT 3.5 turbo", # From the transcript
|
||||
]
|
||||
|
||||
DOCX_COMMENT_TEST_STRINGS = [
|
||||
"314b0a30-5b04-470b-b9f7-eed2c2bec74a",
|
||||
"49e168b7-d2ae-407f-a055-2167576f39a1",
|
||||
"## d666f1f7-46cb-42bd-9a39-9a39cf2a509f",
|
||||
"# Abstract",
|
||||
"# Introduction",
|
||||
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||
"This is a test comment. 12df-321a",
|
||||
"Yet another comment in the doc. 55yiyi-asd09",
|
||||
]
|
||||
|
||||
BLOG_TEST_URL = "https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math"
|
||||
BLOG_TEST_STRINGS = [
|
||||
"Large language models (LLMs) are powerful tools that can generate natural language texts for various applications, such as chatbots, summarization, translation, and more. GPT-4 is currently the state of the art LLM in the world. Is model selection irrelevant? What about inference parameters?",
|
||||
"an example where high cost can easily prevent a generic complex",
|
||||
]
|
||||
|
||||
LLM_TEST_STRINGS = [
|
||||
"5bda1dd6",
|
||||
]
|
||||
|
||||
PPTX_TEST_STRINGS = [
|
||||
"2cdda5c8-e50e-4db4-b5f0-9722a649f455",
|
||||
"04191ea8-5c73-4215-a1d3-1cfb43aaaf12",
|
||||
"44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a",
|
||||
"1b92870d-e3b5-4e65-8153-919f4ff45592",
|
||||
"AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
|
||||
"a3f6004b-6f4f-4ea8-bee3-3741f4dc385f", # chart title
|
||||
"2003", # chart value
|
||||
]
|
||||
|
||||
|
||||
# --- Helper Functions ---
|
||||
def validate_strings(result, expected_strings, exclude_strings=None):
|
||||
"""Validate presence or absence of specific strings."""
|
||||
text_content = result.text_content.replace("\\", "")
|
||||
for string in expected_strings:
|
||||
assert string in text_content
|
||||
if exclude_strings:
|
||||
for string in exclude_strings:
|
||||
assert string not in text_content
|
||||
|
||||
|
||||
def test_stream_info_operations() -> None:
|
||||
"""Test operations performed on StreamInfo objects."""
|
||||
|
||||
stream_info_original = StreamInfo(
|
||||
mimetype="mimetype.1",
|
||||
extension="extension.1",
|
||||
charset="charset.1",
|
||||
filename="filename.1",
|
||||
local_path="local_path.1",
|
||||
url="url.1",
|
||||
)
|
||||
|
||||
# Check updating all attributes by keyword
|
||||
keywords = ["mimetype", "extension", "charset", "filename", "local_path", "url"]
|
||||
for keyword in keywords:
|
||||
updated_stream_info = stream_info_original.copy_and_update(
|
||||
**{keyword: f"{keyword}.2"}
|
||||
)
|
||||
|
||||
# Make sure the targted attribute is updated
|
||||
assert getattr(updated_stream_info, keyword) == f"{keyword}.2"
|
||||
|
||||
# Make sure the other attributes are unchanged
|
||||
for k in keywords:
|
||||
if k != keyword:
|
||||
assert getattr(stream_info_original, k) == getattr(
|
||||
updated_stream_info, k
|
||||
)
|
||||
|
||||
# Check updating all attributes by passing a new StreamInfo object
|
||||
keywords = ["mimetype", "extension", "charset", "filename", "local_path", "url"]
|
||||
for keyword in keywords:
|
||||
updated_stream_info = stream_info_original.copy_and_update(
|
||||
StreamInfo(**{keyword: f"{keyword}.2"})
|
||||
)
|
||||
|
||||
# Make sure the targted attribute is updated
|
||||
assert getattr(updated_stream_info, keyword) == f"{keyword}.2"
|
||||
|
||||
# Make sure the other attributes are unchanged
|
||||
for k in keywords:
|
||||
if k != keyword:
|
||||
assert getattr(stream_info_original, k) == getattr(
|
||||
updated_stream_info, k
|
||||
)
|
||||
|
||||
# Check mixing and matching
|
||||
updated_stream_info = stream_info_original.copy_and_update(
|
||||
StreamInfo(extension="extension.2", filename="filename.2"),
|
||||
mimetype="mimetype.3",
|
||||
charset="charset.3",
|
||||
)
|
||||
assert updated_stream_info.extension == "extension.2"
|
||||
assert updated_stream_info.filename == "filename.2"
|
||||
assert updated_stream_info.mimetype == "mimetype.3"
|
||||
assert updated_stream_info.charset == "charset.3"
|
||||
assert updated_stream_info.local_path == "local_path.1"
|
||||
assert updated_stream_info.url == "url.1"
|
||||
|
||||
# Check multiple StreamInfo objects
|
||||
updated_stream_info = stream_info_original.copy_and_update(
|
||||
StreamInfo(extension="extension.4", filename="filename.5"),
|
||||
StreamInfo(mimetype="mimetype.6", charset="charset.7"),
|
||||
)
|
||||
assert updated_stream_info.extension == "extension.4"
|
||||
assert updated_stream_info.filename == "filename.5"
|
||||
assert updated_stream_info.mimetype == "mimetype.6"
|
||||
assert updated_stream_info.charset == "charset.7"
|
||||
assert updated_stream_info.local_path == "local_path.1"
|
||||
assert updated_stream_info.url == "url.1"
|
||||
|
||||
|
||||
def test_data_uris() -> None:
|
||||
# Test basic parsing of data URIs
|
||||
data_uri = "data:text/plain;base64,SGVsbG8sIFdvcmxkIQ=="
|
||||
mime_type, attributes, data = parse_data_uri(data_uri)
|
||||
assert mime_type == "text/plain"
|
||||
assert len(attributes) == 0
|
||||
assert data == b"Hello, World!"
|
||||
|
||||
data_uri = "data:base64,SGVsbG8sIFdvcmxkIQ=="
|
||||
mime_type, attributes, data = parse_data_uri(data_uri)
|
||||
assert mime_type is None
|
||||
assert len(attributes) == 0
|
||||
assert data == b"Hello, World!"
|
||||
|
||||
data_uri = "data:text/plain;charset=utf-8;base64,SGVsbG8sIFdvcmxkIQ=="
|
||||
mime_type, attributes, data = parse_data_uri(data_uri)
|
||||
assert mime_type == "text/plain"
|
||||
assert len(attributes) == 1
|
||||
assert attributes["charset"] == "utf-8"
|
||||
assert data == b"Hello, World!"
|
||||
|
||||
data_uri = "data:,Hello%2C%20World%21"
|
||||
mime_type, attributes, data = parse_data_uri(data_uri)
|
||||
assert mime_type is None
|
||||
assert len(attributes) == 0
|
||||
assert data == b"Hello, World!"
|
||||
|
||||
data_uri = "data:text/plain,Hello%2C%20World%21"
|
||||
mime_type, attributes, data = parse_data_uri(data_uri)
|
||||
assert mime_type == "text/plain"
|
||||
assert len(attributes) == 0
|
||||
assert data == b"Hello, World!"
|
||||
|
||||
data_uri = "data:text/plain;charset=utf-8,Hello%2C%20World%21"
|
||||
mime_type, attributes, data = parse_data_uri(data_uri)
|
||||
assert mime_type == "text/plain"
|
||||
assert len(attributes) == 1
|
||||
assert attributes["charset"] == "utf-8"
|
||||
assert data == b"Hello, World!"
|
||||
|
||||
|
||||
def test_file_uris() -> None:
|
||||
# Test file URI with an empty host
|
||||
file_uri = "file:///path/to/file.txt"
|
||||
netloc, path = file_uri_to_path(file_uri)
|
||||
assert netloc is None
|
||||
assert path == "/path/to/file.txt"
|
||||
|
||||
# Test file URI with no host
|
||||
file_uri = "file:/path/to/file.txt"
|
||||
netloc, path = file_uri_to_path(file_uri)
|
||||
assert netloc is None
|
||||
assert path == "/path/to/file.txt"
|
||||
|
||||
# Test file URI with localhost
|
||||
file_uri = "file://localhost/path/to/file.txt"
|
||||
netloc, path = file_uri_to_path(file_uri)
|
||||
assert netloc == "localhost"
|
||||
assert path == "/path/to/file.txt"
|
||||
|
||||
# Test file URI with query parameters
|
||||
file_uri = "file:///path/to/file.txt?param=value"
|
||||
netloc, path = file_uri_to_path(file_uri)
|
||||
assert netloc is None
|
||||
assert path == "/path/to/file.txt"
|
||||
|
||||
# Test file URI with fragment
|
||||
file_uri = "file:///path/to/file.txt#fragment"
|
||||
netloc, path = file_uri_to_path(file_uri)
|
||||
assert netloc is None
|
||||
assert path == "/path/to/file.txt"
|
||||
|
||||
|
||||
def test_docx_comments() -> None:
|
||||
# Test DOCX processing, with comments and setting style_map on init
|
||||
markitdown_with_style_map = MarkItDown(style_map="comment-reference => ")
|
||||
result = markitdown_with_style_map.convert(
|
||||
os.path.join(TEST_FILES_DIR, "test_with_comment.docx")
|
||||
)
|
||||
validate_strings(result, DOCX_COMMENT_TEST_STRINGS)
|
||||
|
||||
|
||||
def test_docx_equations() -> None:
|
||||
markitdown = MarkItDown()
|
||||
docx_file = os.path.join(TEST_FILES_DIR, "equations.docx")
|
||||
result = markitdown.convert(docx_file)
|
||||
|
||||
# Check for inline equation m=1 (wrapped with single $) is present
|
||||
assert "$m=1$" in result.text_content, "Inline equation $m=1$ not found"
|
||||
|
||||
# Find block equations wrapped with double $$ and check if they are present
|
||||
block_equations = re.findall(r"\$\$(.+?)\$\$", result.text_content)
|
||||
assert block_equations, "No block equations found in the document."
|
||||
|
||||
|
||||
def test_input_as_strings() -> None:
|
||||
markitdown = MarkItDown()
|
||||
|
||||
# Test input from a stream
|
||||
input_data = b"<html><body><h1>Test</h1></body></html>"
|
||||
result = markitdown.convert_stream(io.BytesIO(input_data))
|
||||
assert "# Test" in result.text_content
|
||||
|
||||
# Test input with leading blank characters
|
||||
input_data = b" \n\n\n<html><body><h1>Test</h1></body></html>"
|
||||
result = markitdown.convert_stream(io.BytesIO(input_data))
|
||||
assert "# Test" in result.text_content
|
||||
|
||||
|
||||
def test_deeply_nested_html_fallback() -> None:
|
||||
"""Large, deeply nested HTML should fall back to plain-text extraction
|
||||
instead of silently returning unconverted HTML (issue #1636).
|
||||
|
||||
Note: This test uses sys.setrecursionlimit to guarantee a RecursionError
|
||||
regardless of the host environment's default limit, making it deterministic
|
||||
across different platforms and CI configurations.
|
||||
"""
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
markitdown = MarkItDown()
|
||||
|
||||
# Use a small recursion limit so the test is environment-independent.
|
||||
# We restore the original limit in a finally block to avoid side-effects.
|
||||
original_limit = sys.getrecursionlimit()
|
||||
low_limit = 200 # well below markdownify's traversal depth for depth=500
|
||||
|
||||
# Build HTML with nesting deep enough to trigger RecursionError
|
||||
depth = 500
|
||||
html = "<html><body>"
|
||||
for _ in range(depth):
|
||||
html += '<div style="margin-left:10px">'
|
||||
html += "<p>Deep content with <b>bold text</b></p>"
|
||||
for _ in range(depth):
|
||||
html += "</div>"
|
||||
html += "</body></html>"
|
||||
|
||||
try:
|
||||
sys.setrecursionlimit(low_limit)
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warnings.simplefilter("always")
|
||||
result = markitdown.convert_stream(
|
||||
io.BytesIO(html.encode("utf-8")),
|
||||
file_extension=".html",
|
||||
)
|
||||
|
||||
# Should have emitted a warning about the fallback
|
||||
recursion_warnings = [x for x in w if "deeply nested" in str(x.message)]
|
||||
assert len(recursion_warnings) > 0
|
||||
finally:
|
||||
sys.setrecursionlimit(original_limit)
|
||||
|
||||
# The output should contain the text content, not raw HTML
|
||||
assert "Deep content" in result.markdown
|
||||
assert "bold text" in result.markdown
|
||||
assert "<div" not in result.markdown
|
||||
assert "<p>" not in result.markdown
|
||||
|
||||
|
||||
def test_doc_rlink() -> None:
|
||||
# Test for: CVE-2025-11849
|
||||
markitdown = MarkItDown()
|
||||
|
||||
# Document with rlink
|
||||
docx_file = os.path.join(TEST_FILES_DIR, "rlink.docx")
|
||||
|
||||
# Directory containing the target rlink file
|
||||
rlink_tmp_dir = os.path.abspath(os.sep + "tmp")
|
||||
|
||||
# Ensure the tmp directory exists
|
||||
if not os.path.exists(rlink_tmp_dir):
|
||||
pytest.skip(f"Skipping rlink test; {rlink_tmp_dir} directory does not exist.")
|
||||
return
|
||||
|
||||
rlink_file_path = os.path.join(rlink_tmp_dir, "test_rlink.txt")
|
||||
rlink_content = "de658225-569e-4e3d-9ed2-cfb6abf927fc"
|
||||
b64_prefix = (
|
||||
"ZGU2NTgyMjUtNTY5ZS00ZTNkLTllZDItY2ZiNmFiZjk" # base64 prefix of rlink_content
|
||||
)
|
||||
|
||||
if os.path.exists(rlink_file_path):
|
||||
with open(rlink_file_path, "r", encoding="utf-8") as f:
|
||||
existing_content = f.read()
|
||||
if existing_content != rlink_content:
|
||||
raise ValueError(
|
||||
f"Existing {rlink_file_path} content does not match expected content."
|
||||
)
|
||||
else:
|
||||
with open(rlink_file_path, "w", encoding="utf-8") as f:
|
||||
f.write(rlink_content)
|
||||
|
||||
try:
|
||||
result = markitdown.convert(docx_file, keep_data_uris=True).text_content
|
||||
assert (
|
||||
b64_prefix not in result
|
||||
) # Make sure the target file was NOT embedded in the output
|
||||
finally:
|
||||
os.remove(rlink_file_path)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
skip_remote,
|
||||
reason="do not run tests that query external urls",
|
||||
)
|
||||
def test_markitdown_remote() -> None:
|
||||
markitdown = MarkItDown()
|
||||
|
||||
# By URL
|
||||
result = markitdown.convert(PDF_TEST_URL)
|
||||
for test_string in PDF_TEST_STRINGS:
|
||||
assert test_string in result.text_content
|
||||
|
||||
# Youtube
|
||||
# result = markitdown.convert(YOUTUBE_TEST_URL)
|
||||
# for test_string in YOUTUBE_TEST_STRINGS:
|
||||
# assert test_string in result.text_content
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
skip_remote,
|
||||
reason="do not run remotely run speech transcription tests",
|
||||
)
|
||||
def test_speech_transcription() -> None:
|
||||
markitdown = MarkItDown()
|
||||
|
||||
# Test WAV files, MP3 and M4A files
|
||||
for file_name in ["test.wav", "test.mp3", "test.m4a"]:
|
||||
result = markitdown.convert(os.path.join(TEST_FILES_DIR, file_name))
|
||||
result_lower = result.text_content.lower()
|
||||
assert (
|
||||
("1" in result_lower or "one" in result_lower)
|
||||
and ("2" in result_lower or "two" in result_lower)
|
||||
and ("3" in result_lower or "three" in result_lower)
|
||||
and ("4" in result_lower or "four" in result_lower)
|
||||
and ("5" in result_lower or "five" in result_lower)
|
||||
)
|
||||
|
||||
|
||||
def test_exceptions() -> None:
|
||||
# Check that an exception is raised when trying to convert an unsupported format
|
||||
markitdown = MarkItDown()
|
||||
with pytest.raises(UnsupportedFormatException):
|
||||
markitdown.convert(os.path.join(TEST_FILES_DIR, "random.bin"))
|
||||
|
||||
# Check that an exception is raised when trying to convert a file that is corrupted
|
||||
with pytest.raises(FileConversionException) as exc_info:
|
||||
markitdown.convert(
|
||||
os.path.join(TEST_FILES_DIR, "random.bin"), file_extension=".pptx"
|
||||
)
|
||||
assert len(exc_info.value.attempts) == 1
|
||||
assert type(exc_info.value.attempts[0].converter).__name__ == "PptxConverter"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
skip_exiftool,
|
||||
reason="do not run if exiftool is not installed",
|
||||
)
|
||||
def test_markitdown_exiftool() -> None:
|
||||
which_exiftool = shutil.which("exiftool")
|
||||
assert which_exiftool is not None
|
||||
|
||||
# Test explicitly setting the location of exiftool
|
||||
markitdown = MarkItDown(exiftool_path=which_exiftool)
|
||||
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.jpg"))
|
||||
for key in JPG_TEST_EXIFTOOL:
|
||||
target = f"{key}: {JPG_TEST_EXIFTOOL[key]}"
|
||||
assert target in result.text_content
|
||||
|
||||
# Test setting the exiftool path through an environment variable
|
||||
os.environ["EXIFTOOL_PATH"] = which_exiftool
|
||||
markitdown = MarkItDown()
|
||||
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.jpg"))
|
||||
for key in JPG_TEST_EXIFTOOL:
|
||||
target = f"{key}: {JPG_TEST_EXIFTOOL[key]}"
|
||||
assert target in result.text_content
|
||||
|
||||
# Test some other media types
|
||||
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.mp3"))
|
||||
for key in MP3_TEST_EXIFTOOL:
|
||||
target = f"{key}: {MP3_TEST_EXIFTOOL[key]}"
|
||||
assert target in result.text_content
|
||||
|
||||
|
||||
def test_markitdown_llm_parameters() -> None:
|
||||
"""Test that LLM parameters are correctly passed to the client."""
|
||||
mock_client = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [
|
||||
MagicMock(
|
||||
message=MagicMock(
|
||||
content="Test caption with red circle and blue square 5bda1dd6"
|
||||
)
|
||||
)
|
||||
]
|
||||
mock_client.chat.completions.create.return_value = mock_response
|
||||
|
||||
test_prompt = "You are a professional test prompt."
|
||||
markitdown = MarkItDown(
|
||||
llm_client=mock_client, llm_model="gpt-4o", llm_prompt=test_prompt
|
||||
)
|
||||
|
||||
# Test image file
|
||||
markitdown.convert(os.path.join(TEST_FILES_DIR, "test_llm.jpg"))
|
||||
|
||||
# Verify the prompt was passed to the OpenAI API
|
||||
assert mock_client.chat.completions.create.called
|
||||
call_args = mock_client.chat.completions.create.call_args
|
||||
messages = call_args[1]["messages"]
|
||||
assert len(messages) == 1
|
||||
assert messages[0]["content"][0]["text"] == test_prompt
|
||||
|
||||
# Reset the mock for the next test
|
||||
mock_client.chat.completions.create.reset_mock()
|
||||
|
||||
# TODO: may only use one test after the llm caption method duplicate has been removed:
|
||||
# https://github.com/microsoft/markitdown/pull/1254
|
||||
# Test PPTX file
|
||||
markitdown.convert(os.path.join(TEST_FILES_DIR, "test.pptx"))
|
||||
|
||||
# Verify the prompt was passed to the OpenAI API for PPTX images too
|
||||
assert mock_client.chat.completions.create.called
|
||||
call_args = mock_client.chat.completions.create.call_args
|
||||
messages = call_args[1]["messages"]
|
||||
assert len(messages) == 1
|
||||
assert messages[0]["content"][0]["text"] == test_prompt
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
skip_llm,
|
||||
reason="do not run llm tests without a key",
|
||||
)
|
||||
def test_markitdown_llm() -> None:
|
||||
client = openai.OpenAI()
|
||||
markitdown = MarkItDown(llm_client=client, llm_model="gpt-4o")
|
||||
|
||||
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_llm.jpg"))
|
||||
for test_string in LLM_TEST_STRINGS:
|
||||
assert test_string in result.text_content
|
||||
|
||||
# This is not super precise. It would also accept "red square", "blue circle",
|
||||
# "the square is not blue", etc. But it's sufficient for this test.
|
||||
for test_string in ["red", "circle", "blue", "square"]:
|
||||
assert test_string in result.text_content.lower()
|
||||
|
||||
# Images embedded in PPTX files
|
||||
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.pptx"))
|
||||
# LLM Captions are included
|
||||
for test_string in LLM_TEST_STRINGS:
|
||||
assert test_string in result.text_content
|
||||
# Standard alt text is included
|
||||
validate_strings(result, PPTX_TEST_STRINGS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""Runs this file's tests from the command line."""
|
||||
for test in [
|
||||
test_stream_info_operations,
|
||||
test_data_uris,
|
||||
test_file_uris,
|
||||
test_docx_comments,
|
||||
test_input_as_strings,
|
||||
test_markitdown_remote,
|
||||
test_speech_transcription,
|
||||
test_exceptions,
|
||||
test_doc_rlink,
|
||||
test_markitdown_exiftool,
|
||||
test_markitdown_llm_parameters,
|
||||
test_markitdown_llm,
|
||||
]:
|
||||
print(f"Running {test.__name__}...", end="")
|
||||
test()
|
||||
print("OK")
|
||||
print("All tests passed!")
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3 -m pytest
|
||||
import os
|
||||
import time
|
||||
import pytest
|
||||
import base64
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
if __name__ == "__main__":
|
||||
from _test_vectors import GENERAL_TEST_VECTORS, DATA_URI_TEST_VECTORS
|
||||
else:
|
||||
from ._test_vectors import GENERAL_TEST_VECTORS, DATA_URI_TEST_VECTORS
|
||||
|
||||
from markitdown import (
|
||||
MarkItDown,
|
||||
StreamInfo,
|
||||
)
|
||||
|
||||
skip_remote = (
|
||||
True if os.environ.get("GITHUB_ACTIONS") else False
|
||||
) # Don't run these tests in CI
|
||||
|
||||
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")
|
||||
TEST_FILES_URL = "https://raw.githubusercontent.com/microsoft/markitdown/refs/heads/main/packages/markitdown/tests/test_files"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", GENERAL_TEST_VECTORS)
|
||||
def test_guess_stream_info(test_vector):
|
||||
"""Test the ability to guess stream info."""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
local_path = os.path.join(TEST_FILES_DIR, test_vector.filename)
|
||||
expected_extension = os.path.splitext(test_vector.filename)[1]
|
||||
|
||||
with open(local_path, "rb") as stream:
|
||||
guesses = markitdown._get_stream_info_guesses(
|
||||
stream,
|
||||
base_guess=StreamInfo(
|
||||
filename=os.path.basename(test_vector.filename),
|
||||
local_path=local_path,
|
||||
extension=expected_extension,
|
||||
),
|
||||
)
|
||||
|
||||
# For some limited exceptions, we can't guarantee the exact
|
||||
# mimetype or extension, so we'll special-case them here.
|
||||
if test_vector.filename in [
|
||||
"test_outlook_msg.msg",
|
||||
]:
|
||||
return
|
||||
|
||||
assert guesses[0].mimetype == test_vector.mimetype
|
||||
assert guesses[0].extension == expected_extension
|
||||
assert guesses[0].charset == test_vector.charset
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", GENERAL_TEST_VECTORS)
|
||||
def test_convert_local(test_vector):
|
||||
"""Test the conversion of a local file."""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
result = markitdown.convert(
|
||||
os.path.join(TEST_FILES_DIR, test_vector.filename), url=test_vector.url
|
||||
)
|
||||
for string in test_vector.must_include:
|
||||
assert string in result.markdown
|
||||
for string in test_vector.must_not_include:
|
||||
assert string not in result.markdown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", GENERAL_TEST_VECTORS)
|
||||
def test_convert_stream_with_hints(test_vector):
|
||||
"""Test the conversion of a stream with full stream info."""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
stream_info = StreamInfo(
|
||||
extension=os.path.splitext(test_vector.filename)[1],
|
||||
mimetype=test_vector.mimetype,
|
||||
charset=test_vector.charset,
|
||||
)
|
||||
|
||||
with open(os.path.join(TEST_FILES_DIR, test_vector.filename), "rb") as stream:
|
||||
result = markitdown.convert(
|
||||
stream, stream_info=stream_info, url=test_vector.url
|
||||
)
|
||||
for string in test_vector.must_include:
|
||||
assert string in result.markdown
|
||||
for string in test_vector.must_not_include:
|
||||
assert string not in result.markdown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", GENERAL_TEST_VECTORS)
|
||||
def test_convert_stream_without_hints(test_vector):
|
||||
"""Test the conversion of a stream with no stream info."""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
with open(os.path.join(TEST_FILES_DIR, test_vector.filename), "rb") as stream:
|
||||
result = markitdown.convert(stream, url=test_vector.url)
|
||||
for string in test_vector.must_include:
|
||||
assert string in result.markdown
|
||||
for string in test_vector.must_not_include:
|
||||
assert string not in result.markdown
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
skip_remote,
|
||||
reason="do not run tests that query external urls",
|
||||
)
|
||||
@pytest.mark.parametrize("test_vector", GENERAL_TEST_VECTORS)
|
||||
def test_convert_http_uri(test_vector):
|
||||
"""Test the conversion of an HTTP:// or HTTPS:// URI."""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
time.sleep(1) # Ensure we don't hit rate limits
|
||||
|
||||
result = markitdown.convert(
|
||||
TEST_FILES_URL + "/" + test_vector.filename,
|
||||
url=test_vector.url, # Mock where this file would be found
|
||||
)
|
||||
for string in test_vector.must_include:
|
||||
assert string in result.markdown
|
||||
for string in test_vector.must_not_include:
|
||||
assert string not in result.markdown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", GENERAL_TEST_VECTORS)
|
||||
def test_convert_file_uri(test_vector):
|
||||
"""Test the conversion of a file:// URI."""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
result = markitdown.convert(
|
||||
Path(os.path.join(TEST_FILES_DIR, test_vector.filename)).as_uri(),
|
||||
url=test_vector.url,
|
||||
)
|
||||
for string in test_vector.must_include:
|
||||
assert string in result.markdown
|
||||
for string in test_vector.must_not_include:
|
||||
assert string not in result.markdown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", GENERAL_TEST_VECTORS)
|
||||
def test_convert_data_uri(test_vector):
|
||||
"""Test the conversion of a data URI."""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
data = ""
|
||||
with open(os.path.join(TEST_FILES_DIR, test_vector.filename), "rb") as stream:
|
||||
data = base64.b64encode(stream.read()).decode("utf-8")
|
||||
mimetype = test_vector.mimetype
|
||||
data_uri = f"data:{mimetype};base64,{data}"
|
||||
|
||||
result = markitdown.convert(
|
||||
data_uri,
|
||||
url=test_vector.url,
|
||||
)
|
||||
for string in test_vector.must_include:
|
||||
assert string in result.markdown
|
||||
for string in test_vector.must_not_include:
|
||||
assert string not in result.markdown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", DATA_URI_TEST_VECTORS)
|
||||
def test_convert_keep_data_uris(test_vector):
|
||||
"""Test API functionality when keep_data_uris is enabled"""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
# Test local file conversion
|
||||
result = markitdown.convert(
|
||||
os.path.join(TEST_FILES_DIR, test_vector.filename),
|
||||
keep_data_uris=True,
|
||||
url=test_vector.url,
|
||||
)
|
||||
|
||||
for string in test_vector.must_include:
|
||||
assert string in result.markdown
|
||||
for string in test_vector.must_not_include:
|
||||
assert string not in result.markdown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_vector", DATA_URI_TEST_VECTORS)
|
||||
def test_convert_stream_keep_data_uris(test_vector):
|
||||
"""Test the conversion of a stream with no stream info."""
|
||||
markitdown = MarkItDown()
|
||||
|
||||
stream_info = StreamInfo(
|
||||
extension=os.path.splitext(test_vector.filename)[1],
|
||||
mimetype=test_vector.mimetype,
|
||||
charset=test_vector.charset,
|
||||
)
|
||||
|
||||
with open(os.path.join(TEST_FILES_DIR, test_vector.filename), "rb") as stream:
|
||||
result = markitdown.convert(
|
||||
stream, stream_info=stream_info, keep_data_uris=True, url=test_vector.url
|
||||
)
|
||||
|
||||
for string in test_vector.must_include:
|
||||
assert string in result.markdown
|
||||
for string in test_vector.must_not_include:
|
||||
assert string not in result.markdown
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""Runs this file's tests from the command line."""
|
||||
|
||||
# General tests
|
||||
for test_function in [
|
||||
test_guess_stream_info,
|
||||
test_convert_local,
|
||||
test_convert_stream_with_hints,
|
||||
test_convert_stream_without_hints,
|
||||
test_convert_http_uri,
|
||||
test_convert_file_uri,
|
||||
test_convert_data_uri,
|
||||
]:
|
||||
for test_vector in GENERAL_TEST_VECTORS:
|
||||
print(
|
||||
f"Running {test_function.__name__} on {test_vector.filename}...", end=""
|
||||
)
|
||||
test_function(test_vector)
|
||||
print("OK")
|
||||
|
||||
# Data URI tests
|
||||
for test_function in [
|
||||
test_convert_keep_data_uris,
|
||||
test_convert_stream_keep_data_uris,
|
||||
]:
|
||||
for test_vector in DATA_URI_TEST_VECTORS:
|
||||
print(
|
||||
f"Running {test_function.__name__} on {test_vector.filename}...", end=""
|
||||
)
|
||||
test_function(test_vector)
|
||||
print("OK")
|
||||
|
||||
print("All tests passed!")
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3 -m pytest
|
||||
"""Tests for MasterFormat-style partial numbering in PDF conversion."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import pytest
|
||||
|
||||
from markitdown import MarkItDown
|
||||
from markitdown.converters._pdf_converter import PARTIAL_NUMBERING_PATTERN
|
||||
|
||||
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")
|
||||
|
||||
|
||||
class TestMasterFormatPartialNumbering:
|
||||
"""Test handling of MasterFormat-style partial numbering (.1, .2, etc.)."""
|
||||
|
||||
def test_partial_numbering_pattern_regex(self):
|
||||
"""Test that the partial numbering regex pattern correctly matches."""
|
||||
|
||||
# Should match partial numbering patterns
|
||||
assert PARTIAL_NUMBERING_PATTERN.match(".1") is not None
|
||||
assert PARTIAL_NUMBERING_PATTERN.match(".2") is not None
|
||||
assert PARTIAL_NUMBERING_PATTERN.match(".10") is not None
|
||||
assert PARTIAL_NUMBERING_PATTERN.match(".99") is not None
|
||||
|
||||
# Should NOT match other patterns
|
||||
assert PARTIAL_NUMBERING_PATTERN.match("1.") is None
|
||||
assert PARTIAL_NUMBERING_PATTERN.match("1.2") is None
|
||||
assert PARTIAL_NUMBERING_PATTERN.match(".1.2") is None
|
||||
assert PARTIAL_NUMBERING_PATTERN.match("text") is None
|
||||
assert PARTIAL_NUMBERING_PATTERN.match(".a") is None
|
||||
assert PARTIAL_NUMBERING_PATTERN.match("") is None
|
||||
|
||||
def test_masterformat_partial_numbering_not_split(self):
|
||||
"""Test that MasterFormat partial numbering stays with associated text.
|
||||
|
||||
MasterFormat documents use partial numbering like:
|
||||
.1 The intent of this Request for Proposal...
|
||||
.2 Available information relative to...
|
||||
|
||||
These should NOT be split into separate table columns, but kept
|
||||
as coherent text lines with the number followed by its description.
|
||||
"""
|
||||
pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf")
|
||||
|
||||
markitdown = MarkItDown()
|
||||
result = markitdown.convert(pdf_path)
|
||||
text_content = result.text_content
|
||||
|
||||
# Partial numberings should NOT appear isolated on their own lines
|
||||
# If they're isolated, it means the parser incorrectly split them from their text
|
||||
lines = text_content.split("\n")
|
||||
isolated_numberings = []
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
# Check if line contains ONLY a partial numbering (with possible whitespace/pipes)
|
||||
cleaned = stripped.replace("|", "").strip()
|
||||
if cleaned in [".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".10"]:
|
||||
isolated_numberings.append(stripped)
|
||||
|
||||
assert len(isolated_numberings) == 0, (
|
||||
f"Partial numberings should not be isolated from their text. "
|
||||
f"Found isolated: {isolated_numberings}"
|
||||
)
|
||||
|
||||
# Verify that partial numberings appear WITH following text on the same line
|
||||
# Look for patterns like ".1 The intent" or ".1 Some text"
|
||||
partial_with_text = re.findall(r"\.\d+\s+\w+", text_content)
|
||||
assert (
|
||||
len(partial_with_text) > 0
|
||||
), "Expected to find partial numberings followed by text on the same line"
|
||||
|
||||
def test_masterformat_content_preserved(self):
|
||||
"""Test that MasterFormat document content is fully preserved."""
|
||||
pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf")
|
||||
|
||||
markitdown = MarkItDown()
|
||||
result = markitdown.convert(pdf_path)
|
||||
text_content = result.text_content
|
||||
|
||||
# Verify key content from the MasterFormat document is preserved
|
||||
expected_content = [
|
||||
"RFP for Construction Management Services",
|
||||
"Section 00 00 43",
|
||||
"Instructions to Respondents",
|
||||
"Ken Sargent House",
|
||||
"INTENT",
|
||||
"Request for Proposal",
|
||||
"KEN SARGENT HOUSE",
|
||||
"GRANDE PRAIRIE, ALBERTA",
|
||||
"Section 00 00 45",
|
||||
]
|
||||
|
||||
for content in expected_content:
|
||||
assert (
|
||||
content in text_content
|
||||
), f"Expected content '{content}' not found in extracted text"
|
||||
|
||||
# Verify partial numbering is followed by text on the same line
|
||||
# .1 should be followed by "The intent" on the same line
|
||||
assert re.search(
|
||||
r"\.1\s+The intent", text_content
|
||||
), "Partial numbering .1 should be followed by 'The intent' text"
|
||||
|
||||
# .2 should be followed by "Available information" on the same line
|
||||
assert re.search(
|
||||
r"\.2\s+Available information", text_content
|
||||
), "Partial numbering .2 should be followed by 'Available information' text"
|
||||
|
||||
# Ensure text content is not empty and has reasonable length
|
||||
assert (
|
||||
len(text_content.strip()) > 100
|
||||
), "MasterFormat document should have substantial text content"
|
||||
|
||||
def test_merge_partial_numbering_with_empty_lines_between(self):
|
||||
"""Test that partial numberings merge correctly even with empty lines between.
|
||||
|
||||
When PDF extractors produce output like:
|
||||
.1
|
||||
|
||||
The intent of this Request...
|
||||
|
||||
The merge logic should still combine them properly.
|
||||
"""
|
||||
pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf")
|
||||
|
||||
markitdown = MarkItDown()
|
||||
result = markitdown.convert(pdf_path)
|
||||
text_content = result.text_content
|
||||
|
||||
# The merged result should have .1 and .2 followed by text
|
||||
# Check that we don't have patterns like ".1\n\nThe intent" (unmerged)
|
||||
lines = text_content.split("\n")
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
# If we find an isolated partial numbering, the merge failed
|
||||
if stripped in [".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8"]:
|
||||
# Check if next non-empty line exists and wasn't merged
|
||||
for j in range(i + 1, min(i + 3, len(lines))):
|
||||
if lines[j].strip():
|
||||
pytest.fail(
|
||||
f"Partial numbering '{stripped}' on line {i} was not "
|
||||
f"merged with following text '{lines[j].strip()[:30]}...'"
|
||||
)
|
||||
break
|
||||
|
||||
def test_multiple_partial_numberings_all_merged(self):
|
||||
"""Test that all partial numberings in a document are properly merged."""
|
||||
pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf")
|
||||
|
||||
markitdown = MarkItDown()
|
||||
result = markitdown.convert(pdf_path)
|
||||
text_content = result.text_content
|
||||
|
||||
# Count occurrences of merged partial numberings (number followed by text)
|
||||
merged_count = len(re.findall(r"\.\d+\s+[A-Za-z]", text_content))
|
||||
|
||||
# Count isolated partial numberings (number alone on a line)
|
||||
isolated_count = 0
|
||||
for line in text_content.split("\n"):
|
||||
stripped = line.strip()
|
||||
if re.match(r"^\.\d+$", stripped):
|
||||
isolated_count += 1
|
||||
|
||||
assert (
|
||||
merged_count >= 2
|
||||
), f"Expected at least 2 merged partial numberings, found {merged_count}"
|
||||
assert (
|
||||
isolated_count == 0
|
||||
), f"Found {isolated_count} isolated partial numberings that weren't merged"
|
||||
@@ -0,0 +1,364 @@
|
||||
#!/usr/bin/env python3 -m pytest
|
||||
"""Tests for PDF converter memory optimization.
|
||||
|
||||
Verifies that:
|
||||
- page.close() is called after processing each page (frees cached data)
|
||||
- Plain-text PDFs fall back to pdfminer when no form pages are found
|
||||
- Mixed PDFs use form extraction only on form-style pages
|
||||
- Memory stays constant regardless of page count
|
||||
"""
|
||||
|
||||
import gc
|
||||
import io
|
||||
import os
|
||||
import tracemalloc
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from markitdown import MarkItDown
|
||||
|
||||
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")
|
||||
|
||||
|
||||
def _has_fpdf2() -> bool:
|
||||
try:
|
||||
import fpdf # noqa: F401
|
||||
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def _make_form_page():
|
||||
"""Create a mock page with 3-column table-like word positions."""
|
||||
page = MagicMock()
|
||||
page.width = 612
|
||||
page.close = MagicMock()
|
||||
page.extract_words.return_value = [
|
||||
{"text": "Name", "x0": 50, "x1": 100, "top": 10, "bottom": 20},
|
||||
{"text": "Value", "x0": 250, "x1": 300, "top": 10, "bottom": 20},
|
||||
{"text": "Unit", "x0": 450, "x1": 500, "top": 10, "bottom": 20},
|
||||
{"text": "Alpha", "x0": 50, "x1": 100, "top": 30, "bottom": 40},
|
||||
{"text": "100", "x0": 250, "x1": 280, "top": 30, "bottom": 40},
|
||||
{"text": "kg", "x0": 450, "x1": 470, "top": 30, "bottom": 40},
|
||||
{"text": "Beta", "x0": 50, "x1": 100, "top": 50, "bottom": 60},
|
||||
{"text": "200", "x0": 250, "x1": 280, "top": 50, "bottom": 60},
|
||||
{"text": "lb", "x0": 450, "x1": 470, "top": 50, "bottom": 60},
|
||||
]
|
||||
return page
|
||||
|
||||
|
||||
def _make_plain_page():
|
||||
"""Create a mock page with single-line paragraph (no table structure)."""
|
||||
page = MagicMock()
|
||||
page.width = 612
|
||||
page.close = MagicMock()
|
||||
page.extract_words.return_value = [
|
||||
{
|
||||
"text": "This is a long paragraph of plain text.",
|
||||
"x0": 50,
|
||||
"x1": 550,
|
||||
"top": 10,
|
||||
"bottom": 20,
|
||||
},
|
||||
]
|
||||
page.extract_text.return_value = "This is a long paragraph of plain text."
|
||||
return page
|
||||
|
||||
|
||||
def _mock_pdfplumber_open(pages):
|
||||
"""Return a mock pdfplumber.open that yields the given pages."""
|
||||
|
||||
def mock_open(stream):
|
||||
mock_pdf = MagicMock()
|
||||
mock_pdf.pages = pages
|
||||
mock_pdf.__enter__ = MagicMock(return_value=mock_pdf)
|
||||
mock_pdf.__exit__ = MagicMock(return_value=False)
|
||||
return mock_pdf
|
||||
|
||||
return mock_open
|
||||
|
||||
|
||||
class TestPdfMemoryOptimization:
|
||||
"""Test that PDF conversion cleans up per-page caches to limit memory."""
|
||||
|
||||
def test_page_close_called_on_every_page(self):
|
||||
"""Verify page.close() is called on every page during conversion.
|
||||
|
||||
This ensures cached word/layout data is freed after each page,
|
||||
preventing O(n) memory growth with page count.
|
||||
"""
|
||||
num_pages = 20
|
||||
pages = [_make_form_page() for _ in range(num_pages)]
|
||||
|
||||
with patch(
|
||||
"markitdown.converters._pdf_converter.pdfplumber"
|
||||
) as mock_pdfplumber:
|
||||
mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages)
|
||||
|
||||
md = MarkItDown()
|
||||
buf = io.BytesIO(b"fake pdf content")
|
||||
from markitdown import StreamInfo
|
||||
|
||||
md.convert_stream(
|
||||
buf,
|
||||
stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"),
|
||||
)
|
||||
|
||||
# page.close() must be called on ALL pages
|
||||
for i, page in enumerate(pages):
|
||||
assert page.close.called, (
|
||||
f"page.close() was NOT called on page {i} — "
|
||||
"this would cause memory to accumulate"
|
||||
)
|
||||
|
||||
def test_plain_text_pdf_falls_back_to_pdfminer(self):
|
||||
"""Verify all-plain-text PDFs fall back to pdfminer.
|
||||
|
||||
When no page has form-style content, the converter should discard
|
||||
pdfplumber results and use pdfminer for the whole document (better
|
||||
text spacing for prose).
|
||||
"""
|
||||
num_pages = 50
|
||||
pages = [_make_plain_page() for _ in range(num_pages)]
|
||||
|
||||
with patch(
|
||||
"markitdown.converters._pdf_converter.pdfplumber"
|
||||
) as mock_pdfplumber, patch(
|
||||
"markitdown.converters._pdf_converter.pdfminer"
|
||||
) as mock_pdfminer:
|
||||
mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages)
|
||||
mock_pdfminer.high_level.extract_text.return_value = "Plain text content"
|
||||
|
||||
md = MarkItDown()
|
||||
buf = io.BytesIO(b"fake pdf content")
|
||||
from markitdown import StreamInfo
|
||||
|
||||
result = md.convert_stream(
|
||||
buf,
|
||||
stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"),
|
||||
)
|
||||
|
||||
# pdfminer should be used for the final text extraction
|
||||
assert mock_pdfminer.high_level.extract_text.called, (
|
||||
"pdfminer.high_level.extract_text was not called — "
|
||||
"plain-text PDFs should fall back to pdfminer"
|
||||
)
|
||||
assert result.text_content is not None
|
||||
|
||||
def test_plain_text_pdf_still_closes_all_pages(self):
|
||||
"""Even for plain-text PDFs, page.close() must be called on every page."""
|
||||
num_pages = 30
|
||||
pages = [_make_plain_page() for _ in range(num_pages)]
|
||||
|
||||
with patch(
|
||||
"markitdown.converters._pdf_converter.pdfplumber"
|
||||
) as mock_pdfplumber, patch(
|
||||
"markitdown.converters._pdf_converter.pdfminer"
|
||||
) as mock_pdfminer:
|
||||
mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages)
|
||||
mock_pdfminer.high_level.extract_text.return_value = "text"
|
||||
|
||||
md = MarkItDown()
|
||||
buf = io.BytesIO(b"fake pdf content")
|
||||
from markitdown import StreamInfo
|
||||
|
||||
md.convert_stream(
|
||||
buf,
|
||||
stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"),
|
||||
)
|
||||
|
||||
for i, page in enumerate(pages):
|
||||
assert (
|
||||
page.close.called
|
||||
), f"page.close() was NOT called on plain-text page {i}"
|
||||
|
||||
def test_mixed_pdf_uses_form_extraction_per_page(self):
|
||||
"""In a mixed PDF, form pages get table extraction while plain pages don't.
|
||||
|
||||
Ensures we don't miss form-style pages and don't waste work
|
||||
running form extraction on plain-text pages.
|
||||
"""
|
||||
# Pages 0,2,4 are form-style; pages 1,3 are plain text
|
||||
pages = [
|
||||
_make_form_page(), # 0 - form
|
||||
_make_plain_page(), # 1 - plain
|
||||
_make_form_page(), # 2 - form
|
||||
_make_plain_page(), # 3 - plain
|
||||
_make_form_page(), # 4 - form
|
||||
]
|
||||
|
||||
with patch(
|
||||
"markitdown.converters._pdf_converter.pdfplumber"
|
||||
) as mock_pdfplumber:
|
||||
mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages)
|
||||
|
||||
md = MarkItDown()
|
||||
buf = io.BytesIO(b"fake pdf content")
|
||||
from markitdown import StreamInfo
|
||||
|
||||
result = md.convert_stream(
|
||||
buf,
|
||||
stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"),
|
||||
)
|
||||
|
||||
# All pages should have close() called
|
||||
for i, page in enumerate(pages):
|
||||
assert page.close.called, f"page.close() not called on page {i}"
|
||||
|
||||
# Form pages (0,2,4) should have extract_words called
|
||||
for i in [0, 2, 4]:
|
||||
assert pages[
|
||||
i
|
||||
].extract_words.called, f"extract_words not called on form page {i}"
|
||||
|
||||
# Result should contain table content from form pages
|
||||
assert result.text_content is not None
|
||||
assert (
|
||||
"|" in result.text_content
|
||||
), "Expected markdown table pipes in output from form-style pages"
|
||||
|
||||
def test_only_one_pdfplumber_open_call(self):
|
||||
"""Verify pdfplumber.open is called exactly once (single pass)."""
|
||||
pages = [_make_form_page() for _ in range(10)]
|
||||
|
||||
with patch(
|
||||
"markitdown.converters._pdf_converter.pdfplumber"
|
||||
) as mock_pdfplumber:
|
||||
mock_pdfplumber.open.side_effect = _mock_pdfplumber_open(pages)
|
||||
|
||||
md = MarkItDown()
|
||||
buf = io.BytesIO(b"fake pdf content")
|
||||
from markitdown import StreamInfo
|
||||
|
||||
md.convert_stream(
|
||||
buf,
|
||||
stream_info=StreamInfo(extension=".pdf", mimetype="application/pdf"),
|
||||
)
|
||||
|
||||
assert mock_pdfplumber.open.call_count == 1, (
|
||||
f"Expected 1 pdfplumber.open call (single pass), "
|
||||
f"got {mock_pdfplumber.open.call_count}"
|
||||
)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not os.path.exists(os.path.join(TEST_FILES_DIR, "test.pdf")),
|
||||
reason="test.pdf not available",
|
||||
)
|
||||
def test_real_pdf_page_cleanup(self):
|
||||
"""Integration test: verify page.close() is called with a real PDF."""
|
||||
import pdfplumber
|
||||
|
||||
close_call_count = 0
|
||||
original_close = pdfplumber.page.Page.close
|
||||
|
||||
def tracking_close(self):
|
||||
nonlocal close_call_count
|
||||
close_call_count += 1
|
||||
original_close(self)
|
||||
|
||||
with patch.object(pdfplumber.page.Page, "close", tracking_close):
|
||||
md = MarkItDown()
|
||||
pdf_path = os.path.join(TEST_FILES_DIR, "test.pdf")
|
||||
md.convert(pdf_path)
|
||||
|
||||
assert (
|
||||
close_call_count > 0
|
||||
), "page.close() was never called during PDF conversion"
|
||||
|
||||
|
||||
def _generate_table_pdf(num_pages: int) -> bytes:
|
||||
"""Generate a PDF with table-like content on every page."""
|
||||
from fpdf import FPDF
|
||||
|
||||
pdf = FPDF()
|
||||
pdf.set_auto_page_break(auto=False)
|
||||
for page_num in range(num_pages):
|
||||
pdf.add_page()
|
||||
pdf.set_font("Helvetica", size=10)
|
||||
pdf.set_xy(10, 10)
|
||||
pdf.cell(60, 8, "Parameter", border=1)
|
||||
pdf.cell(60, 8, "Value", border=1)
|
||||
pdf.cell(60, 8, "Unit", border=1)
|
||||
pdf.ln()
|
||||
for row in range(20):
|
||||
y = 18 + row * 8
|
||||
if y > 270:
|
||||
break
|
||||
pdf.set_xy(10, y)
|
||||
pdf.cell(60, 8, f"Param_{page_num}_{row}", border=1)
|
||||
pdf.cell(60, 8, f"{(page_num * 100 + row) * 1.23:.2f}", border=1)
|
||||
pdf.cell(60, 8, "kg/m2", border=1)
|
||||
return pdf.output()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _has_fpdf2(),
|
||||
reason="fpdf2 not installed",
|
||||
)
|
||||
class TestPdfMemoryBenchmark:
|
||||
"""Benchmark: verify memory stays constant with page.close() fix."""
|
||||
|
||||
def test_memory_does_not_grow_linearly(self):
|
||||
"""Peak memory for 200 pages should be far less than without the fix.
|
||||
|
||||
Without page.close(), 200 pages uses ~225 MiB (linear growth).
|
||||
With the fix, peak memory should stay under 30 MiB.
|
||||
"""
|
||||
from markitdown import StreamInfo
|
||||
|
||||
num_pages = 200
|
||||
pdf_bytes = _generate_table_pdf(num_pages)
|
||||
|
||||
gc.collect()
|
||||
tracemalloc.start()
|
||||
|
||||
md = MarkItDown()
|
||||
buf = io.BytesIO(pdf_bytes)
|
||||
md.convert_stream(buf, stream_info=StreamInfo(extension=".pdf"))
|
||||
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
|
||||
peak_mib = peak / 1024 / 1024
|
||||
# Without the fix this would be ~225 MiB. With the fix it should
|
||||
# be well under 30 MiB. Use a generous threshold to avoid flaky
|
||||
# failures on different machines.
|
||||
assert peak_mib < 30, (
|
||||
f"Peak memory {peak_mib:.1f} MiB for {num_pages} pages is too high. "
|
||||
f"Expected < 30 MiB with page.close() fix."
|
||||
)
|
||||
|
||||
def test_memory_constant_across_page_counts(self):
|
||||
"""Peak memory should not scale linearly with page count.
|
||||
|
||||
Converts 50-page and 200-page PDFs and asserts the peak memory
|
||||
ratio is much less than the 4x page count ratio.
|
||||
"""
|
||||
from markitdown import StreamInfo
|
||||
|
||||
results = {}
|
||||
for num_pages in [50, 200]:
|
||||
pdf_bytes = _generate_table_pdf(num_pages)
|
||||
|
||||
gc.collect()
|
||||
tracemalloc.start()
|
||||
|
||||
md = MarkItDown()
|
||||
buf = io.BytesIO(pdf_bytes)
|
||||
md.convert_stream(buf, stream_info=StreamInfo(extension=".pdf"))
|
||||
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
results[num_pages] = peak
|
||||
|
||||
ratio = results[200] / results[50]
|
||||
# With O(n) memory growth the ratio would be ~4x.
|
||||
# With the fix it should be close to 1x (well under 2x).
|
||||
assert ratio < 2.0, (
|
||||
f"Memory ratio 200p/50p = {ratio:.2f}x — "
|
||||
f"expected < 2.0x (constant memory). "
|
||||
f"50p={results[50] / 1024 / 1024:.1f} MiB, "
|
||||
f"200p={results[200] / 1024 / 1024:.1f} MiB"
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user