wehub-resource-sync
c3749daf48
Tests / test-linux (3.13) (push) Failing after 0s
Tests / test-linux (3.11) (push) Failing after 1s
Tests / lint (push) Failing after 0s
Tests / test-linux (3.9) (push) Failing after 1s
Docker / build (push) Failing after 1s
Docker / build-gpu (push) Failing after 2s
Tests / test-windows (push) Has been cancelled
Tests / test-macos (push) Has been cancelled
1554 lines
67 KiB
Python
1554 lines
67 KiB
Python
"""Tests for format_miner (mempalace 3.3.6, integrated in PR #1555).
|
|
|
|
Covers all 14 fringe cases in the format-coverage spec plus orchestrator
|
|
behavior. MarkItDown is mocked at the seam for most tests; a few guarded
|
|
integration tests exercise live transformers when the optional extras are
|
|
installed.
|
|
|
|
Run with:
|
|
pytest tests/test_format_miner.py -v
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
|
|
from mempalace.format_miner import ( # noqa: E402
|
|
DEFAULT_MAX_FILE_SIZE,
|
|
SUPPORTED_FORMATS,
|
|
ExtractionStatus,
|
|
decode_robust,
|
|
extract_text,
|
|
is_icloud_dataless,
|
|
scan_formats,
|
|
)
|
|
|
|
|
|
def _make_symlink_or_skip(link: Path, target: Path) -> None:
|
|
"""Create ``link`` pointing to ``target``, or ``pytest.skip`` when the
|
|
platform/process can't create symlinks.
|
|
|
|
Windows without ``SeCreateSymbolicLinkPrivilege`` raises ``OSError``
|
|
(``WinError 1314``) from ``Path.symlink_to()`` BEFORE any product code
|
|
runs, which surfaces as a hard test failure even though the failure
|
|
has nothing to do with the code under test. Per PR #1555 review (Igor):
|
|
symlink tests must skip cleanly in environments without privileges
|
|
rather than fail spuriously.
|
|
"""
|
|
try:
|
|
link.symlink_to(target)
|
|
except OSError as exc:
|
|
pytest.skip(f"symlink creation not permitted on this platform/user: {exc}")
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Module surface
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_supported_formats_covers_office_suite():
|
|
assert ".pdf" in SUPPORTED_FORMATS
|
|
assert ".rtf" in SUPPORTED_FORMATS
|
|
assert ".docx" in SUPPORTED_FORMATS
|
|
assert ".xlsx" in SUPPORTED_FORMATS
|
|
assert ".pptx" in SUPPORTED_FORMATS
|
|
assert ".epub" in SUPPORTED_FORMATS
|
|
|
|
|
|
def test_supported_formats_normalised_lowercase():
|
|
for ext in SUPPORTED_FORMATS:
|
|
assert ext == ext.lower()
|
|
assert ext.startswith(".")
|
|
|
|
|
|
def test_extraction_status_enum_has_all_documented_codes():
|
|
expected = {
|
|
"OK",
|
|
"SKIP_TOO_LARGE",
|
|
"SKIP_CLOUD_ONLY",
|
|
"SKIP_EMPTY",
|
|
"SKIP_NO_MARKITDOWN",
|
|
"SKIP_NO_STRIPRTF",
|
|
"SKIP_ENCRYPTED",
|
|
"SKIP_PERMISSION",
|
|
"SKIP_BROKEN_SYMLINK",
|
|
"SKIP_UNRECOGNIZED",
|
|
"SKIP_EXTRACTION_ERROR",
|
|
"SKIP_MISSING_FORMAT_DEPS",
|
|
"SKIP_NETWORK_TIMEOUT",
|
|
"SKIP_UNREADABLE",
|
|
}
|
|
actual = {status.name for status in ExtractionStatus}
|
|
missing = expected - actual
|
|
assert not missing, f"Missing status codes: {missing}"
|
|
|
|
|
|
def test_default_max_file_size_matches_existing_miner():
|
|
assert DEFAULT_MAX_FILE_SIZE == 500 * 1024 * 1024
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 12 — unrecognized extension → skip with note (test first because
|
|
# it covers the simplest dispatch path)
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_unrecognized_extension(tmp_path: Path):
|
|
f = tmp_path / "thing.xyz"
|
|
f.write_text("not a real format")
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_UNRECOGNIZED
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 5 — empty file → skip silently
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_empty_file(tmp_path: Path):
|
|
f = tmp_path / "blank.pdf"
|
|
f.write_bytes(b"")
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_EMPTY
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 2 — file too large → skip with note
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_too_large(tmp_path: Path):
|
|
f = tmp_path / "huge.pdf"
|
|
f.write_bytes(b"%PDF-1.4\n" + b"\x00" * 1024)
|
|
text, status = extract_text(f, max_file_size=128)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_TOO_LARGE
|
|
|
|
|
|
def test_fringe_too_large_respects_caller_max(tmp_path: Path):
|
|
"""Caller can raise the cap for legitimate big files."""
|
|
f = tmp_path / "huge.pdf"
|
|
f.write_bytes(b"%PDF-1.4\n" + b"\x00" * 2048)
|
|
# When the cap is generous, size alone won't trigger SKIP_TOO_LARGE.
|
|
# (MarkItDown will be invoked; we mock it so the test doesn't require it.)
|
|
with patch("mempalace.format_miner._extract_via_markitdown", return_value="dummy text"):
|
|
text, status = extract_text(f, max_file_size=1024 * 1024)
|
|
assert status == ExtractionStatus.OK
|
|
assert text == "dummy text"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 1 — MarkItDown not installed → clear error
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_missing_markitdown(tmp_path: Path):
|
|
f = tmp_path / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nstub")
|
|
with patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
side_effect=ImportError("No module named 'markitdown'"),
|
|
):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_NO_MARKITDOWN
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 4 — encrypted PDF → MarkItDown raises; we catch + skip + note
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_encrypted_pdf(tmp_path: Path):
|
|
f = tmp_path / "locked.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nencrypted stub")
|
|
|
|
class _PasswordError(Exception):
|
|
pass
|
|
|
|
with patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
side_effect=_PasswordError("File has not been decrypted"),
|
|
):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_ENCRYPTED
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 6 — permission denied → catch OSError, skip
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_permission_denied(tmp_path: Path):
|
|
f = tmp_path / "denied.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nstub")
|
|
with patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
side_effect=PermissionError("Permission denied"),
|
|
):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_PERMISSION
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 7 — symlink to nothing → catch, skip
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_broken_symlink(tmp_path: Path):
|
|
target = tmp_path / "does-not-exist.pdf"
|
|
link = tmp_path / "broken-link.pdf"
|
|
_make_symlink_or_skip(link, target)
|
|
assert link.is_symlink()
|
|
text, status = extract_text(link)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_BROKEN_SYMLINK
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 3 — iCloud cloud-only file detection
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_icloud_placeholder_extension(tmp_path: Path):
|
|
# macOS sometimes leaves a literal .icloud placeholder for offloaded files
|
|
f = tmp_path / "doc.pdf.icloud"
|
|
f.write_bytes(b"placeholder")
|
|
assert is_icloud_dataless(f) is True
|
|
|
|
|
|
def test_fringe_icloud_skip_extraction(tmp_path: Path):
|
|
# Cloud-only file. extract_text should not call MarkItDown and should
|
|
# return SKIP_CLOUD_ONLY.
|
|
f = tmp_path / "doc.pdf.icloud"
|
|
f.write_bytes(b"placeholder")
|
|
with patch("mempalace.format_miner._extract_via_markitdown") as mocked:
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_CLOUD_ONLY
|
|
mocked.assert_not_called()
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 8 — encoding fallback (utf-8 → cp1252 → utf-8-replace)
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_decode_robust_clean_utf8():
|
|
assert decode_robust(b"hello \xe2\x9c\xa8 world") == "hello ✨ world"
|
|
|
|
|
|
def test_decode_robust_cp1252_fallback():
|
|
# 0x91 = U+2018 left single quote in cp1252; invalid as standalone utf-8
|
|
raw = b"hello \x91world\x92"
|
|
result = decode_robust(raw)
|
|
assert isinstance(result, str)
|
|
assert "world" in result
|
|
|
|
|
|
def test_decode_robust_never_raises():
|
|
raw = b"\xff\xfe\xfd\xfc" + b"some text"
|
|
result = decode_robust(raw)
|
|
assert isinstance(result, str)
|
|
assert "some text" in result
|
|
|
|
|
|
def test_decode_robust_empty():
|
|
assert decode_robust(b"") == ""
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 9 — Windows path differences (pathlib semantics)
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_extract_text_accepts_pathlib_path(tmp_path: Path):
|
|
"""Accept Path objects without coercing to str (Windows-safe)."""
|
|
f = tmp_path / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nstub")
|
|
with patch("mempalace.format_miner._extract_via_markitdown", return_value="content"):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
|
|
|
|
def test_extract_text_accepts_string_path(tmp_path: Path):
|
|
"""Accept str paths for callers that pre-stringify."""
|
|
f = tmp_path / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nstub")
|
|
with patch("mempalace.format_miner._extract_via_markitdown", return_value="content"):
|
|
text, status = extract_text(str(f))
|
|
assert status == ExtractionStatus.OK
|
|
|
|
|
|
def test_supported_format_check_case_insensitive(tmp_path: Path):
|
|
"""Windows often shows uppercase extensions; we still recognize them."""
|
|
f = tmp_path / "doc.PDF"
|
|
f.write_bytes(b"%PDF-1.4\nstub")
|
|
with patch("mempalace.format_miner._extract_via_markitdown", return_value="content"):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 10 — MarkItDown internal crash on malformed file
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_markitdown_generic_crash(tmp_path: Path):
|
|
f = tmp_path / "malformed.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nmalformed")
|
|
with patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
side_effect=RuntimeError("internal converter explosion"),
|
|
):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR
|
|
|
|
|
|
def test_fringe_markitdown_returns_none(tmp_path: Path):
|
|
"""MarkItDown can return None for some inputs; treat as extraction error."""
|
|
f = tmp_path / "weird.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nweird")
|
|
with patch("mempalace.format_miner._extract_via_markitdown", return_value=None):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Fringe Case 11 — network/sync drive timeout
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_fringe_network_timeout(tmp_path: Path):
|
|
f = tmp_path / "remote.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nstub")
|
|
with patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
side_effect=TimeoutError("operation timed out"),
|
|
):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_NETWORK_TIMEOUT
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Happy paths — formats we explicitly target
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_happy_path_pdf(tmp_path: Path):
|
|
f = tmp_path / "research.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nstub")
|
|
with patch("mempalace.format_miner._extract_via_markitdown", return_value="# Research\n\nbody"):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
assert text == "# Research\n\nbody"
|
|
|
|
|
|
def test_happy_path_docx(tmp_path: Path):
|
|
f = tmp_path / "notes.docx"
|
|
f.write_bytes(b"PK\x03\x04docx-stub")
|
|
with patch("mempalace.format_miner._extract_via_markitdown", return_value="# Notes\n\nbody"):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
assert text.startswith("# Notes")
|
|
|
|
|
|
def test_happy_path_rtf(tmp_path: Path):
|
|
"""RTF routes to striprtf, NOT MarkItDown.
|
|
|
|
MarkItDown 0.1.5 does not actually convert .rtf — it returns the raw
|
|
source unchanged. striprtf is the correct transformer for this format.
|
|
"""
|
|
f = tmp_path / "memo.rtf"
|
|
f.write_bytes(b"{\\rtf1\\ansi memo}")
|
|
with patch("mempalace.format_miner._extract_via_striprtf", return_value="memo body"):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
assert text == "memo body"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Striprtf path — RTF transformer routing (Fringe Case 13: SKIP_NO_STRIPRTF)
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_rtf_routes_to_striprtf_not_markitdown(tmp_path: Path):
|
|
"""Critical invariant: .rtf must NEVER touch MarkItDown.
|
|
|
|
MarkItDown 0.1.5 returns raw RTF source for .rtf inputs (verified live
|
|
against a local RTF test set on 2026-05-19). Routing .rtf through striprtf is
|
|
the bugfix.
|
|
"""
|
|
f = tmp_path / "memo.rtf"
|
|
f.write_bytes(b"{\\rtf1\\ansi memo}")
|
|
with (
|
|
patch("mempalace.format_miner._extract_via_markitdown") as mock_md,
|
|
patch("mempalace.format_miner._extract_via_striprtf", return_value="memo body") as mock_rtf,
|
|
):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
assert text == "memo body"
|
|
mock_md.assert_not_called()
|
|
mock_rtf.assert_called_once()
|
|
|
|
|
|
def test_non_rtf_does_not_touch_striprtf(tmp_path: Path):
|
|
"""Inverse invariant: .pdf / .docx / etc. must not hit striprtf."""
|
|
f = tmp_path / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4\nstub")
|
|
with (
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="pdf text") as mock_md,
|
|
patch("mempalace.format_miner._extract_via_striprtf") as mock_rtf,
|
|
):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
assert text == "pdf text"
|
|
mock_md.assert_called_once()
|
|
mock_rtf.assert_not_called()
|
|
|
|
|
|
def test_fringe_missing_striprtf(tmp_path: Path):
|
|
"""Fringe Case 13 — striprtf not installed → SKIP_NO_STRIPRTF + clear install msg."""
|
|
f = tmp_path / "memo.rtf"
|
|
f.write_bytes(b"{\\rtf1\\ansi memo}")
|
|
with patch(
|
|
"mempalace.format_miner._extract_via_striprtf",
|
|
side_effect=ImportError("No module named 'striprtf'"),
|
|
):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_NO_STRIPRTF
|
|
|
|
|
|
def test_fringe_striprtf_crash(tmp_path: Path):
|
|
"""striprtf raising any other exception → SKIP_EXTRACTION_ERROR."""
|
|
f = tmp_path / "broken.rtf"
|
|
f.write_bytes(b"{\\rtf1\\ansi broken}")
|
|
with patch(
|
|
"mempalace.format_miner._extract_via_striprtf",
|
|
side_effect=RuntimeError("rtf parse explosion"),
|
|
):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR
|
|
|
|
|
|
def test_fringe_striprtf_returns_none(tmp_path: Path):
|
|
"""striprtf returning None → SKIP_EXTRACTION_ERROR (same shape as MarkItDown)."""
|
|
f = tmp_path / "weird.rtf"
|
|
f.write_bytes(b"{\\rtf1\\ansi weird}")
|
|
with patch("mempalace.format_miner._extract_via_striprtf", return_value=None):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR
|
|
|
|
|
|
def test_fringe_striprtf_empty_output(tmp_path: Path):
|
|
"""striprtf returning empty string → SKIP_EXTRACTION_ERROR.
|
|
|
|
An RTF that strips to zero characters is not useful as a drawer; treat
|
|
as extraction failure rather than file an empty drawer.
|
|
"""
|
|
f = tmp_path / "blank-after-strip.rtf"
|
|
f.write_bytes(b"{\\rtf1\\ansi}")
|
|
with patch("mempalace.format_miner._extract_via_striprtf", return_value=""):
|
|
text, status = extract_text(f)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_EXTRACTION_ERROR
|
|
|
|
|
|
def test_rtf_uppercase_extension_also_routes_to_striprtf(tmp_path: Path):
|
|
"""Windows case-insensitive: .RTF must also route to striprtf."""
|
|
f = tmp_path / "memo.RTF"
|
|
f.write_bytes(b"{\\rtf1\\ansi memo}")
|
|
with (
|
|
patch("mempalace.format_miner._extract_via_markitdown") as mock_md,
|
|
patch("mempalace.format_miner._extract_via_striprtf", return_value="memo body") as mock_rtf,
|
|
):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
mock_md.assert_not_called()
|
|
mock_rtf.assert_called_once()
|
|
|
|
|
|
def test_happy_path_xlsx(tmp_path: Path):
|
|
f = tmp_path / "spreadsheet.xlsx"
|
|
f.write_bytes(b"PK\x03\x04xlsx-stub")
|
|
with patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
return_value="| col1 | col2 |\n|---|---|\n| a | b |",
|
|
):
|
|
text, status = extract_text(f)
|
|
assert status == ExtractionStatus.OK
|
|
assert "col1" in text
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# scan_formats — directory walker, returns Path objects sorted
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_scan_formats_finds_supported_only(tmp_path: Path):
|
|
(tmp_path / "a.pdf").write_bytes(b"pdf")
|
|
(tmp_path / "b.txt").write_text("text") # unsupported
|
|
(tmp_path / "c.docx").write_bytes(b"docx")
|
|
(tmp_path / "d.rtf").write_bytes(b"rtf")
|
|
found = {f.name for f in scan_formats(tmp_path)}
|
|
assert "a.pdf" in found
|
|
assert "c.docx" in found
|
|
assert "d.rtf" in found
|
|
assert "b.txt" not in found
|
|
|
|
|
|
def test_scan_formats_walks_subdirectories(tmp_path: Path):
|
|
nested = tmp_path / "deep" / "nested"
|
|
nested.mkdir(parents=True)
|
|
(nested / "buried.pdf").write_bytes(b"pdf")
|
|
found = {f.name for f in scan_formats(tmp_path)}
|
|
assert "buried.pdf" in found
|
|
|
|
|
|
def test_scan_formats_skips_hidden_dirs(tmp_path: Path):
|
|
"""Don't descend into .git, .venv, __pycache__, etc. — same SKIP_DIRS as miner."""
|
|
hidden = tmp_path / ".git"
|
|
hidden.mkdir()
|
|
(hidden / "hidden.pdf").write_bytes(b"pdf")
|
|
(tmp_path / "visible.pdf").write_bytes(b"pdf")
|
|
found = {f.name for f in scan_formats(tmp_path)}
|
|
assert "visible.pdf" in found
|
|
assert "hidden.pdf" not in found
|
|
|
|
|
|
def test_scan_formats_skips_ds_store(tmp_path: Path):
|
|
(tmp_path / ".DS_Store").write_bytes(b"")
|
|
(tmp_path / "real.pdf").write_bytes(b"pdf")
|
|
found = {f.name for f in scan_formats(tmp_path)}
|
|
assert "real.pdf" in found
|
|
assert ".DS_Store" not in found
|
|
|
|
|
|
def test_scan_formats_returns_empty_for_missing_dir(tmp_path: Path):
|
|
"""scan_formats handles a path that doesn't exist (no crash, empty list)."""
|
|
missing = tmp_path / "does-not-exist"
|
|
assert scan_formats(missing) == []
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# decode_robust — exercise the cp1252 path explicitly
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_decode_robust_pure_cp1252():
|
|
"""Bytes that are invalid UTF-8 but valid CP1252 → second-attempt success."""
|
|
raw = b"\x91hello\x92"
|
|
result = decode_robust(raw)
|
|
assert isinstance(result, str)
|
|
assert "hello" in result
|
|
# 0x91 / 0x92 are CP1252 smart quotes that decode without error
|
|
assert "�" not in result, "CP1252 path should not need the replace fallback"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Note on stat() error branches in extract_text:
|
|
# Dedicated PermissionError / FileNotFoundError / OSError stat-handler tests
|
|
# were attempted but tripped on Python 3.13 pathlib internals (patching
|
|
# Path.stat globally interferes with .exists() / .is_symlink() which call
|
|
# stat under the hood). The handlers are kept as defensive guards and
|
|
# remain documented as uncovered branches — the rest of the suite (60+
|
|
# tests + live integration on 63 real archive files) provides the
|
|
# end-to-end safety net.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Live transformer body tests — skipped when transformer isn't installed.
|
|
# These exercise the actual adapter code that mocked tests can't reach.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_extract_via_striprtf_live(tmp_path: Path):
|
|
"""End-to-end: real striprtf converts a real RTF blob to plain text.
|
|
|
|
Skipped if striprtf isn't installed (e.g., environments without the
|
|
[extract] extra). When installed, this exercises the actual adapter
|
|
body that mocked tests bypass.
|
|
"""
|
|
pytest.importorskip("striprtf.striprtf")
|
|
from mempalace.format_miner import _extract_via_striprtf
|
|
|
|
rtf = (
|
|
b"{\\rtf1\\ansi\\ansicpg1252\n{\\fonttbl\\f0\\fnil Helvetica;}\n"
|
|
b"\\f0\\fs24 Hello from a real RTF blob.}"
|
|
)
|
|
f = tmp_path / "live.rtf"
|
|
f.write_bytes(rtf)
|
|
text = _extract_via_striprtf(f)
|
|
assert text is not None
|
|
assert "Hello from a real RTF blob" in text
|
|
assert "\\rtf1" not in text, "raw RTF control codes leaked into output"
|
|
|
|
|
|
def test_extract_via_markitdown_live_pdf(tmp_path: Path):
|
|
"""End-to-end: real MarkItDown converts a real PDF blob to text.
|
|
|
|
Skipped if markitdown isn't installed (3.10+ only, optional extra).
|
|
Also skipped if only the placeholder ``markitdown`` package is present
|
|
(the real Microsoft package exposes the ``MarkItDown`` class).
|
|
"""
|
|
try:
|
|
from markitdown import MarkItDown # noqa: F401
|
|
except ImportError:
|
|
pytest.skip("real Microsoft markitdown not installed (needs Python 3.10+)")
|
|
from mempalace.format_miner import _extract_via_markitdown
|
|
|
|
# Minimal valid PDF that contains the literal text "hello pdf"
|
|
pdf_bytes = (
|
|
b"%PDF-1.1\n"
|
|
b"1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n"
|
|
b"2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj\n"
|
|
b"3 0 obj<</Type/Page/Parent 2 0 R/Resources<</Font<</F1 4 0 R>>>>"
|
|
b"/MediaBox[0 0 612 792]/Contents 5 0 R>>endobj\n"
|
|
b"4 0 obj<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>endobj\n"
|
|
b"5 0 obj<</Length 44>>stream\n"
|
|
b"BT /F1 24 Tf 100 700 Td (hello pdf) Tj ET\n"
|
|
b"endstream endobj\n"
|
|
b"xref\n0 6\n0000000000 65535 f \n"
|
|
b"trailer<</Size 6/Root 1 0 R>>\n"
|
|
b"startxref\n0\n%%EOF\n"
|
|
)
|
|
f = tmp_path / "live.pdf"
|
|
f.write_bytes(pdf_bytes)
|
|
# The adapter shouldn't crash; if MarkItDown returns text, it includes our marker.
|
|
text = _extract_via_markitdown(f)
|
|
# Allow None (MarkItDown may not parse this minimal PDF cleanly), but the
|
|
# adapter path itself must run without exception.
|
|
if text is not None:
|
|
assert isinstance(text, str)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# mine_formats — orchestrator that walks a directory, transforms files,
|
|
# chunks the extracted text, and files drawers into the palace. Mirrors the
|
|
# shape of mine_convos / mine. The collection + lock primitives are mocked
|
|
# so these tests don't write to a real palace.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.fixture
|
|
def _mine_formats_mocks(tmp_path: Path):
|
|
"""Common mocks for mine_formats orchestrator tests.
|
|
|
|
Patches:
|
|
- scan_formats : returns the files we hand it
|
|
- get_collection : returns a MagicMock collection
|
|
- mine_lock : no-op context manager
|
|
- file_already_mined : False (file not yet mined)
|
|
- _extract_via_striprtf / _extract_via_markitdown : mocked at test level
|
|
"""
|
|
from unittest.mock import MagicMock, patch
|
|
from contextlib import contextmanager
|
|
|
|
collection = MagicMock()
|
|
collection.delete = MagicMock()
|
|
collection.upsert = MagicMock()
|
|
|
|
@contextmanager
|
|
def _fake_lock(source_file):
|
|
yield
|
|
|
|
with (
|
|
patch("mempalace.format_miner.get_collection", return_value=collection) as p_coll,
|
|
patch("mempalace.format_miner.mine_lock", side_effect=_fake_lock) as p_lock,
|
|
patch("mempalace.format_miner.file_already_mined", return_value=False) as p_mined,
|
|
):
|
|
yield {
|
|
"collection": collection,
|
|
"get_collection": p_coll,
|
|
"mine_lock": p_lock,
|
|
"file_already_mined": p_mined,
|
|
"tmp_path": tmp_path,
|
|
}
|
|
|
|
|
|
def test_mine_formats_walks_directory(_mine_formats_mocks):
|
|
"""mine_formats must use scan_formats to discover supported files."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
(tmp / "doc.pdf").write_bytes(b"pdf")
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[]) as p_scan,
|
|
patch("mempalace.format_miner._extract_via_markitdown"),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
p_scan.assert_called_once()
|
|
|
|
|
|
def test_mine_formats_skips_already_mined_files(_mine_formats_mocks):
|
|
"""If file_already_mined returns True, extract_text should not be called."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
_mine_formats_mocks["file_already_mined"].return_value = True
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown") as p_md,
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
p_md.assert_not_called()
|
|
|
|
|
|
def test_mine_formats_skips_extraction_failures(_mine_formats_mocks):
|
|
"""When extract_text returns a SKIP status, no real drawers are upserted.
|
|
|
|
A sentinel upsert IS expected (so the file isn't re-extracted on every
|
|
re-mine). The sentinel carries ``is_sentinel=True`` to distinguish it
|
|
from a content drawer.
|
|
"""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "bad.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
side_effect=RuntimeError("converter blew up"),
|
|
),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
# Any upsert that fired must be a sentinel, never a content drawer.
|
|
for call in _mine_formats_mocks["collection"].upsert.call_args_list:
|
|
metas = call.kwargs.get("metadatas") or call.args[2]
|
|
for m in metas:
|
|
assert m.get("is_sentinel") is True, (
|
|
f"unexpected non-sentinel upsert on extraction failure: {m}"
|
|
)
|
|
|
|
|
|
def test_mine_formats_files_drawers_for_ok_extractions(_mine_formats_mocks):
|
|
"""When extract_text returns OK + text, mine_formats chunks and upserts drawers."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "good.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
long_text = "This is a sufficiently long extracted text. " * 30
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value=long_text),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
# upsert should have been called at least once
|
|
assert _mine_formats_mocks["collection"].upsert.called
|
|
|
|
|
|
def test_mine_formats_dry_run_does_not_open_collection(_mine_formats_mocks):
|
|
"""dry_run=True must not call get_collection or upsert any drawers."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
_mine_formats_mocks["get_collection"].reset_mock()
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="some text"),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), dry_run=True)
|
|
_mine_formats_mocks["get_collection"].assert_not_called()
|
|
_mine_formats_mocks["collection"].upsert.assert_not_called()
|
|
|
|
|
|
def test_mine_formats_respects_limit(_mine_formats_mocks):
|
|
"""limit=N should restrict processing to the first N files."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
files = []
|
|
for i in range(5):
|
|
p = tmp / f"f{i}.pdf"
|
|
p.write_bytes(b"%PDF-1.4 stub")
|
|
files.append(p)
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=files),
|
|
patch(
|
|
"mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50
|
|
) as p_md,
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), limit=2)
|
|
# Only 2 of the 5 files should have been extracted
|
|
assert p_md.call_count == 2
|
|
|
|
|
|
def test_mine_formats_limit_skips_already_mined(_mine_formats_mocks):
|
|
"""--limit N counts only new work, not already-mined skips (#1535)."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
files = []
|
|
for i in range(6):
|
|
p = tmp / f"f{i}.pdf"
|
|
p.write_bytes(b"%PDF-1.4 stub")
|
|
files.append(p)
|
|
|
|
call_idx = 0
|
|
|
|
def fake_already_mined(collection, source_file, **kwargs):
|
|
nonlocal call_idx
|
|
call_idx += 1
|
|
return call_idx <= 4
|
|
|
|
_mine_formats_mocks["file_already_mined"].side_effect = fake_already_mined
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=files),
|
|
patch(
|
|
"mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50
|
|
) as p_md,
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), limit=1)
|
|
assert p_md.call_count == 1
|
|
|
|
|
|
def test_mine_formats_wing_defaults_from_directory_name(_mine_formats_mocks):
|
|
"""When wing=None, the directory's basename becomes the wing."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
target_dir = tmp / "my_research_corpus"
|
|
target_dir.mkdir()
|
|
f = target_dir / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50),
|
|
):
|
|
mine_formats(format_dir=str(target_dir), palace_path=str(tmp / "palace"))
|
|
# Inspect the metadata passed to upsert — the wing should derive from the dir name
|
|
call_args = _mine_formats_mocks["collection"].upsert.call_args
|
|
if call_args is not None:
|
|
metas = call_args.kwargs.get("metadatas") or call_args.args[2]
|
|
assert metas[0]["wing"] == "my_research_corpus"
|
|
|
|
|
|
def test_mine_formats_wing_override(_mine_formats_mocks):
|
|
"""Explicit wing= param overrides the directory-name default."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50),
|
|
):
|
|
mine_formats(
|
|
format_dir=str(tmp),
|
|
palace_path=str(tmp / "palace"),
|
|
wing="custom_wing_name",
|
|
)
|
|
call_args = _mine_formats_mocks["collection"].upsert.call_args
|
|
if call_args is not None:
|
|
metas = call_args.kwargs.get("metadatas") or call_args.args[2]
|
|
assert metas[0]["wing"] == "custom_wing_name"
|
|
|
|
|
|
def test_mine_formats_ingest_mode_metadata_is_extract(_mine_formats_mocks):
|
|
"""Drawers from mine_formats must carry ingest_mode='extract' so they're
|
|
distinguishable from project / convo drawers in the palace."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long text " * 50),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
call_args = _mine_formats_mocks["collection"].upsert.call_args
|
|
assert call_args is not None
|
|
metas = call_args.kwargs.get("metadatas") or call_args.args[2]
|
|
assert metas[0]["ingest_mode"] == "extract"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Bot-review parity fixes (PR #1555 follow-up commit, 2026-05-19):
|
|
# - palace.SKIP_DIRS reuse (covered indirectly by scan_formats tests)
|
|
# - scan_formats skips symlinks
|
|
# - mine_formats uses check_mtime=True
|
|
# - source_mtime + hall + entities in drawer metadata
|
|
# - per-file try/except so one bad file doesn't crash the whole mine
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_scan_formats_skips_symlinks(tmp_path: Path):
|
|
"""Symlinks must be skipped — mirrors miner.py and convo_miner.py."""
|
|
real = tmp_path / "real.pdf"
|
|
real.write_bytes(b"%PDF-1.4 stub")
|
|
link = tmp_path / "alias.pdf"
|
|
_make_symlink_or_skip(link, real)
|
|
found = {f.name for f in scan_formats(tmp_path)}
|
|
assert "real.pdf" in found
|
|
assert "alias.pdf" not in found
|
|
|
|
|
|
def test_mine_formats_uses_check_mtime_true(_mine_formats_mocks):
|
|
"""mine_formats must call file_already_mined with check_mtime=True so
|
|
updated documents get re-mined (matches miner.py semantics)."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
# Inspect every file_already_mined call — they must all use check_mtime=True
|
|
calls = _mine_formats_mocks["file_already_mined"].call_args_list
|
|
assert calls, "file_already_mined should have been called at least once"
|
|
for call in calls:
|
|
# check_mtime can be in kwargs or positional. Inspect both.
|
|
kwargs = call.kwargs
|
|
assert kwargs.get("check_mtime") is True, f"check_mtime must be True, got {kwargs}"
|
|
|
|
|
|
def test_mine_formats_records_source_mtime_in_drawer_metadata(_mine_formats_mocks):
|
|
"""Each drawer must carry source_mtime so file_already_mined(check_mtime=True)
|
|
can detect updates on re-mine."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
upsert_calls = _mine_formats_mocks["collection"].upsert.call_args_list
|
|
# Find the content upsert (not the sentinel)
|
|
found_mtime = False
|
|
for call in upsert_calls:
|
|
metas = call.kwargs.get("metadatas") or call.args[2]
|
|
for m in metas:
|
|
if m.get("is_sentinel"):
|
|
continue
|
|
assert "source_mtime" in m, f"missing source_mtime in drawer meta: {m}"
|
|
assert isinstance(m["source_mtime"], (int, float))
|
|
found_mtime = True
|
|
assert found_mtime, "no non-sentinel drawer upsert observed"
|
|
|
|
|
|
def test_mine_formats_records_hall_in_drawer_metadata(_mine_formats_mocks):
|
|
"""Each drawer must carry a 'hall' tag — matches miner.py drawer quality."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
upsert_calls = _mine_formats_mocks["collection"].upsert.call_args_list
|
|
found_hall = False
|
|
for call in upsert_calls:
|
|
metas = call.kwargs.get("metadatas") or call.args[2]
|
|
for m in metas:
|
|
if m.get("is_sentinel"):
|
|
continue
|
|
assert "hall" in m, f"missing hall in drawer meta: {m}"
|
|
assert isinstance(m["hall"], str)
|
|
found_hall = True
|
|
assert found_hall, "no non-sentinel drawer upsert observed"
|
|
|
|
|
|
def test_mine_formats_continues_after_per_file_error(_mine_formats_mocks):
|
|
"""One bad file must not crash the whole mine — the loop continues."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
bad = tmp / "broken.pdf"
|
|
bad.write_bytes(b"%PDF-1.4 stub")
|
|
good = tmp / "fine.pdf"
|
|
good.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
# Make chunk_text crash on the first file but succeed on the second.
|
|
call_count = {"n": 0}
|
|
|
|
def chunk_text_first_fails(content, source_file, **kwargs):
|
|
call_count["n"] += 1
|
|
if call_count["n"] == 1:
|
|
raise RuntimeError("simulated chunker explosion")
|
|
return [{"content": "good chunk content here " * 5, "chunk_index": 0}]
|
|
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[bad, good]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="text"),
|
|
# Patch the module-level binding in format_miner (hoisted in the
|
|
# PR #1555 polish work — Gemini #3). The old patch target
|
|
# ``mempalace.miner.chunk_text`` no longer works because
|
|
# format_miner now binds chunk_text at its own module scope.
|
|
patch("mempalace.format_miner.chunk_text", side_effect=chunk_text_first_fails),
|
|
):
|
|
# Must not raise
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
|
|
# The second file should still have produced an upsert
|
|
upsert_calls = _mine_formats_mocks["collection"].upsert.call_args_list
|
|
non_sentinel_upserts = [
|
|
call
|
|
for call in upsert_calls
|
|
if not any(
|
|
(m or {}).get("is_sentinel") for m in (call.kwargs.get("metadatas") or call.args[2])
|
|
)
|
|
]
|
|
assert non_sentinel_upserts, (
|
|
"the good file should still have produced a content upsert after the bad file errored"
|
|
)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Amendment #3 — parity with miner.py: detect_room + tunnel computation.
|
|
# Tests written FIRST (TDD) to prove the gaps exist, then format_miner.py is
|
|
# amended to make them pass. Pattern mirrors miner.py exactly.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_mine_formats_calls_load_config_for_rooms(_mine_formats_mocks):
|
|
"""mine_formats must load mempalace.yaml (via load_config) to get the
|
|
rooms list — same as miner.py:1154. Without this, drawers fall back to
|
|
a single 'documents' room."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
fake_config = {
|
|
"wing": "wing_aya",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config) as p_cfg,
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
p_cfg.assert_called_once()
|
|
|
|
|
|
def test_mine_formats_calls_detect_room_per_file(_mine_formats_mocks):
|
|
"""mine_formats must call detect_room(filepath, content, rooms, project_path)
|
|
for each file — same as miner.py:904. Hardcoding room='documents' is the
|
|
bug; this test fails until detect_room is wired in."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
fake_config = {
|
|
"wing": "wing_aya",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
patch("mempalace.format_miner.detect_room", return_value="family") as p_room,
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
p_room.assert_called_once()
|
|
# First positional arg should be the file Path; last arg should be project_path
|
|
call_args = p_room.call_args
|
|
# detect_room(filepath, content, rooms, project_path)
|
|
assert len(call_args.args) == 4 or "filepath" in call_args.kwargs
|
|
|
|
|
|
def test_mine_formats_uses_detected_room_in_drawer_metadata(_mine_formats_mocks):
|
|
"""The room field on the drawer metadata must reflect what detect_room
|
|
returned, NOT the hardcoded 'documents'. This is the visible bug from
|
|
the 2026-05-19 mine: every drawer landed in room='documents'."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
fake_config = {
|
|
"wing": "wing_aya",
|
|
"rooms": [{"name": "family", "keywords": ["family"]}],
|
|
}
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
patch("mempalace.format_miner.detect_room", return_value="family"),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"))
|
|
upsert_calls = _mine_formats_mocks["collection"].upsert.call_args_list
|
|
found_room = None
|
|
for call in upsert_calls:
|
|
metas = call.kwargs.get("metadatas") or call.args[2]
|
|
for m in metas:
|
|
if m.get("is_sentinel"):
|
|
continue
|
|
found_room = m.get("room")
|
|
break
|
|
if found_room:
|
|
break
|
|
assert found_room == "family", (
|
|
f"drawer room must be 'family' (detect_room's return), not {found_room!r}"
|
|
)
|
|
|
|
|
|
def test_mine_formats_calls_compute_topic_tunnels_after_loop(_mine_formats_mocks):
|
|
"""After the per-file loop, mine_formats must call
|
|
_compute_topic_tunnels_for_wing(wing) exactly once. Mirrors miner.py:1241.
|
|
Without this, cross-wing topic tunnels never materialize for format-mined
|
|
wings (audit confirmed 0 tunnels in the 2026-05-19 v6 mine)."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
fake_config = {
|
|
"wing": "wing_aya",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
patch("mempalace.format_miner.detect_room", return_value="documents"),
|
|
patch("mempalace.format_miner._compute_topic_tunnels_for_wing", return_value=0) as p_tun,
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_aya")
|
|
p_tun.assert_called_once_with("wing_aya")
|
|
|
|
|
|
def test_mine_formats_tunnel_failure_does_not_crash_mine(_mine_formats_mocks):
|
|
"""If _compute_topic_tunnels_for_wing raises, the mine must still
|
|
complete (summary still prints, no exception propagates). Mirrors the
|
|
try/except wrap at miner.py:1244-1249."""
|
|
from unittest.mock import patch
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = _mine_formats_mocks["tmp_path"]
|
|
f = tmp / "doc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
fake_config = {
|
|
"wing": "wing_aya",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
patch("mempalace.format_miner.detect_room", return_value="documents"),
|
|
patch(
|
|
"mempalace.format_miner._compute_topic_tunnels_for_wing",
|
|
side_effect=RuntimeError("simulated tunnel-compute failure"),
|
|
),
|
|
):
|
|
# Must NOT raise
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_aya")
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# PR #1555 review (Igor) — missing-format-deps surfacing + pyproject extras
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_extract_text_missing_format_dep_returns_distinct_status(tmp_path: Path, monkeypatch):
|
|
"""A MarkItDown ``MissingDependencyException`` (raised when a per-format
|
|
sub-extra like ``markitdown[pdf]`` isn't installed) must surface as
|
|
``SKIP_MISSING_FORMAT_DEPS`` — NOT the generic ``SKIP_EXTRACTION_ERROR``.
|
|
|
|
Per PR #1555 review (Igor): real PDFs hit this in production today
|
|
and users see "extraction error" with no signal that the fix is
|
|
``pip install markitdown[pdf]``. The dispatcher must catch by type
|
|
name so the real markitdown package doesn't have to be import-
|
|
resolvable for the catch to fire.
|
|
"""
|
|
from mempalace import format_miner
|
|
|
|
# Build a fake exception class with the right __name__ so the
|
|
# type-name catch in extract_text recognises it without requiring
|
|
# the real markitdown to be installed in the test environment.
|
|
class MissingDependencyException(Exception):
|
|
pass
|
|
|
|
def fake_extract(p):
|
|
raise MissingDependencyException(
|
|
"MarkItDown failed: install with `pip install markitdown[pdf]`"
|
|
)
|
|
|
|
monkeypatch.setattr(format_miner, "_extract_via_markitdown", fake_extract)
|
|
|
|
pdf = tmp_path / "fake.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
text, status = extract_text(pdf)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_MISSING_FORMAT_DEPS, (
|
|
f"expected SKIP_MISSING_FORMAT_DEPS, got {status}"
|
|
)
|
|
|
|
|
|
def test_pyproject_extract_extra_pulls_markitdown_format_subdeps():
|
|
"""The ``mempalace[extract]`` extra must include MarkItDown's per-format
|
|
sub-extras (``pdf``, ``docx``, ``pptx``, ``xlsx``) — without them, real
|
|
PDF/DOCX/etc files hit ``MissingDependencyException`` at runtime even
|
|
after ``pip install mempalace[extract]``. Per PR #1555 review (Igor).
|
|
|
|
``.rtf`` is covered by the separate ``striprtf`` dependency and ``.epub``
|
|
ships in base MarkItDown (``EpubConverter`` uses ``beautifulsoup4``
|
|
which is a base requirement, not extra-gated), so neither needs a
|
|
sub-extra here.
|
|
"""
|
|
try:
|
|
import tomllib # 3.11+
|
|
except ImportError: # pragma: no cover — only hit on 3.9/3.10
|
|
import tomli as tomllib # type: ignore[import-not-found, no-redef]
|
|
|
|
pyproject_path = Path(__file__).resolve().parent.parent / "pyproject.toml"
|
|
with open(pyproject_path, "rb") as f:
|
|
data = tomllib.load(f)
|
|
|
|
extract = data["project"]["optional-dependencies"]["extract"]
|
|
extract_str = " ".join(extract).lower()
|
|
|
|
# Accept any layout — comma-separated ``markitdown[docx,pdf,pptx,xlsx]``
|
|
# or separate ``markitdown[pdf]`` entries — as long as each format
|
|
# appears inside SOME ``markitdown[...]`` bracketed group.
|
|
import re
|
|
|
|
bracketed = "".join(re.findall(r"markitdown\[([^\]]+)\]", extract_str))
|
|
for sub in ("pdf", "docx", "pptx", "xlsx"):
|
|
assert sub in bracketed, (
|
|
f"[extract] must include markitdown[{sub}]; "
|
|
f"got bracketed extras: {bracketed!r}; full extract: {extract}"
|
|
)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# PR #1555 review polish — bot feedback items addressed post-merge.
|
|
# Five behavioral changes; the trivial items (path expanduser, signature
|
|
# annotations, docstring/doc updates) don't need their own RED tests.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_extract_text_nonexistent_regular_file_returns_unreadable_not_broken_symlink(
|
|
tmp_path: Path,
|
|
):
|
|
"""A non-existent REGULAR file (not a symlink) must return
|
|
``SKIP_UNREADABLE``, not ``SKIP_BROKEN_SYMLINK``.
|
|
|
|
Per PR #1555 review (Copilot #8): the ``FileNotFoundError`` arm in
|
|
``extract_text`` was unconditionally mapping to ``SKIP_BROKEN_SYMLINK``,
|
|
which is misleading when the path was a regular file that got deleted
|
|
between scan and extract. ``SKIP_BROKEN_SYMLINK`` should only fire
|
|
when ``p.is_symlink()`` is true.
|
|
"""
|
|
p = tmp_path / "deleted.pdf"
|
|
assert not p.exists()
|
|
assert not p.is_symlink()
|
|
text, status = extract_text(p)
|
|
assert text is None
|
|
assert status == ExtractionStatus.SKIP_UNREADABLE, (
|
|
f"Expected SKIP_UNREADABLE for non-existent regular file (file was "
|
|
f"deleted between scan and extract); got {status}. "
|
|
f"SKIP_BROKEN_SYMLINK should only fire when the path is_symlink()."
|
|
)
|
|
|
|
|
|
def test_mine_formats_passes_extract_mode_format_to_file_already_mined(monkeypatch, tmp_path: Path):
|
|
"""``mine_formats`` must call ``file_already_mined`` with
|
|
``extract_mode='format'`` so format-mode idempotency is scoped to its
|
|
own drawer set. Otherwise drawers from convo_miner / project miner
|
|
on the same source file falsely indicate "already mined" to the
|
|
format miner (and vice versa).
|
|
|
|
Per PR #1555 review (Copilot #11 + #12). Both call sites — the
|
|
pre-lock check in ``mine_formats`` and the post-lock recheck in
|
|
``_file_chunks_locked`` — must pass the kwarg.
|
|
"""
|
|
from mempalace import format_miner
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
calls: list = []
|
|
|
|
def fake_check(collection, source_file, check_mtime=False, extract_mode=None):
|
|
calls.append(
|
|
{
|
|
"source_file": source_file,
|
|
"check_mtime": check_mtime,
|
|
"extract_mode": extract_mode,
|
|
}
|
|
)
|
|
return False # never already mined — let the mine proceed
|
|
|
|
monkeypatch.setattr(format_miner, "file_already_mined", fake_check)
|
|
|
|
tmp = tmp_path / "src"
|
|
tmp.mkdir()
|
|
f = tmp / "x.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
fake_config = {
|
|
"wing": "wing_test",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 50),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
patch("mempalace.format_miner.detect_room", return_value="documents"),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test")
|
|
|
|
assert calls, "expected file_already_mined to be called at least once"
|
|
for call in calls:
|
|
assert call["extract_mode"] == "format", (
|
|
f"file_already_mined call missing extract_mode='format': {call}"
|
|
)
|
|
|
|
|
|
def test_mine_formats_does_not_write_sentinel_for_skip_no_markitdown(monkeypatch, tmp_path: Path):
|
|
"""Sentinel writing must be skipped when the status is a transient
|
|
"missing optional dependency" (``SKIP_NO_MARKITDOWN``,
|
|
``SKIP_NO_STRIPRTF``, ``SKIP_MISSING_FORMAT_DEPS``). Otherwise the
|
|
file is permanently marked as "already mined" — installing the missing
|
|
extra later does NOT trigger a re-mine, which is a real user surprise.
|
|
|
|
Per PR #1555 review (Copilot #14).
|
|
"""
|
|
from mempalace import format_miner
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
register_calls: list = []
|
|
|
|
def fake_register(collection, source_file, wing, agent):
|
|
register_calls.append(source_file)
|
|
|
|
monkeypatch.setattr(format_miner, "_register_file", fake_register)
|
|
|
|
tmp = tmp_path / "src"
|
|
tmp.mkdir()
|
|
f = tmp / "y.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
fake_config = {
|
|
"wing": "wing_test",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
|
|
# Force the extract path to raise ImportError so we hit SKIP_NO_MARKITDOWN.
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
side_effect=ImportError("no markitdown"),
|
|
),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
patch("mempalace.format_miner.detect_room", return_value="documents"),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test")
|
|
|
|
assert str(f) not in register_calls, (
|
|
f"sentinel was written for SKIP_NO_MARKITDOWN file; should be skipped "
|
|
f"so installing markitdown later triggers a re-mine. "
|
|
f"register_calls={register_calls}"
|
|
)
|
|
|
|
|
|
def test_mine_formats_does_not_write_sentinel_for_skip_missing_format_deps(
|
|
monkeypatch, tmp_path: Path
|
|
):
|
|
"""Same as the SKIP_NO_MARKITDOWN test but for SKIP_MISSING_FORMAT_DEPS
|
|
(raised when markitdown is installed but a per-format sub-extra like
|
|
``markitdown[pdf]`` is missing).
|
|
"""
|
|
from mempalace import format_miner
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
register_calls: list = []
|
|
|
|
def fake_register(collection, source_file, wing, agent):
|
|
register_calls.append(source_file)
|
|
|
|
monkeypatch.setattr(format_miner, "_register_file", fake_register)
|
|
|
|
tmp = tmp_path / "src"
|
|
tmp.mkdir()
|
|
f = tmp / "z.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
fake_config = {
|
|
"wing": "wing_test",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
|
|
class _FakeMissingDep(Exception):
|
|
pass
|
|
|
|
_FakeMissingDep.__name__ = "MissingDependencyException"
|
|
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch(
|
|
"mempalace.format_miner._extract_via_markitdown",
|
|
side_effect=_FakeMissingDep("install markitdown[pdf]"),
|
|
),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
patch("mempalace.format_miner.detect_room", return_value="documents"),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test")
|
|
|
|
assert str(f) not in register_calls, (
|
|
f"sentinel was written for SKIP_MISSING_FORMAT_DEPS file; should be "
|
|
f"skipped so installing markitdown[<fmt>] later triggers a re-mine. "
|
|
f"register_calls={register_calls}"
|
|
)
|
|
|
|
|
|
def test_mine_formats_catches_unexpected_exception_and_prints_summary(
|
|
monkeypatch, tmp_path: Path, capsys
|
|
):
|
|
"""If an unexpected error happens during mining (something the
|
|
per-file try/except doesn't catch — e.g., the topic-tunnel block
|
|
blowing up in a way the inner handler misses), mine_formats must
|
|
catch it at the outer level, print a partial-progress summary, and
|
|
clean up the PID file. Not crash with a traceback to the user.
|
|
|
|
Per PR #1555 review (Gemini #5).
|
|
"""
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
tmp = tmp_path / "src"
|
|
tmp.mkdir()
|
|
f = tmp / "exc.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
fake_config = {
|
|
"wing": "wing_test",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
|
|
# Patch the file enumeration step to raise something unexpected —
|
|
# this fires OUTSIDE the per-file try/except. Without an outer
|
|
# ``except Exception`` clause, the traceback propagates and the
|
|
# summary block never prints.
|
|
def angry_enumerate(*args, **kwargs):
|
|
raise RuntimeError("simulated outer-loop explosion")
|
|
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", side_effect=angry_enumerate),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
):
|
|
# Must NOT raise — outer except Exception should catch.
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test")
|
|
|
|
|
|
def test_mine_formats_threads_chunk_size_from_user_config(monkeypatch, tmp_path: Path):
|
|
"""``mine_formats`` must pass ``chunk_size`` (and ``chunk_overlap``,
|
|
``min_chunk_size``) from MempalaceConfig through to ``chunk_text``.
|
|
Currently the config is loaded only to validate readability; the
|
|
custom chunk parameters are never used during format-mode mining,
|
|
so users who tuned their config see no effect.
|
|
|
|
Per PR #1555 review (Gemini #3).
|
|
"""
|
|
from mempalace import format_miner
|
|
from mempalace.format_miner import mine_formats
|
|
|
|
chunk_calls: list = []
|
|
|
|
def fake_chunk_text(content, source_file, **kwargs):
|
|
chunk_calls.append(kwargs)
|
|
return [{"content": content, "chunk_index": 0}]
|
|
|
|
monkeypatch.setattr(format_miner, "chunk_text", fake_chunk_text)
|
|
|
|
tmp = tmp_path / "src"
|
|
tmp.mkdir()
|
|
f = tmp / "config.pdf"
|
|
f.write_bytes(b"%PDF-1.4 stub")
|
|
|
|
fake_config = {
|
|
"wing": "wing_test",
|
|
"rooms": [{"name": "documents", "keywords": ["documents"]}],
|
|
}
|
|
|
|
# Inject custom config values via a fake MempalaceConfig.
|
|
class _FakeMempalaceConfig:
|
|
chunk_size = 1234
|
|
chunk_overlap = 56
|
|
min_chunk_size = 78
|
|
|
|
@property
|
|
def palace_path(self):
|
|
return str(tmp_path / "palace")
|
|
|
|
monkeypatch.setattr(format_miner, "MempalaceConfig", _FakeMempalaceConfig)
|
|
|
|
with (
|
|
patch("mempalace.format_miner.scan_formats", return_value=[f]),
|
|
patch("mempalace.format_miner._extract_via_markitdown", return_value="long " * 200),
|
|
patch("mempalace.format_miner.load_config", return_value=fake_config),
|
|
patch("mempalace.format_miner.detect_room", return_value="documents"),
|
|
patch("mempalace.format_miner.file_already_mined", return_value=False),
|
|
):
|
|
mine_formats(format_dir=str(tmp), palace_path=str(tmp / "palace"), wing="wing_test")
|
|
|
|
assert chunk_calls, "expected chunk_text to be called at least once"
|
|
call = chunk_calls[0]
|
|
assert call.get("chunk_size") == 1234, (
|
|
f"chunk_text called without user's chunk_size from MempalaceConfig "
|
|
f"(got {call.get('chunk_size')}, expected 1234). kwargs={call}"
|
|
)
|
|
assert call.get("chunk_overlap") == 56, (
|
|
f"chunk_text called without user's chunk_overlap "
|
|
f"(got {call.get('chunk_overlap')}, expected 56). kwargs={call}"
|
|
)
|
|
assert call.get("min_chunk_size") == 78, (
|
|
f"chunk_text called without user's min_chunk_size "
|
|
f"(got {call.get('min_chunk_size')}, expected 78). kwargs={call}"
|
|
)
|