chore: import upstream snapshot with attribution

tools/mtmd/tests/test-1-ground-truth.txt

@@ -0,0 +1,24 @@
+
+ A Powdery Surface
+  Is Closely Explored
+
+By JOHN NOBLE WILFORD
+Special to The New York Times
+
+HOUSTON, Monday, July 21—Men have landed and walked on the moon.
+
+Two Americans, astronauts of Apollo 11, steered their fragile four-legged lunar module safely and smoothly to the historic landing yesterday at 4:17:40 P.M., Eastern daylight time.
+
+Neil A. Armstrong, the 38-year-old civilian commander, radioed to earth and the mission control room here:
+
+"Houston, Tranquility Base here. The Eagle has landed."
+
+The first men to reach the moon—Mr. Armstrong and his co-pilot, Col. Edwin E. Aldrin Jr. of the Air Force—brought their ship to rest on a level, rock-strewn plain near the southwestern shore of the arid Sea of Tranquility.
+
+About six and a half hours later, Mr. Armstrong opened the landing craft's hatch, stepped slowly down the ladder and declared as he planted the first human footprint on the lunar crust:
+
+"That's one small step for man, one giant leap for mankind."
+
+His first step on the moon came at 10:56:20 P.M., as a television camera outside the craft transmitted his every move to an awed and excited audience of hundreds of millions of people on earth.
+
+Tentative Steps Test Soil

tools/mtmd/tests/test-deepseek-ocr.py

@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+"""
+Evaluates llama.cpp's DeepSeek-OCR by comparing its output for a test
+image to the actual text in part of that image.
+
+Runs each test image through mtmd-cli, calculates CER and chrF for
+its output, and holds them against the HF model's scores.
+"""
+
+import argparse
+import logging
+import re
+import subprocess
+import sys
+import unicodedata
+from dataclasses import dataclass
+from pathlib import Path
+
+logger = logging.getLogger("deepseek-ocr-test")
+
+RUN_TIMEOUT = 300
+
+
+@dataclass
+class ModelSpec:
+    key: str
+    label: str
+    model_arg: str
+    mmproj_arg: str
+    model_default: str
+    mmproj_default: str
+    prompt: str = "Free OCR."
+    n_predict: int = 512
+    n_ctx: int | None = None
+    # Unlimited-OCR's "document parsing" prompt emits <|det|> grounding markup that
+    # the HF reference strips in result.md; drop it before scoring to match.
+    strip_grounding: bool = False
+    # v2/Unlimited loop on hard tiles; DRY caps it the way HF's
+    # no_repeat_ngram_size does. v1 scores fine without it.
+    dry: bool = False
+
+
+@dataclass
+class TestCase:
+    model_key: str
+    label: str
+    image: str
+    ground_truth: str
+    hf_cer: float
+    hf_chrf: float
+    cer_tol: float
+    chrf_tol: float
+
+    @property
+    def cer_max(self) -> float:
+        return self.hf_cer + self.cer_tol
+
+    @property
+    def chrf_min(self) -> float:
+        return self.hf_chrf - self.chrf_tol
+
+
+MODELS = {
+    "v1": ModelSpec(
+        key="v1", label="DeepSeek-OCR",
+        model_arg="--llama-model", mmproj_arg="--mmproj",
+        model_default="gguf_models/deepseek-ai/deepseek-ocr-bf16.gguf",
+        mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-bf16.gguf",
+    ),
+    "v2": ModelSpec(
+        key="v2", label="DeepSeek-OCR-2",
+        model_arg="--llama-model-2", mmproj_arg="--mmproj-2",
+        model_default="gguf_models/deepseek-ai/deepseek-ocr-2-bf16.gguf",
+        mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-2-bf16.gguf",
+        # v2 keeps generating past 512 on multi-tile; give it room to match the HF ref.
+        n_predict=2048,
+        dry=True,
+    ),
+    "unlimited": ModelSpec(
+        key="unlimited", label="Unlimited-OCR",
+        model_arg="--llama-model-unlimited", mmproj_arg="--mmproj-unlimited",
+        model_default="gguf_models/baidu/unlimited-ocr-bf16.gguf",
+        mmproj_default="gguf_models/baidu/mmproj-unlimited-ocr-bf16.gguf",
+        # "Free OCR." immediately emits EOS on this checkpoint; the HF reference
+        # (demo/unlimited_ocr_scores.py) uses "document parsing.", which grounds.
+        prompt="document parsing.",
+        # Grounding emits ~3x the tokens of plain OCR, so it needs a larger budget
+        # and context to reach the article body the ground truth covers.
+        n_predict=4096,
+        n_ctx=16384,
+        strip_grounding=True,
+        dry=True,
+    ),
+}
+
+CASES = [
+    TestCase(
+        model_key="v1", label="single-view scan",
+        image="tools/mtmd/test-1.jpeg",
+        ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+        # Fragile image: the HF ref itself swings ~0.286-0.314 across precision
+        # configs -- hence the wide tol. llama.cpp bf16 ~0.322/63.8.
+        hf_cer=0.3140, hf_chrf=67.57, cer_tol=0.04, chrf_tol=5.0,
+    ),
+    TestCase(
+        model_key="v2", label="single-view scan",
+        image="tools/mtmd/test-1.jpeg",
+        ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+        # 640x488 is below the 768 tiling threshold -- single 1024 global view.
+        # hf_cer/hf_chrf are the deepseek-ai repo's own scores (ImageOps.pad);
+        # the transformers HF processor is *not* the reference -- its pad_to_square
+        # is one pixel off and lands at ~0.69 instead.
+        hf_cer=0.7761, hf_chrf=28.70, cer_tol=0.12, chrf_tol=8.0,
+    ),
+    TestCase(
+        model_key="v1", label="multi-tile (dynamic resolution)",
+        image="tools/mtmd/tests/test-1-positive.png",
+        ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+        # 429x806 -- 806 > 640 triggers the v1 "Gundam" path: (1,2) grid ->
+        # 2 local 640 tiles + 1 global 1024 view. Regression guard for the
+        # tiling preprocessor -- a broken tile path craters the score.
+        # hf_cer/hf_chrf are HF v1's measured scores -- it reads this clean crop exactly.
+        hf_cer=0.0000, hf_chrf=100.00, cer_tol=0.03, chrf_tol=3.0,
+    ),
+    TestCase(
+        model_key="v2", label="multi-tile (dynamic resolution)",
+        image="tools/mtmd/tests/test-1-positive.png",
+        ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+        # 429x806 -- 806 > 768 triggers the v2 path: (1,2) grid ->
+        # 2 local 768 tiles + 1 global 1024 view = 545 image tokens.
+        hf_cer=0.0236, hf_chrf=97.05, cer_tol=0.03, chrf_tol=3.0,
+    ),
+    TestCase(
+        model_key="unlimited", label="single-view scan",
+        image="tools/mtmd/test-1.jpeg",
+        ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
+        # HF reference: Unlimited-OCR scoring (gundam, bf16) on this image/ground-truth.
+        # Decoder runs full MHA, not R-SWA; the band absorbs that gap + bf16 variance.
+        hf_cer=0.1869, hf_chrf=75.23, cer_tol=0.06, chrf_tol=6.0,
+    ),
+]
+
+
+GROUNDING_TAG_RE = re.compile(r"<\|(ref|det)\|>.*?<\|/\1\|>", re.DOTALL)
+
+
+def strip_grounding(text: str) -> str:
+    """Drop <|ref|>..<|/ref|> / <|det|>..<|/det|> grounding markup, matching the
+    cleaned result.md the HF reference scores against."""
+    return GROUNDING_TAG_RE.sub("", text)
+
+
+def arg_dest(flag: str) -> str:
+    return flag.lstrip("-").replace("-", "_")
+
+
+def verdict(ok: bool) -> str:
+    return "PASS" if ok else "FAIL"
+
+
+def normalize_text(text: str) -> str:
+    """NFC-normalize and collapse whitespace, so line-wrap and spacing
+    don't count as CER errors."""
+    return " ".join(unicodedata.normalize("NFC", text).split())
+
+
+def locally_align(expected: str, ocr_out: str) -> str:
+    """Return the span of `ocr_out` that best matches `expected`.
+
+    The ground truth covers part of the article body.
+    But the test image includes half of the newspaper's front page.
+    Fuzzy partial-ratio matching picks out
+    the body so the unrelated text doesn't disturb CER / chrF.
+    """
+    from rapidfuzz import fuzz
+    alignment = fuzz.partial_ratio_alignment(expected, ocr_out)
+    if alignment is None or alignment.dest_end <= alignment.dest_start:
+        return ocr_out
+    return ocr_out[alignment.dest_start:alignment.dest_end]
+
+
+def compute_cer(expected: str, ocr_out: str) -> float:
+    """Character Error Rate. Lower is better.
+    CER: fraction of characters you'd insert/delete/substitute to fix the output; 0 = perfect."""
+    import jiwer
+    return jiwer.cer(expected, ocr_out)
+
+
+def compute_chrf(expected: str, ocr_out: str) -> float:
+    """chrF score on 0-100. Higher is better.
+    chrF: F-score over shared character n-grams; more forgiving of small word/spacing drift than CER.
+    """
+    from sacrebleu.metrics import CHRF
+    return CHRF().sentence_score(ocr_out, [expected]).score
+
+
+def run_mtmd_cli(spec: "ModelSpec", model_path, mmproj_path, image_path, bin_path) -> str:
+    """Run mtmd-cli on the image and return its output."""
+    cmd = [
+        str(bin_path),
+        "-m", str(model_path),
+        "--mmproj", str(mmproj_path),
+        "--image", str(image_path),
+        "-p", spec.prompt,
+        "--chat-template", "deepseek-ocr",
+        "--temp", "0",
+        "--flash-attn", "off",  # match the HF "eager" attention reference
+        "--no-warmup",
+        "-n", str(spec.n_predict),  # cap loops on hard images (KV would otherwise fill)
+    ]
+    if spec.dry:
+        # HF decodes with no_repeat_ngram_size; llama.cpp's analog is DRY.
+        # Default DRY breakers include "\n", so they are cleared below.
+        cmd += [
+            "--dry-multiplier", "0.8",
+            "--dry-base", "1.75",
+            "--dry-allowed-length", "2",
+            "--dry-penalty-last-n", "-1",
+            "--dry-sequence-breaker", "none",
+        ]
+    if spec.n_ctx is not None:
+        cmd += ["-c", str(spec.n_ctx)]
+    logger.debug(f"  command: {' '.join(cmd)}")
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=False, timeout=RUN_TIMEOUT)
+    except subprocess.TimeoutExpired as e:
+        if e.stderr:
+            logger.error("llama.cpp stderr:\n%s", e.stderr.decode("utf-8", errors="replace"))
+        raise RuntimeError(f"llama-mtmd-cli timed out after {RUN_TIMEOUT}s")
+
+    if result.returncode != 0:
+        logger.error("llama.cpp stderr:\n%s", result.stderr.decode("utf-8", errors="replace"))
+        raise RuntimeError(f"llama-mtmd-cli failed with code {result.returncode}")
+
+    output = result.stdout.decode("utf-8", errors="replace").strip()
+    if spec.strip_grounding:
+        output = strip_grounding(output)
+    if not output:
+        raise RuntimeError("llama-mtmd-cli produced no output on stdout")
+    logger.info(f"  output: {len(output)} chars")
+    return output
+
+
+def read_expected_text(file_path: Path) -> str:
+    with open(file_path, "r", encoding="utf-8") as f:
+        return f.read().strip()
+
+
+def evaluate(case: "TestCase", expected: str, ocr_out: str) -> bool:
+    expected = normalize_text(expected)
+    ocr_out = normalize_text(ocr_out)
+    aligned = locally_align(expected, ocr_out)
+
+    logger.debug(f"\n--- expected (normalized) ---\n{expected}")
+    logger.debug(f"\n--- OCR output (normalized) ---\n{ocr_out}")
+    logger.debug(f"\n--- aligned span ---\n{aligned}")
+
+    cer = compute_cer(expected, aligned)
+    chrf = compute_chrf(expected, aligned)
+
+    cer_pass = cer <= case.cer_max
+    chrf_pass = chrf >= case.chrf_min
+    passed = cer_pass and chrf_pass
+
+    logger.info("")
+    logger.info("=" * 60)
+    logger.info("OCR evaluation:")
+    logger.info("=" * 60)
+    logger.info(f"  CER               {cer:>7.4f}    (HF {case.hf_cer:.4f}, <= {case.cer_max:>7.4f}  -> {verdict(cer_pass)})")
+    logger.info(f"  chrF (0-100)      {chrf:>7.2f}    (HF {case.hf_chrf:.2f}, >= {case.chrf_min:>7.2f}  -> {verdict(chrf_pass)})")
+    logger.info(f"  Expected chars    {len(expected):>7}")
+    logger.info(f"  Aligned chars     {len(aligned):>7} (of {len(ocr_out)} OCR chars)")
+    logger.info("")
+    logger.info(f"  Result: {verdict(passed)}")
+    logger.info("=" * 60)
+    return passed
+
+
+def argument_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(description="Compare llama.cpp DeepSeek-OCR output with a ground-truth transcript")
+    ap.add_argument("--llama-bin", default="build/bin/llama-mtmd-cli",
+                    help="Path to llama-mtmd-cli binary (relative to repo root or absolute)")
+    for spec in MODELS.values():
+        ap.add_argument(spec.model_arg, default=spec.model_default,
+                        help=f"Path to the {spec.label} GGUF model (relative to repo root or absolute)")
+        ap.add_argument(spec.mmproj_arg, default=spec.mmproj_default,
+                        help=f"Path to the {spec.label} mmproj GGUF file (relative to repo root or absolute)")
+    ap.add_argument("--verbose", action="store_true",
+                    help="Also log the expected, OCR, and aligned text")
+    return ap
+
+
+def configure_logging(verbose: bool) -> None:
+    logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO,
+                        format="%(message)s")
+
+
+def resolve_path(path: str, base: Path) -> Path:
+    p = Path(path)
+    return p if p.is_absolute() else base / p
+
+
+def main() -> int:
+    args = argument_parser().parse_args()
+    configure_logging(args.verbose)
+
+    repo_root = Path(__file__).resolve().parents[3]  # tests -> mtmd -> tools -> repo root
+    binary = resolve_path(args.llama_bin, repo_root)
+
+    if not binary.exists():
+        logger.error(f"Error: binary not found: {binary}")
+        return 1
+
+    logger.info("=" * 60)
+    logger.info("DeepSeek-OCR: llama.cpp vs HF parity check")
+    logger.info("=" * 60)
+
+    results = {}
+    for case in CASES:
+        model_spec = MODELS[case.model_key]
+        title = f"{model_spec.label} -- {case.label}"
+
+        logger.info("")
+        logger.info(f"=== {title} ===")
+
+        model = resolve_path(getattr(args, arg_dest(model_spec.model_arg)), repo_root)
+        mmproj = resolve_path(getattr(args, arg_dest(model_spec.mmproj_arg)), repo_root)
+        image = resolve_path(case.image, repo_root)
+        ground_truth = resolve_path(case.ground_truth, repo_root)
+
+        missing = [(lbl, p) for lbl, p in [("model", model), ("mmproj", mmproj),
+                                           ("image", image), ("ground-truth", ground_truth)]
+                   if not p.exists()]
+        if missing:
+            for lbl, p in missing:
+                logger.error(f"  Error: {lbl} not found: {p}")
+            results[title] = False
+            continue
+
+        expected = read_expected_text(ground_truth)
+        logger.info(f"  Image: {case.image}")
+        logger.info(f"  Expected text: {len(expected)} chars")
+        logger.info(f"  Running llama.cpp prompt {model_spec.prompt!r}")
+        try:
+            ocr_out = run_mtmd_cli(model_spec, model, mmproj, image, binary)
+        except RuntimeError as e:
+            logger.error(f"  Error: {e}")
+            results[title] = False
+            continue
+
+        results[title] = evaluate(case, expected, ocr_out)
+
+    logger.info("")
+    logger.info("=== Summary ===")
+    for title, ok in results.items():
+        logger.info(f"  {title:<48} {verdict(ok)}")
+    all_passed = all(results.values())
+    logger.info(f"Overall: {verdict(all_passed)}")
+
+    return 0 if all_passed else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/mtmd/tests/tests-requirements.txt

@@ -0,0 +1,3 @@
+jiwer
+sacrebleu
+rapidfuzz