chore: import upstream snapshot with attribution

plugins/plugin-vision/native/doctr.cpp/CMakeLists.txt

@@ -0,0 +1,58 @@
+cmake_minimum_required(VERSION 3.20)
+project(doctr LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+option(DOCTR_BUILD_SHARED "Build shared library (bun:ffi)" ON)
+option(DOCTR_WITH_METAL   "Use Metal backend on macOS"     OFF)
+option(DOCTR_WITH_CUDA    "Use CUDA backend"               OFF)
+
+# ----------------------------------------------------------------------------
+# ggml dependency
+# ----------------------------------------------------------------------------
+# The runtime depends on ggml. The expected layout is:
+#   plugins/plugin-local-inference/native/llama.cpp/ggml/  (existing submodule)
+# We point at that to avoid pinning a second copy. If the path is missing the
+# build still produces libdoctr.<ext> with a fallback ABI (DOCTR_HAVE_GGML
+# undefined) so the FFI surface exists for the JS layer to call into — it will throw a
+# clear "GGUF not ready" at runtime.
+
+set(DOCTR_GGML_DIR "" CACHE PATH "Path to a ggml source tree (defaults to llama.cpp's vendored copy)")
+if(NOT DOCTR_GGML_DIR)
+    set(DOCTR_GGML_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../plugin-local-inference/native/llama.cpp/ggml")
+endif()
+
+set(_doctr_has_ggml FALSE)
+if(EXISTS "${DOCTR_GGML_DIR}/include/ggml.h")
+    set(_doctr_has_ggml TRUE)
+    add_subdirectory(${DOCTR_GGML_DIR} ggml-build EXCLUDE_FROM_ALL)
+endif()
+
+# ----------------------------------------------------------------------------
+# Library
+# ----------------------------------------------------------------------------
+set(_doctr_sources
+    src/doctr_det.cpp
+    src/doctr_rec.cpp
+)
+
+if(DOCTR_BUILD_SHARED)
+    add_library(doctr SHARED ${_doctr_sources})
+else()
+    add_library(doctr STATIC ${_doctr_sources})
+endif()
+
+target_include_directories(doctr PUBLIC include)
+
+if(_doctr_has_ggml)
+    target_compile_definitions(doctr PRIVATE DOCTR_HAVE_GGML=1)
+    target_link_libraries(doctr PRIVATE ggml)
+    if(DOCTR_WITH_METAL)
+        target_compile_definitions(doctr PRIVATE GGML_USE_METAL=1)
+    endif()
+    if(DOCTR_WITH_CUDA)
+        target_compile_definitions(doctr PRIVATE GGML_USE_CUDA=1)
+    endif()
+endif()

plugins/plugin-vision/native/doctr.cpp/README.md

@@ -0,0 +1,62 @@
+# doctr.cpp — ggml port of doCTR
+
+C++ port of [Mindee doCTR](https://github.com/mindee/doctr) built directly on
+[ggml](https://github.com/ggml-org/ggml). Two stages:
+
+- **Detection** — `db_mobilenet_v3_large` backbone + DBNet head → probability map.
+- **Recognition** — `crnn_mobilenet_v3_small` backbone + BiLSTM + CTC head → per-crop logits.
+
+The post-processing (DBNet contour → bbox, CTC greedy decode) stays in
+TypeScript — both are trivial and runtime-portable. This C++ library runs only
+the forward pass; the JS caller orchestrates det → crop → rec → decode.
+
+## Status
+
+**Phase 1 (current):** FFI surface scaffolded; weight conversion script
+authored; build glue authored. **GGUF weight files are not yet built.** The TS
+binding throws a clear error until `vision/doctr-det.gguf` and
+`vision/doctr-rec.gguf` are present on disk.
+
+## Build (when implemented)
+
+```bash
+cd plugins/plugin-vision/native/doctr.cpp
+cmake -B build -S . -DGGML_METAL=ON   # macOS arm64
+cmake --build build --config Release
+```
+
+Produces a single shared library `libdoctr.dylib` / `.so` / `.dll` consumed
+via `bun:ffi` from `plugin-vision/src/native/doctr-ffi.ts`.
+
+## Convert weights (when implemented)
+
+```bash
+python scripts/convert.py \
+  --variant db_mobilenet_v3_large \
+  --out vision/doctr-det.gguf
+
+python scripts/convert.py \
+  --variant crnn_mobilenet_v3_small \
+  --out vision/doctr-rec.gguf
+```
+
+The detection variant writes a single tensor graph + mean/std metadata.
+The recognition variant additionally writes the character vocabulary as a
+`doctr.charset` KV entry inside the GGUF file.
+
+## ABI
+
+See `include/doctr.h`. The ABI is intentionally minimal:
+
+```c
+doctr_det_ctx * doctr_det_init(const char * gguf_path);
+int  doctr_det_run(doctr_det_ctx *, const float * rgb_chw, int h, int w,
+                   float * out_prob, int * out_h, int * out_w);
+void doctr_det_free(doctr_det_ctx *);
+
+doctr_rec_ctx * doctr_rec_init(const char * gguf_path);
+int  doctr_rec_run(doctr_rec_ctx *, const float * rgb_chw, int h, int w,
+                   float * out_logits, int * out_T, int * out_C);
+const char * doctr_rec_charset(doctr_rec_ctx *);
+void doctr_rec_free(doctr_rec_ctx *);
+```

plugins/plugin-vision/native/doctr.cpp/include/doctr.h

@@ -0,0 +1,91 @@
+// doctr.h — C ABI for the ggml-backed doCTR runtime.
+//
+// Stable across detection / recognition variants. Both variants own their own
+// context; they're built around ggml's compute-graph + GGUF weight loader and
+// expose only the forward pass. Post-processing (DBNet contouring, CTC decode)
+// happens in TypeScript so that the C side stays a pure tensor pipeline.
+//
+// Threading model: each context is single-threaded. Callers wanting parallel
+// recognition over multiple crops should hold a pool of contexts.
+
+#ifndef DOCTR_H
+#define DOCTR_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct doctr_det_ctx doctr_det_ctx;
+typedef struct doctr_rec_ctx doctr_rec_ctx;
+
+// Return codes. Anything non-zero is an error.
+#define DOCTR_OK              0
+#define DOCTR_ERR_FILE       -1
+#define DOCTR_ERR_FORMAT     -2
+#define DOCTR_ERR_OOM        -3
+#define DOCTR_ERR_SHAPE      -4
+#define DOCTR_ERR_BACKEND    -5
+
+// === Detection ===
+//
+// Loads a `db_mobilenet_v3_large`-based detection model from GGUF. Expected
+// metadata KV entries:
+//   - "doctr.det.variant"   = "db_mobilenet_v3_large"
+//   - "doctr.det.mean"      = float[3]   (per-channel RGB mean, 0..1)
+//   - "doctr.det.std"       = float[3]   (per-channel RGB std, 0..1)
+//   - "doctr.det.input_h"   = int        (typical 1024)
+//   - "doctr.det.input_w"   = int        (typical 1024)
+//
+// Returns NULL on failure; check stderr for diagnostics.
+doctr_det_ctx * doctr_det_init(const char * gguf_path);
+
+// rgb_chw: CHW float32 RGB, normalized externally to [0,1] (the C side
+// applies the model's mean/std from GGUF metadata).
+// h, w: spatial dims of rgb_chw. Must match the GGUF input_h/input_w
+// (callers letterbox/resize beforehand).
+// out_prob: caller-allocated. Size must be (h/4) * (w/4) float32.
+// out_h, out_w: filled with the actual probability-map dims (h/4, w/4).
+int doctr_det_run(doctr_det_ctx * ctx,
+                  const float * rgb_chw,
+                  int h, int w,
+                  float * out_prob,
+                  int * out_h, int * out_w);
+
+void doctr_det_free(doctr_det_ctx * ctx);
+
+// === Recognition ===
+//
+// Loads a `crnn_mobilenet_v3_small`-based recognition model. Expected GGUF
+// metadata:
+//   - "doctr.rec.variant"   = "crnn_mobilenet_v3_small"
+//   - "doctr.rec.mean"      = float[3]
+//   - "doctr.rec.std"       = float[3]
+//   - "doctr.rec.input_h"   = int       (typical 32)
+//   - "doctr.rec.input_w"   = int       (typical 128)
+//   - "doctr.rec.charset"   = utf8 string (newline-separated)
+doctr_rec_ctx * doctr_rec_init(const char * gguf_path);
+
+// rgb_chw: CHW float32 RGB crop normalized to [0,1].
+// h must equal input_h (32). w is dynamic up to model max.
+// out_logits: caller-allocated. Size must be at least T * C float32
+// where T = w/8 (typical CRNN stride) and C = charset.size()+1.
+// out_T, out_C: written by the call.
+int doctr_rec_run(doctr_rec_ctx * ctx,
+                  const float * rgb_chw,
+                  int h, int w,
+                  float * out_logits,
+                  int * out_T, int * out_C);
+
+// Returns the embedded UTF-8 charset string (newline-separated, owned by ctx).
+const char * doctr_rec_charset(doctr_rec_ctx * ctx);
+
+void doctr_rec_free(doctr_rec_ctx * ctx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // DOCTR_H

plugins/plugin-vision/native/doctr.cpp/scripts/convert.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""
+Convert Mindee doCTR PyTorch checkpoints to GGUF for the doctr.cpp runtime.
+
+Usage:
+    python scripts/convert.py --variant db_mobilenet_v3_large --out vision/doctr-det.gguf
+    python scripts/convert.py --variant crnn_mobilenet_v3_small --out vision/doctr-rec.gguf
+
+This script defines the conversion entrypoint and expected GGUF contract. The
+actual tensor-name mapping table must be completed on a build host with
+`python-doctr` and `gguf` installed, then validated end-to-end against
+src/doctr_det.cpp / src/doctr_rec.cpp.
+
+Requirements (install before running):
+    pip install python-doctr[torch] gguf numpy
+
+Tensor naming convention written to the GGUF file:
+    Detection:
+        backbone.stem.conv.weight
+        backbone.stem.bn.{weight,bias,running_mean,running_var}
+        backbone.blocks.<i>.conv1.weight
+        backbone.blocks.<i>.bn1.{weight,bias,running_mean,running_var}
+        ... (per inverted-residual block; see torchvision MobileNetV3 mapping)
+        head.conv1.weight
+        head.bn1.weight
+        head.up1.weight
+        head.up2.weight
+        head.out.weight
+    Recognition:
+        backbone.* (mobilenetv3-small mapping)
+        lstm.weight_ih_l{0,1}
+        lstm.weight_hh_l{0,1}
+        lstm.bias_ih_l{0,1}
+        lstm.bias_hh_l{0,1}
+        head.weight
+        head.bias
+
+Metadata KV entries:
+    "doctr.det.variant" | "doctr.rec.variant" : str
+    "doctr.<stage>.mean"     : f32[3]
+    "doctr.<stage>.std"      : f32[3]
+    "doctr.<stage>.input_h"  : i32
+    "doctr.<stage>.input_w"  : i32
+    "doctr.rec.charset"      : str (utf-8, newline-separated)
+"""
+
+import argparse
+import sys
+
+VALID_VARIANTS = (
+    "db_mobilenet_v3_large",
+    "crnn_mobilenet_v3_small",
+)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--variant", required=True, choices=VALID_VARIANTS)
+    parser.add_argument("--out", required=True, help="Output GGUF path")
+    parser.add_argument(
+        "--quantize",
+        default="f16",
+        choices=["f32", "f16", "q4_0", "q8_0"],
+        help="Tensor quantization for conv/linear weights",
+    )
+    args = parser.parse_args()
+
+    try:
+        from doctr.models import recognition, detection  # noqa: F401
+    except ImportError:
+        print(
+            "python-doctr not installed. Install with: pip install 'python-doctr[torch]'",
+            file=sys.stderr,
+        )
+        return 2
+
+    try:
+        import gguf  # noqa: F401
+    except ImportError:
+        print(
+            "gguf library not installed. Install with: pip install gguf",
+            file=sys.stderr,
+        )
+        return 2
+
+    print(f"[convert] variant={args.variant} out={args.out} quantize={args.quantize}",
+          file=sys.stderr)
+    print(
+        "[convert] WEIGHT MAPPING UNAVAILABLE — run this on a build host with "
+        "the full python-doctr environment and fill in the per-tensor mapping "
+        "table per the docstring above.",
+        file=sys.stderr,
+    )
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

plugins/plugin-vision/native/doctr.cpp/src/doctr_det.cpp

@@ -0,0 +1,112 @@
+// doctr_det.cpp — detection forward pass (db_mobilenet_v3_large + DBNet head).
+//
+// This file pins the detection API and high-level graph structure. Builds that
+// do not link a complete ggml dependency tree refuse initialization through the
+// explicit nullptr path below. The matching CMakeLists.txt vendors `ggml` as a
+// git submodule; keeping this file compilable-in-isolation lets the plugin
+// build while the native model runtime remains unavailable.
+
+#include "doctr.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#if defined(DOCTR_HAVE_GGML)
+#  include "ggml.h"
+#  include "ggml-backend.h"
+#endif
+
+struct doctr_det_ctx {
+    std::string gguf_path;
+
+    // Hyperparameters from GGUF metadata.
+    int   input_h     = 1024;
+    int   input_w     = 1024;
+    float mean[3]     = {0.798f, 0.785f, 0.772f};   // doCTR defaults
+    float std_[3]     = {0.264f, 0.275f, 0.286f};
+
+#if defined(DOCTR_HAVE_GGML)
+    struct ggml_context  * gctx   = nullptr;
+    ggml_backend_t         backend = nullptr;
+    // The compute graph + parameter tensors get built lazily on the first
+    // run, so the model file can be opened and validated without paying for
+    // the graph allocation cost.
+    struct ggml_cgraph   * graph  = nullptr;
+#endif
+};
+
+extern "C" doctr_det_ctx * doctr_det_init(const char * gguf_path) {
+    if (!gguf_path) return nullptr;
+
+    auto * ctx = new (std::nothrow) doctr_det_ctx();
+    if (!ctx) return nullptr;
+    ctx->gguf_path = gguf_path;
+
+#if defined(DOCTR_HAVE_GGML)
+    // 1. Open GGUF, validate `doctr.det.variant == db_mobilenet_v3_large`.
+    // 2. Read mean/std/input_{h,w} metadata into ctx.
+    // 3. Pick backend (Metal on darwin, CUDA when available, CPU else).
+    // 4. Load conv/bn/linear parameter tensors into ctx->gctx.
+    //
+    // The runtime loader is intentionally unavailable until it mirrors the
+    // tensor naming implemented by the conversion harness in `scripts/convert.py`.
+    // See README.md for the conversion pipeline.
+    fprintf(stderr,
+            "[doctr_det] init called for %s — GGML path not yet wired; weights must be built first.\n",
+            gguf_path);
+    delete ctx;
+    return nullptr;
+#else
+    // Build without ggml linked: refuse to initialize at all so the caller's
+    // JS layer can throw a clear "GGUF not ready" error.
+    fprintf(stderr,
+            "[doctr_det] built without DOCTR_HAVE_GGML — weights cannot load.\n");
+    delete ctx;
+    return nullptr;
+#endif
+}
+
+extern "C" int doctr_det_run(doctr_det_ctx * ctx,
+                             const float * rgb_chw,
+                             int h, int w,
+                             float * out_prob,
+                             int * out_h, int * out_w) {
+    if (!ctx || !rgb_chw || !out_prob || !out_h || !out_w) {
+        return DOCTR_ERR_SHAPE;
+    }
+    if (h != ctx->input_h || w != ctx->input_w) {
+        return DOCTR_ERR_SHAPE;
+    }
+#if defined(DOCTR_HAVE_GGML)
+    // Forward pass:
+    //   1. apply mean/std normalization in-place to a scratch tensor
+    //   2. run db_mobilenet_v3_large backbone (12 inverted-residual blocks,
+    //      hidden-state dims [16,24,40,80,112,160])
+    //   3. FPN-like neck producing a single (B, 256, H/4, W/4) feature map
+    //   4. DBNet head: 3x3 conv → conv-transpose ×2 → 1x1 conv → sigmoid
+    //
+    // Output: (1, 1, H/4, W/4) probability map copied into out_prob.
+    *out_h = h / 4;
+    *out_w = w / 4;
+    std::memset(out_prob, 0, sizeof(float) * (*out_h) * (*out_w));
+    return DOCTR_ERR_BACKEND;
+#else
+    (void)out_prob;
+    *out_h = 0;
+    *out_w = 0;
+    return DOCTR_ERR_BACKEND;
+#endif
+}
+
+extern "C" void doctr_det_free(doctr_det_ctx * ctx) {
+    if (!ctx) return;
+#if defined(DOCTR_HAVE_GGML)
+    if (ctx->graph)   { /* ggml_graph_free handled by gctx */ }
+    if (ctx->gctx)    { ggml_free(ctx->gctx); }
+    if (ctx->backend) { ggml_backend_free(ctx->backend); }
+#endif
+    delete ctx;
+}

plugins/plugin-vision/native/doctr.cpp/src/doctr_rec.cpp

@@ -0,0 +1,103 @@
+// doctr_rec.cpp — recognition forward pass (crnn_mobilenet_v3_small + BiLSTM).
+//
+// Same scaffolding rationale as doctr_det.cpp: API + graph structure are
+// pinned; the ggml-backed loader/forward is gated behind DOCTR_HAVE_GGML and
+// will be wired once the GGUF weight files exist.
+
+#include "doctr.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#if defined(DOCTR_HAVE_GGML)
+#  include "ggml.h"
+#  include "ggml-backend.h"
+#endif
+
+struct doctr_rec_ctx {
+    std::string gguf_path;
+    std::string charset;        // utf-8, newline separated
+    int charset_size = 0;       // number of glyph entries (charset_size+1 logits)
+
+    int input_h = 32;
+    int input_w = 128;          // dynamic; this is the maximum
+    float mean[3] = {0.694f, 0.695f, 0.693f};
+    float std_[3] = {0.299f, 0.296f, 0.301f};
+
+#if defined(DOCTR_HAVE_GGML)
+    struct ggml_context  * gctx    = nullptr;
+    ggml_backend_t         backend = nullptr;
+    struct ggml_cgraph   * graph   = nullptr;
+#endif
+};
+
+extern "C" doctr_rec_ctx * doctr_rec_init(const char * gguf_path) {
+    if (!gguf_path) return nullptr;
+    auto * ctx = new (std::nothrow) doctr_rec_ctx();
+    if (!ctx) return nullptr;
+    ctx->gguf_path = gguf_path;
+
+#if defined(DOCTR_HAVE_GGML)
+    // 1. Open GGUF, validate `doctr.rec.variant == crnn_mobilenet_v3_small`.
+    // 2. Read mean/std/input_h/input_w and charset KV.
+    // 3. Pick backend.
+    // 4. Load conv/bn/linear + LSTM gate weights.
+    fprintf(stderr,
+            "[doctr_rec] init called for %s — GGML path not yet wired; weights must be built first.\n",
+            gguf_path);
+    delete ctx;
+    return nullptr;
+#else
+    fprintf(stderr,
+            "[doctr_rec] built without DOCTR_HAVE_GGML — weights cannot load.\n");
+    delete ctx;
+    return nullptr;
+#endif
+}
+
+extern "C" int doctr_rec_run(doctr_rec_ctx * ctx,
+                             const float * rgb_chw,
+                             int h, int w,
+                             float * out_logits,
+                             int * out_T, int * out_C) {
+    if (!ctx || !rgb_chw || !out_logits || !out_T || !out_C) {
+        return DOCTR_ERR_SHAPE;
+    }
+    if (h != ctx->input_h) return DOCTR_ERR_SHAPE;
+    if (w <= 0 || w > ctx->input_w) return DOCTR_ERR_SHAPE;
+
+#if defined(DOCTR_HAVE_GGML)
+    // Forward pass:
+    //   1. normalize input with mean/std
+    //   2. mobilenetv3-small backbone — outputs (1, 256, 1, w/8)
+    //   3. squeeze height -> (1, 256, w/8)
+    //   4. BiLSTM 128 hidden × 2 layers
+    //   5. linear projection to (charset_size + 1)
+    //
+    // Output: (T, C) row-major float32 logits.
+    *out_T = w / 8;
+    *out_C = ctx->charset_size + 1;
+    std::memset(out_logits, 0, sizeof(float) * (*out_T) * (*out_C));
+    return DOCTR_ERR_BACKEND;
+#else
+    *out_T = 0;
+    *out_C = 0;
+    return DOCTR_ERR_BACKEND;
+#endif
+}
+
+extern "C" const char * doctr_rec_charset(doctr_rec_ctx * ctx) {
+    return ctx ? ctx->charset.c_str() : nullptr;
+}
+
+extern "C" void doctr_rec_free(doctr_rec_ctx * ctx) {
+    if (!ctx) return;
+#if defined(DOCTR_HAVE_GGML)
+    if (ctx->gctx)    { ggml_free(ctx->gctx); }
+    if (ctx->backend) { ggml_backend_free(ctx->backend); }
+#endif
+    delete ctx;
+}