elizaos--eliza/plugins/plugin-vision/native/macos-vision-ocr.swift

// macOS Apple Vision OCR helper for @elizaos/plugin-vision (issue #9105 — per-OS
// native OCR fallback). Reads PNG/JPEG bytes from stdin, runs an accurate
// `VNRecognizeTextRequest` with language correction, and prints ONE JSON object
// on stdout:
//
//   {"lines":[{"text":..,"confidence":..,"boundingBox":{"x":..,"y":..,"width":..,"height":..}}],"fullText":..}
//
// Vision reports normalized, BOTTOM-LEFT-origin bounding boxes (x,y,width,height
// in 0..1, y growing upward). The other plugin-vision OCR providers use
// TOP-LEFT-origin PIXEL coordinates (display-absolute convention), so we convert
// here: pixelX = x*W, pixelY = (1 - y - height)*H, and scale width/height by the
// image dimensions. Empty/zero results still print a well-formed empty object so
// the Node side never has to special-case a missing stdout.

import AppKit
import Foundation
import Vision

struct OcrLine: Encodable {
    let text: String
    let confidence: Double
    let boundingBox: BBox
}

struct BBox: Encodable {
    let x: Double
    let y: Double
    let width: Double
    let height: Double
}

struct OcrPayload: Encodable {
    let lines: [OcrLine]
    let fullText: String
}

func emit(_ payload: OcrPayload) {
    let encoder = JSONEncoder()
    encoder.outputFormatting = [.withoutEscapingSlashes]
    if let data = try? encoder.encode(payload),
        let json = String(data: data, encoding: .utf8)
    {
        print(json)
    } else {
        print("{\"lines\":[],\"fullText\":\"\"}")
    }
}

let emptyPayload = OcrPayload(lines: [], fullText: "")

// Read the full image from stdin.
let inputData = FileHandle.standardInput.readDataToEndOfFile()
guard !inputData.isEmpty, let nsImage = NSImage(data: inputData),
    let cgImage = nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil)
else {
    emit(emptyPayload)
    exit(0)
}

let pixelWidth = Double(cgImage.width)
let pixelHeight = Double(cgImage.height)
if pixelWidth <= 0 || pixelHeight <= 0 {
    emit(emptyPayload)
    exit(0)
}

let request = VNRecognizeTextRequest()
request.recognitionLevel = .accurate
request.usesLanguageCorrection = true

let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
do {
    try handler.perform([request])
} catch {
    emit(emptyPayload)
    exit(0)
}

guard let observations = request.results, !observations.isEmpty else {
    emit(emptyPayload)
    exit(0)
}

var lines: [OcrLine] = []
for observation in observations {
    guard let candidate = observation.topCandidates(1).first else { continue }
    let text = candidate.string
    if text.isEmpty { continue }

    // `observation.boundingBox` is normalized, bottom-left origin. Convert to
    // top-left-origin pixel coordinates matching the other OCR providers.
    let box = observation.boundingBox
    let pixelX = box.origin.x * pixelWidth
    let pixelY = (1.0 - box.origin.y - box.size.height) * pixelHeight
    let pixelW = box.size.width * pixelWidth
    let pixelH = box.size.height * pixelHeight

    lines.append(
        OcrLine(
            text: text,
            confidence: Double(candidate.confidence),
            boundingBox: BBox(
                x: pixelX,
                y: pixelY,
                width: pixelW,
                height: pixelH
            )
        )
    )
}

let fullText = lines.map { $0.text }.joined(separator: "\n")
emit(OcrPayload(lines: lines, fullText: fullText))