chore: import upstream snapshot with attribution

apps/pii/engines.py

@@ -0,0 +1,267 @@
+"""Analyzer engine builders for the PII service.
+
+Two NER engines share one recognizer surface:
+
+- spacy (default): the 5 large spaCy models do NER (PERSON/LOCATION/NRP/
+  DATE_TIME) and tokenization.
+- gliner (opt-in): one multilingual GLiNER model does NER on CPU or GPU;
+  small spaCy models remain only for tokenization + lemmas.
+
+Both engines register the identical regex/checksum recognizer set (Presidio
+defaults, EXTRA_RECOGNIZERS, VIN) — only the source of the 4 NER entity types
+differs. Side-effect free: importing this module loads no models.
+"""
+
+import importlib.util
+
+import spacy.util
+from presidio_analyzer import AnalyzerEngine, Pattern, PatternRecognizer
+from presidio_analyzer.nlp_engine import NlpEngineProvider
+from presidio_analyzer.predefined_recognizers import (
+    AuAbnRecognizer,
+    AuAcnRecognizer,
+    AuMedicareRecognizer,
+    AuTfnRecognizer,
+    EsNieRecognizer,
+    EsNifRecognizer,
+    FiPersonalIdentityCodeRecognizer,
+    GLiNERRecognizer,
+    InAadhaarRecognizer,
+    InPanRecognizer,
+    InPassportRecognizer,
+    InVehicleRegistrationRecognizer,
+    InVoterRecognizer,
+    ItDriverLicenseRecognizer,
+    ItFiscalCodeRecognizer,
+    ItIdentityCardRecognizer,
+    ItPassportRecognizer,
+    ItVatCodeRecognizer,
+    PlPeselRecognizer,
+    SgFinRecognizer,
+    SgUenRecognizer,
+    UkNinoRecognizer,
+)
+
+# Languages served. Each needs its spaCy model installed in the image; the
+# es/it/pl/fi predefined recognizers (ES_NIF, IT_FISCAL_CODE, PL_PESEL, ...)
+# auto-load once their NLP engine is present.
+NLP_CONFIGURATION = {
+    "nlp_engine_name": "spacy",
+    "models": [
+        {"lang_code": "en", "model_name": "en_core_web_lg"},
+        {"lang_code": "es", "model_name": "es_core_news_lg"},
+        {"lang_code": "it", "model_name": "it_core_news_lg"},
+        {"lang_code": "pl", "model_name": "pl_core_news_lg"},
+        {"lang_code": "fi", "model_name": "fi_core_news_lg"},
+    ],
+}
+SUPPORTED_LANGUAGES = [m["lang_code"] for m in NLP_CONFIGURATION["models"]]
+
+# The gliner engine still needs a spaCy pipeline per language: the regex
+# recognizers consume NlpArtifacts and the LemmaContextAwareEnhancer boosts
+# scores from surrounding lemmas. The small models (~12-40MB each vs ~400MB
+# large) keep tokenization + lemmas intact while GLiNER owns NER. Blank
+# pipelines ("blank:xx") are not an option: Presidio's SpacyNlpEngine treats
+# unknown model names as pip packages and tries to download them.
+# labels_to_ignore strips the small models' NER output from NlpArtifacts —
+# correctness comes from removing SpacyRecognizer in build_gliner_analyzer;
+# this only silences unmapped-label noise.
+GLINER_NLP_CONFIGURATION = {
+    "nlp_engine_name": "spacy",
+    "models": [
+        {"lang_code": "en", "model_name": "en_core_web_sm"},
+        {"lang_code": "es", "model_name": "es_core_news_sm"},
+        {"lang_code": "it", "model_name": "it_core_news_sm"},
+        {"lang_code": "pl", "model_name": "pl_core_news_sm"},
+        {"lang_code": "fi", "model_name": "fi_core_news_sm"},
+    ],
+    "ner_model_configuration": {
+        "labels_to_ignore": [
+            "CARDINAL", "DATE", "EVENT", "FAC", "GPE", "LANGUAGE", "LAW",
+            "LOC", "MISC", "MONEY", "NORP", "ORDINAL", "ORG", "PER",
+            "PERCENT", "PERSON", "PRODUCT", "QUANTITY", "TIME", "WORK_OF_ART",
+        ],
+    },
+}
+
+# Zero-shot label prompts -> the 4 Presidio NER entities GLiNER owns. Multiple
+# prompts per entity trade a little inference cost for recall; tune against
+# scripts/bench_engines.py output.
+GLINER_ENTITY_MAPPING = {
+    "person": "PERSON",
+    "name": "PERSON",
+    "location": "LOCATION",
+    "address": "LOCATION",
+    "date": "DATE_TIME",
+    "time": "DATE_TIME",
+    "nationality": "NRP",
+    "religious group": "NRP",
+    "political group": "NRP",
+    "ethnic group": "NRP",
+}
+
+# Predefined recognizers Presidio ships but does NOT load into the default
+# registry — they must be added explicitly. Each carries its own
+# supported_language, so it fires under that language once its NLP model is
+# loaded. en: UK/AU/IN/SG locale ids; es/it/pl/fi: national ids.
+EXTRA_RECOGNIZERS = [
+    UkNinoRecognizer,
+    AuAbnRecognizer,
+    AuAcnRecognizer,
+    AuTfnRecognizer,
+    AuMedicareRecognizer,
+    InPanRecognizer,
+    InAadhaarRecognizer,
+    InVehicleRegistrationRecognizer,
+    InVoterRecognizer,
+    InPassportRecognizer,
+    SgFinRecognizer,
+    SgUenRecognizer,
+    EsNifRecognizer,
+    EsNieRecognizer,
+    ItFiscalCodeRecognizer,
+    ItDriverLicenseRecognizer,
+    ItVatCodeRecognizer,
+    ItPassportRecognizer,
+    ItIdentityCardRecognizer,
+    PlPeselRecognizer,
+    FiPersonalIdentityCodeRecognizer,
+]
+
+
+class VinRecognizer(PatternRecognizer):
+    """VIN (17 chars, A-Z/0-9 excluding I/O/Q) with ISO 3779 check-digit
+    validation (position 9). Validation makes accidental matches on arbitrary
+    17-char codes (request ids, SKUs, tokens) extremely unlikely. Some
+    non-North-American VINs omit the check digit and are skipped — an
+    intentional bias toward precision.
+    """
+
+    _TRANSLIT = {
+        **{str(d): d for d in range(10)},
+        "A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "H": 8,
+        "J": 1, "K": 2, "L": 3, "M": 4, "N": 5, "P": 7, "R": 9,
+        "S": 2, "T": 3, "U": 4, "V": 5, "W": 6, "X": 7, "Y": 8, "Z": 9,
+    }
+    _WEIGHTS = [8, 7, 6, 5, 4, 3, 2, 10, 0, 9, 8, 7, 6, 5, 4, 3, 2]
+
+    def validate_result(self, pattern_text: str):
+        vin = pattern_text.upper()
+        if len(vin) != 17:
+            return False
+        try:
+            total = sum(self._TRANSLIT[c] * w for c, w in zip(vin, self._WEIGHTS))
+        except KeyError:
+            return False
+        check = total % 11
+        expected = "X" if check == 10 else str(check)
+        return vin[8] == expected
+
+
+class SharedModelGLiNERRecognizer(GLiNERRecognizer):
+    """Per-language GLiNER recognizer sharing ONE loaded model.
+
+    Presidio routes recognizers by supported_language, so the registry holds
+    one instance per served language — but each instance's load() would pull
+    its own ~1.2GB model copy. The first instance loads (an ImportError from
+    a missing gliner package propagates — fail fast in the lean image); the
+    rest reuse the cached model.
+    """
+
+    _shared_models: dict = {}
+
+    def load(self) -> None:
+        key = (self.model_name, self.map_location)
+        cached = self._shared_models.get(key)
+        if cached is None:
+            super().load()
+            self._shared_models[key] = self.gliner
+        else:
+            self.gliner = cached
+
+    def analyze(self, text, entities, nlp_artifacts=None):
+        """GLiNERRecognizer appends any requested entity it doesn't know as an
+        ad-hoc zero-shot label and returns its hits. The analyzer passes ALL
+        supported entities (~40) when a request doesn't narrow them, which
+        would prompt GLiNER for CREDIT_CARD/VIN/ES_NIF/... — wrong scope, and
+        inference cost scales with label count. Restrict to the NER entities
+        this recognizer owns."""
+        requested = [e for e in (entities or self.supported_entities) if e in self.supported_entities]
+        if not requested:
+            return []
+        return super().analyze(text, requested, nlp_artifacts)
+
+
+def _register_common_recognizers(analyzer: AnalyzerEngine) -> None:
+    """Regex/checksum recognizers shared by both engines."""
+    # VIN is language-agnostic, so register it under every served language —
+    # a recognizer only fires for the language the caller routes to.
+    vin_pattern = Pattern(name="vin", regex=r"\b[A-HJ-NPR-Z0-9]{17}\b", score=0.7)
+    for language in SUPPORTED_LANGUAGES:
+        analyzer.registry.add_recognizer(
+            VinRecognizer(
+                supported_entity="VIN",
+                patterns=[vin_pattern],
+                context=["vin", "vehicle", "chassis"],
+                supported_language=language,
+            )
+        )
+    for recognizer_cls in EXTRA_RECOGNIZERS:
+        analyzer.registry.add_recognizer(recognizer_cls())
+
+
+def build_spacy_analyzer() -> AnalyzerEngine:
+    nlp_engine = NlpEngineProvider(nlp_configuration=NLP_CONFIGURATION).create_engine()
+    analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES)
+    _register_common_recognizers(analyzer)
+    return analyzer
+
+
+def build_gliner_analyzer(model_name: str, device: str | None) -> AnalyzerEngine:
+    """GLiNER engine: one multilingual zero-shot model replaces spaCy NER for
+    PERSON/LOCATION/NRP/DATE_TIME; everything else is unchanged.
+
+    :param model_name: HuggingFace id of the GLiNER model.
+    :param device: torch device ("cpu", "cuda", "cuda:0"); None auto-detects
+        via Presidio's device_detector (cuda when available, else cpu).
+    """
+    # Fail fast with an actionable message when gliner deps are missing (e.g.
+    # a custom-built image without them). Without these checks Presidio would
+    # try to pip-download the missing spaCy models at startup (a silent
+    # network fallback that dies with an unrelated pip permission error), and
+    # the gliner ImportError would surface only later.
+    if importlib.util.find_spec("gliner") is None:
+        raise RuntimeError(
+            "PII_ENGINE=gliner but the gliner package is not installed; "
+            "use the stock pii image (docker/pii.Dockerfile ships torch + gliner)"
+        )
+    missing = [
+        m["model_name"]
+        for m in GLINER_NLP_CONFIGURATION["models"]
+        if not spacy.util.is_package(m["model_name"])
+    ]
+    if missing:
+        raise RuntimeError(
+            f"PII_ENGINE=gliner needs spaCy models {missing}; "
+            "use the stock pii image (docker/pii.Dockerfile ships them)"
+        )
+    nlp_engine = NlpEngineProvider(nlp_configuration=GLINER_NLP_CONFIGURATION).create_engine()
+    analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES)
+    # The default registry wires SpacyRecognizer per language; with GLiNER
+    # owning the NER entities it would emit duplicate/competing spans from the
+    # small models' ner pipe. remove_recognizer only logs when nothing matched,
+    # so assert the removal actually happened.
+    analyzer.registry.remove_recognizer("SpacyRecognizer")
+    if any(r.name == "SpacyRecognizer" for r in analyzer.registry.recognizers):
+        raise RuntimeError("SpacyRecognizer removal failed; Presidio registry layout changed")
+    for language in SUPPORTED_LANGUAGES:
+        analyzer.registry.add_recognizer(
+            SharedModelGLiNERRecognizer(
+                entity_mapping=GLINER_ENTITY_MAPPING,
+                model_name=model_name,
+                map_location=device,
+                supported_language=language,
+            )
+        )
+    _register_common_recognizers(analyzer)
+    return analyzer

apps/pii/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@sim/pii",
+  "version": "0.0.0",
+  "private": true,
+  "description": "PII detection + anonymization service (Microsoft Presidio, FastAPI). Python service built as a container image (docker/pii.Dockerfile); not part of the JS/turbo build."
+}

apps/pii/requirements-dev.txt

@@ -0,0 +1,5 @@
+# Test-only deps. Unit tests need requirements.txt + this file (no models);
+# integration tests additionally need the models baked into the docker images
+# (see tests/test_integration.py).
+pytest==9.0.3
+httpx==0.28.1

apps/pii/requirements-gliner.txt

@@ -0,0 +1,10 @@
+# Extras for the opt-in GLiNER engine — installed ONLY in the `gliner`
+# Dockerfile target, on top of requirements.txt. Pinned for reproducible image
+# builds; bump deliberately. presidio-analyzer 2.2.362 requires
+# gliner >=0.2.13,<1.0.0.
+#
+# torch is pinned in the Dockerfile instead: the CPU and CUDA targets install
+# the same version from different wheel indexes.
+gliner==0.2.27
+transformers==5.3.0
+huggingface_hub==1.3.0

apps/pii/requirements.txt

@@ -0,0 +1,10 @@
+# Pinned for reproducible image builds. Bump deliberately.
+presidio-analyzer==2.2.362
+presidio-anonymizer==2.2.362
+spacy==3.8.14
+fastapi==0.138.0
+uvicorn[standard]==0.49.0
+
+# The English spaCy model (en_core_web_lg, ~400MB) is fetched + pinned in the
+# Dockerfile via curl-with-retry rather than here — a direct pip wheel URL
+# truncates on flaky networks and fails wheel validation.

apps/pii/scripts/bench_engines.py

@@ -0,0 +1,206 @@
+"""Benchmark + parity harness for the spacy vs gliner NER engines.
+
+Runs the same payload through both engines and reports per-engine throughput
+(batch analyze, the production /redact_batch path) and per-text latency, plus
+an accuracy diff over the 4 NER entity types (PERSON/LOCATION/NRP/DATE_TIME).
+Non-NER (regex/checksum) results must be identical between engines — both
+register the same recognizers — so any mismatch there is a wiring bug and the
+script exits non-zero.
+
+Meant to run inside the pii image (both engines ship in it):
+
+    docker run --rm <pii-image> python scripts/bench_engines.py
+    docker run --rm -v $PWD/texts.json:/data.json <pii-image> \\
+        python scripts/bench_engines.py --payload /data.json
+
+Payload format: JSON list of {"text": str, "language": str} objects.
+This doubles as the tuning harness for GLINER_ENTITY_MAPPING label prompts.
+"""
+
+import argparse
+import json
+import statistics
+import sys
+import time
+from collections import defaultdict
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import engines  # noqa: E402
+
+# Entities sourced from the NER models rather than regex/checksum patterns.
+# ORGANIZATION is emitted by the spacy engine's NER on unfiltered requests but
+# is not in the app's supported set and has no GLiNER mapping — it shows up in
+# the NER diff (spacy-only) rather than failing the regex-parity gate.
+NER_ENTITIES = {"PERSON", "LOCATION", "NRP", "DATE_TIME", "ORGANIZATION"}
+DEFAULT_PAYLOAD = Path(__file__).resolve().parent / "bench_payload.json"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--payload", type=Path, default=DEFAULT_PAYLOAD)
+    parser.add_argument("--engines", default="spacy,gliner")
+    parser.add_argument("--runs", type=int, default=3)
+    parser.add_argument("--warmup", type=int, default=1)
+    parser.add_argument("--device", default=None, help="torch device for gliner (default: auto)")
+    parser.add_argument("--gliner-model", default="urchade/gliner_multi_pii-v1")
+    parser.add_argument("--max-examples", type=int, default=10)
+    parser.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+    return parser.parse_args()
+
+
+def build(engine: str, args) -> tuple:
+    started = time.perf_counter()
+    if engine == "spacy":
+        analyzer = engines.build_spacy_analyzer()
+    elif engine == "gliner":
+        analyzer = engines.build_gliner_analyzer(model_name=args.gliner_model, device=args.device)
+    else:
+        raise ValueError(f"Unknown engine {engine!r}")
+    return analyzer, time.perf_counter() - started
+
+
+def analyze_all(analyzer, items) -> list[list]:
+    """One analyze() call per text, in payload order."""
+    return [analyzer.analyze(text=item["text"], language=item["language"]) for item in items]
+
+
+def bench(analyzer, items, runs: int, warmup: int) -> dict:
+    for _ in range(warmup):
+        analyze_all(analyzer, items)
+    run_times = []
+    latencies = []
+    for _ in range(runs):
+        run_started = time.perf_counter()
+        for item in items:
+            text_started = time.perf_counter()
+            analyzer.analyze(text=item["text"], language=item["language"])
+            latencies.append(time.perf_counter() - text_started)
+        run_times.append(time.perf_counter() - run_started)
+    total_chars = sum(len(item["text"]) for item in items)
+    avg_run = statistics.mean(run_times)
+    return {
+        "texts_per_sec": len(items) / avg_run,
+        "chars_per_sec": total_chars / avg_run,
+        "latency_p50_ms": statistics.median(latencies) * 1000,
+        "latency_p95_ms": statistics.quantiles(latencies, n=20)[18] * 1000,
+    }
+
+
+def spans(results, keep_ner: bool) -> set:
+    return {
+        (r.entity_type, r.start, r.end)
+        for r in results
+        if (r.entity_type in NER_ENTITIES) == keep_ner
+    }
+
+
+def iou(a: tuple, b: tuple) -> float:
+    inter = max(0, min(a[2], b[2]) - max(a[1], b[1]))
+    union = max(a[2], b[2]) - min(a[1], b[1])
+    return inter / union if union else 0.0
+
+
+def diff_ner(items, results_a, results_b, max_examples: int) -> dict:
+    """Per-entity-type agreement between two engines (span IoU >= 0.5)."""
+    per_type = defaultdict(lambda: {"a_total": 0, "b_total": 0, "matched": 0})
+    examples = []
+    for item, res_a, res_b in zip(items, results_a, results_b):
+        a = sorted(spans(res_a, keep_ner=True))
+        b = sorted(spans(res_b, keep_ner=True))
+        unmatched_b = set(b)
+        for span_a in a:
+            per_type[span_a[0]]["a_total"] += 1
+            match = next(
+                (s for s in unmatched_b if s[0] == span_a[0] and iou(span_a, s) >= 0.5), None
+            )
+            if match:
+                per_type[span_a[0]]["matched"] += 1
+                unmatched_b.discard(match)
+        for span_b in b:
+            per_type[span_b[0]]["b_total"] += 1
+        only_a = [s for s in a if not any(s[0] == t[0] and iou(s, t) >= 0.5 for t in b)]
+        only_b = sorted(unmatched_b)
+        if (only_a or only_b) and len(examples) < max_examples:
+            examples.append(
+                {
+                    "text": item["text"],
+                    "language": item["language"],
+                    "only_a": [f"{t}[{s}:{e}]={item['text'][s:e]!r}" for t, s, e in only_a],
+                    "only_b": [f"{t}[{s}:{e}]={item['text'][s:e]!r}" for t, s, e in only_b],
+                }
+            )
+    return {"per_type": dict(per_type), "examples": examples}
+
+
+def diff_regex(items, results_a, results_b) -> list:
+    """Non-NER results must be identical: same recognizers on both engines."""
+    mismatches = []
+    for item, res_a, res_b in zip(items, results_a, results_b):
+        a = spans(res_a, keep_ner=False)
+        b = spans(res_b, keep_ner=False)
+        if a != b:
+            mismatches.append({"text": item["text"], "only_a": sorted(a - b), "only_b": sorted(b - a)})
+    return mismatches
+
+
+def main() -> int:
+    args = parse_args()
+    items = json.loads(args.payload.read_text())
+    engine_names = [e.strip() for e in args.engines.split(",") if e.strip()]
+
+    report = {"payload": str(args.payload), "texts": len(items), "engines": {}}
+    results_by_engine = {}
+    for name in engine_names:
+        analyzer, build_secs = build(name, args)
+        stats = bench(analyzer, items, runs=args.runs, warmup=args.warmup)
+        stats["build_secs"] = build_secs
+        report["engines"][name] = stats
+        results_by_engine[name] = analyze_all(analyzer, items)
+
+    exit_code = 0
+    if set(engine_names) >= {"spacy", "gliner"}:
+        report["ner_diff"] = diff_ner(
+            items, results_by_engine["spacy"], results_by_engine["gliner"], args.max_examples
+        )
+        regex_mismatches = diff_regex(
+            items, results_by_engine["spacy"], results_by_engine["gliner"]
+        )
+        report["regex_mismatches"] = regex_mismatches
+        if regex_mismatches:
+            exit_code = 1
+
+    if args.json:
+        print(json.dumps(report, indent=2, default=str))
+        return exit_code
+
+    for name, stats in report["engines"].items():
+        print(f"\n== {name} ==")
+        print(f"  build:        {stats['build_secs']:.1f}s")
+        print(f"  throughput:   {stats['texts_per_sec']:.2f} texts/s  ({stats['chars_per_sec']:.0f} chars/s)")
+        print(f"  latency:      p50 {stats['latency_p50_ms']:.1f}ms  p95 {stats['latency_p95_ms']:.1f}ms")
+    if "ner_diff" in report:
+        print("\n== NER parity (spacy=a vs gliner=b, span IoU>=0.5) ==")
+        for entity, counts in sorted(report["ner_diff"]["per_type"].items()):
+            print(
+                f"  {entity:<10} spacy={counts['a_total']:<4} gliner={counts['b_total']:<4} "
+                f"matched={counts['matched']}"
+            )
+        for example in report["ner_diff"]["examples"]:
+            print(f"\n  [{example['language']}] {example['text']}")
+            if example["only_a"]:
+                print(f"    spacy only:  {', '.join(example['only_a'])}")
+            if example["only_b"]:
+                print(f"    gliner only: {', '.join(example['only_b'])}")
+        if report["regex_mismatches"]:
+            print("\n!! REGEX MISMATCHES (wiring bug — engines must agree on non-NER):")
+            for mismatch in report["regex_mismatches"]:
+                print(f"  {mismatch}")
+        else:
+            print("\n  regex/checksum entities: identical across engines ✓")
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

apps/pii/scripts/bench_payload.json

@@ -0,0 +1,97 @@
+[
+  { "text": "My name is John Smith and I live in Paris with my wife Marie.", "language": "en" },
+  {
+    "text": "Dr. Angela Rodriguez will see you on March 14, 2026 at 3:30 PM in the Boston clinic.",
+    "language": "en"
+  },
+  {
+    "text": "Contact Sarah O'Connor at sarah.oconnor@example.com or call (212) 555-0123.",
+    "language": "en"
+  },
+  { "text": "The package ships to 45 Queen Street, Toronto, next Tuesday.", "language": "en" },
+  {
+    "text": "Ahmed is a practicing Muslim from Egypt who moved to Berlin in 2019.",
+    "language": "en"
+  },
+  { "text": "She is a Catholic Norwegian citizen born on 12/05/1988.", "language": "en" },
+  {
+    "text": "Payment with card 4111111111111111 was declined yesterday at 10am.",
+    "language": "en"
+  },
+  {
+    "text": "The vehicle VIN 1HGCM82633A004352 was registered to James Wilson in Ohio.",
+    "language": "en"
+  },
+  {
+    "text": "His National Insurance number is AB123456C and he lives in Manchester.",
+    "language": "en"
+  },
+  {
+    "text": "Meeting rescheduled: Friday, 9 January 2026, 14:00, with Priya Natarajan from Mumbai.",
+    "language": "en"
+  },
+  { "text": "Send the invoice to accounts@acme.io before the end of Q1 2026.", "language": "en" },
+  {
+    "text": "Klaus, a German engineer, and his Buddhist colleague Mei flew from Munich to Osaka.",
+    "language": "en"
+  },
+  {
+    "text": "The Democrats and Republicans debated in Washington on election night.",
+    "language": "en"
+  },
+  {
+    "text": "No PII here: the quarterly revenue grew 14% and margins held steady.",
+    "language": "en"
+  },
+  {
+    "text": "Server request id a7f3k2m9x1q8w5z2b is not a VIN and not a person.",
+    "language": "en"
+  },
+  {
+    "text": "Me llamo María García y vivo en Madrid desde el 3 de mayo de 2020.",
+    "language": "es"
+  },
+  { "text": "Mi NIF es 12345678Z y mi correo es maria.garcia@ejemplo.es.", "language": "es" },
+  {
+    "text": "El señor Javier Morales, ciudadano mexicano, llegó a Barcelona el lunes.",
+    "language": "es"
+  },
+  { "text": "La reunión con Carmen será el 15 de junio de 2026 en Sevilla.", "language": "es" },
+  {
+    "text": "Los musulmanes y los católicos convivieron durante siglos en Córdoba.",
+    "language": "es"
+  },
+  { "text": "Mi chiamo Marco Rossi e abito a Roma vicino al Colosseo.", "language": "it" },
+  {
+    "text": "Il codice fiscale di Maria Rossi è RSSMRA85T10A562S, nata il 10 dicembre 1985.",
+    "language": "it"
+  },
+  {
+    "text": "Giulia Bianchi, cittadina italiana, si trasferì a Milano nel gennaio 2021.",
+    "language": "it"
+  },
+  {
+    "text": "L'appuntamento con il dottor Ferrari è fissato per il 20 marzo 2026 a Torino.",
+    "language": "it"
+  },
+  { "text": "Nazywam się Jan Kowalski i mieszkam w Warszawie od 2015 roku.", "language": "pl" },
+  {
+    "text": "Mój numer PESEL to 44051401359, urodziłem się 14 maja 1944 w Krakowie.",
+    "language": "pl"
+  },
+  {
+    "text": "Anna Nowak, obywatelka polska, spotka się z nami we wtorek 12 maja 2026.",
+    "language": "pl"
+  },
+  { "text": "Katolicy i protestanci wspólnie świętowali w Gdańsku.", "language": "pl" },
+  {
+    "text": "Nimeni on Matti Virtanen ja asun Helsingissä Töölön kaupunginosassa.",
+    "language": "fi"
+  },
+  {
+    "text": "Henkilötunnukseni on 131052-308T ja synnyin Tampereella lokakuussa 1952.",
+    "language": "fi"
+  },
+  { "text": "Liisa Korhonen muutti Ouluun maanantaina 5. tammikuuta 2026.", "language": "fi" },
+  { "text": "Suomalaiset ja ruotsalaiset kilpailevat jääkiekossa joka vuosi.", "language": "fi" }
+]

apps/pii/server.py

@@ -0,0 +1,291 @@
+"""Combined Presidio REST service: analyzer + anonymizer on one port.
+
+Constructs one warm AnalyzerEngine (multi-language NLP + a native check-digit
+VIN recognizer) and one AnonymizerEngine at startup, exposing stock-compatible
+endpoints so a single PRESIDIO_URL serves both.
+
+NER engine selection (see engines.py):
+- PII_ENGINE=spacy (default): the 5 large spaCy models, unchanged behavior.
+- PII_ENGINE=gliner: one multilingual GLiNER model for PERSON/LOCATION/NRP/
+  DATE_TIME. The stock image ships both engines, so this is a pure env flip.
+  PII_DEVICE picks cpu/cuda (unset = auto-detect), PII_GLINER_MODEL overrides
+  the model id. The same code runs on CPU and GPU. Each uvicorn worker
+  (PII_WORKERS) loads its own GLiNER model copy — into GPU memory when on
+  cuda — so GPU deployments generally want PII_WORKERS=1 per GPU, unlike the
+  CPU/spacy path where workers scale with vCPUs.
+"""
+
+import logging
+import os
+import time
+from typing import Any
+
+from engines import build_gliner_analyzer, build_spacy_analyzer
+from fastapi import FastAPI
+from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine, RecognizerResult
+from presidio_anonymizer import AnonymizerEngine
+from presidio_anonymizer.entities import OperatorConfig
+from pydantic import BaseModel
+
+PII_ENGINE = os.environ.get("PII_ENGINE", "spacy")
+if PII_ENGINE not in ("spacy", "gliner"):
+    raise ValueError(f"Invalid PII_ENGINE={PII_ENGINE!r}; expected 'spacy' or 'gliner'")
+# Empty/unset -> None -> auto-detect (cuda when torch sees a GPU, else cpu).
+PII_DEVICE = os.environ.get("PII_DEVICE") or None
+PII_GLINER_MODEL = os.environ.get("PII_GLINER_MODEL", "urchade/gliner_multi_pii-v1")
+
+# Propagates to uvicorn's root handler, so timing lands in the container log stream.
+logger = logging.getLogger("sim.pii")
+
+
+def build_analyzer() -> AnalyzerEngine:
+    if PII_ENGINE == "gliner":
+        return build_gliner_analyzer(model_name=PII_GLINER_MODEL, device=PII_DEVICE)
+    return build_spacy_analyzer()
+
+
+logger.info("building analyzer engine=%s device=%s", PII_ENGINE, PII_DEVICE or "auto")
+analyzer = build_analyzer()
+batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)
+anonymizer = AnonymizerEngine()
+
+app = FastAPI(title="Sim Presidio", docs_url=None, redoc_url=None)
+
+
+class AnalyzeRequest(BaseModel):
+    text: str
+    language: str = "en"
+    entities: list[str] | None = None
+    score_threshold: float | None = None
+    return_decision_process: bool = False
+
+
+class AnalyzeBatchRequest(BaseModel):
+    texts: list[str]
+    language: str = "en"
+    entities: list[str] | None = None
+    score_threshold: float | None = None
+
+
+class AnonymizeRequest(BaseModel):
+    text: str
+    analyzer_results: list[dict[str, Any]] = []
+    anonymizers: dict[str, dict[str, Any]] | None = None
+    operators: dict[str, dict[str, Any]] | None = None
+
+
+class AnonymizeBatchItem(BaseModel):
+    text: str
+    analyzer_results: list[dict[str, Any]] = []
+
+
+class AnonymizeBatchRequest(BaseModel):
+    items: list[AnonymizeBatchItem] = []
+    anonymizers: dict[str, dict[str, Any]] | None = None
+    operators: dict[str, dict[str, Any]] | None = None
+
+
+class RedactRequest(BaseModel):
+    text: str
+    language: str = "en"
+    entities: list[str] | None = None
+    score_threshold: float | None = None
+    anonymizers: dict[str, dict[str, Any]] | None = None
+    operators: dict[str, dict[str, Any]] | None = None
+
+
+class RedactBatchRequest(BaseModel):
+    texts: list[str]
+    language: str = "en"
+    entities: list[str] | None = None
+    score_threshold: float | None = None
+    anonymizers: dict[str, dict[str, Any]] | None = None
+    operators: dict[str, dict[str, Any]] | None = None
+
+
+def build_operators(
+    raw_operators: dict[str, dict[str, Any]] | None,
+) -> dict[str, OperatorConfig] | None:
+    if not raw_operators:
+        return None
+    operators: dict[str, OperatorConfig] = {}
+    for entity, raw_cfg in raw_operators.items():
+        op_cfg = dict(raw_cfg)
+        op_type = op_cfg.pop("type", "replace")
+        operators[entity] = OperatorConfig(op_type, op_cfg)
+    return operators
+
+
+def run_anonymize(
+    text: str,
+    raw_results: list[dict[str, Any]],
+    operators: dict[str, OperatorConfig] | None,
+):
+    analyzer_results = [
+        RecognizerResult(
+            entity_type=r["entity_type"],
+            start=r["start"],
+            end=r["end"],
+            score=r.get("score", 1.0),
+        )
+        for r in raw_results
+    ]
+    return anonymizer.anonymize(
+        text=text,
+        analyzer_results=analyzer_results,
+        operators=operators,
+    )
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.get("/supportedentities")
+def supported_entities(language: str = "en") -> list[str]:
+    return analyzer.get_supported_entities(language)
+
+
+@app.post("/analyze")
+def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]:
+    started = time.perf_counter()
+    results = analyzer.analyze(
+        text=req.text,
+        language=req.language,
+        entities=req.entities or None,
+        score_threshold=req.score_threshold,
+        return_decision_process=req.return_decision_process,
+    )
+    logger.info(
+        "analyze lang=%s chars=%d entities=%d duration_ms=%.1f",
+        req.language,
+        len(req.text),
+        len(results),
+        (time.perf_counter() - started) * 1000,
+    )
+    return [r.to_dict() for r in results]
+
+
+@app.post("/analyze_batch")
+def analyze_batch(req: AnalyzeBatchRequest) -> list[list[dict[str, Any]]]:
+    """Analyze many texts in one pass (spaCy nlp.pipe), returning one span list
+    per input in request order — the batched counterpart to /analyze."""
+    results = batch_analyzer.analyze_iterator(
+        texts=req.texts,
+        language=req.language,
+        entities=req.entities or None,
+        score_threshold=req.score_threshold,
+    )
+    return [[r.to_dict() for r in per_text] for per_text in results]
+
+
+@app.post("/anonymize")
+def anonymize(req: AnonymizeRequest) -> dict[str, Any]:
+    started = time.perf_counter()
+    operators = build_operators(req.anonymizers or req.operators)
+    result = run_anonymize(req.text, req.analyzer_results, operators)
+    logger.info(
+        "anonymize chars=%d spans=%d duration_ms=%.1f",
+        len(req.text),
+        len(req.analyzer_results),
+        (time.perf_counter() - started) * 1000,
+    )
+    return {
+        "text": result.text,
+        "items": [
+            {
+                "operator": item.operator,
+                "entity_type": item.entity_type,
+                "start": item.start,
+                "end": item.end,
+                "text": item.text,
+            }
+            for item in result.items
+        ],
+    }
+
+
+@app.post("/anonymize_batch")
+def anonymize_batch(req: AnonymizeBatchRequest) -> dict[str, list[str]]:
+    """Mask many texts in one pass, returning masked text per item in request
+    order — the batched counterpart to /anonymize. Anonymization is pure string
+    work (no NLP), so callers should send only items with detected spans."""
+    operators = build_operators(req.anonymizers or req.operators)
+    return {
+        "texts": [
+            run_anonymize(item.text, item.analyzer_results, operators).text
+            for item in req.items
+        ]
+    }
+
+
+@app.post("/redact")
+def redact(req: RedactRequest) -> dict[str, str]:
+    """Analyze + anonymize one text in a single round-trip (the combined
+    counterpart to /analyze followed by /anonymize). Returns masked text; a text
+    with no detected PII passes through unchanged. The analyzer results feed the
+    anonymizer directly (no dict round-trip)."""
+    started = time.perf_counter()
+    operators = build_operators(req.anonymizers or req.operators)
+    results = analyzer.analyze(
+        text=req.text,
+        language=req.language,
+        entities=req.entities or None,
+        score_threshold=req.score_threshold,
+    )
+    text = (
+        req.text
+        if not results
+        else anonymizer.anonymize(
+            text=req.text, analyzer_results=results, operators=operators
+        ).text
+    )
+    logger.info(
+        "redact lang=%s chars=%d spans=%d duration_ms=%.1f",
+        req.language,
+        len(req.text),
+        len(results),
+        (time.perf_counter() - started) * 1000,
+    )
+    return {"text": text}
+
+
+@app.post("/redact_batch")
+def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]:
+    """Analyze + anonymize many texts in a single round-trip (the combined
+    counterpart to /analyze_batch followed by /anonymize_batch). Returns masked
+    text per input in request order; texts with no detected PII pass through
+    unchanged. Analysis batches through spaCy nlp.pipe; the analyzer results feed
+    the anonymizer directly (no dict round-trip), and anonymization runs only on
+    texts that actually matched."""
+    started = time.perf_counter()
+    operators = build_operators(req.anonymizers or req.operators)
+    analyzed = list(
+        batch_analyzer.analyze_iterator(
+            texts=req.texts,
+            language=req.language,
+            entities=req.entities or None,
+            score_threshold=req.score_threshold,
+        )
+    )
+    masked: list[str] = []
+    total_spans = 0
+    for text, per_text in zip(req.texts, analyzed):
+        if not per_text:
+            masked.append(text)
+            continue
+        total_spans += len(per_text)
+        masked.append(
+            anonymizer.anonymize(
+                text=text, analyzer_results=per_text, operators=operators
+            ).text
+        )
+    logger.info(
+        "redact_batch lang=%s texts=%d spans=%d duration_ms=%.1f",
+        req.language,
+        len(req.texts),
+        total_spans,
+        (time.perf_counter() - started) * 1000,
+    )
+    return {"texts": masked}

apps/pii/tests/conftest.py

@@ -0,0 +1,5 @@
+import sys
+from pathlib import Path
+
+# server.py / engines.py live one level up (repo: apps/pii, image: /app).
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

apps/pii/tests/test_engines.py

@@ -0,0 +1,105 @@
+"""Unit tests for engines.py — no models, no downloads, no network.
+
+Run: pip install -r requirements.txt -r requirements-dev.txt && python -m pytest tests
+"""
+
+import importlib.util
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from presidio_analyzer.predefined_recognizers.ner import gliner_recognizer
+
+import engines
+
+PII_DIR = Path(__file__).resolve().parent.parent
+
+
+class FakeModel:
+    def __init__(self):
+        self.seen_labels: list[list[str]] = []
+
+    def predict_entities(self, text, labels, flat_ner=True, threshold=0.3, multi_label=False):
+        self.seen_labels.append(list(labels))
+        return [{"label": "person", "score": 0.92, "start": 0, "end": 4, "text": text[0:4]}]
+
+
+class FakeGLiNER:
+    calls = 0
+
+    @classmethod
+    def from_pretrained(cls, model_name, **kwargs):
+        cls.calls += 1
+        return FakeModel()
+
+
+@pytest.fixture
+def fake_gliner(monkeypatch):
+    monkeypatch.setattr(gliner_recognizer, "GLiNER", FakeGLiNER)
+    engines.SharedModelGLiNERRecognizer._shared_models.clear()
+    FakeGLiNER.calls = 0
+    yield FakeGLiNER
+    engines.SharedModelGLiNERRecognizer._shared_models.clear()
+
+
+def make_recognizer(language: str):
+    return engines.SharedModelGLiNERRecognizer(
+        entity_mapping=engines.GLINER_ENTITY_MAPPING,
+        model_name="fake/model",
+        map_location="cpu",
+        supported_language=language,
+    )
+
+
+def test_invalid_pii_engine_fails_import():
+    result = subprocess.run(
+        [sys.executable, "-c", "import server"],
+        cwd=PII_DIR,
+        env={**os.environ, "PII_ENGINE": "bogus"},
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode != 0
+    assert "Invalid PII_ENGINE" in result.stderr
+
+
+@pytest.mark.skipif(
+    importlib.util.find_spec("gliner") is not None,
+    reason="fail-fast path only exists when gliner is not installed",
+)
+def test_build_gliner_analyzer_fails_fast_without_gliner():
+    with pytest.raises(RuntimeError, match="gliner package is not installed"):
+        engines.build_gliner_analyzer(model_name="fake/model", device="cpu")
+
+
+def test_shared_model_loads_once_across_languages(fake_gliner):
+    first = make_recognizer("en")
+    second = make_recognizer("es")
+    assert fake_gliner.calls == 1
+    assert first.gliner is second.gliner
+
+
+def test_analyze_never_prompts_gliner_with_foreign_entities(fake_gliner):
+    recognizer = make_recognizer("en")
+    all_supported = ["PERSON", "LOCATION", "NRP", "DATE_TIME", "CREDIT_CARD", "VIN", "ES_NIF"]
+    results = recognizer.analyze("John went home", entities=all_supported)
+    for labels in recognizer.gliner.seen_labels:
+        assert set(labels) <= set(engines.GLINER_ENTITY_MAPPING)
+    assert results and results[0].entity_type == "PERSON"
+
+
+def test_analyze_skips_inference_when_no_owned_entity_requested(fake_gliner):
+    recognizer = make_recognizer("en")
+    assert recognizer.analyze("4111111111111111", entities=["CREDIT_CARD"]) == []
+    assert recognizer.gliner.seen_labels == []
+
+
+def test_entity_mapping_targets_exactly_the_ner_entities():
+    assert set(engines.GLINER_ENTITY_MAPPING.values()) == {
+        "PERSON",
+        "LOCATION",
+        "NRP",
+        "DATE_TIME",
+    }

apps/pii/tests/test_integration.py

@@ -0,0 +1,102 @@
+"""Integration tests — exercise the real engines end-to-end via the FastAPI app.
+
+Requires the models present, so run inside the built images (gated behind
+RUN_PII_INTEGRATION to keep plain `pytest` runs model-free):
+
+    # spacy regression (default engine)
+    docker run --rm -e RUN_PII_INTEGRATION=1 <pii-image> python -m pytest tests
+
+    # gliner engine
+    docker run --rm -e RUN_PII_INTEGRATION=1 -e PII_ENGINE=gliner <pii-image> \
+        python -m pytest tests/test_integration.py
+
+The suite adapts to PII_ENGINE: shared assertions always run, engine-specific
+ones only for the active engine.
+"""
+
+import os
+
+import pytest
+
+if not os.environ.get("RUN_PII_INTEGRATION"):
+    pytest.skip(
+        "integration tests need the built image (RUN_PII_INTEGRATION=1)",
+        allow_module_level=True,
+    )
+
+from fastapi.testclient import TestClient
+
+import server
+
+ENGINE = server.PII_ENGINE
+client = TestClient(server.app)
+
+
+def redact_batch(texts, language="en"):
+    response = client.post("/redact_batch", json={"texts": texts, "language": language})
+    assert response.status_code == 200
+    return response.json()["texts"]
+
+
+def test_health():
+    assert client.get("/health").json() == {"status": "ok"}
+
+
+def test_masks_person_and_email():
+    [masked] = redact_batch(["My name is John Smith, email john.smith@example.com."])
+    assert "<PERSON>" in masked
+    assert "<EMAIL_ADDRESS>" in masked
+    assert "John Smith" not in masked
+    assert "john.smith@example.com" not in masked
+
+
+def test_masks_location_and_phone():
+    [masked] = redact_batch(["I live in Paris, call me at (212) 555-0123."])
+    assert "<LOCATION>" in masked
+    assert "<PHONE_NUMBER>" in masked
+    assert "Paris" not in masked
+
+
+def test_regex_recognizers_fire_in_non_english_languages():
+    [masked] = redact_batch(["Mi NIF es 12345678Z."], language="es")
+    assert "<ES_NIF>" in masked
+    # On the spacy engine the it_core_news_lg NER tags the fiscal code as
+    # ORGANIZATION and outscores the pattern recognizer, so only assert the
+    # value is masked; the exact label is checked on the gliner engine where
+    # spaCy NER can't compete.
+    [masked] = redact_batch(["Il codice fiscale è RSSMRA85T10A562S."], language="it")
+    assert "RSSMRA85T10A562S" not in masked
+    if ENGINE == "gliner":
+        assert "<IT_FISCAL_CODE>" in masked
+
+
+def test_vin_checksum_recognizer_fires():
+    [masked] = redact_batch(["The car VIN is 1HGCM82633A004352."])
+    assert "<VIN>" in masked
+
+
+def test_no_pii_passes_through_unchanged():
+    # NB: "Quarterly" would be tagged DATE_TIME by the spacy engine — keep
+    # this text free of anything either engine considers an entity.
+    text = "Revenue grew and margins held steady."
+    assert redact_batch([text]) == [text]
+
+
+@pytest.mark.skipif(ENGINE != "gliner", reason="gliner-only wiring assertions")
+def test_gliner_registry_has_no_spacy_recognizer():
+    names = {r.name for r in server.analyzer.registry.recognizers}
+    assert "SpacyRecognizer" not in names
+    assert "GLiNERRecognizer" in names
+
+
+@pytest.mark.skipif(ENGINE != "gliner", reason="gliner-only wiring assertions")
+def test_gliner_supported_entities_keep_ner_types():
+    supported = set(server.analyzer.get_supported_entities("en"))
+    assert {"PERSON", "LOCATION", "NRP", "DATE_TIME"} <= supported
+
+
+@pytest.mark.skipif(ENGINE != "spacy", reason="spacy-only wiring assertions")
+def test_spacy_registry_unchanged():
+    names = {r.name for r in server.analyzer.registry.recognizers}
+    assert "SpacyRecognizer" in names
+    assert "GLiNERRecognizer" not in names