chore: import upstream snapshot with attribution

apps/pii/tests/conftest.py

@@ -0,0 +1,5 @@
+import sys
+from pathlib import Path
+
+# server.py / engines.py live one level up (repo: apps/pii, image: /app).
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

apps/pii/tests/test_engines.py

@@ -0,0 +1,105 @@
+"""Unit tests for engines.py — no models, no downloads, no network.
+
+Run: pip install -r requirements.txt -r requirements-dev.txt && python -m pytest tests
+"""
+
+import importlib.util
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from presidio_analyzer.predefined_recognizers.ner import gliner_recognizer
+
+import engines
+
+PII_DIR = Path(__file__).resolve().parent.parent
+
+
+class FakeModel:
+    def __init__(self):
+        self.seen_labels: list[list[str]] = []
+
+    def predict_entities(self, text, labels, flat_ner=True, threshold=0.3, multi_label=False):
+        self.seen_labels.append(list(labels))
+        return [{"label": "person", "score": 0.92, "start": 0, "end": 4, "text": text[0:4]}]
+
+
+class FakeGLiNER:
+    calls = 0
+
+    @classmethod
+    def from_pretrained(cls, model_name, **kwargs):
+        cls.calls += 1
+        return FakeModel()
+
+
+@pytest.fixture
+def fake_gliner(monkeypatch):
+    monkeypatch.setattr(gliner_recognizer, "GLiNER", FakeGLiNER)
+    engines.SharedModelGLiNERRecognizer._shared_models.clear()
+    FakeGLiNER.calls = 0
+    yield FakeGLiNER
+    engines.SharedModelGLiNERRecognizer._shared_models.clear()
+
+
+def make_recognizer(language: str):
+    return engines.SharedModelGLiNERRecognizer(
+        entity_mapping=engines.GLINER_ENTITY_MAPPING,
+        model_name="fake/model",
+        map_location="cpu",
+        supported_language=language,
+    )
+
+
+def test_invalid_pii_engine_fails_import():
+    result = subprocess.run(
+        [sys.executable, "-c", "import server"],
+        cwd=PII_DIR,
+        env={**os.environ, "PII_ENGINE": "bogus"},
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode != 0
+    assert "Invalid PII_ENGINE" in result.stderr
+
+
+@pytest.mark.skipif(
+    importlib.util.find_spec("gliner") is not None,
+    reason="fail-fast path only exists when gliner is not installed",
+)
+def test_build_gliner_analyzer_fails_fast_without_gliner():
+    with pytest.raises(RuntimeError, match="gliner package is not installed"):
+        engines.build_gliner_analyzer(model_name="fake/model", device="cpu")
+
+
+def test_shared_model_loads_once_across_languages(fake_gliner):
+    first = make_recognizer("en")
+    second = make_recognizer("es")
+    assert fake_gliner.calls == 1
+    assert first.gliner is second.gliner
+
+
+def test_analyze_never_prompts_gliner_with_foreign_entities(fake_gliner):
+    recognizer = make_recognizer("en")
+    all_supported = ["PERSON", "LOCATION", "NRP", "DATE_TIME", "CREDIT_CARD", "VIN", "ES_NIF"]
+    results = recognizer.analyze("John went home", entities=all_supported)
+    for labels in recognizer.gliner.seen_labels:
+        assert set(labels) <= set(engines.GLINER_ENTITY_MAPPING)
+    assert results and results[0].entity_type == "PERSON"
+
+
+def test_analyze_skips_inference_when_no_owned_entity_requested(fake_gliner):
+    recognizer = make_recognizer("en")
+    assert recognizer.analyze("4111111111111111", entities=["CREDIT_CARD"]) == []
+    assert recognizer.gliner.seen_labels == []
+
+
+def test_entity_mapping_targets_exactly_the_ner_entities():
+    assert set(engines.GLINER_ENTITY_MAPPING.values()) == {
+        "PERSON",
+        "LOCATION",
+        "NRP",
+        "DATE_TIME",
+    }

apps/pii/tests/test_integration.py

@@ -0,0 +1,102 @@
+"""Integration tests — exercise the real engines end-to-end via the FastAPI app.
+
+Requires the models present, so run inside the built images (gated behind
+RUN_PII_INTEGRATION to keep plain `pytest` runs model-free):
+
+    # spacy regression (default engine)
+    docker run --rm -e RUN_PII_INTEGRATION=1 <pii-image> python -m pytest tests
+
+    # gliner engine
+    docker run --rm -e RUN_PII_INTEGRATION=1 -e PII_ENGINE=gliner <pii-image> \
+        python -m pytest tests/test_integration.py
+
+The suite adapts to PII_ENGINE: shared assertions always run, engine-specific
+ones only for the active engine.
+"""
+
+import os
+
+import pytest
+
+if not os.environ.get("RUN_PII_INTEGRATION"):
+    pytest.skip(
+        "integration tests need the built image (RUN_PII_INTEGRATION=1)",
+        allow_module_level=True,
+    )
+
+from fastapi.testclient import TestClient
+
+import server
+
+ENGINE = server.PII_ENGINE
+client = TestClient(server.app)
+
+
+def redact_batch(texts, language="en"):
+    response = client.post("/redact_batch", json={"texts": texts, "language": language})
+    assert response.status_code == 200
+    return response.json()["texts"]
+
+
+def test_health():
+    assert client.get("/health").json() == {"status": "ok"}
+
+
+def test_masks_person_and_email():
+    [masked] = redact_batch(["My name is John Smith, email john.smith@example.com."])
+    assert "<PERSON>" in masked
+    assert "<EMAIL_ADDRESS>" in masked
+    assert "John Smith" not in masked
+    assert "john.smith@example.com" not in masked
+
+
+def test_masks_location_and_phone():
+    [masked] = redact_batch(["I live in Paris, call me at (212) 555-0123."])
+    assert "<LOCATION>" in masked
+    assert "<PHONE_NUMBER>" in masked
+    assert "Paris" not in masked
+
+
+def test_regex_recognizers_fire_in_non_english_languages():
+    [masked] = redact_batch(["Mi NIF es 12345678Z."], language="es")
+    assert "<ES_NIF>" in masked
+    # On the spacy engine the it_core_news_lg NER tags the fiscal code as
+    # ORGANIZATION and outscores the pattern recognizer, so only assert the
+    # value is masked; the exact label is checked on the gliner engine where
+    # spaCy NER can't compete.
+    [masked] = redact_batch(["Il codice fiscale è RSSMRA85T10A562S."], language="it")
+    assert "RSSMRA85T10A562S" not in masked
+    if ENGINE == "gliner":
+        assert "<IT_FISCAL_CODE>" in masked
+
+
+def test_vin_checksum_recognizer_fires():
+    [masked] = redact_batch(["The car VIN is 1HGCM82633A004352."])
+    assert "<VIN>" in masked
+
+
+def test_no_pii_passes_through_unchanged():
+    # NB: "Quarterly" would be tagged DATE_TIME by the spacy engine — keep
+    # this text free of anything either engine considers an entity.
+    text = "Revenue grew and margins held steady."
+    assert redact_batch([text]) == [text]
+
+
+@pytest.mark.skipif(ENGINE != "gliner", reason="gliner-only wiring assertions")
+def test_gliner_registry_has_no_spacy_recognizer():
+    names = {r.name for r in server.analyzer.registry.recognizers}
+    assert "SpacyRecognizer" not in names
+    assert "GLiNERRecognizer" in names
+
+
+@pytest.mark.skipif(ENGINE != "gliner", reason="gliner-only wiring assertions")
+def test_gliner_supported_entities_keep_ner_types():
+    supported = set(server.analyzer.get_supported_entities("en"))
+    assert {"PERSON", "LOCATION", "NRP", "DATE_TIME"} <= supported
+
+
+@pytest.mark.skipif(ENGINE != "spacy", reason="spacy-only wiring assertions")
+def test_spacy_registry_unchanged():
+    names = {r.name for r in server.analyzer.registry.recognizers}
+    assert "SpacyRecognizer" in names
+    assert "GLiNERRecognizer" not in names