chore: import upstream snapshot with attribution

frigate/data_processing/common/audio_transcription/model.py

@@ -0,0 +1,83 @@
+"""Set up audio transcription models based on model size."""
+
+import logging
+import os
+
+import sherpa_onnx
+
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.const import MODEL_CACHE_DIR
+from frigate.data_processing.types import AudioTranscriptionModel
+from frigate.util.downloader import ModelDownloader
+
+logger = logging.getLogger(__name__)
+
+
+class AudioTranscriptionModelRunner:
+    def __init__(
+        self,
+        device: str = "CPU",
+        model_size: str = "small",
+    ):
+        self.model: AudioTranscriptionModel = None
+        self.requestor = InterProcessRequestor()
+
+        if model_size == "large":
+            # use the Whisper download function instead of our own
+            # Import dynamically to avoid crashes on systems without AVX support
+            from faster_whisper.utils import download_model
+
+            logger.debug("Downloading Whisper audio transcription model")
+            download_model(
+                size_or_id="small" if device == "cuda" else "tiny",
+                local_files_only=False,
+                cache_dir=os.path.join(MODEL_CACHE_DIR, "whisper"),
+            )
+            logger.debug("Whisper audio transcription model downloaded")
+
+        else:
+            # small model as default
+            download_path = os.path.join(MODEL_CACHE_DIR, "sherpa-onnx")
+            HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co")
+            self.model_files = {
+                "encoder.onnx": f"{HF_ENDPOINT}/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26/resolve/main/encoder-epoch-99-avg-1-chunk-16-left-128.onnx",
+                "decoder.onnx": f"{HF_ENDPOINT}/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26/resolve/main/decoder-epoch-99-avg-1-chunk-16-left-128.onnx",
+                "joiner.onnx": f"{HF_ENDPOINT}/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26/resolve/main/joiner-epoch-99-avg-1-chunk-16-left-128.onnx",
+                "tokens.txt": f"{HF_ENDPOINT}/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26/resolve/main/tokens.txt",
+            }
+
+            if not all(
+                os.path.exists(os.path.join(download_path, n))
+                for n in self.model_files.keys()
+            ):
+                self.downloader = ModelDownloader(
+                    model_name="sherpa-onnx",
+                    download_path=download_path,
+                    file_names=list(self.model_files.keys()),
+                    download_func=self.__download_models,
+                )
+                self.downloader.ensure_model_files()
+                self.downloader.wait_for_download()
+
+            self.model = sherpa_onnx.OnlineRecognizer.from_transducer(
+                tokens=os.path.join(MODEL_CACHE_DIR, "sherpa-onnx/tokens.txt"),
+                encoder=os.path.join(MODEL_CACHE_DIR, "sherpa-onnx/encoder.onnx"),
+                decoder=os.path.join(MODEL_CACHE_DIR, "sherpa-onnx/decoder.onnx"),
+                joiner=os.path.join(MODEL_CACHE_DIR, "sherpa-onnx/joiner.onnx"),
+                num_threads=2,
+                sample_rate=16000,
+                feature_dim=80,
+                enable_endpoint_detection=True,
+                rule1_min_trailing_silence=2.4,
+                rule2_min_trailing_silence=1.2,
+                rule3_min_utterance_length=300,
+                decoding_method="greedy_search",
+                provider="cpu",
+            )
+
+    def __download_models(self, path: str) -> None:
+        try:
+            file_name = os.path.basename(path)
+            ModelDownloader.download_from_url(self.model_files[file_name], path)
+        except Exception as e:
+            logger.error(f"Failed to download {path}: {e}")

frigate/data_processing/common/face/model.py

@@ -0,0 +1,436 @@
+import logging
+import os
+import queue
+import threading
+from abc import ABC, abstractmethod
+
+import cv2
+import numpy as np
+from scipy import stats
+
+from frigate.config import FrigateConfig
+from frigate.const import FACE_DIR, MODEL_CACHE_DIR
+from frigate.embeddings.onnx.face_embedding import ArcfaceEmbedding, FaceNetEmbedding
+from frigate.log import redirect_output_to_logger
+
+logger = logging.getLogger(__name__)
+
+
+class FaceRecognizer(ABC):
+    """Face recognition runner."""
+
+    def __init__(self, config: FrigateConfig) -> None:
+        self.config = config
+        self.landmark_detector: cv2.face.Facemark | None = None
+        self.init_landmark_detector()
+
+    @abstractmethod
+    def build(self) -> None:
+        """Build face recognition model."""
+        pass
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Clear current built model."""
+        pass
+
+    @abstractmethod
+    def classify(self, face_image: np.ndarray) -> tuple[str, float] | None:
+        pass
+
+    @redirect_output_to_logger(logger, logging.DEBUG)  # type: ignore[misc]
+    def init_landmark_detector(self) -> None:
+        landmark_model = os.path.join(MODEL_CACHE_DIR, "facedet/landmarkdet.yaml")
+
+        if os.path.exists(landmark_model):
+            landmark_detector = cv2.face.createFacemarkLBF()
+            landmark_detector.loadModel(landmark_model)
+            self.landmark_detector = landmark_detector
+
+    def align_face(
+        self,
+        image: np.ndarray,
+        output_width: int,
+        output_height: int,
+    ) -> np.ndarray:
+        if not self.landmark_detector:
+            raise ValueError("Landmark detector not initialized")
+
+        # landmark is run on grayscale images
+        if image.ndim == 3:
+            land_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        else:
+            land_image = image
+
+        _, lands = self.landmark_detector.fit(
+            land_image, np.array([(0, 0, land_image.shape[1], land_image.shape[0])])
+        )
+        landmarks: np.ndarray = lands[0][0]
+
+        # get landmarks for eyes
+        leftEyePts = landmarks[42:48]
+        rightEyePts = landmarks[36:42]
+
+        # compute the center of mass for each eye
+        leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
+        rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
+
+        # compute the angle between the eye centroids
+        dY = rightEyeCenter[1] - leftEyeCenter[1]
+        dX = rightEyeCenter[0] - leftEyeCenter[0]
+        angle = np.degrees(np.arctan2(dY, dX)) - 180
+
+        # compute the desired right eye x-coordinate based on the
+        # desired x-coordinate of the left eye
+        desiredRightEyeX = 1.0 - 0.35
+
+        # determine the scale of the new resulting image by taking
+        # the ratio of the distance between eyes in the *current*
+        # image to the ratio of distance between eyes in the
+        # *desired* image
+        dist = np.sqrt((dX**2) + (dY**2))
+        desiredDist = desiredRightEyeX - 0.35
+        desiredDist *= output_width
+        scale = desiredDist / dist
+
+        # compute center (x, y)-coordinates (i.e., the median point)
+        # between the two eyes in the input image
+        # grab the rotation matrix for rotating and scaling the face
+        eyesCenter = (
+            int((leftEyeCenter[0] + rightEyeCenter[0]) // 2),
+            int((leftEyeCenter[1] + rightEyeCenter[1]) // 2),
+        )
+        M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
+
+        # update the translation component of the matrix
+        tX = output_width * 0.5
+        tY = output_height * 0.35
+        M[0, 2] += tX - eyesCenter[0]
+        M[1, 2] += tY - eyesCenter[1]
+
+        # apply the affine transformation
+        return cv2.warpAffine(
+            image, M, (output_width, output_height), flags=cv2.INTER_CUBIC
+        )
+
+    def get_blur_confidence_reduction(self, input: np.ndarray) -> float:
+        """Calculates the reduction in confidence based on the blur of the image."""
+        if not self.config.face_recognition.blur_confidence_filter:
+            return 0.0
+
+        variance = cv2.Laplacian(input, cv2.CV_64F).var()
+        logger.debug(f"face detected with blurriness {variance}")
+
+        if variance < 120:  # image is very blurry
+            return 0.06
+        elif variance < 160:  # image moderately blurry
+            return 0.04
+        elif variance < 200:  # image is slightly blurry
+            return 0.02
+        elif variance < 250:  # image is mostly clear
+            return 0.01
+        else:
+            return 0.0
+
+
+def build_class_mean(
+    embs: list[np.ndarray],
+    trim: float = 0.15,
+    outlier_threshold: float = 0.30,
+    min_keep_frac: float = 0.7,
+    max_iters: int = 3,
+) -> np.ndarray:
+    """Build a class-mean embedding with two-layer outlier protection.
+
+    Layer 1 (iterative, vector-wise): drop whole embeddings whose cosine
+    similarity to the current class mean is below ``outlier_threshold``.
+    Catches mislabeled or corrupted training samples (wrong face in the
+    folder, full-frame screenshots, extreme crops) that per-dimension
+    trimming cannot detect.
+
+    Layer 2 (per-dimension): ``scipy.stats.trim_mean`` on the retained set
+    to smooth per-component noise (lighting, expression, alignment jitter).
+
+    Collections with fewer than 5 images bypass outlier rejection — too few
+    samples to establish a reliable class center.
+    """
+    arr = np.stack(embs, axis=0)
+
+    if len(arr) < 5:
+        return np.asarray(stats.trim_mean(arr, trim, axis=0))
+
+    keep = np.ones(len(arr), dtype=bool)
+    floor = max(5, int(np.ceil(min_keep_frac * len(arr))))
+
+    for _ in range(max_iters):
+        mean = stats.trim_mean(arr[keep], trim, axis=0)
+        m_norm = mean / (np.linalg.norm(mean) + 1e-9)
+        e_norms = arr / (np.linalg.norm(arr, axis=1, keepdims=True) + 1e-9)
+        cos = e_norms @ m_norm
+        new_keep = cos >= outlier_threshold
+
+        if new_keep.sum() < floor:
+            top = np.argsort(-cos)[:floor]
+            new_keep = np.zeros(len(arr), dtype=bool)
+            new_keep[top] = True
+
+        if np.array_equal(new_keep, keep):
+            break
+        keep = new_keep
+
+    dropped = int((~keep).sum())
+
+    if dropped:
+        logger.debug(
+            f"Vector-wise outlier filter dropped {dropped}/{len(arr)} embeddings"
+        )
+
+    return np.asarray(stats.trim_mean(arr[keep], trim, axis=0))
+
+
+def similarity_to_confidence(
+    cosine_similarity: float,
+    median: float = 0.3,
+    range_width: float = 0.6,
+    slope_factor: float = 12,
+) -> float:
+    """
+    Default sigmoid function to map cosine similarity to confidence.
+
+    Args:
+        cosine_similarity (float): The input cosine similarity.
+        median (float): Assumed median of cosine similarity distribution.
+        range_width (float): Assumed range of cosine similarity distribution (90th percentile - 10th percentile).
+        slope_factor (float): Adjusts the steepness of the curve.
+
+    Returns:
+        float: The confidence score.
+    """
+
+    # Calculate slope and bias
+    slope = slope_factor / range_width
+    bias = median
+
+    # Calculate confidence
+    confidence: float = 1 / (1 + np.exp(-slope * (cosine_similarity - bias)))
+    return confidence
+
+
+class FaceNetRecognizer(FaceRecognizer):
+    def __init__(self, config: FrigateConfig):
+        super().__init__(config)
+        self.mean_embs: dict[str, np.ndarray] = {}
+        self.face_embedder: FaceNetEmbedding = FaceNetEmbedding()
+        self.model_builder_queue: queue.Queue | None = None
+
+    def clear(self) -> None:
+        self.mean_embs = {}
+
+    def run_build_task(self) -> None:
+        self.model_builder_queue = queue.Queue()
+
+        def build_model() -> None:
+            face_embeddings_map: dict[str, list[np.ndarray]] = {}
+            idx = 0
+
+            dir = FACE_DIR
+            for name in os.listdir(dir):
+                if name == "train":
+                    continue
+
+                face_folder = os.path.join(dir, name)
+
+                if not os.path.isdir(face_folder):
+                    continue
+
+                face_embeddings_map[name] = []
+                for image in os.listdir(face_folder):
+                    img = cv2.imread(os.path.join(face_folder, image))
+
+                    if img is None:
+                        continue  # type: ignore[unreachable]
+
+                    img = self.align_face(img, img.shape[1], img.shape[0])
+                    emb = self.face_embedder([img])[0].squeeze()
+                    face_embeddings_map[name].append(emb)
+
+                idx += 1
+
+            assert self.model_builder_queue is not None
+            self.model_builder_queue.put(face_embeddings_map)
+
+        thread = threading.Thread(target=build_model, daemon=True)
+        thread.start()
+
+    def build(self) -> None:
+        if not self.landmark_detector:
+            self.init_landmark_detector()
+            return None
+
+        if self.model_builder_queue is not None:
+            try:
+                face_embeddings_map: dict[str, list[np.ndarray]] = (
+                    self.model_builder_queue.get(timeout=0.1)
+                )
+                self.model_builder_queue = None
+            except queue.Empty:
+                return
+        else:
+            self.run_build_task()
+            return
+
+        if not face_embeddings_map:
+            return
+
+        for name, embs in face_embeddings_map.items():
+            if embs:
+                self.mean_embs[name] = build_class_mean(embs)
+
+        logger.debug("Finished building ArcFace model")
+
+    def classify(self, face_image: np.ndarray) -> tuple[str, float] | None:
+        if not self.landmark_detector:
+            return None
+
+        if not self.mean_embs:
+            self.build()
+
+            if not self.mean_embs:
+                return None
+
+        # face recognition is best run on grayscale images
+
+        # get blur factor before aligning face
+        blur_reduction = self.get_blur_confidence_reduction(face_image)
+
+        # align face and run recognition
+        img = self.align_face(face_image, face_image.shape[1], face_image.shape[0])
+        embedding = self.face_embedder([img])[0].squeeze()
+
+        score: float = 0
+        label = ""
+
+        for name, mean_emb in self.mean_embs.items():
+            dot_product = np.dot(embedding, mean_emb)
+            magnitude_A = np.linalg.norm(embedding)
+            magnitude_B = np.linalg.norm(mean_emb)
+
+            cosine_similarity = dot_product / (magnitude_A * magnitude_B)
+            confidence = similarity_to_confidence(
+                cosine_similarity, median=0.5, range_width=0.6
+            )
+
+            if confidence > score:
+                score = confidence
+                label = name
+
+        return label, max(0, round(score - blur_reduction, 2))
+
+
+class ArcFaceRecognizer(FaceRecognizer):
+    def __init__(self, config: FrigateConfig):
+        super().__init__(config)
+        self.mean_embs: dict[str, np.ndarray] = {}
+        self.face_embedder: ArcfaceEmbedding = ArcfaceEmbedding(config.face_recognition)
+        self.model_builder_queue: queue.Queue | None = None
+
+    def clear(self) -> None:
+        self.mean_embs = {}
+
+    def run_build_task(self) -> None:
+        self.model_builder_queue = queue.Queue()
+
+        def build_model() -> None:
+            face_embeddings_map: dict[str, list[np.ndarray]] = {}
+            idx = 0
+
+            dir = FACE_DIR
+            for name in os.listdir(dir):
+                if name == "train":
+                    continue
+
+                face_folder = os.path.join(dir, name)
+
+                if not os.path.isdir(face_folder):
+                    continue
+
+                face_embeddings_map[name] = []
+                for image in os.listdir(face_folder):
+                    img = cv2.imread(os.path.join(face_folder, image))
+
+                    if img is None:
+                        continue  # type: ignore[unreachable]
+
+                    img = self.align_face(img, img.shape[1], img.shape[0])
+                    emb = self.face_embedder([img])[0].squeeze()  # type: ignore[arg-type]
+                    face_embeddings_map[name].append(emb)
+
+                idx += 1
+
+            assert self.model_builder_queue is not None
+            self.model_builder_queue.put(face_embeddings_map)
+
+        thread = threading.Thread(target=build_model, daemon=True)
+        thread.start()
+
+    def build(self) -> None:
+        if not self.landmark_detector:
+            self.init_landmark_detector()
+            return None
+
+        if self.model_builder_queue is not None:
+            try:
+                face_embeddings_map: dict[str, list[np.ndarray]] = (
+                    self.model_builder_queue.get(timeout=0.1)
+                )
+                self.model_builder_queue = None
+            except queue.Empty:
+                return
+        else:
+            self.run_build_task()
+            return
+
+        if not face_embeddings_map:
+            return
+
+        for name, embs in face_embeddings_map.items():
+            if embs:
+                self.mean_embs[name] = build_class_mean(embs)
+
+        logger.debug("Finished building ArcFace model")
+
+    def classify(self, face_image: np.ndarray) -> tuple[str, float] | None:
+        if not self.landmark_detector:
+            return None
+
+        if not self.mean_embs:
+            self.build()
+
+            if not self.mean_embs:
+                return None
+
+        # face recognition is best run on grayscale images
+
+        # get blur reduction before aligning face
+        blur_reduction = self.get_blur_confidence_reduction(face_image)
+
+        # align face and run recognition
+        img = self.align_face(face_image, face_image.shape[1], face_image.shape[0])
+        embedding = self.face_embedder([img])[0].squeeze()  # type: ignore[arg-type]
+
+        score: float = 0
+        label = ""
+
+        for name, mean_emb in self.mean_embs.items():
+            dot_product = np.dot(embedding, mean_emb)
+            magnitude_A = np.linalg.norm(embedding)
+            magnitude_B = np.linalg.norm(mean_emb)
+
+            cosine_similarity = dot_product / (magnitude_A * magnitude_B)
+            confidence = similarity_to_confidence(cosine_similarity)
+
+            if confidence > score:
+                score = confidence
+                label = name
+
+        return label, max(0, round(score - blur_reduction, 2))

frigate/data_processing/common/license_plate/model.py

@@ -0,0 +1,37 @@
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.embeddings.onnx.lpr_embedding import (
+    LicensePlateDetector,
+    PaddleOCRClassification,
+    PaddleOCRDetection,
+    PaddleOCRRecognition,
+)
+
+from ...types import DataProcessorModelRunner
+
+
+class LicensePlateModelRunner(DataProcessorModelRunner):
+    def __init__(
+        self,
+        requestor: InterProcessRequestor,
+        device: str = "CPU",
+        model_size: str = "small",
+    ):
+        super().__init__(requestor, device, model_size)
+        self.detection_model = PaddleOCRDetection(
+            model_size=model_size, requestor=requestor, device=device
+        )
+        self.classification_model = PaddleOCRClassification(
+            model_size=model_size, requestor=requestor, device=device
+        )
+        self.recognition_model = PaddleOCRRecognition(
+            model_size=model_size, requestor=requestor, device=device
+        )
+        self.yolov9_detection_model = LicensePlateDetector(
+            model_size=model_size, requestor=requestor, device=device
+        )
+
+        # Load all models once
+        self.detection_model._load_model_and_utils()
+        self.classification_model._load_model_and_utils()
+        self.recognition_model._load_model_and_utils()
+        self.yolov9_detection_model._load_model_and_utils()

frigate/data_processing/post/api.py

@@ -0,0 +1,65 @@
+"""Local or remote processors to handle post processing."""
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any
+
+from frigate.config import FrigateConfig
+
+from ..types import DataProcessorMetrics, DataProcessorModelRunner, PostProcessDataEnum
+
+logger = logging.getLogger(__name__)
+
+
+class PostProcessorApi(ABC):
+    @abstractmethod
+    def __init__(
+        self,
+        config: FrigateConfig,
+        metrics: DataProcessorMetrics,
+        model_runner: DataProcessorModelRunner | None,
+    ) -> None:
+        self.config = config
+        self.metrics = metrics
+        self.model_runner = model_runner
+        pass
+
+    @abstractmethod
+    def process_data(
+        self, data: dict[str, Any], data_type: PostProcessDataEnum
+    ) -> None:
+        """Processes the data of data type.
+        Args:
+            data (dict): containing data about the input.
+            data_type (enum): Describing the data that is being processed.
+
+        Returns:
+            None.
+        """
+        pass
+
+    @abstractmethod
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | str | None:
+        """Handle metadata requests.
+        Args:
+            request_data (dict): containing data about requested change to process.
+
+        Returns:
+            None if request was not handled, otherwise return response.
+        """
+        pass
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Handle a config change notification.
+
+        Called for every config update published under ``config/``.
+        Processors should override this to check the topic and act only
+        on changes relevant to them. Default is a no-op.
+
+        Args:
+            topic: The config topic that changed.
+            payload: The updated configuration object.
+        """
+        pass

frigate/data_processing/post/audio_transcription.py

@@ -0,0 +1,220 @@
+"""Handle post-processing for audio transcription."""
+
+import logging
+import os
+import threading
+import time
+from typing import Any
+
+from peewee import DoesNotExist
+
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.const import (
+    CACHE_DIR,
+    MODEL_CACHE_DIR,
+    UPDATE_AUDIO_TRANSCRIPTION_STATE,
+    UPDATE_EVENT_DESCRIPTION,
+)
+from frigate.data_processing.types import PostProcessDataEnum
+from frigate.embeddings.embeddings import Embeddings
+from frigate.types import TrackedObjectUpdateTypesEnum
+from frigate.util.audio import get_audio_from_recording
+
+from ..types import DataProcessorMetrics
+from .api import PostProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+class AudioTranscriptionPostProcessor(PostProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        embeddings: Embeddings,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics, None)
+        self.config = config
+        self.requestor = requestor
+        self.embeddings = embeddings
+        self.recognizer = None
+        self.transcription_lock = threading.Lock()
+        self.transcription_thread: threading.Thread | None = None
+        self.transcription_running = False
+
+        # faster-whisper handles model downloading automatically
+        self.model_path = os.path.join(MODEL_CACHE_DIR, "whisper")
+        os.makedirs(self.model_path, exist_ok=True)
+
+        self.__build_recognizer()
+
+    def __build_recognizer(self) -> None:
+        try:
+            # Import dynamically to avoid crashes on systems without AVX support
+            from faster_whisper import WhisperModel
+
+            self.recognizer = WhisperModel(
+                model_size_or_path="small",
+                device="cuda"
+                if self.config.audio_transcription.device == "GPU"
+                else "cpu",
+                download_root=self.model_path,
+                local_files_only=False,  # Allow downloading if not cached
+                compute_type="int8",
+            )
+            logger.debug("Audio transcription (recordings) initialized")
+        except Exception as e:
+            logger.error(f"Failed to initialize recordings audio transcription: {e}")
+            self.recognizer = None
+
+    def process_data(
+        self, data: dict[str, Any], data_type: PostProcessDataEnum
+    ) -> None:
+        """Transcribe audio from a recording.
+
+        Args:
+            data (dict): Contains data about the input (event_id, camera, etc.).
+            data_type (enum): Describes the data being processed (recording or tracked_object).
+
+        Returns:
+            None
+        """
+        event_id = data["event_id"]
+        camera_name = data["camera"]
+
+        if data_type == PostProcessDataEnum.recording:
+            start_ts = data["frame_time"]
+            recordings_available_through = data["recordings_available"]
+            end_ts = min(recordings_available_through, start_ts + 60)  # Default 60s
+
+        elif data_type == PostProcessDataEnum.tracked_object:
+            obj_data = data["event"]["data"]
+            obj_data["id"] = data["event"]["id"]
+            obj_data["camera"] = data["event"]["camera"]
+            start_ts = data["event"]["start_time"]
+            end_ts = data["event"].get(
+                "end_time", start_ts + 60
+            )  # Use end_time if available
+
+        else:
+            logger.error("No data type passed to audio transcription post-processing")
+            return
+
+        try:
+            audio_data = get_audio_from_recording(
+                self.config.cameras[camera_name].ffmpeg,
+                camera_name,
+                start_ts,
+                end_ts,
+                sample_rate=16000,
+            )
+
+            if not audio_data:
+                logger.debug(f"No audio data extracted for {event_id}")
+                return
+
+            transcription = self.__transcribe_audio(audio_data)
+            if not transcription:
+                logger.debug("No transcription generated from audio")
+                return
+
+            logger.debug(f"Transcribed audio for {event_id}: '{transcription}'")
+
+            self.requestor.send_data(
+                UPDATE_EVENT_DESCRIPTION,
+                {
+                    "type": TrackedObjectUpdateTypesEnum.description,
+                    "id": event_id,
+                    "description": transcription,
+                    "camera": camera_name,
+                },
+            )
+
+            # Embed the description if semantic search is enabled
+            if self.config.semantic_search.enabled:
+                self.embeddings.embed_description(event_id, transcription)
+
+        except DoesNotExist:
+            logger.debug("No recording found for audio transcription post-processing")
+            return
+        except Exception as e:
+            logger.error(f"Error in audio transcription post-processing: {e}")
+
+    def __transcribe_audio(self, audio_data: bytes) -> str | None:
+        """Transcribe WAV audio data using faster-whisper."""
+        if not self.recognizer:
+            logger.debug("Recognizer not initialized")
+            return None
+
+        try:  # type: ignore[unreachable]
+            # Save audio data to a temporary wav (faster-whisper expects a file)
+            temp_wav = os.path.join(CACHE_DIR, f"temp_audio_{int(time.time())}.wav")
+            with open(temp_wav, "wb") as f:
+                f.write(audio_data)
+
+            segments, info = self.recognizer.transcribe(
+                temp_wav,
+                language=self.config.audio_transcription.language,
+                beam_size=5,
+            )
+
+            os.remove(temp_wav)
+
+            # Combine all segment texts
+            text = " ".join(segment.text.strip() for segment in segments)
+            if not text:
+                return None
+
+            logger.debug(
+                "Detected language '%s' with probability %f",
+                info.language,
+                info.language_probability,
+            )
+
+            return text
+        except Exception as e:
+            logger.error(f"Error transcribing audio: {e}")
+            return None
+
+    def _transcription_wrapper(self, event: dict[str, Any]) -> None:
+        """Wrapper to run transcription and reset running flag when done."""
+        try:
+            self.process_data(
+                {
+                    "event_id": event["id"],
+                    "camera": event["camera"],
+                    "event": event,
+                },
+                PostProcessDataEnum.tracked_object,
+            )
+        finally:
+            with self.transcription_lock:
+                self.transcription_running = False
+                self.transcription_thread = None
+
+            self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "idle")
+
+    def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
+        if topic == "transcribe_audio":
+            event = request_data["event"]
+
+            with self.transcription_lock:
+                if self.transcription_running:
+                    logger.warning(
+                        "Audio transcription for a speech event is already running."
+                    )
+                    return "in_progress"
+
+                # Mark as running and start the thread
+                self.transcription_running = True
+                self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "processing")
+
+                self.transcription_thread = threading.Thread(
+                    target=self._transcription_wrapper, args=(event,), daemon=True
+                )
+                self.transcription_thread.start()
+                return "started"
+
+        return None

frigate/data_processing/post/license_plate.py

@@ -0,0 +1,244 @@
+"""Handle post processing for license plate recognition."""
+
+import datetime
+import logging
+from typing import Any
+
+import cv2
+import numpy as np
+from peewee import DoesNotExist
+
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.data_processing.common.license_plate.mixin import (
+    WRITE_DEBUG_IMAGES,
+    LicensePlateProcessingMixin,
+)
+from frigate.data_processing.common.license_plate.model import (
+    LicensePlateModelRunner,
+)
+from frigate.data_processing.types import PostProcessDataEnum
+from frigate.models import Recordings
+from frigate.util.image import get_image_from_recording
+
+from ..types import DataProcessorMetrics
+from .api import PostProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):  # type: ignore[misc]
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+        model_runner: LicensePlateModelRunner,
+        detected_license_plates: dict[str, dict[str, Any]],
+    ):
+        self.requestor = requestor
+        self.detected_license_plates = detected_license_plates
+        self.model_runner = model_runner
+        self.lpr_config = config.lpr
+        self.config = config
+        self.sub_label_publisher = sub_label_publisher
+        super().__init__(config, metrics, model_runner)
+
+    CONFIG_UPDATE_TOPIC = "config/lpr"
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Update LPR config at runtime."""
+        if topic != self.CONFIG_UPDATE_TOPIC:
+            return
+
+        self.lpr_config = payload
+        logger.debug("LPR post-processor config updated dynamically")
+
+    def process_data(
+        self, data: dict[str, Any], data_type: PostProcessDataEnum
+    ) -> None:
+        """Look for license plates in recording stream image
+        Args:
+            data (dict): containing data about the input.
+            data_type (enum): Describing the data that is being processed.
+
+        Returns:
+            None.
+        """
+        # don't run LPR post processing for now
+        return
+
+        event_id = data["event_id"]  # type: ignore[unreachable]
+        camera_name = data["camera"]
+
+        if data_type == PostProcessDataEnum.recording:
+            obj_data = data["obj_data"]
+            frame_time = obj_data["frame_time"]
+            recordings_available_through = data["recordings_available"]
+
+            if frame_time > recordings_available_through:
+                logger.debug(
+                    f"LPR post processing: No recordings available for this frame time {frame_time}, available through {recordings_available_through}"
+                )
+
+        elif data_type == PostProcessDataEnum.tracked_object:
+            # non-functional, need to think about snapshot time
+            obj_data = data["event"]["data"]
+            obj_data["id"] = data["event"]["id"]
+            obj_data["camera"] = data["event"]["camera"]
+            # TODO: snapshot time?
+            frame_time = data["event"]["start_time"]
+
+        else:
+            logger.error("No data type passed to LPR postprocessing")
+            return
+
+        recording_query = (
+            Recordings.select(
+                Recordings.path,
+                Recordings.start_time,
+            )
+            .where(
+                (frame_time >= Recordings.start_time)
+                & (frame_time <= Recordings.end_time)
+            )
+            .where(Recordings.camera == camera_name)
+            .order_by(Recordings.start_time.desc())
+            .limit(1)
+        )
+
+        try:
+            recording: Recordings = recording_query.get()
+            time_in_segment = frame_time - recording.start_time
+            codec = "mjpeg"
+
+            image_data = get_image_from_recording(
+                self.config.ffmpeg, recording.path, time_in_segment, codec, None
+            )
+
+            if not image_data:
+                logger.debug(
+                    "LPR post processing: Unable to fetch license plate from recording"
+                )
+
+            # Convert bytes to numpy array
+            image_array = np.frombuffer(image_data, dtype=np.uint8)
+
+            if len(image_array) == 0:
+                logger.debug("LPR post processing: No image")
+                return
+
+            image = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
+
+        except DoesNotExist:
+            logger.debug("Error fetching license plate for postprocessing")
+            return
+
+        if WRITE_DEBUG_IMAGES:
+            cv2.imwrite(
+                f"debug/frames/lpr_post_{datetime.datetime.now().timestamp()}.jpg",
+                image,
+            )
+
+        # convert to yuv for processing
+        frame = cv2.cvtColor(image, cv2.COLOR_BGR2YUV_I420)
+
+        detect_width = self.config.cameras[camera_name].detect.width
+        detect_height = self.config.cameras[camera_name].detect.height
+
+        # Scale the boxes based on detect dimensions
+        scale_x = image.shape[1] / detect_width
+        scale_y = image.shape[0] / detect_height
+
+        # Determine which box to enlarge based on detection mode
+        if "license_plate" not in self.config.cameras[camera_name].objects.track:
+            # Scale and enlarge the car box
+            box = obj_data.get("box")
+            if not box:
+                return
+
+            # Scale original car box to detection dimensions
+            left = int(box[0] * scale_x)
+            top = int(box[1] * scale_y)
+            right = int(box[2] * scale_x)
+            bottom = int(box[3] * scale_y)
+            box = [left, top, right, bottom]
+        else:
+            # Get the license plate box from attributes
+            if not obj_data.get("current_attributes"):
+                return
+
+            license_plate = None
+            for attr in obj_data["current_attributes"]:
+                if attr.get("label") != "license_plate":
+                    continue
+                if license_plate is None or attr.get("score", 0.0) > license_plate.get(
+                    "score", 0.0
+                ):
+                    license_plate = attr
+
+            if not license_plate or not license_plate.get("box"):
+                return
+
+            # Scale license plate box to detection dimensions
+            orig_box = license_plate["box"]
+            left = int(orig_box[0] * scale_x)
+            top = int(orig_box[1] * scale_y)
+            right = int(orig_box[2] * scale_x)
+            bottom = int(orig_box[3] * scale_y)
+            box = [left, top, right, bottom]
+
+        width_box = right - left
+        height_box = bottom - top
+
+        # Enlarge box slightly to account for drift in detect vs recording stream
+        enlarge_factor = 0.3
+        new_left = max(0, int(left - (width_box * enlarge_factor / 2)))
+        new_top = max(0, int(top - (height_box * enlarge_factor / 2)))
+        new_right = min(image.shape[1], int(right + (width_box * enlarge_factor / 2)))
+        new_bottom = min(
+            image.shape[0], int(bottom + (height_box * enlarge_factor / 2))
+        )
+
+        keyframe_obj_data = obj_data.copy()
+        if "license_plate" not in self.config.cameras[camera_name].objects.track:
+            # car box
+            keyframe_obj_data["box"] = [new_left, new_top, new_right, new_bottom]
+        else:
+            # Update the license plate box in the attributes
+            new_attributes = []
+            for attr in obj_data["current_attributes"]:
+                if attr.get("label") == "license_plate":
+                    new_attr = attr.copy()
+                    new_attr["box"] = [new_left, new_top, new_right, new_bottom]
+                    new_attributes.append(new_attr)
+                else:
+                    new_attributes.append(attr)
+            keyframe_obj_data["current_attributes"] = new_attributes
+
+        # run the frame through lpr processing
+        logger.debug(f"Post processing plate: {event_id}, {frame_time}")
+        self.lpr_process(keyframe_obj_data, frame)
+
+    def handle_request(self, topic: str, request_data: dict) -> dict[str, Any] | None:
+        if topic == EmbeddingsRequestEnum.reprocess_plate.value:
+            event = request_data["event"]
+
+            self.process_data(
+                {
+                    "event_id": event["id"],
+                    "camera": event["camera"],
+                    "event": event,
+                },
+                PostProcessDataEnum.tracked_object,
+            )
+
+            return {
+                "message": "Successfully requested reprocessing of license plate.",
+                "success": True,
+            }
+
+        return None

frigate/data_processing/post/object_descriptions.py

@@ -0,0 +1,381 @@
+"""Post processor for object descriptions using GenAI."""
+
+import datetime
+import logging
+import os
+import threading
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import cv2
+import numpy as np
+from peewee import DoesNotExist
+
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import CameraConfig, FrigateConfig
+from frigate.const import CLIPS_DIR, UPDATE_EVENT_DESCRIPTION
+from frigate.data_processing.post.semantic_trigger import SemanticTriggerProcessor
+from frigate.data_processing.types import PostProcessDataEnum
+from frigate.genai.manager import GenAIClientManager
+from frigate.models import Event
+from frigate.types import TrackedObjectUpdateTypesEnum
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed
+from frigate.util.file import get_event_thumbnail_bytes, load_event_snapshot_image
+from frigate.util.image import create_thumbnail, ensure_jpeg_bytes
+
+if TYPE_CHECKING:
+    from frigate.embeddings.embeddings import Embeddings
+
+from ..post.api import PostProcessorApi
+from ..types import DataProcessorMetrics
+
+logger = logging.getLogger(__name__)
+
+MAX_THUMBNAILS = 10
+
+
+class ObjectDescriptionProcessor(PostProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        embeddings: "Embeddings",
+        requestor: InterProcessRequestor,
+        metrics: DataProcessorMetrics,
+        genai_manager: GenAIClientManager,
+        semantic_trigger_processor: SemanticTriggerProcessor | None,
+    ):
+        super().__init__(config, metrics, None)
+        self.config = config
+        self.embeddings = embeddings
+        self.requestor = requestor
+        self.metrics = metrics
+        self.genai_manager = genai_manager
+        self.semantic_trigger_processor = semantic_trigger_processor
+        self.tracked_events: dict[str, list[Any]] = {}
+        self.early_request_sent: dict[str, bool] = {}
+        self.object_desc_speed = InferenceSpeed(self.metrics.object_desc_speed)
+        self.object_desc_dps = EventsPerSecond()
+        self.object_desc_dps.start()
+
+    def __handle_frame_update(
+        self, camera: str, data: dict, yuv_frame: np.ndarray
+    ) -> None:
+        """Handle an update to a frame for an object."""
+        camera_config = self.config.cameras[camera]
+
+        # no need to save our own thumbnails if genai is not enabled
+        # or if the object has become stationary
+        if not data["stationary"]:
+            if data["id"] not in self.tracked_events:
+                self.tracked_events[data["id"]] = []
+
+            data["thumbnail"] = create_thumbnail(yuv_frame, data["box"])
+
+            # Limit the number of thumbnails saved
+            if len(self.tracked_events[data["id"]]) >= MAX_THUMBNAILS:
+                # Always keep the first thumbnail for the event
+                self.tracked_events[data["id"]].pop(1)
+
+            self.tracked_events[data["id"]].append(data)
+
+        # check if we're configured to send an early request after a minimum number of updates received
+        if camera_config.objects.genai.send_triggers.after_significant_updates:
+            if (
+                len(self.tracked_events.get(data["id"], []))
+                >= camera_config.objects.genai.send_triggers.after_significant_updates
+                and data["id"] not in self.early_request_sent
+            ):
+                if data["has_clip"] and data["has_snapshot"]:
+                    try:
+                        event: Event = Event.get(Event.id == data["id"])
+                    except DoesNotExist:
+                        logger.error(f"Event {data['id']} not found")
+                        return
+
+                    if (
+                        not camera_config.objects.genai.objects
+                        or event.label in camera_config.objects.genai.objects
+                    ) and (
+                        not camera_config.objects.genai.required_zones
+                        or set(data["entered_zones"])
+                        & set(camera_config.objects.genai.required_zones)
+                    ):
+                        logger.debug(f"{camera} sending early request to GenAI")
+
+                        self.early_request_sent[data["id"]] = True
+                        # Copy thumbnails to avoid holding references after cleanup
+                        thumbnails_copy = [
+                            data["thumbnail"][:] if data.get("thumbnail") else None
+                            for data in self.tracked_events[data["id"]]
+                            if data.get("thumbnail")
+                        ]
+                        threading.Thread(
+                            target=self._genai_embed_description,
+                            name=f"_genai_embed_description_{event.id}",
+                            daemon=True,
+                            args=(
+                                event,
+                                thumbnails_copy,
+                            ),
+                        ).start()
+
+    def __handle_frame_finalize(
+        self, camera: str, event: Event, thumbnail: bytes
+    ) -> None:
+        """Handle the finalization of a frame."""
+        camera_config = self.config.cameras[camera]
+
+        if (
+            camera_config.objects.genai.enabled
+            and camera_config.objects.genai.send_triggers.tracked_object_end
+            and (
+                not camera_config.objects.genai.objects
+                or event.label in camera_config.objects.genai.objects
+            )
+            and (
+                not camera_config.objects.genai.required_zones
+                or set(event.zones) & set(camera_config.objects.genai.required_zones)
+            )
+        ):
+            self._process_genai_description(event, camera_config, thumbnail)
+        else:
+            self.cleanup_event(str(event.id))
+
+    def __regenerate_description(self, event_id: str, source: str, force: bool) -> None:
+        """Regenerate the description for an event."""
+        try:
+            event: Event = Event.get(Event.id == event_id)
+        except DoesNotExist:
+            logger.error(f"Event {event_id} not found for description regeneration")
+            return
+
+        camera_config = self.config.cameras[str(event.camera)]
+        if not camera_config.objects.genai.enabled and not force:
+            logger.error(f"GenAI not enabled for camera {event.camera}")
+            return
+
+        thumbnail = get_event_thumbnail_bytes(event)
+
+        if thumbnail is None:
+            logger.error("No thumbnail available for %s", event.id)
+            return
+
+        # ensure we have a jpeg to pass to the model
+        thumbnail = ensure_jpeg_bytes(thumbnail)
+
+        logger.debug(
+            f"Trying {source} regeneration for {event}, has_snapshot: {event.has_snapshot}"
+        )
+
+        if event.has_snapshot and source == "snapshot":
+            snapshot_image = self._read_and_crop_snapshot(event)
+            if not snapshot_image:
+                return
+
+        embed_image = (
+            [snapshot_image]
+            if event.has_snapshot and source == "snapshot"
+            # Copy thumbnails to avoid holding references
+            else (
+                [
+                    data["thumbnail"][:] if data.get("thumbnail") else None
+                    for data in self.tracked_events[event_id]
+                    if data.get("thumbnail")
+                ]
+                if len(self.tracked_events.get(event_id, [])) > 0
+                else [thumbnail]
+            )
+        )
+
+        self._genai_embed_description(
+            event, [img for img in embed_image if img is not None]
+        )
+
+    def process_data(self, frame_data: dict, data_type: PostProcessDataEnum) -> None:
+        """Process a frame update."""
+        self.metrics.object_desc_dps.value = self.object_desc_dps.eps()
+
+        if data_type != PostProcessDataEnum.tracked_object:
+            return
+
+        if self.genai_manager.description_client is None:
+            return
+
+        state: str | None = frame_data.get("state", None)
+
+        if state is not None:
+            logger.debug(f"Processing {state} for {frame_data['camera']}")
+
+        if state == "update":
+            self.__handle_frame_update(
+                frame_data["camera"], frame_data["data"], frame_data["yuv_frame"]
+            )
+        elif state == "finalize":
+            self.__handle_frame_finalize(
+                frame_data["camera"], frame_data["event"], frame_data["thumbnail"]
+            )
+
+    def handle_request(self, topic: str, data: dict[str, Any]) -> str | None:
+        """Handle a request."""
+        if topic == "regenerate_description":
+            self.__regenerate_description(
+                data["event_id"], data["source"], data["force"]
+            )
+        return None
+
+    def cleanup_event(self, event_id: str) -> None:
+        """Clean up tracked event data to prevent memory leaks.
+
+        This should be called when an event ends, regardless of whether
+        genai processing is triggered.
+        """
+        if event_id in self.tracked_events:
+            del self.tracked_events[event_id]
+        if event_id in self.early_request_sent:
+            del self.early_request_sent[event_id]
+
+    def _read_and_crop_snapshot(self, event: Event) -> bytes | None:
+        """Read, decode, and crop the snapshot image."""
+
+        try:
+            img, _ = load_event_snapshot_image(event)
+            if img is None:
+                logger.error(f"Cannot load snapshot for {event.id}, file not found")
+                return None
+
+            # Crop snapshot based on region
+            # provide full image if region doesn't exist (manual events)
+            height, width = img.shape[:2]
+            x1_rel, y1_rel, width_rel, height_rel = event.data.get(  # type: ignore[attr-defined]
+                "region", [0, 0, 1, 1]
+            )
+            x1, y1 = int(x1_rel * width), int(y1_rel * height)
+
+            cropped_image = img[
+                y1 : y1 + int(height_rel * height),
+                x1 : x1 + int(width_rel * width),
+            ]
+
+            _, buffer = cv2.imencode(".jpg", cropped_image)
+
+            return buffer.tobytes()
+        except Exception:
+            return None
+
+    def _process_genai_description(
+        self, event: Event, camera_config: CameraConfig, thumbnail: bytes
+    ) -> None:
+        event_id = str(event.id)
+
+        if event.has_snapshot and camera_config.objects.genai.use_snapshot:
+            snapshot_image = self._read_and_crop_snapshot(event)
+
+            if not snapshot_image:
+                self.cleanup_event(event_id)
+                return
+
+        num_thumbnails = len(self.tracked_events.get(event_id, []))
+
+        # ensure we have a jpeg to pass to the model
+        thumbnail = ensure_jpeg_bytes(thumbnail)
+
+        embed_image = (
+            [snapshot_image]
+            if event.has_snapshot and camera_config.objects.genai.use_snapshot
+            # Copy thumbnails to avoid holding references after cleanup
+            else (
+                [
+                    data["thumbnail"][:] if data.get("thumbnail") else None
+                    for data in self.tracked_events[event_id]
+                    if data.get("thumbnail")
+                ]
+                if num_thumbnails > 0
+                else [thumbnail]
+            )
+        )
+
+        if camera_config.objects.genai.debug_save_thumbnails and num_thumbnails > 0:
+            logger.debug(f"Saving {num_thumbnails} thumbnails for event {event_id}")
+
+            Path(os.path.join(CLIPS_DIR, f"genai-requests/{event_id}")).mkdir(
+                parents=True, exist_ok=True
+            )
+
+            for idx, data in enumerate(self.tracked_events[event_id], 1):
+                jpg_bytes: bytes | None = data["thumbnail"]
+
+                if jpg_bytes is None:
+                    logger.warning(f"Unable to save thumbnail {idx} for {event_id}.")
+                else:
+                    with open(
+                        os.path.join(
+                            CLIPS_DIR,
+                            f"genai-requests/{event_id}/{idx}.jpg",
+                        ),
+                        "wb",
+                    ) as j:
+                        j.write(jpg_bytes)
+
+        # Generate the description. Call happens in a thread since it is network bound.
+        threading.Thread(
+            target=self._genai_embed_description,
+            name=f"_genai_embed_description_{event_id}",
+            daemon=True,
+            args=(
+                event,
+                embed_image,
+            ),
+        ).start()
+
+        # Clean up tracked events and early request state
+        self.cleanup_event(event_id)
+
+    def _genai_embed_description(self, event: Event, thumbnails: list[bytes]) -> None:
+        """Embed the description for an event."""
+        start = datetime.datetime.now().timestamp()
+        camera_config = self.config.cameras[str(event.camera)]
+        client = self.genai_manager.description_client
+
+        if client is None:
+            return
+
+        description = client.generate_object_description(
+            camera_config, thumbnails, event
+        )
+
+        if not description:
+            logger.debug("Failed to generate description for %s", event.id)
+            return
+
+        # fire and forget description update
+        self.requestor.send_data(
+            UPDATE_EVENT_DESCRIPTION,
+            {
+                "type": TrackedObjectUpdateTypesEnum.description,
+                "id": event.id,
+                "description": description,
+                "camera": event.camera,
+            },
+        )
+
+        # Embed the description
+        if self.config.semantic_search.enabled:
+            self.embeddings.embed_description(str(event.id), description)
+
+            # Check semantic trigger for this description
+            if self.semantic_trigger_processor is not None:
+                self.semantic_trigger_processor.process_data(
+                    {"event_id": event.id, "camera": event.camera, "type": "text"},
+                    PostProcessDataEnum.tracked_object,
+                )
+
+        # Update inference timing metrics
+        self.object_desc_speed.update(datetime.datetime.now().timestamp() - start)
+        self.object_desc_dps.update()
+
+        logger.debug(
+            "Generated description for %s (%d images): %s",
+            event.id,
+            len(thumbnails),
+            description,
+        )

frigate/data_processing/post/review_descriptions.py

@@ -0,0 +1,614 @@
+"""Post processor for review items to get descriptions."""
+
+import copy
+import datetime
+import logging
+import math
+import os
+import shutil
+import threading
+from pathlib import Path
+from typing import Any
+
+import cv2
+from peewee import DoesNotExist
+from titlecase import titlecase
+
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.config.camera import CameraConfig
+from frigate.config.camera.review import GenAIReviewConfig, ImageSourceEnum
+from frigate.const import (
+    ATTRIBUTE_LABEL_DISPLAY_MAP,
+    CACHE_DIR,
+    CLIPS_DIR,
+    UPDATE_REVIEW_DESCRIPTION,
+)
+from frigate.data_processing.types import PostProcessDataEnum
+from frigate.genai import GenAIClient
+from frigate.genai.manager import GenAIClientManager
+from frigate.models import Recordings, ReviewSegment
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed
+from frigate.util.image import get_image_from_recording
+
+from ..post.api import PostProcessorApi
+from ..types import DataProcessorMetrics
+
+logger = logging.getLogger(__name__)
+
+RECORDING_BUFFER_EXTENSION_PERCENT = 0.10
+MIN_RECORDING_DURATION = 10
+MAX_IMAGE_TOKENS = 24000
+MAX_FRAMES_PER_SECOND = 1
+
+
+class ReviewDescriptionProcessor(PostProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        metrics: DataProcessorMetrics,
+        genai_manager: GenAIClientManager,
+    ):
+        super().__init__(config, metrics, None)
+        self.requestor = requestor
+        self.metrics = metrics
+        self.genai_manager = genai_manager
+        self.review_desc_speed = InferenceSpeed(self.metrics.review_desc_speed)
+        self.review_desc_dps = EventsPerSecond()
+        self.review_desc_dps.start()
+
+    def calculate_frame_count(
+        self,
+        camera: str,
+        duration: float,
+        image_source: ImageSourceEnum = ImageSourceEnum.preview,
+        height: int = 480,
+    ) -> int:
+        """Calculate optimal number of frames based on event duration, context size,
+        image source, and resolution.
+
+        Per-image token cost is asked of the GenAI provider so providers that know
+        their model's true cost (e.g. llama.cpp can probe the loaded mmproj) can
+        diverge from the default ~1-token-per-1250-pixels heuristic. The frame
+        budget is bounded by:
+          - remaining context window after prompt + response reservations
+          - a fixed MAX_IMAGE_TOKENS ceiling
+          - MAX_FRAMES_PER_SECOND x duration, to avoid drowning short events in
+            near-duplicate frames where the model latches onto the redundant middle
+            and skips the start/end action
+        """
+        client = self.genai_manager.description_client
+
+        if client is None:
+            return 3
+
+        context_size = client.get_context_size()
+        camera_config = self.config.cameras[camera]
+
+        detect_width = camera_config.detect.width
+        detect_height = camera_config.detect.height
+
+        if not detect_width or not detect_height:
+            aspect_ratio = 16 / 9
+        else:
+            aspect_ratio = detect_width / detect_height
+
+        if image_source == ImageSourceEnum.recordings:
+            if aspect_ratio >= 1:
+                # Landscape or square: constrain height
+                width = int(height * aspect_ratio)
+            else:
+                # Portrait: constrain width
+                width = height
+                height = int(width / aspect_ratio)
+        else:
+            if aspect_ratio >= 1:
+                # Landscape or square: constrain height
+                target_height = 180
+                width = int(target_height * aspect_ratio)
+                height = target_height
+            else:
+                # Portrait: constrain width
+                target_width = 180
+                width = target_width
+                height = int(target_width / aspect_ratio)
+
+        tokens_per_image = client.estimate_image_tokens(width, height)
+        prompt_tokens = 3800
+        response_tokens = 300
+        context_budget = context_size - prompt_tokens - response_tokens
+        image_token_budget = min(context_budget, MAX_IMAGE_TOKENS)
+        max_frames_by_tokens = int(image_token_budget / tokens_per_image)
+        max_frames_by_duration = int(duration * MAX_FRAMES_PER_SECOND)
+        max_frames = min(max_frames_by_tokens, max_frames_by_duration)
+        return max(max_frames, 3)
+
+    def process_data(
+        self, data: dict[str, Any], data_type: PostProcessDataEnum
+    ) -> None:
+        self.metrics.review_desc_dps.value = self.review_desc_dps.eps()
+
+        if data_type != PostProcessDataEnum.review:
+            return
+
+        if self.genai_manager.description_client is None:
+            return
+
+        camera = data["after"]["camera"]
+        camera_config = self.config.cameras[camera]
+
+        if not camera_config.review.genai.enabled:
+            return
+
+        id = data["after"]["id"]
+
+        if data["type"] == "new" or data["type"] == "update":
+            return
+        else:
+            final_data = data["after"]
+
+            if (
+                final_data["severity"] == "alert"
+                and not camera_config.review.genai.alerts
+            ):
+                return
+            elif (
+                final_data["severity"] == "detection"
+                and not camera_config.review.genai.detections
+            ):
+                return
+
+            image_source = camera_config.review.genai.image_source
+
+            if image_source == ImageSourceEnum.recordings:
+                duration = final_data["end_time"] - final_data["start_time"]
+                buffer_extension = min(5, duration * RECORDING_BUFFER_EXTENSION_PERCENT)
+
+                # Ensure minimum total duration for short review items
+                # This provides better context for brief events
+                total_duration = duration + (2 * buffer_extension)
+                if total_duration < MIN_RECORDING_DURATION:
+                    # Expand buffer to reach minimum duration, still respecting max of 5s per side
+                    additional_buffer_per_side = (MIN_RECORDING_DURATION - duration) / 2
+                    buffer_extension = min(5, additional_buffer_per_side)
+
+                final_data["start_time"] -= buffer_extension
+                final_data["end_time"] += buffer_extension
+
+                thumbs = self.get_recording_frames(
+                    camera,
+                    final_data["start_time"],
+                    final_data["end_time"],
+                    height=480,  # Use 480p for good balance between quality and token usage
+                )
+
+                if not thumbs:
+                    # Fallback to preview frames if no recordings available
+                    logger.warning(
+                        f"No recording frames found for {camera}, falling back to preview frames"
+                    )
+                    thumbs = self.get_preview_frames_as_bytes(
+                        camera,
+                        final_data["start_time"],
+                        final_data["end_time"],
+                        final_data["thumb_path"],
+                        id,
+                        camera_config.review.genai.debug_save_thumbnails,
+                    )
+                elif camera_config.review.genai.debug_save_thumbnails:
+                    # Save debug thumbnails for recordings
+                    Path(os.path.join(CLIPS_DIR, "genai-requests", id)).mkdir(
+                        parents=True, exist_ok=True
+                    )
+                    for idx, frame_bytes in enumerate(thumbs):
+                        with open(
+                            os.path.join(CLIPS_DIR, f"genai-requests/{id}/{idx}.jpg"),
+                            "wb",
+                        ) as f:
+                            f.write(frame_bytes)
+            else:
+                # Use preview frames
+                thumbs = self.get_preview_frames_as_bytes(
+                    camera,
+                    final_data["start_time"],
+                    final_data["end_time"],
+                    final_data["thumb_path"],
+                    id,
+                    camera_config.review.genai.debug_save_thumbnails,
+                )
+
+            # kickoff analysis
+            self.review_desc_dps.update()
+            threading.Thread(
+                target=run_analysis,
+                args=(
+                    self.requestor,
+                    self.genai_manager.description_client,
+                    self.review_desc_speed,
+                    camera_config,
+                    final_data,
+                    thumbs,
+                    camera_config.review.genai,
+                    list(self.config.model.merged_labelmap.values()),
+                    self.config.model.all_attributes,
+                ),
+            ).start()
+
+    def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
+        if topic == EmbeddingsRequestEnum.summarize_review.value:
+            start_ts = request_data["start_ts"]
+            end_ts = request_data["end_ts"]
+            logger.debug(
+                f"Found GenAI Review Summary request for {start_ts} to {end_ts}"
+            )
+
+            # Query all review segments with camera and time information
+            segments: list[dict[str, Any]] = [
+                {
+                    "camera": r["camera"].replace("_", " ").title(),
+                    "start_time": r["start_time"],
+                    "end_time": r["end_time"],
+                    "metadata": r["data"]["metadata"],
+                }
+                for r in (
+                    ReviewSegment.select(
+                        ReviewSegment.camera,
+                        ReviewSegment.start_time,
+                        ReviewSegment.end_time,
+                        ReviewSegment.data,
+                    )
+                    .where(
+                        (ReviewSegment.data["metadata"].is_null(False))
+                        & (ReviewSegment.start_time < end_ts)
+                        & (ReviewSegment.end_time > start_ts)
+                    )
+                    .order_by(ReviewSegment.start_time.asc())
+                    .dicts()
+                    .iterator()
+                )
+            ]
+
+            if len(segments) == 0:
+                logger.debug("No review items with metadata found during time period")
+                return "No activity was found during this time period."
+
+            # Identify primary items (important items that need review)
+            primary_segments = [
+                seg
+                for seg in segments
+                if seg["metadata"].get("potential_threat_level", 0) > 0
+                or seg["metadata"].get("other_concerns")
+            ]
+
+            if not primary_segments:
+                return "No concerns were found during this time period."
+
+            # Build hierarchical structure: each primary event with its contextual items
+            events_with_context = []
+
+            for primary_seg in primary_segments:
+                # Start building the primary event structure
+                primary_item = copy.deepcopy(primary_seg["metadata"])
+                primary_item["camera"] = primary_seg["camera"]
+                primary_item["start_time"] = primary_seg["start_time"]
+                primary_item["end_time"] = primary_seg["end_time"]
+
+                # Find overlapping contextual items from other cameras
+                primary_start = primary_seg["start_time"]
+                primary_end = primary_seg["end_time"]
+                primary_camera = primary_seg["camera"]
+                contextual_items = []
+                seen_contextual_cameras = set()
+
+                for seg in segments:
+                    seg_camera = seg["camera"]
+
+                    if seg_camera == primary_camera:
+                        continue
+
+                    if seg in primary_segments:
+                        continue
+
+                    seg_start = seg["start_time"]
+                    seg_end = seg["end_time"]
+
+                    if seg_start < primary_end and primary_start < seg_end:
+                        # Avoid duplicates if same camera has multiple overlapping segments
+                        if seg_camera not in seen_contextual_cameras:
+                            contextual_item = copy.deepcopy(seg["metadata"])
+                            contextual_item["camera"] = seg_camera
+                            contextual_item["start_time"] = seg_start
+                            contextual_item["end_time"] = seg_end
+                            contextual_items.append(contextual_item)
+                            seen_contextual_cameras.add(seg_camera)
+
+                # Add context array to primary item
+                primary_item["context"] = contextual_items
+                events_with_context.append(primary_item)
+
+            total_context_items = sum(
+                len(event.get("context", [])) for event in events_with_context
+            )
+            logger.debug(
+                f"Summary includes {len(events_with_context)} primary events with "
+                f"{total_context_items} total contextual items"
+            )
+
+            if self.config.review.genai.debug_save_thumbnails:
+                Path(
+                    os.path.join(CLIPS_DIR, "genai-requests", f"{start_ts}-{end_ts}")
+                ).mkdir(parents=True, exist_ok=True)
+
+            client = self.genai_manager.description_client
+
+            if client is None:
+                return None
+
+            return client.generate_review_summary(
+                start_ts,
+                end_ts,
+                events_with_context,
+                self.config.review.genai.preferred_language,
+                self.config.review.genai.debug_save_thumbnails,
+            )
+        else:
+            return None
+
+    def get_cache_frames(
+        self,
+        camera: str,
+        start_time: float,
+        end_time: float,
+    ) -> list[str]:
+        preview_dir = os.path.join(CACHE_DIR, "preview_frames")
+        file_start = f"preview_{camera}-"
+        start_file = f"{file_start}{start_time}.webp"
+        end_file = f"{file_start}{end_time}.webp"
+
+        camera_files = [
+            entry.name
+            for entry in os.scandir(preview_dir)
+            if entry.name.startswith(file_start)
+        ]
+        camera_files.sort()
+
+        all_frames: list[str] = []
+
+        for file in camera_files:
+            if file < start_file:
+                if len(all_frames):
+                    all_frames[0] = os.path.join(preview_dir, file)
+                else:
+                    all_frames.append(os.path.join(preview_dir, file))
+
+                continue
+
+            if file > end_file:
+                all_frames.append(os.path.join(preview_dir, file))
+                break
+
+            all_frames.append(os.path.join(preview_dir, file))
+
+        frame_count = len(all_frames)
+        desired_frame_count = self.calculate_frame_count(
+            camera, duration=end_time - start_time
+        )
+
+        if frame_count <= desired_frame_count:
+            return all_frames
+
+        selected_frames = []
+        step_size = (frame_count - 1) / (desired_frame_count - 1)
+
+        for i in range(desired_frame_count):
+            index = round(i * step_size)
+            selected_frames.append(all_frames[index])
+
+        return selected_frames
+
+    def get_recording_frames(
+        self,
+        camera: str,
+        start_time: float,
+        end_time: float,
+        height: int = 480,
+    ) -> list[bytes]:
+        """Get frames from recordings at specified timestamps."""
+        duration = end_time - start_time
+        desired_frame_count = self.calculate_frame_count(
+            camera, duration, ImageSourceEnum.recordings, height
+        )
+
+        # Calculate evenly spaced timestamps throughout the duration
+        if desired_frame_count == 1:
+            timestamps = [start_time + duration / 2]
+        else:
+            step = duration / (desired_frame_count - 1)
+            timestamps = [start_time + (i * step) for i in range(desired_frame_count)]
+
+        def extract_frame_from_recording(ts: float) -> bytes | None:
+            """Extract a single frame from recording at given timestamp."""
+            try:
+                recording = (
+                    Recordings.select(
+                        Recordings.path,
+                        Recordings.start_time,
+                    )
+                    .where((ts >= Recordings.start_time) & (ts <= Recordings.end_time))
+                    .where(Recordings.camera == camera)
+                    .order_by(Recordings.start_time.desc())
+                    .limit(1)
+                    .get()
+                )
+
+                time_in_segment = ts - recording.start_time
+                return get_image_from_recording(
+                    self.config.ffmpeg,
+                    recording.path,
+                    time_in_segment,
+                    "mjpeg",
+                    height=height,
+                )
+            except DoesNotExist:
+                return None
+
+        frames = []
+
+        for timestamp in timestamps:
+            try:
+                # Try to extract frame at exact timestamp
+                image_data = extract_frame_from_recording(timestamp)
+
+                if not image_data:
+                    # Try with rounded timestamp as fallback
+                    rounded_timestamp = math.ceil(timestamp)
+                    image_data = extract_frame_from_recording(rounded_timestamp)
+
+                if image_data:
+                    frames.append(image_data)
+                else:
+                    logger.warning(
+                        f"No recording found for {camera} at timestamp {timestamp}"
+                    )
+            except Exception as e:
+                logger.error(
+                    f"Error extracting frame from recording for {camera} at {timestamp}: {e}"
+                )
+                continue
+
+        return frames
+
+    def get_preview_frames_as_bytes(
+        self,
+        camera: str,
+        start_time: float,
+        end_time: float,
+        thumb_path_fallback: str,
+        review_id: str,
+        save_debug: bool,
+    ) -> list[bytes]:
+        """Get preview frames and convert them to JPEG bytes.
+
+        Args:
+            camera: Camera name
+            start_time: Start timestamp
+            end_time: End timestamp
+            thumb_path_fallback: Fallback thumbnail path if no preview frames found
+            review_id: Review item ID for debug saving
+            save_debug: Whether to save debug thumbnails
+
+        Returns:
+            List of JPEG image bytes
+        """
+        frame_paths = self.get_cache_frames(camera, start_time, end_time)
+        if not frame_paths:
+            frame_paths = [thumb_path_fallback]
+
+        thumbs = []
+        for idx, thumb_path in enumerate(frame_paths):
+            thumb_data = cv2.imread(thumb_path)
+
+            if thumb_data is None:
+                logger.warning(  # type: ignore[unreachable]
+                    "Could not read preview frame at %s, skipping", thumb_path
+                )
+                continue
+
+            ret, jpg = cv2.imencode(
+                ".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100]
+            )
+            if ret:
+                thumbs.append(jpg.tobytes())
+
+            if save_debug:
+                Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
+                    parents=True, exist_ok=True
+                )
+                shutil.copy(
+                    thumb_path,
+                    os.path.join(CLIPS_DIR, f"genai-requests/{review_id}/{idx}.webp"),
+                )
+
+        return thumbs
+
+
+def run_analysis(
+    requestor: InterProcessRequestor,
+    genai_client: GenAIClient,
+    review_inference_speed: InferenceSpeed,
+    camera_config: CameraConfig,
+    final_data: dict[str, Any],
+    thumbs: list[bytes],
+    genai_config: GenAIReviewConfig,
+    labelmap_objects: list[str],
+    attribute_labels: list[str],
+) -> None:
+    start = datetime.datetime.now().timestamp()
+
+    # Format zone names using zone config friendly names if available
+    formatted_zones = []
+    for zone_name in final_data["data"]["zones"]:
+        if zone_name in camera_config.zones:
+            formatted_zones.append(
+                camera_config.zones[zone_name].get_formatted_name(zone_name)
+            )
+
+    analytics_data = {
+        "id": final_data["id"],
+        "camera": camera_config.get_formatted_name(),
+        "zones": formatted_zones,
+        "start": datetime.datetime.fromtimestamp(final_data["start_time"]).strftime(
+            "%A, %I:%M %p"
+        ),
+        "duration": round(final_data["end_time"] - final_data["start_time"]),
+    }
+
+    unified_objects = []
+
+    objects_list = final_data["data"]["objects"]
+    sub_labels_list = final_data["data"]["sub_labels"]
+
+    for i, verified_label in enumerate(final_data["data"]["verified_objects"]):
+        object_type = verified_label.replace("-verified", "").replace("_", " ")
+        name = titlecase(sub_labels_list[i].replace("_", " "))
+        unified_objects.append(f"{name} ← {object_type}")
+
+    for label in objects_list:
+        if "-verified" in label:
+            continue
+        elif label in labelmap_objects:
+            object_type = label.replace("_", " ")
+
+            if label in attribute_labels:
+                display_name = ATTRIBUTE_LABEL_DISPLAY_MAP.get(label, object_type)
+                unified_objects.append(f"{display_name} (delivery/service)")
+            else:
+                unified_objects.append(object_type)
+
+    analytics_data["unified_objects"] = unified_objects
+
+    metadata = genai_client.generate_review_description(
+        analytics_data,
+        thumbs,
+        genai_config.additional_concerns,
+        genai_config.preferred_language,
+        genai_config.debug_save_thumbnails,
+        genai_config.activity_context_prompt,
+    )
+    review_inference_speed.update(datetime.datetime.now().timestamp() - start)
+
+    if not metadata:
+        return None
+
+    prev_data = copy.deepcopy(final_data)
+    final_data["data"]["metadata"] = metadata.model_dump()
+    requestor.send_data(
+        UPDATE_REVIEW_DESCRIPTION,
+        {
+            "type": "genai",
+            "before": {k: v for k, v in prev_data.items()},
+            "after": {k: v for k, v in final_data.items()},
+        },
+    )

frigate/data_processing/post/semantic_trigger.py

@@ -0,0 +1,275 @@
+"""Post time processor to trigger actions based on similar embeddings."""
+
+import datetime
+import json
+import logging
+import os
+from typing import Any
+
+import cv2
+import numpy as np
+from peewee import DoesNotExist
+
+from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
+    EventMetadataTypeEnum,
+)
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.const import CONFIG_DIR
+from frigate.data_processing.types import PostProcessDataEnum
+from frigate.db.sqlitevecq import SqliteVecQueueDatabase
+from frigate.embeddings.embeddings import Embeddings
+from frigate.embeddings.util import ZScoreNormalization
+from frigate.models import Event, Trigger
+from frigate.util.builtin import cosine_distance
+from frigate.util.file import get_event_thumbnail_bytes
+
+from ..post.api import PostProcessorApi
+from ..types import DataProcessorMetrics
+
+logger = logging.getLogger(__name__)
+
+WRITE_DEBUG_IMAGES = False
+
+
+class SemanticTriggerProcessor(PostProcessorApi):
+    def __init__(
+        self,
+        db: SqliteVecQueueDatabase,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+        embeddings: Embeddings,
+    ) -> None:
+        super().__init__(config, metrics, None)
+        self.db = db
+        self.embeddings = embeddings
+        self.requestor = requestor
+        self.sub_label_publisher = sub_label_publisher
+        self.trigger_embeddings: list[np.ndarray] = []
+
+        self.thumb_stats = ZScoreNormalization()
+        self.desc_stats = ZScoreNormalization()
+
+        # load stats from disk
+        try:
+            with open(os.path.join(CONFIG_DIR, ".search_stats.json")) as f:
+                data = json.loads(f.read())
+                self.thumb_stats.from_dict(data["thumb_stats"])
+                self.desc_stats.from_dict(data["desc_stats"])
+        except FileNotFoundError:
+            pass
+
+    def process_data(
+        self, data: dict[str, Any], data_type: PostProcessDataEnum
+    ) -> None:
+        event_id = data["event_id"]
+        camera = data["camera"]
+        process_type = data["type"]
+
+        if self.config.cameras[camera].semantic_search.triggers is None:
+            return
+
+        triggers = (
+            Trigger.select(
+                Trigger.camera,
+                Trigger.name,
+                Trigger.data,
+                Trigger.type,
+                Trigger.embedding,
+                Trigger.threshold,
+            )
+            .where(Trigger.camera == camera)
+            .dicts()
+            .iterator()
+        )
+
+        for trigger in triggers:
+            if (
+                trigger["name"]
+                not in self.config.cameras[camera].semantic_search.triggers
+                or not self.config.cameras[camera]
+                .semantic_search.triggers[trigger["name"]]
+                .enabled
+            ):
+                logger.debug(
+                    f"Trigger {trigger['name']} is disabled for camera {camera}"
+                )
+                continue
+
+            logger.debug(
+                f"Processing {trigger['type']} trigger for {event_id} on {trigger['camera']}: {trigger['name']}"
+            )
+
+            trigger_embedding = np.frombuffer(trigger["embedding"], dtype=np.float32)
+
+            # Get embeddings based on type
+            thumbnail_embedding = None
+            description_embedding = None
+
+            if process_type == "image":
+                cursor = self.db.execute_sql(
+                    """
+                    SELECT thumbnail_embedding FROM vec_thumbnails WHERE id = ?
+                    """,
+                    [event_id],
+                )
+                row = cursor.fetchone() if cursor else None
+                if row:
+                    thumbnail_embedding = np.frombuffer(row[0], dtype=np.float32)
+
+            if process_type == "text":
+                cursor = self.db.execute_sql(
+                    """
+                    SELECT description_embedding FROM vec_descriptions WHERE id = ?
+                    """,
+                    [event_id],
+                )
+                row = cursor.fetchone() if cursor else None
+                if row:
+                    description_embedding = np.frombuffer(row[0], dtype=np.float32)
+
+            # Skip processing if we don't have any embeddings
+            if thumbnail_embedding is None and description_embedding is None:
+                logger.debug(f"No embeddings found for {event_id}")
+                return
+
+            # Determine which embedding to compare based on trigger type
+            if (
+                trigger["type"] in ["text", "thumbnail"]
+                and thumbnail_embedding is not None
+            ):
+                data_embedding = thumbnail_embedding
+                normalized_distance = self.thumb_stats.normalize(
+                    [cosine_distance(data_embedding, trigger_embedding)],
+                    save_stats=False,
+                )[0]
+            elif trigger["type"] == "description" and description_embedding is not None:
+                data_embedding = description_embedding
+                normalized_distance = self.desc_stats.normalize(
+                    [cosine_distance(data_embedding, trigger_embedding)],
+                    save_stats=False,
+                )[0]
+
+            else:
+                continue
+
+            similarity = 1 - normalized_distance
+
+            logger.debug(
+                f"Trigger {trigger['name']} ({trigger['data'] if trigger['type'] == 'text' or trigger['type'] == 'description' else 'image'}): "
+                f"normalized distance: {normalized_distance:.4f}, "
+                f"similarity: {similarity:.4f}, threshold: {trigger['threshold']}"
+            )
+
+            # Check if similarity meets threshold
+            if similarity >= trigger["threshold"]:
+                logger.debug(
+                    f"Trigger {trigger['name']} activated with similarity {similarity:.4f}"
+                )
+
+                # Update the trigger's last_triggered and triggering_event_id
+                Trigger.update(
+                    last_triggered=datetime.datetime.now(), triggering_event_id=event_id
+                ).where(
+                    Trigger.camera == camera, Trigger.name == trigger["name"]
+                ).execute()
+
+                # Always publish MQTT message
+                self.requestor.send_data(
+                    "triggers",
+                    json.dumps(
+                        {
+                            "name": trigger["name"],
+                            "camera": camera,
+                            "event_id": event_id,
+                            "type": trigger["type"],
+                            "score": similarity,
+                        }
+                    ),
+                )
+
+                friendly_name = (
+                    self.config.cameras[camera]
+                    .semantic_search.triggers[trigger["name"]]
+                    .friendly_name
+                )
+
+                if (
+                    self.config.cameras[camera]
+                    .semantic_search.triggers[trigger["name"]]
+                    .actions
+                ):
+                    # handle actions for the trigger
+                    # notifications already handled by webpush
+                    if (
+                        "sub_label"
+                        in self.config.cameras[camera]
+                        .semantic_search.triggers[trigger["name"]]
+                        .actions
+                    ):
+                        self.sub_label_publisher.publish(
+                            (event_id, friendly_name, similarity),
+                            EventMetadataTypeEnum.sub_label,
+                        )
+                    if (
+                        "attribute"
+                        in self.config.cameras[camera]
+                        .semantic_search.triggers[trigger["name"]]
+                        .actions
+                    ):
+                        self.sub_label_publisher.publish(
+                            (
+                                event_id,
+                                trigger["name"],
+                                trigger["type"],
+                                similarity,
+                            ),
+                            EventMetadataTypeEnum.attribute.value,
+                        )
+
+            if WRITE_DEBUG_IMAGES:
+                try:
+                    event: Event = Event.get(Event.id == event_id)
+                except DoesNotExist:
+                    return
+
+                # Skip the event if not an object
+                if event.data.get("type") != "object":  # type: ignore[attr-defined]
+                    return
+
+                thumbnail_bytes = get_event_thumbnail_bytes(event)
+
+                if thumbnail_bytes is None:
+                    return
+
+                nparr = np.frombuffer(thumbnail_bytes, np.uint8)
+                thumbnail = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+
+                font_scale = 0.5
+                font = cv2.FONT_HERSHEY_SIMPLEX
+                cv2.putText(
+                    thumbnail,
+                    f"{similarity:.4f}",
+                    (10, 30),
+                    font,
+                    fontScale=font_scale,
+                    color=(0, 255, 0),
+                    thickness=2,
+                )
+
+                current_time = int(datetime.datetime.now().timestamp())
+                cv2.imwrite(
+                    f"debug/frames/trigger-{event_id}_{current_time}.jpg",
+                    thumbnail,
+                )
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | str | None:
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        pass

frigate/data_processing/post/types.py

@@ -0,0 +1,45 @@
+from typing import Annotated
+
+from pydantic import BaseModel, ConfigDict, Field, StringConstraints
+
+ObservationItem = Annotated[str, StringConstraints(min_length=20, max_length=200)]
+
+
+class ReviewMetadata(BaseModel):
+    model_config = ConfigDict(extra="ignore", protected_namespaces=())
+
+    observations: list[ObservationItem] = Field(
+        ...,
+        min_length=3,
+        max_length=8,
+        description="Enumerate the significant observations across all frames, in chronological order.",
+    )
+    scene: str = Field(
+        min_length=150,
+        max_length=600,
+        description="A chronological narrative of what happens from start to finish, drawing directly from the items in observations.",
+    )
+    title: str = Field(
+        max_length=80,
+        description="Title for the activity.",
+    )
+    shortSummary: str = Field(
+        min_length=70,
+        max_length=140,
+        description="A brief summary for the activity.",
+    )
+    confidence: float = Field(
+        ge=0.0,
+        le=1.0,
+        description="Confidence in the analysis as a decimal between 0.0 and 1.0, where 0.0 means no confidence and 1.0 means complete confidence. Express ONLY as a decimal.",
+    )
+    potential_threat_level: int = Field(
+        ge=0,
+        le=2,
+        description="Threat level: 0 = normal, 1 = suspicious, 2 = critical threat.",
+    )
+    other_concerns: list[str] | None = Field(
+        default=None,
+        description="Other concerns highlighted by the user that are observed.",
+    )
+    time: str | None = Field(default=None, description="Time of activity.")

frigate/data_processing/real_time/api.py

@@ -0,0 +1,201 @@
+"""Local only processors for handling real time object processing."""
+
+import logging
+import threading
+from abc import ABC, abstractmethod
+from collections import deque
+from collections.abc import Callable
+from concurrent.futures import Future
+from queue import Empty, Full, Queue
+from typing import Any
+
+import numpy as np
+
+from frigate.config import FrigateConfig
+
+from ..types import DataProcessorMetrics
+
+logger = logging.getLogger(__name__)
+
+
+class RealTimeProcessorApi(ABC):
+    @abstractmethod
+    def __init__(
+        self,
+        config: FrigateConfig,
+        metrics: DataProcessorMetrics,
+    ) -> None:
+        self.config = config
+        self.metrics = metrics
+        pass
+
+    @abstractmethod
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        """Processes the frame with object data.
+        Args:
+            obj_data (dict): containing data about focused object in frame.
+            frame (ndarray): full yuv frame.
+
+        Returns:
+            None.
+        """
+        pass
+
+    @abstractmethod
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        """Handle metadata requests.
+        Args:
+            topic (str): topic that dictates what work is requested.
+            request_data (dict): containing data about requested change to process.
+
+        Returns:
+            None if request was not handled, otherwise return response.
+        """
+        pass
+
+    @abstractmethod
+    def expire_object(self, object_id: str, camera: str) -> None:
+        """Handle objects that are no longer detected.
+        Args:
+            object_id (str): id of object that is no longer detected.
+            camera (str): name of camera that object was detected on.
+
+        Returns:
+            None.
+        """
+        pass
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Handle a config change notification.
+
+        Called for every config update published under ``config/``.
+        Processors should override this to check the topic and act only
+        on changes relevant to them. Default is a no-op.
+
+        Args:
+            topic: The config topic that changed.
+            payload: The updated configuration object.
+        """
+        pass
+
+    def drain_results(self) -> list[dict[str, Any]]:
+        """Return pending results that need IPC side-effects.
+
+        Deferred processors accumulate results on a worker thread.
+        The maintainer calls this each loop iteration to collect them
+        and perform publishes on the main thread.
+
+        Synchronous processors return an empty list (default).
+        """
+        return []
+
+    def shutdown(self) -> None:
+        """Stop any background work and release resources.
+
+        Called when the processor is being removed or the maintainer
+        is shutting down. Default is a no-op for synchronous processors.
+        """
+        pass
+
+
+class DeferredRealtimeProcessorApi(RealTimeProcessorApi):
+    """Base class for processors that offload heavy work to a background thread.
+
+    Subclasses implement:
+      - process_frame(): do cheap gating + crop + copy, then call _enqueue_task()
+      - _process_task(task): heavy work (inference, consensus) on the worker thread
+      - handle_request(): optionally use _enqueue_request() for sync request/response
+      - expire_object(): call _enqueue_task() with a control message
+
+    The worker thread owns all processor state. No locks are needed because
+    only the worker mutates state. Results that need IPC are placed in
+    _pending_results via _emit_result(), and the maintainer drains them
+    each loop iteration.
+    """
+
+    def __init__(
+        self,
+        config: FrigateConfig,
+        metrics: DataProcessorMetrics,
+        max_queue: int = 8,
+    ) -> None:
+        super().__init__(config, metrics)
+        self._task_queue: Queue = Queue(maxsize=max_queue)
+        self._pending_results: deque[dict[str, Any]] = deque()
+        self._results_lock = threading.Lock()
+        self._stop_event = threading.Event()
+        self._worker = threading.Thread(
+            target=self._drain_loop,
+            daemon=True,
+            name=f"{type(self).__name__}_worker",
+        )
+        self._worker.start()
+
+    def _drain_loop(self) -> None:
+        """Worker thread main loop — drains the task queue until stopped."""
+        while not self._stop_event.is_set():
+            try:
+                task = self._task_queue.get(timeout=0.5)
+            except Empty:
+                continue
+
+            if (
+                isinstance(task, tuple)
+                and len(task) == 2
+                and isinstance(task[1], Future)
+            ):
+                # Request/response: (callable_and_args, future)
+                (func, args), future = task
+                try:
+                    result = func(args)
+                    future.set_result(result)
+                except Exception as e:
+                    future.set_exception(e)
+            else:
+                try:
+                    self._process_task(task)
+                except Exception:
+                    logger.exception("Error processing deferred task")
+
+    def _enqueue_task(self, task: Any) -> bool:
+        """Enqueue a task for the worker. Returns False if queue is full (dropped)."""
+        try:
+            self._task_queue.put_nowait(task)
+            return True
+        except Full:
+            logger.debug("Deferred processor queue full, dropping task")
+            return False
+
+    def _enqueue_request(self, func: Callable, args: Any, timeout: float = 10.0) -> Any:
+        """Enqueue a request and block until the worker returns a result."""
+        future: Future = Future()
+        self._task_queue.put(((func, args), future), timeout=timeout)
+        return future.result(timeout=timeout)
+
+    def _emit_result(self, result: dict[str, Any]) -> None:
+        """Called by the worker thread to stage a result for the maintainer."""
+        with self._results_lock:
+            self._pending_results.append(result)
+
+    def drain_results(self) -> list[dict[str, Any]]:
+        """Called by the maintainer on the main thread to collect pending results."""
+        with self._results_lock:
+            results = list(self._pending_results)
+            self._pending_results.clear()
+        return results
+
+    def shutdown(self) -> None:
+        """Signal the worker to stop and wait for it to finish."""
+        self._stop_event.set()
+        self._worker.join(timeout=5.0)
+
+    @abstractmethod
+    def _process_task(self, task: Any) -> None:
+        """Process a single task on the worker thread.
+
+        Subclasses implement inference, consensus, training image saves here.
+        Call _emit_result() to stage results for the maintainer to publish.
+        """
+        pass

frigate/data_processing/real_time/audio_transcription.py

@@ -0,0 +1,279 @@
+"""Handle processing audio for speech transcription using sherpa-onnx with FFmpeg pipe."""
+
+import logging
+import os
+import queue
+import threading
+from typing import Any
+
+import numpy as np
+
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import CameraConfig, FrigateConfig
+from frigate.const import MODEL_CACHE_DIR
+from frigate.data_processing.common.audio_transcription.model import (
+    AudioTranscriptionModelRunner,
+)
+from frigate.data_processing.real_time.whisper_online import (
+    FasterWhisperASR,
+    OnlineASRProcessor,
+)
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        camera_config: CameraConfig,
+        requestor: InterProcessRequestor,
+        model_runner: AudioTranscriptionModelRunner,
+        metrics: DataProcessorMetrics,
+        stop_event: threading.Event,
+    ):
+        super().__init__(config, metrics)
+        self.config = config
+        self.camera_config = camera_config
+        self.requestor = requestor
+        self.stream: Any = None
+        self.whisper_model: FasterWhisperASR | None = None
+        self.model_runner = model_runner
+        self.transcription_segments: list[str] = []
+        self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue()
+        self.stop_event = stop_event
+
+    def __build_recognizer(self) -> None:
+        try:
+            if self.config.audio_transcription.model_size == "large":
+                # Whisper models need to be per-process and can only run one stream at a time
+                # TODO: try parallel: https://github.com/SYSTRAN/faster-whisper/issues/100
+                logger.debug(f"Loading Whisper model for {self.camera_config.name}")
+                self.whisper_model = FasterWhisperASR(
+                    modelsize="tiny",
+                    device="cuda"
+                    if self.config.audio_transcription.device == "GPU"
+                    else "cpu",
+                    lan=self.config.audio_transcription.language,
+                    model_dir=os.path.join(MODEL_CACHE_DIR, "whisper"),
+                )
+                self.whisper_model.use_vad()
+                self.stream = OnlineASRProcessor(
+                    asr=self.whisper_model,
+                )
+            else:
+                logger.debug(f"Loading sherpa stream for {self.camera_config.name}")
+                self.stream = self.model_runner.model.create_stream()
+            logger.debug(
+                f"Audio transcription (live) initialized for {self.camera_config.name}"
+            )
+        except Exception as e:
+            logger.error(
+                f"Failed to initialize live streaming audio transcription: {e}"
+            )
+
+    def __process_audio_stream(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
+        if (
+            self.model_runner.model is None
+            and self.config.audio_transcription.model_size == "small"
+        ):
+            logger.debug("Audio transcription (live) model not initialized")
+            return None
+
+        if not self.stream:
+            self.__build_recognizer()
+
+        try:
+            if audio_data.dtype != np.float32:
+                audio_data = audio_data.astype(np.float32)
+
+            if audio_data.max() > 1.0 or audio_data.min() < -1.0:
+                audio_data = audio_data / 32768.0  # Normalize from int16
+
+            rms = float(np.sqrt(np.mean(np.absolute(np.square(audio_data)))))
+            logger.debug(f"Audio chunk size: {audio_data.size}, RMS: {rms:.4f}")
+
+            if self.config.audio_transcription.model_size == "large":
+                # large model
+                self.stream.insert_audio_chunk(audio_data)
+                output = self.stream.process_iter()
+                text = output[2].strip()
+                is_endpoint = (
+                    text.endswith((".", "!", "?"))
+                    and sum(len(str(lines)) for lines in self.transcription_segments)
+                    > 300
+                )
+
+                if text:
+                    self.transcription_segments.append(text)
+                concatenated_text = " ".join(self.transcription_segments)
+                logger.debug(f"Concatenated transcription: '{concatenated_text}'")
+                text = concatenated_text
+
+            else:
+                # small model
+                self.stream.accept_waveform(16000, audio_data)
+
+                while self.model_runner.model.is_ready(self.stream):
+                    self.model_runner.model.decode_stream(self.stream)
+
+                text = self.model_runner.model.get_result(self.stream).strip()
+                is_endpoint = self.model_runner.model.is_endpoint(self.stream)
+
+            logger.debug(f"Transcription result: '{text}'")
+
+            if not text:
+                logger.debug("No transcription, returning")
+                return None
+
+            logger.debug(f"Endpoint detected: {is_endpoint}")
+
+            if is_endpoint and self.config.audio_transcription.model_size == "small":
+                # reset sherpa if we've reached an endpoint
+                self.model_runner.model.reset(self.stream)
+
+            return text, is_endpoint
+        except Exception as e:
+            logger.error(f"Error processing audio stream: {e}")
+            return None
+
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        pass
+
+    def process_audio(self, obj_data: dict[str, Any], audio: np.ndarray) -> bool | None:
+        if audio is None or audio.size == 0:
+            logger.debug("No audio data provided for transcription")
+            return None
+
+        # enqueue audio data for processing in the thread
+        self.audio_queue.put((obj_data, audio))
+        return None
+
+    def run(self) -> None:
+        """Run method for the transcription thread to process queued audio data."""
+        logger.debug(
+            f"Starting audio transcription thread for {self.camera_config.name}"
+        )
+
+        # start with an empty transcription
+        self.requestor.send_data(
+            f"{self.camera_config.name}/audio/transcription",
+            "",
+        )
+
+        while not self.stop_event.is_set():
+            try:
+                # Get audio data from queue with a timeout to check stop_event
+                _, audio = self.audio_queue.get(timeout=0.1)
+                result = self.__process_audio_stream(audio)
+
+                if not result:
+                    continue
+
+                text, is_endpoint = result
+                logger.debug(f"Transcribed audio: '{text}', Endpoint: {is_endpoint}")
+
+                self.requestor.send_data(
+                    f"{self.camera_config.name}/audio/transcription", text
+                )
+
+                self.audio_queue.task_done()
+
+                if is_endpoint:
+                    self.reset()
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.error(f"Error processing audio in thread: {e}")
+                self.audio_queue.task_done()
+
+        logger.debug(
+            f"Stopping audio transcription thread for {self.camera_config.name}"
+        )
+
+    def clear_audio_queue(self) -> None:
+        # Clear the audio queue
+        while not self.audio_queue.empty():
+            try:
+                self.audio_queue.get_nowait()
+                self.audio_queue.task_done()
+            except queue.Empty:
+                break
+
+    def reset(self) -> None:
+        if self.config.audio_transcription.model_size == "large":
+            # get final output from whisper
+            output = self.stream.finish()
+            self.transcription_segments = []
+
+            self.requestor.send_data(
+                f"{self.camera_config.name}/audio/transcription",
+                (output[2].strip() + " "),
+            )
+
+            # reset whisper
+            self.stream.init()
+            self.transcription_segments = []
+        else:
+            # reset sherpa
+            self.model_runner.model.reset(self.stream)
+
+        logger.debug("Stream reset")
+
+    def check_unload_model(self) -> None:
+        # regularly called in the loop in audio maintainer
+        if (
+            self.config.audio_transcription.model_size == "large"
+            and self.whisper_model is not None
+        ):
+            logger.debug(f"Unloading Whisper model for {self.camera_config.name}")
+            self.clear_audio_queue()
+            self.transcription_segments = []
+            self.stream = None
+            self.whisper_model = None
+
+            self.requestor.send_data(
+                f"{self.camera_config.name}/audio/transcription",
+                "",
+            )
+        if (
+            self.config.audio_transcription.model_size == "small"
+            and self.stream is not None
+        ):
+            logger.debug(f"Clearing sherpa stream for {self.camera_config.name}")
+            self.stream = None
+
+            self.requestor.send_data(
+                f"{self.camera_config.name}/audio/transcription",
+                "",
+            )
+
+    def stop(self) -> None:
+        """Stop the transcription thread and clean up."""
+        self.stop_event.set()
+        # Clear the queue to prevent processing stale data
+        while not self.audio_queue.empty():
+            try:
+                self.audio_queue.get_nowait()
+                self.audio_queue.task_done()
+            except queue.Empty:
+                break
+        logger.debug(
+            f"Transcription thread stop signaled for {self.camera_config.name}"
+        )
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if topic == "clear_audio_recognizer":
+            self.stream = None
+            self.__build_recognizer()
+            return {"message": "Audio recognizer cleared and rebuilt", "success": True}
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        pass

frigate/data_processing/real_time/bird.py

@@ -0,0 +1,193 @@
+"""Handle processing images to classify birds."""
+
+import logging
+import os
+from typing import Any
+
+import cv2
+import numpy as np
+
+from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
+    EventMetadataTypeEnum,
+)
+from frigate.config import FrigateConfig
+from frigate.const import MODEL_CACHE_DIR
+from frigate.log import suppress_stderr_during
+from frigate.util.image import calculate_region
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+try:
+    from tflite_runtime.interpreter import Interpreter
+except ModuleNotFoundError:
+    from ai_edge_litert.interpreter import Interpreter
+
+logger = logging.getLogger(__name__)
+
+
+class BirdRealTimeProcessor(RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics)
+        self.interpreter: Interpreter | None = None
+        self.sub_label_publisher = sub_label_publisher
+        self.tensor_input_details: list[dict[str, Any]] | None = None
+        self.tensor_output_details: list[dict[str, Any]] | None = None
+        self.detected_birds: dict[str, float] = {}
+        self.labelmap: dict[int, str] = {}
+
+        GITHUB_RAW_ENDPOINT = os.environ.get(
+            "GITHUB_RAW_ENDPOINT", "https://raw.githubusercontent.com"
+        )
+        download_path = os.path.join(MODEL_CACHE_DIR, "bird")
+        self.model_files = {
+            "bird.tflite": f"{GITHUB_RAW_ENDPOINT}/google-coral/test_data/master/mobilenet_v2_1.0_224_inat_bird_quant.tflite",
+            "birdmap.txt": f"{GITHUB_RAW_ENDPOINT}/google-coral/test_data/master/inat_bird_labels.txt",
+        }
+
+        if not all(
+            os.path.exists(os.path.join(download_path, n))
+            for n in self.model_files.keys()
+        ):
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            self.downloader = ModelDownloader(
+                model_name="bird",
+                download_path=download_path,
+                file_names=list(self.model_files.keys()),
+                download_func=self.__download_models,
+                complete_func=self.__build_detector,
+            )
+            self.downloader.ensure_model_files()
+        else:
+            self.__build_detector()
+
+    def __download_models(self, path: str) -> None:
+        try:
+            file_name = os.path.basename(path)
+
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            ModelDownloader.download_from_url(self.model_files[file_name], path)
+        except Exception as e:
+            logger.error(f"Failed to download {path}: {e}")
+
+    def __build_detector(self) -> None:
+        # Suppress TFLite delegate creation messages that bypass Python logging
+        with suppress_stderr_during("tflite_interpreter_init"):
+            self.interpreter = Interpreter(
+                model_path=os.path.join(MODEL_CACHE_DIR, "bird/bird.tflite"),
+                num_threads=2,
+            )
+            self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+
+        i = 0
+
+        with open(os.path.join(MODEL_CACHE_DIR, "bird/birdmap.txt")) as f:
+            line = f.readline()
+            while line:
+                start = line.find("(")
+                end = line.find(")")
+                self.labelmap[i] = line[start + 1 : end]
+                i += 1
+                line = f.readline()
+
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        if (
+            not self.interpreter
+            or not self.tensor_input_details
+            or not self.tensor_output_details
+        ):
+            return
+
+        if obj_data["label"] != "bird":
+            return
+
+        x, y, x2, y2 = calculate_region(
+            frame.shape,
+            obj_data["box"][0],
+            obj_data["box"][1],
+            obj_data["box"][2],
+            obj_data["box"][3],
+            int(
+                max(
+                    obj_data["box"][1] - obj_data["box"][0],
+                    obj_data["box"][3] - obj_data["box"][2],
+                )
+                * 1.1
+            ),
+            1.0,
+        )
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        input = rgb[
+            y:y2,
+            x:x2,
+        ]
+
+        if input.shape != (224, 224):
+            try:
+                input = cv2.resize(input, (224, 224))
+            except Exception:
+                logger.warning("Failed to resize image for bird classification")
+                return
+
+        input = np.expand_dims(input, axis=0)
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        probs = res / res.sum(axis=0)
+        best_id = int(np.argmax(probs))
+
+        if best_id == 964:
+            logger.debug("No bird classification was detected.")
+            return
+
+        score = round(probs[best_id], 2)
+
+        if score < self.config.classification.bird.threshold:
+            logger.debug(f"Score {score} is not above required threshold")
+            return
+
+        previous_score = self.detected_birds.get(obj_data["id"], 0.0)
+
+        if score <= previous_score:
+            logger.debug(f"Score {score} is worse than previous score {previous_score}")
+            return
+
+        self.sub_label_publisher.publish(
+            (obj_data["id"], self.labelmap[best_id], score),
+            EventMetadataTypeEnum.sub_label.value,
+        )
+        self.detected_birds[obj_data["id"]] = score
+
+    CONFIG_UPDATE_TOPIC = "config/classification"
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Update bird classification config at runtime."""
+        if topic != self.CONFIG_UPDATE_TOPIC:
+            return
+
+        self.config.classification = payload
+        logger.debug("Bird classification config updated dynamically")
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        if object_id in self.detected_birds:
+            self.detected_birds.pop(object_id)

frigate/data_processing/real_time/custom_classification.py

@@ -0,0 +1,744 @@
+"""Real time processor that works with classification tflite models."""
+
+import datetime
+import logging
+import os
+from typing import Any
+
+import cv2
+import numpy as np
+
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.config.classification import CustomClassificationConfig
+from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR
+from frigate.log import suppress_stderr_during
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels
+from frigate.util.image import calculate_region
+from frigate.util.object import box_overlaps
+
+from ..types import DataProcessorMetrics
+from .api import DeferredRealtimeProcessorApi
+
+try:
+    from tflite_runtime.interpreter import Interpreter
+except ModuleNotFoundError:
+    from ai_edge_litert.interpreter import Interpreter
+
+logger = logging.getLogger(__name__)
+
+MAX_OBJECT_CLASSIFICATIONS = 16
+
+
+class CustomStateClassificationProcessor(DeferredRealtimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        model_config: CustomClassificationConfig,
+        requestor: InterProcessRequestor,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics, max_queue=4)
+        self.model_config = model_config
+
+        if not self.model_config.name:
+            raise ValueError("Custom classification model name must be set.")
+
+        self.requestor = requestor
+        self.model_dir = os.path.join(MODEL_CACHE_DIR, self.model_config.name)
+        self.train_dir = os.path.join(CLIPS_DIR, self.model_config.name, "train")
+        self.interpreter: Interpreter | None = None
+        self.tensor_input_details: list[dict[str, Any]] | None = None
+        self.tensor_output_details: list[dict[str, Any]] | None = None
+        self.labelmap: dict[int, str] = {}
+        self.classifications_per_second = EventsPerSecond()
+        self.state_history: dict[str, dict[str, Any]] = {}
+
+        if (
+            self.metrics
+            and self.model_config.name in self.metrics.classification_speeds
+        ):
+            self.inference_speed: InferenceSpeed | None = InferenceSpeed(
+                self.metrics.classification_speeds[self.model_config.name]
+            )
+        else:
+            self.inference_speed = None
+
+        self.last_run = datetime.datetime.now().timestamp()
+        self.__build_detector()
+
+    def __build_detector(self) -> None:
+        model_path = os.path.join(self.model_dir, "model.tflite")
+        labelmap_path = os.path.join(self.model_dir, "labelmap.txt")
+
+        if not os.path.exists(model_path) or not os.path.exists(labelmap_path):
+            self.interpreter = None
+            self.tensor_input_details = None
+            self.tensor_output_details = None
+            self.labelmap = {}
+            return
+
+        # Suppress TFLite delegate creation messages that bypass Python logging
+        with suppress_stderr_during("tflite_interpreter_init"):
+            self.interpreter = Interpreter(
+                model_path=model_path,
+                num_threads=2,
+            )
+            self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+        self.labelmap = load_labels(labelmap_path, prefill=0, indexed=False)
+        self.classifications_per_second.start()
+
+    def __update_metrics(self, duration: float) -> None:
+        self.classifications_per_second.update()
+        if self.inference_speed:
+            self.inference_speed.update(duration)
+
+    def _should_save_image(
+        self, camera: str, detected_state: str, score: float = 1.0
+    ) -> bool:
+        """
+        Determine if we should save the image for training.
+        Save when:
+        - State is changing or being verified (regardless of score)
+        - Score is less than 100% (even if state matches, useful for training)
+        Don't save when:
+        - State is stable (matches current_state) AND score is 100%
+        """
+        if camera not in self.state_history:
+            # First detection for this camera, save it
+            return True
+
+        verification = self.state_history[camera]
+        current_state = verification.get("current_state")
+        pending_state = verification.get("pending_state")
+
+        # Save if there's a pending state change being verified
+        if pending_state is not None:
+            return True
+
+        # Save if the detected state differs from the current verified state
+        # (state is changing)
+        if current_state is not None and detected_state != current_state:
+            return True
+
+        # If score is less than 100%, save even if state matches
+        # (useful for training to improve confidence)
+        if score < 1.0:
+            return True
+
+        # Don't save if state is stable (detected_state == current_state) AND score is 100%
+        return False
+
+    def verify_state_change(self, camera: str, detected_state: str) -> str | None:
+        """
+        Verify state change requires 3 consecutive identical states before publishing.
+        Returns state to publish or None if verification not complete.
+        """
+        if camera not in self.state_history:
+            self.state_history[camera] = {
+                "current_state": None,
+                "pending_state": None,
+                "consecutive_count": 0,
+            }
+
+        verification = self.state_history[camera]
+
+        if detected_state == verification["current_state"]:
+            verification["pending_state"] = None
+            verification["consecutive_count"] = 0
+            return None
+
+        if detected_state == verification["pending_state"]:
+            verification["consecutive_count"] += 1
+
+            if verification["consecutive_count"] >= 3:
+                verification["current_state"] = detected_state
+                verification["pending_state"] = None
+                verification["consecutive_count"] = 0
+                return detected_state
+        else:
+            verification["pending_state"] = detected_state
+            verification["consecutive_count"] = 1
+            logger.debug(
+                f"New state '{detected_state}' detected for {camera}, need {3 - verification['consecutive_count']} more consecutive detections"
+            )
+
+        return None
+
+    def process_frame(self, frame_data: dict[str, Any], frame: np.ndarray) -> None:
+        if (
+            not self.model_config.name
+            or not self.model_config.state_config
+            or not self.tensor_input_details
+            or not self.tensor_output_details
+        ):
+            return
+
+        if self.metrics and self.model_config.name in self.metrics.classification_cps:
+            self.metrics.classification_cps[
+                self.model_config.name
+            ].value = self.classifications_per_second.eps()
+        camera = str(frame_data.get("camera"))
+
+        if camera not in self.model_config.state_config.cameras:
+            return
+
+        camera_config = self.model_config.state_config.cameras[camera]
+        crop = [
+            camera_config.crop[0] * self.config.cameras[camera].detect.width,
+            camera_config.crop[1] * self.config.cameras[camera].detect.height,
+            camera_config.crop[2] * self.config.cameras[camera].detect.width,
+            camera_config.crop[3] * self.config.cameras[camera].detect.height,
+        ]
+        should_run = False
+
+        now = datetime.datetime.now().timestamp()
+        if (
+            self.model_config.state_config.interval
+            and now > self.last_run + self.model_config.state_config.interval
+        ):
+            self.last_run = now
+            should_run = True
+
+        if (
+            not should_run
+            and self.model_config.state_config.motion
+            and any([box_overlaps(crop, mb) for mb in frame_data.get("motion", [])])
+        ):
+            # classification should run at most once per second
+            if now > self.last_run + 1:
+                self.last_run = now
+                should_run = True
+
+        # Shortcut: always run if we have a pending state verification to complete
+        if (
+            not should_run
+            and camera in self.state_history
+            and self.state_history[camera]["pending_state"] is not None
+            and now > self.last_run + 0.5
+        ):
+            self.last_run = now
+            should_run = True
+            logger.debug(
+                f"Running verification check for pending state: {self.state_history[camera]['pending_state']} ({self.state_history[camera]['consecutive_count']}/3)"
+            )
+
+        if not should_run:
+            return
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        height, width = rgb.shape[:2]
+
+        # Convert normalized crop coordinates to pixel values
+        x1 = int(camera_config.crop[0] * width)
+        y1 = int(camera_config.crop[1] * height)
+        x2 = int(camera_config.crop[2] * width)
+        y2 = int(camera_config.crop[3] * height)
+
+        # Clip coordinates to frame boundaries
+        x1 = max(0, min(x1, width))
+        y1 = max(0, min(y1, height))
+        x2 = max(0, min(x2, width))
+        y2 = max(0, min(y2, height))
+
+        if x2 <= x1 or y2 <= y1:
+            logger.warning(
+                f"Invalid crop coordinates for {camera}: [{x1}, {y1}, {x2}, {y2}]"
+            )
+            return
+
+        cropped_frame = rgb[y1:y2, x1:x2]
+
+        try:
+            resized_frame = cv2.resize(cropped_frame, (224, 224))
+        except Exception:
+            logger.warning("Failed to resize image for state classification")
+            return
+
+        # Copy for training image saves on worker thread
+        crop_bgr = cv2.cvtColor(cropped_frame, cv2.COLOR_RGB2BGR)
+
+        self._enqueue_task(("classify", camera, now, resized_frame, crop_bgr))
+
+    def _process_task(self, task: Any) -> None:
+        kind = task[0]
+        if kind == "classify":
+            _, camera, timestamp, resized_frame, crop_bgr = task
+            self._classify_state(camera, timestamp, resized_frame, crop_bgr)
+        elif kind == "reload":
+            self.__build_detector()
+
+    def _classify_state(
+        self,
+        camera: str,
+        timestamp: float,
+        resized_frame: np.ndarray,
+        crop_bgr: np.ndarray,
+    ) -> None:
+        if self.interpreter is None:
+            # When interpreter is None, always save (score is 0.0, which is < 1.0)
+            if self._should_save_image(camera, "unknown", 0.0):
+                save_attempts = (
+                    self.model_config.save_attempts
+                    if self.model_config.save_attempts is not None
+                    else 100
+                )
+                write_classification_attempt(
+                    self.train_dir,
+                    crop_bgr,
+                    "none-none",
+                    timestamp,
+                    "unknown",
+                    0.0,
+                    max_files=save_attempts,
+                )
+            return
+
+        if not self.tensor_input_details or not self.tensor_output_details:
+            return
+
+        input = np.expand_dims(resized_frame, axis=0)
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        probs = res / res.sum(axis=0)
+        logger.debug(
+            f"{self.model_config.name} Ran state classification with probabilities: {probs}"
+        )
+        best_id = int(np.argmax(probs))
+        score = round(probs[best_id], 2)
+        self.__update_metrics(datetime.datetime.now().timestamp() - timestamp)
+
+        detected_state = self.labelmap[best_id]
+
+        if self._should_save_image(camera, detected_state, score):
+            save_attempts = (
+                self.model_config.save_attempts
+                if self.model_config.save_attempts is not None
+                else 100
+            )
+            write_classification_attempt(
+                self.train_dir,
+                crop_bgr,
+                "none-none",
+                timestamp,
+                detected_state,
+                score,
+                max_files=save_attempts,
+            )
+
+        if score < self.model_config.threshold:
+            logger.debug(
+                f"Score {score} below threshold {self.model_config.threshold}, skipping verification"
+            )
+            return
+
+        verified_state = self.verify_state_change(camera, detected_state)
+
+        if verified_state is not None:
+            self._emit_result(
+                {
+                    "type": "classification",
+                    "processor": "state",
+                    "model_name": self.model_config.name,
+                    "camera": camera,
+                    "state": verified_state,
+                }
+            )
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if topic == EmbeddingsRequestEnum.reload_classification_model.value:
+            if request_data.get("model_name") == self.model_config.name:
+
+                def _do_reload(data: dict[str, Any]) -> dict[str, Any]:
+                    self.__build_detector()
+                    logger.info(
+                        f"Successfully loaded updated model for {self.model_config.name}"
+                    )
+                    return {
+                        "success": True,
+                        "message": f"Loaded {self.model_config.name} model.",
+                    }
+
+                result: dict[str, Any] = self._enqueue_request(_do_reload, request_data)
+                return result
+            else:
+                return None
+        else:
+            return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        pass
+
+
+class CustomObjectClassificationProcessor(DeferredRealtimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        model_config: CustomClassificationConfig,
+        sub_label_publisher: EventMetadataPublisher,
+        requestor: InterProcessRequestor,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics, max_queue=8)
+        self.model_config = model_config
+
+        if not self.model_config.name:
+            raise ValueError("Custom classification model name must be set.")
+
+        self.model_dir = os.path.join(MODEL_CACHE_DIR, self.model_config.name)
+        self.train_dir = os.path.join(CLIPS_DIR, self.model_config.name, "train")
+        self.interpreter: Interpreter | None = None
+        self.sub_label_publisher = sub_label_publisher
+        self.requestor = requestor
+        self.tensor_input_details: list[dict[str, Any]] | None = None
+        self.tensor_output_details: list[dict[str, Any]] | None = None
+        self.classification_history: dict[str, list[tuple[str, float, float]]] = {}
+        self.labelmap: dict[int, str] = {}
+        self.classifications_per_second = EventsPerSecond()
+
+        if (
+            self.metrics
+            and self.model_config.name in self.metrics.classification_speeds
+        ):
+            self.inference_speed: InferenceSpeed | None = InferenceSpeed(
+                self.metrics.classification_speeds[self.model_config.name]
+            )
+        else:
+            self.inference_speed = None
+
+        self.__build_detector()
+
+    def __build_detector(self) -> None:
+        model_path = os.path.join(self.model_dir, "model.tflite")
+        labelmap_path = os.path.join(self.model_dir, "labelmap.txt")
+
+        if not os.path.exists(model_path) or not os.path.exists(labelmap_path):
+            self.interpreter = None
+            self.tensor_input_details = None
+            self.tensor_output_details = None
+            self.labelmap = {}
+            return
+
+        # Suppress TFLite delegate creation messages that bypass Python logging
+        with suppress_stderr_during("tflite_interpreter_init"):
+            self.interpreter = Interpreter(
+                model_path=model_path,
+                num_threads=2,
+            )
+            self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+        self.labelmap = load_labels(labelmap_path, prefill=0, indexed=False)
+
+    def __update_metrics(self, duration: float) -> None:
+        self.classifications_per_second.update()
+        if self.inference_speed:
+            self.inference_speed.update(duration)
+
+    def get_weighted_score(
+        self,
+        object_id: str,
+        current_label: str,
+        current_score: float,
+        current_time: float,
+    ) -> tuple[str | None, float]:
+        """
+        Determine weighted score based on history to prevent false positives/negatives.
+        Requires 60% of attempts to agree on a label before publishing.
+        Returns (weighted_label, weighted_score) or (None, 0.0) if no weighted score.
+        """
+        if object_id not in self.classification_history:
+            self.classification_history[object_id] = []
+            logger.debug(f"Created new classification history for {object_id}")
+
+        self.classification_history[object_id].append(
+            (current_label, current_score, current_time)
+        )
+
+        history = self.classification_history[object_id]
+        logger.debug(
+            f"History for {object_id}: {len(history)} entries, latest=({current_label}, {current_score})"
+        )
+
+        if len(history) < 3:
+            logger.debug(
+                f"History for {object_id} has {len(history)} entries, need at least 3"
+            )
+            return None, 0.0
+
+        label_counts: dict[str, int] = {}
+        label_scores: dict[str, list[float]] = {}
+        total_attempts = len(history)
+
+        for label, score, timestamp in history:
+            if label not in label_counts:
+                label_counts[label] = 0
+                label_scores[label] = []
+
+            label_counts[label] += 1
+            label_scores[label].append(score)
+
+        best_label = max(label_counts, key=lambda k: label_counts[k])
+        best_count = label_counts[best_label]
+
+        consensus_threshold = total_attempts * 0.6
+        logger.debug(
+            f"Consensus calc for {object_id}: label_counts={label_counts}, "
+            f"best_label={best_label}, best_count={best_count}, "
+            f"total={total_attempts}, threshold={consensus_threshold}"
+        )
+
+        if best_count < consensus_threshold:
+            logger.debug(
+                f"No consensus for {object_id}: {best_count} < {consensus_threshold}"
+            )
+            return None, 0.0
+
+        avg_score = sum(label_scores[best_label]) / len(label_scores[best_label])
+
+        if best_label == "none":
+            logger.debug(f"Filtering 'none' label for {object_id}")
+            return None, 0.0
+
+        logger.debug(
+            f"Consensus reached for {object_id}: {best_label} with avg_score={avg_score}"
+        )
+        return best_label, avg_score
+
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        if (
+            not self.model_config.name
+            or not self.model_config.object_config
+            or not self.tensor_input_details
+            or not self.tensor_output_details
+        ):
+            return
+
+        if self.metrics and self.model_config.name in self.metrics.classification_cps:
+            self.metrics.classification_cps[
+                self.model_config.name
+            ].value = self.classifications_per_second.eps()
+
+        if obj_data["false_positive"]:
+            return
+
+        if obj_data["label"] not in self.model_config.object_config.objects:
+            return
+
+        if obj_data.get("end_time") is not None:
+            return
+
+        object_id = obj_data["id"]
+
+        if (
+            object_id in self.classification_history
+            and len(self.classification_history[object_id])
+            >= MAX_OBJECT_CLASSIFICATIONS
+        ):
+            return
+
+        now = datetime.datetime.now().timestamp()
+        x, y, x2, y2 = calculate_region(
+            frame.shape,
+            obj_data["box"][0],
+            obj_data["box"][1],
+            obj_data["box"][2],
+            obj_data["box"][3],
+            max(
+                obj_data["box"][2] - obj_data["box"][0],
+                obj_data["box"][3] - obj_data["box"][1],
+            ),
+            1.0,
+        )
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        crop = rgb[y:y2, x:x2]
+
+        try:
+            resized_crop = cv2.resize(crop, (224, 224))
+        except Exception:
+            logger.warning("Failed to resize image for object classification")
+            return
+
+        # Copy crop for training images (will be used on worker thread)
+        crop_bgr = cv2.cvtColor(crop, cv2.COLOR_RGB2BGR)
+
+        self._enqueue_task(
+            ("classify", object_id, obj_data["camera"], now, resized_crop, crop_bgr)
+        )
+
+    def _process_task(self, task: Any) -> None:
+        kind = task[0]
+        if kind == "classify":
+            _, object_id, camera, timestamp, resized_crop, crop_bgr = task
+            self._classify_object(object_id, camera, timestamp, resized_crop, crop_bgr)
+        elif kind == "expire":
+            _, object_id = task
+            if object_id in self.classification_history:
+                self.classification_history.pop(object_id)
+        elif kind == "reload":
+            self.__build_detector()
+
+    def _classify_object(
+        self,
+        object_id: str,
+        camera: str,
+        timestamp: float,
+        resized_crop: np.ndarray,
+        crop_bgr: np.ndarray,
+    ) -> None:
+        if self.interpreter is None:
+            save_attempts = (
+                self.model_config.save_attempts
+                if self.model_config.save_attempts is not None
+                else 200
+            )
+            write_classification_attempt(
+                self.train_dir,
+                crop_bgr,
+                object_id,
+                timestamp,
+                "unknown",
+                0.0,
+                max_files=save_attempts,
+            )
+
+            # Still track history even when model doesn't exist to respect MAX_OBJECT_CLASSIFICATIONS
+            # Add an entry with "unknown" label so the history limit is enforced
+            if object_id not in self.classification_history:
+                self.classification_history[object_id] = []
+
+            self.classification_history[object_id].append(("unknown", 0.0, timestamp))
+            return
+
+        if not self.tensor_input_details or not self.tensor_output_details:
+            return
+
+        input = np.expand_dims(resized_crop, axis=0)
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        probs = res / res.sum(axis=0)
+        logger.debug(
+            f"{self.model_config.name} Ran object classification with probabilities: {probs}"
+        )
+        best_id = int(np.argmax(probs))
+        score = round(probs[best_id], 2)
+        self.__update_metrics(datetime.datetime.now().timestamp() - timestamp)
+
+        save_attempts = (
+            self.model_config.save_attempts
+            if self.model_config.save_attempts is not None
+            else 200
+        )
+        write_classification_attempt(
+            self.train_dir,
+            crop_bgr,
+            object_id,
+            timestamp,
+            self.labelmap[best_id],
+            score,
+            max_files=save_attempts,
+        )
+
+        if score < self.model_config.threshold:
+            logger.debug(
+                f"{self.model_config.name}: Score {score} < threshold {self.model_config.threshold} for {object_id}, skipping"
+            )
+            return
+
+        sub_label = self.labelmap[best_id]
+
+        logger.debug(
+            f"{self.model_config.name}: Object {object_id} passed threshold with sub_label={sub_label}, score={score}"
+        )
+
+        consensus_label, consensus_score = self.get_weighted_score(
+            object_id, sub_label, score, timestamp
+        )
+
+        logger.debug(
+            f"{self.model_config.name}: get_weighted_score returned consensus_label={consensus_label}, consensus_score={consensus_score} for {object_id}"
+        )
+
+        if consensus_label is not None and self.model_config.object_config is not None:
+            self._emit_result(
+                {
+                    "type": "classification",
+                    "processor": "object",
+                    "model_name": self.model_config.name,
+                    "classification_type": self.model_config.object_config.classification_type,
+                    "object_id": object_id,
+                    "camera": camera,
+                    "timestamp": timestamp,
+                    "label": consensus_label,
+                    "score": consensus_score,
+                }
+            )
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if topic == EmbeddingsRequestEnum.reload_classification_model.value:
+            if request_data.get("model_name") == self.model_config.name:
+
+                def _do_reload(data: dict[str, Any]) -> dict[str, Any]:
+                    self.__build_detector()
+                    logger.info(
+                        f"Successfully loaded updated model for {self.model_config.name}"
+                    )
+                    return {
+                        "success": True,
+                        "message": f"Loaded {self.model_config.name} model.",
+                    }
+
+                result: dict[str, Any] = self._enqueue_request(_do_reload, request_data)
+                return result
+            else:
+                return None
+        else:
+            return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        self._enqueue_task(("expire", object_id))
+
+
+def write_classification_attempt(
+    folder: str,
+    frame: np.ndarray,
+    event_id: str,
+    timestamp: float,
+    label: str,
+    score: float,
+    max_files: int = 100,
+) -> None:
+    if "-" in label:
+        label = label.replace("-", "_")
+
+    file = os.path.join(folder, f"{event_id}-{timestamp}-{label}-{score}.webp")
+    os.makedirs(folder, exist_ok=True)
+    cv2.imwrite(file, frame)
+
+    # delete oldest face image if maximum is reached
+    try:
+        files = sorted(
+            filter(lambda f: f.endswith(".webp"), os.listdir(folder)),
+            key=lambda f: os.path.getctime(os.path.join(folder, f)),
+            reverse=True,
+        )
+
+        if len(files) > max_files:
+            os.unlink(os.path.join(folder, files[-1]))
+    except (FileNotFoundError, OSError):
+        pass

frigate/data_processing/real_time/face.py

@@ -0,0 +1,566 @@
+"""Handle processing images for face detection and recognition."""
+
+import base64
+import datetime
+import json
+import logging
+import os
+import shutil
+from pathlib import Path
+from typing import Any
+
+import cv2
+import numpy as np
+
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
+    EventMetadataTypeEnum,
+)
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.const import FACE_DIR, MODEL_CACHE_DIR
+from frigate.data_processing.common.face.model import (
+    ArcFaceRecognizer,
+    FaceNetRecognizer,
+    FaceRecognizer,
+)
+from frigate.types import TrackedObjectUpdateTypesEnum
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed
+from frigate.util.image import area
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+MAX_DETECTION_HEIGHT = 1080
+MAX_FACES_ATTEMPTS_AFTER_REC = 6
+MAX_FACE_ATTEMPTS = 12
+
+
+class FaceRealTimeProcessor(RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics)
+        self.face_config = config.face_recognition
+        self.requestor = requestor
+        self.sub_label_publisher = sub_label_publisher
+        self.face_detector: cv2.FaceDetectorYN | None = None
+        self.requires_face_detection = "face" not in self.config.objects.all_objects
+        self.person_face_history: dict[str, list[tuple[str, float, int]]] = {}
+        self.camera_current_people: dict[str, list[str]] = {}
+        self.recognizer: FaceRecognizer
+        self.faces_per_second = EventsPerSecond()
+        self.inference_speed = InferenceSpeed(self.metrics.face_rec_speed)
+
+        GITHUB_ENDPOINT = os.environ.get("GITHUB_ENDPOINT", "https://github.com")
+
+        download_path = os.path.join(MODEL_CACHE_DIR, "facedet")
+        self.model_files = {
+            "facedet.onnx": f"{GITHUB_ENDPOINT}/NickM-27/facenet-onnx/releases/download/v1.0/facedet.onnx",
+            "landmarkdet.yaml": f"{GITHUB_ENDPOINT}/NickM-27/facenet-onnx/releases/download/v1.0/landmarkdet.yaml",
+        }
+
+        if not all(
+            os.path.exists(os.path.join(download_path, n))
+            for n in self.model_files.keys()
+        ):
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            self.downloader = ModelDownloader(
+                model_name="facedet",
+                download_path=download_path,
+                file_names=list(self.model_files.keys()),
+                download_func=self.__download_models,
+                complete_func=self.__build_detector,
+            )
+            self.downloader.ensure_model_files()
+        else:
+            self.__build_detector()
+
+        self.label_map: dict[int, str] = {}
+
+        if self.face_config.model_size == "small":
+            self.recognizer = FaceNetRecognizer(self.config)
+        else:
+            self.recognizer = ArcFaceRecognizer(self.config)
+
+        self.recognizer.build()
+
+    CONFIG_UPDATE_TOPIC = "config/face_recognition"
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Update face recognition config at runtime."""
+        if topic != self.CONFIG_UPDATE_TOPIC:
+            return
+
+        previous_min_area = self.config.face_recognition.min_area
+        self.config.face_recognition = payload
+        self.face_config = payload
+
+        for camera_config in self.config.cameras.values():
+            if camera_config.face_recognition.min_area == previous_min_area:
+                camera_config.face_recognition.min_area = payload.min_area
+
+        logger.debug("Face recognition config updated dynamically")
+
+    def __download_models(self, path: str) -> None:
+        try:
+            file_name = os.path.basename(path)
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            ModelDownloader.download_from_url(self.model_files[file_name], path)
+        except Exception as e:
+            logger.error(f"Failed to download {path}: {e}")
+
+    def __build_detector(self) -> None:
+        self.face_detector = cv2.FaceDetectorYN.create(
+            os.path.join(MODEL_CACHE_DIR, "facedet/facedet.onnx"),
+            config="",
+            input_size=(320, 320),
+            score_threshold=0.5,
+            nms_threshold=0.3,
+        )
+        self.faces_per_second.start()
+
+    def __detect_face(
+        self, input: np.ndarray, threshold: float
+    ) -> tuple[int, int, int, int] | None:
+        """Detect faces in input image."""
+        if not self.face_detector:
+            return None
+
+        # YN face detector fails at extreme definitions
+        # this rescales to a size that can properly detect faces
+        # still retaining plenty of detail
+        if input.shape[0] > MAX_DETECTION_HEIGHT:
+            scale_factor = MAX_DETECTION_HEIGHT / input.shape[0]
+            new_width = int(scale_factor * input.shape[1])
+            input = cv2.resize(input, (new_width, MAX_DETECTION_HEIGHT))
+        else:
+            scale_factor = 1
+
+        self.face_detector.setInputSize((input.shape[1], input.shape[0]))
+        faces = self.face_detector.detect(input)
+
+        if faces is None or faces[1] is None:
+            return None  # type: ignore[unreachable]
+
+        face = None
+
+        for _, potential_face in enumerate(faces[1]):
+            if potential_face[-1] < threshold:
+                continue
+
+            raw_bbox = potential_face[0:4].astype(np.uint16)
+            x: int = int(max(raw_bbox[0], 0) / scale_factor)
+            y: int = int(max(raw_bbox[1], 0) / scale_factor)
+            w: int = int(raw_bbox[2] / scale_factor)
+            h: int = int(raw_bbox[3] / scale_factor)
+            bbox = (x, y, x + w, y + h)
+
+            if face is None or area(bbox) > area(face):  # type: ignore[unreachable]
+                face = bbox
+
+        return face
+
+    def __update_metrics(self, duration: float) -> None:
+        self.faces_per_second.update()
+        self.inference_speed.update(duration)
+
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        """Look for faces in image."""
+        self.metrics.face_rec_fps.value = self.faces_per_second.eps()
+        camera = obj_data["camera"]
+
+        if not self.config.cameras[camera].face_recognition.enabled:
+            logger.debug(f"Face recognition disabled for camera {camera}, skipping")
+            return
+
+        start = datetime.datetime.now().timestamp()
+        id = obj_data["id"]
+
+        # don't run for non person objects
+        if obj_data.get("label") != "person":
+            logger.debug("Not processing face for a non person object.")
+            return
+
+        # don't overwrite sub label for objects that have a sub label
+        # that is not a face
+        if obj_data.get("sub_label") and id not in self.person_face_history:
+            logger.debug(
+                f"Not processing face due to existing sub label: {obj_data.get('sub_label')}."
+            )
+            return
+
+        # check if we have hit limits
+        if (
+            id in self.person_face_history
+            and len(self.person_face_history[id]) >= MAX_FACES_ATTEMPTS_AFTER_REC
+        ):
+            # if we are at max attempts after rec and we have a rec
+            if obj_data.get("sub_label"):
+                logger.debug(
+                    "Not processing due to hitting max attempts after true recognition."
+                )
+                return
+
+            # if we don't have a rec and are at max attempts
+            if len(self.person_face_history[id]) >= MAX_FACE_ATTEMPTS:
+                logger.debug("Not processing due to hitting max rec attempts.")
+                return
+
+        face: dict[str, Any] | None = None
+
+        if self.requires_face_detection:
+            logger.debug("Running manual face detection.")
+            person_box = obj_data.get("box")
+
+            if not person_box:
+                logger.debug(f"No person box available for {id}")
+                return
+
+            # YuNet (cv2.FaceDetectorYN) is trained on BGR
+            bgr = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+            left, top, right, bottom = person_box
+            person = bgr[top:bottom, left:right]
+            face_box = self.__detect_face(person, self.face_config.detection_threshold)
+
+            if not face_box:
+                logger.debug("Detected no faces for person object.")
+                return
+
+            face_frame = person[
+                max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
+                max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
+            ]
+
+            # check that face is correct size
+            if area(face_box) < self.config.cameras[camera].face_recognition.min_area:
+                logger.debug(
+                    f"Detected face that is smaller than the min_area {face} < {self.config.cameras[camera].face_recognition.min_area}"
+                )
+                return
+
+        else:
+            # don't run for object without attributes
+            if not obj_data.get("current_attributes"):
+                logger.debug("No attributes to parse.")
+                return
+
+            attributes: list[dict[str, Any]] = obj_data.get("current_attributes", [])
+            for attr in attributes:
+                if attr.get("label") != "face":
+                    continue
+
+                if face is None or attr.get("score", 0.0) > face.get("score", 0.0):
+                    face = attr
+
+            # no faces detected in this frame
+            if not face:
+                logger.debug(f"No face attributes found for {id}")
+                return
+
+            face_box = face.get("box")
+
+            # check that face is valid
+            if (
+                not face_box
+                or area(face_box)
+                < self.config.cameras[camera].face_recognition.min_area
+            ):
+                logger.debug(f"Invalid face box {face}")
+                return
+
+            face_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+
+            face_frame = face_frame[
+                max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
+                max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
+            ]
+
+        res = self.recognizer.classify(face_frame)
+
+        if not res:
+            logger.debug(f"Face recognizer returned no result for {id}")
+            self.__update_metrics(datetime.datetime.now().timestamp() - start)
+            return
+
+        sub_label, score = res
+
+        if score <= self.face_config.unknown_score:
+            sub_label = "unknown"
+
+        logger.debug(
+            f"Detected best face for person as: {sub_label} with probability {score}"
+        )
+
+        self.write_face_attempt(
+            face_frame, id, datetime.datetime.now().timestamp(), sub_label, score
+        )
+
+        if id not in self.person_face_history:
+            self.person_face_history[id] = []
+
+            if camera not in self.camera_current_people:
+                self.camera_current_people[camera] = []
+
+            self.camera_current_people[camera].append(id)
+
+        self.person_face_history[id].append(
+            (sub_label, score, face_frame.shape[0] * face_frame.shape[1])
+        )
+        (weighted_sub_label, weighted_score) = self.weighted_average(
+            self.person_face_history[id]
+        )
+
+        self.requestor.send_data(
+            "tracked_object_update",
+            json.dumps(
+                {
+                    "type": TrackedObjectUpdateTypesEnum.face,
+                    "name": weighted_sub_label,
+                    "score": weighted_score,
+                    "id": id,
+                    "camera": camera,
+                    "timestamp": start,
+                }
+            ),
+        )
+
+        if weighted_score >= self.face_config.recognition_threshold:
+            self.sub_label_publisher.publish(
+                (id, weighted_sub_label, weighted_score),
+                EventMetadataTypeEnum.sub_label.value,
+            )
+
+        self.__update_metrics(datetime.datetime.now().timestamp() - start)
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if topic == EmbeddingsRequestEnum.clear_face_classifier.value:
+            self.recognizer.clear()
+            return {"success": True, "message": "Face classifier cleared."}
+        elif topic == EmbeddingsRequestEnum.recognize_face.value:
+            img = cv2.imdecode(
+                np.frombuffer(base64.b64decode(request_data["image"]), dtype=np.uint8),
+                cv2.IMREAD_COLOR,
+            )
+
+            # detect faces with lower confidence since we expect the face
+            # to be visible in uploaded images
+            face_box = self.__detect_face(img, 0.5)
+
+            if not face_box:
+                return {"message": "No face was detected.", "success": False}
+
+            face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
+            res = self.recognizer.classify(face)
+
+            if not res:
+                return {"success": False, "message": "No face was recognized."}
+
+            sub_label, score = res
+
+            if score <= self.face_config.unknown_score:
+                sub_label = "unknown"
+
+            return {"success": True, "score": score, "face_name": sub_label}
+        elif topic == EmbeddingsRequestEnum.register_face.value:
+            label = request_data["face_name"]
+
+            if request_data.get("cropped"):
+                thumbnail = request_data["image"]
+            else:
+                img = cv2.imdecode(
+                    np.frombuffer(
+                        base64.b64decode(request_data["image"]), dtype=np.uint8
+                    ),
+                    cv2.IMREAD_COLOR,
+                )
+
+                # detect faces with lower confidence since we expect the face
+                # to be visible in uploaded images
+                face_box = self.__detect_face(img, 0.5)
+
+                if not face_box:
+                    return {
+                        "message": "No face was detected.",
+                        "success": False,
+                    }
+
+                face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
+                _, thumbnail = cv2.imencode(
+                    ".webp", face, [int(cv2.IMWRITE_WEBP_QUALITY), 100]
+                )
+
+            # write face to library
+            folder = os.path.join(FACE_DIR, label)
+            file = os.path.join(
+                folder, f"{label}_{datetime.datetime.now().timestamp()}.webp"
+            )
+            os.makedirs(folder, exist_ok=True)
+
+            # save face image
+            with open(file, "wb") as output:
+                output.write(thumbnail.tobytes())
+
+            self.recognizer.clear()
+            return {
+                "message": "Successfully registered face.",
+                "success": True,
+            }
+        elif topic == EmbeddingsRequestEnum.reprocess_face.value:
+            current_file: str = request_data["image_file"]
+            (id_time, id_rand, timestamp, _, _) = current_file.split("-")
+            img = None
+            id = f"{id_time}-{id_rand}"
+
+            if current_file:
+                img = cv2.imread(current_file)
+
+            if img is None:
+                return {  # type: ignore[unreachable]
+                    "message": "Invalid image file.",
+                    "success": False,
+                }
+
+            res = self.recognizer.classify(img)
+
+            if not res:
+                return {
+                    "message": "Model is still training, please try again in a few moments.",
+                    "success": False,
+                }
+
+            sub_label, score = res
+
+            if score <= self.face_config.unknown_score:
+                sub_label = "unknown"
+
+            if "-" in sub_label:
+                sub_label = sub_label.replace("-", "_")
+
+            if self.config.face_recognition.save_attempts:
+                # write face to library
+                folder = os.path.join(FACE_DIR, "train")
+                os.makedirs(folder, exist_ok=True)
+                new_file = os.path.join(
+                    folder, f"{id}-{timestamp}-{sub_label}-{score}.webp"
+                )
+                shutil.move(current_file, new_file)
+
+            return {
+                "message": f"Successfully reprocessed face. Result: {sub_label} (score: {score:.2f})",
+                "success": True,
+                "face_name": sub_label,
+                "score": score,
+            }
+
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        if object_id in self.person_face_history:
+            self.person_face_history.pop(object_id)
+
+            if object_id in self.camera_current_people.get(camera, []):
+                self.camera_current_people[camera].remove(object_id)
+
+    def weighted_average(
+        self, results_list: list[tuple[str, float, int]], max_weight: int = 4000
+    ) -> tuple[str | None, float]:
+        """
+        Calculates a robust weighted average, capping the area weight and giving more weight to higher scores.
+
+        Args:
+            results_list: A list of tuples, where each tuple contains (name, score, face_area).
+            max_weight: The maximum weight to apply based on face area.
+
+        Returns:
+            A tuple containing the prominent name and its weighted average score, or (None, 0.0) if the list is empty.
+        """
+        if not results_list:
+            return None, 0.0
+
+        counts: dict[str, int] = {}
+        weighted_scores: dict[str, float] = {}
+        total_weights: dict[str, float] = {}
+
+        for name, score, face_area in results_list:
+            if name == "unknown":
+                continue
+
+            if name not in weighted_scores:
+                counts[name] = 0
+                weighted_scores[name] = 0.0
+                total_weights[name] = 0.0
+
+            # increase count
+            counts[name] += 1
+
+            # Capped weight based on face area
+            weight: float = min(face_area, max_weight)
+
+            # Score-based weighting (higher scores get more weight)
+            weight *= (score - self.face_config.unknown_score) * 10
+            weighted_scores[name] += score * weight
+            total_weights[name] += weight
+
+        if not weighted_scores:
+            return None, 0.0
+
+        best_name = max(weighted_scores, key=lambda k: weighted_scores[k])
+
+        # If the number of faces for this person < min_faces, we are not confident it is a correct result
+        if counts[best_name] < self.face_config.min_faces:
+            return None, 0.0
+
+        # If the best name has the same number of results as another name, we are not confident it is a correct result
+        for name, count in counts.items():
+            if name != best_name and counts[best_name] == count:
+                return None, 0.0
+
+        weighted_average = weighted_scores[best_name] / total_weights[best_name]
+
+        return best_name, weighted_average
+
+    def write_face_attempt(
+        self,
+        frame: np.ndarray,
+        event_id: str,
+        timestamp: float,
+        sub_label: str,
+        score: float,
+    ) -> None:
+        if self.config.face_recognition.save_attempts:
+            # write face to library
+            folder = os.path.join(FACE_DIR, "train")
+
+            if "-" in sub_label:
+                sub_label = sub_label.replace("-", "_")
+
+            file = os.path.join(
+                folder, f"{event_id}-{timestamp}-{sub_label}-{score}.webp"
+            )
+            os.makedirs(folder, exist_ok=True)
+            cv2.imwrite(file, frame)
+
+            files = sorted(
+                filter(lambda f: f.endswith(".webp"), os.listdir(folder)),
+                key=lambda f: os.path.getctime(os.path.join(folder, f)),
+                reverse=True,
+            )
+
+            # delete oldest face image if maximum is reached
+            if len(files) > self.config.face_recognition.save_attempts:
+                Path(os.path.join(folder, files[-1])).unlink(missing_ok=True)

frigate/data_processing/real_time/license_plate.py

@@ -0,0 +1,76 @@
+"""Handle processing images for face detection and recognition."""
+
+import logging
+from typing import Any
+
+import numpy as np
+
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.data_processing.common.license_plate.mixin import (
+    LicensePlateProcessingMixin,
+)
+from frigate.data_processing.common.license_plate.model import (
+    LicensePlateModelRunner,
+)
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+        model_runner: LicensePlateModelRunner,
+        detected_license_plates: dict[str, dict[str, Any]],
+    ):
+        self.requestor = requestor
+        self.detected_license_plates = detected_license_plates
+        self.model_runner = model_runner
+        self.lpr_config = config.lpr
+        self.config = config
+        self.sub_label_publisher = sub_label_publisher
+        self.camera_current_cars: dict[str, list[str]] = {}
+        super().__init__(config, metrics)
+
+    CONFIG_UPDATE_TOPIC = "config/lpr"
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Update LPR config at runtime."""
+        if topic != self.CONFIG_UPDATE_TOPIC:
+            return
+
+        previous_min_area = self.config.lpr.min_area
+        self.config.lpr = payload
+        self.lpr_config = payload
+
+        for camera_config in self.config.cameras.values():
+            if camera_config.lpr.min_area == previous_min_area:
+                camera_config.lpr.min_area = payload.min_area
+
+        logger.debug("LPR config updated dynamically")
+
+    def process_frame(
+        self,
+        obj_data: dict[str, Any],
+        frame: np.ndarray,
+        dedicated_lpr: bool = False,
+    ) -> None:
+        """Look for license plates in image."""
+        self.lpr_process(obj_data, frame, dedicated_lpr)
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        """Expire lpr objects."""
+        self.lpr_expire(object_id, camera)

frigate/data_processing/types.py

@@ -0,0 +1,69 @@
+"""Embeddings types."""
+
+from __future__ import annotations
+
+from enum import Enum
+from multiprocessing.managers import DictProxy, SyncManager, ValueProxy
+from typing import Any
+
+import sherpa_onnx
+
+from frigate.data_processing.real_time.whisper_online import FasterWhisperASR
+
+
+class DataProcessorMetrics:
+    image_embeddings_speed: ValueProxy[float]
+    image_embeddings_eps: ValueProxy[float]
+    text_embeddings_speed: ValueProxy[float]
+    text_embeddings_eps: ValueProxy[float]
+    face_rec_speed: ValueProxy[float]
+    face_rec_fps: ValueProxy[float]
+    alpr_speed: ValueProxy[float]
+    alpr_pps: ValueProxy[float]
+    yolov9_lpr_speed: ValueProxy[float]
+    yolov9_lpr_pps: ValueProxy[float]
+    review_desc_speed: ValueProxy[float]
+    review_desc_dps: ValueProxy[float]
+    object_desc_speed: ValueProxy[float]
+    object_desc_dps: ValueProxy[float]
+    classification_speeds: DictProxy[str, ValueProxy[float]]
+    classification_cps: DictProxy[str, ValueProxy[float]]
+
+    def __init__(self, manager: SyncManager, custom_classification_models: list[str]):
+        self.image_embeddings_speed = manager.Value("d", 0.0)
+        self.image_embeddings_eps = manager.Value("d", 0.0)
+        self.text_embeddings_speed = manager.Value("d", 0.0)
+        self.text_embeddings_eps = manager.Value("d", 0.0)
+        self.face_rec_speed = manager.Value("d", 0.0)
+        self.face_rec_fps = manager.Value("d", 0.0)
+        self.alpr_speed = manager.Value("d", 0.0)
+        self.alpr_pps = manager.Value("d", 0.0)
+        self.yolov9_lpr_speed = manager.Value("d", 0.0)
+        self.yolov9_lpr_pps = manager.Value("d", 0.0)
+        self.review_desc_speed = manager.Value("d", 0.0)
+        self.review_desc_dps = manager.Value("d", 0.0)
+        self.object_desc_speed = manager.Value("d", 0.0)
+        self.object_desc_dps = manager.Value("d", 0.0)
+        self.classification_speeds = manager.dict()
+        self.classification_cps = manager.dict()
+
+        if custom_classification_models:
+            for key in custom_classification_models:
+                self.classification_speeds[key] = manager.Value("d", 0.0)
+                self.classification_cps[key] = manager.Value("d", 0.0)
+
+
+class DataProcessorModelRunner:
+    def __init__(self, requestor: Any, device: str = "CPU", model_size: str = "large"):
+        self.requestor = requestor
+        self.device = device
+        self.model_size = model_size
+
+
+class PostProcessDataEnum(str, Enum):
+    recording = "recording"
+    review = "review"
+    tracked_object = "tracked_object"
+
+
+AudioTranscriptionModel = FasterWhisperASR | sherpa_onnx.OnlineRecognizer | None