chore: import upstream snapshot with attribution

frigate/data_processing/real_time/api.py

@@ -0,0 +1,201 @@
+"""Local only processors for handling real time object processing."""
+
+import logging
+import threading
+from abc import ABC, abstractmethod
+from collections import deque
+from collections.abc import Callable
+from concurrent.futures import Future
+from queue import Empty, Full, Queue
+from typing import Any
+
+import numpy as np
+
+from frigate.config import FrigateConfig
+
+from ..types import DataProcessorMetrics
+
+logger = logging.getLogger(__name__)
+
+
+class RealTimeProcessorApi(ABC):
+    @abstractmethod
+    def __init__(
+        self,
+        config: FrigateConfig,
+        metrics: DataProcessorMetrics,
+    ) -> None:
+        self.config = config
+        self.metrics = metrics
+        pass
+
+    @abstractmethod
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        """Processes the frame with object data.
+        Args:
+            obj_data (dict): containing data about focused object in frame.
+            frame (ndarray): full yuv frame.
+
+        Returns:
+            None.
+        """
+        pass
+
+    @abstractmethod
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        """Handle metadata requests.
+        Args:
+            topic (str): topic that dictates what work is requested.
+            request_data (dict): containing data about requested change to process.
+
+        Returns:
+            None if request was not handled, otherwise return response.
+        """
+        pass
+
+    @abstractmethod
+    def expire_object(self, object_id: str, camera: str) -> None:
+        """Handle objects that are no longer detected.
+        Args:
+            object_id (str): id of object that is no longer detected.
+            camera (str): name of camera that object was detected on.
+
+        Returns:
+            None.
+        """
+        pass
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Handle a config change notification.
+
+        Called for every config update published under ``config/``.
+        Processors should override this to check the topic and act only
+        on changes relevant to them. Default is a no-op.
+
+        Args:
+            topic: The config topic that changed.
+            payload: The updated configuration object.
+        """
+        pass
+
+    def drain_results(self) -> list[dict[str, Any]]:
+        """Return pending results that need IPC side-effects.
+
+        Deferred processors accumulate results on a worker thread.
+        The maintainer calls this each loop iteration to collect them
+        and perform publishes on the main thread.
+
+        Synchronous processors return an empty list (default).
+        """
+        return []
+
+    def shutdown(self) -> None:
+        """Stop any background work and release resources.
+
+        Called when the processor is being removed or the maintainer
+        is shutting down. Default is a no-op for synchronous processors.
+        """
+        pass
+
+
+class DeferredRealtimeProcessorApi(RealTimeProcessorApi):
+    """Base class for processors that offload heavy work to a background thread.
+
+    Subclasses implement:
+      - process_frame(): do cheap gating + crop + copy, then call _enqueue_task()
+      - _process_task(task): heavy work (inference, consensus) on the worker thread
+      - handle_request(): optionally use _enqueue_request() for sync request/response
+      - expire_object(): call _enqueue_task() with a control message
+
+    The worker thread owns all processor state. No locks are needed because
+    only the worker mutates state. Results that need IPC are placed in
+    _pending_results via _emit_result(), and the maintainer drains them
+    each loop iteration.
+    """
+
+    def __init__(
+        self,
+        config: FrigateConfig,
+        metrics: DataProcessorMetrics,
+        max_queue: int = 8,
+    ) -> None:
+        super().__init__(config, metrics)
+        self._task_queue: Queue = Queue(maxsize=max_queue)
+        self._pending_results: deque[dict[str, Any]] = deque()
+        self._results_lock = threading.Lock()
+        self._stop_event = threading.Event()
+        self._worker = threading.Thread(
+            target=self._drain_loop,
+            daemon=True,
+            name=f"{type(self).__name__}_worker",
+        )
+        self._worker.start()
+
+    def _drain_loop(self) -> None:
+        """Worker thread main loop — drains the task queue until stopped."""
+        while not self._stop_event.is_set():
+            try:
+                task = self._task_queue.get(timeout=0.5)
+            except Empty:
+                continue
+
+            if (
+                isinstance(task, tuple)
+                and len(task) == 2
+                and isinstance(task[1], Future)
+            ):
+                # Request/response: (callable_and_args, future)
+                (func, args), future = task
+                try:
+                    result = func(args)
+                    future.set_result(result)
+                except Exception as e:
+                    future.set_exception(e)
+            else:
+                try:
+                    self._process_task(task)
+                except Exception:
+                    logger.exception("Error processing deferred task")
+
+    def _enqueue_task(self, task: Any) -> bool:
+        """Enqueue a task for the worker. Returns False if queue is full (dropped)."""
+        try:
+            self._task_queue.put_nowait(task)
+            return True
+        except Full:
+            logger.debug("Deferred processor queue full, dropping task")
+            return False
+
+    def _enqueue_request(self, func: Callable, args: Any, timeout: float = 10.0) -> Any:
+        """Enqueue a request and block until the worker returns a result."""
+        future: Future = Future()
+        self._task_queue.put(((func, args), future), timeout=timeout)
+        return future.result(timeout=timeout)
+
+    def _emit_result(self, result: dict[str, Any]) -> None:
+        """Called by the worker thread to stage a result for the maintainer."""
+        with self._results_lock:
+            self._pending_results.append(result)
+
+    def drain_results(self) -> list[dict[str, Any]]:
+        """Called by the maintainer on the main thread to collect pending results."""
+        with self._results_lock:
+            results = list(self._pending_results)
+            self._pending_results.clear()
+        return results
+
+    def shutdown(self) -> None:
+        """Signal the worker to stop and wait for it to finish."""
+        self._stop_event.set()
+        self._worker.join(timeout=5.0)
+
+    @abstractmethod
+    def _process_task(self, task: Any) -> None:
+        """Process a single task on the worker thread.
+
+        Subclasses implement inference, consensus, training image saves here.
+        Call _emit_result() to stage results for the maintainer to publish.
+        """
+        pass

frigate/data_processing/real_time/audio_transcription.py

@@ -0,0 +1,279 @@
+"""Handle processing audio for speech transcription using sherpa-onnx with FFmpeg pipe."""
+
+import logging
+import os
+import queue
+import threading
+from typing import Any
+
+import numpy as np
+
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import CameraConfig, FrigateConfig
+from frigate.const import MODEL_CACHE_DIR
+from frigate.data_processing.common.audio_transcription.model import (
+    AudioTranscriptionModelRunner,
+)
+from frigate.data_processing.real_time.whisper_online import (
+    FasterWhisperASR,
+    OnlineASRProcessor,
+)
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+class AudioTranscriptionRealTimeProcessor(RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        camera_config: CameraConfig,
+        requestor: InterProcessRequestor,
+        model_runner: AudioTranscriptionModelRunner,
+        metrics: DataProcessorMetrics,
+        stop_event: threading.Event,
+    ):
+        super().__init__(config, metrics)
+        self.config = config
+        self.camera_config = camera_config
+        self.requestor = requestor
+        self.stream: Any = None
+        self.whisper_model: FasterWhisperASR | None = None
+        self.model_runner = model_runner
+        self.transcription_segments: list[str] = []
+        self.audio_queue: queue.Queue[tuple[dict[str, Any], np.ndarray]] = queue.Queue()
+        self.stop_event = stop_event
+
+    def __build_recognizer(self) -> None:
+        try:
+            if self.config.audio_transcription.model_size == "large":
+                # Whisper models need to be per-process and can only run one stream at a time
+                # TODO: try parallel: https://github.com/SYSTRAN/faster-whisper/issues/100
+                logger.debug(f"Loading Whisper model for {self.camera_config.name}")
+                self.whisper_model = FasterWhisperASR(
+                    modelsize="tiny",
+                    device="cuda"
+                    if self.config.audio_transcription.device == "GPU"
+                    else "cpu",
+                    lan=self.config.audio_transcription.language,
+                    model_dir=os.path.join(MODEL_CACHE_DIR, "whisper"),
+                )
+                self.whisper_model.use_vad()
+                self.stream = OnlineASRProcessor(
+                    asr=self.whisper_model,
+                )
+            else:
+                logger.debug(f"Loading sherpa stream for {self.camera_config.name}")
+                self.stream = self.model_runner.model.create_stream()
+            logger.debug(
+                f"Audio transcription (live) initialized for {self.camera_config.name}"
+            )
+        except Exception as e:
+            logger.error(
+                f"Failed to initialize live streaming audio transcription: {e}"
+            )
+
+    def __process_audio_stream(self, audio_data: np.ndarray) -> tuple[str, bool] | None:
+        if (
+            self.model_runner.model is None
+            and self.config.audio_transcription.model_size == "small"
+        ):
+            logger.debug("Audio transcription (live) model not initialized")
+            return None
+
+        if not self.stream:
+            self.__build_recognizer()
+
+        try:
+            if audio_data.dtype != np.float32:
+                audio_data = audio_data.astype(np.float32)
+
+            if audio_data.max() > 1.0 or audio_data.min() < -1.0:
+                audio_data = audio_data / 32768.0  # Normalize from int16
+
+            rms = float(np.sqrt(np.mean(np.absolute(np.square(audio_data)))))
+            logger.debug(f"Audio chunk size: {audio_data.size}, RMS: {rms:.4f}")
+
+            if self.config.audio_transcription.model_size == "large":
+                # large model
+                self.stream.insert_audio_chunk(audio_data)
+                output = self.stream.process_iter()
+                text = output[2].strip()
+                is_endpoint = (
+                    text.endswith((".", "!", "?"))
+                    and sum(len(str(lines)) for lines in self.transcription_segments)
+                    > 300
+                )
+
+                if text:
+                    self.transcription_segments.append(text)
+                concatenated_text = " ".join(self.transcription_segments)
+                logger.debug(f"Concatenated transcription: '{concatenated_text}'")
+                text = concatenated_text
+
+            else:
+                # small model
+                self.stream.accept_waveform(16000, audio_data)
+
+                while self.model_runner.model.is_ready(self.stream):
+                    self.model_runner.model.decode_stream(self.stream)
+
+                text = self.model_runner.model.get_result(self.stream).strip()
+                is_endpoint = self.model_runner.model.is_endpoint(self.stream)
+
+            logger.debug(f"Transcription result: '{text}'")
+
+            if not text:
+                logger.debug("No transcription, returning")
+                return None
+
+            logger.debug(f"Endpoint detected: {is_endpoint}")
+
+            if is_endpoint and self.config.audio_transcription.model_size == "small":
+                # reset sherpa if we've reached an endpoint
+                self.model_runner.model.reset(self.stream)
+
+            return text, is_endpoint
+        except Exception as e:
+            logger.error(f"Error processing audio stream: {e}")
+            return None
+
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        pass
+
+    def process_audio(self, obj_data: dict[str, Any], audio: np.ndarray) -> bool | None:
+        if audio is None or audio.size == 0:
+            logger.debug("No audio data provided for transcription")
+            return None
+
+        # enqueue audio data for processing in the thread
+        self.audio_queue.put((obj_data, audio))
+        return None
+
+    def run(self) -> None:
+        """Run method for the transcription thread to process queued audio data."""
+        logger.debug(
+            f"Starting audio transcription thread for {self.camera_config.name}"
+        )
+
+        # start with an empty transcription
+        self.requestor.send_data(
+            f"{self.camera_config.name}/audio/transcription",
+            "",
+        )
+
+        while not self.stop_event.is_set():
+            try:
+                # Get audio data from queue with a timeout to check stop_event
+                _, audio = self.audio_queue.get(timeout=0.1)
+                result = self.__process_audio_stream(audio)
+
+                if not result:
+                    continue
+
+                text, is_endpoint = result
+                logger.debug(f"Transcribed audio: '{text}', Endpoint: {is_endpoint}")
+
+                self.requestor.send_data(
+                    f"{self.camera_config.name}/audio/transcription", text
+                )
+
+                self.audio_queue.task_done()
+
+                if is_endpoint:
+                    self.reset()
+
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.error(f"Error processing audio in thread: {e}")
+                self.audio_queue.task_done()
+
+        logger.debug(
+            f"Stopping audio transcription thread for {self.camera_config.name}"
+        )
+
+    def clear_audio_queue(self) -> None:
+        # Clear the audio queue
+        while not self.audio_queue.empty():
+            try:
+                self.audio_queue.get_nowait()
+                self.audio_queue.task_done()
+            except queue.Empty:
+                break
+
+    def reset(self) -> None:
+        if self.config.audio_transcription.model_size == "large":
+            # get final output from whisper
+            output = self.stream.finish()
+            self.transcription_segments = []
+
+            self.requestor.send_data(
+                f"{self.camera_config.name}/audio/transcription",
+                (output[2].strip() + " "),
+            )
+
+            # reset whisper
+            self.stream.init()
+            self.transcription_segments = []
+        else:
+            # reset sherpa
+            self.model_runner.model.reset(self.stream)
+
+        logger.debug("Stream reset")
+
+    def check_unload_model(self) -> None:
+        # regularly called in the loop in audio maintainer
+        if (
+            self.config.audio_transcription.model_size == "large"
+            and self.whisper_model is not None
+        ):
+            logger.debug(f"Unloading Whisper model for {self.camera_config.name}")
+            self.clear_audio_queue()
+            self.transcription_segments = []
+            self.stream = None
+            self.whisper_model = None
+
+            self.requestor.send_data(
+                f"{self.camera_config.name}/audio/transcription",
+                "",
+            )
+        if (
+            self.config.audio_transcription.model_size == "small"
+            and self.stream is not None
+        ):
+            logger.debug(f"Clearing sherpa stream for {self.camera_config.name}")
+            self.stream = None
+
+            self.requestor.send_data(
+                f"{self.camera_config.name}/audio/transcription",
+                "",
+            )
+
+    def stop(self) -> None:
+        """Stop the transcription thread and clean up."""
+        self.stop_event.set()
+        # Clear the queue to prevent processing stale data
+        while not self.audio_queue.empty():
+            try:
+                self.audio_queue.get_nowait()
+                self.audio_queue.task_done()
+            except queue.Empty:
+                break
+        logger.debug(
+            f"Transcription thread stop signaled for {self.camera_config.name}"
+        )
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if topic == "clear_audio_recognizer":
+            self.stream = None
+            self.__build_recognizer()
+            return {"message": "Audio recognizer cleared and rebuilt", "success": True}
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        pass

frigate/data_processing/real_time/bird.py

@@ -0,0 +1,193 @@
+"""Handle processing images to classify birds."""
+
+import logging
+import os
+from typing import Any
+
+import cv2
+import numpy as np
+
+from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
+    EventMetadataTypeEnum,
+)
+from frigate.config import FrigateConfig
+from frigate.const import MODEL_CACHE_DIR
+from frigate.log import suppress_stderr_during
+from frigate.util.image import calculate_region
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+try:
+    from tflite_runtime.interpreter import Interpreter
+except ModuleNotFoundError:
+    from ai_edge_litert.interpreter import Interpreter
+
+logger = logging.getLogger(__name__)
+
+
+class BirdRealTimeProcessor(RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics)
+        self.interpreter: Interpreter | None = None
+        self.sub_label_publisher = sub_label_publisher
+        self.tensor_input_details: list[dict[str, Any]] | None = None
+        self.tensor_output_details: list[dict[str, Any]] | None = None
+        self.detected_birds: dict[str, float] = {}
+        self.labelmap: dict[int, str] = {}
+
+        GITHUB_RAW_ENDPOINT = os.environ.get(
+            "GITHUB_RAW_ENDPOINT", "https://raw.githubusercontent.com"
+        )
+        download_path = os.path.join(MODEL_CACHE_DIR, "bird")
+        self.model_files = {
+            "bird.tflite": f"{GITHUB_RAW_ENDPOINT}/google-coral/test_data/master/mobilenet_v2_1.0_224_inat_bird_quant.tflite",
+            "birdmap.txt": f"{GITHUB_RAW_ENDPOINT}/google-coral/test_data/master/inat_bird_labels.txt",
+        }
+
+        if not all(
+            os.path.exists(os.path.join(download_path, n))
+            for n in self.model_files.keys()
+        ):
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            self.downloader = ModelDownloader(
+                model_name="bird",
+                download_path=download_path,
+                file_names=list(self.model_files.keys()),
+                download_func=self.__download_models,
+                complete_func=self.__build_detector,
+            )
+            self.downloader.ensure_model_files()
+        else:
+            self.__build_detector()
+
+    def __download_models(self, path: str) -> None:
+        try:
+            file_name = os.path.basename(path)
+
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            ModelDownloader.download_from_url(self.model_files[file_name], path)
+        except Exception as e:
+            logger.error(f"Failed to download {path}: {e}")
+
+    def __build_detector(self) -> None:
+        # Suppress TFLite delegate creation messages that bypass Python logging
+        with suppress_stderr_during("tflite_interpreter_init"):
+            self.interpreter = Interpreter(
+                model_path=os.path.join(MODEL_CACHE_DIR, "bird/bird.tflite"),
+                num_threads=2,
+            )
+            self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+
+        i = 0
+
+        with open(os.path.join(MODEL_CACHE_DIR, "bird/birdmap.txt")) as f:
+            line = f.readline()
+            while line:
+                start = line.find("(")
+                end = line.find(")")
+                self.labelmap[i] = line[start + 1 : end]
+                i += 1
+                line = f.readline()
+
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        if (
+            not self.interpreter
+            or not self.tensor_input_details
+            or not self.tensor_output_details
+        ):
+            return
+
+        if obj_data["label"] != "bird":
+            return
+
+        x, y, x2, y2 = calculate_region(
+            frame.shape,
+            obj_data["box"][0],
+            obj_data["box"][1],
+            obj_data["box"][2],
+            obj_data["box"][3],
+            int(
+                max(
+                    obj_data["box"][1] - obj_data["box"][0],
+                    obj_data["box"][3] - obj_data["box"][2],
+                )
+                * 1.1
+            ),
+            1.0,
+        )
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        input = rgb[
+            y:y2,
+            x:x2,
+        ]
+
+        if input.shape != (224, 224):
+            try:
+                input = cv2.resize(input, (224, 224))
+            except Exception:
+                logger.warning("Failed to resize image for bird classification")
+                return
+
+        input = np.expand_dims(input, axis=0)
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        probs = res / res.sum(axis=0)
+        best_id = int(np.argmax(probs))
+
+        if best_id == 964:
+            logger.debug("No bird classification was detected.")
+            return
+
+        score = round(probs[best_id], 2)
+
+        if score < self.config.classification.bird.threshold:
+            logger.debug(f"Score {score} is not above required threshold")
+            return
+
+        previous_score = self.detected_birds.get(obj_data["id"], 0.0)
+
+        if score <= previous_score:
+            logger.debug(f"Score {score} is worse than previous score {previous_score}")
+            return
+
+        self.sub_label_publisher.publish(
+            (obj_data["id"], self.labelmap[best_id], score),
+            EventMetadataTypeEnum.sub_label.value,
+        )
+        self.detected_birds[obj_data["id"]] = score
+
+    CONFIG_UPDATE_TOPIC = "config/classification"
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Update bird classification config at runtime."""
+        if topic != self.CONFIG_UPDATE_TOPIC:
+            return
+
+        self.config.classification = payload
+        logger.debug("Bird classification config updated dynamically")
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        if object_id in self.detected_birds:
+            self.detected_birds.pop(object_id)

frigate/data_processing/real_time/custom_classification.py

@@ -0,0 +1,744 @@
+"""Real time processor that works with classification tflite models."""
+
+import datetime
+import logging
+import os
+from typing import Any
+
+import cv2
+import numpy as np
+
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.config.classification import CustomClassificationConfig
+from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR
+from frigate.log import suppress_stderr_during
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels
+from frigate.util.image import calculate_region
+from frigate.util.object import box_overlaps
+
+from ..types import DataProcessorMetrics
+from .api import DeferredRealtimeProcessorApi
+
+try:
+    from tflite_runtime.interpreter import Interpreter
+except ModuleNotFoundError:
+    from ai_edge_litert.interpreter import Interpreter
+
+logger = logging.getLogger(__name__)
+
+MAX_OBJECT_CLASSIFICATIONS = 16
+
+
+class CustomStateClassificationProcessor(DeferredRealtimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        model_config: CustomClassificationConfig,
+        requestor: InterProcessRequestor,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics, max_queue=4)
+        self.model_config = model_config
+
+        if not self.model_config.name:
+            raise ValueError("Custom classification model name must be set.")
+
+        self.requestor = requestor
+        self.model_dir = os.path.join(MODEL_CACHE_DIR, self.model_config.name)
+        self.train_dir = os.path.join(CLIPS_DIR, self.model_config.name, "train")
+        self.interpreter: Interpreter | None = None
+        self.tensor_input_details: list[dict[str, Any]] | None = None
+        self.tensor_output_details: list[dict[str, Any]] | None = None
+        self.labelmap: dict[int, str] = {}
+        self.classifications_per_second = EventsPerSecond()
+        self.state_history: dict[str, dict[str, Any]] = {}
+
+        if (
+            self.metrics
+            and self.model_config.name in self.metrics.classification_speeds
+        ):
+            self.inference_speed: InferenceSpeed | None = InferenceSpeed(
+                self.metrics.classification_speeds[self.model_config.name]
+            )
+        else:
+            self.inference_speed = None
+
+        self.last_run = datetime.datetime.now().timestamp()
+        self.__build_detector()
+
+    def __build_detector(self) -> None:
+        model_path = os.path.join(self.model_dir, "model.tflite")
+        labelmap_path = os.path.join(self.model_dir, "labelmap.txt")
+
+        if not os.path.exists(model_path) or not os.path.exists(labelmap_path):
+            self.interpreter = None
+            self.tensor_input_details = None
+            self.tensor_output_details = None
+            self.labelmap = {}
+            return
+
+        # Suppress TFLite delegate creation messages that bypass Python logging
+        with suppress_stderr_during("tflite_interpreter_init"):
+            self.interpreter = Interpreter(
+                model_path=model_path,
+                num_threads=2,
+            )
+            self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+        self.labelmap = load_labels(labelmap_path, prefill=0, indexed=False)
+        self.classifications_per_second.start()
+
+    def __update_metrics(self, duration: float) -> None:
+        self.classifications_per_second.update()
+        if self.inference_speed:
+            self.inference_speed.update(duration)
+
+    def _should_save_image(
+        self, camera: str, detected_state: str, score: float = 1.0
+    ) -> bool:
+        """
+        Determine if we should save the image for training.
+        Save when:
+        - State is changing or being verified (regardless of score)
+        - Score is less than 100% (even if state matches, useful for training)
+        Don't save when:
+        - State is stable (matches current_state) AND score is 100%
+        """
+        if camera not in self.state_history:
+            # First detection for this camera, save it
+            return True
+
+        verification = self.state_history[camera]
+        current_state = verification.get("current_state")
+        pending_state = verification.get("pending_state")
+
+        # Save if there's a pending state change being verified
+        if pending_state is not None:
+            return True
+
+        # Save if the detected state differs from the current verified state
+        # (state is changing)
+        if current_state is not None and detected_state != current_state:
+            return True
+
+        # If score is less than 100%, save even if state matches
+        # (useful for training to improve confidence)
+        if score < 1.0:
+            return True
+
+        # Don't save if state is stable (detected_state == current_state) AND score is 100%
+        return False
+
+    def verify_state_change(self, camera: str, detected_state: str) -> str | None:
+        """
+        Verify state change requires 3 consecutive identical states before publishing.
+        Returns state to publish or None if verification not complete.
+        """
+        if camera not in self.state_history:
+            self.state_history[camera] = {
+                "current_state": None,
+                "pending_state": None,
+                "consecutive_count": 0,
+            }
+
+        verification = self.state_history[camera]
+
+        if detected_state == verification["current_state"]:
+            verification["pending_state"] = None
+            verification["consecutive_count"] = 0
+            return None
+
+        if detected_state == verification["pending_state"]:
+            verification["consecutive_count"] += 1
+
+            if verification["consecutive_count"] >= 3:
+                verification["current_state"] = detected_state
+                verification["pending_state"] = None
+                verification["consecutive_count"] = 0
+                return detected_state
+        else:
+            verification["pending_state"] = detected_state
+            verification["consecutive_count"] = 1
+            logger.debug(
+                f"New state '{detected_state}' detected for {camera}, need {3 - verification['consecutive_count']} more consecutive detections"
+            )
+
+        return None
+
+    def process_frame(self, frame_data: dict[str, Any], frame: np.ndarray) -> None:
+        if (
+            not self.model_config.name
+            or not self.model_config.state_config
+            or not self.tensor_input_details
+            or not self.tensor_output_details
+        ):
+            return
+
+        if self.metrics and self.model_config.name in self.metrics.classification_cps:
+            self.metrics.classification_cps[
+                self.model_config.name
+            ].value = self.classifications_per_second.eps()
+        camera = str(frame_data.get("camera"))
+
+        if camera not in self.model_config.state_config.cameras:
+            return
+
+        camera_config = self.model_config.state_config.cameras[camera]
+        crop = [
+            camera_config.crop[0] * self.config.cameras[camera].detect.width,
+            camera_config.crop[1] * self.config.cameras[camera].detect.height,
+            camera_config.crop[2] * self.config.cameras[camera].detect.width,
+            camera_config.crop[3] * self.config.cameras[camera].detect.height,
+        ]
+        should_run = False
+
+        now = datetime.datetime.now().timestamp()
+        if (
+            self.model_config.state_config.interval
+            and now > self.last_run + self.model_config.state_config.interval
+        ):
+            self.last_run = now
+            should_run = True
+
+        if (
+            not should_run
+            and self.model_config.state_config.motion
+            and any([box_overlaps(crop, mb) for mb in frame_data.get("motion", [])])
+        ):
+            # classification should run at most once per second
+            if now > self.last_run + 1:
+                self.last_run = now
+                should_run = True
+
+        # Shortcut: always run if we have a pending state verification to complete
+        if (
+            not should_run
+            and camera in self.state_history
+            and self.state_history[camera]["pending_state"] is not None
+            and now > self.last_run + 0.5
+        ):
+            self.last_run = now
+            should_run = True
+            logger.debug(
+                f"Running verification check for pending state: {self.state_history[camera]['pending_state']} ({self.state_history[camera]['consecutive_count']}/3)"
+            )
+
+        if not should_run:
+            return
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        height, width = rgb.shape[:2]
+
+        # Convert normalized crop coordinates to pixel values
+        x1 = int(camera_config.crop[0] * width)
+        y1 = int(camera_config.crop[1] * height)
+        x2 = int(camera_config.crop[2] * width)
+        y2 = int(camera_config.crop[3] * height)
+
+        # Clip coordinates to frame boundaries
+        x1 = max(0, min(x1, width))
+        y1 = max(0, min(y1, height))
+        x2 = max(0, min(x2, width))
+        y2 = max(0, min(y2, height))
+
+        if x2 <= x1 or y2 <= y1:
+            logger.warning(
+                f"Invalid crop coordinates for {camera}: [{x1}, {y1}, {x2}, {y2}]"
+            )
+            return
+
+        cropped_frame = rgb[y1:y2, x1:x2]
+
+        try:
+            resized_frame = cv2.resize(cropped_frame, (224, 224))
+        except Exception:
+            logger.warning("Failed to resize image for state classification")
+            return
+
+        # Copy for training image saves on worker thread
+        crop_bgr = cv2.cvtColor(cropped_frame, cv2.COLOR_RGB2BGR)
+
+        self._enqueue_task(("classify", camera, now, resized_frame, crop_bgr))
+
+    def _process_task(self, task: Any) -> None:
+        kind = task[0]
+        if kind == "classify":
+            _, camera, timestamp, resized_frame, crop_bgr = task
+            self._classify_state(camera, timestamp, resized_frame, crop_bgr)
+        elif kind == "reload":
+            self.__build_detector()
+
+    def _classify_state(
+        self,
+        camera: str,
+        timestamp: float,
+        resized_frame: np.ndarray,
+        crop_bgr: np.ndarray,
+    ) -> None:
+        if self.interpreter is None:
+            # When interpreter is None, always save (score is 0.0, which is < 1.0)
+            if self._should_save_image(camera, "unknown", 0.0):
+                save_attempts = (
+                    self.model_config.save_attempts
+                    if self.model_config.save_attempts is not None
+                    else 100
+                )
+                write_classification_attempt(
+                    self.train_dir,
+                    crop_bgr,
+                    "none-none",
+                    timestamp,
+                    "unknown",
+                    0.0,
+                    max_files=save_attempts,
+                )
+            return
+
+        if not self.tensor_input_details or not self.tensor_output_details:
+            return
+
+        input = np.expand_dims(resized_frame, axis=0)
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        probs = res / res.sum(axis=0)
+        logger.debug(
+            f"{self.model_config.name} Ran state classification with probabilities: {probs}"
+        )
+        best_id = int(np.argmax(probs))
+        score = round(probs[best_id], 2)
+        self.__update_metrics(datetime.datetime.now().timestamp() - timestamp)
+
+        detected_state = self.labelmap[best_id]
+
+        if self._should_save_image(camera, detected_state, score):
+            save_attempts = (
+                self.model_config.save_attempts
+                if self.model_config.save_attempts is not None
+                else 100
+            )
+            write_classification_attempt(
+                self.train_dir,
+                crop_bgr,
+                "none-none",
+                timestamp,
+                detected_state,
+                score,
+                max_files=save_attempts,
+            )
+
+        if score < self.model_config.threshold:
+            logger.debug(
+                f"Score {score} below threshold {self.model_config.threshold}, skipping verification"
+            )
+            return
+
+        verified_state = self.verify_state_change(camera, detected_state)
+
+        if verified_state is not None:
+            self._emit_result(
+                {
+                    "type": "classification",
+                    "processor": "state",
+                    "model_name": self.model_config.name,
+                    "camera": camera,
+                    "state": verified_state,
+                }
+            )
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if topic == EmbeddingsRequestEnum.reload_classification_model.value:
+            if request_data.get("model_name") == self.model_config.name:
+
+                def _do_reload(data: dict[str, Any]) -> dict[str, Any]:
+                    self.__build_detector()
+                    logger.info(
+                        f"Successfully loaded updated model for {self.model_config.name}"
+                    )
+                    return {
+                        "success": True,
+                        "message": f"Loaded {self.model_config.name} model.",
+                    }
+
+                result: dict[str, Any] = self._enqueue_request(_do_reload, request_data)
+                return result
+            else:
+                return None
+        else:
+            return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        pass
+
+
+class CustomObjectClassificationProcessor(DeferredRealtimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        model_config: CustomClassificationConfig,
+        sub_label_publisher: EventMetadataPublisher,
+        requestor: InterProcessRequestor,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics, max_queue=8)
+        self.model_config = model_config
+
+        if not self.model_config.name:
+            raise ValueError("Custom classification model name must be set.")
+
+        self.model_dir = os.path.join(MODEL_CACHE_DIR, self.model_config.name)
+        self.train_dir = os.path.join(CLIPS_DIR, self.model_config.name, "train")
+        self.interpreter: Interpreter | None = None
+        self.sub_label_publisher = sub_label_publisher
+        self.requestor = requestor
+        self.tensor_input_details: list[dict[str, Any]] | None = None
+        self.tensor_output_details: list[dict[str, Any]] | None = None
+        self.classification_history: dict[str, list[tuple[str, float, float]]] = {}
+        self.labelmap: dict[int, str] = {}
+        self.classifications_per_second = EventsPerSecond()
+
+        if (
+            self.metrics
+            and self.model_config.name in self.metrics.classification_speeds
+        ):
+            self.inference_speed: InferenceSpeed | None = InferenceSpeed(
+                self.metrics.classification_speeds[self.model_config.name]
+            )
+        else:
+            self.inference_speed = None
+
+        self.__build_detector()
+
+    def __build_detector(self) -> None:
+        model_path = os.path.join(self.model_dir, "model.tflite")
+        labelmap_path = os.path.join(self.model_dir, "labelmap.txt")
+
+        if not os.path.exists(model_path) or not os.path.exists(labelmap_path):
+            self.interpreter = None
+            self.tensor_input_details = None
+            self.tensor_output_details = None
+            self.labelmap = {}
+            return
+
+        # Suppress TFLite delegate creation messages that bypass Python logging
+        with suppress_stderr_during("tflite_interpreter_init"):
+            self.interpreter = Interpreter(
+                model_path=model_path,
+                num_threads=2,
+            )
+            self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+        self.labelmap = load_labels(labelmap_path, prefill=0, indexed=False)
+
+    def __update_metrics(self, duration: float) -> None:
+        self.classifications_per_second.update()
+        if self.inference_speed:
+            self.inference_speed.update(duration)
+
+    def get_weighted_score(
+        self,
+        object_id: str,
+        current_label: str,
+        current_score: float,
+        current_time: float,
+    ) -> tuple[str | None, float]:
+        """
+        Determine weighted score based on history to prevent false positives/negatives.
+        Requires 60% of attempts to agree on a label before publishing.
+        Returns (weighted_label, weighted_score) or (None, 0.0) if no weighted score.
+        """
+        if object_id not in self.classification_history:
+            self.classification_history[object_id] = []
+            logger.debug(f"Created new classification history for {object_id}")
+
+        self.classification_history[object_id].append(
+            (current_label, current_score, current_time)
+        )
+
+        history = self.classification_history[object_id]
+        logger.debug(
+            f"History for {object_id}: {len(history)} entries, latest=({current_label}, {current_score})"
+        )
+
+        if len(history) < 3:
+            logger.debug(
+                f"History for {object_id} has {len(history)} entries, need at least 3"
+            )
+            return None, 0.0
+
+        label_counts: dict[str, int] = {}
+        label_scores: dict[str, list[float]] = {}
+        total_attempts = len(history)
+
+        for label, score, timestamp in history:
+            if label not in label_counts:
+                label_counts[label] = 0
+                label_scores[label] = []
+
+            label_counts[label] += 1
+            label_scores[label].append(score)
+
+        best_label = max(label_counts, key=lambda k: label_counts[k])
+        best_count = label_counts[best_label]
+
+        consensus_threshold = total_attempts * 0.6
+        logger.debug(
+            f"Consensus calc for {object_id}: label_counts={label_counts}, "
+            f"best_label={best_label}, best_count={best_count}, "
+            f"total={total_attempts}, threshold={consensus_threshold}"
+        )
+
+        if best_count < consensus_threshold:
+            logger.debug(
+                f"No consensus for {object_id}: {best_count} < {consensus_threshold}"
+            )
+            return None, 0.0
+
+        avg_score = sum(label_scores[best_label]) / len(label_scores[best_label])
+
+        if best_label == "none":
+            logger.debug(f"Filtering 'none' label for {object_id}")
+            return None, 0.0
+
+        logger.debug(
+            f"Consensus reached for {object_id}: {best_label} with avg_score={avg_score}"
+        )
+        return best_label, avg_score
+
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        if (
+            not self.model_config.name
+            or not self.model_config.object_config
+            or not self.tensor_input_details
+            or not self.tensor_output_details
+        ):
+            return
+
+        if self.metrics and self.model_config.name in self.metrics.classification_cps:
+            self.metrics.classification_cps[
+                self.model_config.name
+            ].value = self.classifications_per_second.eps()
+
+        if obj_data["false_positive"]:
+            return
+
+        if obj_data["label"] not in self.model_config.object_config.objects:
+            return
+
+        if obj_data.get("end_time") is not None:
+            return
+
+        object_id = obj_data["id"]
+
+        if (
+            object_id in self.classification_history
+            and len(self.classification_history[object_id])
+            >= MAX_OBJECT_CLASSIFICATIONS
+        ):
+            return
+
+        now = datetime.datetime.now().timestamp()
+        x, y, x2, y2 = calculate_region(
+            frame.shape,
+            obj_data["box"][0],
+            obj_data["box"][1],
+            obj_data["box"][2],
+            obj_data["box"][3],
+            max(
+                obj_data["box"][2] - obj_data["box"][0],
+                obj_data["box"][3] - obj_data["box"][1],
+            ),
+            1.0,
+        )
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        crop = rgb[y:y2, x:x2]
+
+        try:
+            resized_crop = cv2.resize(crop, (224, 224))
+        except Exception:
+            logger.warning("Failed to resize image for object classification")
+            return
+
+        # Copy crop for training images (will be used on worker thread)
+        crop_bgr = cv2.cvtColor(crop, cv2.COLOR_RGB2BGR)
+
+        self._enqueue_task(
+            ("classify", object_id, obj_data["camera"], now, resized_crop, crop_bgr)
+        )
+
+    def _process_task(self, task: Any) -> None:
+        kind = task[0]
+        if kind == "classify":
+            _, object_id, camera, timestamp, resized_crop, crop_bgr = task
+            self._classify_object(object_id, camera, timestamp, resized_crop, crop_bgr)
+        elif kind == "expire":
+            _, object_id = task
+            if object_id in self.classification_history:
+                self.classification_history.pop(object_id)
+        elif kind == "reload":
+            self.__build_detector()
+
+    def _classify_object(
+        self,
+        object_id: str,
+        camera: str,
+        timestamp: float,
+        resized_crop: np.ndarray,
+        crop_bgr: np.ndarray,
+    ) -> None:
+        if self.interpreter is None:
+            save_attempts = (
+                self.model_config.save_attempts
+                if self.model_config.save_attempts is not None
+                else 200
+            )
+            write_classification_attempt(
+                self.train_dir,
+                crop_bgr,
+                object_id,
+                timestamp,
+                "unknown",
+                0.0,
+                max_files=save_attempts,
+            )
+
+            # Still track history even when model doesn't exist to respect MAX_OBJECT_CLASSIFICATIONS
+            # Add an entry with "unknown" label so the history limit is enforced
+            if object_id not in self.classification_history:
+                self.classification_history[object_id] = []
+
+            self.classification_history[object_id].append(("unknown", 0.0, timestamp))
+            return
+
+        if not self.tensor_input_details or not self.tensor_output_details:
+            return
+
+        input = np.expand_dims(resized_crop, axis=0)
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        probs = res / res.sum(axis=0)
+        logger.debug(
+            f"{self.model_config.name} Ran object classification with probabilities: {probs}"
+        )
+        best_id = int(np.argmax(probs))
+        score = round(probs[best_id], 2)
+        self.__update_metrics(datetime.datetime.now().timestamp() - timestamp)
+
+        save_attempts = (
+            self.model_config.save_attempts
+            if self.model_config.save_attempts is not None
+            else 200
+        )
+        write_classification_attempt(
+            self.train_dir,
+            crop_bgr,
+            object_id,
+            timestamp,
+            self.labelmap[best_id],
+            score,
+            max_files=save_attempts,
+        )
+
+        if score < self.model_config.threshold:
+            logger.debug(
+                f"{self.model_config.name}: Score {score} < threshold {self.model_config.threshold} for {object_id}, skipping"
+            )
+            return
+
+        sub_label = self.labelmap[best_id]
+
+        logger.debug(
+            f"{self.model_config.name}: Object {object_id} passed threshold with sub_label={sub_label}, score={score}"
+        )
+
+        consensus_label, consensus_score = self.get_weighted_score(
+            object_id, sub_label, score, timestamp
+        )
+
+        logger.debug(
+            f"{self.model_config.name}: get_weighted_score returned consensus_label={consensus_label}, consensus_score={consensus_score} for {object_id}"
+        )
+
+        if consensus_label is not None and self.model_config.object_config is not None:
+            self._emit_result(
+                {
+                    "type": "classification",
+                    "processor": "object",
+                    "model_name": self.model_config.name,
+                    "classification_type": self.model_config.object_config.classification_type,
+                    "object_id": object_id,
+                    "camera": camera,
+                    "timestamp": timestamp,
+                    "label": consensus_label,
+                    "score": consensus_score,
+                }
+            )
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if topic == EmbeddingsRequestEnum.reload_classification_model.value:
+            if request_data.get("model_name") == self.model_config.name:
+
+                def _do_reload(data: dict[str, Any]) -> dict[str, Any]:
+                    self.__build_detector()
+                    logger.info(
+                        f"Successfully loaded updated model for {self.model_config.name}"
+                    )
+                    return {
+                        "success": True,
+                        "message": f"Loaded {self.model_config.name} model.",
+                    }
+
+                result: dict[str, Any] = self._enqueue_request(_do_reload, request_data)
+                return result
+            else:
+                return None
+        else:
+            return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        self._enqueue_task(("expire", object_id))
+
+
+def write_classification_attempt(
+    folder: str,
+    frame: np.ndarray,
+    event_id: str,
+    timestamp: float,
+    label: str,
+    score: float,
+    max_files: int = 100,
+) -> None:
+    if "-" in label:
+        label = label.replace("-", "_")
+
+    file = os.path.join(folder, f"{event_id}-{timestamp}-{label}-{score}.webp")
+    os.makedirs(folder, exist_ok=True)
+    cv2.imwrite(file, frame)
+
+    # delete oldest face image if maximum is reached
+    try:
+        files = sorted(
+            filter(lambda f: f.endswith(".webp"), os.listdir(folder)),
+            key=lambda f: os.path.getctime(os.path.join(folder, f)),
+            reverse=True,
+        )
+
+        if len(files) > max_files:
+            os.unlink(os.path.join(folder, files[-1]))
+    except (FileNotFoundError, OSError):
+        pass

frigate/data_processing/real_time/face.py

@@ -0,0 +1,566 @@
+"""Handle processing images for face detection and recognition."""
+
+import base64
+import datetime
+import json
+import logging
+import os
+import shutil
+from pathlib import Path
+from typing import Any
+
+import cv2
+import numpy as np
+
+from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
+    EventMetadataTypeEnum,
+)
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.const import FACE_DIR, MODEL_CACHE_DIR
+from frigate.data_processing.common.face.model import (
+    ArcFaceRecognizer,
+    FaceNetRecognizer,
+    FaceRecognizer,
+)
+from frigate.types import TrackedObjectUpdateTypesEnum
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed
+from frigate.util.image import area
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+MAX_DETECTION_HEIGHT = 1080
+MAX_FACES_ATTEMPTS_AFTER_REC = 6
+MAX_FACE_ATTEMPTS = 12
+
+
+class FaceRealTimeProcessor(RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics)
+        self.face_config = config.face_recognition
+        self.requestor = requestor
+        self.sub_label_publisher = sub_label_publisher
+        self.face_detector: cv2.FaceDetectorYN | None = None
+        self.requires_face_detection = "face" not in self.config.objects.all_objects
+        self.person_face_history: dict[str, list[tuple[str, float, int]]] = {}
+        self.camera_current_people: dict[str, list[str]] = {}
+        self.recognizer: FaceRecognizer
+        self.faces_per_second = EventsPerSecond()
+        self.inference_speed = InferenceSpeed(self.metrics.face_rec_speed)
+
+        GITHUB_ENDPOINT = os.environ.get("GITHUB_ENDPOINT", "https://github.com")
+
+        download_path = os.path.join(MODEL_CACHE_DIR, "facedet")
+        self.model_files = {
+            "facedet.onnx": f"{GITHUB_ENDPOINT}/NickM-27/facenet-onnx/releases/download/v1.0/facedet.onnx",
+            "landmarkdet.yaml": f"{GITHUB_ENDPOINT}/NickM-27/facenet-onnx/releases/download/v1.0/landmarkdet.yaml",
+        }
+
+        if not all(
+            os.path.exists(os.path.join(download_path, n))
+            for n in self.model_files.keys()
+        ):
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            self.downloader = ModelDownloader(
+                model_name="facedet",
+                download_path=download_path,
+                file_names=list(self.model_files.keys()),
+                download_func=self.__download_models,
+                complete_func=self.__build_detector,
+            )
+            self.downloader.ensure_model_files()
+        else:
+            self.__build_detector()
+
+        self.label_map: dict[int, str] = {}
+
+        if self.face_config.model_size == "small":
+            self.recognizer = FaceNetRecognizer(self.config)
+        else:
+            self.recognizer = ArcFaceRecognizer(self.config)
+
+        self.recognizer.build()
+
+    CONFIG_UPDATE_TOPIC = "config/face_recognition"
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Update face recognition config at runtime."""
+        if topic != self.CONFIG_UPDATE_TOPIC:
+            return
+
+        previous_min_area = self.config.face_recognition.min_area
+        self.config.face_recognition = payload
+        self.face_config = payload
+
+        for camera_config in self.config.cameras.values():
+            if camera_config.face_recognition.min_area == previous_min_area:
+                camera_config.face_recognition.min_area = payload.min_area
+
+        logger.debug("Face recognition config updated dynamically")
+
+    def __download_models(self, path: str) -> None:
+        try:
+            file_name = os.path.basename(path)
+            # conditionally import ModelDownloader
+            from frigate.util.downloader import ModelDownloader
+
+            ModelDownloader.download_from_url(self.model_files[file_name], path)
+        except Exception as e:
+            logger.error(f"Failed to download {path}: {e}")
+
+    def __build_detector(self) -> None:
+        self.face_detector = cv2.FaceDetectorYN.create(
+            os.path.join(MODEL_CACHE_DIR, "facedet/facedet.onnx"),
+            config="",
+            input_size=(320, 320),
+            score_threshold=0.5,
+            nms_threshold=0.3,
+        )
+        self.faces_per_second.start()
+
+    def __detect_face(
+        self, input: np.ndarray, threshold: float
+    ) -> tuple[int, int, int, int] | None:
+        """Detect faces in input image."""
+        if not self.face_detector:
+            return None
+
+        # YN face detector fails at extreme definitions
+        # this rescales to a size that can properly detect faces
+        # still retaining plenty of detail
+        if input.shape[0] > MAX_DETECTION_HEIGHT:
+            scale_factor = MAX_DETECTION_HEIGHT / input.shape[0]
+            new_width = int(scale_factor * input.shape[1])
+            input = cv2.resize(input, (new_width, MAX_DETECTION_HEIGHT))
+        else:
+            scale_factor = 1
+
+        self.face_detector.setInputSize((input.shape[1], input.shape[0]))
+        faces = self.face_detector.detect(input)
+
+        if faces is None or faces[1] is None:
+            return None  # type: ignore[unreachable]
+
+        face = None
+
+        for _, potential_face in enumerate(faces[1]):
+            if potential_face[-1] < threshold:
+                continue
+
+            raw_bbox = potential_face[0:4].astype(np.uint16)
+            x: int = int(max(raw_bbox[0], 0) / scale_factor)
+            y: int = int(max(raw_bbox[1], 0) / scale_factor)
+            w: int = int(raw_bbox[2] / scale_factor)
+            h: int = int(raw_bbox[3] / scale_factor)
+            bbox = (x, y, x + w, y + h)
+
+            if face is None or area(bbox) > area(face):  # type: ignore[unreachable]
+                face = bbox
+
+        return face
+
+    def __update_metrics(self, duration: float) -> None:
+        self.faces_per_second.update()
+        self.inference_speed.update(duration)
+
+    def process_frame(self, obj_data: dict[str, Any], frame: np.ndarray) -> None:
+        """Look for faces in image."""
+        self.metrics.face_rec_fps.value = self.faces_per_second.eps()
+        camera = obj_data["camera"]
+
+        if not self.config.cameras[camera].face_recognition.enabled:
+            logger.debug(f"Face recognition disabled for camera {camera}, skipping")
+            return
+
+        start = datetime.datetime.now().timestamp()
+        id = obj_data["id"]
+
+        # don't run for non person objects
+        if obj_data.get("label") != "person":
+            logger.debug("Not processing face for a non person object.")
+            return
+
+        # don't overwrite sub label for objects that have a sub label
+        # that is not a face
+        if obj_data.get("sub_label") and id not in self.person_face_history:
+            logger.debug(
+                f"Not processing face due to existing sub label: {obj_data.get('sub_label')}."
+            )
+            return
+
+        # check if we have hit limits
+        if (
+            id in self.person_face_history
+            and len(self.person_face_history[id]) >= MAX_FACES_ATTEMPTS_AFTER_REC
+        ):
+            # if we are at max attempts after rec and we have a rec
+            if obj_data.get("sub_label"):
+                logger.debug(
+                    "Not processing due to hitting max attempts after true recognition."
+                )
+                return
+
+            # if we don't have a rec and are at max attempts
+            if len(self.person_face_history[id]) >= MAX_FACE_ATTEMPTS:
+                logger.debug("Not processing due to hitting max rec attempts.")
+                return
+
+        face: dict[str, Any] | None = None
+
+        if self.requires_face_detection:
+            logger.debug("Running manual face detection.")
+            person_box = obj_data.get("box")
+
+            if not person_box:
+                logger.debug(f"No person box available for {id}")
+                return
+
+            # YuNet (cv2.FaceDetectorYN) is trained on BGR
+            bgr = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+            left, top, right, bottom = person_box
+            person = bgr[top:bottom, left:right]
+            face_box = self.__detect_face(person, self.face_config.detection_threshold)
+
+            if not face_box:
+                logger.debug("Detected no faces for person object.")
+                return
+
+            face_frame = person[
+                max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
+                max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
+            ]
+
+            # check that face is correct size
+            if area(face_box) < self.config.cameras[camera].face_recognition.min_area:
+                logger.debug(
+                    f"Detected face that is smaller than the min_area {face} < {self.config.cameras[camera].face_recognition.min_area}"
+                )
+                return
+
+        else:
+            # don't run for object without attributes
+            if not obj_data.get("current_attributes"):
+                logger.debug("No attributes to parse.")
+                return
+
+            attributes: list[dict[str, Any]] = obj_data.get("current_attributes", [])
+            for attr in attributes:
+                if attr.get("label") != "face":
+                    continue
+
+                if face is None or attr.get("score", 0.0) > face.get("score", 0.0):
+                    face = attr
+
+            # no faces detected in this frame
+            if not face:
+                logger.debug(f"No face attributes found for {id}")
+                return
+
+            face_box = face.get("box")
+
+            # check that face is valid
+            if (
+                not face_box
+                or area(face_box)
+                < self.config.cameras[camera].face_recognition.min_area
+            ):
+                logger.debug(f"Invalid face box {face}")
+                return
+
+            face_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+
+            face_frame = face_frame[
+                max(0, face_box[1]) : min(frame.shape[0], face_box[3]),
+                max(0, face_box[0]) : min(frame.shape[1], face_box[2]),
+            ]
+
+        res = self.recognizer.classify(face_frame)
+
+        if not res:
+            logger.debug(f"Face recognizer returned no result for {id}")
+            self.__update_metrics(datetime.datetime.now().timestamp() - start)
+            return
+
+        sub_label, score = res
+
+        if score <= self.face_config.unknown_score:
+            sub_label = "unknown"
+
+        logger.debug(
+            f"Detected best face for person as: {sub_label} with probability {score}"
+        )
+
+        self.write_face_attempt(
+            face_frame, id, datetime.datetime.now().timestamp(), sub_label, score
+        )
+
+        if id not in self.person_face_history:
+            self.person_face_history[id] = []
+
+            if camera not in self.camera_current_people:
+                self.camera_current_people[camera] = []
+
+            self.camera_current_people[camera].append(id)
+
+        self.person_face_history[id].append(
+            (sub_label, score, face_frame.shape[0] * face_frame.shape[1])
+        )
+        (weighted_sub_label, weighted_score) = self.weighted_average(
+            self.person_face_history[id]
+        )
+
+        self.requestor.send_data(
+            "tracked_object_update",
+            json.dumps(
+                {
+                    "type": TrackedObjectUpdateTypesEnum.face,
+                    "name": weighted_sub_label,
+                    "score": weighted_score,
+                    "id": id,
+                    "camera": camera,
+                    "timestamp": start,
+                }
+            ),
+        )
+
+        if weighted_score >= self.face_config.recognition_threshold:
+            self.sub_label_publisher.publish(
+                (id, weighted_sub_label, weighted_score),
+                EventMetadataTypeEnum.sub_label.value,
+            )
+
+        self.__update_metrics(datetime.datetime.now().timestamp() - start)
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        if topic == EmbeddingsRequestEnum.clear_face_classifier.value:
+            self.recognizer.clear()
+            return {"success": True, "message": "Face classifier cleared."}
+        elif topic == EmbeddingsRequestEnum.recognize_face.value:
+            img = cv2.imdecode(
+                np.frombuffer(base64.b64decode(request_data["image"]), dtype=np.uint8),
+                cv2.IMREAD_COLOR,
+            )
+
+            # detect faces with lower confidence since we expect the face
+            # to be visible in uploaded images
+            face_box = self.__detect_face(img, 0.5)
+
+            if not face_box:
+                return {"message": "No face was detected.", "success": False}
+
+            face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
+            res = self.recognizer.classify(face)
+
+            if not res:
+                return {"success": False, "message": "No face was recognized."}
+
+            sub_label, score = res
+
+            if score <= self.face_config.unknown_score:
+                sub_label = "unknown"
+
+            return {"success": True, "score": score, "face_name": sub_label}
+        elif topic == EmbeddingsRequestEnum.register_face.value:
+            label = request_data["face_name"]
+
+            if request_data.get("cropped"):
+                thumbnail = request_data["image"]
+            else:
+                img = cv2.imdecode(
+                    np.frombuffer(
+                        base64.b64decode(request_data["image"]), dtype=np.uint8
+                    ),
+                    cv2.IMREAD_COLOR,
+                )
+
+                # detect faces with lower confidence since we expect the face
+                # to be visible in uploaded images
+                face_box = self.__detect_face(img, 0.5)
+
+                if not face_box:
+                    return {
+                        "message": "No face was detected.",
+                        "success": False,
+                    }
+
+                face = img[face_box[1] : face_box[3], face_box[0] : face_box[2]]
+                _, thumbnail = cv2.imencode(
+                    ".webp", face, [int(cv2.IMWRITE_WEBP_QUALITY), 100]
+                )
+
+            # write face to library
+            folder = os.path.join(FACE_DIR, label)
+            file = os.path.join(
+                folder, f"{label}_{datetime.datetime.now().timestamp()}.webp"
+            )
+            os.makedirs(folder, exist_ok=True)
+
+            # save face image
+            with open(file, "wb") as output:
+                output.write(thumbnail.tobytes())
+
+            self.recognizer.clear()
+            return {
+                "message": "Successfully registered face.",
+                "success": True,
+            }
+        elif topic == EmbeddingsRequestEnum.reprocess_face.value:
+            current_file: str = request_data["image_file"]
+            (id_time, id_rand, timestamp, _, _) = current_file.split("-")
+            img = None
+            id = f"{id_time}-{id_rand}"
+
+            if current_file:
+                img = cv2.imread(current_file)
+
+            if img is None:
+                return {  # type: ignore[unreachable]
+                    "message": "Invalid image file.",
+                    "success": False,
+                }
+
+            res = self.recognizer.classify(img)
+
+            if not res:
+                return {
+                    "message": "Model is still training, please try again in a few moments.",
+                    "success": False,
+                }
+
+            sub_label, score = res
+
+            if score <= self.face_config.unknown_score:
+                sub_label = "unknown"
+
+            if "-" in sub_label:
+                sub_label = sub_label.replace("-", "_")
+
+            if self.config.face_recognition.save_attempts:
+                # write face to library
+                folder = os.path.join(FACE_DIR, "train")
+                os.makedirs(folder, exist_ok=True)
+                new_file = os.path.join(
+                    folder, f"{id}-{timestamp}-{sub_label}-{score}.webp"
+                )
+                shutil.move(current_file, new_file)
+
+            return {
+                "message": f"Successfully reprocessed face. Result: {sub_label} (score: {score:.2f})",
+                "success": True,
+                "face_name": sub_label,
+                "score": score,
+            }
+
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        if object_id in self.person_face_history:
+            self.person_face_history.pop(object_id)
+
+            if object_id in self.camera_current_people.get(camera, []):
+                self.camera_current_people[camera].remove(object_id)
+
+    def weighted_average(
+        self, results_list: list[tuple[str, float, int]], max_weight: int = 4000
+    ) -> tuple[str | None, float]:
+        """
+        Calculates a robust weighted average, capping the area weight and giving more weight to higher scores.
+
+        Args:
+            results_list: A list of tuples, where each tuple contains (name, score, face_area).
+            max_weight: The maximum weight to apply based on face area.
+
+        Returns:
+            A tuple containing the prominent name and its weighted average score, or (None, 0.0) if the list is empty.
+        """
+        if not results_list:
+            return None, 0.0
+
+        counts: dict[str, int] = {}
+        weighted_scores: dict[str, float] = {}
+        total_weights: dict[str, float] = {}
+
+        for name, score, face_area in results_list:
+            if name == "unknown":
+                continue
+
+            if name not in weighted_scores:
+                counts[name] = 0
+                weighted_scores[name] = 0.0
+                total_weights[name] = 0.0
+
+            # increase count
+            counts[name] += 1
+
+            # Capped weight based on face area
+            weight: float = min(face_area, max_weight)
+
+            # Score-based weighting (higher scores get more weight)
+            weight *= (score - self.face_config.unknown_score) * 10
+            weighted_scores[name] += score * weight
+            total_weights[name] += weight
+
+        if not weighted_scores:
+            return None, 0.0
+
+        best_name = max(weighted_scores, key=lambda k: weighted_scores[k])
+
+        # If the number of faces for this person < min_faces, we are not confident it is a correct result
+        if counts[best_name] < self.face_config.min_faces:
+            return None, 0.0
+
+        # If the best name has the same number of results as another name, we are not confident it is a correct result
+        for name, count in counts.items():
+            if name != best_name and counts[best_name] == count:
+                return None, 0.0
+
+        weighted_average = weighted_scores[best_name] / total_weights[best_name]
+
+        return best_name, weighted_average
+
+    def write_face_attempt(
+        self,
+        frame: np.ndarray,
+        event_id: str,
+        timestamp: float,
+        sub_label: str,
+        score: float,
+    ) -> None:
+        if self.config.face_recognition.save_attempts:
+            # write face to library
+            folder = os.path.join(FACE_DIR, "train")
+
+            if "-" in sub_label:
+                sub_label = sub_label.replace("-", "_")
+
+            file = os.path.join(
+                folder, f"{event_id}-{timestamp}-{sub_label}-{score}.webp"
+            )
+            os.makedirs(folder, exist_ok=True)
+            cv2.imwrite(file, frame)
+
+            files = sorted(
+                filter(lambda f: f.endswith(".webp"), os.listdir(folder)),
+                key=lambda f: os.path.getctime(os.path.join(folder, f)),
+                reverse=True,
+            )
+
+            # delete oldest face image if maximum is reached
+            if len(files) > self.config.face_recognition.save_attempts:
+                Path(os.path.join(folder, files[-1])).unlink(missing_ok=True)

frigate/data_processing/real_time/license_plate.py

@@ -0,0 +1,76 @@
+"""Handle processing images for face detection and recognition."""
+
+import logging
+from typing import Any
+
+import numpy as np
+
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.data_processing.common.license_plate.mixin import (
+    LicensePlateProcessingMixin,
+)
+from frigate.data_processing.common.license_plate.model import (
+    LicensePlateModelRunner,
+)
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+logger = logging.getLogger(__name__)
+
+
+class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+        model_runner: LicensePlateModelRunner,
+        detected_license_plates: dict[str, dict[str, Any]],
+    ):
+        self.requestor = requestor
+        self.detected_license_plates = detected_license_plates
+        self.model_runner = model_runner
+        self.lpr_config = config.lpr
+        self.config = config
+        self.sub_label_publisher = sub_label_publisher
+        self.camera_current_cars: dict[str, list[str]] = {}
+        super().__init__(config, metrics)
+
+    CONFIG_UPDATE_TOPIC = "config/lpr"
+
+    def update_config(self, topic: str, payload: Any) -> None:
+        """Update LPR config at runtime."""
+        if topic != self.CONFIG_UPDATE_TOPIC:
+            return
+
+        previous_min_area = self.config.lpr.min_area
+        self.config.lpr = payload
+        self.lpr_config = payload
+
+        for camera_config in self.config.cameras.values():
+            if camera_config.lpr.min_area == previous_min_area:
+                camera_config.lpr.min_area = payload.min_area
+
+        logger.debug("LPR config updated dynamically")
+
+    def process_frame(
+        self,
+        obj_data: dict[str, Any],
+        frame: np.ndarray,
+        dedicated_lpr: bool = False,
+    ) -> None:
+        """Look for license plates in image."""
+        self.lpr_process(obj_data, frame, dedicated_lpr)
+
+    def handle_request(
+        self, topic: str, request_data: dict[str, Any]
+    ) -> dict[str, Any] | None:
+        return None
+
+    def expire_object(self, object_id: str, camera: str) -> None:
+        """Expire lpr objects."""
+        self.lpr_expire(object_id, camera)