chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,65 @@
|
||||
"""Local or remote processors to handle post processing."""
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
from frigate.config import FrigateConfig
|
||||
|
||||
from ..types import DataProcessorMetrics, DataProcessorModelRunner, PostProcessDataEnum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PostProcessorApi(ABC):
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
self,
|
||||
config: FrigateConfig,
|
||||
metrics: DataProcessorMetrics,
|
||||
model_runner: DataProcessorModelRunner | None,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.metrics = metrics
|
||||
self.model_runner = model_runner
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def process_data(
|
||||
self, data: dict[str, Any], data_type: PostProcessDataEnum
|
||||
) -> None:
|
||||
"""Processes the data of data type.
|
||||
Args:
|
||||
data (dict): containing data about the input.
|
||||
data_type (enum): Describing the data that is being processed.
|
||||
|
||||
Returns:
|
||||
None.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def handle_request(
|
||||
self, topic: str, request_data: dict[str, Any]
|
||||
) -> dict[str, Any] | str | None:
|
||||
"""Handle metadata requests.
|
||||
Args:
|
||||
request_data (dict): containing data about requested change to process.
|
||||
|
||||
Returns:
|
||||
None if request was not handled, otherwise return response.
|
||||
"""
|
||||
pass
|
||||
|
||||
def update_config(self, topic: str, payload: Any) -> None:
|
||||
"""Handle a config change notification.
|
||||
|
||||
Called for every config update published under ``config/``.
|
||||
Processors should override this to check the topic and act only
|
||||
on changes relevant to them. Default is a no-op.
|
||||
|
||||
Args:
|
||||
topic: The config topic that changed.
|
||||
payload: The updated configuration object.
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Handle post-processing for audio transcription."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from peewee import DoesNotExist
|
||||
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.const import (
|
||||
CACHE_DIR,
|
||||
MODEL_CACHE_DIR,
|
||||
UPDATE_AUDIO_TRANSCRIPTION_STATE,
|
||||
UPDATE_EVENT_DESCRIPTION,
|
||||
)
|
||||
from frigate.data_processing.types import PostProcessDataEnum
|
||||
from frigate.embeddings.embeddings import Embeddings
|
||||
from frigate.types import TrackedObjectUpdateTypesEnum
|
||||
from frigate.util.audio import get_audio_from_recording
|
||||
|
||||
from ..types import DataProcessorMetrics
|
||||
from .api import PostProcessorApi
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AudioTranscriptionPostProcessor(PostProcessorApi):
|
||||
def __init__(
|
||||
self,
|
||||
config: FrigateConfig,
|
||||
requestor: InterProcessRequestor,
|
||||
embeddings: Embeddings,
|
||||
metrics: DataProcessorMetrics,
|
||||
):
|
||||
super().__init__(config, metrics, None)
|
||||
self.config = config
|
||||
self.requestor = requestor
|
||||
self.embeddings = embeddings
|
||||
self.recognizer = None
|
||||
self.transcription_lock = threading.Lock()
|
||||
self.transcription_thread: threading.Thread | None = None
|
||||
self.transcription_running = False
|
||||
|
||||
# faster-whisper handles model downloading automatically
|
||||
self.model_path = os.path.join(MODEL_CACHE_DIR, "whisper")
|
||||
os.makedirs(self.model_path, exist_ok=True)
|
||||
|
||||
self.__build_recognizer()
|
||||
|
||||
def __build_recognizer(self) -> None:
|
||||
try:
|
||||
# Import dynamically to avoid crashes on systems without AVX support
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
self.recognizer = WhisperModel(
|
||||
model_size_or_path="small",
|
||||
device="cuda"
|
||||
if self.config.audio_transcription.device == "GPU"
|
||||
else "cpu",
|
||||
download_root=self.model_path,
|
||||
local_files_only=False, # Allow downloading if not cached
|
||||
compute_type="int8",
|
||||
)
|
||||
logger.debug("Audio transcription (recordings) initialized")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize recordings audio transcription: {e}")
|
||||
self.recognizer = None
|
||||
|
||||
def process_data(
|
||||
self, data: dict[str, Any], data_type: PostProcessDataEnum
|
||||
) -> None:
|
||||
"""Transcribe audio from a recording.
|
||||
|
||||
Args:
|
||||
data (dict): Contains data about the input (event_id, camera, etc.).
|
||||
data_type (enum): Describes the data being processed (recording or tracked_object).
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
event_id = data["event_id"]
|
||||
camera_name = data["camera"]
|
||||
|
||||
if data_type == PostProcessDataEnum.recording:
|
||||
start_ts = data["frame_time"]
|
||||
recordings_available_through = data["recordings_available"]
|
||||
end_ts = min(recordings_available_through, start_ts + 60) # Default 60s
|
||||
|
||||
elif data_type == PostProcessDataEnum.tracked_object:
|
||||
obj_data = data["event"]["data"]
|
||||
obj_data["id"] = data["event"]["id"]
|
||||
obj_data["camera"] = data["event"]["camera"]
|
||||
start_ts = data["event"]["start_time"]
|
||||
end_ts = data["event"].get(
|
||||
"end_time", start_ts + 60
|
||||
) # Use end_time if available
|
||||
|
||||
else:
|
||||
logger.error("No data type passed to audio transcription post-processing")
|
||||
return
|
||||
|
||||
try:
|
||||
audio_data = get_audio_from_recording(
|
||||
self.config.cameras[camera_name].ffmpeg,
|
||||
camera_name,
|
||||
start_ts,
|
||||
end_ts,
|
||||
sample_rate=16000,
|
||||
)
|
||||
|
||||
if not audio_data:
|
||||
logger.debug(f"No audio data extracted for {event_id}")
|
||||
return
|
||||
|
||||
transcription = self.__transcribe_audio(audio_data)
|
||||
if not transcription:
|
||||
logger.debug("No transcription generated from audio")
|
||||
return
|
||||
|
||||
logger.debug(f"Transcribed audio for {event_id}: '{transcription}'")
|
||||
|
||||
self.requestor.send_data(
|
||||
UPDATE_EVENT_DESCRIPTION,
|
||||
{
|
||||
"type": TrackedObjectUpdateTypesEnum.description,
|
||||
"id": event_id,
|
||||
"description": transcription,
|
||||
"camera": camera_name,
|
||||
},
|
||||
)
|
||||
|
||||
# Embed the description if semantic search is enabled
|
||||
if self.config.semantic_search.enabled:
|
||||
self.embeddings.embed_description(event_id, transcription)
|
||||
|
||||
except DoesNotExist:
|
||||
logger.debug("No recording found for audio transcription post-processing")
|
||||
return
|
||||
except Exception as e:
|
||||
logger.error(f"Error in audio transcription post-processing: {e}")
|
||||
|
||||
def __transcribe_audio(self, audio_data: bytes) -> str | None:
|
||||
"""Transcribe WAV audio data using faster-whisper."""
|
||||
if not self.recognizer:
|
||||
logger.debug("Recognizer not initialized")
|
||||
return None
|
||||
|
||||
try: # type: ignore[unreachable]
|
||||
# Save audio data to a temporary wav (faster-whisper expects a file)
|
||||
temp_wav = os.path.join(CACHE_DIR, f"temp_audio_{int(time.time())}.wav")
|
||||
with open(temp_wav, "wb") as f:
|
||||
f.write(audio_data)
|
||||
|
||||
segments, info = self.recognizer.transcribe(
|
||||
temp_wav,
|
||||
language=self.config.audio_transcription.language,
|
||||
beam_size=5,
|
||||
)
|
||||
|
||||
os.remove(temp_wav)
|
||||
|
||||
# Combine all segment texts
|
||||
text = " ".join(segment.text.strip() for segment in segments)
|
||||
if not text:
|
||||
return None
|
||||
|
||||
logger.debug(
|
||||
"Detected language '%s' with probability %f",
|
||||
info.language,
|
||||
info.language_probability,
|
||||
)
|
||||
|
||||
return text
|
||||
except Exception as e:
|
||||
logger.error(f"Error transcribing audio: {e}")
|
||||
return None
|
||||
|
||||
def _transcription_wrapper(self, event: dict[str, Any]) -> None:
|
||||
"""Wrapper to run transcription and reset running flag when done."""
|
||||
try:
|
||||
self.process_data(
|
||||
{
|
||||
"event_id": event["id"],
|
||||
"camera": event["camera"],
|
||||
"event": event,
|
||||
},
|
||||
PostProcessDataEnum.tracked_object,
|
||||
)
|
||||
finally:
|
||||
with self.transcription_lock:
|
||||
self.transcription_running = False
|
||||
self.transcription_thread = None
|
||||
|
||||
self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "idle")
|
||||
|
||||
def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
|
||||
if topic == "transcribe_audio":
|
||||
event = request_data["event"]
|
||||
|
||||
with self.transcription_lock:
|
||||
if self.transcription_running:
|
||||
logger.warning(
|
||||
"Audio transcription for a speech event is already running."
|
||||
)
|
||||
return "in_progress"
|
||||
|
||||
# Mark as running and start the thread
|
||||
self.transcription_running = True
|
||||
self.requestor.send_data(UPDATE_AUDIO_TRANSCRIPTION_STATE, "processing")
|
||||
|
||||
self.transcription_thread = threading.Thread(
|
||||
target=self._transcription_wrapper, args=(event,), daemon=True
|
||||
)
|
||||
self.transcription_thread.start()
|
||||
return "started"
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Handle post processing for license plate recognition."""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from peewee import DoesNotExist
|
||||
|
||||
from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
|
||||
from frigate.comms.event_metadata_updater import EventMetadataPublisher
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.data_processing.common.license_plate.mixin import (
|
||||
WRITE_DEBUG_IMAGES,
|
||||
LicensePlateProcessingMixin,
|
||||
)
|
||||
from frigate.data_processing.common.license_plate.model import (
|
||||
LicensePlateModelRunner,
|
||||
)
|
||||
from frigate.data_processing.types import PostProcessDataEnum
|
||||
from frigate.models import Recordings
|
||||
from frigate.util.image import get_image_from_recording
|
||||
|
||||
from ..types import DataProcessorMetrics
|
||||
from .api import PostProcessorApi
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi): # type: ignore[misc]
|
||||
def __init__(
|
||||
self,
|
||||
config: FrigateConfig,
|
||||
requestor: InterProcessRequestor,
|
||||
sub_label_publisher: EventMetadataPublisher,
|
||||
metrics: DataProcessorMetrics,
|
||||
model_runner: LicensePlateModelRunner,
|
||||
detected_license_plates: dict[str, dict[str, Any]],
|
||||
):
|
||||
self.requestor = requestor
|
||||
self.detected_license_plates = detected_license_plates
|
||||
self.model_runner = model_runner
|
||||
self.lpr_config = config.lpr
|
||||
self.config = config
|
||||
self.sub_label_publisher = sub_label_publisher
|
||||
super().__init__(config, metrics, model_runner)
|
||||
|
||||
CONFIG_UPDATE_TOPIC = "config/lpr"
|
||||
|
||||
def update_config(self, topic: str, payload: Any) -> None:
|
||||
"""Update LPR config at runtime."""
|
||||
if topic != self.CONFIG_UPDATE_TOPIC:
|
||||
return
|
||||
|
||||
self.lpr_config = payload
|
||||
logger.debug("LPR post-processor config updated dynamically")
|
||||
|
||||
def process_data(
|
||||
self, data: dict[str, Any], data_type: PostProcessDataEnum
|
||||
) -> None:
|
||||
"""Look for license plates in recording stream image
|
||||
Args:
|
||||
data (dict): containing data about the input.
|
||||
data_type (enum): Describing the data that is being processed.
|
||||
|
||||
Returns:
|
||||
None.
|
||||
"""
|
||||
# don't run LPR post processing for now
|
||||
return
|
||||
|
||||
event_id = data["event_id"] # type: ignore[unreachable]
|
||||
camera_name = data["camera"]
|
||||
|
||||
if data_type == PostProcessDataEnum.recording:
|
||||
obj_data = data["obj_data"]
|
||||
frame_time = obj_data["frame_time"]
|
||||
recordings_available_through = data["recordings_available"]
|
||||
|
||||
if frame_time > recordings_available_through:
|
||||
logger.debug(
|
||||
f"LPR post processing: No recordings available for this frame time {frame_time}, available through {recordings_available_through}"
|
||||
)
|
||||
|
||||
elif data_type == PostProcessDataEnum.tracked_object:
|
||||
# non-functional, need to think about snapshot time
|
||||
obj_data = data["event"]["data"]
|
||||
obj_data["id"] = data["event"]["id"]
|
||||
obj_data["camera"] = data["event"]["camera"]
|
||||
# TODO: snapshot time?
|
||||
frame_time = data["event"]["start_time"]
|
||||
|
||||
else:
|
||||
logger.error("No data type passed to LPR postprocessing")
|
||||
return
|
||||
|
||||
recording_query = (
|
||||
Recordings.select(
|
||||
Recordings.path,
|
||||
Recordings.start_time,
|
||||
)
|
||||
.where(
|
||||
(frame_time >= Recordings.start_time)
|
||||
& (frame_time <= Recordings.end_time)
|
||||
)
|
||||
.where(Recordings.camera == camera_name)
|
||||
.order_by(Recordings.start_time.desc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
try:
|
||||
recording: Recordings = recording_query.get()
|
||||
time_in_segment = frame_time - recording.start_time
|
||||
codec = "mjpeg"
|
||||
|
||||
image_data = get_image_from_recording(
|
||||
self.config.ffmpeg, recording.path, time_in_segment, codec, None
|
||||
)
|
||||
|
||||
if not image_data:
|
||||
logger.debug(
|
||||
"LPR post processing: Unable to fetch license plate from recording"
|
||||
)
|
||||
|
||||
# Convert bytes to numpy array
|
||||
image_array = np.frombuffer(image_data, dtype=np.uint8)
|
||||
|
||||
if len(image_array) == 0:
|
||||
logger.debug("LPR post processing: No image")
|
||||
return
|
||||
|
||||
image = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
|
||||
|
||||
except DoesNotExist:
|
||||
logger.debug("Error fetching license plate for postprocessing")
|
||||
return
|
||||
|
||||
if WRITE_DEBUG_IMAGES:
|
||||
cv2.imwrite(
|
||||
f"debug/frames/lpr_post_{datetime.datetime.now().timestamp()}.jpg",
|
||||
image,
|
||||
)
|
||||
|
||||
# convert to yuv for processing
|
||||
frame = cv2.cvtColor(image, cv2.COLOR_BGR2YUV_I420)
|
||||
|
||||
detect_width = self.config.cameras[camera_name].detect.width
|
||||
detect_height = self.config.cameras[camera_name].detect.height
|
||||
|
||||
# Scale the boxes based on detect dimensions
|
||||
scale_x = image.shape[1] / detect_width
|
||||
scale_y = image.shape[0] / detect_height
|
||||
|
||||
# Determine which box to enlarge based on detection mode
|
||||
if "license_plate" not in self.config.cameras[camera_name].objects.track:
|
||||
# Scale and enlarge the car box
|
||||
box = obj_data.get("box")
|
||||
if not box:
|
||||
return
|
||||
|
||||
# Scale original car box to detection dimensions
|
||||
left = int(box[0] * scale_x)
|
||||
top = int(box[1] * scale_y)
|
||||
right = int(box[2] * scale_x)
|
||||
bottom = int(box[3] * scale_y)
|
||||
box = [left, top, right, bottom]
|
||||
else:
|
||||
# Get the license plate box from attributes
|
||||
if not obj_data.get("current_attributes"):
|
||||
return
|
||||
|
||||
license_plate = None
|
||||
for attr in obj_data["current_attributes"]:
|
||||
if attr.get("label") != "license_plate":
|
||||
continue
|
||||
if license_plate is None or attr.get("score", 0.0) > license_plate.get(
|
||||
"score", 0.0
|
||||
):
|
||||
license_plate = attr
|
||||
|
||||
if not license_plate or not license_plate.get("box"):
|
||||
return
|
||||
|
||||
# Scale license plate box to detection dimensions
|
||||
orig_box = license_plate["box"]
|
||||
left = int(orig_box[0] * scale_x)
|
||||
top = int(orig_box[1] * scale_y)
|
||||
right = int(orig_box[2] * scale_x)
|
||||
bottom = int(orig_box[3] * scale_y)
|
||||
box = [left, top, right, bottom]
|
||||
|
||||
width_box = right - left
|
||||
height_box = bottom - top
|
||||
|
||||
# Enlarge box slightly to account for drift in detect vs recording stream
|
||||
enlarge_factor = 0.3
|
||||
new_left = max(0, int(left - (width_box * enlarge_factor / 2)))
|
||||
new_top = max(0, int(top - (height_box * enlarge_factor / 2)))
|
||||
new_right = min(image.shape[1], int(right + (width_box * enlarge_factor / 2)))
|
||||
new_bottom = min(
|
||||
image.shape[0], int(bottom + (height_box * enlarge_factor / 2))
|
||||
)
|
||||
|
||||
keyframe_obj_data = obj_data.copy()
|
||||
if "license_plate" not in self.config.cameras[camera_name].objects.track:
|
||||
# car box
|
||||
keyframe_obj_data["box"] = [new_left, new_top, new_right, new_bottom]
|
||||
else:
|
||||
# Update the license plate box in the attributes
|
||||
new_attributes = []
|
||||
for attr in obj_data["current_attributes"]:
|
||||
if attr.get("label") == "license_plate":
|
||||
new_attr = attr.copy()
|
||||
new_attr["box"] = [new_left, new_top, new_right, new_bottom]
|
||||
new_attributes.append(new_attr)
|
||||
else:
|
||||
new_attributes.append(attr)
|
||||
keyframe_obj_data["current_attributes"] = new_attributes
|
||||
|
||||
# run the frame through lpr processing
|
||||
logger.debug(f"Post processing plate: {event_id}, {frame_time}")
|
||||
self.lpr_process(keyframe_obj_data, frame)
|
||||
|
||||
def handle_request(self, topic: str, request_data: dict) -> dict[str, Any] | None:
|
||||
if topic == EmbeddingsRequestEnum.reprocess_plate.value:
|
||||
event = request_data["event"]
|
||||
|
||||
self.process_data(
|
||||
{
|
||||
"event_id": event["id"],
|
||||
"camera": event["camera"],
|
||||
"event": event,
|
||||
},
|
||||
PostProcessDataEnum.tracked_object,
|
||||
)
|
||||
|
||||
return {
|
||||
"message": "Successfully requested reprocessing of license plate.",
|
||||
"success": True,
|
||||
}
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,381 @@
|
||||
"""Post processor for object descriptions using GenAI."""
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from peewee import DoesNotExist
|
||||
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import CameraConfig, FrigateConfig
|
||||
from frigate.const import CLIPS_DIR, UPDATE_EVENT_DESCRIPTION
|
||||
from frigate.data_processing.post.semantic_trigger import SemanticTriggerProcessor
|
||||
from frigate.data_processing.types import PostProcessDataEnum
|
||||
from frigate.genai.manager import GenAIClientManager
|
||||
from frigate.models import Event
|
||||
from frigate.types import TrackedObjectUpdateTypesEnum
|
||||
from frigate.util.builtin import EventsPerSecond, InferenceSpeed
|
||||
from frigate.util.file import get_event_thumbnail_bytes, load_event_snapshot_image
|
||||
from frigate.util.image import create_thumbnail, ensure_jpeg_bytes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from frigate.embeddings.embeddings import Embeddings
|
||||
|
||||
from ..post.api import PostProcessorApi
|
||||
from ..types import DataProcessorMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_THUMBNAILS = 10
|
||||
|
||||
|
||||
class ObjectDescriptionProcessor(PostProcessorApi):
|
||||
def __init__(
|
||||
self,
|
||||
config: FrigateConfig,
|
||||
embeddings: "Embeddings",
|
||||
requestor: InterProcessRequestor,
|
||||
metrics: DataProcessorMetrics,
|
||||
genai_manager: GenAIClientManager,
|
||||
semantic_trigger_processor: SemanticTriggerProcessor | None,
|
||||
):
|
||||
super().__init__(config, metrics, None)
|
||||
self.config = config
|
||||
self.embeddings = embeddings
|
||||
self.requestor = requestor
|
||||
self.metrics = metrics
|
||||
self.genai_manager = genai_manager
|
||||
self.semantic_trigger_processor = semantic_trigger_processor
|
||||
self.tracked_events: dict[str, list[Any]] = {}
|
||||
self.early_request_sent: dict[str, bool] = {}
|
||||
self.object_desc_speed = InferenceSpeed(self.metrics.object_desc_speed)
|
||||
self.object_desc_dps = EventsPerSecond()
|
||||
self.object_desc_dps.start()
|
||||
|
||||
def __handle_frame_update(
|
||||
self, camera: str, data: dict, yuv_frame: np.ndarray
|
||||
) -> None:
|
||||
"""Handle an update to a frame for an object."""
|
||||
camera_config = self.config.cameras[camera]
|
||||
|
||||
# no need to save our own thumbnails if genai is not enabled
|
||||
# or if the object has become stationary
|
||||
if not data["stationary"]:
|
||||
if data["id"] not in self.tracked_events:
|
||||
self.tracked_events[data["id"]] = []
|
||||
|
||||
data["thumbnail"] = create_thumbnail(yuv_frame, data["box"])
|
||||
|
||||
# Limit the number of thumbnails saved
|
||||
if len(self.tracked_events[data["id"]]) >= MAX_THUMBNAILS:
|
||||
# Always keep the first thumbnail for the event
|
||||
self.tracked_events[data["id"]].pop(1)
|
||||
|
||||
self.tracked_events[data["id"]].append(data)
|
||||
|
||||
# check if we're configured to send an early request after a minimum number of updates received
|
||||
if camera_config.objects.genai.send_triggers.after_significant_updates:
|
||||
if (
|
||||
len(self.tracked_events.get(data["id"], []))
|
||||
>= camera_config.objects.genai.send_triggers.after_significant_updates
|
||||
and data["id"] not in self.early_request_sent
|
||||
):
|
||||
if data["has_clip"] and data["has_snapshot"]:
|
||||
try:
|
||||
event: Event = Event.get(Event.id == data["id"])
|
||||
except DoesNotExist:
|
||||
logger.error(f"Event {data['id']} not found")
|
||||
return
|
||||
|
||||
if (
|
||||
not camera_config.objects.genai.objects
|
||||
or event.label in camera_config.objects.genai.objects
|
||||
) and (
|
||||
not camera_config.objects.genai.required_zones
|
||||
or set(data["entered_zones"])
|
||||
& set(camera_config.objects.genai.required_zones)
|
||||
):
|
||||
logger.debug(f"{camera} sending early request to GenAI")
|
||||
|
||||
self.early_request_sent[data["id"]] = True
|
||||
# Copy thumbnails to avoid holding references after cleanup
|
||||
thumbnails_copy = [
|
||||
data["thumbnail"][:] if data.get("thumbnail") else None
|
||||
for data in self.tracked_events[data["id"]]
|
||||
if data.get("thumbnail")
|
||||
]
|
||||
threading.Thread(
|
||||
target=self._genai_embed_description,
|
||||
name=f"_genai_embed_description_{event.id}",
|
||||
daemon=True,
|
||||
args=(
|
||||
event,
|
||||
thumbnails_copy,
|
||||
),
|
||||
).start()
|
||||
|
||||
def __handle_frame_finalize(
|
||||
self, camera: str, event: Event, thumbnail: bytes
|
||||
) -> None:
|
||||
"""Handle the finalization of a frame."""
|
||||
camera_config = self.config.cameras[camera]
|
||||
|
||||
if (
|
||||
camera_config.objects.genai.enabled
|
||||
and camera_config.objects.genai.send_triggers.tracked_object_end
|
||||
and (
|
||||
not camera_config.objects.genai.objects
|
||||
or event.label in camera_config.objects.genai.objects
|
||||
)
|
||||
and (
|
||||
not camera_config.objects.genai.required_zones
|
||||
or set(event.zones) & set(camera_config.objects.genai.required_zones)
|
||||
)
|
||||
):
|
||||
self._process_genai_description(event, camera_config, thumbnail)
|
||||
else:
|
||||
self.cleanup_event(str(event.id))
|
||||
|
||||
def __regenerate_description(self, event_id: str, source: str, force: bool) -> None:
|
||||
"""Regenerate the description for an event."""
|
||||
try:
|
||||
event: Event = Event.get(Event.id == event_id)
|
||||
except DoesNotExist:
|
||||
logger.error(f"Event {event_id} not found for description regeneration")
|
||||
return
|
||||
|
||||
camera_config = self.config.cameras[str(event.camera)]
|
||||
if not camera_config.objects.genai.enabled and not force:
|
||||
logger.error(f"GenAI not enabled for camera {event.camera}")
|
||||
return
|
||||
|
||||
thumbnail = get_event_thumbnail_bytes(event)
|
||||
|
||||
if thumbnail is None:
|
||||
logger.error("No thumbnail available for %s", event.id)
|
||||
return
|
||||
|
||||
# ensure we have a jpeg to pass to the model
|
||||
thumbnail = ensure_jpeg_bytes(thumbnail)
|
||||
|
||||
logger.debug(
|
||||
f"Trying {source} regeneration for {event}, has_snapshot: {event.has_snapshot}"
|
||||
)
|
||||
|
||||
if event.has_snapshot and source == "snapshot":
|
||||
snapshot_image = self._read_and_crop_snapshot(event)
|
||||
if not snapshot_image:
|
||||
return
|
||||
|
||||
embed_image = (
|
||||
[snapshot_image]
|
||||
if event.has_snapshot and source == "snapshot"
|
||||
# Copy thumbnails to avoid holding references
|
||||
else (
|
||||
[
|
||||
data["thumbnail"][:] if data.get("thumbnail") else None
|
||||
for data in self.tracked_events[event_id]
|
||||
if data.get("thumbnail")
|
||||
]
|
||||
if len(self.tracked_events.get(event_id, [])) > 0
|
||||
else [thumbnail]
|
||||
)
|
||||
)
|
||||
|
||||
self._genai_embed_description(
|
||||
event, [img for img in embed_image if img is not None]
|
||||
)
|
||||
|
||||
def process_data(self, frame_data: dict, data_type: PostProcessDataEnum) -> None:
|
||||
"""Process a frame update."""
|
||||
self.metrics.object_desc_dps.value = self.object_desc_dps.eps()
|
||||
|
||||
if data_type != PostProcessDataEnum.tracked_object:
|
||||
return
|
||||
|
||||
if self.genai_manager.description_client is None:
|
||||
return
|
||||
|
||||
state: str | None = frame_data.get("state", None)
|
||||
|
||||
if state is not None:
|
||||
logger.debug(f"Processing {state} for {frame_data['camera']}")
|
||||
|
||||
if state == "update":
|
||||
self.__handle_frame_update(
|
||||
frame_data["camera"], frame_data["data"], frame_data["yuv_frame"]
|
||||
)
|
||||
elif state == "finalize":
|
||||
self.__handle_frame_finalize(
|
||||
frame_data["camera"], frame_data["event"], frame_data["thumbnail"]
|
||||
)
|
||||
|
||||
def handle_request(self, topic: str, data: dict[str, Any]) -> str | None:
|
||||
"""Handle a request."""
|
||||
if topic == "regenerate_description":
|
||||
self.__regenerate_description(
|
||||
data["event_id"], data["source"], data["force"]
|
||||
)
|
||||
return None
|
||||
|
||||
def cleanup_event(self, event_id: str) -> None:
|
||||
"""Clean up tracked event data to prevent memory leaks.
|
||||
|
||||
This should be called when an event ends, regardless of whether
|
||||
genai processing is triggered.
|
||||
"""
|
||||
if event_id in self.tracked_events:
|
||||
del self.tracked_events[event_id]
|
||||
if event_id in self.early_request_sent:
|
||||
del self.early_request_sent[event_id]
|
||||
|
||||
def _read_and_crop_snapshot(self, event: Event) -> bytes | None:
|
||||
"""Read, decode, and crop the snapshot image."""
|
||||
|
||||
try:
|
||||
img, _ = load_event_snapshot_image(event)
|
||||
if img is None:
|
||||
logger.error(f"Cannot load snapshot for {event.id}, file not found")
|
||||
return None
|
||||
|
||||
# Crop snapshot based on region
|
||||
# provide full image if region doesn't exist (manual events)
|
||||
height, width = img.shape[:2]
|
||||
x1_rel, y1_rel, width_rel, height_rel = event.data.get( # type: ignore[attr-defined]
|
||||
"region", [0, 0, 1, 1]
|
||||
)
|
||||
x1, y1 = int(x1_rel * width), int(y1_rel * height)
|
||||
|
||||
cropped_image = img[
|
||||
y1 : y1 + int(height_rel * height),
|
||||
x1 : x1 + int(width_rel * width),
|
||||
]
|
||||
|
||||
_, buffer = cv2.imencode(".jpg", cropped_image)
|
||||
|
||||
return buffer.tobytes()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _process_genai_description(
|
||||
self, event: Event, camera_config: CameraConfig, thumbnail: bytes
|
||||
) -> None:
|
||||
event_id = str(event.id)
|
||||
|
||||
if event.has_snapshot and camera_config.objects.genai.use_snapshot:
|
||||
snapshot_image = self._read_and_crop_snapshot(event)
|
||||
|
||||
if not snapshot_image:
|
||||
self.cleanup_event(event_id)
|
||||
return
|
||||
|
||||
num_thumbnails = len(self.tracked_events.get(event_id, []))
|
||||
|
||||
# ensure we have a jpeg to pass to the model
|
||||
thumbnail = ensure_jpeg_bytes(thumbnail)
|
||||
|
||||
embed_image = (
|
||||
[snapshot_image]
|
||||
if event.has_snapshot and camera_config.objects.genai.use_snapshot
|
||||
# Copy thumbnails to avoid holding references after cleanup
|
||||
else (
|
||||
[
|
||||
data["thumbnail"][:] if data.get("thumbnail") else None
|
||||
for data in self.tracked_events[event_id]
|
||||
if data.get("thumbnail")
|
||||
]
|
||||
if num_thumbnails > 0
|
||||
else [thumbnail]
|
||||
)
|
||||
)
|
||||
|
||||
if camera_config.objects.genai.debug_save_thumbnails and num_thumbnails > 0:
|
||||
logger.debug(f"Saving {num_thumbnails} thumbnails for event {event_id}")
|
||||
|
||||
Path(os.path.join(CLIPS_DIR, f"genai-requests/{event_id}")).mkdir(
|
||||
parents=True, exist_ok=True
|
||||
)
|
||||
|
||||
for idx, data in enumerate(self.tracked_events[event_id], 1):
|
||||
jpg_bytes: bytes | None = data["thumbnail"]
|
||||
|
||||
if jpg_bytes is None:
|
||||
logger.warning(f"Unable to save thumbnail {idx} for {event_id}.")
|
||||
else:
|
||||
with open(
|
||||
os.path.join(
|
||||
CLIPS_DIR,
|
||||
f"genai-requests/{event_id}/{idx}.jpg",
|
||||
),
|
||||
"wb",
|
||||
) as j:
|
||||
j.write(jpg_bytes)
|
||||
|
||||
# Generate the description. Call happens in a thread since it is network bound.
|
||||
threading.Thread(
|
||||
target=self._genai_embed_description,
|
||||
name=f"_genai_embed_description_{event_id}",
|
||||
daemon=True,
|
||||
args=(
|
||||
event,
|
||||
embed_image,
|
||||
),
|
||||
).start()
|
||||
|
||||
# Clean up tracked events and early request state
|
||||
self.cleanup_event(event_id)
|
||||
|
||||
def _genai_embed_description(self, event: Event, thumbnails: list[bytes]) -> None:
|
||||
"""Embed the description for an event."""
|
||||
start = datetime.datetime.now().timestamp()
|
||||
camera_config = self.config.cameras[str(event.camera)]
|
||||
client = self.genai_manager.description_client
|
||||
|
||||
if client is None:
|
||||
return
|
||||
|
||||
description = client.generate_object_description(
|
||||
camera_config, thumbnails, event
|
||||
)
|
||||
|
||||
if not description:
|
||||
logger.debug("Failed to generate description for %s", event.id)
|
||||
return
|
||||
|
||||
# fire and forget description update
|
||||
self.requestor.send_data(
|
||||
UPDATE_EVENT_DESCRIPTION,
|
||||
{
|
||||
"type": TrackedObjectUpdateTypesEnum.description,
|
||||
"id": event.id,
|
||||
"description": description,
|
||||
"camera": event.camera,
|
||||
},
|
||||
)
|
||||
|
||||
# Embed the description
|
||||
if self.config.semantic_search.enabled:
|
||||
self.embeddings.embed_description(str(event.id), description)
|
||||
|
||||
# Check semantic trigger for this description
|
||||
if self.semantic_trigger_processor is not None:
|
||||
self.semantic_trigger_processor.process_data(
|
||||
{"event_id": event.id, "camera": event.camera, "type": "text"},
|
||||
PostProcessDataEnum.tracked_object,
|
||||
)
|
||||
|
||||
# Update inference timing metrics
|
||||
self.object_desc_speed.update(datetime.datetime.now().timestamp() - start)
|
||||
self.object_desc_dps.update()
|
||||
|
||||
logger.debug(
|
||||
"Generated description for %s (%d images): %s",
|
||||
event.id,
|
||||
len(thumbnails),
|
||||
description,
|
||||
)
|
||||
@@ -0,0 +1,614 @@
|
||||
"""Post processor for review items to get descriptions."""
|
||||
|
||||
import copy
|
||||
import datetime
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import cv2
|
||||
from peewee import DoesNotExist
|
||||
from titlecase import titlecase
|
||||
|
||||
from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.config.camera import CameraConfig
|
||||
from frigate.config.camera.review import GenAIReviewConfig, ImageSourceEnum
|
||||
from frigate.const import (
|
||||
ATTRIBUTE_LABEL_DISPLAY_MAP,
|
||||
CACHE_DIR,
|
||||
CLIPS_DIR,
|
||||
UPDATE_REVIEW_DESCRIPTION,
|
||||
)
|
||||
from frigate.data_processing.types import PostProcessDataEnum
|
||||
from frigate.genai import GenAIClient
|
||||
from frigate.genai.manager import GenAIClientManager
|
||||
from frigate.models import Recordings, ReviewSegment
|
||||
from frigate.util.builtin import EventsPerSecond, InferenceSpeed
|
||||
from frigate.util.image import get_image_from_recording
|
||||
|
||||
from ..post.api import PostProcessorApi
|
||||
from ..types import DataProcessorMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RECORDING_BUFFER_EXTENSION_PERCENT = 0.10
|
||||
MIN_RECORDING_DURATION = 10
|
||||
MAX_IMAGE_TOKENS = 24000
|
||||
MAX_FRAMES_PER_SECOND = 1
|
||||
|
||||
|
||||
class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
def __init__(
|
||||
self,
|
||||
config: FrigateConfig,
|
||||
requestor: InterProcessRequestor,
|
||||
metrics: DataProcessorMetrics,
|
||||
genai_manager: GenAIClientManager,
|
||||
):
|
||||
super().__init__(config, metrics, None)
|
||||
self.requestor = requestor
|
||||
self.metrics = metrics
|
||||
self.genai_manager = genai_manager
|
||||
self.review_desc_speed = InferenceSpeed(self.metrics.review_desc_speed)
|
||||
self.review_desc_dps = EventsPerSecond()
|
||||
self.review_desc_dps.start()
|
||||
|
||||
def calculate_frame_count(
|
||||
self,
|
||||
camera: str,
|
||||
duration: float,
|
||||
image_source: ImageSourceEnum = ImageSourceEnum.preview,
|
||||
height: int = 480,
|
||||
) -> int:
|
||||
"""Calculate optimal number of frames based on event duration, context size,
|
||||
image source, and resolution.
|
||||
|
||||
Per-image token cost is asked of the GenAI provider so providers that know
|
||||
their model's true cost (e.g. llama.cpp can probe the loaded mmproj) can
|
||||
diverge from the default ~1-token-per-1250-pixels heuristic. The frame
|
||||
budget is bounded by:
|
||||
- remaining context window after prompt + response reservations
|
||||
- a fixed MAX_IMAGE_TOKENS ceiling
|
||||
- MAX_FRAMES_PER_SECOND x duration, to avoid drowning short events in
|
||||
near-duplicate frames where the model latches onto the redundant middle
|
||||
and skips the start/end action
|
||||
"""
|
||||
client = self.genai_manager.description_client
|
||||
|
||||
if client is None:
|
||||
return 3
|
||||
|
||||
context_size = client.get_context_size()
|
||||
camera_config = self.config.cameras[camera]
|
||||
|
||||
detect_width = camera_config.detect.width
|
||||
detect_height = camera_config.detect.height
|
||||
|
||||
if not detect_width or not detect_height:
|
||||
aspect_ratio = 16 / 9
|
||||
else:
|
||||
aspect_ratio = detect_width / detect_height
|
||||
|
||||
if image_source == ImageSourceEnum.recordings:
|
||||
if aspect_ratio >= 1:
|
||||
# Landscape or square: constrain height
|
||||
width = int(height * aspect_ratio)
|
||||
else:
|
||||
# Portrait: constrain width
|
||||
width = height
|
||||
height = int(width / aspect_ratio)
|
||||
else:
|
||||
if aspect_ratio >= 1:
|
||||
# Landscape or square: constrain height
|
||||
target_height = 180
|
||||
width = int(target_height * aspect_ratio)
|
||||
height = target_height
|
||||
else:
|
||||
# Portrait: constrain width
|
||||
target_width = 180
|
||||
width = target_width
|
||||
height = int(target_width / aspect_ratio)
|
||||
|
||||
tokens_per_image = client.estimate_image_tokens(width, height)
|
||||
prompt_tokens = 3800
|
||||
response_tokens = 300
|
||||
context_budget = context_size - prompt_tokens - response_tokens
|
||||
image_token_budget = min(context_budget, MAX_IMAGE_TOKENS)
|
||||
max_frames_by_tokens = int(image_token_budget / tokens_per_image)
|
||||
max_frames_by_duration = int(duration * MAX_FRAMES_PER_SECOND)
|
||||
max_frames = min(max_frames_by_tokens, max_frames_by_duration)
|
||||
return max(max_frames, 3)
|
||||
|
||||
def process_data(
|
||||
self, data: dict[str, Any], data_type: PostProcessDataEnum
|
||||
) -> None:
|
||||
self.metrics.review_desc_dps.value = self.review_desc_dps.eps()
|
||||
|
||||
if data_type != PostProcessDataEnum.review:
|
||||
return
|
||||
|
||||
if self.genai_manager.description_client is None:
|
||||
return
|
||||
|
||||
camera = data["after"]["camera"]
|
||||
camera_config = self.config.cameras[camera]
|
||||
|
||||
if not camera_config.review.genai.enabled:
|
||||
return
|
||||
|
||||
id = data["after"]["id"]
|
||||
|
||||
if data["type"] == "new" or data["type"] == "update":
|
||||
return
|
||||
else:
|
||||
final_data = data["after"]
|
||||
|
||||
if (
|
||||
final_data["severity"] == "alert"
|
||||
and not camera_config.review.genai.alerts
|
||||
):
|
||||
return
|
||||
elif (
|
||||
final_data["severity"] == "detection"
|
||||
and not camera_config.review.genai.detections
|
||||
):
|
||||
return
|
||||
|
||||
image_source = camera_config.review.genai.image_source
|
||||
|
||||
if image_source == ImageSourceEnum.recordings:
|
||||
duration = final_data["end_time"] - final_data["start_time"]
|
||||
buffer_extension = min(5, duration * RECORDING_BUFFER_EXTENSION_PERCENT)
|
||||
|
||||
# Ensure minimum total duration for short review items
|
||||
# This provides better context for brief events
|
||||
total_duration = duration + (2 * buffer_extension)
|
||||
if total_duration < MIN_RECORDING_DURATION:
|
||||
# Expand buffer to reach minimum duration, still respecting max of 5s per side
|
||||
additional_buffer_per_side = (MIN_RECORDING_DURATION - duration) / 2
|
||||
buffer_extension = min(5, additional_buffer_per_side)
|
||||
|
||||
final_data["start_time"] -= buffer_extension
|
||||
final_data["end_time"] += buffer_extension
|
||||
|
||||
thumbs = self.get_recording_frames(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
height=480, # Use 480p for good balance between quality and token usage
|
||||
)
|
||||
|
||||
if not thumbs:
|
||||
# Fallback to preview frames if no recordings available
|
||||
logger.warning(
|
||||
f"No recording frames found for {camera}, falling back to preview frames"
|
||||
)
|
||||
thumbs = self.get_preview_frames_as_bytes(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
final_data["thumb_path"],
|
||||
id,
|
||||
camera_config.review.genai.debug_save_thumbnails,
|
||||
)
|
||||
elif camera_config.review.genai.debug_save_thumbnails:
|
||||
# Save debug thumbnails for recordings
|
||||
Path(os.path.join(CLIPS_DIR, "genai-requests", id)).mkdir(
|
||||
parents=True, exist_ok=True
|
||||
)
|
||||
for idx, frame_bytes in enumerate(thumbs):
|
||||
with open(
|
||||
os.path.join(CLIPS_DIR, f"genai-requests/{id}/{idx}.jpg"),
|
||||
"wb",
|
||||
) as f:
|
||||
f.write(frame_bytes)
|
||||
else:
|
||||
# Use preview frames
|
||||
thumbs = self.get_preview_frames_as_bytes(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
final_data["thumb_path"],
|
||||
id,
|
||||
camera_config.review.genai.debug_save_thumbnails,
|
||||
)
|
||||
|
||||
# kickoff analysis
|
||||
self.review_desc_dps.update()
|
||||
threading.Thread(
|
||||
target=run_analysis,
|
||||
args=(
|
||||
self.requestor,
|
||||
self.genai_manager.description_client,
|
||||
self.review_desc_speed,
|
||||
camera_config,
|
||||
final_data,
|
||||
thumbs,
|
||||
camera_config.review.genai,
|
||||
list(self.config.model.merged_labelmap.values()),
|
||||
self.config.model.all_attributes,
|
||||
),
|
||||
).start()
|
||||
|
||||
def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
|
||||
if topic == EmbeddingsRequestEnum.summarize_review.value:
|
||||
start_ts = request_data["start_ts"]
|
||||
end_ts = request_data["end_ts"]
|
||||
logger.debug(
|
||||
f"Found GenAI Review Summary request for {start_ts} to {end_ts}"
|
||||
)
|
||||
|
||||
# Query all review segments with camera and time information
|
||||
segments: list[dict[str, Any]] = [
|
||||
{
|
||||
"camera": r["camera"].replace("_", " ").title(),
|
||||
"start_time": r["start_time"],
|
||||
"end_time": r["end_time"],
|
||||
"metadata": r["data"]["metadata"],
|
||||
}
|
||||
for r in (
|
||||
ReviewSegment.select(
|
||||
ReviewSegment.camera,
|
||||
ReviewSegment.start_time,
|
||||
ReviewSegment.end_time,
|
||||
ReviewSegment.data,
|
||||
)
|
||||
.where(
|
||||
(ReviewSegment.data["metadata"].is_null(False))
|
||||
& (ReviewSegment.start_time < end_ts)
|
||||
& (ReviewSegment.end_time > start_ts)
|
||||
)
|
||||
.order_by(ReviewSegment.start_time.asc())
|
||||
.dicts()
|
||||
.iterator()
|
||||
)
|
||||
]
|
||||
|
||||
if len(segments) == 0:
|
||||
logger.debug("No review items with metadata found during time period")
|
||||
return "No activity was found during this time period."
|
||||
|
||||
# Identify primary items (important items that need review)
|
||||
primary_segments = [
|
||||
seg
|
||||
for seg in segments
|
||||
if seg["metadata"].get("potential_threat_level", 0) > 0
|
||||
or seg["metadata"].get("other_concerns")
|
||||
]
|
||||
|
||||
if not primary_segments:
|
||||
return "No concerns were found during this time period."
|
||||
|
||||
# Build hierarchical structure: each primary event with its contextual items
|
||||
events_with_context = []
|
||||
|
||||
for primary_seg in primary_segments:
|
||||
# Start building the primary event structure
|
||||
primary_item = copy.deepcopy(primary_seg["metadata"])
|
||||
primary_item["camera"] = primary_seg["camera"]
|
||||
primary_item["start_time"] = primary_seg["start_time"]
|
||||
primary_item["end_time"] = primary_seg["end_time"]
|
||||
|
||||
# Find overlapping contextual items from other cameras
|
||||
primary_start = primary_seg["start_time"]
|
||||
primary_end = primary_seg["end_time"]
|
||||
primary_camera = primary_seg["camera"]
|
||||
contextual_items = []
|
||||
seen_contextual_cameras = set()
|
||||
|
||||
for seg in segments:
|
||||
seg_camera = seg["camera"]
|
||||
|
||||
if seg_camera == primary_camera:
|
||||
continue
|
||||
|
||||
if seg in primary_segments:
|
||||
continue
|
||||
|
||||
seg_start = seg["start_time"]
|
||||
seg_end = seg["end_time"]
|
||||
|
||||
if seg_start < primary_end and primary_start < seg_end:
|
||||
# Avoid duplicates if same camera has multiple overlapping segments
|
||||
if seg_camera not in seen_contextual_cameras:
|
||||
contextual_item = copy.deepcopy(seg["metadata"])
|
||||
contextual_item["camera"] = seg_camera
|
||||
contextual_item["start_time"] = seg_start
|
||||
contextual_item["end_time"] = seg_end
|
||||
contextual_items.append(contextual_item)
|
||||
seen_contextual_cameras.add(seg_camera)
|
||||
|
||||
# Add context array to primary item
|
||||
primary_item["context"] = contextual_items
|
||||
events_with_context.append(primary_item)
|
||||
|
||||
total_context_items = sum(
|
||||
len(event.get("context", [])) for event in events_with_context
|
||||
)
|
||||
logger.debug(
|
||||
f"Summary includes {len(events_with_context)} primary events with "
|
||||
f"{total_context_items} total contextual items"
|
||||
)
|
||||
|
||||
if self.config.review.genai.debug_save_thumbnails:
|
||||
Path(
|
||||
os.path.join(CLIPS_DIR, "genai-requests", f"{start_ts}-{end_ts}")
|
||||
).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
client = self.genai_manager.description_client
|
||||
|
||||
if client is None:
|
||||
return None
|
||||
|
||||
return client.generate_review_summary(
|
||||
start_ts,
|
||||
end_ts,
|
||||
events_with_context,
|
||||
self.config.review.genai.preferred_language,
|
||||
self.config.review.genai.debug_save_thumbnails,
|
||||
)
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_cache_frames(
|
||||
self,
|
||||
camera: str,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> list[str]:
|
||||
preview_dir = os.path.join(CACHE_DIR, "preview_frames")
|
||||
file_start = f"preview_{camera}-"
|
||||
start_file = f"{file_start}{start_time}.webp"
|
||||
end_file = f"{file_start}{end_time}.webp"
|
||||
|
||||
camera_files = [
|
||||
entry.name
|
||||
for entry in os.scandir(preview_dir)
|
||||
if entry.name.startswith(file_start)
|
||||
]
|
||||
camera_files.sort()
|
||||
|
||||
all_frames: list[str] = []
|
||||
|
||||
for file in camera_files:
|
||||
if file < start_file:
|
||||
if len(all_frames):
|
||||
all_frames[0] = os.path.join(preview_dir, file)
|
||||
else:
|
||||
all_frames.append(os.path.join(preview_dir, file))
|
||||
|
||||
continue
|
||||
|
||||
if file > end_file:
|
||||
all_frames.append(os.path.join(preview_dir, file))
|
||||
break
|
||||
|
||||
all_frames.append(os.path.join(preview_dir, file))
|
||||
|
||||
frame_count = len(all_frames)
|
||||
desired_frame_count = self.calculate_frame_count(
|
||||
camera, duration=end_time - start_time
|
||||
)
|
||||
|
||||
if frame_count <= desired_frame_count:
|
||||
return all_frames
|
||||
|
||||
selected_frames = []
|
||||
step_size = (frame_count - 1) / (desired_frame_count - 1)
|
||||
|
||||
for i in range(desired_frame_count):
|
||||
index = round(i * step_size)
|
||||
selected_frames.append(all_frames[index])
|
||||
|
||||
return selected_frames
|
||||
|
||||
def get_recording_frames(
|
||||
self,
|
||||
camera: str,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
height: int = 480,
|
||||
) -> list[bytes]:
|
||||
"""Get frames from recordings at specified timestamps."""
|
||||
duration = end_time - start_time
|
||||
desired_frame_count = self.calculate_frame_count(
|
||||
camera, duration, ImageSourceEnum.recordings, height
|
||||
)
|
||||
|
||||
# Calculate evenly spaced timestamps throughout the duration
|
||||
if desired_frame_count == 1:
|
||||
timestamps = [start_time + duration / 2]
|
||||
else:
|
||||
step = duration / (desired_frame_count - 1)
|
||||
timestamps = [start_time + (i * step) for i in range(desired_frame_count)]
|
||||
|
||||
def extract_frame_from_recording(ts: float) -> bytes | None:
|
||||
"""Extract a single frame from recording at given timestamp."""
|
||||
try:
|
||||
recording = (
|
||||
Recordings.select(
|
||||
Recordings.path,
|
||||
Recordings.start_time,
|
||||
)
|
||||
.where((ts >= Recordings.start_time) & (ts <= Recordings.end_time))
|
||||
.where(Recordings.camera == camera)
|
||||
.order_by(Recordings.start_time.desc())
|
||||
.limit(1)
|
||||
.get()
|
||||
)
|
||||
|
||||
time_in_segment = ts - recording.start_time
|
||||
return get_image_from_recording(
|
||||
self.config.ffmpeg,
|
||||
recording.path,
|
||||
time_in_segment,
|
||||
"mjpeg",
|
||||
height=height,
|
||||
)
|
||||
except DoesNotExist:
|
||||
return None
|
||||
|
||||
frames = []
|
||||
|
||||
for timestamp in timestamps:
|
||||
try:
|
||||
# Try to extract frame at exact timestamp
|
||||
image_data = extract_frame_from_recording(timestamp)
|
||||
|
||||
if not image_data:
|
||||
# Try with rounded timestamp as fallback
|
||||
rounded_timestamp = math.ceil(timestamp)
|
||||
image_data = extract_frame_from_recording(rounded_timestamp)
|
||||
|
||||
if image_data:
|
||||
frames.append(image_data)
|
||||
else:
|
||||
logger.warning(
|
||||
f"No recording found for {camera} at timestamp {timestamp}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error extracting frame from recording for {camera} at {timestamp}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
return frames
|
||||
|
||||
def get_preview_frames_as_bytes(
|
||||
self,
|
||||
camera: str,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
thumb_path_fallback: str,
|
||||
review_id: str,
|
||||
save_debug: bool,
|
||||
) -> list[bytes]:
|
||||
"""Get preview frames and convert them to JPEG bytes.
|
||||
|
||||
Args:
|
||||
camera: Camera name
|
||||
start_time: Start timestamp
|
||||
end_time: End timestamp
|
||||
thumb_path_fallback: Fallback thumbnail path if no preview frames found
|
||||
review_id: Review item ID for debug saving
|
||||
save_debug: Whether to save debug thumbnails
|
||||
|
||||
Returns:
|
||||
List of JPEG image bytes
|
||||
"""
|
||||
frame_paths = self.get_cache_frames(camera, start_time, end_time)
|
||||
if not frame_paths:
|
||||
frame_paths = [thumb_path_fallback]
|
||||
|
||||
thumbs = []
|
||||
for idx, thumb_path in enumerate(frame_paths):
|
||||
thumb_data = cv2.imread(thumb_path)
|
||||
|
||||
if thumb_data is None:
|
||||
logger.warning( # type: ignore[unreachable]
|
||||
"Could not read preview frame at %s, skipping", thumb_path
|
||||
)
|
||||
continue
|
||||
|
||||
ret, jpg = cv2.imencode(
|
||||
".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100]
|
||||
)
|
||||
if ret:
|
||||
thumbs.append(jpg.tobytes())
|
||||
|
||||
if save_debug:
|
||||
Path(os.path.join(CLIPS_DIR, "genai-requests", review_id)).mkdir(
|
||||
parents=True, exist_ok=True
|
||||
)
|
||||
shutil.copy(
|
||||
thumb_path,
|
||||
os.path.join(CLIPS_DIR, f"genai-requests/{review_id}/{idx}.webp"),
|
||||
)
|
||||
|
||||
return thumbs
|
||||
|
||||
|
||||
def run_analysis(
|
||||
requestor: InterProcessRequestor,
|
||||
genai_client: GenAIClient,
|
||||
review_inference_speed: InferenceSpeed,
|
||||
camera_config: CameraConfig,
|
||||
final_data: dict[str, Any],
|
||||
thumbs: list[bytes],
|
||||
genai_config: GenAIReviewConfig,
|
||||
labelmap_objects: list[str],
|
||||
attribute_labels: list[str],
|
||||
) -> None:
|
||||
start = datetime.datetime.now().timestamp()
|
||||
|
||||
# Format zone names using zone config friendly names if available
|
||||
formatted_zones = []
|
||||
for zone_name in final_data["data"]["zones"]:
|
||||
if zone_name in camera_config.zones:
|
||||
formatted_zones.append(
|
||||
camera_config.zones[zone_name].get_formatted_name(zone_name)
|
||||
)
|
||||
|
||||
analytics_data = {
|
||||
"id": final_data["id"],
|
||||
"camera": camera_config.get_formatted_name(),
|
||||
"zones": formatted_zones,
|
||||
"start": datetime.datetime.fromtimestamp(final_data["start_time"]).strftime(
|
||||
"%A, %I:%M %p"
|
||||
),
|
||||
"duration": round(final_data["end_time"] - final_data["start_time"]),
|
||||
}
|
||||
|
||||
unified_objects = []
|
||||
|
||||
objects_list = final_data["data"]["objects"]
|
||||
sub_labels_list = final_data["data"]["sub_labels"]
|
||||
|
||||
for i, verified_label in enumerate(final_data["data"]["verified_objects"]):
|
||||
object_type = verified_label.replace("-verified", "").replace("_", " ")
|
||||
name = titlecase(sub_labels_list[i].replace("_", " "))
|
||||
unified_objects.append(f"{name} ← {object_type}")
|
||||
|
||||
for label in objects_list:
|
||||
if "-verified" in label:
|
||||
continue
|
||||
elif label in labelmap_objects:
|
||||
object_type = label.replace("_", " ")
|
||||
|
||||
if label in attribute_labels:
|
||||
display_name = ATTRIBUTE_LABEL_DISPLAY_MAP.get(label, object_type)
|
||||
unified_objects.append(f"{display_name} (delivery/service)")
|
||||
else:
|
||||
unified_objects.append(object_type)
|
||||
|
||||
analytics_data["unified_objects"] = unified_objects
|
||||
|
||||
metadata = genai_client.generate_review_description(
|
||||
analytics_data,
|
||||
thumbs,
|
||||
genai_config.additional_concerns,
|
||||
genai_config.preferred_language,
|
||||
genai_config.debug_save_thumbnails,
|
||||
genai_config.activity_context_prompt,
|
||||
)
|
||||
review_inference_speed.update(datetime.datetime.now().timestamp() - start)
|
||||
|
||||
if not metadata:
|
||||
return None
|
||||
|
||||
prev_data = copy.deepcopy(final_data)
|
||||
final_data["data"]["metadata"] = metadata.model_dump()
|
||||
requestor.send_data(
|
||||
UPDATE_REVIEW_DESCRIPTION,
|
||||
{
|
||||
"type": "genai",
|
||||
"before": {k: v for k, v in prev_data.items()},
|
||||
"after": {k: v for k, v in final_data.items()},
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,275 @@
|
||||
"""Post time processor to trigger actions based on similar embeddings."""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from peewee import DoesNotExist
|
||||
|
||||
from frigate.comms.event_metadata_updater import (
|
||||
EventMetadataPublisher,
|
||||
EventMetadataTypeEnum,
|
||||
)
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.const import CONFIG_DIR
|
||||
from frigate.data_processing.types import PostProcessDataEnum
|
||||
from frigate.db.sqlitevecq import SqliteVecQueueDatabase
|
||||
from frigate.embeddings.embeddings import Embeddings
|
||||
from frigate.embeddings.util import ZScoreNormalization
|
||||
from frigate.models import Event, Trigger
|
||||
from frigate.util.builtin import cosine_distance
|
||||
from frigate.util.file import get_event_thumbnail_bytes
|
||||
|
||||
from ..post.api import PostProcessorApi
|
||||
from ..types import DataProcessorMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WRITE_DEBUG_IMAGES = False
|
||||
|
||||
|
||||
class SemanticTriggerProcessor(PostProcessorApi):
|
||||
def __init__(
|
||||
self,
|
||||
db: SqliteVecQueueDatabase,
|
||||
config: FrigateConfig,
|
||||
requestor: InterProcessRequestor,
|
||||
sub_label_publisher: EventMetadataPublisher,
|
||||
metrics: DataProcessorMetrics,
|
||||
embeddings: Embeddings,
|
||||
) -> None:
|
||||
super().__init__(config, metrics, None)
|
||||
self.db = db
|
||||
self.embeddings = embeddings
|
||||
self.requestor = requestor
|
||||
self.sub_label_publisher = sub_label_publisher
|
||||
self.trigger_embeddings: list[np.ndarray] = []
|
||||
|
||||
self.thumb_stats = ZScoreNormalization()
|
||||
self.desc_stats = ZScoreNormalization()
|
||||
|
||||
# load stats from disk
|
||||
try:
|
||||
with open(os.path.join(CONFIG_DIR, ".search_stats.json")) as f:
|
||||
data = json.loads(f.read())
|
||||
self.thumb_stats.from_dict(data["thumb_stats"])
|
||||
self.desc_stats.from_dict(data["desc_stats"])
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def process_data(
|
||||
self, data: dict[str, Any], data_type: PostProcessDataEnum
|
||||
) -> None:
|
||||
event_id = data["event_id"]
|
||||
camera = data["camera"]
|
||||
process_type = data["type"]
|
||||
|
||||
if self.config.cameras[camera].semantic_search.triggers is None:
|
||||
return
|
||||
|
||||
triggers = (
|
||||
Trigger.select(
|
||||
Trigger.camera,
|
||||
Trigger.name,
|
||||
Trigger.data,
|
||||
Trigger.type,
|
||||
Trigger.embedding,
|
||||
Trigger.threshold,
|
||||
)
|
||||
.where(Trigger.camera == camera)
|
||||
.dicts()
|
||||
.iterator()
|
||||
)
|
||||
|
||||
for trigger in triggers:
|
||||
if (
|
||||
trigger["name"]
|
||||
not in self.config.cameras[camera].semantic_search.triggers
|
||||
or not self.config.cameras[camera]
|
||||
.semantic_search.triggers[trigger["name"]]
|
||||
.enabled
|
||||
):
|
||||
logger.debug(
|
||||
f"Trigger {trigger['name']} is disabled for camera {camera}"
|
||||
)
|
||||
continue
|
||||
|
||||
logger.debug(
|
||||
f"Processing {trigger['type']} trigger for {event_id} on {trigger['camera']}: {trigger['name']}"
|
||||
)
|
||||
|
||||
trigger_embedding = np.frombuffer(trigger["embedding"], dtype=np.float32)
|
||||
|
||||
# Get embeddings based on type
|
||||
thumbnail_embedding = None
|
||||
description_embedding = None
|
||||
|
||||
if process_type == "image":
|
||||
cursor = self.db.execute_sql(
|
||||
"""
|
||||
SELECT thumbnail_embedding FROM vec_thumbnails WHERE id = ?
|
||||
""",
|
||||
[event_id],
|
||||
)
|
||||
row = cursor.fetchone() if cursor else None
|
||||
if row:
|
||||
thumbnail_embedding = np.frombuffer(row[0], dtype=np.float32)
|
||||
|
||||
if process_type == "text":
|
||||
cursor = self.db.execute_sql(
|
||||
"""
|
||||
SELECT description_embedding FROM vec_descriptions WHERE id = ?
|
||||
""",
|
||||
[event_id],
|
||||
)
|
||||
row = cursor.fetchone() if cursor else None
|
||||
if row:
|
||||
description_embedding = np.frombuffer(row[0], dtype=np.float32)
|
||||
|
||||
# Skip processing if we don't have any embeddings
|
||||
if thumbnail_embedding is None and description_embedding is None:
|
||||
logger.debug(f"No embeddings found for {event_id}")
|
||||
return
|
||||
|
||||
# Determine which embedding to compare based on trigger type
|
||||
if (
|
||||
trigger["type"] in ["text", "thumbnail"]
|
||||
and thumbnail_embedding is not None
|
||||
):
|
||||
data_embedding = thumbnail_embedding
|
||||
normalized_distance = self.thumb_stats.normalize(
|
||||
[cosine_distance(data_embedding, trigger_embedding)],
|
||||
save_stats=False,
|
||||
)[0]
|
||||
elif trigger["type"] == "description" and description_embedding is not None:
|
||||
data_embedding = description_embedding
|
||||
normalized_distance = self.desc_stats.normalize(
|
||||
[cosine_distance(data_embedding, trigger_embedding)],
|
||||
save_stats=False,
|
||||
)[0]
|
||||
|
||||
else:
|
||||
continue
|
||||
|
||||
similarity = 1 - normalized_distance
|
||||
|
||||
logger.debug(
|
||||
f"Trigger {trigger['name']} ({trigger['data'] if trigger['type'] == 'text' or trigger['type'] == 'description' else 'image'}): "
|
||||
f"normalized distance: {normalized_distance:.4f}, "
|
||||
f"similarity: {similarity:.4f}, threshold: {trigger['threshold']}"
|
||||
)
|
||||
|
||||
# Check if similarity meets threshold
|
||||
if similarity >= trigger["threshold"]:
|
||||
logger.debug(
|
||||
f"Trigger {trigger['name']} activated with similarity {similarity:.4f}"
|
||||
)
|
||||
|
||||
# Update the trigger's last_triggered and triggering_event_id
|
||||
Trigger.update(
|
||||
last_triggered=datetime.datetime.now(), triggering_event_id=event_id
|
||||
).where(
|
||||
Trigger.camera == camera, Trigger.name == trigger["name"]
|
||||
).execute()
|
||||
|
||||
# Always publish MQTT message
|
||||
self.requestor.send_data(
|
||||
"triggers",
|
||||
json.dumps(
|
||||
{
|
||||
"name": trigger["name"],
|
||||
"camera": camera,
|
||||
"event_id": event_id,
|
||||
"type": trigger["type"],
|
||||
"score": similarity,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
friendly_name = (
|
||||
self.config.cameras[camera]
|
||||
.semantic_search.triggers[trigger["name"]]
|
||||
.friendly_name
|
||||
)
|
||||
|
||||
if (
|
||||
self.config.cameras[camera]
|
||||
.semantic_search.triggers[trigger["name"]]
|
||||
.actions
|
||||
):
|
||||
# handle actions for the trigger
|
||||
# notifications already handled by webpush
|
||||
if (
|
||||
"sub_label"
|
||||
in self.config.cameras[camera]
|
||||
.semantic_search.triggers[trigger["name"]]
|
||||
.actions
|
||||
):
|
||||
self.sub_label_publisher.publish(
|
||||
(event_id, friendly_name, similarity),
|
||||
EventMetadataTypeEnum.sub_label,
|
||||
)
|
||||
if (
|
||||
"attribute"
|
||||
in self.config.cameras[camera]
|
||||
.semantic_search.triggers[trigger["name"]]
|
||||
.actions
|
||||
):
|
||||
self.sub_label_publisher.publish(
|
||||
(
|
||||
event_id,
|
||||
trigger["name"],
|
||||
trigger["type"],
|
||||
similarity,
|
||||
),
|
||||
EventMetadataTypeEnum.attribute.value,
|
||||
)
|
||||
|
||||
if WRITE_DEBUG_IMAGES:
|
||||
try:
|
||||
event: Event = Event.get(Event.id == event_id)
|
||||
except DoesNotExist:
|
||||
return
|
||||
|
||||
# Skip the event if not an object
|
||||
if event.data.get("type") != "object": # type: ignore[attr-defined]
|
||||
return
|
||||
|
||||
thumbnail_bytes = get_event_thumbnail_bytes(event)
|
||||
|
||||
if thumbnail_bytes is None:
|
||||
return
|
||||
|
||||
nparr = np.frombuffer(thumbnail_bytes, np.uint8)
|
||||
thumbnail = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
|
||||
|
||||
font_scale = 0.5
|
||||
font = cv2.FONT_HERSHEY_SIMPLEX
|
||||
cv2.putText(
|
||||
thumbnail,
|
||||
f"{similarity:.4f}",
|
||||
(10, 30),
|
||||
font,
|
||||
fontScale=font_scale,
|
||||
color=(0, 255, 0),
|
||||
thickness=2,
|
||||
)
|
||||
|
||||
current_time = int(datetime.datetime.now().timestamp())
|
||||
cv2.imwrite(
|
||||
f"debug/frames/trigger-{event_id}_{current_time}.jpg",
|
||||
thumbnail,
|
||||
)
|
||||
|
||||
def handle_request(
|
||||
self, topic: str, request_data: dict[str, Any]
|
||||
) -> dict[str, Any] | str | None:
|
||||
return None
|
||||
|
||||
def expire_object(self, object_id: str, camera: str) -> None:
|
||||
pass
|
||||
@@ -0,0 +1,45 @@
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StringConstraints
|
||||
|
||||
ObservationItem = Annotated[str, StringConstraints(min_length=20, max_length=200)]
|
||||
|
||||
|
||||
class ReviewMetadata(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", protected_namespaces=())
|
||||
|
||||
observations: list[ObservationItem] = Field(
|
||||
...,
|
||||
min_length=3,
|
||||
max_length=8,
|
||||
description="Enumerate the significant observations across all frames, in chronological order.",
|
||||
)
|
||||
scene: str = Field(
|
||||
min_length=150,
|
||||
max_length=600,
|
||||
description="A chronological narrative of what happens from start to finish, drawing directly from the items in observations.",
|
||||
)
|
||||
title: str = Field(
|
||||
max_length=80,
|
||||
description="Title for the activity.",
|
||||
)
|
||||
shortSummary: str = Field(
|
||||
min_length=70,
|
||||
max_length=140,
|
||||
description="A brief summary for the activity.",
|
||||
)
|
||||
confidence: float = Field(
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Confidence in the analysis as a decimal between 0.0 and 1.0, where 0.0 means no confidence and 1.0 means complete confidence. Express ONLY as a decimal.",
|
||||
)
|
||||
potential_threat_level: int = Field(
|
||||
ge=0,
|
||||
le=2,
|
||||
description="Threat level: 0 = normal, 1 = suspicious, 2 = critical threat.",
|
||||
)
|
||||
other_concerns: list[str] | None = Field(
|
||||
default=None,
|
||||
description="Other concerns highlighted by the user that are observed.",
|
||||
)
|
||||
time: str | None = Field(default=None, description="Time of activity.")
|
||||
Reference in New Issue
Block a user