sgl-project--sglang/python/sglang/multimodal_gen/runtime/realtime/states/camera_control.py

# SPDX-License-Identifier: Apache-2.0

from __future__ import annotations

from collections.abc import Callable
from typing import Any

from sglang.multimodal_gen.runtime.realtime.control_signals import (
    ControlScriptQueue,
    ControlStateQueue,
    ControlStateTransition,
    parse_control_event_payload,
)

CameraActionNormalizer = Callable[[list[Any]], list[str]]
CameraActionValidator = Callable[[Any], list[list[str]]]


def _identity_actions(actions: list[Any]) -> list[str]:
    return list(actions)


class RealtimeCameraControlState:
    """Session-local camera-control buffer shared by realtime model adapters.

    Camera controls arrive in two shapes:

    1. Script mode: ``list[list[str]]`` where each item is one output-frame's
       held actions. The script is consumed once from a FIFO and padded with
       neutral ``[]`` frames after it runs out.
    2. State mode: timestamped transitions such as "W is currently held".
       ``ControlStateQueue`` expands that continuous state into the next
       chunk and can pulse a short key press for a minimum number of frames.

    The two modes are intentionally exclusive. A new script clears state mode,
    and new state transitions clear script mode, so adapters never merge two
    camera timelines accidentally. ``sample_camera_actions`` returns ``None``
    only when no control should be sent; otherwise it returns exactly
    ``chunk_size`` frames, with ``[]`` meaning neutral/no-op for that frame.
    """

    def __init__(
        self,
        *,
        min_pulse_items: int = 1,
        script_maxlen: int = 512,
        max_transitions: int = 512,
        normalize_state_actions: CameraActionNormalizer = _identity_actions,
    ) -> None:
        # stores state-mode control signals
        self.camera_state_queue = ControlStateQueue(
            default_item=[],
            min_pulse_items=min_pulse_items,
            max_transitions=max_transitions,
        )
        # stores script-mode control signals
        # script-mode signals take precedence over state-mode signals, see sample_camera_actions
        self.camera_script_queue = ControlScriptQueue(
            "camera_actions",
            max_events=script_maxlen,
            default_item=[],
        )
        self.latest_sampled_event_id: int | None = None
        self._normalize_state_actions = normalize_state_actions

    def clear(self) -> None:
        """Reset all camera controls owned by this realtime session."""
        self.camera_state_queue.clear()
        self.camera_script_queue.clear()
        self.latest_sampled_event_id = None

    def receive_camera_action_script(
        self,
        camera_actions: list[list[str]],
        *,
        event_id: int | None = None,
    ) -> None:
        """Replace active controls with a finite per-frame script."""
        self.camera_state_queue.clear()
        self.camera_script_queue.push_script(
            [list(actions) for actions in camera_actions],
            event_id=event_id,
        )

    def receive_camera_state_transitions(
        self,
        transitions: list[ControlStateTransition],
    ) -> None:
        """Replace the script with continuous state transitions."""
        self.camera_script_queue.clear()
        self.camera_state_queue.push_many(transitions)

    def receive_camera_state(
        self,
        actions: list[str],
        *,
        event_id: int | None = None,
        timestamp_ms: int | None = None,
    ) -> None:
        self.receive_camera_state_transitions(
            [
                ControlStateTransition(
                    payload=self._normalize_state_actions(actions),
                    seq_id=event_id,
                    timestamp_ms=timestamp_ms,
                )
            ]
        )

    def receive_camera_control_event_payload(
        self,
        payload: Any,
        *,
        event_id: int | None,
        validate_camera_actions: CameraActionValidator,
    ) -> str:
        """Parse an external camera event (from endpoint) and install it as script or state."""
        parsed = parse_control_event_payload(
            payload,
            event_id=event_id,
            kind="camera_actions",
            normalize_state_payload=self._normalize_state_actions,
            validate_script_payload=validate_camera_actions,
        )
        if parsed.mode == "state":
            transitions = parsed.payload
            self.receive_camera_state_transitions(transitions)
            return f"kind=camera_actions, mode=state, transitions={len(transitions)}"

        camera_actions = parsed.payload
        self.receive_camera_action_script(camera_actions, event_id=event_id)
        return f"kind=camera_actions, mode=script, frames={len(camera_actions)}"

    def sample_camera_actions(self, chunk_size: int) -> list[list[str]] | None:
        """Core method, return the next chunk-sized camera action window.

        Script mode has priority because it represents an explicit finite
        timeline. State mode is sampled only when no script is pending.
        """
        # Script mode wins: it is an explicit finite timeline and should not be
        # merged with held-key state from the live control path.
        if self.camera_script_queue.has_script():
            return self._sample_camera_script(chunk_size)
        # State mode is the WebUI path: held controls persist across chunks until
        # a new transition changes the current state.
        action_list = self.camera_state_queue.sample_chunk(chunk_size)
        if action_list is None:
            return None
        self.latest_sampled_event_id = self.camera_state_queue.latest_sampled_seq_id()
        return [list(actions) for actions in action_list]

    def _sample_camera_script(self, chunk_size: int) -> list[list[str]]:
        chunk = self.camera_script_queue.sample_script(chunk_size)
        chunk = [list(actions) for actions in chunk]
        self.latest_sampled_event_id = self.camera_script_queue.last_sampled_seq_id()
        return chunk