chore: import upstream snapshot with attribution

python/sglang/multimodal_gen/runtime/realtime/__init__.py

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""session-scoped realtime state, control signals, and runtime-only helpers"""
+
+from sglang.multimodal_gen.runtime.realtime.control_signals import (
+    ControlScriptQueue,
+    ControlSignal,
+    ControlSignalQueue,
+    ControlSignalSamplingParams,
+    ControlStateQueue,
+    ControlStateTransition,
+    ParsedControlEventPayload,
+    parse_control_event_payload,
+)
+from sglang.multimodal_gen.runtime.realtime.session import (
+    BaseRealtimeState,
+    RealtimeSession,
+    RealtimeSessionCache,
+)
+from sglang.multimodal_gen.runtime.realtime.states import (
+    RealtimeCameraControlState,
+    RealtimeCausalDecodeState,
+    RealtimeCausalDiTState,
+    get_realtime_causal_dit_state,
+)
+
+__all__ = [
+    "BaseRealtimeState",
+    "ControlScriptQueue",
+    "ControlSignal",
+    "ControlSignalQueue",
+    "ControlSignalSamplingParams",
+    "ControlStateQueue",
+    "ControlStateTransition",
+    "ParsedControlEventPayload",
+    "RealtimeCameraControlState",
+    "RealtimeCausalDecodeState",
+    "RealtimeCausalDiTState",
+    "RealtimeSession",
+    "RealtimeSessionCache",
+    "get_realtime_causal_dit_state",
+    "parse_control_event_payload",
+]

python/sglang/multimodal_gen/runtime/realtime/control_signals.py

@@ -0,0 +1,464 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""Realtime control signal primitives.
+
+This module owns the small, model-agnostic data structures that turn external
+realtime control inputs into chunk-sized payloads consumed by model adapters.
+It intentionally does not know about cameras, LingBot, or SANA-WM semantics;
+callers provide validation/normalization functions and decide how sampled
+payloads map to request ``condition_inputs``.
+
+There are two control modes:
+
+* Script mode is a finite per-frame timeline, for example
+  ``[["w"], ["w"], [], ...]``. It is already expanded by the caller, so
+  ``ControlScriptQueue`` consumes it in order and pads the tail with a neutral
+  default item when configured. Script mode is useful for tests, presets, and
+  deterministic replay.
+* State mode is a level-triggered stream of transitions, for example "these
+  keys are currently held". ``ControlStateQueue`` keeps the latest state and
+  samples a stable chunk from it; a short non-default pulse can be preserved
+  even when press/release transitions arrive between two render chunks. State
+  mode is the natural shape for live keyboard/gamepad controls.
+
+``ControlSignalQueue`` is the lower-level FIFO for discrete signals. It is used
+directly for one-shot controls such as prompt updates, and wrapped by
+``ControlScriptQueue`` for finite timelines. ``ControlStateQueue`` is separate
+because held controls need stateful sampling rather than FIFO draining.
+"""
+
+from __future__ import annotations
+
+import copy
+from collections import deque
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+_MISSING = object()
+
+
+@dataclass(frozen=True)
+class ControlSignal:
+    """a discrete, minimal input signal for a specific control kind.
+
+    Generated from script-mode inputs such as receive_camera_action_script
+    Consumed by chunk samplers such as sample_camera_actions
+
+    """
+
+    # camera_actions, prompt
+    kind: str
+    # two formats:
+    # 1. script mode: a sequence of flatten actions (e.g., [["w"], ["w"], ["a"], []])
+    # 2. state mode: a sequence of state changes (e.g., "actions": ["w"], "client_ts_ms": 1000)
+    payload: Any
+    # the timestep of this signal
+    timestamp_ms: int | None = None
+    seq_id: int | None = None
+
+
+@dataclass(frozen=True)
+class ControlStateTransition:
+    payload: Any
+    timestamp_ms: int | None = None
+    seq_id: int | None = None
+
+
+@dataclass(frozen=True)
+class ControlSignalSamplingParams:
+    chunk_size: int
+    default_item: Any = _MISSING
+    repeat_last: bool = True
+    repeat_last_across_empty_chunks: bool = False
+
+
+@dataclass(frozen=True)
+class ParsedControlEventPayload:
+    mode: str
+    payload: Any
+
+
+ControlStatePayloadNormalizer = Callable[[list[Any]], Any]
+ControlScriptPayloadValidator = Callable[[Any], Any]
+
+
+def parse_control_event_payload(
+    payload: Any,
+    *,
+    event_id: int | None,
+    kind: str,
+    normalize_state_payload: ControlStatePayloadNormalizer,
+    validate_script_payload: ControlScriptPayloadValidator,
+) -> ParsedControlEventPayload:
+    """parse external control event from endpoint"""
+    if isinstance(payload, dict) and payload.get("mode") == "state":
+        return ParsedControlEventPayload(
+            mode="state",
+            payload=_control_state_transitions_from_event_payload(
+                payload,
+                event_id=event_id,
+                kind=kind,
+                normalize_state_payload=normalize_state_payload,
+            ),
+        )
+    return ParsedControlEventPayload(
+        mode="script",
+        payload=validate_script_payload(payload),
+    )
+
+
+def _control_state_transitions_from_event_payload(
+    payload: dict[str, Any],
+    *,
+    event_id: int | None,
+    kind: str,
+    normalize_state_payload: ControlStatePayloadNormalizer,
+) -> list[ControlStateTransition]:
+    transitions = payload.get("transitions")
+    if not isinstance(transitions, list):
+        raise ValueError(f"{kind} state payload requires transitions")
+    result = []
+    for transition in transitions:
+        if not isinstance(transition, dict):
+            raise ValueError(f"{kind} transition must be a map")
+        actions = transition.get("actions")
+        if not isinstance(actions, list):
+            raise ValueError(f"{kind} transition actions must be a list")
+        timestamp_ms = transition.get("client_ts_ms")
+        if timestamp_ms is not None:
+            timestamp_ms = int(timestamp_ms)
+        result.append(
+            ControlStateTransition(
+                payload=normalize_state_payload(actions),
+                seq_id=event_id,
+                timestamp_ms=timestamp_ms,
+            )
+        )
+    return result
+
+
+class ControlSignalQueue:
+    """FIFO storage for discrete realtime control signals
+
+    Script-mode controls and one-shot signals are already expressed as discrete
+    payloads, so sampling only consumes queued signals and applies the requested
+    padding strategy.
+    """
+
+    def __init__(
+        self,
+        max_events: int | Mapping[str, int] = 512,
+    ) -> None:
+        self._max_events = max_events
+        # [control_kind, deque of control signals]
+        self._signals: dict[str, deque[ControlSignal]] = {}
+        self._last_payloads: dict[str, Any] = {}
+        self._last_sampled_seq_ids: dict[str, int | None] = {}
+        self._seen_kinds: set[str] = set()
+
+    def push(
+        self,
+        kind: str,
+        payload: Any,
+        *,
+        event_id: int | None = None,
+        timestamp_ms: int | None = None,
+        expand_payload: bool = True,
+    ) -> None:
+        queue = self._queue_for(kind)
+        for signal in self._iter_signals(
+            kind,
+            payload,
+            event_id=event_id,
+            timestamp_ms=timestamp_ms,
+            expand_payload=expand_payload,
+        ):
+            queue.append(signal)
+        self._seen_kinds.add(kind)
+
+    def replace(
+        self,
+        kind: str,
+        payload: Any,
+        *,
+        event_id: int | None = None,
+        timestamp_ms: int | None = None,
+        expand_payload: bool = True,
+    ) -> None:
+        self.clear_kind(kind)
+        self.push(
+            kind,
+            payload,
+            event_id=event_id,
+            timestamp_ms=timestamp_ms,
+            expand_payload=expand_payload,
+        )
+
+    def pop_latest(self, kind: str) -> Any | None:
+        queue = self._signals.get(kind)
+        if not queue:
+            return None
+        signal = queue.pop()
+        latest_payload = signal.payload
+        self._last_payloads[kind] = signal.payload
+        self._last_sampled_seq_ids[kind] = signal.seq_id
+        queue.clear()
+        self._seen_kinds.add(kind)
+        return latest_payload
+
+    def has_events(self, kind: str) -> bool:
+        queue = self._signals.get(kind)
+        return bool(queue)
+
+    def sample_chunk(
+        self,
+        kind: str,
+        params: ControlSignalSamplingParams,
+    ) -> list[Any] | None:
+        """sample queued signals for one realtime chunk"""
+        if params.chunk_size <= 0:
+            return None
+
+        chunk: list[Any] = []
+        queue = self._signals.get(kind)
+        self._drain_signals(kind, queue, chunk, params.chunk_size)
+
+        if len(chunk) == 0 and kind not in self._seen_kinds:
+            if params.default_item is _MISSING:
+                return None
+            return [params.default_item for _ in range(params.chunk_size)]
+
+        if len(chunk) == 0:
+            if params.repeat_last_across_empty_chunks and kind in self._last_payloads:
+                return [self._last_payloads[kind] for _ in range(params.chunk_size)]
+            if params.default_item is _MISSING:
+                return None
+            return [params.default_item for _ in range(params.chunk_size)]
+
+        if not params.repeat_last:
+            return chunk
+
+        pad_item = self._last_payloads.get(kind, params.default_item)
+        if pad_item is _MISSING:
+            return chunk
+        while len(chunk) < params.chunk_size:
+            chunk.append(pad_item)
+        return chunk
+
+    def clear(self) -> None:
+        self._signals.clear()
+        self._last_payloads.clear()
+        self._last_sampled_seq_ids.clear()
+        self._seen_kinds.clear()
+
+    def clear_kind(self, kind: str) -> None:
+        self._signals.pop(kind, None)
+        self._last_payloads.pop(kind, None)
+        self._last_sampled_seq_ids.pop(kind, None)
+        self._seen_kinds.discard(kind)
+
+    def last_sampled_seq_id(self, kind: str) -> int | None:
+        return self._last_sampled_seq_ids.get(kind)
+
+    def _queue_for(self, kind: str) -> deque[ControlSignal]:
+        queue = self._signals.get(kind)
+        if queue is None:
+            if isinstance(self._max_events, Mapping):
+                maxlen = self._max_events.get(kind, 512)
+            else:
+                maxlen = self._max_events
+            queue = deque(maxlen=maxlen)
+            self._signals[kind] = queue
+        return queue
+
+    def _iter_signals(
+        self,
+        kind: str,
+        payload: Any,
+        *,
+        event_id: int | None,
+        timestamp_ms: int | None,
+        expand_payload: bool,
+    ):
+        items = (
+            payload
+            if self._should_expand_payload(payload, expand_payload)
+            else (payload,)
+        )
+        for item in items:
+            if isinstance(item, ControlSignal):
+                if item.kind != kind:
+                    raise ValueError(
+                        "control signal kind "
+                        f"{item.kind!r} does not match queue kind {kind!r}"
+                    )
+                yield item
+            else:
+                yield ControlSignal(
+                    kind=kind,
+                    payload=item,
+                    timestamp_ms=timestamp_ms,
+                    seq_id=event_id,
+                )
+
+    @staticmethod
+    def _should_expand_payload(payload: Any, expand_payload: bool) -> bool:
+        return (
+            expand_payload
+            and isinstance(payload, Sequence)
+            and not isinstance(payload, (str, bytes, bytearray))
+        )
+
+    def _drain_signals(
+        self,
+        kind: str,
+        signals: deque[ControlSignal] | None,
+        chunk: list[Any],
+        chunk_size: int,
+    ) -> None:
+        while signals and len(chunk) < chunk_size:
+            signal = signals.popleft()
+            chunk.append(signal.payload)
+            self._last_payloads[kind] = signal.payload
+            self._last_sampled_seq_ids[kind] = signal.seq_id
+
+
+class ControlScriptQueue:
+    """Script-mode queue for finite per-frame control timelines."""
+
+    def __init__(
+        self,
+        kind: str,
+        *,
+        max_events: int = 512,
+        default_item: Any = _MISSING,
+    ) -> None:
+        self.kind = kind
+        self.default_item = default_item
+        self._signals = ControlSignalQueue(max_events={kind: max_events})
+
+    def clear(self) -> None:
+        self._signals.clear_kind(self.kind)
+
+    def push_script(
+        self,
+        script: Sequence[Any],
+        *,
+        event_id: int | None = None,
+    ) -> None:
+        self.clear()
+        self._signals.push(self.kind, script, event_id=event_id)
+
+    def has_script(self) -> bool:
+        return self._signals.has_events(self.kind)
+
+    def sample_script(self, chunk_size: int) -> list[Any]:
+        chunk = self._signals.sample_chunk(
+            self.kind,
+            ControlSignalSamplingParams(
+                chunk_size=chunk_size,
+                default_item=self.default_item,
+                repeat_last=False,
+            ),
+        )
+        if chunk is None:
+            chunk = []
+        while len(chunk) < chunk_size and self.default_item is not _MISSING:
+            chunk.append(self._copy_item(self.default_item))
+        return chunk
+
+    def last_sampled_seq_id(self) -> int | None:
+        return self._signals.last_sampled_seq_id(self.kind)
+
+    @staticmethod
+    def _copy_item(item: Any) -> Any:
+        if isinstance(item, list):
+            return copy.deepcopy(item)
+        if isinstance(item, dict):
+            return copy.deepcopy(item)
+        return item
+
+
+class ControlStateQueue:
+    """State-mode sampler for level-triggered realtime controls."""
+
+    def __init__(
+        self,
+        *,
+        default_item: Any,
+        min_pulse_items: int = 1,
+        max_transitions: int = 512,
+    ) -> None:
+        self.default_item = default_item
+        self.min_pulse_items = min_pulse_items
+        self._pending: deque[ControlStateTransition] = deque(maxlen=max_transitions)
+        self._current_item = default_item
+        self._current_seq_id: int | None = None
+        self._latest_sampled_seq_id: int | None = None
+
+    def clear(self) -> None:
+        self._pending.clear()
+        self._current_item = self.default_item
+        self._current_seq_id = None
+        self._latest_sampled_seq_id = None
+
+    def push(self, transition: ControlStateTransition) -> None:
+        self._pending.append(transition)
+
+    def push_many(self, transitions: Sequence[ControlStateTransition]) -> None:
+        for transition in transitions:
+            self.push(transition)
+
+    def sample_chunk(self, chunk_size: int) -> list[Any] | None:
+        if chunk_size <= 0:
+            return None
+
+        transitions = self._drain_pending()
+        if not transitions:
+            self._latest_sampled_seq_id = self._current_seq_id
+            return [self._copy_item(self._current_item) for _ in range(chunk_size)]
+
+        pulse = self._latest_non_default_transition(transitions)
+        final = transitions[-1]
+        self._current_item = final.payload
+        self._current_seq_id = final.seq_id
+
+        if pulse is not None and pulse.payload != final.payload:
+            pulse_items = min(self.min_pulse_items, chunk_size)
+            chunk = [self._copy_item(pulse.payload) for _ in range(pulse_items)]
+            chunk.extend(
+                self._copy_item(final.payload) for _ in range(chunk_size - pulse_items)
+            )
+            self._latest_sampled_seq_id = (
+                final.seq_id if len(chunk) > pulse_items else pulse.seq_id
+            )
+            return chunk
+
+        self._latest_sampled_seq_id = final.seq_id
+        return [self._copy_item(final.payload) for _ in range(chunk_size)]
+
+    def latest_sampled_seq_id(self) -> int | None:
+        return self._latest_sampled_seq_id
+
+    def _drain_pending(self) -> list[ControlStateTransition]:
+        transitions = list(self._pending)
+        self._pending.clear()
+        return transitions
+
+    def _latest_non_default_transition(
+        self,
+        transitions: Sequence[ControlStateTransition],
+    ) -> ControlStateTransition | None:
+        for transition in reversed(transitions):
+            if transition.payload != self.default_item:
+                return transition
+        return None
+
+    @staticmethod
+    def _copy_item(item: Any) -> Any:
+        if isinstance(item, list):
+            return copy.deepcopy(item)
+        if isinstance(item, dict):
+            return copy.deepcopy(item)
+        return item

python/sglang/multimodal_gen/runtime/realtime/session.py

@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from collections import OrderedDict
+from typing import Any
+
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+
+class BaseRealtimeState:
+    """per-session state owned by pipeline stages"""
+
+    def dispose(self) -> None:
+        pass
+
+
+class RealtimeSession:
+    """reusable state container across realtime request chunks"""
+
+    def __init__(self) -> None:
+        self._states: dict[type[BaseRealtimeState], BaseRealtimeState] = {}
+
+    @staticmethod
+    def resolve_session_id(req: Any) -> str | None:
+        session_id = req.realtime_session_id
+        if isinstance(session_id, str) and session_id:
+            return session_id
+        return None
+
+    def get_or_create_state(
+        self, state_cls: type[BaseRealtimeState]
+    ) -> BaseRealtimeState:
+        """returns the BaseRealtimeState instance hold by the current RealtimeSession"""
+        state = self._states.get(state_cls)
+        if state is None:
+            state = state_cls()
+            self._states[state_cls] = state
+        return state
+
+    def get_state(self, state_cls: type[BaseRealtimeState]) -> BaseRealtimeState | None:
+        return self._states.get(state_cls)
+
+    def dispose(self) -> None:
+        for state in list(self._states.values()):
+            state.dispose()
+        self._states.clear()
+
+
+class RealtimeSessionCache:
+    """lru cache that binds incoming chunks to persistent realtime sessions"""
+
+    def __init__(self, max_sessions: int = 64) -> None:
+        self.max_sessions = max_sessions
+        self._sessions: OrderedDict[str, RealtimeSession] = OrderedDict()
+
+    def _dispose_session(
+        self, session_id: str, session: RealtimeSession | None
+    ) -> None:
+        if session is None:
+            return
+        try:
+            session.dispose()
+        except Exception as e:
+            logger.warning(
+                "Failed to dispose realtime session cache entry %s: %s",
+                session_id,
+                e,
+            )
+
+    def release(self, session_id: str) -> bool:
+        session = self._sessions.pop(session_id, None)
+        released = session is not None
+        self._dispose_session(session_id, session)
+        logger.info(
+            "Realtime session release: session_id=%s released=%s",
+            session_id,
+            released,
+        )
+        return released
+
+    def attach(self, req: Any) -> None:
+        session_id = RealtimeSession.resolve_session_id(req)
+        if session_id is None:
+            return
+
+        if session_id not in self._sessions:
+            if req.block_idx > 0:
+                raise ValueError(
+                    "Missing realtime session state for "
+                    f"session_id={session_id} block_idx={req.block_idx}."
+                )
+            self._sessions[session_id] = req.session or RealtimeSession()
+        elif req.block_idx == 0:
+            old_session = self._sessions[session_id]
+            new_session = req.session or RealtimeSession()
+            if old_session is not new_session:
+                self._dispose_session(session_id, old_session)
+            self._sessions[session_id] = new_session
+            logger.info("Realtime session reset: session_id=%s", session_id)
+
+        req.session = self._sessions[session_id]
+        self._sessions.move_to_end(session_id)
+        self._evict_stale_sessions()
+
+    def _evict_stale_sessions(self) -> None:
+        while len(self._sessions) > self.max_sessions:
+            stale_session_id, stale_session = self._sessions.popitem(last=False)
+            self._dispose_session(stale_session_id, stale_session)
+            logger.debug("Evicted stale realtime session cache: %s", stale_session_id)

python/sglang/multimodal_gen/runtime/realtime/states/__init__.py

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: Apache-2.0
+
+"""reusable session-scoped state implementations for realtime pipelines"""
+
+from sglang.multimodal_gen.runtime.realtime.states.camera_control import (
+    RealtimeCameraControlState,
+)
+from sglang.multimodal_gen.runtime.realtime.states.causal import (
+    RealtimeCausalDecodeState,
+    RealtimeCausalDiTState,
+    get_realtime_causal_dit_state,
+)
+
+__all__ = [
+    "RealtimeCameraControlState",
+    "RealtimeCausalDecodeState",
+    "RealtimeCausalDiTState",
+    "get_realtime_causal_dit_state",
+]

python/sglang/multimodal_gen/runtime/realtime/states/camera_control.py

@@ -0,0 +1,156 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from sglang.multimodal_gen.runtime.realtime.control_signals import (
+    ControlScriptQueue,
+    ControlStateQueue,
+    ControlStateTransition,
+    parse_control_event_payload,
+)
+
+CameraActionNormalizer = Callable[[list[Any]], list[str]]
+CameraActionValidator = Callable[[Any], list[list[str]]]
+
+
+def _identity_actions(actions: list[Any]) -> list[str]:
+    return list(actions)
+
+
+class RealtimeCameraControlState:
+    """Session-local camera-control buffer shared by realtime model adapters.
+
+    Camera controls arrive in two shapes:
+
+    1. Script mode: ``list[list[str]]`` where each item is one output-frame's
+       held actions. The script is consumed once from a FIFO and padded with
+       neutral ``[]`` frames after it runs out.
+    2. State mode: timestamped transitions such as "W is currently held".
+       ``ControlStateQueue`` expands that continuous state into the next
+       chunk and can pulse a short key press for a minimum number of frames.
+
+    The two modes are intentionally exclusive. A new script clears state mode,
+    and new state transitions clear script mode, so adapters never merge two
+    camera timelines accidentally. ``sample_camera_actions`` returns ``None``
+    only when no control should be sent; otherwise it returns exactly
+    ``chunk_size`` frames, with ``[]`` meaning neutral/no-op for that frame.
+    """
+
+    def __init__(
+        self,
+        *,
+        min_pulse_items: int = 1,
+        script_maxlen: int = 512,
+        max_transitions: int = 512,
+        normalize_state_actions: CameraActionNormalizer = _identity_actions,
+    ) -> None:
+        # stores state-mode control signals
+        self.camera_state_queue = ControlStateQueue(
+            default_item=[],
+            min_pulse_items=min_pulse_items,
+            max_transitions=max_transitions,
+        )
+        # stores script-mode control signals
+        # script-mode signals take precedence over state-mode signals, see sample_camera_actions
+        self.camera_script_queue = ControlScriptQueue(
+            "camera_actions",
+            max_events=script_maxlen,
+            default_item=[],
+        )
+        self.latest_sampled_event_id: int | None = None
+        self._normalize_state_actions = normalize_state_actions
+
+    def clear(self) -> None:
+        """Reset all camera controls owned by this realtime session."""
+        self.camera_state_queue.clear()
+        self.camera_script_queue.clear()
+        self.latest_sampled_event_id = None
+
+    def receive_camera_action_script(
+        self,
+        camera_actions: list[list[str]],
+        *,
+        event_id: int | None = None,
+    ) -> None:
+        """Replace active controls with a finite per-frame script."""
+        self.camera_state_queue.clear()
+        self.camera_script_queue.push_script(
+            [list(actions) for actions in camera_actions],
+            event_id=event_id,
+        )
+
+    def receive_camera_state_transitions(
+        self,
+        transitions: list[ControlStateTransition],
+    ) -> None:
+        """Replace the script with continuous state transitions."""
+        self.camera_script_queue.clear()
+        self.camera_state_queue.push_many(transitions)
+
+    def receive_camera_state(
+        self,
+        actions: list[str],
+        *,
+        event_id: int | None = None,
+        timestamp_ms: int | None = None,
+    ) -> None:
+        self.receive_camera_state_transitions(
+            [
+                ControlStateTransition(
+                    payload=self._normalize_state_actions(actions),
+                    seq_id=event_id,
+                    timestamp_ms=timestamp_ms,
+                )
+            ]
+        )
+
+    def receive_camera_control_event_payload(
+        self,
+        payload: Any,
+        *,
+        event_id: int | None,
+        validate_camera_actions: CameraActionValidator,
+    ) -> str:
+        """Parse an external camera event (from endpoint) and install it as script or state."""
+        parsed = parse_control_event_payload(
+            payload,
+            event_id=event_id,
+            kind="camera_actions",
+            normalize_state_payload=self._normalize_state_actions,
+            validate_script_payload=validate_camera_actions,
+        )
+        if parsed.mode == "state":
+            transitions = parsed.payload
+            self.receive_camera_state_transitions(transitions)
+            return f"kind=camera_actions, mode=state, transitions={len(transitions)}"
+
+        camera_actions = parsed.payload
+        self.receive_camera_action_script(camera_actions, event_id=event_id)
+        return f"kind=camera_actions, mode=script, frames={len(camera_actions)}"
+
+    def sample_camera_actions(self, chunk_size: int) -> list[list[str]] | None:
+        """Core method, return the next chunk-sized camera action window.
+
+        Script mode has priority because it represents an explicit finite
+        timeline. State mode is sampled only when no script is pending.
+        """
+        # Script mode wins: it is an explicit finite timeline and should not be
+        # merged with held-key state from the live control path.
+        if self.camera_script_queue.has_script():
+            return self._sample_camera_script(chunk_size)
+        # State mode is the WebUI path: held controls persist across chunks until
+        # a new transition changes the current state.
+        action_list = self.camera_state_queue.sample_chunk(chunk_size)
+        if action_list is None:
+            return None
+        self.latest_sampled_event_id = self.camera_state_queue.latest_sampled_seq_id()
+        return [list(actions) for actions in action_list]
+
+    def _sample_camera_script(self, chunk_size: int) -> list[list[str]]:
+        chunk = self.camera_script_queue.sample_script(chunk_size)
+        chunk = [list(actions) for actions in chunk]
+        self.latest_sampled_event_id = self.camera_script_queue.last_sampled_seq_id()
+        return chunk

python/sglang/multimodal_gen/runtime/realtime/states/causal.py

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any, TypeVar, cast
+
+from sglang.multimodal_gen.runtime.realtime.session import (
+    BaseRealtimeState,
+    RealtimeSession,
+)
+
+
+class RealtimeCausalDiTState(BaseRealtimeState):
+    """persist causal DiT cache, chunk frontier, and output buffer"""
+
+    def __init__(self):
+        super().__init__()
+        self.kv_cache: Any = None
+        self.crossattn_cache: Any = None
+        self.runtime_cache: dict = {}
+        self.current_chunk_start_frame: int = 0
+        self.chunk_idx: int = 0
+        self.chunk_indices: list[int] = [0]
+        self.latents: Any = None
+        self.scheduler: Any = None
+
+    def dispose(self) -> None:
+        self.kv_cache = None
+        self.crossattn_cache = None
+        self.runtime_cache.clear()
+        self.current_chunk_start_frame = 0
+        self.chunk_idx = 0
+        self.chunk_indices = [0]
+        self.latents = None
+        self.scheduler = None
+
+
+RealtimeCausalDiTStateT = TypeVar(
+    "RealtimeCausalDiTStateT", bound=RealtimeCausalDiTState
+)
+
+
+def get_realtime_causal_dit_state(
+    session: RealtimeSession,
+    state_cls: type[RealtimeCausalDiTStateT] = RealtimeCausalDiTState,
+) -> RealtimeCausalDiTStateT:
+    return cast(RealtimeCausalDiTStateT, session.get_or_create_state(state_cls))
+
+
+class RealtimeCausalDecodeState(BaseRealtimeState):
+    """persist causal VAE decode cache and output frontier across chunks"""
+
+    def __init__(self):
+        super().__init__()
+        self.conv_cache: dict | None = None
+        self.next_dec_idx: int = 0
+
+    def dispose(self) -> None:
+        self.conv_cache = None
+        self.next_dec_idx = 0