# Copyright 2024 NVIDIA CORPORATION & AFFILIATES # Licensed under the Apache License, Version 2.0 # SPDX-License-Identifier: Apache-2.0 """Shared camera-control core. One implementation of the pose math + velocity model, used by BOTH the action-string inference rollout (``action_string_to_c2w``) and the interactive realtime demo, so the two can never drift. Only the *input surface* differs: the inference path maps DSL letters, the demo maps physical keys — both produce the same canonical controls and run through the same smoother + integrator. Coordinate convention: OpenCV (``+X right, +Y down, +Z forward``); poses are camera-to-world, starting from the identity. Control scheme (unified): W / S forward / back translation (along heading) A / D yaw left / right rotation (DSL: a/d ; demo: A/D) Up / Down pitch up / down rotation (DSL: i/k ; demo: ArrowUp/Down) Left / Right strafe left / right translation (DSL: j/l ; demo: ArrowLeft/Right) """ from __future__ import annotations import math from dataclasses import dataclass import numpy as np FPS = 16 # Gentle default motion magnitudes (per frame), shared by demo + inference. DEFAULT_TRANSLATION_SPEED = 0.025 DEFAULT_ROTATION_SPEED_DEG = 0.6 DEFAULT_PITCH_LIMIT_DEG = 60.0 # Exponential-smoothing time constants (seconds): faster ramp on press, slower # coast on release so motion eases to a stop instead of snapping. TAU_PRESS = 0.45 TAU_COAST = 1.0 # Canonical control tokens. CONTROL_TOKENS = frozenset( {"forward", "back", "strafe_left", "strafe_right", "yaw_left", "yaw_right", "pitch_up", "pitch_down"} ) # Input-surface mappings -> canonical controls (the ONLY thing that differs # between the two entry points). DSL_KEY_TO_CONTROL: dict[str, str] = { "w": "forward", "s": "back", "a": "yaw_left", "d": "yaw_right", "i": "pitch_up", "k": "pitch_down", "j": "strafe_left", "l": "strafe_right", } DEMO_KEY_TO_CONTROL: dict[str, str] = { "w": "forward", "s": "back", "a": "yaw_left", "d": "yaw_right", "up": "pitch_up", "down": "pitch_down", "left": "strafe_left", "right": "strafe_right", } def rot_x(angle_rad: float) -> np.ndarray: c, s = math.cos(angle_rad), math.sin(angle_rad) return np.array([[1.0, 0.0, 0.0], [0.0, c, -s], [0.0, s, c]], dtype=np.float64) def rot_y(angle_rad: float) -> np.ndarray: c, s = math.cos(angle_rad), math.sin(angle_rad) return np.array([[c, 0.0, s], [0.0, 1.0, 0.0], [-s, 0.0, c]], dtype=np.float64) @dataclass class VelocityState: """Per-frame velocity: translation (tx forward, sx strafe-right) + rotation (yaw right+, pitch up+), in the same units as the motion magnitudes.""" tx: float = 0.0 sx: float = 0.0 yaw: float = 0.0 pitch: float = 0.0 def snap_to(self, target: VelocityState) -> None: self.tx, self.sx, self.yaw, self.pitch = target.tx, target.sx, target.yaw, target.pitch def step_toward(self, target: VelocityState, dt: float) -> None: for attr in ("tx", "sx", "yaw", "pitch"): cur = getattr(self, attr) tgt = getattr(target, attr) tau = TAU_PRESS if abs(tgt) > 1e-12 else TAU_COAST alpha = 1.0 - math.exp(-dt / tau) setattr(self, attr, cur + alpha * (tgt - cur)) def controls_to_target_velocity( controls: set[str], *, translation_speed: float = DEFAULT_TRANSLATION_SPEED, rotation_speed_rad: float | None = None, ) -> VelocityState: """Map a set of canonical control tokens to a target velocity.""" if rotation_speed_rad is None: rotation_speed_rad = math.radians(DEFAULT_ROTATION_SPEED_DEG) fwd = (1.0 if "forward" in controls else 0.0) - (1.0 if "back" in controls else 0.0) strafe = (1.0 if "strafe_right" in controls else 0.0) - (1.0 if "strafe_left" in controls else 0.0) yaw = (1.0 if "yaw_right" in controls else 0.0) - (1.0 if "yaw_left" in controls else 0.0) pit = (1.0 if "pitch_up" in controls else 0.0) - (1.0 if "pitch_down" in controls else 0.0) return VelocityState( tx=fwd * translation_speed, sx=strafe * translation_speed, yaw=yaw * rotation_speed_rad, pitch=pit * rotation_speed_rad, ) class CameraPoseIntegrator: """Integrate per-frame velocity into a camera-to-world pose (the shared core). ``rot_y(yaw) @ R @ rot_x(pitch)`` for rotation; translation is on the horizontal (y=0) plane along the projected forward / right axes. Pitch saturates at ``±pitch_limit``. """ def __init__(self, pitch_limit_rad: float = math.radians(DEFAULT_PITCH_LIMIT_DEG)) -> None: self.pose = np.eye(4, dtype=np.float64) self.pitch = 0.0 self.pitch_limit = float(pitch_limit_rad) def step(self, v: VelocityState) -> np.ndarray: new_pitch = max(-self.pitch_limit, min(self.pitch_limit, self.pitch + v.pitch)) pitch_step = new_pitch - self.pitch self.pitch = new_pitch R = self.pose[:3, :3] R_new = rot_y(v.yaw) @ R @ rot_x(pitch_step) fwd = R_new[:, 2].copy() fwd[1] = 0.0 rgt = R_new[:, 0].copy() rgt[1] = 0.0 fn = float(np.linalg.norm(fwd)) rn = float(np.linalg.norm(rgt)) if fn > 0: fwd /= fn + 1e-6 if rn > 0: rgt /= rn + 1e-6 T_ = self.pose[:3, 3] + fwd * v.tx + rgt * v.sx self.pose = np.eye(4, dtype=np.float64) self.pose[:3, :3] = R_new self.pose[:3, 3] = T_ return self.pose.copy()