#!/usr/bin/env python3 """ Example using an example depth dataset from NYU. https://cs.nyu.edu/~fergus/datasets/nyu_depth_v2.html """ from __future__ import annotations import argparse import os import sys import zipfile from datetime import datetime from pathlib import Path from typing import Any, Final import cv2 import numpy as np import numpy.typing as npt import requests from tqdm import tqdm import rerun as rr # pip install rerun-sdk import rerun.blueprint as rrb DESCRIPTION = """ # RGBD Visualizes an example recording from [the NYUD dataset](https://cs.nyu.edu/~silberman/datasets/nyu_depth_v2.html) with RGB and Depth channels. The full source code for this example is available [on GitHub](https://github.com/rerun-io/rerun/blob/latest/examples/python/rgbd). """ DEPTH_IMAGE_SCALING: Final = 1e4 DATASET_DIR: Final = Path(os.path.dirname(__file__)) / "dataset" DATASET_URL_BASE: Final = "https://static.rerun.io/rgbd_dataset" DATASET_URL_BASE_ALTERNATE: Final = "https://cs.nyu.edu/~fergus/datasets/nyu_depth_v2.html" AVAILABLE_RECORDINGS: Final = ["cafe", "basements", "studies", "office_kitchens", "playroooms"] def parse_timestamp(filename: str) -> datetime: """Parse the timestamp portion of the filename.""" file_name_parts = filename.split("-") time = file_name_parts[len(file_name_parts) - 2] return datetime.fromtimestamp(float(time)) def read_image_bgr(buf: bytes) -> npt.NDArray[np.uint8]: """Decode an image provided in `buf`, and interpret it as RGB data.""" np_buf: npt.NDArray[np.uint8] = np.ndarray(shape=(1, len(buf)), dtype=np.uint8, buffer=buf) img_bgr: npt.NDArray[Any] = cv2.imdecode(np_buf, cv2.IMREAD_COLOR) return img_bgr def read_depth_image(buf: bytes) -> npt.NDArray[Any]: """Decode an image provided in `buf`.""" np_buf: npt.NDArray[np.uint8] = np.ndarray(shape=(1, len(buf)), dtype=np.uint8, buffer=buf) img: npt.NDArray[Any] = cv2.imdecode(np_buf, cv2.IMREAD_UNCHANGED) return img def log_nyud_data(recording_path: Path, subset_idx: int, frames: int) -> None: rr.log("world", rr.ViewCoordinates.RIGHT_HAND_Y_DOWN, static=True) with zipfile.ZipFile(recording_path, "r") as archive: archive_dirs = [f.filename for f in archive.filelist if f.is_dir()] print(f"Using recording subset {subset_idx} ([0 - {len(archive_dirs) - 1}] available).") dir_to_log = archive_dirs[subset_idx] subset = [ f for f in archive.filelist if f.filename.startswith(dir_to_log) and (f.filename.endswith(".ppm") or f.filename.endswith(".pgm")) ] files_with_timestamps = [(parse_timestamp(f.filename), f) for f in subset] files_with_timestamps.sort(key=lambda t: t[0]) if len(files_with_timestamps) > frames: files_with_timestamps = files_with_timestamps[:frames] for time, f in files_with_timestamps: rr.set_time("time", timestamp=time.timestamp()) if f.filename.endswith(".ppm"): buf = archive.read(f) img_bgr = read_image_bgr(buf) rr.log("world/camera/image/rgb", rr.Image(img_bgr, color_model="BGR").compress(jpeg_quality=95)) elif f.filename.endswith(".pgm"): buf = archive.read(f) img_depth = read_depth_image(buf) # Log the camera transforms: rr.log( "world/camera/image", rr.Pinhole( resolution=[img_depth.shape[1], img_depth.shape[0]], focal_length=0.7 * img_depth.shape[1], # Intentionally off-center to demonstrate that we support it principal_point=[0.45 * img_depth.shape[1], 0.55 * img_depth.shape[0]], ), ) # Log the depth image to the cameras image-space: rr.log("world/camera/image/depth", rr.DepthImage(img_depth, meter=DEPTH_IMAGE_SCALING)) def ensure_recording_downloaded(name: str) -> Path: recording_filename = f"{name}.zip" recording_path = DATASET_DIR / recording_filename if recording_path.exists(): return recording_path url = f"{DATASET_URL_BASE}/{recording_filename}" alternate_url = f"{DATASET_URL_BASE_ALTERNATE}/{recording_filename}" os.makedirs(DATASET_DIR, exist_ok=True) try: try: print(f"downloading {url} to {recording_path}") download_progress(url, recording_path) except ValueError: print(f"Failed to download from {url}, trying backup URL {alternate_url} instead") download_progress(alternate_url, recording_path) except BaseException: recording_path.unlink(missing_ok=True) raise return recording_path def download_progress(url: str, dst: Path) -> None: """ Download file with tqdm progress bar. From: """ resp = requests.get(url, stream=True) if resp.status_code != 200: raise ValueError(f"Failed to download file (status code: {resp.status_code})") total = int(resp.headers.get("content-length", 0)) chunk_size = 1024 * 1024 # Can also replace 'file' with a io.BytesIO object with ( open(dst, "wb") as file, tqdm( desc=dst.name, total=total, unit="iB", unit_scale=True, unit_divisor=1024, ) as bar, ): for data in resp.iter_content(chunk_size=chunk_size): size = file.write(data) bar.update(size) def main() -> None: parser = argparse.ArgumentParser(description="Example using an example depth dataset from NYU.") parser.add_argument( "--recording", type=str, choices=AVAILABLE_RECORDINGS, default=AVAILABLE_RECORDINGS[0], help="Name of the NYU Depth Dataset V2 recording", ) parser.add_argument("--subset-idx", type=int, default=0, help="The index of the subset of the recording to use.") parser.add_argument( "--frames", type=int, default=sys.maxsize, help="If specified, limits the number of frames logged", ) rr.script_add_args(parser) args = parser.parse_args() rr.script_setup( args, "rerun_example_rgbd", default_blueprint=rrb.Horizontal( rrb.Vertical( rrb.Spatial3DView(name="3D", origin="world"), rrb.TextDocumentView(name="Description", origin="/description"), row_shares=[7, 3], ), rrb.Vertical( # Put the origin for both 2D spaces where the pinhole is logged. Doing so allows them to understand how they're connected to the 3D space. # This enables interactions like clicking on a point in the 3D space to show the corresponding point in the 2D spaces and vice versa. rrb.Spatial2DView( name="RGB & Depth", origin="world/camera/image", overrides={"world/camera/image/rgb": rr.Image.from_fields(opacity=0.5)}, ), rrb.Tabs( rrb.Spatial2DView(name="RGB", origin="world/camera/image", contents="world/camera/image/rgb"), rrb.Spatial2DView(name="Depth", origin="world/camera/image", contents="world/camera/image/depth"), ), name="2D", row_shares=[3, 3, 2], ), column_shares=[2, 1], ), ) recording_path = ensure_recording_downloaded(args.recording) rr.log("description", rr.TextDocument(DESCRIPTION, media_type=rr.MediaType.MARKDOWN), static=True) log_nyud_data( recording_path=recording_path, subset_idx=args.subset_idx, frames=args.frames, ) rr.script_teardown(args) if __name__ == "__main__": main()