from collections import deque from contextlib import contextmanager from dataclasses import dataclass from typing import Callable, Deque, Dict, List, Optional import torch class DeviceTimer: def __init__(self, reporter: Callable): self._intervals: Deque[_TimingInterval] = deque() self._reporters: List[Callable] = [reporter] def add_reporter(self, reporter: Callable): self._reporters.append(reporter) @contextmanager def wrap(self, metadata: Dict): self._intervals.append(_TimingInterval.create()) try: yield finally: self._intervals[-1].end(metadata=metadata) self._report() def _report(self): while len(self._intervals) > 0: interval = self._intervals[0] if not interval.end_event.query(): break self._intervals.popleft() elapsed = interval.elapsed_time() / 1000.0 for reporter in self._reporters: reporter(t=elapsed, **interval.metadata) class GapTimer(DeviceTimer): """Measures GPU idle gaps between consecutive uses of a stream. Where DeviceTimer.wrap() measures the duration *inside* a block, GapTimer.wrap() measures the time *between* consecutive blocks (gap = next_block_start - last_block_end). """ def __init__(self, reporter: Callable): super().__init__(reporter) self._pending: Optional[_TimingInterval] = None @contextmanager def wrap(self, metadata: Dict): if self._pending is not None: self._pending.end(metadata=metadata) self._intervals.append(self._pending) self._pending = None self._report() try: yield finally: self._pending = _TimingInterval.create() def cancel(self): """Discard a pending gap (e.g. server went idle).""" self._pending = None @dataclass class _TimingInterval: start_event: torch.cuda.Event end_event: Optional[torch.cuda.Event] = None metadata: Optional[Dict] = None @staticmethod def create(): start_event = torch.cuda.Event(enable_timing=True) start_event.record() return _TimingInterval(start_event=start_event) def end(self, metadata: Dict): end_event = torch.cuda.Event(enable_timing=True) end_event.record() assert self.end_event is None self.end_event = end_event self.metadata = metadata def elapsed_time(self) -> float: return self.start_event.elapsed_time(self.end_event)