chore: import upstream snapshot with attribution

python/ray/data/_internal/issue_detection/__init__.py

@@ -0,0 +1,20 @@
+from ray.data._internal.issue_detection.detectors.hanging_detector import (
+    HangingExecutionIssueDetector,
+    HangingExecutionIssueDetectorConfig,
+)
+from ray.data._internal.issue_detection.issue_detector import Issue, IssueDetector
+from ray.data._internal.issue_detection.issue_detector_configuration import (
+    IssueDetectorsConfiguration,
+)
+from ray.data._internal.issue_detection.issue_detector_manager import (
+    IssueDetectorManager,
+)
+
+__all__ = [
+    "Issue",
+    "IssueDetector",
+    "IssueDetectorManager",
+    "IssueDetectorsConfiguration",
+    "HangingExecutionIssueDetector",
+    "HangingExecutionIssueDetectorConfig",
+]

python/ray/data/_internal/issue_detection/detectors/__init__.py

@@ -0,0 +1,21 @@
+from ray.data._internal.issue_detection.detectors.hanging_detector import (
+    HangingExecutionIssueDetector,
+    HangingExecutionIssueDetectorConfig,
+)
+from ray.data._internal.issue_detection.detectors.hash_shuffle_detector import (
+    HashShuffleAggregatorIssueDetector,
+    HashShuffleAggregatorIssueDetectorConfig,
+)
+from ray.data._internal.issue_detection.detectors.high_memory_detector import (
+    HighMemoryIssueDetector,
+    HighMemoryIssueDetectorConfig,
+)
+
+__all__ = [
+    "HangingExecutionIssueDetector",
+    "HangingExecutionIssueDetectorConfig",
+    "HashShuffleAggregatorIssueDetector",
+    "HashShuffleAggregatorIssueDetectorConfig",
+    "HighMemoryIssueDetector",
+    "HighMemoryIssueDetectorConfig",
+]

python/ray/data/_internal/issue_detection/detectors/hanging_detector.py

@@ -0,0 +1,282 @@
+import logging
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, DefaultDict, Dict, List, Optional, Union
+
+import requests
+
+import ray
+from ray.data._internal.execution.interfaces.op_runtime_metrics import RunningTaskInfo
+from ray.data._internal.issue_detection.issue_detector import (
+    Issue,
+    IssueDetector,
+    IssueType,
+)
+from ray.util.state import get_task
+from ray.util.state.common import TaskState
+from ray.util.state.exception import RayStateApiException
+
+if TYPE_CHECKING:
+    from ray.data._internal.execution.interfaces.physical_operator import (
+        PhysicalOperator,
+    )
+    from ray.data._internal.execution.streaming_executor import StreamingExecutor
+
+# Default minimum count of tasks before using adaptive thresholds
+DEFAULT_OP_TASK_STATS_MIN_COUNT = 10
+# Default multiple of standard deviations to use as hanging threshold
+DEFAULT_OP_TASK_STATS_STD_FACTOR = 10
+# Default detection time interval.
+DEFAULT_DETECTION_TIME_INTERVAL_S = 30.0
+
+logger = logging.getLogger(__name__)
+
+OpId = str
+TaskIdx = int
+# Map of operator id -> task index -> hanging execution state.
+HangingOpTasks = DefaultDict[OpId, Dict[TaskIdx, "HangingExecutionState"]]
+
+
+def _format_timestamp(epoch: float) -> str:
+    """Format a ``time.time()`` epoch value as a human-readable UTC string."""
+    return datetime.fromtimestamp(epoch, tz=timezone.utc).strftime(
+        "%Y-%m-%d %H:%M:%S %Z"
+    )
+
+
+@dataclass
+class TaskMetadata:
+    """Subset of TaskState fields relevant for hanging detection."""
+
+    attempt_number: int
+    node_id: str
+    pid: int
+
+    @classmethod
+    def from_task_state(cls, task_state: TaskState) -> "TaskMetadata":
+        return cls(
+            attempt_number=task_state.attempt_number,
+            node_id=task_state.node_id,
+            pid=task_state.worker_pid,
+        )
+
+
+@dataclass
+class HangingExecutionState:
+    operator_id: OpId
+    task_idx: TaskIdx
+    task_id: ray.TaskID
+    task_metadata: Optional[TaskMetadata]
+    bytes_output: int
+    # NOTE This is from perf_couinter()
+    start_time_hanging: float
+
+    def hanging_time(self):
+        return time.perf_counter() - self.start_time_hanging
+
+
+@dataclass
+class HangingExecutionIssueDetectorConfig:
+    op_task_stats_min_count: int = field(default=DEFAULT_OP_TASK_STATS_MIN_COUNT)
+    op_task_stats_std_factor: float = field(default=DEFAULT_OP_TASK_STATS_STD_FACTOR)
+    detection_time_interval_s: float = DEFAULT_DETECTION_TIME_INTERVAL_S
+
+
+class HangingExecutionIssueDetector(IssueDetector):
+    def __init__(
+        self,
+        dataset_id: str,
+        operators: List["PhysicalOperator"],
+        config: HangingExecutionIssueDetectorConfig,
+    ):
+        self._dataset_id = dataset_id
+        self._operators = operators
+        self._detector_cfg = config
+
+        self._op_task_stats_min_count = self._detector_cfg.op_task_stats_min_count
+        self._op_task_stats_std_factor_threshold = (
+            self._detector_cfg.op_task_stats_std_factor
+        )
+
+        # Map of operator id to Dict[task index, state]
+        self._hanging_op_tasks: HangingOpTasks = defaultdict(dict)
+
+    @classmethod
+    def from_executor(
+        cls, executor: "StreamingExecutor"
+    ) -> "HangingExecutionIssueDetector":
+        """Factory method to create a HangingExecutionIssueDetector from a StreamingExecutor.
+
+        Args:
+            executor: The StreamingExecutor instance to extract dependencies from.
+
+        Returns:
+            An instance of HangingExecutionIssueDetector.
+        """
+        operators = list(executor._topology.keys()) if executor._topology else []
+        ctx = executor._data_context
+        return cls(
+            dataset_id=executor._dataset_id,
+            operators=operators,
+            config=ctx.issue_detectors_config.hanging_detector_config,
+        )
+
+    def _create_issue(
+        self,
+        operator: "PhysicalOperator",
+        hanging_execution_state: HangingExecutionState,
+    ) -> Issue:
+
+        hes = hanging_execution_state
+        op_task_stats = operator.metrics.op_task_duration_stats
+        avg_duration = op_task_stats.mean
+        stdev = op_task_stats.stddev
+
+        meta = hes.task_metadata
+        task_info = ""
+        if meta is not None:
+            task_info = f"(pid={meta.pid}, node_id={meta.node_id}, attempt={meta.attempt_number}) "
+
+        hanging_time = hes.hanging_time()
+        hanging_since = _format_timestamp(time.time() - hanging_time)
+
+        message = (
+            f"A task (task_id={hes.task_id}) of operator "
+            f"{operator.name} {task_info}has been running or stuck in scheduling for "
+            f"{hanging_time:.2f}s, which is longer than the average task "
+            f"duration + z-score * stddev of this operator "
+            f"({avg_duration:.2f} + "
+            f"{self._op_task_stats_std_factor_threshold} * "
+            f"{stdev:.2f}s). "
+            f"Last time task produced output or made any progress was {hanging_since}. "
+            f"If this message persists, please check "
+            f"the stack trace of the task for potential hanging "
+            f"issues. To adjust the z-score value, set "
+            f"`ray.data.DataContext.get_current()"
+            f".issue_detectors_config.hanging_detector_config"
+            f".op_task_stats_std_factor`."
+        )
+
+        return Issue(
+            dataset_name=self._dataset_id,
+            operator_id=hes.operator_id,
+            issue_type=IssueType.HANGING,
+            message=message,
+        )
+
+    def _refresh_state(
+        self,
+        operator: "PhysicalOperator",
+        task_idx: TaskIdx,
+        old_state: Optional[HangingExecutionState],
+        task_info: RunningTaskInfo,
+    ) -> HangingExecutionState:
+        """Build a HangingExecutionState, fetching task metadata lazily.
+
+        Task metadata (pid, node_id, attempt) is fetched from the Ray
+        State API only when unknown or potentially stale (e.g. after the
+        task made progress then stalled again).
+        """
+        task_metadata: Optional[TaskMetadata] = None
+        if old_state is not None:
+            task_metadata = old_state.task_metadata
+        else:
+            task_metadata = get_latest_state_for_task(task_info.task_id)
+
+        return HangingExecutionState(
+            operator_id=operator.id,
+            task_idx=task_idx,
+            task_id=task_info.task_id,
+            task_metadata=task_metadata,
+            bytes_output=task_info.bytes_output,
+            start_time_hanging=task_info.last_updated,
+        )
+
+    def detect(self) -> List[Issue]:
+
+        issues: List[Issue] = []
+        # Build fresh maps each cycle so that tasks which finished or
+        # dropped below the threshold are automatically pruned.
+        hanging_op_tasks: HangingOpTasks = defaultdict(dict)
+
+        for operator in self._operators:
+            if operator.has_execution_finished():
+                continue
+
+            op_metrics = operator.metrics
+            op_task_stats = op_metrics.op_task_duration_stats
+            # 1) Skip if not reached minimum task count
+            if op_task_stats.num_samples < self._op_task_stats_min_count:
+                continue
+
+            # 2) Skip if under threshold of mean + z-score * stddev
+            mean = op_task_stats.mean
+            stddev = op_task_stats.stddev
+            threshold = mean + self._op_task_stats_std_factor_threshold * stddev
+
+            for task_idx, task_info in op_metrics._running_tasks.items():
+
+                time_since_last_update = time.perf_counter() - task_info.last_updated
+                if time_since_last_update <= threshold:
+                    continue
+
+                old_state = self._hanging_op_tasks[operator.id].get(task_idx)
+
+                new_state = self._refresh_state(
+                    operator=operator,
+                    task_idx=task_idx,
+                    old_state=old_state,
+                    task_info=task_info,
+                )
+
+                hanging_op_tasks[operator.id][task_idx] = new_state
+
+                if old_state == new_state:
+                    continue
+
+                issues.append(
+                    self._create_issue(
+                        operator=operator, hanging_execution_state=new_state
+                    )
+                )
+
+        self._hanging_op_tasks = hanging_op_tasks
+        return issues
+
+    def detection_time_interval_s(self) -> float:
+        return self._detector_cfg.detection_time_interval_s
+
+
+def get_latest_state_for_task(task_id: ray.TaskID) -> Optional[TaskMetadata]:
+    """Query the Ray State API for the latest attempt of a task.
+
+    Returns a TaskMetadata with the highest attempt_number when multiple
+    attempts exist, or None if the task is not found (can happen when
+    get_task() is called shortly after submission, before the state API
+    has indexed it) or the API is unreachable.
+    """
+    try:
+        # NOTE: timeout is set to 1 because ray will take max(1, timeout).
+        # TODO(Justin): Make this asynchronous
+        task_state: Union[TaskState, List[TaskState], None] = get_task(
+            task_id.hex(),
+            timeout=1,
+            _explain=True,
+        )
+    except (RayStateApiException, requests.exceptions.RequestException):
+        logger.debug(f"Failed to grab task state with task_id={task_id}", exc_info=True)
+        return None
+    except Exception:
+        logger.debug(
+            f"Unexpected error when grabbing task state with task_id={task_id}",
+            exc_info=True,
+        )
+        return None
+    if isinstance(task_state, list):
+        # get the latest task
+        task_state = max(task_state, key=lambda ts: ts.attempt_number, default=None)
+    if task_state is None:
+        return None
+    return TaskMetadata.from_task_state(task_state)

python/ray/data/_internal/issue_detection/detectors/hash_shuffle_detector.py

@@ -0,0 +1,145 @@
+import time
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List
+
+import ray
+from ray.data._internal.execution.operators.hash_shuffle import (
+    AggregatorHealthInfo,
+    HashShuffleOperator,
+)
+from ray.data._internal.issue_detection.issue_detector import (
+    Issue,
+    IssueDetector,
+    IssueType,
+)
+from ray.data._internal.util import GiB
+
+if TYPE_CHECKING:
+    from ray.data._internal.execution.interfaces.physical_operator import (
+        PhysicalOperator,
+    )
+    from ray.data._internal.execution.streaming_executor import StreamingExecutor
+
+
+@dataclass
+class HashShuffleAggregatorIssueDetectorConfig:
+    """Configuration for HashShuffleAggregatorIssueDetector."""
+
+    detection_time_interval_s: float = 30.0
+    min_wait_time_s: float = 300.0
+
+
+class HashShuffleAggregatorIssueDetector(IssueDetector):
+    """Detector for hash shuffle aggregator health issues."""
+
+    def __init__(
+        self,
+        dataset_id: str,
+        operators: List["PhysicalOperator"],
+        config: HashShuffleAggregatorIssueDetectorConfig,
+    ):
+        self._dataset_id = dataset_id
+        self._operators = operators
+        self._detector_cfg = config
+        self._last_warning_times = {}  # Track per-operator warning times
+
+    @classmethod
+    def from_executor(
+        cls, executor: "StreamingExecutor"
+    ) -> "HashShuffleAggregatorIssueDetector":
+        """Factory method to create a HashShuffleAggregatorIssueDetector from a StreamingExecutor.
+
+        Args:
+            executor: The StreamingExecutor instance to extract dependencies from.
+
+        Returns:
+            An instance of HashShuffleAggregatorIssueDetector.
+        """
+        operators = list(executor._topology.keys()) if executor._topology else []
+        ctx = executor._data_context
+        return cls(
+            dataset_id=executor._dataset_id,
+            operators=operators,
+            config=ctx.issue_detectors_config.hash_shuffle_detector_config,
+        )
+
+    def detect(self) -> List[Issue]:
+        issues = []
+        current_time = time.time()
+
+        # Find all hash shuffle operators in the topology
+        for op in self._operators:
+            if not isinstance(op, HashShuffleOperator):
+                continue
+
+            # Skip if operator doesn't have aggregator pool yet
+            if op._aggregator_pool is None:
+                continue
+
+            pool = op._aggregator_pool
+            aggregator_info = pool.check_aggregator_health()
+
+            if aggregator_info is None:
+                continue
+
+            # Check if we should emit a warning for this operator
+            should_warn = self._should_emit_warning(
+                op.id, current_time, aggregator_info
+            )
+
+            if should_warn:
+                message = self._format_health_warning(aggregator_info)
+                issues.append(
+                    Issue(
+                        dataset_name=self._dataset_id,
+                        operator_id=op.id,
+                        issue_type=IssueType.HANGING,
+                        message=message,
+                    )
+                )
+                self._last_warning_times[op.id] = current_time
+
+        return issues
+
+    def detection_time_interval_s(self) -> float:
+        return self._detector_cfg.detection_time_interval_s
+
+    def _should_emit_warning(
+        self, op_id: str, current_time: float, info: AggregatorHealthInfo
+    ) -> bool:
+        """Check if we should emit a warning for this operator."""
+        if not info.has_unready_aggregators:
+            # Clear warning time if all aggregators are healthy
+            self._last_warning_times.pop(op_id, None)
+            return False
+
+        # Check if enough time has passed since start
+        if current_time - info.started_at < self._detector_cfg.min_wait_time_s:
+            return False
+
+        # Check if enough time has passed since last warning
+        last_warning = self._last_warning_times.get(op_id)
+        if last_warning is None:
+            return True
+
+        return current_time - last_warning >= self.detection_time_interval_s()
+
+    def _format_health_warning(self, info: AggregatorHealthInfo) -> str:
+        """Format the health warning message."""
+        available_resources = ray.available_resources()
+        available_cpus = available_resources.get("CPU", 0)
+        cluster_resources = ray.cluster_resources()
+        total_memory = cluster_resources.get("memory", 0)
+        available_memory = available_resources.get("memory", 0)
+
+        return (
+            f"Only {info.ready_aggregators} out of {info.total_aggregators} "
+            f"hash-shuffle aggregators are ready after {info.wait_time:.1f} secs. "
+            f"This might indicate resource contention for cluster resources "
+            f"(available CPUs: {available_cpus}, required CPUs: {info.required_resources.cpu}). "
+            f"Cluster only has {available_memory / GiB:.2f} GiB available memory, required memory: {info.required_resources.memory / GiB:.2f} GiB. "
+            f"{total_memory / GiB:.2f} GiB total memory. "
+            f"Consider increasing cluster size or reducing the number of aggregators "
+            f"via `DataContext.max_hash_shuffle_aggregators`. "
+            f"Will continue checking every {self.detection_time_interval_s()}s."
+        )

python/ray/data/_internal/issue_detection/detectors/high_memory_detector.py

@@ -0,0 +1,124 @@
+import textwrap
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Dict, List
+
+from ray.data._internal.execution.operators.map_operator import (
+    MapOperator,
+    get_safe_default_logical_memory,
+)
+from ray.data._internal.execution.util import memory_string
+from ray.data._internal.issue_detection.issue_detector import (
+    Issue,
+    IssueDetector,
+    IssueType,
+)
+
+if TYPE_CHECKING:
+    from ray.data._internal.execution.interfaces.physical_operator import (
+        PhysicalOperator,
+    )
+    from ray.data._internal.execution.streaming_executor import StreamingExecutor
+
+HIGH_MEMORY_PERIODIC_WARNING = """
+Operator '{op_name}' uses {memory_per_task} of memory per task on average, but Ray
+only requests {initial_memory_request} per task at the start of the pipeline.
+
+To avoid out-of-memory errors, consider setting `memory={memory_per_task}` in the
+appropriate function or method call. (This might be unnecessary if the number of
+concurrent tasks is low.)
+
+To change the frequency of this warning, set
+`DataContext.get_current().issue_detectors_config.high_memory_detector_config.detection_time_interval_s`,
+or disable the warning by setting value to -1. (current value:
+{detection_time_interval_s})
+"""  # noqa: E501
+
+
+@dataclass
+class HighMemoryIssueDetectorConfig:
+    detection_time_interval_s: float = 30
+
+
+class HighMemoryIssueDetector(IssueDetector):
+    def __init__(
+        self,
+        dataset_id: str,
+        operators: List["PhysicalOperator"],
+        config: HighMemoryIssueDetectorConfig,
+    ):
+        self._dataset_id = dataset_id
+        self._detector_cfg = config
+        self._operators = operators
+
+        self._initial_memory_requests: Dict[MapOperator, int] = {}
+        for op in operators:
+            if isinstance(op, MapOperator):
+                self._initial_memory_requests[op] = (
+                    op._get_dynamic_ray_remote_args().get("memory") or 0
+                )
+
+    @classmethod
+    def from_executor(cls, executor: "StreamingExecutor") -> "HighMemoryIssueDetector":
+        """Factory method to create a HighMemoryIssueDetector from a StreamingExecutor.
+
+        Args:
+            executor: The StreamingExecutor instance to extract dependencies from.
+
+        Returns:
+            An instance of HighMemoryIssueDetector.
+        """
+        operators = list(executor._topology.keys()) if executor._topology else []
+        ctx = executor._data_context
+        return cls(
+            dataset_id=executor._dataset_id,
+            operators=operators,
+            config=ctx.issue_detectors_config.high_memory_detector_config,
+        )
+
+    def detect(self) -> List[Issue]:
+        issues = []
+        for op in self._operators:
+            if not isinstance(op, MapOperator):
+                continue
+
+            if op.metrics.average_max_uss_per_task is None:
+                continue
+
+            remote_args = op._get_dynamic_ray_remote_args()
+            safe_memory_per_task = get_safe_default_logical_memory(remote_args)
+
+            if (
+                op.metrics.average_max_uss_per_task > self._initial_memory_requests[op]
+                and op.metrics.average_max_uss_per_task >= safe_memory_per_task
+            ):
+                message = HIGH_MEMORY_PERIODIC_WARNING.format(
+                    op_name=op.name,
+                    memory_per_task=memory_string(op.metrics.average_max_uss_per_task),
+                    initial_memory_request=memory_string(
+                        self._initial_memory_requests[op]
+                    ),
+                    detection_time_interval_s=self.detection_time_interval_s(),
+                )
+                issues.append(
+                    Issue(
+                        dataset_name=self._dataset_id,
+                        operator_id=op.id,
+                        issue_type=IssueType.HIGH_MEMORY,
+                        message=_format_message(message),
+                    )
+                )
+
+        return issues
+
+    def detection_time_interval_s(self) -> float:
+        return self._detector_cfg.detection_time_interval_s
+
+
+def _format_message(message: str) -> str:
+    # Apply some formatting to make the message look nicer when printed.
+    formatted_paragraphs = []
+    for paragraph in message.split("\n\n"):
+        formatted_paragraph = textwrap.fill(paragraph, break_long_words=False).strip()
+        formatted_paragraphs.append(formatted_paragraph)
+    formatted_message = "\n\n".join(formatted_paragraphs)
+    return "\n\n" + formatted_message + "\n"

python/ray/data/_internal/issue_detection/issue_detector.py

@@ -0,0 +1,44 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import TYPE_CHECKING, List
+
+if TYPE_CHECKING:
+    from ray.data._internal.execution.streaming_executor import StreamingExecutor
+
+
+class IssueType(str, Enum):
+    HANGING = "hanging"
+    HIGH_MEMORY = "high memory"
+
+
+@dataclass
+class Issue:
+    dataset_name: str
+    operator_id: str
+    message: str
+    issue_type: IssueType
+
+
+class IssueDetector(ABC):
+    @classmethod
+    @abstractmethod
+    def from_executor(cls, executor: "StreamingExecutor") -> "IssueDetector":
+        """Factory method to create an issue detector from a StreamingExecutor.
+
+        Args:
+            executor: The StreamingExecutor instance to extract dependencies from.
+
+        Returns:
+            An instance of the issue detector.
+        """
+        pass
+
+    @abstractmethod
+    def detect(self) -> List[Issue]:
+        pass
+
+    @abstractmethod
+    def detection_time_interval_s(self) -> float:
+        """Time interval between detections, or -1 if not enabled."""
+        pass

python/ray/data/_internal/issue_detection/issue_detector_configuration.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass, field
+from typing import List, Type
+
+from ray.data._internal.issue_detection.detectors import (
+    HangingExecutionIssueDetectorConfig,
+    HashShuffleAggregatorIssueDetector,
+    HashShuffleAggregatorIssueDetectorConfig,
+    HighMemoryIssueDetector,
+    HighMemoryIssueDetectorConfig,
+)
+from ray.data._internal.issue_detection.issue_detector import IssueDetector
+
+
+@dataclass
+class IssueDetectorsConfiguration:
+    hanging_detector_config: HangingExecutionIssueDetectorConfig = field(
+        default_factory=HangingExecutionIssueDetectorConfig
+    )
+    hash_shuffle_detector_config: HashShuffleAggregatorIssueDetectorConfig = field(
+        default_factory=HashShuffleAggregatorIssueDetectorConfig
+    )
+    high_memory_detector_config: HighMemoryIssueDetectorConfig = field(
+        default_factory=HighMemoryIssueDetectorConfig
+    )
+    detectors: List[Type[IssueDetector]] = field(
+        default_factory=lambda: [
+            # TODO(Justin): Enable once it's non-blocking
+            # HangingExecutionIssueDetector,
+            HashShuffleAggregatorIssueDetector,
+            HighMemoryIssueDetector,
+        ]
+    )

python/ray/data/_internal/issue_detection/issue_detector_manager.py

@@ -0,0 +1,112 @@
+import logging
+import threading
+import time
+from typing import TYPE_CHECKING, Dict, List, Set, Tuple
+
+from ray.core.generated.export_dataset_operator_event_pb2 import (
+    ExportDatasetOperatorEventData as ProtoOperatorEventData,
+)
+from ray.data._internal.issue_detection.issue_detector import (
+    Issue,
+    IssueDetector,
+    IssueType,
+)
+from ray.data._internal.operator_event_exporter import (
+    OperatorEvent,
+    format_export_issue_event_name,
+    get_operator_event_exporter,
+)
+
+if TYPE_CHECKING:
+    from ray.data._internal.execution.interfaces.physical_operator import (
+        PhysicalOperator,
+    )
+    from ray.data._internal.execution.streaming_executor import StreamingExecutor
+
+logger = logging.getLogger(__name__)
+
+
+class IssueDetectorManager:
+    def __init__(self, executor: "StreamingExecutor"):
+        ctx = executor._data_context
+        self._issue_detectors: List[IssueDetector] = [
+            cls.from_executor(executor) for cls in ctx.issue_detectors_config.detectors
+        ]
+        self._last_detection_times: Dict[IssueDetector, float] = {
+            detector: time.perf_counter() for detector in self._issue_detectors
+        }
+        self.executor = executor
+        self._operator_event_exporter = get_operator_event_exporter()
+        # Set of detected (issue_type, operator) pairs for usage collection.
+        self._detected_issues: Set[Tuple[IssueType, "PhysicalOperator"]] = set()
+        # We protect the above set with a lock to avoid race conditions between the executor thread, that invokes the detectors (adding to the set of detected issues), and the
+        # consumer thread that checks the set of detected issues on shutdown (in the usage callback).
+        self._detected_issues_lock = threading.Lock()
+
+    def invoke_detectors(self) -> None:
+        curr_time = time.perf_counter()
+        issues = []
+        for detector in self._issue_detectors:
+            if detector.detection_time_interval_s() == -1:
+                continue
+
+            if (
+                curr_time - self._last_detection_times[detector]
+                > detector.detection_time_interval_s()
+            ):
+                issues.extend(detector.detect())
+
+                self._last_detection_times[detector] = time.perf_counter()
+
+        self._report_issues(issues)
+
+    def _report_issues(self, issues: List[Issue]) -> None:
+        operators: Dict[str, "PhysicalOperator"] = {}
+        op_to_id: Dict["PhysicalOperator", str] = {}
+        for i, operator in enumerate(self.executor._topology.keys()):
+            operators[operator.id] = operator
+            op_to_id[operator] = self.executor._get_operator_id(operator, i)
+            # Reset issue detector metrics for each operator so that previous issues
+            # don't affect the current ones.
+            operator.metrics._issue_detector_hanging = 0
+            operator.metrics._issue_detector_high_memory = 0
+
+        for issue in issues:
+            logger.warning(issue.message)
+            operator = operators.get(issue.operator_id)
+            if not operator:
+                continue
+
+            with self._detected_issues_lock:
+                self._detected_issues.add((issue.issue_type, operator))
+
+            issue_event_type = format_export_issue_event_name(issue.issue_type)
+            if (
+                self._operator_event_exporter is not None
+                and issue_event_type
+                in ProtoOperatorEventData.DatasetOperatorEventType.keys()
+            ):
+                event_time = time.time()
+                operator_event = OperatorEvent(
+                    dataset_id=issue.dataset_name,
+                    operator_id=op_to_id[operator],
+                    operator_name=operator.name,
+                    event_time=event_time,
+                    event_type=issue_event_type,
+                    message=issue.message,
+                )
+                self._operator_event_exporter.export_operator_event(operator_event)
+
+            if issue.issue_type == IssueType.HANGING:
+                operator.metrics._issue_detector_hanging += 1
+            if issue.issue_type == IssueType.HIGH_MEMORY:
+                operator.metrics._issue_detector_high_memory += 1
+        if len(issues) > 0:
+            logger.warning(
+                f"Found {len(issues)} issues. To disable issue detection, run DataContext.get_current().issue_detectors_config.detectors = []."
+            )
+
+    def get_detected_issues(self) -> Set[Tuple[IssueType, "PhysicalOperator"]]:
+        """Return a copy of the detected (issue_type, operator) pairs."""
+        with self._detected_issues_lock:
+            return set(self._detected_issues)