chore: import upstream snapshot with attribution

python/ray/data/_internal/datasource_v2/__init__.py

@@ -0,0 +1,3 @@
+from typing import TypeVar
+
+InputSplit = TypeVar("InputSplit")

python/ray/data/_internal/datasource_v2/chunkers/file_chunker.py

@@ -0,0 +1,286 @@
+"""File chunkers for DataSourceV2.
+
+A ``FileChunker`` decides how a single listed file is split into one or
+more parallel-read units. The indexer drives the chunker once per file
+and emits one manifest row per chunk; downstream the partitioner /
+reader carry the per-chunk metadata through to the read task.
+"""
+
+import abc
+import logging
+import math
+from typing import (
+    TYPE_CHECKING,
+    Iterable,
+    Optional,
+    Tuple,
+    Type,
+    TypedDict,
+    TypeVar,
+    cast,
+    get_type_hints,
+)
+
+from ray.data._internal.util import MiB, infer_compression
+from ray.util.annotations import DeveloperAPI
+
+if TYPE_CHECKING:
+    from pyarrow.fs import FileSystem
+
+logger = logging.getLogger(__name__)
+
+
+class ChunkMetadata(TypedDict):
+    """Base interface for chunk metadata types."""
+
+    pass
+
+
+_ChunkMetadataT = TypeVar("_ChunkMetadataT", bound=ChunkMetadata)
+
+
+def create_chunk_metadata(cls: Type[_ChunkMetadataT], **kwargs) -> _ChunkMetadataT:
+    """Create a metadata instance with validation, ensure the keys are correct."""
+    required_keys = list(get_type_hints(cls).keys())
+
+    missing_keys = [key for key in required_keys if key not in kwargs]
+    if missing_keys:
+        raise ValueError(f"Missing required keys: {missing_keys}")
+
+    extra_keys = [key for key in kwargs if key not in required_keys]
+    if extra_keys:
+        raise ValueError(f"Unexpected keys: {extra_keys}")
+
+    return cast(_ChunkMetadataT, kwargs)
+
+
+class LineDelimitedFileChunkMetadata(ChunkMetadata):
+    """Metadata for line-delimited file chunks."""
+
+    chunk_byte_start_idx: int
+    chunk_byte_end_idx: int
+
+
+class ParquetFileChunkMetadata(ChunkMetadata):
+    """Metadata for Parquet file chunks.
+
+    A chunk is an explicit, half-open range of consecutive row groups
+    ``[row_group_start, row_group_end)`` within a single file, computed at
+    listing time from the file's footer. The reader slices the fragment to
+    exactly this range — no estimation or read-time reconciliation.
+    """
+
+    row_group_start: int  # inclusive
+    row_group_end: int  # exclusive
+
+
+@DeveloperAPI
+class FileChunker(abc.ABC):
+    """Abstract base class for chunking files into smaller pieces for parallel processing.
+
+    File chunkers determine how large files should be split into chunks that can be
+    processed in parallel. Different file formats may require different chunking strategies.
+
+    For example:
+    - Line-delimited files (JSONL, CSV) can be chunked by byte ranges
+    - Parquet files can be chunked by row groups
+    """
+
+    # Whether ``generate_chunk_metadatas`` performs file I/O (e.g. reading a
+    # Parquet footer). When True, the indexer fans chunking across its thread
+    # pool so the per-file reads parallelize even for a single input
+    # directory. When False, the indexer chunks inline (no thread hand-off).
+    reads_file_metadata: bool = False
+
+    @abc.abstractmethod
+    def generate_chunk_metadatas(
+        self,
+        path: str,
+        file_size: int,
+        filesystem: Optional["FileSystem"] = None,
+    ) -> Iterable[Tuple[Optional[ChunkMetadata], int]]:
+        """Generate metadata for file chunks.
+
+        Args:
+            path: The file path being chunked.
+            file_size: The total size in bytes of the file to be chunked.
+            filesystem: PyArrow filesystem used to read per-file metadata
+                (e.g. the Parquet footer). Ignored by chunkers that do not
+                read file metadata.
+
+        Returns:
+            An iterable of tuples containing (metadata, chunk_size) where metadata
+            describes the chunk and chunk_size is the size of the chunk in bytes.
+            Metadata can be None for chunks that don't require metadata
+            (e.g., whole file processing).
+        """
+        ...
+
+
+@DeveloperAPI
+class WholeFileChunker(FileChunker):
+    """File chunker that treats the whole file as a single chunk.
+
+    This chunker is used when files should be processed as a single unit,
+    typically for smaller files or when the file format doesn't support
+    efficient chunking (e.g., compressed files).
+
+    Yields a single chunk with no metadata, indicating the entire file
+    should be processed as one unit.
+    """
+
+    def generate_chunk_metadatas(
+        self,
+        path: str,
+        file_size: int,
+        filesystem: Optional["FileSystem"] = None,
+    ) -> Iterable[Tuple[Optional[ChunkMetadata], int]]:
+        yield None, file_size
+
+
+@DeveloperAPI
+class LineDelimitedFileChunker(FileChunker):
+    """File chunker for line-delimited files (JSONL, CSV, TSV, etc.).
+
+    This chunker splits files into fixed-size byte chunks (default: 256 MiB)
+    and provides metadata about the byte ranges for each chunk. The actual
+    line boundaries are handled by the reader to ensure complete records.
+    """
+
+    _CHUNK_BYTE_SIZE = 256 * MiB  # 256 MiB
+
+    def generate_chunk_metadatas(
+        self,
+        path: str,
+        file_size: int,
+        filesystem: Optional["FileSystem"] = None,
+    ) -> Iterable[Tuple[Optional[ChunkMetadata], int]]:
+        compression = infer_compression(path)
+        if compression is not None:
+            yield None, file_size
+        else:
+            num_chunks = math.ceil(file_size / self._CHUNK_BYTE_SIZE)
+            for chunk_idx in range(num_chunks):
+                chunk_start = self._CHUNK_BYTE_SIZE * chunk_idx
+                chunk_end = min(self._CHUNK_BYTE_SIZE * (chunk_idx + 1), file_size)
+                chunk_size = chunk_end - chunk_start
+                yield (
+                    create_chunk_metadata(
+                        LineDelimitedFileChunkMetadata,
+                        chunk_byte_start_idx=chunk_start,
+                        chunk_byte_end_idx=chunk_end,
+                    ),
+                    chunk_size,
+                )
+
+
+@DeveloperAPI
+class ParquetFileChunker(FileChunker):
+    """File chunker for Parquet files.
+
+    Reads each file's footer at listing time and chunks on **true row-group
+    boundaries**: consecutive row groups are bundled into a chunk until the
+    bundle's on-disk size reaches ``target_chunk_size`` (always at least one
+    row group per chunk). Each chunk carries an explicit half-open row-group
+    range, so the reader slices to exactly those row groups with no
+    estimation or read-time reconciliation, and the listing stage never
+    produces empty read tasks.
+
+    The row group is Parquet's atomic read unit, so a chunk can never be
+    smaller than a single row group. With the default target (which falls
+    back to ``DataContext.target_min_block_size``), a file's row groups map
+    1:1 to chunks unless they are smaller than the target, in which case
+    consecutive small row groups are bundled to avoid an excessive number of
+    tiny chunks.
+    """
+
+    # Hard fallback used only when neither an explicit target nor the
+    # DataContext size knobs are set.
+    _FALLBACK_TARGET_CHUNK_SIZE = 1 * MiB
+
+    # Footer reads are file I/O — let the indexer parallelize them.
+    reads_file_metadata: bool = True
+
+    def __init__(self, target_chunk_size: Optional[int] = None):
+        from ray.data.context import DataContext
+
+        ctx = DataContext.get_current()
+        # Resolve with explicit ``is not None`` checks rather than ``or`` so an
+        # explicit ``0`` (e.g. to force one row group per chunk) isn't treated as
+        # "unset" and silently overridden by a falsy-coalescing fallback.
+        if target_chunk_size is not None:
+            self._target_chunk_size = target_chunk_size
+        elif ctx.parquet_chunker_target_chunk_size is not None:
+            self._target_chunk_size = ctx.parquet_chunker_target_chunk_size
+        elif ctx.target_min_block_size is not None:
+            self._target_chunk_size = ctx.target_min_block_size
+        else:
+            self._target_chunk_size = self._FALLBACK_TARGET_CHUNK_SIZE
+
+    def generate_chunk_metadatas(
+        self,
+        path: str,
+        file_size: int,
+        filesystem: Optional["FileSystem"] = None,
+    ) -> Iterable[Tuple[Optional[ChunkMetadata], int]]:
+        import pyarrow.parquet as pq
+
+        try:
+            # Reads only the Parquet footer (file metadata), not data.
+            metadata = pq.read_metadata(path, filesystem=filesystem)
+        except Exception as e:
+            # Corrupt / unreadable footer (or a non-Parquet file that slipped
+            # through). Fall back to a single whole-file chunk so the file is
+            # still read rather than dropped.
+            logger.debug(
+                "Could not read Parquet footer for chunking (%s): %s; "
+                "falling back to a whole-file chunk.",
+                path,
+                e,
+            )
+            yield None, file_size
+            return
+
+        num_row_groups = metadata.num_row_groups
+        if num_row_groups == 0:
+            yield None, file_size
+            return
+
+        # Greedily bundle consecutive row groups until the running on-disk
+        # size reaches the target. Always emit at least one row group per
+        # chunk (the atomic read unit).
+        start = 0
+        running_size = 0
+        for rg_idx in range(num_row_groups):
+            rg_meta = metadata.row_group(rg_idx)
+            # On-disk (compressed) row-group size. ``RowGroupMetaData`` exposes
+            # only the *uncompressed* ``total_byte_size``; the on-disk size lives
+            # on each ``ColumnChunkMetaData``, so sum the per-column compressed
+            # sizes. Keeping chunk sizes in on-disk units matches the manifest's
+            # ``file_sizes`` and the ``×encoding_ratio`` in-memory estimator.
+            rg_size = sum(
+                rg_meta.column(c).total_compressed_size
+                for c in range(rg_meta.num_columns)
+            )
+            if running_size > 0 and running_size + rg_size > self._target_chunk_size:
+                yield (
+                    create_chunk_metadata(
+                        ParquetFileChunkMetadata,
+                        row_group_start=start,
+                        row_group_end=rg_idx,
+                    ),
+                    running_size,
+                )
+                start = rg_idx
+                running_size = 0
+            running_size += rg_size
+
+        # Flush the final bundle.
+        yield (
+            create_chunk_metadata(
+                ParquetFileChunkMetadata,
+                row_group_start=start,
+                row_group_end=num_row_groups,
+            ),
+            running_size,
+        )

python/ray/data/_internal/datasource_v2/chunkers/parquet_file_chunking_utils.py

@@ -0,0 +1,49 @@
+"""Parquet file-level chunking helpers for DataSourceV2.
+
+Maps planner chunk metadata (``ParquetFileChunkMetadata``) to PyArrow
+``ParquetFileFragment`` subsets for parallel reads. Chunk metadata carries
+an explicit half-open row-group range computed at listing time from the
+file's footer, so no estimation or reconciliation is needed here.
+"""
+from typing import List, Tuple
+
+import pyarrow.dataset as pds
+
+from ray.data._internal.datasource_v2.chunkers.file_chunker import (
+    ParquetFileChunkMetadata,
+)
+
+
+def _fragments_from_chunk_metadata(
+    fragment: pds.ParquetFileFragment,
+    chunk_metadata: ParquetFileChunkMetadata,
+) -> List[Tuple[pds.ParquetFileFragment, int]]:
+    """Slice ``fragment`` into per-row-group sub-fragments for the chunk's range.
+
+    The chunk carries an explicit ``[row_group_start, row_group_end)`` range.
+    Returns one ``(ParquetFileFragment, file_row_offset)`` pair per row group
+    in that range, where ``file_row_offset`` is the sum of ``num_rows`` across
+    all row groups that precede the sub-fragment in the underlying file.
+    Callers seed per-fragment hashing offsets with this value so sub-fragments
+    of the same file don't collide on ``(path, 0, n)``.
+
+    The range is defensively clamped to the file's actual row-group count;
+    since ranges are computed from the same footer the reader sees, the clamp
+    is a no-op in practice and never drops real row groups.
+    """
+    start = chunk_metadata["row_group_start"]
+    end = chunk_metadata["row_group_end"]
+    metadata = fragment.metadata
+    total_row_groups = metadata.num_row_groups
+
+    start = min(start, total_row_groups)
+    end = min(end, total_row_groups)
+
+    file_row_offset = sum(metadata.row_group(i).num_rows for i in range(start))
+    sub_fragments: List[Tuple[pds.ParquetFileFragment, int]] = []
+    for row_group_index in range(start, end):
+        sub_fragments.append(
+            (fragment.subset(row_group_ids=[row_group_index]), file_row_offset)
+        )
+        file_row_offset += metadata.row_group(row_group_index).num_rows
+    return sub_fragments

python/ray/data/_internal/datasource_v2/datasource_v2.py

@@ -0,0 +1,191 @@
+"""
+DataSourceV2 API - Unified Abstraction for Reading Data Sources
+
+This module defines a unified, extensible API for reading data from diverse sources
+in Ray Data. The API provides a common abstraction layer that enables datasources to
+declaratively expose their capabilities—such as filter pushdown, projection pruning,
+and parallel reads—while allowing the execution engine to leverage these capabilities
+transparently.
+
+Core Principles:
+- Modularity: Separate concerns (indexing, scanning, reading)
+- Expressivity: Declarative capability exposure via mixins
+- Extensibility: Easy to add new datasources with custom optimizations
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Generic,
+    Optional,
+)
+
+import pyarrow as pa
+
+from ray.data._internal.datasource_v2 import InputSplit
+from ray.data._internal.datasource_v2.listing.file_indexer import FileIndexer
+from ray.util.annotations import DeveloperAPI
+
+if TYPE_CHECKING:
+    from pyarrow.fs import FileSystem
+
+    from ray.data._internal.datasource_v2.readers.in_memory_size_estimator import (
+        InMemorySizeEstimator,
+    )
+    from ray.data._internal.datasource_v2.scanners.scanner import Scanner
+
+
+@DeveloperAPI
+class DatasourceCategory(Enum):
+    """Categories of datasources with different capability profiles.
+
+    Each category has a distinct set of applicable optimizations:
+    - FILE_BASED: Local/cloud files (parquet, csv, json, images)
+    - DATABASE: SQL databases (postgres, mysql, snowflake)
+    - DATA_LAKE: Table formats (iceberg, delta, hudi)
+    - IN_MEMORY: In-process data (pandas, numpy, arrow)
+    - SYNTHETIC: Generated data (range, range_tensor)
+    - STREAMING: Unbounded sources (kafka, kinesis)
+    """
+
+    FILE_BASED = "file_based"
+    DATABASE = "database"
+    DATA_LAKE = "data_lake"
+    IN_MEMORY = "in_memory"
+    SYNTHETIC = "synthetic"
+    STREAMING = "streaming"
+
+
+@DeveloperAPI
+class DataSourceV2(ABC, Generic[InputSplit]):
+    """Abstract base class for V2 datasources.
+
+    DataSourceV2 is the entry point for reading data from a source. It provides:
+    1. File listing (for file-based sources) - via _get_file_indexer()
+    2. Schema inference
+    3. Size estimation
+    4. Scanner creation
+
+    Subclasses should implement the abstract methods and can optionally
+    override _get_file_indexer() and get_size_estimator() for file-based sources.
+
+    Example::
+
+        datasource = ParquetDatasourceV2()
+        indexer = datasource._get_file_indexer()
+        # List files with optional sampling
+        for manifest in indexer.list_files(paths, filesystem=fs):
+            schema = datasource.infer_schema(manifest)
+            break  # Just need first manifest for schema
+        scanner = datasource.create_scanner(schema)
+        scanner = scanner.prune_columns(["col1", "col2"])
+        reader = scanner.create_reader()
+        for table in reader.read(manifest):
+            process(table)
+    """
+
+    def __init__(self, name: str, category: DatasourceCategory):
+        """Initialize the datasource.
+
+        Args:
+            name: Human-readable name for this datasource.
+            category: Category of this datasource.
+        """
+        self._name = name
+        self._category = category
+        # File-based subclasses set this to ``False`` in their ``__init__``
+        # when the user-supplied paths are in the ``local://`` scheme —
+        # the driver node is the only one that can read those files.
+        # ``_read_datasource_v2`` consults the flag to decide whether to
+        # pin read tasks via a ``label_selector``.
+        self._supports_distributed_reads: bool = True
+
+    @property
+    def name(self) -> str:
+        """Human-readable name for this datasource."""
+        return self._name
+
+    @property
+    def category(self) -> DatasourceCategory:
+        """Category of this datasource."""
+        return self._category
+
+    @property
+    def supports_distributed_reads(self) -> bool:
+        """Whether read tasks may run on any cluster node.
+
+        Defaults to ``True``. File-based subclasses (e.g.
+        :class:`ParquetDatasourceV2`) flip this to ``False`` when the
+        user supplies ``local://``-scheme paths so ``_read_datasource_v2``
+        can pin reads to the driver node via a ``ray.io/node-id``
+        label selector. Mirrors V1 ``Datasource.supports_distributed_reads``.
+        """
+        return self._supports_distributed_reads
+
+    def _get_file_indexer(self) -> Optional[FileIndexer]:
+        """Return FileIndexer component if applicable.
+
+        Override this for file-based datasources to provide file discovery.
+
+        Returns:
+            FileIndexer instance, or None for non-file-based sources.
+        """
+        return None
+
+    def get_size_estimator(self) -> Optional[InMemorySizeEstimator]:
+        """Return size estimator for this datasource.
+
+        Override this to provide format-specific size estimation.
+
+        Returns:
+            InMemorySizeEstimator instance, or None if not supported.
+        """
+        return None
+
+    @abstractmethod
+    def infer_schema(self, sample: InputSplit) -> pa.Schema:
+        """Infer schema from a sample of data.
+
+        Args:
+            sample: Sample data to infer schema from.
+
+        Returns:
+            PyArrow Schema inferred from the sample.
+
+        Raises:
+            ValueError: If schema cannot be inferred from the sample.
+        """
+        ...
+
+    @abstractmethod
+    def create_scanner(
+        self,
+        schema: pa.Schema,
+        filesystem: Optional["FileSystem"] = None,
+        **options: Any,
+    ) -> Scanner[InputSplit]:
+        """Create a Scanner for reading data.
+
+        Args:
+            schema: Schema for the data to read.
+            filesystem: Optional filesystem for file-based sources.
+            **options: Additional datasource-specific options.
+
+        Returns:
+            Configured Scanner instance.
+        """
+        ...
+
+    def resolve_partitioning(self, sample: InputSplit) -> Optional[Any]:
+        """Return a partitioning descriptor derived from ``sample``, or ``None``.
+
+        Override this for file-based sources whose partition keys must be
+        discovered from a sample path (e.g. hive layouts where field names
+        are not known up front). The resolved descriptor is passed into
+        :meth:`create_scanner`.
+        """
+        return None

python/ray/data/_internal/datasource_v2/listing/file_indexer.py

@@ -0,0 +1,376 @@
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Callable, Iterable, Iterator, List, Optional, Tuple, Union
+
+from pyarrow.fs import FileSystem
+
+from ray._common.utils import env_integer
+from ray.data._internal.datasource_v2.chunkers.file_chunker import (
+    ChunkMetadata,
+    FileChunker,
+    WholeFileChunker,
+)
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.listing.file_pruners import FilePruner
+from ray.data._internal.datasource_v2.listing.indexing_utils import (
+    _get_file_infos,
+    _get_path_contents,
+)
+from ray.data._internal.dynamic_work_queue import parallel_process_work_stealing
+from ray.data._internal.util import make_async_gen
+from ray.data.block import BlockColumn
+from ray.data.datasource.path_util import _resolve_paths_and_filesystem
+
+logger = logging.getLogger(__name__)
+
+
+class FileIndexer(ABC):
+    @property
+    @abstractmethod
+    def file_chunker(self) -> FileChunker:
+        """The file chunker that this indexer uses."""
+        ...
+
+    @abstractmethod
+    def list_files(
+        self,
+        paths: "BlockColumn",
+        *,
+        filesystem: "FileSystem",
+        pruners: Optional[List[FilePruner]] = None,
+        preserve_order: bool = False,
+    ) -> Iterable[FileManifest]:
+        """List files and their on-disk sizes for the given path.
+
+        Args:
+            paths: A column of paths pointing to files or directories.
+            filesystem: A PyArrow filesystem object.
+            pruners: A list of file pruners to apply.
+            preserve_order: Whether to preserve order in file listing.
+
+        Returns:
+            An iterator of `FileManifest` objects, each of which contains a file path
+            and the on-disk size of the file in bytes.
+        """
+        ...
+
+
+@dataclass(frozen=True)
+class FileInfo:
+    """File information for file listing."""
+
+    path: str
+    size: Optional[int]
+
+
+@dataclass(frozen=True)
+class _TraversalWorkItem:
+    """Work item for parallel directory traversal. Distinguishes seed paths
+    (user-provided, need resolution) from subdir paths (from filesystem listing,
+    use directly to avoid redundant resolution that breaks non-local
+    filesystems)."""
+
+    # Could be a file path or a directory path.
+    path: str
+    # True for subdirectories discovered during traversal; False for seed input paths.
+    is_discovered_subdir: bool = False
+    # Original seed-path index used to restore deterministic ordering when requested.
+    input_path_index: Optional[int] = None
+    # Top-level path the traversal started from, used to scope hidden-prefix
+    # exclusion to entries whose path relative to the root is hidden.
+    root_path: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class OrderedFileResult:
+    """File result with order information for sorting when preserve_order is True."""
+
+    input_path_index: int
+    # The leaf file path.
+    file_path: str
+    file_info: FileInfo
+
+
+class NonSamplingFileIndexer(FileIndexer):
+    """A file indexer that exhaustively lists files.
+
+    This implementation works with paths that point to files or directories,
+    although it's slow if you try to list lots of paths pointing to files
+    rather than a single directory.
+    """
+
+    _DEFAULT_MAX_PATHS_PER_OUTPUT = env_integer(
+        "RAY_DATA_MAX_PATHS_PER_LIST_FILES_OUTPUT", 1000
+    )
+
+    _DEFAULT_NUM_WORKERS = env_integer("RAY_DATA_LIST_FILES_THREADED_NUM_WORKERS", 8)
+
+    def __init__(
+        self,
+        *,
+        ignore_missing_paths: bool,
+        num_workers: Optional[int] = None,
+        max_paths_per_output: Optional[int] = None,
+        file_chunker: Optional[FileChunker] = None,
+    ):
+        self._ignore_missing_paths = ignore_missing_paths
+        self._max_paths_per_output = (
+            max_paths_per_output
+            if max_paths_per_output is not None
+            else self._DEFAULT_MAX_PATHS_PER_OUTPUT
+        )
+        self._num_workers = (
+            num_workers if num_workers is not None else self._DEFAULT_NUM_WORKERS
+        )
+        self._queue_size_per_thread = env_integer(
+            "RAY_DATA_LIST_FILES_QUEUE_SIZE_PER_THREAD",
+            self._max_paths_per_output * 4,
+        )
+        self._file_chunker: FileChunker = (
+            file_chunker if file_chunker is not None else WholeFileChunker()
+        )
+
+    @property
+    def file_chunker(self) -> FileChunker:
+        """The file chunker that this indexer uses.
+
+        Exposed primarily for tests and shuffle-aware planning code that needs
+        to introspect or override the chunking strategy.
+        """
+        return self._file_chunker
+
+    def list_files(
+        self,
+        paths: "BlockColumn",
+        *,
+        filesystem: "FileSystem",
+        pruners: Optional[List[FilePruner]] = None,
+        preserve_order: bool = False,
+    ) -> Iterable[FileManifest]:
+        file_info_iterator = (
+            self._get_file_info_iterator_threaded(paths, filesystem, preserve_order)
+            if self._num_workers > 1
+            else self._get_file_info_iterator_sequential(paths, filesystem)
+        )
+
+        # Stage pipeline: list → prune (cheap, inline) → chunk (may read
+        # per-file metadata) → batch into manifests. Pruning runs *before*
+        # chunking so we never read a footer for a file we'd discard.
+        pruned = self._filter_file_infos(file_info_iterator, pruners or [])
+        chunk_records = self._generate_chunk_records(pruned, filesystem, preserve_order)
+        yield from self._batch_chunk_records_to_manifests(chunk_records)
+
+    def _get_file_info_iterator_sequential(
+        self,
+        paths: "BlockColumn",
+        filesystem: "FileSystem",
+    ) -> Iterable[FileInfo]:
+        for input_path in paths.to_pylist():
+            resolved_paths, _ = _resolve_paths_and_filesystem(input_path, filesystem)
+            assert len(resolved_paths) == 1
+
+            for path, file_size in _get_file_infos(
+                resolved_paths[0], filesystem, self._ignore_missing_paths
+            ):
+                yield FileInfo(path=path, size=file_size)
+
+    def _get_file_info_iterator_threaded(
+        self,
+        paths: "BlockColumn",
+        filesystem: "FileSystem",
+        preserve_order: bool = False,
+    ) -> Iterable[FileInfo]:
+        """Threaded file info iterator with work stealing for parallel directory
+        traversal. Subdirectories are added as work items for idle workers to
+        process."""
+        paths_list = paths.to_pylist()
+        if len(paths_list) == 0:
+            return
+
+        num_workers = self._num_workers
+
+        seed_items = [
+            _TraversalWorkItem(
+                path=p,
+                is_discovered_subdir=False,
+                input_path_index=i if preserve_order else None,
+            )
+            for i, p in enumerate(paths_list)
+        ]
+
+        def process_fn(
+            item: _TraversalWorkItem,
+            add_work: Callable[[_TraversalWorkItem], None],
+            add_result: Callable[[Union[OrderedFileResult, FileInfo]], None],
+        ) -> None:
+            """Process a single item, adding discovered subdirs as work and
+            files as results."""
+            input_path_index = item.input_path_index
+
+            if item.is_discovered_subdir:
+                # Subdir paths from filesystem listing: use directly. Re-resolution
+                # would infer LocalFileSystem for scheme-less paths on S3/GCS,
+                # and add redundant overhead.
+                path = item.path
+                root_path = item.root_path
+            else:
+                # Seed paths from user: resolve once to get normalized path + fs.
+                resolved_paths, _ = _resolve_paths_and_filesystem(item.path, filesystem)
+                assert len(resolved_paths) == 1
+                path = resolved_paths[0]
+                root_path = path
+
+            contents = _get_path_contents(
+                path, filesystem, self._ignore_missing_paths, root_path=root_path
+            )
+            for file_path, file_size in contents.files:
+                file_info_result = FileInfo(path=file_path, size=file_size)
+                if preserve_order:
+                    add_result(
+                        OrderedFileResult(
+                            input_path_index=input_path_index,
+                            file_path=file_path,
+                            file_info=file_info_result,
+                        )
+                    )
+                else:
+                    add_result(file_info_result)
+            for subdir_path in contents.subdirs:
+                add_work(
+                    _TraversalWorkItem(
+                        path=subdir_path,
+                        is_discovered_subdir=True,
+                        input_path_index=input_path_index,
+                        root_path=root_path,
+                    )
+                )
+
+        def _ordered_result_key(result: OrderedFileResult) -> Tuple[int, str]:
+            return (result.input_path_index, result.file_path)
+
+        if preserve_order:
+            for result in parallel_process_work_stealing(
+                seed_items=seed_items,
+                process_fn=process_fn,
+                num_workers=num_workers,
+                preserve_order=True,
+                order_key=_ordered_result_key,
+            ):
+                # Ordered mode returns `OrderedFileResult` for sorting, so unwrap.
+                yield result.file_info
+        else:
+            yield from parallel_process_work_stealing(
+                seed_items=seed_items,
+                process_fn=process_fn,
+                num_workers=num_workers,
+            )
+
+    def _filter_file_infos(
+        self,
+        file_infos: Iterable[FileInfo],
+        pruners: List[FilePruner],
+    ) -> Iterator[FileInfo]:
+        """Drop zero-size and pruned files before any per-file metadata read."""
+        for file_info in file_infos:
+            if file_info.size is None or file_info.size == 0:
+                logger.warning(f"Skipping zero-size file: {file_info.path!r}")
+                continue
+            if not all(pruner.should_include(file_info.path) for pruner in pruners):
+                continue
+            yield file_info
+
+    def _generate_chunk_records(
+        self,
+        file_infos: Iterable[FileInfo],
+        filesystem: "FileSystem",
+        preserve_order: bool,
+    ) -> Iterator[Tuple[str, int, Optional[ChunkMetadata]]]:
+        """Drive the chunker per file, yielding ``(path, chunk_size, metadata)``.
+
+        When the chunker reads per-file metadata (e.g. ``ParquetFileChunker``
+        reading footers), fan the work across the indexer's thread pool so the
+        I/O parallelizes even for a single input directory — ``make_async_gen``
+        over the *discovered files*, not the input paths. Chunkers that don't
+        read metadata (whole-file / line-delimited) are driven inline to avoid
+        a pointless thread hand-off.
+        """
+        chunker = self._file_chunker
+
+        def chunk_file(
+            fi: FileInfo,
+        ) -> List[Tuple[str, int, Optional[ChunkMetadata]]]:
+            return [
+                (fi.path, chunk_size, chunk_metadata)
+                for chunk_metadata, chunk_size in chunker.generate_chunk_metadatas(
+                    fi.path, fi.size, filesystem
+                )
+            ]
+
+        if chunker.reads_file_metadata and self._num_workers > 1:
+            # Fan per-file footer reads across the thread pool. ``make_async_gen``
+            # only preserves ordering for a 1:1 map (one output per input item), so
+            # emit ONE record list per file and flatten here. Yielding chunk rows
+            # individually would let its round-robin merge interleave chunks from
+            # the files processed concurrently -- breaking per-file contiguity and
+            # discovery order under ``preserve_order=True``.
+            def chunk_files(
+                infos: Iterator[FileInfo],
+            ) -> Iterator[List[Tuple[str, int, Optional[ChunkMetadata]]]]:
+                for fi in infos:
+                    yield chunk_file(fi)
+
+            for records in make_async_gen(
+                # ``iter(...)`` so a non-iterator iterable (e.g. a list from a
+                # test or subclass) is still consumed correctly by the helper.
+                base_iterator=iter(file_infos),
+                fn=chunk_files,
+                preserve_ordering=preserve_order,
+                num_workers=self._num_workers,
+                buffer_size=self._queue_size_per_thread,
+            ):
+                yield from records
+        else:
+            for fi in file_infos:
+                yield from chunk_file(fi)
+
+    def _batch_chunk_records_to_manifests(
+        self,
+        chunk_records: Iterable[Tuple[str, int, Optional[ChunkMetadata]]],
+    ) -> Iterable[FileManifest]:
+        """Batch chunk records into ``FileManifest`` blocks of bounded size."""
+        running_paths: List[str] = []
+        running_file_sizes: List[int] = []
+        running_chunk_metadatas: List[Optional[ChunkMetadata]] = []
+        manifests_count = 0
+        chunks_count = 0
+
+        for path, chunk_size, chunk_metadata in chunk_records:
+            running_paths.append(path)
+            running_file_sizes.append(chunk_size)
+            running_chunk_metadatas.append(chunk_metadata)
+            chunks_count += 1
+
+            if len(running_paths) >= self._max_paths_per_output:
+                manifests_count += 1
+                yield FileManifest.construct_manifest(
+                    running_paths,
+                    running_file_sizes,
+                    running_chunk_metadatas,
+                )
+                running_paths = []
+                running_file_sizes = []
+                running_chunk_metadatas = []
+
+        if running_paths:
+            manifests_count += 1
+            yield FileManifest.construct_manifest(
+                running_paths,
+                running_file_sizes,
+                running_chunk_metadatas,
+            )
+
+        logger.debug(
+            f"Listing files: constructed {manifests_count} manifests "
+            f"with {chunks_count} file chunks"
+        )

python/ray/data/_internal/datasource_v2/listing/file_manifest.py

@@ -0,0 +1,141 @@
+from functools import cached_property
+from typing import List, Optional
+
+import numpy as np
+import pyarrow as pa
+
+from ray.data._internal.datasource_v2.chunkers.file_chunker import ChunkMetadata
+from ray.data.block import Block, BlockAccessor, BlockColumnAccessor
+
+# File manifest column names
+PATH_COLUMN_NAME = "__path"
+FILE_SIZE_COLUMN_NAME = "__file_size"
+FILE_CHUNK_METADATA_COLUMN_NAME = "__file_chunk_metadata"
+
+
+class FileManifest:
+    """Structured view over file paths, sizes, and per-chunk metadata.
+
+    Provides structured access to file paths, sizes, and chunk metadata. This avoids
+    making implicit assumptions about block structure as data moves between file
+    listing, partitioning, and reading stages.
+
+    All extracted views (i.e., `paths`, `file_sizes`, `file_chunk_metadatas`) share
+    the same row order as the underlying block. Any transformation must preserve this.
+
+    Each row represents a single chunk of a file. For unchunked files (whole-file
+    reads), the chunk-metadata entry is ``None`` and ``file_sizes`` equals the
+    on-disk file size. For chunked files, multiple rows can share the same path
+    but carry different chunk metadata.
+    """
+
+    def __init__(self, block: Block):
+        """Create a new `FileManifest` from a block.
+
+        Args:
+            block: Block with `PATH_COLUMN_NAME`, `FILE_SIZE_COLUMN_NAME`, and
+                `FILE_CHUNK_METADATA_COLUMN_NAME` columns. Any other columns are
+                optional and treated as input data.
+        """
+        column_names = BlockAccessor.for_block(block).column_names()
+        assert FILE_SIZE_COLUMN_NAME in column_names
+        assert PATH_COLUMN_NAME in column_names
+        assert FILE_CHUNK_METADATA_COLUMN_NAME in column_names
+
+        self._block = block
+
+        self._paths = block[PATH_COLUMN_NAME]
+        self._file_sizes = block[FILE_SIZE_COLUMN_NAME]
+        self._file_chunk_metadatas = block[FILE_CHUNK_METADATA_COLUMN_NAME]
+
+    def __len__(self) -> int:
+        return len(self._block)
+
+    def __repr__(self):
+        return f"<{self.__class__.__name__} length={len(self._block)}>"
+
+    # TODO Use arrow arrays instead of numpy for these properties.
+    @cached_property
+    def paths(self) -> np.ndarray:
+        return BlockColumnAccessor.for_column(self._paths).to_numpy()
+
+    @cached_property
+    def file_sizes(self) -> np.ndarray:
+        return BlockColumnAccessor.for_column(self._file_sizes).to_numpy()
+
+    @cached_property
+    def file_chunk_metadatas(self) -> np.ndarray:
+        return BlockColumnAccessor.for_column(self._file_chunk_metadatas).to_numpy()
+
+    def as_block(self) -> Block:
+        """Return the underlying block for the `FileManifest`.
+
+        This doesn't make a copy of the underlying data.
+        """
+        return self._block
+
+    @classmethod
+    def concat(cls, manifests: List["FileManifest"]) -> "FileManifest":
+        """Return a new `FileManifest` whose rows are the concatenation of
+        ``manifests`` in order.
+
+        Row alignment of ``paths`` / ``file_sizes`` is preserved because
+        each input already satisfies it.
+        """
+        assert len(manifests) > 0, "concat requires at least one manifest"
+        if len(manifests) == 1:
+            return manifests[0]
+
+        merged = pa.concat_tables(
+            [
+                BlockAccessor.for_block(manifest._block).to_arrow()
+                for manifest in manifests
+            ]
+        )
+        return cls(merged)
+
+    def shuffle(self, seed: Optional[int]) -> "FileManifest":
+        """Return a new `FileManifest` with rows permuted.
+
+        Args:
+            seed: Random seed. ``None`` for non-deterministic shuffling.
+                When set, input rows are first sorted by path so the shuffle
+                is reproducible regardless of upstream listing order
+                (the threaded ``FileIndexer`` doesn't preserve order).
+
+        Returns:
+            A new `FileManifest` with the same rows in a shuffled order. The
+            underlying row alignment between `paths` and `file_sizes` is
+            preserved because the permutation is applied to the block as a
+            whole.
+        """
+        n = len(self)
+        if n <= 1:
+            return self
+        block = self._block
+        if seed is not None:
+            sort_indices = pa.compute.sort_indices(
+                BlockAccessor.for_block(block).to_arrow(),
+                sort_keys=[(PATH_COLUMN_NAME, "ascending")],
+            )
+            block = block.take(sort_indices)
+        permutation = np.random.default_rng(seed).permutation(n)
+        return FileManifest(block.take(permutation))
+
+    @classmethod
+    def construct_manifest(
+        cls,
+        paths: List[str],
+        sizes: List[int],
+        chunk_metadatas: List[Optional[ChunkMetadata]],
+    ) -> "FileManifest":
+        assert len(paths) == len(sizes) == len(chunk_metadatas)
+
+        block = pa.table(
+            {
+                PATH_COLUMN_NAME: paths,
+                FILE_SIZE_COLUMN_NAME: sizes,
+                FILE_CHUNK_METADATA_COLUMN_NAME: chunk_metadatas,
+            }
+        )
+        return cls(block)

python/ray/data/_internal/datasource_v2/listing/file_pruners.py

@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from ray.data.datasource import PathPartitionFilter
+from ray.data.datasource.path_util import _has_file_extension
+
+
+class FilePruner(ABC):
+    """Generic file-level filter applied during listing."""
+
+    @abstractmethod
+    def should_include(self, path: str) -> bool:
+        """Return True if this file should be included, False to skip it."""
+        ...
+
+
+class FileExtensionPruner(FilePruner):
+    """Skip files that don't match the expected extensions."""
+
+    def __init__(self, file_extensions: List[str]):
+        self._file_extensions = file_extensions
+
+    def should_include(self, path: str) -> bool:
+        return _has_file_extension(path, self._file_extensions)
+
+
+class PartitionPruner(FilePruner):
+    """Skip files based on partition column predicates (e.g., hive partitioning)."""
+
+    def __init__(self, partition_filter: PathPartitionFilter):
+        self._filter = partition_filter
+
+    def should_include(self, path: str) -> bool:
+        return self._filter.apply(path)

python/ray/data/_internal/datasource_v2/listing/indexing_utils.py

@@ -0,0 +1,123 @@
+import logging
+from dataclasses import dataclass
+from typing import Iterable, List, Optional, Tuple
+
+import pyarrow as pa
+from pyarrow.fs import FileSelector, FileType
+
+from ray.data.datasource.file_meta_provider import _handle_read_os_error
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class PathContents:
+    """Contents of a path: files (path, size) and subdirectories to expand."""
+
+    files: List[Tuple[str, Optional[int]]]
+    subdirs: List[str]
+
+
+def _expand_directory(
+    base_path: str,
+    filesystem: pa.fs.FileSystem,
+    ignore_missing_path: bool,
+    *,
+    root_path: Optional[str] = None,
+) -> PathContents:
+    """List one level of a directory.
+
+    Hidden-prefix (``.``/``_``) exclusion is applied relative to ``root_path``
+    (the top-level path the traversal started from), not the immediate parent,
+    so a nested entry is only excluded when its path *relative to the root*
+    begins with an excluded prefix. When ``root_path`` is ``None`` it defaults
+    to ``base_path``.
+    """
+    exclude_prefixes = [".", "_"]
+
+    if root_path is None:
+        root_path = base_path
+
+    selector = FileSelector(
+        base_path, recursive=False, allow_not_found=ignore_missing_path
+    )
+    children = filesystem.get_file_info(selector)
+
+    # Lineage reconstruction doesn't work if tasks aren't deterministic, and
+    # `filesystem.get_file_info` might return files in a non-deterministic order. So, we
+    # sort the files.
+    assert isinstance(children, list), type(children)
+    children.sort(key=lambda file_: file_.path)
+
+    files: List[Tuple[str, Optional[int]]] = []
+    subdirs: List[str] = []
+
+    for child in children:
+        if not child.path.startswith(root_path):
+            continue
+
+        relative = child.path[len(root_path) :].lstrip("/")
+        if any(relative.startswith(prefix) for prefix in exclude_prefixes):
+            continue
+
+        if child.type == FileType.File:
+            files.append((child.path, child.size))
+        elif child.type == FileType.Directory:
+            subdirs.append(child.path)
+        elif child.type == FileType.UNKNOWN:
+            logger.warning(f"Discovered file with unknown type: '{child.path}'")
+            continue
+        else:
+            assert child.type == FileType.NotFound
+            raise FileNotFoundError(child.path)
+
+    return PathContents(files=files, subdirs=subdirs)
+
+
+def _get_path_contents(
+    path: str,
+    filesystem: pa.fs.FileSystem,
+    ignore_missing_path: bool,
+    *,
+    root_path: Optional[str] = None,
+) -> PathContents:
+    """Get files and subdirs for a path. Handles File, Directory, and NotFound.
+
+    Only one level of a directory is expanded; discovered subdirectories are
+    returned in :attr:`PathContents.subdirs` for the caller to expand.
+    """
+    try:
+        file_info = filesystem.get_file_info(path)
+    except OSError as e:
+        _handle_read_os_error(e, path)
+
+    if file_info.type == FileType.File:
+        return PathContents(files=[(path, file_info.size)], subdirs=[])
+    elif file_info.type == FileType.Directory:
+        return _expand_directory(
+            path, filesystem, ignore_missing_path, root_path=root_path
+        )
+    elif file_info.type == FileType.NotFound and ignore_missing_path:
+        return PathContents(files=[], subdirs=[])
+    else:
+        raise FileNotFoundError(path)
+
+
+def _get_file_infos(
+    path: str,
+    filesystem: pa.fs.FileSystem,
+    ignore_missing_path: bool,
+    *,
+    _root_path: Optional[str] = None,
+) -> Iterable[Tuple[str, Optional[int]]]:
+    """Recursively expand a path (file or directory) into ``(path, size)`` tuples."""
+    if _root_path is None:
+        _root_path = path
+    contents = _get_path_contents(
+        path, filesystem, ignore_missing_path, root_path=_root_path
+    )
+    yield from contents.files
+    for subdir in contents.subdirs:
+        yield from _get_file_infos(
+            subdir, filesystem, ignore_missing_path, _root_path=_root_path
+        )

python/ray/data/_internal/datasource_v2/listing/listing_utils.py

@@ -0,0 +1,159 @@
+from typing import TYPE_CHECKING, Iterable, List, Optional
+
+import pyarrow as pa
+
+from ray.data._internal.datasource_v2.listing.file_manifest import (
+    PATH_COLUMN_NAME,
+    FileManifest,
+)
+from ray.data._internal.datasource_v2.listing.file_pruners import (
+    FileExtensionPruner,
+    FilePruner,
+    PartitionPruner,
+)
+from ray.data._internal.datasource_v2.partitioners.file_partitioner import (
+    FilePartitioner,
+)
+from ray.data._internal.delegating_block_builder import DelegatingBlockBuilder
+from ray.data._internal.execution.interfaces.task_context import TaskContext
+from ray.data.block import Block, BlockAccessor
+
+if TYPE_CHECKING:
+    from pyarrow.fs import FileSystem
+
+    from ray.data._internal.datasource_v2.listing.file_indexer import FileIndexer
+    from ray.data.datasource.file_based_datasource import FileShuffleConfig
+    from ray.data.datasource.partitioning import PathPartitionFilter
+
+
+def partition_files(
+    blocks: Iterable[Block],
+    _: TaskContext,
+    partitioner: FilePartitioner,
+) -> Iterable[Block]:
+    for block in blocks:
+        partitioner.add_input(FileManifest(block))
+        while partitioner.has_partition():
+            yield partitioner.next_partition().as_block()
+
+    partitioner.finalize()
+    while partitioner.has_partition():
+        yield partitioner.next_partition().as_block()
+
+
+def _build_pruners(
+    file_extensions: Optional[List[str]],
+    partition_filter: Optional["PathPartitionFilter"],
+) -> List[FilePruner]:
+    pruners: List[FilePruner] = []
+    if file_extensions is not None:
+        pruners.append(FileExtensionPruner(file_extensions))
+    if partition_filter is not None:
+        pruners.append(PartitionPruner(partition_filter))
+    return pruners
+
+
+def list_files_for_each_block(
+    blocks: Iterable[Block],
+    _: TaskContext,
+    *,
+    indexer: "FileIndexer",
+    filesystem: "FileSystem",
+    file_extensions: Optional[List[str]] = None,
+    partition_filter: Optional["PathPartitionFilter"] = None,
+    preserve_order: bool = False,
+) -> Iterable[Block]:
+    """Expand path blocks into ``FileManifest`` blocks.
+
+    Each input block carries a single ``__path`` column of path strings.
+    For every path, the indexer is invoked to produce a stream of
+    ``FileManifest`` objects; each manifest's backing block is yielded.
+    Pruners are constructed once per task from ``file_extensions`` /
+    ``partition_filter`` — keeps pruner construction out of the
+    ``_read_datasource_v2`` entry point.
+    """
+    pruners = _build_pruners(file_extensions, partition_filter)
+    for block in blocks:
+        for manifest in indexer.list_files(
+            block[PATH_COLUMN_NAME],
+            filesystem=filesystem,
+            pruners=pruners,
+            preserve_order=preserve_order,
+        ):
+            if len(manifest) > 0:
+                yield manifest.as_block()
+
+
+def shuffle_files(
+    blocks: Iterable[Block],
+    _: TaskContext,
+    *,
+    shuffle_config: "FileShuffleConfig",
+    execution_idx: int,
+) -> Iterable[Block]:
+    """Concatenate manifest blocks and shuffle rows with the seeded RNG.
+
+    Runs in a single task (`plan_list_files_op` sets `should_parallelize=False`
+    when shuffle is requested) so we have the full manifest before
+    shuffling. Emits one merged manifest block. Determinism comes from
+    ``FileManifest.shuffle`` which sorts by path before applying the
+    permutation — this protects against non-deterministic upstream
+    indexer yield order.
+    """
+    builder = DelegatingBlockBuilder()
+    for block in blocks:
+        if len(block) > 0:
+            builder.add_block(block)
+
+    combined = builder.build()
+    if len(combined) == 0:
+        return
+
+    seed = shuffle_config.get_seed(execution_idx)
+    yield FileManifest(combined).shuffle(seed).as_block()
+
+
+def sample_files(
+    indexer: "FileIndexer",
+    paths: List[str],
+    filesystem: "FileSystem",
+    pruners: Optional[List[FilePruner]] = None,
+    max_files: int = 16,
+) -> FileManifest:
+    """Drive the indexer until up to ``max_files`` files arrive; return them.
+
+    Used for driver-side schema inference in ``_read_datasource_v2``.
+    Sampling more than one file lets callers unify schemas (e.g., if the
+    first file has an all-null column, later files' non-null types can
+    promote it). No caching — the returned manifest is discarded after
+    schema inference, and the ``ListFiles`` op lists the same paths
+    again on workers at execution time.
+    """
+    assert max_files >= 1
+    paths_column = pa.array(paths, type=pa.string())
+    collected: List[FileManifest] = []
+    collected_rows = 0
+    for manifest in indexer.list_files(
+        paths_column,
+        filesystem=filesystem,
+        pruners=pruners or [],
+        preserve_order=True,
+    ):
+        if len(manifest) == 0:
+            continue
+        remaining = max_files - collected_rows
+        if len(manifest) <= remaining:
+            collected.append(manifest)
+            collected_rows += len(manifest)
+        else:
+            collected.append(
+                FileManifest(
+                    BlockAccessor.for_block(manifest.as_block()).slice(0, remaining)
+                )
+            )
+            collected_rows = max_files
+        if collected_rows >= max_files:
+            break
+    if not collected:
+        return FileManifest.construct_manifest(paths=[], sizes=[], chunk_metadatas=[])
+    return FileManifest.concat(collected)

python/ray/data/_internal/datasource_v2/logical_optimizers.py

@@ -0,0 +1,122 @@
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, List, Optional, Set, Tuple
+
+from ray.data.expressions import Expr
+from ray.util.annotations import DeveloperAPI
+
+if TYPE_CHECKING:
+    from ray.data._internal.datasource_v2.scanners.scanner import Scanner
+
+
+@DeveloperAPI
+class SupportsFilterPushdown(ABC):
+    """Mixin for scanners that support filter/predicate pushdown.
+
+    Filter pushdown allows predicates to be evaluated at the data source level,
+    reducing the amount of data that needs to be read and transferred.
+    """
+
+    @abstractmethod
+    def push_filters(self, predicate: "Expr") -> Tuple["Scanner", Optional["Expr"]]:
+        """Push a filter predicate down to the scanner.
+
+        Args:
+            predicate: Expression representing the filter condition.
+
+        Returns:
+            Tuple of (new_scanner, residual_predicate) where:
+            - new_scanner: New Scanner instance with the filter applied
+            - residual_predicate: Any part of the predicate that couldn't be
+              pushed down and must be applied post-scan. None if fully pushed.
+        """
+        ...
+
+
+@DeveloperAPI
+class SupportsColumnPruning(ABC):
+    """Mixin for scanners that support column pruning/projection pushdown.
+
+    Column pruning allows reading only the columns needed by the query,
+    which is especially beneficial for columnar formats like Parquet.
+    """
+
+    @abstractmethod
+    def prune_columns(self, columns: List[str]) -> "Scanner":
+        """Prune the scanner to only read the specified columns.
+
+        Args:
+            columns: List of column names to read.
+
+        Returns:
+            New Scanner instance configured to read only the specified columns.
+        """
+        ...
+
+    @abstractmethod
+    def pruned_column_names(self) -> Optional[Tuple[str, ...]]:
+        """Physical column names selected after pruning, if any.
+
+        Returns:
+            ``None`` when no pruning has been applied (read all columns).
+            A tuple (possibly empty) after :meth:`prune_columns` has been
+            applied, listing on-disk / reader column names in read order.
+        """
+        ...
+
+
+@DeveloperAPI
+class SupportsLimitPushdown(ABC):
+    """Mixin for scanners that support limit pushdown.
+
+    Limit pushdown allows the scanner to stop early once the required number
+    of rows has been read.
+    """
+
+    @abstractmethod
+    def push_limit(self, limit: int) -> "Scanner":
+        """Push a row limit down to the scanner.
+
+        Args:
+            limit: Maximum number of rows to read.
+
+        Returns:
+            New Scanner instance with the limit applied.
+        """
+        ...
+
+
+@DeveloperAPI
+class SupportsPartitionPruning(ABC):
+    """Mixin for scanners that support partition pruning.
+
+    Partition pruning allows skipping entire files/partitions based on
+    predicates that reference partition columns.
+    """
+
+    @property
+    @abstractmethod
+    def partition_columns(self) -> Set[str]:
+        """Names of columns that are partition keys.
+
+        Callers (e.g. the predicate-pushdown rule) use this to decide
+        whether a predicate should be routed through :meth:`push_filters`
+        (data columns) or :meth:`prune_partitions` (partition columns).
+        Must be fully populated by schema inference at planning time.
+        """
+        ...
+
+    @abstractmethod
+    def prune_partitions(self, predicate: "Expr") -> "Scanner":
+        """Prune partitions based on a predicate.
+
+        The scanner determines its partition columns from its
+        ``Partitioning`` configuration, which is fully populated
+        by schema inference at planning time.
+
+        Args:
+            predicate: Expression to evaluate against partition values.
+
+        Returns:
+            New Scanner instance with partition pruning applied.
+        """
+        ...

python/ray/data/_internal/datasource_v2/parquet_datasource_v2.py

@@ -0,0 +1,309 @@
+"""Concrete ``DataSourceV2`` for Parquet files.
+
+Wires the V2 listing (`NonSamplingFileIndexer`, driven by the upstream
+`ListFiles` op), scanning (`ParquetScanner`), and reading
+(`ParquetFileReader`) components against a user-supplied path set.
+Constructed from `read_api.read_parquet` when
+`DataContext.use_datasource_v2` is set.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List, Literal, Optional, Union
+
+import pyarrow as pa
+from typing_extensions import override
+
+from ray.data._internal.datasource.parquet_datasource import (
+    ParquetDatasource,
+    check_for_legacy_tensor_type,
+)
+from ray.data._internal.datasource_v2.chunkers.file_chunker import (
+    FileChunker,
+    ParquetFileChunker,
+)
+from ray.data._internal.datasource_v2.datasource_v2 import (
+    DatasourceCategory,
+    DataSourceV2,
+)
+from ray.data._internal.datasource_v2.listing.file_indexer import (
+    FileIndexer,
+    NonSamplingFileIndexer,
+)
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.readers.file_reader import (
+    INCLUDE_PATHS_COLUMN_NAME,
+)
+from ray.data._internal.datasource_v2.readers.in_memory_size_estimator import (
+    ParquetInMemorySizeEstimator,
+)
+from ray.data._internal.datasource_v2.scanners.parquet_scanner import ParquetScanner
+from ray.data._internal.util import _is_local_scheme
+from ray.data.context import DataContext
+from ray.data.datasource.partitioning import (
+    Partitioning,
+    PartitionStyle,
+    PathPartitionParser,
+    _partition_field_types_to_pa_schema,
+)
+from ray.data.datasource.path_util import _resolve_paths_and_filesystem
+from ray.util.annotations import DeveloperAPI
+
+if TYPE_CHECKING:
+    from pyarrow.fs import FileSystem
+
+    from ray.data.datasource.file_based_datasource import FileShuffleConfig
+
+
+@DeveloperAPI
+class ParquetDatasourceV2(DataSourceV2[FileManifest]):
+    """V2 Parquet datasource.
+
+    Listing is delegated to :class:`NonSamplingFileIndexer` driven by the
+    ``ListFiles`` logical op; scanning and reading are delegated to
+    :class:`ParquetScanner` and :class:`ParquetFileReader`. Schema
+    inference reads the first file's footer only and augments with
+    partition/path columns as configured.
+    """
+
+    def __init__(
+        self,
+        paths: List[str],
+        *,
+        filesystem: Optional["FileSystem"] = None,
+        partitioning: Optional[Partitioning] = Partitioning(PartitionStyle.HIVE),
+        file_extensions: Optional[List[str]] = None,
+        ignore_missing_paths: bool = False,
+        include_paths: bool = False,
+        include_row_hash: bool = False,
+        shuffle: Optional[Union[Literal["files"], "FileShuffleConfig"]] = None,
+        arrow_parquet_args: Optional[dict] = None,
+        schema: Optional[pa.Schema] = None,
+        parquet_format_kwargs: Optional[dict] = None,
+        file_chunker: Optional[FileChunker] = None,
+    ):
+        super().__init__(name="ParquetV2", category=DatasourceCategory.FILE_BASED)
+        # Capture the ``local://`` check against the *original* paths;
+        # ``_resolve_paths_and_filesystem`` below strips the scheme, so
+        # introspecting ``self._paths`` after construction can't tell a
+        # plain local path from a ``local://`` one. ``_supports_distributed_reads``
+        # is exposed by the base-class ``supports_distributed_reads``
+        # property and consumed by ``_read_datasource_v2``.
+        self._supports_distributed_reads = not _is_local_scheme(paths)
+        resolved_paths, resolved_filesystem = _resolve_paths_and_filesystem(
+            paths, filesystem
+        )
+        self._paths: List[str] = resolved_paths
+        self._filesystem = resolved_filesystem
+        self._partitioning = partitioning
+        self._file_extensions = file_extensions or ParquetDatasource._FILE_EXTENSIONS
+        self._ignore_missing_paths = ignore_missing_paths
+        self._include_paths = include_paths
+        self._include_row_hash = include_row_hash
+        self._shuffle = shuffle
+        self._arrow_parquet_args = arrow_parquet_args or {}
+        # ``pds.ParquetFileFormat`` kwargs forwarded from the deprecated
+        # ``read_parquet(dataset_kwargs=...)`` arg. Spread into the format
+        # built by ``ParquetFileReader._make_format``.
+        self._parquet_format_kwargs = parquet_format_kwargs or {}
+        # User-supplied schema override. When set, ``infer_schema`` returns
+        # it verbatim (plus partition/path augmentation) rather than reading
+        # footers, and the scanner pins it on the pyarrow dataset so files
+        # are cast to these types at scan time.
+        self._user_schema = schema
+        # Chunker that splits each listed Parquet file into one or more
+        # row-group-aligned read units. Defaults to ``ParquetFileChunker``
+        # (1 GiB target chunk size, or whatever ``DataContext`` configures).
+        # Callers can inject an alternative for tests or shuffle-aware
+        # planning code that wants whole-file reads.
+        self._file_chunker: FileChunker = (
+            file_chunker if file_chunker is not None else ParquetFileChunker()
+        )
+
+    @property
+    def paths(self) -> List[str]:
+        return self._paths
+
+    @property
+    def filesystem(self) -> Optional["FileSystem"]:
+        return self._filesystem
+
+    @property
+    def partitioning(self) -> Optional[Partitioning]:
+        return self._partitioning
+
+    @property
+    def file_extensions(self) -> List[str]:
+        return self._file_extensions
+
+    @property
+    def ignore_missing_paths(self) -> bool:
+        return self._ignore_missing_paths
+
+    @property
+    def include_paths(self) -> bool:
+        return self._include_paths
+
+    @property
+    def shuffle(self) -> Optional[Union[Literal["files"], "FileShuffleConfig"]]:
+        return self._shuffle
+
+    def _get_file_indexer(self) -> FileIndexer:
+        return NonSamplingFileIndexer(
+            ignore_missing_paths=self._ignore_missing_paths,
+            file_chunker=self._file_chunker,
+        )
+
+    def get_size_estimator(self) -> ParquetInMemorySizeEstimator:
+        return ParquetInMemorySizeEstimator()
+
+    @override
+    def resolve_partitioning(self, sample: FileManifest) -> Optional[Partitioning]:
+        """Return ``self._partitioning`` with path-discovered field names.
+
+        Hive partitioning ships with ``field_names=None`` by default and
+        discovers keys from the file path at plan time. Directory
+        partitioning already carries ``field_names`` at construction and
+        needs no discovery. Returns a fresh ``Partitioning`` rather than
+        mutating ``self`` so schema inference stays side-effect-free.
+        """
+        import copy
+
+        if self._partitioning is None or len(sample) == 0:
+            return copy.deepcopy(self._partitioning)
+        if self._partitioning.field_names:
+            return copy.deepcopy(self._partitioning)
+
+        first_path = sample.paths.tolist()[0]
+        parser = PathPartitionParser(self._partitioning)
+        partition_kv = parser(first_path)
+        if not partition_kv:
+            return copy.deepcopy(self._partitioning)
+        return Partitioning(
+            style=self._partitioning.style,
+            base_dir=self._partitioning.base_dir,
+            field_names=list(partition_kv.keys()),
+            field_types=self._partitioning.field_types,
+            filesystem=self._partitioning.filesystem,
+        )
+
+    def infer_schema(self, sample: FileManifest) -> pa.Schema:
+        """Read Parquet footers from the sample manifest; unify and augment.
+
+        When the sample has multiple files, their schemas are unified via
+        ``unify_schemas_with_validation`` so a first file with all-null
+        columns doesn't lock in ``null`` types that can't be cast to the
+        actual types in later files.
+
+        Pure: does not mutate ``self``. Partitioning field-name discovery
+        is delegated to :meth:`resolve_partitioning` so the discovered
+        ``Partitioning`` can flow through ``_read_datasource_v2`` into
+        :meth:`create_scanner` without side effects.
+        """
+        from concurrent.futures import ThreadPoolExecutor
+
+        import pyarrow.parquet as pq
+
+        from ray.data._internal.util import unify_schemas_with_validation
+
+        # Empty sample — typically means the user pointed ``read_parquet``
+        # at an empty directory. Return an empty schema so the rest of
+        # the plan stays valid; downstream ops produce zero blocks and
+        # the executor runs through without error (matches V1).
+        if len(sample) == 0:
+            return self._user_schema if self._user_schema is not None else pa.schema([])
+
+        sample_paths: List[str] = sample.paths.tolist()
+        # Parquet footer reads against high-latency object stores
+        # (S3, GCS) are ~50-100 ms each. Reading the sample's footers in
+        # parallel keeps driver-side schema inference bounded by the
+        # slowest single read rather than the sum. ``executor.map``
+        # preserves input order, which matters because the unified
+        # schema's field order follows the first schema's and
+        # ``sample_paths[0]`` drives partition discovery below.
+        #
+        # NOTE: ``pq.read_schema`` only accepts a ``filesystem=`` kwarg in
+        # recent pyarrow releases; older wheels in CI don't have it. Open
+        # the file through the configured filesystem and hand the file
+        # handle to ``read_schema`` for cross-version compatibility.
+        filesystem = self._filesystem
+
+        if self._user_schema is not None:
+            # Caller pinned the schema — skip footer reads. Partition/path
+            # augmentation below still applies so downstream ops see the
+            # synthesized columns.
+            schema = self._user_schema
+        else:
+
+            def _read_schema(path: str):
+                if filesystem is None:
+                    return pq.read_schema(path)
+                with filesystem.open_input_file(path) as handle:
+                    return pq.read_schema(handle)
+
+            with ThreadPoolExecutor(max_workers=min(len(sample_paths), 16)) as executor:
+                per_file_schemas = list(executor.map(_read_schema, sample_paths))
+            schema = (
+                unify_schemas_with_validation(per_file_schemas) or per_file_schemas[0]
+            )
+            assert isinstance(schema, pa.Schema)
+
+        resolved_partitioning = self.resolve_partitioning(sample)
+        if resolved_partitioning is not None:
+            first_path = sample_paths[0]
+            parser = PathPartitionParser(resolved_partitioning)
+            partition_kv = parser(first_path)
+            # For hive partitioning the parser discovers key names from the
+            # path itself; for directory partitioning it uses ``field_names``.
+            # In both cases ``partition_kv`` is the authoritative list of
+            # partition columns for the first sample file.
+            partition_pa_schema = _partition_field_types_to_pa_schema(
+                field_names=list(partition_kv.keys()),
+                field_types=resolved_partitioning.field_types or {},
+            )
+            for field_name in partition_kv.keys():
+                if schema.get_field_index(field_name) == -1:
+                    pa_type = partition_pa_schema.field(field_name).type
+                    schema = schema.append(pa.field(field_name, pa_type))
+
+        if (
+            self._include_paths
+            and schema.get_field_index(INCLUDE_PATHS_COLUMN_NAME) == -1
+        ):
+            schema = schema.append(pa.field(INCLUDE_PATHS_COLUMN_NAME, pa.string()))
+
+        if self._include_row_hash:
+            # ``row_hash`` is synthesized post-read as ``uint64``. Replace
+            # the field type when the file already has a ``row_hash``
+            # column (matches V1 ``_derive_schema``); otherwise append.
+            idx = schema.get_field_index("row_hash")
+            if idx == -1:
+                schema = schema.append(pa.field("row_hash", pa.uint64()))
+            elif schema.field(idx).type != pa.uint64():
+                schema = schema.set(idx, pa.field("row_hash", pa.uint64()))
+
+        check_for_legacy_tensor_type(schema)
+        return schema
+
+    def create_scanner(
+        self,
+        schema: pa.Schema,
+        filesystem: Optional["FileSystem"] = None,
+        **options: Any,
+    ) -> ParquetScanner:
+        # Callers (``_read_datasource_v2``) supply the sample-resolved
+        # ``Partitioning`` via ``options["partitioning"]`` so the
+        # datasource itself stays immutable — fall back to the
+        # constructor-provided one for direct users of this API.
+        partitioning = options.get("partitioning", self._partitioning)
+        return ParquetScanner(
+            schema=schema,
+            filesystem=filesystem or self._filesystem,
+            partitioning=partitioning,
+            include_paths=self._include_paths,
+            include_row_hash=self._include_row_hash,
+            shuffle=self._shuffle,
+            ignore_prefixes=options.get("ignore_prefixes"),
+            target_block_size=DataContext.get_current().target_max_block_size,
+            parquet_format_kwargs=dict(self._parquet_format_kwargs),
+        )

python/ray/data/_internal/datasource_v2/partitioners/file_partitioner.py

@@ -0,0 +1,51 @@
+from abc import ABC, abstractmethod
+
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+
+
+class FilePartitioner(ABC):
+    """Abstract base class for partitioning file manifests.
+
+    A ``FilePartitioner`` groups file paths and their associated metadata into new
+    file manifests based on a specific partitioning strategy.
+
+    Implementations must be deterministic to ensure consistent partitioning across
+    retries.
+    """
+
+    @abstractmethod
+    def add_input(self, input_manifest: FileManifest):
+        """Add a file manifest to be partitioned.
+
+        Args:
+            input_manifest: A ``FileManifest`` containing paths and metadata to partition.
+        """
+        ...
+
+    @abstractmethod
+    def has_partition(self) -> bool:
+        """Check if there are any partitions available.
+
+        Returns:
+            ``True`` if there are partitions ready to be retrieved via
+            ``next_partition()``, ``False`` otherwise.
+        """
+        ...
+
+    @abstractmethod
+    def next_partition(self) -> FileManifest:
+        """Get the next available partition.
+
+        Returns:
+            A ``FileManifest`` containing the paths and metadata for the next partition.
+        """
+        ...
+
+    @abstractmethod
+    def finalize(self):
+        """Process any remaining files and complete the partitioning.
+
+        This method is called after all inputs have been added via ``add_input()`` to
+        ensure any buffered files are properly partitioned.
+        """
+        ...

python/ray/data/_internal/datasource_v2/partitioners/round_robin_partitioner.py

@@ -0,0 +1,89 @@
+import logging
+
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.partitioners.file_partitioner import (
+    FilePartitioner,
+)
+from ray.data._internal.datasource_v2.readers.in_memory_size_estimator import (
+    InMemorySizeEstimator,
+)
+from ray.data._internal.weighted_round_robin import WeightedRoundRobinPartitioner
+
+logger = logging.getLogger(__name__)
+
+
+class RoundRobinPartitioner(FilePartitioner):
+    """Partitions input paths into blocks based on the in-memory size of files.
+
+    This partitioning ensures read tasks effectively utilize the cluster and
+    produce appropriately-sized blocks
+
+    **Steps:**
+        1. Initialize empty buckets.
+        2. Iterate through input blocks and add paths to buckets. For each path:
+            - If the current bucket falls below `min_bucket_size`, add the path and don't move
+              to the next bucket.
+            - If the current bucket exceeds `min_bucket_size` but not `max_bucket_size`,
+              add the path and move to the next bucket.
+            - If the current bucket exceeds `max_bucket_size`, yield the paths as a block, clear
+              the bucket, and move to the next bucket.
+        3. Yield any remaining paths in the buckets as blocks.
+
+    This algorithm ensures that each block contains [min_bucket_size, max_bucket_size]
+    worth of files.  It's a deterministic algorithm, but it doesn't maintain the order
+    of the input paths.
+    """
+
+    def __init__(
+        self,
+        in_memory_size_estimator: InMemorySizeEstimator,
+        *,
+        min_bucket_size: int,
+        max_bucket_size: int,
+        num_buckets: int,
+    ):
+        self._in_memory_size_estimator = in_memory_size_estimator
+        self._partitioner = WeightedRoundRobinPartitioner(
+            min_bucket_size=min_bucket_size,
+            max_bucket_size=max_bucket_size,
+            num_buckets=num_buckets,
+        )
+
+    def add_input(self, input_manifest: FileManifest):
+        in_memory_size_estimates = (
+            self._in_memory_size_estimator.estimate_in_memory_sizes(input_manifest)
+        )
+        for (
+            file_path,
+            file_size,
+            file_chunk_metadata,
+            in_memory_size_estimate,
+        ) in zip(
+            input_manifest.paths,
+            input_manifest.file_sizes,
+            input_manifest.file_chunk_metadatas,
+            in_memory_size_estimates,
+        ):
+            self._partitioner.add_item(
+                (file_path, file_size, file_chunk_metadata),
+                in_memory_size_estimate,
+            )
+
+    def has_partition(self) -> bool:
+        return self._partitioner.has_partition()
+
+    @property
+    def num_buckets(self) -> int:
+        return self._partitioner.num_buckets
+
+    def next_partition(self) -> FileManifest:
+        partition = self._partitioner.next_partition()
+        paths, file_sizes, file_chunk_metadatas = zip(*partition)
+        return FileManifest.construct_manifest(
+            list(paths),
+            list(file_sizes),
+            list(file_chunk_metadatas),
+        )
+
+    def finalize(self):
+        self._partitioner.finalize()

python/ray/data/_internal/datasource_v2/readers/base_reader.py

@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from typing import Generic, Iterator
+
+import pyarrow as pa
+
+from ray.data._internal.datasource_v2 import InputSplit
+from ray.util.annotations import DeveloperAPI
+
+
+@DeveloperAPI
+class Reader(ABC, Generic[InputSplit]):
+    """Abstract base class for reading data from input buckets.
+
+    Readers execute on workers to actually read data. They receive an InputSplit
+    (e.g., FileManifest for file-based sources) and yield Arrow tables.
+
+    The Reader is created by Scanner.create_reader() and is configured with all
+    pushdown optimizations (columns, predicates, limits) that were applied.
+    """
+
+    @abstractmethod
+    def read(self, input_split: InputSplit) -> Iterator[pa.Table]:
+        """Read data from the input bucket and yield Arrow tables.
+
+        This method is called on workers to perform the actual read operation.
+        It should respect all pushdowns configured on this reader.
+
+        Args:
+            input_split: Work unit describing what data to read.
+
+        Returns:
+            Iterator[pa.Table]: Iterator of PyArrow Tables containing the read data.
+        """
+        ...

python/ray/data/_internal/datasource_v2/readers/file_reader.py

@@ -0,0 +1,513 @@
+from enum import Enum
+from functools import cached_property, partial
+from typing import Any, Iterator, List, Optional, Set, Tuple
+
+import pyarrow as pa
+import pyarrow.dataset as pds
+from pyarrow.fs import FileSystem, LocalFileSystem
+
+from ray._common.utils import env_integer
+from ray.data._internal.arrow_block import _BATCH_SIZE_PRESERVING_STUB_COL_NAME
+from ray.data._internal.datasource.parquet_datasource import _compute_row_hashes
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.readers.base_reader import Reader
+from ray.data._internal.util import iterate_with_retry, make_async_gen
+from ray.data.context import DataContext
+from ray.data.datasource.partitioning import Partitioning, PathPartitionParser
+from ray.data.expressions import Expr
+from ray.util.annotations import DeveloperAPI
+
+# Synthetic column name produced when ``include_paths=True``. Shared with
+# the V2 datasource and scanner layers so all references are spelled the
+# same way.
+INCLUDE_PATHS_COLUMN_NAME = "path"
+
+# https://arrow.apache.org/docs/python/generated/pyarrow.dataset.Scanner.html#pyarrow.dataset.Scanner.from_batches
+# Default is specified by PyArrow.
+_ARROW_DEFAULT_BATCH_SIZE = 131_072
+
+# Number of batches read ahead per scanner. PyArrow's default is 16,
+# which can retain a multi-GB working set when scanning jumbo tensor
+# columns. 8 keeps I/O pipelined on remote filesystems for typical
+# Parquet workloads without doubling memory peak. Drop to 1 via the
+# env var when reading wide tensor columns.
+_ARROW_SCANNER_BATCH_READAHEAD = env_integer(
+    "RAY_DATA_ARROW_SCANNER_BATCH_READAHEAD", 8
+)
+
+# Number of worker threads used to read fragments concurrently per task.
+# Defaults to 4 to overlap remote-filesystem I/O latency across multiple
+# fragments. ``_read_fragment_batches`` caps this to ``len(fragments)``
+# at runtime so single-fragment tasks don't spin up extra workers, and
+# falls back to the sequential path entirely when
+# ``DataContext.execution_options.preserve_order`` is set.
+_DEFAULT_NUM_THREADS = env_integer("RAY_DATA_READ_FILES_NUM_THREADS", 4)
+
+ROW_HASH_COLUMN_NAME = "row_hash"
+
+
+class FileFormat(str, Enum):
+    PARQUET = "parquet"
+    CSV = "csv"
+    FEATHER = "feather"
+    JSON = "json"
+    ARROW = "arrow"
+    IPC = "ipc"
+
+
+@DeveloperAPI
+class FileReader(Reader[FileManifest]):
+    """Reader for file-based sources.
+
+    This reader uses PyArrow's Dataset API which automatically handles:
+    - Column pruning
+    - Filter pushdown (row group pruning)
+    - Batch-level filtering
+    """
+
+    def __init__(
+        self,
+        format: FileFormat,
+        batch_size: int = _ARROW_DEFAULT_BATCH_SIZE,
+        columns: Optional[List[str]] = None,
+        predicate: Optional[Expr] = None,
+        limit: Optional[int] = None,
+        filesystem: Optional[FileSystem] = None,
+        partitioning: Optional[Partitioning] = None,
+        ignore_prefixes: Optional[List[str]] = None,
+        include_paths: bool = False,
+        include_row_hash: bool = False,
+        schema: Optional[pa.Schema] = None,
+    ):
+        """Initialize the reader.
+        Refer to https://arrow.apache.org/docs/python/generated/pyarrow.dataset.dataset.html for more details.
+
+        Args:
+            format: Format of the files to read.
+            batch_size: Number of rows per batch.
+            columns: Columns to read. None means all columns.
+            predicate: Ray Data expression for filtering. Converted to a
+                PyArrow expression at the scanner-kwargs boundary.
+            limit: Maximum number of rows to read.
+            filesystem: Filesystem for reading files.
+            partitioning: Ray ``Partitioning`` object. Partition columns are
+                synthesized per-path via ``PathPartitionParser`` after each
+                batch is read, producing string-typed columns (V1 parity).
+            ignore_prefixes: Prefixes to ignore when reading files. Default is ['.', '_'] set by PyArrow.
+            include_paths: If True, include the source file path in a
+                ``'path'`` column for each row.
+            include_row_hash: If True, include a deterministic uint64 hash
+                per row in a ``'row_hash'`` column. The hash is derived from
+                the source file path and the row's post-filter output
+                position within the fragment, matching V1 semantics. If a
+                ``'row_hash'`` column already exists in the file, it is
+                overwritten.
+            schema: Caller-supplied unified schema used both to override
+                pyarrow's per-fragment inference (so a file whose column
+                is all-null doesn't pin the type to ``null``) and to cast
+                path-derived partition values to their target types when
+                ``Partitioning(field_types=...)`` is set.
+
+        """
+        self._format = format
+        self._columns = columns
+        self._predicate = predicate
+        self._batch_size = batch_size
+        self._limit = limit
+        self._filesystem = filesystem
+        self._partition_parser: Optional[PathPartitionParser] = (
+            PathPartitionParser(partitioning) if partitioning is not None else None
+        )
+        self._ignore_prefixes = ignore_prefixes
+        self._include_paths = include_paths
+        self._include_row_hash = include_row_hash
+        self._schema = schema
+
+    @cached_property
+    def _file_dataset_schema(self) -> Optional[pa.Schema]:
+        """Schema passed to ``pds.dataset`` — partition keys and ``path``
+        stripped out since those are synthesized post-read.
+
+        Pinning the caller-supplied schema at the pyarrow layer is how
+        we cover the "first file has an all-null column, later files
+        have the real type" case (e.g.
+        ``test_read_null_data_in_first_file``): without the pin,
+        pyarrow locks column X to ``null`` across the fragment group
+        and the later string-typed file fails the cast.
+
+        But pyarrow refuses extension-to-extension casts (e.g.
+        ``ArrowTensorTypeV2(shape=X)`` → ``ArrowVariableShapedTensor``),
+        and files with different per-file tensor shapes only unify
+        through ``ArrowVariableShapedTensor``. When the caller schema
+        contains *any* extension column we skip the pin entirely and
+        let pyarrow infer per-file — downstream concat handles the
+        heterogeneous blocks. Losing the all-null promotion in this
+        narrow case is acceptable; the combination of an all-null
+        first file *and* an extension column is uncommon, whereas
+        reading multiple files with variable-shape tensors is a
+        supported V1 feature.
+        """
+        if self._schema is None:
+            return None
+        if any(isinstance(f.type, pa.ExtensionType) for f in self._schema):
+            return None
+        partition_keys = (
+            set(self._partition_parser._scheme.field_names or [])
+            if self._partition_parser is not None
+            else set()
+        )
+        synthesized = {INCLUDE_PATHS_COLUMN_NAME}
+        if self._include_row_hash:
+            # ``row_hash`` is synthesized post-read, and the schema's type
+            # (``uint64``) may not match the on-disk column's type when a
+            # file already carries a ``row_hash`` column. Strip it from the
+            # dataset schema so pyarrow doesn't try to cast.
+            synthesized.add(ROW_HASH_COLUMN_NAME)
+        fields = [
+            f
+            for f in self._schema
+            if f.name not in partition_keys and f.name not in synthesized
+        ]
+        return pa.schema(fields) if fields else None
+
+    def _broadcast_partition_value(
+        self, name: str, value: Any, num_rows: int
+    ) -> pa.Array:
+        """Broadcast a single path-derived partition value to ``num_rows``,
+        casting to the caller-supplied schema's field type if set.
+
+        Values are stringified first (``PathPartitionParser`` in
+        ``explicit`` mode can return arrow-scalar-like non-strings) and
+        then cast to the target type, so ``Partitioning(field_types=
+        {"year": int})`` still promotes them correctly.
+        """
+        str_val = None if value is None else str(value)
+        arr = pa.repeat(pa.scalar(str_val, type=pa.string()), num_rows)
+        if self._schema is not None:
+            idx = self._schema.get_field_index(name)
+            if idx != -1 and self._schema.field(idx).type != pa.string():
+                arr = arr.cast(self._schema.field(idx).type)
+        return arr
+
+    def read(self, input_split: FileManifest) -> Iterator[pa.Table]:
+        """Read data from the input bucket and yield Arrow tables.
+
+        This method is called on workers to perform the actual read operation.
+        It should respect all pushdowns configured on this reader.
+
+        Args:
+            input_split: Work unit describing what data to read.
+
+        Yields:
+            pa.Table: PyArrow Tables containing the read data.
+        """
+        if len(input_split) == 0:
+            return
+
+        # Dedupe paths before handing them to pyarrow. When chunking is on,
+        # a manifest can carry multiple rows per file (each describing a
+        # different row-group slice); pyarrow only needs one fragment per
+        # file, and ``_get_fragments_to_read`` then fans out chunk-level
+        # sub-fragments using the per-row chunk metadata.
+        paths = list(dict.fromkeys(list(input_split.paths)))
+        filesystem = self._filesystem or LocalFileSystem()
+        # Build a ``pds.Dataset`` over *all* manifest paths so pyarrow's
+        # listing + column metadata is shared, but then iterate its
+        # fragments one at a time. ``dataset.scanner(fragments=...)``
+        # at the aggregate level would force a cross-fragment cast —
+        # which breaks variable-shape tensor extensions where each
+        # file has its own ``ArrowTensorTypeV2(shape=...)``. Per-
+        # fragment scanners let pyarrow use the native per-file type,
+        # and downstream concat handles unification.
+        dataset = pds.dataset(
+            source=paths,
+            format=self._make_format(),
+            filesystem=filesystem,
+            schema=self._file_dataset_schema,
+            ignore_prefixes=self._ignore_prefixes,
+        )
+
+        # Split the requested columns into ones the on-disk file has
+        # (pyarrow reads these) and ones we need to synthesize post-read
+        # (hive partition keys, "path"). ``self._columns is None`` means
+        # "no projection" — read every file column and synthesize every
+        # available partition/path column.
+        on_disk_column_names = set(dataset.schema.names)
+        if self._columns is None:
+            columns_to_read_from_file: Optional[List[str]] = None
+            columns_to_synthesize: Optional[Set[str]] = None
+        else:
+            columns_to_read_from_file = [
+                c for c in self._columns if c in on_disk_column_names
+            ]
+            columns_to_synthesize = set(self._columns) - on_disk_column_names
+
+        scanner_kwargs = {
+            "columns": columns_to_read_from_file,
+            "filter": (
+                self._predicate.to_pyarrow() if self._predicate is not None else None
+            ),
+            "batch_size": self._resolve_batch_size(dataset),
+            "batch_readahead": _ARROW_SCANNER_BATCH_READAHEAD,
+        }
+        scanner_kwargs.update(self._arrow_scanner_kwargs())
+
+        rows_read = 0
+        for table, fragment_path, fragment_row_offset in self._read_fragment_batches(
+            dataset, scanner_kwargs, input_split
+        ):
+            if self._limit is not None:
+                if rows_read >= self._limit:
+                    break
+                if len(table) > self._limit - rows_read:
+                    table = table.slice(0, self._limit - rows_read)
+
+            # Build the list of (name, value) pairs to synthesize from
+            # the fragment path: hive partitions + optional ``path``.
+            derived_items: List[Tuple[str, Any]] = []
+            if self._partition_parser is not None:
+                derived_items.extend(self._partition_parser(fragment_path).items())
+            if self._include_paths:
+                derived_items.append((INCLUDE_PATHS_COLUMN_NAME, fragment_path))
+
+            for name, value in derived_items:
+                if (
+                    columns_to_synthesize is not None
+                    and name not in columns_to_synthesize
+                ):
+                    continue
+                if name in table.column_names:
+                    # When the caller schema names a partition key, pyarrow
+                    # expects it in every file and fills it with nulls when
+                    # absent (the hive-typical case). Drop that placeholder
+                    # so the path-derived value below replaces it.
+                    table = table.drop([name])
+                table = table.append_column(
+                    name,
+                    self._broadcast_partition_value(name, value, table.num_rows),
+                )
+
+            # Skip when projection pushdown has narrowed ``columns`` to
+            # exclude ``row_hash`` — the projection below would just drop it.
+            if self._include_row_hash and (
+                columns_to_synthesize is None
+                or ROW_HASH_COLUMN_NAME in columns_to_synthesize
+            ):
+                hashes = _compute_row_hashes(
+                    fragment_path, fragment_row_offset, table.num_rows
+                )
+                if ROW_HASH_COLUMN_NAME in table.column_names:
+                    table = table.drop([ROW_HASH_COLUMN_NAME])
+                table = table.append_column(
+                    ROW_HASH_COLUMN_NAME, pa.array(hashes, type=pa.uint64())
+                )
+
+            if self._columns is not None:
+                # Project/reorder to the caller's requested column order;
+                # drop any that weren't produced (matches V1's lenient
+                # behavior). Always select — an empty projection must
+                # narrow the table to zero columns so the stub-column
+                # guard below handles row preservation.
+                produced = set(table.column_names)
+                projected = [c for c in self._columns if c in produced]
+                table = table.select(projected)
+
+            if table.num_columns == 0 and table.num_rows > 0:
+                # Guards against ``pa.concat_tables`` collapsing rows
+                # when a batch has zero columns (e.g., empty projection
+                # for a count query). The stub column is dropped by
+                # downstream projections.
+                table = table.append_column(
+                    _BATCH_SIZE_PRESERVING_STUB_COL_NAME,
+                    pa.nulls(table.num_rows),
+                )
+
+            self._on_batch_read(table)
+            rows_read += len(table)
+            yield table
+
+    def _resolve_batch_size(self, dataset: pds.Dataset) -> int:
+        """Return the batch size to use for scanning.
+
+        Subclasses can override this to implement adaptive batch sizing.
+        """
+        return self._batch_size
+
+    def _on_batch_read(self, table: pa.Table) -> None:
+        """Hook called after each batch is read.
+
+        Subclasses can override this to update internal state (e.g., refine
+        batch size estimates from actual data).
+        """
+        pass
+
+    def _arrow_scanner_kwargs(self) -> dict:
+        """Additional keyword arguments passed to ``pds.Dataset.scanner()``.
+
+        Subclasses override this to inject format-specific options.
+        """
+        return {}
+
+    def _make_format(self) -> Any:
+        """Format passed to ``pds.dataset(format=...)``.
+
+        Defaults to the format string (e.g. ``"parquet"``); subclasses
+        override to return a configured ``pds.FileFormat`` instance when
+        format-specific options (read options, fragment scan options) need
+        to be threaded through.
+        """
+        return self._format.value
+
+    def _get_fragments_to_read(
+        self,
+        dataset: pds.Dataset,
+        manifest: FileManifest,
+    ) -> List[Tuple[pds.Fragment, int]]:
+        """Return ``(fragment, file_row_offset)`` pairs to scan for this
+        manifest.
+
+        ``file_row_offset`` is the cumulative pre-filter row count of all
+        rows in the underlying file that precede this fragment. It seeds
+        the per-fragment hashing offset so chunked sub-fragments of the
+        same file produce unique ``_compute_row_hashes`` keys instead of
+        colliding on ``(path, 0, n)``.
+
+        Default impl returns one ``(fragment, 0)`` per file in the dataset
+        (paths are deduped in :meth:`read` before the dataset is built).
+        Subclasses that support per-row chunk metadata
+        (e.g. :class:`ParquetFileReader`) override this to fan a single
+        file fragment out into N sub-fragments — one per row-group slice —
+        based on :attr:`FileManifest.file_chunk_metadatas`, each paired
+        with its starting row offset in the file.
+        """
+        return [(fragment, 0) for fragment in dataset.get_fragments()]
+
+    def _read_fragment_batches(
+        self,
+        dataset: pds.Dataset,
+        scanner_kwargs: dict,
+        manifest: FileManifest,
+    ) -> Iterator[Tuple[pa.Table, str, int]]:
+        """Yield non-empty (table, fragment_path, fragment_row_offset) triples.
+
+        ``fragment_row_offset`` is the post-filter row position of the first
+        row of ``table`` within its fragment. ``iterate_with_retry`` skips
+        already-yielded items on retry, so ``offset`` reflects only the
+        rows that actually surface to the caller — matching V1 row-hash
+        semantics even when a fragment fails partway through.
+
+        Retry is scoped per-fragment: if a fragment fails mid-read, only
+        that fragment is re-read (skipping batches already yielded).
+        Wrapping the whole manifest in a single retry would re-iterate
+        fragments that already succeeded and double-emit their batches.
+
+        Each fragment gets its own scanner so pyarrow uses the native
+        per-file schema. A cross-fragment scanner would force a unified
+        schema cast, which refuses extension-to-extension conversion
+        (e.g. variable-shape tensors). V1 ``ParquetDatasource`` follows
+        the same per-fragment pattern via ``fragment.to_batches``.
+
+        When ``RAY_DATA_READ_FILES_NUM_THREADS > 1`` and
+        ``execution_options.preserve_order`` is False, fragments are
+        read concurrently via :func:`make_async_gen`. We still pass
+        ``preserve_ordering=True`` so concurrent reads emit blocks in
+        fragment order; otherwise Ray Data task retries (block
+        reconstruction) could produce a different block sequence.
+
+        ``make_async_gen`` consumes the whole input iterator up front
+        when preserving order. That is acceptable here because the input
+        is the finite fragment manifest from ``_get_fragments_to_read``,
+        which we materialize below anyway. File data is still read lazily
+        by the worker threads.
+        """
+        ctx = DataContext.get_current()
+
+        # ``preserve_ordering=True`` would drain the input iterator
+        # eagerly anyway, so materialize once here to (a) cap
+        # ``num_workers`` at the actual fragment count and (b) avoid
+        # an early-fallback when the manifest has a single fragment.
+        # Subclasses (e.g. ``ParquetFileReader``) override
+        # ``_get_fragments_to_read`` to fan out chunk-level
+        # sub-fragments from the manifest's chunk metadata.
+        fragments_with_offsets = self._get_fragments_to_read(dataset, manifest)
+        if not fragments_with_offsets:
+            return
+
+        num_workers = min(_DEFAULT_NUM_THREADS, len(fragments_with_offsets))
+        if num_workers <= 1 or ctx.execution_options.preserve_order:
+            yield from self._read_fragments_sequential(
+                iter(fragments_with_offsets), scanner_kwargs
+            )
+            return
+
+        # Set `preserve_ordering=True` to ensure deterministic output ordering.
+        # This is required so that Ray Data task retries (block reconstruction)
+        yield from make_async_gen(
+            base_iterator=iter(fragments_with_offsets),
+            fn=partial(self._read_fragments_sequential, scanner_kwargs=scanner_kwargs),
+            preserve_ordering=True,
+            num_workers=num_workers,
+        )
+
+    def _read_fragments_sequential(
+        self,
+        fragments_with_offsets: Iterator[Tuple[pds.Fragment, int]],
+        scanner_kwargs: dict,
+    ) -> Iterator[Tuple[pa.Table, str, int]]:
+        """Read each fragment in ``fragments_with_offsets`` in order, yielding
+        ``(table, fragment_path, fragment_row_offset)`` triples.
+
+        Each input pair is ``(fragment, file_row_offset)``. The yielded
+        ``fragment_row_offset`` starts at ``file_row_offset`` (the row
+        position of the fragment's first row within its underlying file)
+        and accumulates per yielded batch, so the per-fragment row-hash
+        math in :meth:`read` keys off the right window even when chunking
+        fans one file into multiple sub-fragments sharing ``fragment.path``.
+
+        ``iterate_with_retry`` is scoped to a single fragment so a
+        transient I/O failure only re-reads the failing file (skipping
+        batches already yielded), not the whole input.
+
+        This is the per-worker body for the threaded path in
+        :meth:`_read_fragment_batches` (one thread per call, each
+        consuming a disjoint slice of fragments via ``make_async_gen``)
+        and is also the entire read loop for the sequential path.
+        """
+        ctx = DataContext.get_current()
+        for fragment, file_row_offset in fragments_with_offsets:
+            offset = file_row_offset
+            for table in iterate_with_retry(
+                partial(self._iter_fragment_tables, fragment, scanner_kwargs),
+                f"read fragment {fragment.path}",
+                match=ctx.retried_io_errors,
+            ):
+                if table.num_rows > 0:
+                    yield table, fragment.path, offset
+                    offset += table.num_rows
+
+    def _iter_fragment_tables(
+        self,
+        fragment: pds.Fragment,
+        scanner_kwargs: dict,
+    ) -> Iterator[pa.Table]:
+        """Yield Arrow tables for a single fragment.
+
+        Subclasses override this to swap in a format-specific reader for
+        fragments that don't fit the default scanner-based path (e.g.
+        Parquet's ARROW-5030 nested-type fallback).
+
+        When a non-extension caller schema is available we pin it at the
+        scanner so pyarrow null-fills any column the unified schema names
+        but the fragment lacks (V1 parity — ``ParquetDatasource`` passes
+        ``read_schema`` to ``fragment.to_batches``). Falling back to the
+        per-fragment ``physical_schema`` preserves the variable-shape
+        tensor escape hatch already encoded in ``_file_dataset_schema``.
+        """
+        fragment_schema = (
+            self._file_dataset_schema
+            if self._file_dataset_schema is not None
+            else fragment.physical_schema
+        )
+        scanner = fragment.scanner(**scanner_kwargs, schema=fragment_schema)
+        for tagged in scanner.scan_batches():
+            yield pa.Table.from_batches(batches=[tagged.record_batch])

python/ray/data/_internal/datasource_v2/readers/in_memory_size_estimator.py

@@ -0,0 +1,146 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import numpy as np
+
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.readers.file_reader import FileReader
+from ray.data._internal.delegating_block_builder import DelegatingBlockBuilder
+from ray.data.block import BlockAccessor
+from ray.util.annotations import DeveloperAPI
+
+
+@DeveloperAPI
+class InMemorySizeEstimator(ABC):
+    @abstractmethod
+    def estimate_in_memory_sizes(self, manifest: FileManifest) -> np.ndarray:
+        """Estimate the in-memory sizes of the paths in the given manifest.
+
+        Some `FilePartitioner` implementations use this method to ensure that each
+        read task receives an appropriate amount of data. To ensure that file listing
+        is efficient, this method must be cheap to call, on average.
+
+        Args:
+            manifest: A manifest containing the paths and on-disk sizes of the files.
+
+        Returns:
+            The estimated in-memory sizes of the data in bytes.
+        """
+        ...
+
+
+@DeveloperAPI
+class SamplingInMemorySizeEstimator(InMemorySizeEstimator):
+    """Estimates in-memory sizes by reading files.
+
+    This class estimates the in-memory size of files by multiplying the on-disk
+    size by an estimated encoding ratio. If an instance hasn't estimated an encoding
+    ratio yet, it'll read a file to estimate it. Otherwise, it'll use the previously
+    estimated encoding ratio.
+
+    TODO: This approach doesn't work well for formats that produce multiple batches
+    (because we assume a 1:1 encoding ratio) or for formats that vary in encoding
+    ratios (e.g. videos).
+    """
+
+    def __init__(self, reader: "FileReader"):
+        self._reader = reader
+
+        self._encoding_ratio = None
+
+    def estimate_in_memory_sizes(self, manifest: FileManifest) -> np.ndarray:
+        assert np.all(manifest.file_sizes >= 0)
+
+        for path, file_size in zip(manifest.paths, manifest.file_sizes):
+            if self._encoding_ratio is None:
+                # Estimating the encoding ratio can be expensive since it requires
+                # reading the file. So, we only estimate the encoding ratio if we don't
+                # already have one.
+                self._encoding_ratio = self._estimate_encoding_ratio(path, file_size)
+                break
+
+        if self._encoding_ratio is None:
+            # If we couldn't estimate the encoding ratio, assume a 1:1 encoding ratio.
+            return manifest.file_sizes
+        else:
+            return manifest.file_sizes * self._encoding_ratio
+
+    def _estimate_encoding_ratio(
+        self,
+        path: str,
+        file_size: int,
+    ) -> Optional[float]:
+        """
+        Estimate the encoding ratio (in-memory size / on-disk size) for a file.
+
+        Args:
+            path: The path to the file.
+            file_size: The on-disk size of the file/chunk in bytes.
+
+        Returns:
+            The estimated encoding ratio of the file, or `None` if the ratio can't
+            be estimated.
+        """
+        # If the file is empty, we can't estimate the encoding ratio.
+        if not file_size:
+            return None
+
+        # Use ``None`` chunk metadata: the size estimator reads the file whole
+        # to estimate the encoding ratio; chunk-level splitting is irrelevant here.
+        manifest = FileManifest.construct_manifest(
+            [path],
+            [file_size],
+            [None],
+        )
+        batches = self._reader.read(manifest)
+
+        try:
+            first_batch = next(batches)
+        except StopIteration:
+            # If there's no data, we can't estimate the encoding ratio.
+            return None
+
+        try:
+            # Try to read a second batch. If it succeeds, it means the file contains
+            # multiple batches.
+            next(batches)
+        except StopIteration:
+            # Each file contains exactly one batch.
+            builder = DelegatingBlockBuilder()
+            builder.add_batch(first_batch)
+            block = builder.build()
+
+            in_memory_size = BlockAccessor.for_block(block).size_bytes()
+        else:
+            # Each file contains multiple batches.
+            #
+            # NOTE: To avoid reading the entire file to estimate the encoding ratio,
+            # we assume the file is 1:1 encoded. We can't return `None` because if
+            # all files contain multiple batches, then we'd try to re-estimate the
+            # encoding ratio for every file, and that'd be very expensive.
+            in_memory_size = file_size
+
+        return in_memory_size / file_size
+
+
+# Default Parquet encoding ratio: in-memory is ~5x on-disk size.
+# Parquet uses columnar compression and encoding, so Arrow in-memory
+# representation is significantly larger than the on-disk format.
+PARQUET_ENCODING_RATIO_ESTIMATE_DEFAULT = 5
+
+
+@DeveloperAPI
+class ParquetInMemorySizeEstimator(InMemorySizeEstimator):
+    """Estimates in-memory sizes for Parquet files using a fixed encoding ratio.
+
+    Parquet files are typically much smaller on disk than in memory due to
+    columnar compression and encoding. This estimator applies a constant
+    ratio (default 5x) to avoid the overhead of reading file metadata or
+    sampling data, which can be slow for Parquet files and hurt startup time.
+    """
+
+    def __init__(self, encoding_ratio: float = PARQUET_ENCODING_RATIO_ESTIMATE_DEFAULT):
+        self._encoding_ratio = encoding_ratio
+
+    def estimate_in_memory_sizes(self, manifest: FileManifest) -> np.ndarray:
+        return self._encoding_ratio * manifest.file_sizes

python/ray/data/_internal/datasource_v2/readers/parquet_file_reader.py

@@ -0,0 +1,516 @@
+import logging
+import math
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple
+
+import pyarrow as pa
+import pyarrow.dataset as pds
+import pyarrow.parquet as pq
+from pyarrow.fs import FileSystem
+from typing_extensions import override
+
+if TYPE_CHECKING:
+    from ray.data.datasource.partitioning import Partitioning
+
+from ray._common.utils import env_bool, env_integer
+from ray.data._internal.datasource.parquet_datasource import (
+    AUTOLOAD_PICKLE_OBJECT_SCALAR_ENV_VAR,
+    _check_for_pickle_object_columns,
+)
+from ray.data._internal.datasource_v2.chunkers.parquet_file_chunking_utils import (
+    _fragments_from_chunk_metadata,
+)
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.readers.file_reader import (
+    _ARROW_DEFAULT_BATCH_SIZE,
+    FileFormat,
+    FileReader,
+)
+from ray.data._internal.datasource_v2.readers.in_memory_size_estimator import (
+    PARQUET_ENCODING_RATIO_ESTIMATE_DEFAULT,
+)
+from ray.data._internal.util import MiB
+from ray.data.expressions import Expr
+from ray.util.annotations import DeveloperAPI
+from ray.util.debug import log_once
+
+logger = logging.getLogger(__name__)
+
+_UNSET = object()
+
+# Per-stream read-ahead buffer for ``use_buffered_stream=True``. PyArrow's
+# default (~8 KiB) produces many tiny range requests on S3; 8 MiB
+# amortizes per-request latency across meaningful payload sizes.
+_PARQUET_FRAGMENT_BUFFER_SIZE = env_integer(
+    "RAY_DATA_PARQUET_FRAGMENT_BUFFER_SIZE", 8 * MiB
+)
+
+
+def _estimate_batch_size_from_metadata(
+    fragment: pds.ParquetFileFragment,
+    columns: Optional[List[str]],
+    target_block_size: int,
+) -> Optional[int]:
+    """Estimate batch size from Parquet row group metadata without reading data.
+
+    Uses uncompressed column sizes from row group metadata and the encoding
+    ratio to estimate in-memory row size, then computes how many rows fit
+    in ``target_block_size``.
+
+    Args:
+        fragment: A PyArrow Parquet fragment with accessible metadata.
+        columns: Columns being read, or None for all columns.
+        target_block_size: Target in-memory size per batch in bytes.
+
+    Returns:
+        Estimated batch size in rows, or None if metadata is unavailable.
+    """
+    try:
+        # Accessing `metadata` triggers I/O via `EnsureCompleteMetadata()` to
+        # read the Parquet footer. `check_status` maps C++ Status codes to
+        # Python exceptions: IOError (OSError) for I/O failures,
+        # ArrowInvalid for corrupt footers.
+        # https://github.com/apache/arrow/blob/apache-arrow-23.0.0/python/pyarrow/error.pxi#L110-L126
+        metadata: pq.FileMetaData = fragment.metadata
+    except (OSError, pa.ArrowInvalid) as e:
+        logger.debug("Failed to read Parquet metadata for batch size estimation: %s", e)
+        return None
+
+    if metadata is None or metadata.num_row_groups == 0:
+        return None
+
+    row_group_idx: int = (
+        fragment.row_groups[0].id if fragment.row_groups is not None else 0
+    )
+    row_group_meta: pq.RowGroupMetaData = metadata.row_group(row_group_idx)
+    row_group_num_rows: int = row_group_meta.num_rows
+
+    if row_group_num_rows == 0:
+        return None
+
+    if columns is not None:
+        projected_columns = tuple(columns)
+        target_column_indices = []
+        for col_idx in range(row_group_meta.num_columns):
+            leaf_path = row_group_meta.column(col_idx).path_in_schema
+            # Account for nested columns
+            if any(
+                leaf_path == col_name or leaf_path.startswith(f"{col_name}.")
+                for col_name in projected_columns
+            ):
+                target_column_indices.append(col_idx)
+        row_group_uncompressed_size = sum(
+            row_group_meta.column(col_idx).total_uncompressed_size
+            for col_idx in target_column_indices
+        )
+    else:
+        # Sum per-column uncompressed sizes instead of using
+        # row_group_meta.total_byte_size, which can return the *compressed* size
+        # for some files (apache/arrow#48138).
+        row_group_uncompressed_size = sum(
+            row_group_meta.column(col_idx).total_uncompressed_size
+            for col_idx in range(row_group_meta.num_columns)
+        )
+
+    # Estimate the in-memory size of the row group
+    estimated_in_mem_row_group_size = (
+        row_group_uncompressed_size * PARQUET_ENCODING_RATIO_ESTIMATE_DEFAULT
+    )
+
+    estimated_in_mem_row_size = estimated_in_mem_row_group_size / row_group_num_rows
+    if estimated_in_mem_row_size == 0:
+        return None
+
+    # Never request more rows than the row group actually contains.
+    target_batch_size = min(
+        math.ceil(target_block_size / estimated_in_mem_row_size),
+        row_group_num_rows,
+    )
+
+    logger.debug(
+        f"Estimated target batch size to be: {target_batch_size} (with "
+        f"{target_block_size=} bytes and {estimated_in_mem_row_size=} bytes)"
+    )
+
+    return target_batch_size
+
+
+@DeveloperAPI
+class ParquetFileReader(FileReader):
+    """Parquet-specific file reader with adaptive batch sizing.
+
+    Extends :class:`FileReader` with:
+
+    - **Metadata-based batch size estimation**: Uses Parquet row group metadata
+      (uncompressed column sizes) to estimate an optimal batch size before
+      reading any data.
+    - **Adaptive refinement**: After reading each batch, refines the batch size
+      estimate from actual in-memory sizes for subsequent reads.
+
+    For non-Parquet formats, use :class:`FileReader` directly.
+    """
+
+    def __init__(
+        self,
+        batch_size: Optional[int] = None,
+        columns: Optional[List[str]] = None,
+        predicate: Optional[Expr] = None,
+        limit: Optional[int] = None,
+        filesystem: Optional[FileSystem] = None,
+        partitioning: "Optional[Partitioning]" = None,
+        ignore_prefixes: Optional[List[str]] = None,
+        target_block_size: Optional[int] = None,
+        include_paths: bool = False,
+        include_row_hash: bool = False,
+        schema: Optional[pa.Schema] = None,
+        parquet_format_kwargs: Optional[Dict[str, Any]] = None,
+    ):
+        """Initialize the Parquet reader.
+
+        Args:
+            batch_size: Explicit batch size override. If provided, disables
+                adaptive batch sizing.
+            columns: Columns to read. None means all columns.
+            predicate: Ray Data expression for filtering.
+            limit: Maximum number of rows to read.
+            filesystem: Filesystem for reading files.
+            partitioning: Ray ``Partitioning`` for synthesizing partition
+                columns from file paths.
+            ignore_prefixes: Prefixes to ignore when reading files.
+            target_block_size: Target in-memory size per batch in bytes.
+                Used for adaptive batch sizing when ``batch_size`` is not set.
+            include_paths: If True, include the source file path in a
+                ``'path'`` column for each row.
+            include_row_hash: If True, include a deterministic uint64 hash
+                per row in a ``'row_hash'`` column.
+            schema: Caller-supplied unified schema forwarded to the base
+                :class:`FileReader` for per-fragment inference override
+                and partition-column type casting.
+            parquet_format_kwargs: Extra kwargs spread into
+                :class:`pyarrow.dataset.ParquetFileFormat` (e.g.
+                ``coerce_int96_timestamp_unit``, ``pre_buffer``,
+                ``dictionary_columns``). Used to forward the deprecated
+                ``dataset_kwargs`` arg on the V2 path.
+        """
+        super().__init__(
+            format=FileFormat.PARQUET,
+            batch_size=batch_size or _ARROW_DEFAULT_BATCH_SIZE,
+            columns=columns,
+            predicate=predicate,
+            limit=limit,
+            filesystem=filesystem,
+            partitioning=partitioning,
+            ignore_prefixes=ignore_prefixes,
+            include_paths=include_paths,
+            include_row_hash=include_row_hash,
+            schema=schema,
+        )
+        self._explicit_batch_size = batch_size
+        self._allow_pickle_object_columns = env_bool(
+            AUTOLOAD_PICKLE_OBJECT_SCALAR_ENV_VAR, False
+        )
+        self._target_block_size = target_block_size
+        self._parquet_format_kwargs: Dict[str, Any] = parquet_format_kwargs or {}
+        self._sampled_batch_size: int | object = (
+            _UNSET  # pyrefly: ignore[bad-assignment]
+        )
+
+    @override
+    def _make_format(self) -> pds.ParquetFileFormat:
+        return pds.ParquetFileFormat(**self._parquet_format_kwargs)
+
+    @override
+    def _resolve_batch_size(self, dataset: pds.Dataset) -> int:
+        """Determine batch size from explicit setting, metadata, or default.
+
+        Priority: explicit batch_size > sampled estimate > metadata estimate > default.
+
+        On the first call ``_sampled_batch_size`` is ``_UNSET``, so we fall
+        through to the metadata estimate and seed ``_sampled_batch_size`` with
+        the result.  ``_on_batch_read`` later refines it from actual data, and
+        subsequent ``read()`` calls on the same instance use the refined value.
+        """
+        if self._explicit_batch_size is not None:
+            return self._explicit_batch_size
+
+        if self._sampled_batch_size is not _UNSET:
+            return self._sampled_batch_size  # pyrefly: ignore[bad-return]
+
+        batch_size = _ARROW_DEFAULT_BATCH_SIZE
+        if self._target_block_size is not None:
+            first_fragment = next(dataset.get_fragments(), None)
+            if first_fragment is not None:
+                estimated = _estimate_batch_size_from_metadata(
+                    first_fragment, self._columns, self._target_block_size
+                )
+                if estimated is not None:
+                    logger.debug(
+                        "Estimated Parquet batch size: %d rows (target_block_size=%d)",
+                        estimated,
+                        self._target_block_size,
+                    )
+                    batch_size = estimated
+
+        self._sampled_batch_size = batch_size
+        return batch_size
+
+    @override
+    def _on_batch_read(self, table: pa.Table) -> None:
+        """Refine batch size estimate from actual in-memory data."""
+        if self._target_block_size is None or table.nbytes == 0 or table.num_rows == 0:
+            return
+        row_size = table.nbytes / table.num_rows
+        self._sampled_batch_size = max(math.ceil(self._target_block_size / row_size), 1)
+
+    @override
+    def _get_fragments_to_read(
+        self,
+        dataset: pds.Dataset,
+        manifest: FileManifest,
+    ) -> List[Tuple[pds.Fragment, int]]:
+        """Fan file fragments into chunk-level sub-fragments per manifest row.
+
+        For each manifest row, looks up the file's fragment by path and:
+
+        - If ``chunk_metadata`` is ``None`` (whole-file case), the file
+          fragment is yielded as-is with a row offset of 0. This matches
+          ``ParquetFileChunker``'s behavior for files at or below
+          ``target_chunk_size`` and the default ``WholeFileChunker`` for
+          non-chunking callers.
+        - Otherwise the row carries a :class:`ParquetFileChunkMetadata`;
+          we slice the fragment via
+          :func:`~ray.data._internal.datasource_v2.chunkers.parquet_file_chunking_utils._fragments_from_chunk_metadata`
+          which returns one sub-fragment per row group in the chunk's
+          row-group range, paired with the cumulative pre-filter row
+          offset of that row group within the file. The downstream
+          ``_compute_row_hashes`` call uses this offset so row hashes
+          remain unique across sub-fragments that share ``fragment.path``.
+
+        Paths are deduped by :meth:`FileReader.read` before the dataset is
+        built, so the dataset has exactly one fragment per file. The
+        per-row chunk metadata drives the fan-out here, not the dataset
+        itself — multiple manifest rows can share a single path with
+        different chunk indices.
+        """
+        path_to_fragment = {
+            fragment.path: fragment for fragment in dataset.get_fragments()
+        }
+        fragments: List[Tuple[pds.Fragment, int]] = []
+        for path, chunk_metadata in zip(manifest.paths, manifest.file_chunk_metadatas):
+            fragment = path_to_fragment[path]
+            if chunk_metadata is None:
+                fragments.append((fragment, 0))
+            else:
+                fragments.extend(
+                    _fragments_from_chunk_metadata(fragment, chunk_metadata)
+                )
+        return fragments
+
+    @override
+    def _iter_fragment_tables(
+        self,
+        fragment: pds.Fragment,
+        scanner_kwargs: dict,
+    ) -> "Iterator[pa.Table]":
+        for table in self._iter_fragment_tables_without_pickle_check(
+            fragment, scanner_kwargs
+        ):
+            if not self._allow_pickle_object_columns:
+                _check_for_pickle_object_columns(table)
+            yield table
+
+    def _iter_fragment_tables_without_pickle_check(
+        self,
+        fragment: pds.Fragment,
+        scanner_kwargs: dict,
+    ) -> "Iterator[pa.Table]":
+        """Use V1's nested-type fallback path when the fragment has nested
+        columns whose row-group size exceeds Arrow's ~2GB chunking limit
+        (ARROW-5030).
+        """
+        import pyarrow.compute as pc
+
+        from ray.data._internal.arrow_ops.transform_pyarrow import (
+            _align_struct_fields,
+        )
+        from ray.data._internal.datasource.parquet_datasource import (
+            _get_safe_batch_size_for_nested_types,
+            _needs_nested_type_fallback,
+            _resolve_leaf_column_indices,
+            _resolve_read_columns,
+        )
+        from ray.data._internal.planner.plan_expression.expression_visitors import (
+            get_column_references,
+        )
+
+        columns = scanner_kwargs.get("columns")
+        filter_expr: pc.Expression = scanner_kwargs.get("filter")
+        # Include filter-referenced columns in the fallback check: a filter
+        # that touches a large nested column outside the projection still
+        # forces row-level decoding of that column, which would otherwise
+        # hit ARROW-5030 in the normal scanner path.
+        filter_columns = (
+            get_column_references(self._predicate)
+            if self._predicate is not None
+            else None
+        )
+        read_columns = _resolve_read_columns(columns, filter_expr, filter_columns)
+        if not _needs_nested_type_fallback(fragment, read_columns):
+            yield from super()._iter_fragment_tables(fragment, scanner_kwargs)
+            return
+
+        if log_once(f"parquet_nested_fallback_v2:{fragment.path}"):
+            logger.warning(
+                "Using pyarrow.parquet row-level batched reader for '%s' due "
+                "to Arrow nested type chunking limitation (ARROW-5030). "
+                "Consider writing Parquet files with smaller row group sizes "
+                "to avoid this.",
+                fragment.path,
+            )
+
+        batch_size = scanner_kwargs.get("batch_size")
+
+        pf = pq.ParquetFile(
+            fragment.path,
+            filesystem=fragment.filesystem,  # pyrefly: ignore[unexpected-keyword]
+        )
+
+        # Scope the safe batch-size calculation to the columns actually being
+        # decoded so we don't shrink batches based on columns we won't read.
+        leaf_indices = (
+            _resolve_leaf_column_indices(pf.metadata, read_columns)
+            if read_columns is not None and pf.metadata.num_row_groups > 0
+            else None
+        )
+        safe_batch_size = _get_safe_batch_size_for_nested_types(pf, leaf_indices)
+        fallback_batch_size = (
+            min(batch_size, safe_batch_size) if batch_size else safe_batch_size
+        )
+
+        # Apply row-group-level predicate pushdown via fragment.subset; the
+        # row-level filter is applied per-batch below since iter_batches
+        # doesn't accept a filter expression. Under schema evolution the
+        # filter may reference a column absent from this fragment's
+        # physical schema — fragment.subset uses that schema (not the
+        # unified one) and raises ArrowInvalid, so skip row-group pruning
+        # in that case and let the per-batch filter (post null-fill) do
+        # all the row-dropping.
+        fragment_physical_columns = set(fragment.physical_schema.names)
+        filter_touches_missing_column = filter_columns is not None and any(
+            c not in fragment_physical_columns for c in filter_columns
+        )
+        if filter_expr is not None and not filter_touches_missing_column:
+            subset = fragment.subset(filter=filter_expr)
+        else:
+            subset = fragment
+        row_groups = (
+            [rg.id for rg in subset.row_groups]
+            if subset.row_groups is not None
+            else None
+        )
+        if row_groups is not None and len(row_groups) == 0:
+            return
+
+        # ``pq.ParquetFile.iter_batches`` returns batches with the fragment's
+        # physical schema, so the fallback path would otherwise emit tables
+        # that differ from the scanner path (which pins
+        # ``_file_dataset_schema``) in struct field order, integer width,
+        # or missing columns. Align + cast to the same unified schema so
+        # fallback and non-fallback fragments concat cleanly downstream.
+        # Scoped to ``columns`` (not ``read_columns``) since filter-only
+        # columns are projected away before alignment.
+        file_dataset_schema = self._file_dataset_schema
+        if file_dataset_schema is not None and columns is not None:
+            align_schema = pa.schema(
+                [
+                    file_dataset_schema.field(c)
+                    for c in columns
+                    if file_dataset_schema.get_field_index(c) != -1
+                ]
+            )
+        else:
+            align_schema = file_dataset_schema
+
+        # Under schema evolution a filter-referenced column may live in
+        # the unified dataset schema but be absent from this fragment.
+        # The scanner path null-fills such columns via dataset-level
+        # schema pinning; ``pq.ParquetFile.iter_batches`` silently drops
+        # them and then ``table.filter(filter_expr)`` raises
+        # ``ArrowInvalid: No match for FieldRef.Name``. Mirror the
+        # scanner: append a null column of the unified type before the
+        # filter evaluates, so ``null > 15`` resolves to false and the
+        # fragment contributes 0 rows.
+        columns_to_null_fill: List[str] = (
+            [c for c in read_columns if c not in fragment_physical_columns]
+            if read_columns is not None
+            else []
+        )
+        null_fill_type_by_column = {
+            column_name: (
+                file_dataset_schema.field(column_name).type
+                if file_dataset_schema is not None
+                and file_dataset_schema.get_field_index(column_name) != -1
+                else pa.null()
+            )
+            for column_name in columns_to_null_fill
+        }
+
+        for batch in pf.iter_batches(
+            batch_size=fallback_batch_size,
+            columns=read_columns,
+            use_threads=False,
+            row_groups=row_groups,
+        ):
+            table = pa.Table.from_batches([batch])
+            for column_name in columns_to_null_fill:
+                if column_name not in table.column_names:
+                    table = table.append_column(
+                        column_name,
+                        pa.nulls(
+                            table.num_rows,
+                            type=null_fill_type_by_column[column_name],
+                        ),
+                    )
+            if filter_expr is not None:
+                table = table.filter(filter_expr)
+                # Skip downstream select/align/cast on fully-filtered
+                # batches — the caller discards empty tables anyway.
+                if table.num_rows == 0:
+                    continue
+            if columns is not None:
+                table = table.select([c for c in columns if c in table.column_names])
+            if align_schema is not None:
+                table = _align_struct_fields([table], align_schema)[0].cast(
+                    align_schema
+                )
+            yield table
+
+    @override
+    def _arrow_scanner_kwargs(self) -> dict:
+        # ``pre_buffer`` is left at pyarrow's default (``True``). With
+        # ``pre_buffer=True`` pyarrow plans a single coalesced range
+        # request covering all needed column chunks for a fragment and
+        # issues it in one I/O burst, then decodes from memory. With
+        # ``pre_buffer=False`` pyarrow opens a per-column buffered
+        # stream and fetches lazily — fine on narrow schemas (few large
+        # columns) but catastrophic on wide schemas (thousands of small
+        # columns become thousands of range requests). V1
+        # ``ParquetDatasource`` also relies on the default. The
+        # cross-fragment memory accumulation that originally motivated
+        # disabling ``pre_buffer`` (apache/arrow#39808) is already
+        # addressed by V2's per-fragment scanners.
+        #
+        # ``buffer_size`` controls the per-stream read-ahead buffer
+        # pyarrow issues against the filesystem when ``use_buffered_stream``
+        # is on. The default is small (8 KiB), which produces many tiny
+        # range requests on S3. 8 MiB amortizes S3 latency across
+        # meaningful bytes per round-trip. Tunable via env var for
+        # workloads that need a different point on the latency/memory-
+        # peak curve.
+        kwargs: dict = {
+            "fragment_scan_options": pds.ParquetFragmentScanOptions(
+                use_buffered_stream=True,
+                buffer_size=_PARQUET_FRAGMENT_BUFFER_SIZE,
+            ),
+            "fragment_readahead": 1,
+        }
+        return kwargs

python/ray/data/_internal/datasource_v2/scanners/arrow_file_scanner.py

@@ -0,0 +1,201 @@
+import logging
+from dataclasses import dataclass, replace
+from typing import List, Optional, Set, Tuple
+
+import pyarrow as pa
+from pyarrow.fs import FileSystem
+from typing_extensions import override
+
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.logical_optimizers import (
+    SupportsColumnPruning,
+    SupportsFilterPushdown,
+    SupportsLimitPushdown,
+    SupportsPartitionPruning,
+)
+from ray.data._internal.datasource_v2.scanners.file_scanner import FileScanner
+from ray.data.datasource.partitioning import Partitioning, PathPartitionParser
+from ray.data.expressions import Expr
+from ray.util.annotations import DeveloperAPI
+
+logger = logging.getLogger(__name__)
+
+
+@DeveloperAPI
+@dataclass(frozen=True)
+class ArrowFileScanner(
+    FileScanner,
+    SupportsFilterPushdown,
+    SupportsColumnPruning,
+    SupportsLimitPushdown,
+    SupportsPartitionPruning,
+):
+    """Base scanner for file-based datasources that use PyArrow's Dataset API.
+
+    Holds shared Arrow types and options (schema, projection, filesystem,
+    partitioning, etc.). Subclasses set the file format in :meth:`create_reader`.
+
+    Provides default implementations of filter pushdown, column pruning,
+    limit pushdown, and partition pruning that work for all Arrow-backed
+    formats.
+
+    Non-Arrow file formats should subclass :class:`FileScanner` directly.
+    """
+
+    schema: pa.Schema
+    batch_size: Optional[int] = None
+    columns: Optional[Tuple[str, ...]] = None
+    predicate: Optional[Expr] = None
+    partition_predicate: Optional[Expr] = None
+    limit: Optional[int] = None
+    filesystem: Optional[FileSystem] = None
+    partitioning: Optional[Partitioning] = None
+    ignore_prefixes: Optional[List[str]] = None
+
+    @property
+    def partition_columns(self) -> Set[str]:
+        """Return the set of partition column names, or empty if unpartitioned."""
+        if self.partitioning is None:
+            return set()
+        return set(self.partitioning.field_names or [])
+
+    def read_schema(self) -> pa.Schema:
+        """Return the logical schema after column pruning.
+
+        ``columns is None`` → no projection applied, return the full schema.
+        ``columns = ()`` → empty projection (``ds.select_columns([])``),
+        return an empty schema.
+
+        The physical read may still inject a stub column (see
+        ``_BATCH_SIZE_PRESERVING_STUB_COL_NAME``) so that row counts
+        survive a zero-column scan; that stub is an execution-layer detail
+        and is deliberately not reflected in this logical schema.
+        """
+        if self.columns is None:
+            return self.schema
+        fields = []
+        for name in self.columns:
+            idx = self.schema.get_field_index(name)
+            assert idx >= 0, f"Column {name} not found in schema"
+            fields.append(self.schema.field(idx))
+        return pa.schema(fields)
+
+    @override
+    def push_filters(
+        self, predicate: "Expr"
+    ) -> Tuple["ArrowFileScanner", Optional["Expr"]]:
+        """Push filter predicate down to the scanner.
+
+        ANDs the predicate with any existing predicate. The Ray ``Expr`` is
+        retained as the source of truth so the reader can introspect filter
+        columns; conversion to a PyArrow expression happens at the
+        scanner-kwargs boundary in :class:`FileReader`.
+
+        This method handles data-column predicates only. Partition predicates
+        should be pushed via :meth:`prune_partitions` instead; the optimizer
+        is responsible for splitting them before calling either method.
+
+        Args:
+            predicate: Ray Data expression to push down.
+
+        Returns:
+            A pair ``(scanner, residual)`` where ``scanner`` has the predicate
+            merged into its PyArrow filter. ``residual`` is ``None`` because
+            PyArrow handles the full filter at scan time.
+        """
+        if self.predicate is not None:
+            combined = self.predicate & predicate
+        else:
+            combined = predicate
+
+        return replace(self, predicate=combined), None
+
+    @override
+    def prune_columns(self, columns: List[str]) -> "ArrowFileScanner":
+        """Prune to only the specified columns.
+
+        Args:
+            columns: List of column names to keep.
+
+        Returns:
+            New scanner with column pruning applied.
+        """
+        if self.columns:
+            existing = set(self.columns)
+            columns = [c for c in columns if c in existing]
+
+        return replace(self, columns=tuple(columns))
+
+    @override
+    def pruned_column_names(self) -> Optional[Tuple[str, ...]]:
+        return self.columns
+
+    @override
+    def push_limit(self, limit: int) -> "ArrowFileScanner":
+        """Push row limit down to the scanner.
+
+        Args:
+            limit: Maximum number of rows to read.
+
+        Returns:
+            New scanner with limit applied.
+        """
+        current = self.limit
+        new_limit = min(current, limit) if current is not None else limit
+        return replace(self, limit=new_limit)
+
+    @override
+    def prune_partitions(self, predicate: "Expr") -> "ArrowFileScanner":
+        """Store a partition predicate for file-level pruning during plan().
+
+        The predicate is ANDed with any existing partition predicate. Actual
+        file pruning happens in :meth:`plan` when the manifest is available,
+        using :class:`PathPartitionParser` to evaluate partition values from
+        file paths.
+
+        Args:
+            predicate: Expression referencing only partition columns.
+
+        Returns:
+            New scanner with partition predicate stored.
+        """
+        if self.partition_predicate is not None:
+            combined = self.partition_predicate & predicate
+        else:
+            combined = predicate
+
+        return replace(self, partition_predicate=combined)
+
+    @override
+    def prune_manifest(self, manifest: FileManifest) -> FileManifest:
+        """Filter manifest to only files matching ``self.partition_predicate``.
+
+        Called by :func:`plan_read_files_op.do_read` for every incoming
+        manifest block. No-op when either the predicate or the
+        partitioning spec is absent. Uses
+        :class:`PathPartitionParser` to parse partition values from
+        each file path and evaluate the predicate.
+        """
+        if self.partition_predicate is None or self.partitioning is None:
+            return manifest
+
+        parser = PathPartitionParser(self.partitioning)
+        keep_indices = []
+
+        for i, path in enumerate(manifest.paths):
+            if parser.evaluate_predicate_on_partition(path, self.partition_predicate):
+                keep_indices.append(i)
+
+        if len(keep_indices) == len(manifest):
+            return manifest
+
+        pruned_count = len(manifest) - len(keep_indices)
+        logger.debug(
+            "Partition pruning removed %d of %d files",
+            pruned_count,
+            len(manifest),
+        )
+
+        block = manifest.as_block()
+        pruned_block = block.take(keep_indices)
+        return FileManifest(pruned_block)

python/ray/data/_internal/datasource_v2/scanners/file_scanner.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass, field
+from typing import Literal, Union
+
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.scanners.scanner import Scanner
+from ray.data.datasource.file_based_datasource import (
+    FileShuffleConfig,
+    _validate_shuffle_arg,
+)
+from ray.util.annotations import DeveloperAPI
+
+
+@DeveloperAPI
+@dataclass(frozen=True)
+class FileScanner(Scanner[FileManifest]):
+    """Base scanner for file-based datasources.
+
+    Subclasses implement format-specific ``read_schema()`` and
+    ``create_reader()``. Shuffling and parallel bucketing are handled
+    upstream in the ``ListFiles`` transform chain (``shuffle_files`` +
+    ``RoundRobinPartitioner`` via ``plan_list_files_op``), not here.
+
+    PyArrow Dataset-based scanners should subclass ``ArrowFileScanner``;
+    use ``FileScanner`` directly for non-Arrow file formats.
+    """
+
+    # kw_only so subclass dataclasses can declare their own required fields
+    # (like ``ArrowFileScanner.schema``) without running into the "non-default
+    # argument follows default argument" dataclass inheritance rule.
+    shuffle: Union[Literal["files"], FileShuffleConfig, None] = field(
+        default=None, kw_only=True
+    )
+
+    def __post_init__(self) -> None:
+        _validate_shuffle_arg(self.shuffle)
+
+    def prune_manifest(self, manifest: FileManifest) -> FileManifest:
+        """Return a filtered view of ``manifest``.
+
+        Default: identity. Subclasses that support file-level predicate
+        pruning (e.g. :class:`ArrowFileScanner`'s ``partition_predicate``)
+        override this to drop rows whose partition values fail the
+        predicate. Invoked per-block from
+        :func:`plan_read_files_op.do_read`.
+        """
+        return manifest

python/ray/data/_internal/datasource_v2/scanners/parquet_scanner.py

@@ -0,0 +1,88 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
+import pyarrow as pa
+
+from ray.data._internal.datasource.parquet_datasource import (
+    check_for_legacy_tensor_type,
+)
+from ray.data._internal.datasource_v2.readers.file_reader import (
+    INCLUDE_PATHS_COLUMN_NAME,
+    ROW_HASH_COLUMN_NAME,
+)
+from ray.data._internal.datasource_v2.readers.parquet_file_reader import (
+    ParquetFileReader,
+)
+from ray.data._internal.datasource_v2.scanners.arrow_file_scanner import (
+    ArrowFileScanner,
+)
+from ray.util.annotations import DeveloperAPI
+
+
+@DeveloperAPI
+@dataclass(frozen=True)
+class ParquetScanner(ArrowFileScanner):
+    """Parquet-specific scanner implementation.
+
+    Inherits filter pushdown, column pruning, limit pushdown, partition
+    pruning, and file shuffle from ArrowFileScanner. Adds Parquet-specific
+    reader creation with adaptive batch sizing and the Parquet-only
+    legacy-tensor-type schema check.
+    """
+
+    target_block_size: Optional[int] = None
+    include_paths: bool = False
+    include_row_hash: bool = False
+    # Extra kwargs forwarded to ``pds.ParquetFileFormat(**kwargs)`` inside
+    # the per-task ``ParquetFileReader`` (e.g. ``coerce_int96_timestamp_unit``,
+    # ``pre_buffer``, ``dictionary_columns``). Carries the deprecated
+    # ``dataset_kwargs`` payload from ``read_parquet`` to the worker.
+    parquet_format_kwargs: Dict[str, Any] = field(default_factory=dict)
+
+    def read_schema(self) -> pa.Schema:
+        """Return schema after column pruning and tensor check.
+
+        ``path`` and ``row_hash`` are synthesized post-read by the file
+        reader, but only for columns listed in ``self.columns`` (see
+        ``file_reader.read``'s ``columns_to_synthesize`` filter). When a
+        projection has pruned a synthesized column away, advertising it
+        here would put the schema out of sync with the actual blocks — so
+        only append when no projection is active or when it survives.
+        """
+        schema = super().read_schema()
+        synthesized = (
+            (self.include_paths, INCLUDE_PATHS_COLUMN_NAME, pa.string()),
+            (self.include_row_hash, ROW_HASH_COLUMN_NAME, pa.uint64()),
+        )
+        for enabled, name, dtype in synthesized:
+            if not enabled:
+                continue
+            if self.columns is not None and name not in self.columns:
+                continue
+            if schema.get_field_index(name) != -1:
+                continue
+            schema = schema.append(pa.field(name, dtype))
+
+        check_for_legacy_tensor_type(schema)
+        return schema
+
+    def create_reader(self) -> ParquetFileReader:
+        """Create a ParquetFileReader configured for this scanner.
+
+        Returns:
+            ParquetFileReader with all pushdowns and adaptive batch sizing.
+        """
+        return ParquetFileReader(
+            batch_size=self.batch_size,
+            columns=list(self.columns) if self.columns is not None else None,
+            predicate=self.predicate,
+            limit=self.limit,
+            filesystem=self.filesystem,
+            partitioning=self.partitioning,
+            ignore_prefixes=self.ignore_prefixes,
+            target_block_size=self.target_block_size,
+            include_paths=self.include_paths,
+            include_row_hash=self.include_row_hash,
+            schema=self.schema,
+            parquet_format_kwargs=dict(self.parquet_format_kwargs),
+        )

python/ray/data/_internal/datasource_v2/scanners/scanner.py

@@ -0,0 +1,51 @@
+from abc import ABC, abstractmethod
+from typing import Generic
+
+import pyarrow as pa
+
+from ray.data._internal.datasource_v2 import InputSplit
+from ray.data._internal.datasource_v2.readers.base_reader import Reader
+from ray.util.annotations import DeveloperAPI
+
+
+@DeveloperAPI
+class Scanner(ABC, Generic[InputSplit]):
+    """Abstract base class for configured scanners.
+
+    A Scanner represents the logical result of reading data, including applied
+    filters, projections, limits, and other pushdown operations. It is an
+    immutable abstraction: each push operation returns a new Scanner instance
+    via cloning rather than mutation.
+
+    The Scanner is responsible for:
+    1. Determining the output schema after all projections
+    2. Creating Reader instances configured with all pushdowns
+
+    Splitting the input into parallel work units used to live here as a
+    ``plan()`` method. That responsibility now belongs to the listing-side
+    pipeline (``ListFiles`` + ``FilePartitioner``); scanners only
+    need to answer "what schema?" and "give me a reader."
+    """
+
+    @abstractmethod
+    def read_schema(self) -> pa.Schema:
+        """Return the schema that will be produced by this scanner.
+
+        This reflects the schema after all column pruning has been applied.
+
+        Returns:
+            PyArrow Schema describing the output data.
+        """
+        ...
+
+    @abstractmethod
+    def create_reader(self) -> Reader[InputSplit]:
+        """Create a Reader configured for this scanner.
+
+        The returned Reader will have all pushdowns (columns, predicates, limits)
+        applied and is ready to execute on workers.
+
+        Returns:
+            Configured Reader instance.
+        """
+        ...

python/ray/data/_internal/datasource_v2/tests/test_file_partitioners.py

@@ -0,0 +1,144 @@
+from unittest.mock import MagicMock
+
+import numpy as np
+import pyarrow as pa
+import pytest
+
+from ray.data._internal.datasource_v2.listing.file_manifest import (
+    FILE_CHUNK_METADATA_COLUMN_NAME,
+    FILE_SIZE_COLUMN_NAME,
+    PATH_COLUMN_NAME,
+)
+from ray.data._internal.datasource_v2.listing.listing_utils import partition_files
+from ray.data._internal.datasource_v2.partitioners.round_robin_partitioner import (
+    RoundRobinPartitioner,
+)
+from ray.data._internal.datasource_v2.readers.in_memory_size_estimator import (
+    InMemorySizeEstimator,
+)
+from ray.data._internal.weighted_round_robin import WeightedRoundRobinPartitioner
+
+
+@pytest.mark.parametrize(
+    "num_paths, expected_partitions",
+    (
+        # These diagrams represent the state before leftover paths are yielded. Each
+        # column represent a bucket, the height represent the max bucket size, and
+        # numbers represent paths. There are two buckets, the min bucket size is 1, and
+        # the max bucket size is 3.
+        #
+        # | | | |  Yeilds [1].
+        # | | | |
+        # |1| | |
+        [1, [["1"]]],
+        # | | | |  Move to the second bucket because ther first one exceeds the min
+        # | | | |  bucket size (1).
+        # |1| |2|
+        [2, [["1"], ["2"]]],
+        # |5| | |    | | | |  Continue spreading paths because all buckets contain the
+        # |3| |4| -> | | |4|  min bucket size. Once the first bucket is full, yield the
+        # |1| |2|    | | |2|  paths, clear the bucket, and move to the second bucket.
+        [5, [["1", "3", "5"], ["2", "4"]]],
+        # | | |6|    | | | |  The second bucket is full, so we yield the paths, clear
+        # | | |4| -> | | | |  the bucket, and move back to the first.
+        # | | |2|    | | | |
+        [6, [["1", "3", "5"], ["2", "4", "6"]]],
+        # | | | |  Repeat.
+        # | | | |
+        # |7| | |
+        [7, [["1", "3", "5"], ["2", "4", "6"], ["7"]]],
+    ),
+)
+def test_round_robin_partitioner_produces_correct_partitions(
+    num_paths, expected_partitions
+):
+    input_table = pa.Table.from_pydict(
+        {
+            PATH_COLUMN_NAME: [str(i) for i in range(1, num_paths + 1)],
+            FILE_SIZE_COLUMN_NAME: [1] * num_paths,
+            FILE_CHUNK_METADATA_COLUMN_NAME: [None] * num_paths,
+        }
+    )
+
+    class StubInMemorySizeEstimator(InMemorySizeEstimator):
+        def estimate_in_memory_sizes(
+            self,
+            manifest,
+        ) -> np.ndarray:
+            return np.ones(len(manifest))
+
+    outputs = partition_files(
+        iter([input_table]),
+        MagicMock(),
+        partitioner=RoundRobinPartitioner(
+            in_memory_size_estimator=StubInMemorySizeEstimator(),
+            num_buckets=2,
+            min_bucket_size=1,
+            max_bucket_size=3,
+        ),
+    )
+
+    partitions = [output[PATH_COLUMN_NAME].to_pylist() for output in outputs]
+    assert partitions == expected_partitions
+
+
+def test_round_robin_partitioner_with_no_size_estimates():
+    # This tests the case where we don't have size estimates. This can happen if you use
+    # HTTPFileSystem.
+    input_table = pa.Table.from_pydict(
+        {
+            PATH_COLUMN_NAME: ["path0", "path1", "path2"],
+            FILE_SIZE_COLUMN_NAME: [None, None, None],
+            FILE_CHUNK_METADATA_COLUMN_NAME: [None, None, None],
+        }
+    )
+
+    class StubInMemorySizeEstimator(InMemorySizeEstimator):
+        def estimate_in_memory_sizes(
+            self,
+            manifest,
+        ) -> np.ndarray:
+            return manifest.file_sizes
+
+    outputs = partition_files(
+        iter([input_table]),
+        MagicMock(),
+        partitioner=RoundRobinPartitioner(
+            in_memory_size_estimator=StubInMemorySizeEstimator(),
+            num_buckets=2,
+            min_bucket_size=1,
+            max_bucket_size=1,
+        ),
+    )
+    partitions = [output[PATH_COLUMN_NAME].to_pylist() for output in outputs]
+
+    # If in-memory size estimates aren't available, the partitioner should round-robin
+    # the paths across the buckets, disregarding the bucket size limits.
+    assert len(partitions) == 2
+    assert partitions[0] == ["path0", "path2"]
+    assert partitions[1] == ["path1"]
+
+
+def test_weighted_round_robin_partitioner_can_emit_before_overflow():
+    partitioner = WeightedRoundRobinPartitioner(
+        num_buckets=1,
+        min_bucket_size=1,
+        max_bucket_size=3,
+        emit_before_overflow=True,
+    )
+
+    partitioner.add_item("a", 2)
+    partitioner.add_item("b", 2)
+
+    assert partitioner.has_partition()
+    assert partitioner.next_partition() == ["a"]
+
+    partitioner.finalize()
+    assert partitioner.has_partition()
+    assert partitioner.next_partition() == ["b"]
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/data/_internal/datasource_v2/tests/test_parquet_datasource_v2.py

@@ -0,0 +1,379 @@
+"""Unit tests for :class:`ParquetDatasourceV2`.
+
+These tests exercise schema inference, scanner/estimator creation, and
+include-paths schema augmentation against a local tmpdir — they do not
+spin up Ray.
+"""
+
+import os
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from ray.data._internal.datasource_v2.chunkers.file_chunker import (
+    ParquetFileChunker,
+    ParquetFileChunkMetadata,
+    WholeFileChunker,
+    create_chunk_metadata,
+)
+from ray.data._internal.datasource_v2.chunkers.parquet_file_chunking_utils import (
+    _fragments_from_chunk_metadata,
+)
+from ray.data._internal.datasource_v2.listing.file_manifest import FileManifest
+from ray.data._internal.datasource_v2.parquet_datasource_v2 import (
+    ParquetDatasourceV2,
+)
+from ray.data._internal.datasource_v2.readers.in_memory_size_estimator import (
+    ParquetInMemorySizeEstimator,
+)
+from ray.data._internal.datasource_v2.readers.parquet_file_reader import (
+    ParquetFileReader,
+)
+from ray.data._internal.datasource_v2.scanners.parquet_scanner import (
+    ParquetScanner,
+)
+from ray.data.datasource.partitioning import Partitioning, PartitionStyle
+
+
+def _write_parquet(path: str, table: pa.Table) -> None:
+    pq.write_table(table, path)
+
+
+def _manifest_of(paths):
+    sizes = [os.path.getsize(p) for p in paths]
+    return FileManifest.construct_manifest(paths, sizes, [None] * len(paths))
+
+
+def test_infer_schema_unpartitioned(tmp_path):
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"a": [1, 2, 3], "b": ["x", "y", "z"]}))
+
+    datasource = ParquetDatasourceV2([str(file_path)])
+    schema = datasource.infer_schema(_manifest_of([str(file_path)]))
+
+    assert schema.names == ["a", "b"]
+    assert schema.field("a").type == pa.int64()
+    assert schema.field("b").type == pa.string()
+
+
+def test_infer_schema_hive_partitioned(tmp_path):
+    for part in ["a", "b"]:
+        d = tmp_path / f"color={part}"
+        d.mkdir()
+        _write_parquet(str(d / "data.parquet"), pa.table({"x": [1, 2]}))
+
+    first_file = str(tmp_path / "color=a" / "data.parquet")
+    datasource = ParquetDatasourceV2(
+        [str(tmp_path)], partitioning=Partitioning(PartitionStyle.HIVE)
+    )
+    schema = datasource.infer_schema(_manifest_of([first_file]))
+
+    assert "x" in schema.names
+    assert "color" in schema.names
+    assert schema.field("color").type == pa.string()
+
+
+def test_infer_schema_with_include_paths(tmp_path):
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"a": [1, 2]}))
+
+    datasource = ParquetDatasourceV2([str(file_path)], include_paths=True)
+    schema = datasource.infer_schema(_manifest_of([str(file_path)]))
+
+    assert "path" in schema.names
+    assert schema.field("path").type == pa.string()
+
+
+def test_infer_schema_returns_empty_schema_on_empty_manifest(tmp_path):
+    datasource = ParquetDatasourceV2([str(tmp_path)])
+    empty = FileManifest.construct_manifest([], [], [])
+    schema = datasource.infer_schema(empty)
+    assert schema.names == []
+
+
+def test_create_scanner_returns_parquet_scanner(tmp_path):
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"a": [1]}))
+
+    datasource = ParquetDatasourceV2([str(file_path)])
+    schema = datasource.infer_schema(_manifest_of([str(file_path)]))
+    scanner = datasource.create_scanner(schema)
+
+    assert isinstance(scanner, ParquetScanner)
+    assert scanner.schema == schema
+
+
+def test_get_size_estimator_returns_parquet_estimator(tmp_path):
+    datasource = ParquetDatasourceV2([str(tmp_path)])
+    assert isinstance(datasource.get_size_estimator(), ParquetInMemorySizeEstimator)
+
+
+def test_paths_and_filesystem_resolved(tmp_path):
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"a": [1]}))
+
+    datasource = ParquetDatasourceV2([str(file_path)])
+    # _resolve_paths_and_filesystem produces a concrete filesystem even when
+    # the caller passed None.
+    assert datasource.filesystem is not None
+    assert len(datasource.paths) == 1
+
+
+def test_infer_schema_with_include_row_hash(tmp_path):
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"a": [1, 2]}))
+
+    datasource = ParquetDatasourceV2([str(file_path)], include_row_hash=True)
+    schema = datasource.infer_schema(_manifest_of([str(file_path)]))
+
+    assert "row_hash" in schema.names
+    assert schema.field("row_hash").type == pa.uint64()
+
+
+def test_infer_schema_with_include_row_hash_existing_column_promoted_to_uint64(
+    tmp_path,
+):
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"val": [1, 2], "row_hash": [10, 20]}))
+
+    datasource = ParquetDatasourceV2([str(file_path)], include_row_hash=True)
+    schema = datasource.infer_schema(_manifest_of([str(file_path)]))
+
+    assert schema.field("row_hash").type == pa.uint64()
+
+
+def test_create_scanner_propagates_include_row_hash(tmp_path):
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"a": [1]}))
+
+    datasource = ParquetDatasourceV2([str(file_path)], include_row_hash=True)
+    schema = datasource.infer_schema(_manifest_of([str(file_path)]))
+    scanner = datasource.create_scanner(schema)
+
+    assert scanner.include_row_hash is True
+
+
+def test_nested_fallback_handles_schema_evolution(tmp_path, monkeypatch):
+    """Regression: when the nested-type fallback fires on a fragment that
+    lacks a filter-referenced column, the V2 reader must null-fill the
+    missing column instead of letting pyarrow raise. Matches the
+    scanner path, which null-fills via dataset-level schema pinning.
+    """
+    import pyarrow.dataset as pds
+
+    from ray.data._internal.datasource import parquet_datasource
+    from ray.data._internal.datasource_v2.readers.parquet_file_reader import (
+        ParquetFileReader,
+    )
+    from ray.data.expressions import col
+
+    _write_parquet(
+        str(tmp_path / "with_b.parquet"),
+        pa.table({"a": [1, 2, 3], "b": [10, 20, 30]}),
+    )
+    _write_parquet(
+        str(tmp_path / "without_b.parquet"),
+        pa.table({"a": [4, 5, 6]}),
+    )
+
+    unified_schema = pa.schema([("a", pa.int64()), ("b", pa.int64())])
+    predicate = col("b") > 15
+
+    # Force the fallback path; the source-module attribute is what V2's
+    # function-local import resolves to on each call.
+    monkeypatch.setattr(
+        parquet_datasource, "_needs_nested_type_fallback", lambda *a, **kw: True
+    )
+
+    reader = ParquetFileReader(
+        columns=["a"], predicate=predicate, schema=unified_schema
+    )
+    dataset = pds.dataset(str(tmp_path), format="parquet", schema=unified_schema)
+    scanner_kwargs = {
+        "columns": ["a"],
+        "filter": predicate.to_pyarrow(),
+        "batch_size": None,
+    }
+
+    rows_by_fragment = {}
+    for fragment in dataset.get_fragments():
+        tables = list(reader._iter_fragment_tables(fragment, scanner_kwargs))
+        rows_by_fragment[os.path.basename(fragment.path)] = sum(
+            t.num_rows for t in tables
+        )
+
+    # with_b: rows where b > 15 → 2 rows (b=20, b=30)
+    # without_b: b is null-filled → null > 15 is null → 0 rows
+    assert rows_by_fragment == {"with_b.parquet": 2, "without_b.parquet": 0}
+
+
+def test_datasource_defaults_to_parquet_file_chunker(tmp_path):
+    """``ParquetDatasourceV2`` plugs ``ParquetFileChunker`` into its indexer."""
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"a": [1, 2, 3]}))
+
+    datasource = ParquetDatasourceV2([str(file_path)])
+    indexer = datasource._get_file_indexer()
+    assert isinstance(indexer.file_chunker, ParquetFileChunker)
+
+
+def test_datasource_accepts_custom_chunker(tmp_path):
+    """An explicit ``file_chunker`` override propagates to the indexer."""
+    file_path = tmp_path / "data.parquet"
+    _write_parquet(str(file_path), pa.table({"a": [1, 2, 3]}))
+
+    custom = WholeFileChunker()
+    datasource = ParquetDatasourceV2([str(file_path)], file_chunker=custom)
+    indexer = datasource._get_file_indexer()
+    assert indexer.file_chunker is custom
+
+
+def _write_multi_row_group_parquet(path, num_rows: int, row_group_size: int):
+    table = pa.table({"id": list(range(num_rows))})
+    pq.write_table(table, path, row_group_size=row_group_size)
+    return table
+
+
+def test_fragments_from_chunk_metadata_subsets_by_row_group(tmp_path):
+    """``_fragments_from_chunk_metadata`` slices a fragment to the explicit range."""
+    import pyarrow.dataset as pds
+
+    file_path = str(tmp_path / "multi.parquet")
+    # 1000 rows, 100 row groups (row_group_size=10).
+    _write_multi_row_group_parquet(file_path, num_rows=1000, row_group_size=10)
+
+    dataset = pds.dataset(file_path, format="parquet")
+    (fragment,) = dataset.get_fragments()
+    assert fragment.metadata.num_row_groups == 100
+
+    # Explicit range [25, 50) → 25 row groups, starting row offset 250.
+    chunk_md = create_chunk_metadata(
+        ParquetFileChunkMetadata, row_group_start=25, row_group_end=50
+    )
+    sub_fragments = _fragments_from_chunk_metadata(fragment, chunk_md)
+    assert len(sub_fragments) == 25
+    expected_offset = 250  # 25 row groups × 10 rows each precede the range.
+    for sub, offset in sub_fragments:
+        assert len(sub.row_groups) == 1
+        assert offset == expected_offset
+        expected_offset += sub.metadata.row_group(sub.row_groups[0].id).num_rows
+
+
+def test_fragments_from_chunk_metadata_clamps_range_beyond_row_groups(tmp_path):
+    """A range beyond the file's actual row-group count is clamped (no crash)."""
+    import pyarrow.dataset as pds
+
+    file_path = str(tmp_path / "single.parquet")
+    # 5 rows, single row group.
+    _write_multi_row_group_parquet(file_path, num_rows=5, row_group_size=5)
+
+    dataset = pds.dataset(file_path, format="parquet")
+    (fragment,) = dataset.get_fragments()
+    assert fragment.metadata.num_row_groups == 1
+
+    # Fully out-of-range [5, 6) → clamped to [1, 1) → no sub-fragments.
+    chunk_md = create_chunk_metadata(
+        ParquetFileChunkMetadata, row_group_start=5, row_group_end=6
+    )
+    assert _fragments_from_chunk_metadata(fragment, chunk_md) == []
+
+    # Partially out-of-range [0, 9) → clamped to [0, 1) → the one real row group.
+    chunk_md = create_chunk_metadata(
+        ParquetFileChunkMetadata, row_group_start=0, row_group_end=9
+    )
+    sub_fragments = _fragments_from_chunk_metadata(fragment, chunk_md)
+    assert len(sub_fragments) == 1
+    assert sub_fragments[0][1] == 0  # row offset
+
+
+def _read_via_reader(reader, manifest):
+    return list(reader.read(manifest))
+
+
+def test_parquet_file_reader_reads_chunked_manifest(tmp_path):
+    """End-to-end: a manifest with per-chunk rows is read into the same rows
+    as a single whole-file manifest."""
+    file_path = str(tmp_path / "data.parquet")
+    expected_rows = 200
+    _write_multi_row_group_parquet(file_path, num_rows=expected_rows, row_group_size=20)
+    file_size = os.path.getsize(file_path)
+
+    reader_whole = ParquetFileReader()
+    whole_manifest = FileManifest.construct_manifest([file_path], [file_size], [None])
+    whole_tables = _read_via_reader(reader_whole, whole_manifest)
+    whole_rows = pa.concat_tables(whole_tables).column("id").to_pylist()
+
+    # target_chunk_size=1 forces one chunk per row group.
+    chunker = ParquetFileChunker(target_chunk_size=1)
+    chunks = list(chunker.generate_chunk_metadatas(file_path, file_size))
+    assert len(chunks) > 1, "test setup expects ParquetFileChunker to chunk"
+
+    paths = [file_path] * len(chunks)
+    chunk_metadatas = [md for md, _ in chunks]
+    chunk_sizes = [sz for _, sz in chunks]
+    chunked_manifest = FileManifest.construct_manifest(
+        paths, chunk_sizes, chunk_metadatas
+    )
+
+    reader_chunked = ParquetFileReader()
+    chunked_tables = _read_via_reader(reader_chunked, chunked_manifest)
+    chunked_rows = pa.concat_tables(chunked_tables).column("id").to_pylist()
+
+    assert sorted(chunked_rows) == sorted(whole_rows) == list(range(expected_rows))
+
+
+def test_parquet_file_reader_chunked_row_hashes_are_unique(tmp_path):
+    """Row hashes must remain unique across chunked sub-fragments of the
+    same file.
+
+    Regression: ``_read_fragments_sequential`` previously reseeded
+    ``offset=0`` for every fragment. Since chunked sub-fragments share
+    ``fragment.path``, ``_compute_row_hashes(path, 0, n)`` collided across
+    row groups of the same file.
+    """
+    file_path = str(tmp_path / "data.parquet")
+    expected_rows = 200
+    _write_multi_row_group_parquet(file_path, num_rows=expected_rows, row_group_size=20)
+    file_size = os.path.getsize(file_path)
+
+    chunker = ParquetFileChunker(target_chunk_size=1)
+    chunks = list(chunker.generate_chunk_metadatas(file_path, file_size))
+    assert len(chunks) > 1, "test setup expects ParquetFileChunker to chunk"
+
+    paths = [file_path] * len(chunks)
+    chunk_metadatas = [md for md, _ in chunks]
+    chunk_sizes = [sz for _, sz in chunks]
+    chunked_manifest = FileManifest.construct_manifest(
+        paths, chunk_sizes, chunk_metadatas
+    )
+
+    reader = ParquetFileReader(include_row_hash=True)
+    chunked_tables = list(reader.read(chunked_manifest))
+    hashes = pa.concat_tables(chunked_tables).column("row_hash").to_pylist()
+    assert len(hashes) == expected_rows
+    assert (
+        len(set(hashes)) == expected_rows
+    ), "row_hash must be unique across chunked sub-fragments of one file"
+
+
+def test_parquet_file_reader_handles_out_of_range_chunks(tmp_path):
+    """Defensively clamped out-of-range chunk metadata yields no rows, no crash.
+
+    The chunker never emits out-of-range ranges (they're computed from the
+    same footer the reader sees), but a hand-constructed range beyond the
+    file's row groups must be handled gracefully.
+    """
+    file_path = str(tmp_path / "tiny.parquet")
+    # 5 rows, single row group.
+    _write_multi_row_group_parquet(file_path, num_rows=5, row_group_size=5)
+    file_size = os.path.getsize(file_path)
+
+    # Explicit range entirely beyond the file's one row group.
+    out_of_range = create_chunk_metadata(
+        ParquetFileChunkMetadata, row_group_start=3, row_group_end=4
+    )
+    manifest = FileManifest.construct_manifest([file_path], [file_size], [out_of_range])
+
+    reader = ParquetFileReader()
+    tables = list(reader.read(manifest))
+    assert sum(t.num_rows for t in tables) == 0