chore: import upstream snapshot with attribution

python/ray/data/tests/unit/datasource_v2/test_file_chunker.py

@@ -0,0 +1,192 @@
+"""Unit tests for ``FileChunker`` implementations in DataSourceV2."""
+from pathlib import Path
+from typing import cast
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from ray.data._internal.datasource_v2.chunkers.file_chunker import (
+    ChunkMetadata,
+    LineDelimitedFileChunker,
+    LineDelimitedFileChunkMetadata,
+    ParquetFileChunker,
+    ParquetFileChunkMetadata,
+    WholeFileChunker,
+    create_chunk_metadata,
+)
+
+
+def _write_parquet_with_row_groups(
+    path: str, num_row_groups: int, rows_per_group: int = 10
+) -> int:
+    """Write a Parquet file with exactly ``num_row_groups`` row groups.
+
+    Returns the on-disk file size in bytes.
+    """
+    n = num_row_groups * rows_per_group
+    table = pa.table({"a": list(range(n))})
+    pq.write_table(table, path, row_group_size=rows_per_group)
+    return Path(path).stat().st_size
+
+
+class TestCreateChunkMetadata:
+    def test_validates_missing_keys(self):
+        with pytest.raises(ValueError, match="Missing required keys"):
+            create_chunk_metadata(ParquetFileChunkMetadata, row_group_start=0)
+
+    def test_validates_unexpected_keys(self):
+        with pytest.raises(ValueError, match="Unexpected keys"):
+            create_chunk_metadata(
+                ParquetFileChunkMetadata,
+                row_group_start=0,
+                row_group_end=1,
+                extra_field="boom",
+            )
+
+    def test_returns_dict_with_keys(self):
+        md = create_chunk_metadata(
+            ParquetFileChunkMetadata, row_group_start=2, row_group_end=5
+        )
+        assert md == {"row_group_start": 2, "row_group_end": 5}
+
+
+class TestWholeFileChunker:
+    def test_yields_single_none_chunk(self):
+        chunker = WholeFileChunker()
+        chunks = list(chunker.generate_chunk_metadatas("foo.bin", 12345))
+        assert chunks == [(None, 12345)]
+
+    def test_does_not_read_metadata(self):
+        assert WholeFileChunker.reads_file_metadata is False
+
+
+class TestLineDelimitedFileChunker:
+    def test_chunks_uncompressed_file(self):
+        chunker = LineDelimitedFileChunker()
+        # 600MB file at 256MB chunks -> 3 chunks (256, 256, 88).
+        chunks = list(chunker.generate_chunk_metadatas("data.jsonl", 600 * 1024 * 1024))
+        assert len(chunks) == 3
+        for i, (md, size) in enumerate(chunks):
+            assert md is not None
+            md = cast(LineDelimitedFileChunkMetadata, md)
+            assert md["chunk_byte_start_idx"] == i * 256 * 1024 * 1024
+            assert size == md["chunk_byte_end_idx"] - md["chunk_byte_start_idx"]
+        # Final chunk should clip to file_size.
+        last_md = cast(LineDelimitedFileChunkMetadata, chunks[-1][0])
+        assert last_md["chunk_byte_end_idx"] == 600 * 1024 * 1024
+
+    def test_compressed_file_yields_whole(self):
+        chunker = LineDelimitedFileChunker()
+        chunks = list(chunker.generate_chunk_metadatas("data.jsonl.gz", 1024))
+        assert chunks == [(None, 1024)]
+
+    def test_does_not_read_metadata(self):
+        assert LineDelimitedFileChunker.reads_file_metadata is False
+
+
+class TestParquetFileChunker:
+    def test_reads_file_metadata_flag(self):
+        assert ParquetFileChunker.reads_file_metadata is True
+
+    @pytest.mark.parametrize(
+        "target_chunk_size, expected_ranges",
+        [
+            # target < row-group size → K=1, one chunk per row group.
+            (1, [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)]),
+            # target ≫ file size → all row groups bundled into one chunk.
+            (10 * 1024**3, [(0, 5)]),
+        ],
+        ids=["target_below_rg_size", "target_above_file_size"],
+    )
+    def test_row_group_bundling_by_target(
+        self, tmp_path, target_chunk_size, expected_ranges
+    ):
+        p = str(tmp_path / "d.parquet")
+        size = _write_parquet_with_row_groups(p, num_row_groups=5)
+        chunker = ParquetFileChunker(target_chunk_size=target_chunk_size)
+        chunks = list(chunker.generate_chunk_metadatas(p, size))
+        ranges = [(m["row_group_start"], m["row_group_end"]) for m, _ in chunks]
+        assert ranges == expected_ranges
+
+    def test_chunk_size_equals_summed_row_group_bytes(self, tmp_path):
+        p = str(tmp_path / "d.parquet")
+        size = _write_parquet_with_row_groups(p, num_row_groups=4)
+        chunker = ParquetFileChunker(target_chunk_size=1)
+        chunks = list(chunker.generate_chunk_metadatas(p, size))
+        md = pq.read_metadata(p)
+        expected_total = sum(
+            md.row_group(i).column(c).total_compressed_size
+            for i in range(md.num_row_groups)
+            for c in range(md.row_group(i).num_columns)
+        )
+        assert sum(sz for _, sz in chunks) == expected_total
+
+    def test_ranges_are_contiguous_and_cover_all_row_groups(self, tmp_path):
+        """Bundled ranges partition [0, num_row_groups) with no gaps/overlap."""
+        p = str(tmp_path / "d.parquet")
+        # Many small row groups + a moderate target → some bundling.
+        size = _write_parquet_with_row_groups(p, num_row_groups=12, rows_per_group=5)
+        n = pq.read_metadata(p).num_row_groups
+        chunker = ParquetFileChunker(target_chunk_size=2_000)
+        chunks = list(chunker.generate_chunk_metadatas(p, size))
+        ranges = [(m["row_group_start"], m["row_group_end"]) for m, _ in chunks]
+        # Contiguous, starts at 0, ends at n, each non-empty.
+        assert ranges[0][0] == 0
+        assert ranges[-1][1] == n
+        for (_, prev_end), (next_start, _) in zip(ranges, ranges[1:]):
+            assert prev_end == next_start
+        assert all(start < end for start, end in ranges)
+
+    def test_corrupt_footer_falls_back_to_whole_file(self, tmp_path):
+        p = str(tmp_path / "bad.parquet")
+        Path(p).write_bytes(b"this is not a parquet file")
+        chunker = ParquetFileChunker(target_chunk_size=1)
+        chunks = list(chunker.generate_chunk_metadatas(p, 26))
+        assert chunks == [(None, 26)]
+
+    @pytest.mark.parametrize(
+        "ctor_arg, ctx_chunk_size, ctx_min_block, expected",
+        [
+            # ctor arg wins over the context knobs.
+            (2048, 1024, 7777, 2048),
+            # An explicit 0 is honored (resolved with ``is not None``, not
+            # ``or``), so it isn't silently treated as "unset" and overridden.
+            (0, 1024, 7777, 0),
+            # ctx chunk-size knob used when there's no ctor arg.
+            (None, 1024, 7777, 1024),
+            # Falls back to target_min_block_size when the chunk-size knob is unset.
+            (None, None, 7777, 7777),
+        ],
+        ids=["ctor_arg", "ctor_arg_zero", "ctx_knob", "fallback_min_block"],
+    )
+    def test_target_chunk_size_resolution(
+        self, restore_data_context, ctor_arg, ctx_chunk_size, ctx_min_block, expected
+    ):
+        from ray.data.context import DataContext
+
+        ctx = DataContext.get_current()
+        ctx.parquet_chunker_target_chunk_size = ctx_chunk_size
+        ctx.target_min_block_size = ctx_min_block
+        chunker = ParquetFileChunker(target_chunk_size=ctor_arg)
+        assert chunker._target_chunk_size == expected
+
+
+def test_chunk_metadata_subclasses_are_typeddicts():
+    # Ensures the subclasses don't accidentally inherit unrelated keys.
+    pmd: ChunkMetadata = create_chunk_metadata(
+        ParquetFileChunkMetadata, row_group_start=0, row_group_end=1
+    )
+    lmd: ChunkMetadata = create_chunk_metadata(
+        LineDelimitedFileChunkMetadata,
+        chunk_byte_start_idx=0,
+        chunk_byte_end_idx=10,
+    )
+    assert set(pmd.keys()) == {"row_group_start", "row_group_end"}
+    assert set(lmd.keys()) == {"chunk_byte_start_idx", "chunk_byte_end_idx"}
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/data/tests/unit/datasource_v2/test_file_indexer.py

@@ -0,0 +1,352 @@
+import os
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+from pyarrow.fs import LocalFileSystem
+
+from ray.data._internal.datasource_v2.chunkers.file_chunker import (
+    FileChunker,
+    LineDelimitedFileChunker,
+    ParquetFileChunker,
+    WholeFileChunker,
+)
+from ray.data._internal.datasource_v2.listing.file_indexer import (
+    FileInfo,
+    NonSamplingFileIndexer,
+)
+from ray.data._internal.datasource_v2.listing.file_pruners import FileExtensionPruner
+
+
+class _CountingChunker(FileChunker):
+    """Fake chunker that emits a fixed number of chunks per file and reports it
+    reads metadata, so the indexer takes the threaded fan-out path."""
+
+    reads_file_metadata = True
+
+    def __init__(self, chunks_per_file: int):
+        self._chunks_per_file = chunks_per_file
+
+    def generate_chunk_metadatas(self, path, file_size, filesystem=None):
+        for i in range(self._chunks_per_file):
+            yield {"chunk_index": i}, file_size
+
+
+def _list_all(indexer, paths, **kwargs):
+    """Run list_files and flatten all manifests into (path, size) pairs."""
+    pa_paths = pa.array(paths)
+    fs = LocalFileSystem()
+    manifests = list(indexer.list_files(pa_paths, filesystem=fs, **kwargs))
+    results = []
+    for m in manifests:
+        for p, s in zip(m.paths, m.file_sizes):
+            results.append((str(p), int(s)))
+    return sorted(results)
+
+
+@pytest.fixture(params=[1, 2], ids=["sequential", "threaded"])
+def indexer(request):
+    """Yield a NonSamplingFileIndexer using sequential or threaded listing."""
+    return NonSamplingFileIndexer(ignore_missing_paths=False, num_workers=request.param)
+
+
+class TestListFiles:
+    def test_single_file(self, tmp_path, indexer):
+        f = tmp_path / "data.csv"
+        f.write_bytes(b"x" * 42)
+
+        results = _list_all(indexer, [str(f)])
+        assert results == [(str(f), 42)]
+
+    def test_directory(self, tmp_path, indexer):
+        for name in ["a.csv", "b.csv", "c.csv"]:
+            (tmp_path / name).write_bytes(b"x" * 10)
+
+        results = _list_all(indexer, [str(tmp_path)])
+        assert len(results) == 3
+        assert all(size == 10 for _, size in results)
+
+    def test_nested_directories(self, tmp_path, indexer):
+        (tmp_path / "top.csv").write_bytes(b"x" * 100)
+        (tmp_path / "sub").mkdir()
+        (tmp_path / "sub" / "nested.csv").write_bytes(b"x" * 100)
+        (tmp_path / "sub" / "deep").mkdir()
+        (tmp_path / "sub" / "deep" / "leaf.csv").write_bytes(b"x" * 100)
+
+        results = _list_all(indexer, [str(tmp_path)])
+        assert len(results) == 3
+        basenames = sorted(os.path.basename(p) for p, _ in results)
+        assert basenames == ["leaf.csv", "nested.csv", "top.csv"]
+
+    def test_multiple_paths(self, tmp_path, indexer):
+        f1 = tmp_path / "one.csv"
+        f2 = tmp_path / "two.csv"
+        f1.write_bytes(b"x" * 10)
+        f2.write_bytes(b"x" * 20)
+
+        results = _list_all(indexer, [str(f1), str(f2)])
+        assert sorted(results) == [(str(f1), 10), (str(f2), 20)]
+
+    @pytest.mark.parametrize(
+        "filename",
+        [".hidden", "_metadata", "_SUCCESS", ".gitignore"],
+        ids=["dot-prefix", "underscore-prefix", "underscore-upper", "dotfile"],
+    )
+    def test_excludes_hidden_and_metadata_files(self, tmp_path, indexer, filename):
+        (tmp_path / filename).write_bytes(b"x" * 100)
+        (tmp_path / "visible.csv").write_bytes(b"x" * 100)
+
+        results = _list_all(indexer, [str(tmp_path)])
+        assert len(results) == 1
+        assert os.path.basename(results[0][0]) == "visible.csv"
+
+    @pytest.mark.parametrize(
+        "filename",
+        ["_metadata", "_my_file.csv", ".hidden_data"],
+        ids=["underscore-metadata", "underscore-csv", "dot-hidden"],
+    )
+    def test_includes_excluded_prefix_files_in_subdirectories(
+        self, tmp_path, indexer, filename
+    ):
+        """Files whose names start with _ or . should only be excluded when
+        they appear at the top level of the listed directory, not when they
+        appear inside a subdirectory. The relative path from the root is
+        e.g. "subdir/_metadata" which starts with "s", not "_"."""
+        sub = tmp_path / "subdir"
+        sub.mkdir()
+        (sub / filename).write_bytes(b"x" * 50)
+        (sub / "normal.csv").write_bytes(b"x" * 50)
+
+        results = _list_all(indexer, [str(tmp_path)])
+        basenames = sorted(os.path.basename(p) for p, _ in results)
+        assert filename in basenames
+        assert "normal.csv" in basenames
+
+    def test_skips_zero_size_files(self, tmp_path, indexer):
+        (tmp_path / "empty.csv").write_bytes(b"")
+        (tmp_path / "real.csv").write_bytes(b"x" * 50)
+
+        results = _list_all(indexer, [str(tmp_path)])
+        assert len(results) == 1
+        assert os.path.basename(results[0][0]) == "real.csv"
+
+    def test_empty_directory(self, tmp_path, indexer):
+        os.makedirs(tmp_path / "empty_dir", exist_ok=True)
+        results = _list_all(indexer, [str(tmp_path / "empty_dir")])
+        assert results == []
+
+
+class TestPruners:
+    @pytest.fixture
+    def indexer(self):
+        return NonSamplingFileIndexer(ignore_missing_paths=False, num_workers=1)
+
+    @pytest.mark.parametrize(
+        "extensions, expected_basenames",
+        [
+            (["csv"], ["a.csv"]),
+            (["json"], ["b.json"]),
+            (["csv", "json"], ["a.csv", "b.json"]),
+            (["parquet"], []),
+        ],
+        ids=["csv-only", "json-only", "csv-and-json", "no-match"],
+    )
+    def test_extension_pruner(self, tmp_path, indexer, extensions, expected_basenames):
+        (tmp_path / "a.csv").write_bytes(b"x" * 100)
+        (tmp_path / "b.json").write_bytes(b"x" * 100)
+        (tmp_path / "c.txt").write_bytes(b"x" * 100)
+
+        pruner = FileExtensionPruner(extensions)
+        results = _list_all(indexer, [str(tmp_path)], pruners=[pruner])
+        basenames = sorted(os.path.basename(p) for p, _ in results)
+        assert basenames == sorted(expected_basenames)
+
+    def test_multiple_pruners_intersect(self, tmp_path, indexer):
+        """Multiple pruners are AND'd — a file must pass all of them."""
+        (tmp_path / "a.csv").write_bytes(b"x" * 100)
+        (tmp_path / "b.json").write_bytes(b"x" * 100)
+
+        pruner_csv = FileExtensionPruner(["csv", "json"])
+        pruner_json = FileExtensionPruner(["json"])
+        results = _list_all(indexer, [str(tmp_path)], pruners=[pruner_csv, pruner_json])
+        basenames = [os.path.basename(p) for p, _ in results]
+        assert basenames == ["b.json"]
+
+
+class TestMissingPaths:
+    def test_raises_on_missing_path(self, tmp_path):
+        indexer = NonSamplingFileIndexer(ignore_missing_paths=False)
+        missing = str(tmp_path / "nonexistent")
+
+        with pytest.raises(FileNotFoundError):
+            _list_all(indexer, [missing])
+
+    def test_ignores_missing_path(self, tmp_path):
+        indexer = NonSamplingFileIndexer(ignore_missing_paths=True)
+        missing = str(tmp_path / "nonexistent")
+
+        results = _list_all(indexer, [missing])
+        assert results == []
+
+    def test_mixed_existing_and_missing(self, tmp_path):
+        indexer = NonSamplingFileIndexer(ignore_missing_paths=True)
+        real = tmp_path / "real.csv"
+        real.write_bytes(b"x" * 10)
+        missing = str(tmp_path / "gone")
+
+        results = _list_all(indexer, [str(real), missing])
+        assert results == [(str(real), 10)]
+
+
+class TestManifestBatching:
+    def test_splits_into_multiple_manifests(self, tmp_path):
+        indexer = NonSamplingFileIndexer(
+            ignore_missing_paths=False, max_paths_per_output=3
+        )
+
+        for i in range(7):
+            (tmp_path / f"file_{i}.csv").write_bytes(b"x" * 100)
+
+        pa_paths = pa.array([str(tmp_path)])
+        fs = LocalFileSystem()
+        manifests = list(indexer.list_files(pa_paths, filesystem=fs))
+
+        assert len(manifests) == 3  # ceil(7/3)
+        assert len(manifests[0]) == 3
+        assert len(manifests[1]) == 3
+        assert len(manifests[2]) == 1
+
+        total_files = sum(len(m) for m in manifests)
+        assert total_files == 7
+
+
+class TestFileChunkerIntegration:
+    """Cover ``NonSamplingFileIndexer`` interaction with a ``FileChunker``."""
+
+    def test_default_uses_whole_file_chunker(self):
+        indexer = NonSamplingFileIndexer(ignore_missing_paths=False)
+        assert isinstance(indexer.file_chunker, WholeFileChunker)
+
+    def test_explicit_chunker_is_exposed(self):
+        chunker = ParquetFileChunker(target_chunk_size=1024)
+        indexer = NonSamplingFileIndexer(
+            ignore_missing_paths=False, file_chunker=chunker
+        )
+        assert indexer.file_chunker is chunker
+
+    def test_whole_file_chunker_yields_none_chunk_metadata(self, tmp_path):
+        (tmp_path / "a.csv").write_bytes(b"x" * 100)
+        indexer = NonSamplingFileIndexer(ignore_missing_paths=False, num_workers=1)
+        fs = LocalFileSystem()
+        manifests = list(indexer.list_files(pa.array([str(tmp_path)]), filesystem=fs))
+        assert len(manifests) == 1
+        manifest = manifests[0]
+        assert len(manifest) == 1
+        # ``WholeFileChunker`` emits one ``None`` chunk per file.
+        assert list(manifest.file_chunk_metadatas) == [None]
+        assert list(manifest.file_sizes) == [100]
+
+    def test_parquet_chunker_splits_file_on_row_group_boundaries(self, tmp_path):
+        # A real Parquet file with 8 row groups; the chunker reads the footer
+        # at listing time and (target < row-group size) emits one chunk per
+        # row group with an explicit half-open range.
+        table = pa.table({"a": list(range(80))})
+        pq.write_table(table, str(tmp_path / "big.parquet"), row_group_size=10)
+        chunker = ParquetFileChunker(target_chunk_size=1)
+        indexer = NonSamplingFileIndexer(
+            ignore_missing_paths=False,
+            num_workers=1,
+            file_chunker=chunker,
+        )
+        fs = LocalFileSystem()
+        manifests = list(indexer.list_files(pa.array([str(tmp_path)]), filesystem=fs))
+        rows = []
+        for m in manifests:
+            for path, size, md in zip(m.paths, m.file_sizes, m.file_chunk_metadatas):
+                rows.append((str(path), int(size), md))
+
+        # 8 row groups → 8 chunks, contiguous half-open ranges.
+        assert len(rows) == 8
+        ranges = [(md["row_group_start"], md["row_group_end"]) for _, _, md in rows]
+        assert ranges == [(i, i + 1) for i in range(8)]
+
+    def test_parquet_chunker_parallel_footer_reads(self, tmp_path):
+        # With num_workers > 1 the chunker's footer reads fan across the
+        # thread pool (reads_file_metadata=True). Verify correctness is
+        # unaffected: every file's row groups are represented exactly once.
+        for f in range(4):
+            table = pa.table({"a": list(range(30))})
+            pq.write_table(table, str(tmp_path / f"f{f}.parquet"), row_group_size=10)
+        chunker = ParquetFileChunker(target_chunk_size=1)
+        indexer = NonSamplingFileIndexer(
+            ignore_missing_paths=False,
+            num_workers=4,
+            file_chunker=chunker,
+        )
+        fs = LocalFileSystem()
+        manifests = list(indexer.list_files(pa.array([str(tmp_path)]), filesystem=fs))
+        per_path_ranges = {}
+        for m in manifests:
+            for path, _, md in zip(m.paths, m.file_sizes, m.file_chunk_metadatas):
+                per_path_ranges.setdefault(str(path), []).append(
+                    (md["row_group_start"], md["row_group_end"])
+                )
+        # 4 files × 3 row groups each.
+        assert len(per_path_ranges) == 4
+        for ranges in per_path_ranges.values():
+            assert sorted(ranges) == [(0, 1), (1, 2), (2, 3)]
+
+    def test_line_delimited_chunker_byte_ranges(self, tmp_path):
+        (tmp_path / "a.jsonl").write_bytes(b"x" * 10_000)
+        chunker = LineDelimitedFileChunker()
+        # Force smaller chunks via a private override so the unit test
+        # doesn't need a 256 MB file on disk.
+        chunker._CHUNK_BYTE_SIZE = 1024
+        indexer = NonSamplingFileIndexer(
+            ignore_missing_paths=False,
+            num_workers=1,
+            file_chunker=chunker,
+        )
+        fs = LocalFileSystem()
+        manifests = list(indexer.list_files(pa.array([str(tmp_path)]), filesystem=fs))
+        rows = []
+        for m in manifests:
+            for path, size, md in zip(m.paths, m.file_sizes, m.file_chunk_metadatas):
+                rows.append((str(path), int(size), md))
+        assert len(rows) == 10
+        # Byte ranges must tile the file exactly.
+        assert rows[0][2]["chunk_byte_start_idx"] == 0
+        assert rows[-1][2]["chunk_byte_end_idx"] == 10_000
+
+    def test_parallel_chunking_preserves_discovery_order(self):
+        """Regression: with num_workers>1 and preserve_order, a multi-chunk-per-
+        file chunker's records stay grouped per file in discovery order.
+
+        ``make_async_gen`` only preserves order for a 1:1 map, so the indexer
+        must emit one record list per file (not yield chunk rows individually) --
+        otherwise its round-robin merge interleaves chunks across the files
+        processed concurrently, e.g. f0,f1,f2,f3,f0,f1,... instead of f0,f0,f0,...
+        """
+        chunker = _CountingChunker(chunks_per_file=3)
+        indexer = NonSamplingFileIndexer(
+            ignore_missing_paths=False, num_workers=4, file_chunker=chunker
+        )
+        file_infos = [FileInfo(path=f"f{i}.parquet", size=100 + i) for i in range(8)]
+        records = list(
+            indexer._generate_chunk_records(
+                file_infos, filesystem=None, preserve_order=True
+            )
+        )
+        # 8 files x 3 chunks each, contiguous per file in discovery order.
+        assert [path for path, _, _ in records] == [
+            f"f{i}.parquet" for i in range(8) for _ in range(3)
+        ]
+        assert [md["chunk_index"] for _, _, md in records] == [
+            i for _ in range(8) for i in range(3)
+        ]
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/data/tests/unit/datasource_v2/test_list_files_op.py

@@ -0,0 +1,78 @@
+"""Unit tests for :class:`ListFiles` logical op.
+
+Full physical-planning tests live in the CI parquet regression suite
+(they need Ray initialized for ``ray.put`` on the listing input
+bundles). Here we exercise just the logical op shape and the shuffle
+factory semantics.
+"""
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from ray.data._internal.datasource_v2.listing.file_indexer import (
+    NonSamplingFileIndexer,
+)
+from ray.data._internal.datasource_v2.listing.file_manifest import (
+    FILE_SIZE_COLUMN_NAME,
+    PATH_COLUMN_NAME,
+)
+from ray.data._internal.logical.operators import ListFiles
+from ray.data.datasource.file_based_datasource import FileShuffleConfig
+
+
+def _mk_indexer():
+    return NonSamplingFileIndexer(ignore_missing_paths=False)
+
+
+def _mk_list_files(tmp_path, num_files: int = 3, shuffle_seed=None):
+    for i in range(num_files):
+        pq.write_table(pa.table({"x": [i]}), str(tmp_path / f"f{i}.parquet"))
+    paths = [str(tmp_path / f"f{i}.parquet") for i in range(num_files)]
+
+    def _shuffle_factory():
+        if shuffle_seed is None:
+            return None
+        return FileShuffleConfig(seed=shuffle_seed)
+
+    import pyarrow.fs as pafs
+
+    return ListFiles(
+        paths=paths,
+        file_indexer=_mk_indexer(),
+        filesystem=pafs.LocalFileSystem(),
+        source_paths=paths,
+        shuffle_config_factory=_shuffle_factory,
+    )
+
+
+def test_list_files_infers_manifest_schema(tmp_path):
+    op = _mk_list_files(tmp_path, num_files=1)
+    schema = op.infer_schema()
+    assert schema.names == [PATH_COLUMN_NAME, FILE_SIZE_COLUMN_NAME]
+    assert schema.field(PATH_COLUMN_NAME).type == pa.string()
+    assert schema.field(FILE_SIZE_COLUMN_NAME).type == pa.int64()
+
+
+def test_list_files_has_no_input_dependencies(tmp_path):
+    op = _mk_list_files(tmp_path, num_files=1)
+    assert op.input_dependencies == []
+    assert op.num_outputs is None
+    assert op.output_data() is None
+
+
+def test_shuffle_config_factory_none_when_unconfigured(tmp_path):
+    op = _mk_list_files(tmp_path, num_files=1, shuffle_seed=None)
+    assert op.shuffle_config_factory() is None
+
+
+def test_shuffle_config_factory_returns_config_when_seeded(tmp_path):
+    op = _mk_list_files(tmp_path, num_files=1, shuffle_seed=42)
+    config = op.shuffle_config_factory()
+    assert isinstance(config, FileShuffleConfig)
+    assert config.seed == 42
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main([__file__, "-xvs"]))