chore: import upstream snapshot with attribution

doc/source/data/api/_autogen.rst

@@ -0,0 +1,24 @@
+:orphan:
+
+.. # This file is only used to auto-generate API docs.
+.. # It should not be included in the toctree.
+.. #
+.. # For any classes that you want to include in the
+.. # API docs, add them to the list of autosummary
+.. # below, then include the generated ray.data.<class>.rst
+.. # file in your top level rst file.
+
+.. currentmodule:: ray.data
+
+.. autosummary::
+    :nosignatures:
+    :template: autosummary/class_v2.rst
+    :toctree:
+
+    DataIterator
+    Dataset
+    Schema
+    stats.DatasetSummary
+    grouped_data.GroupedData
+    aggregate.AggregateFn
+    aggregate.AggregateFnV2

doc/source/data/api/aggregate.rst

@@ -0,0 +1,34 @@
+.. _aggregations_api_ref:
+
+Aggregation API
+===============
+
+Pass :class:`AggregateFnV2 <ray.data.aggregate.AggregateFnV2>` objects to
+:meth:`Dataset.aggregate() <ray.data.Dataset.aggregate>` or 
+:meth:`Dataset.groupby().aggregate() <ray.data.grouped_data.GroupedData.aggregate>` to 
+compute aggregations.
+
+.. currentmodule:: ray.data.aggregate
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    AggregateFnV2
+    AggregateFn
+    Count
+    Sum
+    Min
+    Max
+    Mean
+    Std
+    AbsMax
+    Quantile
+    Unique
+    AsList
+    CountDistinct
+    ValueCounter
+    MissingValuePercentage
+    ZeroPercentage
+    ApproximateQuantile
+    ApproximateTopK

doc/source/data/api/api.rst

@@ -0,0 +1,22 @@
+.. _data-api:
+
+Ray Data API
+================
+
+.. toctree::
+    :maxdepth: 2
+
+    loading_data.rst
+    saving_data.rst
+    dataset.rst
+    data_iterator.rst
+    execution_options.rst
+    checkpoint.rst
+    aggregate.rst
+    grouped_data.rst
+    expressions.rst
+    datatype.rst
+    data_context.rst
+    preprocessor.rst
+    llm.rst
+    from_other_data_libs.rst

doc/source/data/api/checkpoint.rst

@@ -0,0 +1,18 @@
+.. _checkpoint-api:
+
+Checkpoint API
+==============
+
+.. currentmodule:: ray.data.checkpoint.interfaces
+
+Configuration
+-------------
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+   :template: autosummary/class_without_autosummary.rst
+
+   CheckpointConfig
+   CheckpointBackend
+

doc/source/data/api/data_context.rst

@@ -0,0 +1,17 @@
+.. _data-context-api:
+
+Global configuration
+====================
+
+.. currentmodule:: ray.data.context
+
+.. autoclass:: DataContext
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   DataContext.get_current
+
+
+.. autoclass:: AutoscalingConfig

doc/source/data/api/data_iterator.rst

@@ -0,0 +1,6 @@
+.. _dataset-iterator-api:
+
+DataIterator API
+================
+
+.. include:: ray.data.DataIterator.rst

doc/source/data/api/dataset.rst

@@ -0,0 +1,68 @@
+.. _dataset-api:
+
+Dataset API
+==============
+
+.. include:: ray.data.Dataset.rst
+
+Compute Strategy API
+--------------------
+.. currentmodule:: ray.data
+.. autosummary::
+  :nosignatures:
+  :toctree: doc/
+
+  ActorPoolStrategy
+  TaskPoolStrategy
+
+Mixing API
+----------
+.. currentmodule:: ray.data
+
+.. autoclass:: MixStoppingCondition
+    :members:
+    :undoc-members:
+
+Schema
+------
+.. currentmodule:: ray.data
+
+.. autoclass:: Schema
+    :members:
+
+DatasetSummary
+--------------
+.. currentmodule:: ray.data.stats
+
+.. autoclass:: DatasetSummary
+    :members:
+
+Developer API
+-------------
+
+.. currentmodule:: ray.data
+
+.. autosummary::
+  :nosignatures:
+  :toctree: doc/
+
+  Dataset.to_pandas_refs
+  Dataset.to_numpy_refs
+  Dataset.to_arrow_refs
+  Dataset.iter_internal_ref_bundles
+  block.Block
+  block.BlockExecStats
+  block.BlockMetadata
+  block.BlockAccessor
+  RandomSeedConfig
+
+Deprecated API
+--------------
+
+.. currentmodule:: ray.data
+
+.. autosummary::
+  :nosignatures:
+  :toctree: doc/
+
+  Dataset.iter_tf_batches

doc/source/data/api/datatype.rst

@@ -0,0 +1,18 @@
+.. _datatype-api:
+
+Data types
+==========
+
+.. currentmodule:: ray.data.datatype
+
+Class
+-----
+
+.. autoclass:: DataType
+    :members:
+
+Enumeration
+-----------
+
+.. autoclass:: TypeCategory
+    :members:

doc/source/data/api/execution_options.rst

@@ -0,0 +1,26 @@
+.. _execution-options-api:
+
+ExecutionOptions API
+====================
+
+.. currentmodule:: ray.data
+
+Constructor
+-----------
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+   :template: autosummary/class_without_autosummary.rst
+
+   ExecutionOptions
+
+Resource Options
+----------------
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+   :template: autosummary/class_without_autosummary.rst
+
+   ExecutionResources

doc/source/data/api/expressions.rst

@@ -0,0 +1,218 @@
+.. _expressions-api:
+
+Expressions API
+================
+
+.. currentmodule:: ray.data.expressions
+
+Expressions provide a way to specify column-based operations on datasets.
+Use :func:`col` to reference columns and :func:`lit` to create literal values.
+You can combine these with operators to create complex expressions for filtering,
+transformations, and computations.
+
+Public API
+----------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    star
+    col
+    lit
+    udf
+    pyarrow_udf
+    download
+    monotonically_increasing_id
+    random
+    uuid
+
+Expression Classes
+------------------
+
+These classes represent the structure of expressions. You typically don't need to
+instantiate them directly, but you may encounter them when working with expressions.
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    Expr
+    ColumnExpr
+    LiteralExpr
+    BinaryExpr
+    UnaryExpr
+    UDFExpr
+    StarExpr
+    DownloadExpr
+    MonotonicallyIncreasingIdExpr
+    RandomExpr
+    UUIDExpr
+
+Expression namespaces
+------------------------------------
+
+These namespace classes provide specialized operations for list, string, struct, array, and
+`datetime` columns. You access them through properties on expressions: ``.list``, ``.str``,
+``.struct``, ``.arr``, and ``.dt``.
+
+The following example shows how to use the string namespace to transform text columns:
+
+.. testcode::
+
+    import ray
+    from ray.data.expressions import col
+
+    # Create a dataset with a text column
+    ds = ray.data.from_items([
+        {"name": "alice"},
+        {"name": "bob"},
+        {"name": "charlie"}
+    ])
+
+    # Use the string namespace to uppercase the names
+    ds = ds.with_column("upper_name", col("name").str.upper())
+    ds.show()
+
+.. testoutput::
+
+    {'name': 'alice', 'upper_name': 'ALICE'}
+    {'name': 'bob', 'upper_name': 'BOB'}
+    {'name': 'charlie', 'upper_name': 'CHARLIE'}
+
+The following example demonstrates using the list namespace to work with array columns:
+
+.. testcode::
+
+    import ray
+    from ray.data.expressions import col
+
+    # Create a dataset with list columns
+    ds = ray.data.from_items([
+        {"scores": [85, 90, 78]},
+        {"scores": [92, 88]},
+        {"scores": [76, 82, 88, 91]}
+    ])
+
+    # Use the list namespace to get the length of each list
+    ds = ds.with_column("num_scores", col("scores").list.len())
+    ds.show()
+
+.. testoutput::
+
+    {'scores': [85, 90, 78], 'num_scores': 3}
+    {'scores': [92, 88], 'num_scores': 2}
+    {'scores': [76, 82, 88, 91], 'num_scores': 4}
+
+You can also perform list-specific transformations like sorting and flattening:
+
+.. testcode::
+
+    import ray
+    from ray.data.expressions import col
+
+    ds = ray.data.from_items([
+        {"values": [3, 1, 2], "nested": [[1, 2], [3]]},
+        {"values": [2, None, 5], "nested": [[4], []]}
+    ])
+
+    ds = ds.with_column(
+        "sorted_values", col("values").list.sort(order="descending")
+    )
+    ds = ds.with_column(
+        "flattened_nested", col("nested").list.flatten()
+    )
+    ds.show()
+
+.. testoutput::
+
+    {'values': [3, 1, 2], 'nested': [[1, 2], [3]], 'sorted_values': [3, 2, 1], 'flattened_nested': [1, 2, 3]}
+    {'values': [2, None, 5], 'nested': [[4], []], 'sorted_values': [5, 2, None], 'flattened_nested': [4]}
+
+The following example shows how to use the struct namespace to access nested fields:
+
+.. testcode::
+
+    import ray
+    from ray.data.expressions import col
+
+    # Create a dataset with struct columns
+    ds = ray.data.from_items([
+        {"user": {"name": "alice", "age": 25}},
+        {"user": {"name": "bob", "age": 30}},
+        {"user": {"name": "charlie", "age": 35}}
+    ])
+
+    # Use the struct namespace to extract a specific field
+    ds = ds.with_column("user_name", col("user").struct.field("name"))
+    ds.show()
+
+.. testoutput::
+
+    {'user': {'name': 'alice', 'age': 25}, 'user_name': 'alice'}
+    {'user': {'name': 'bob', 'age': 30}, 'user_name': 'bob'}
+    {'user': {'name': 'charlie', 'age': 35}, 'user_name': 'charlie'}
+
+The following example shows how to use the array namespace to convert fixed-size
+list columns to variable-length lists:
+
+.. testcode::
+
+    import pyarrow as pa
+    import ray
+    from ray.data.expressions import col
+
+    values = pa.array([1, 2, 3, 4])
+    fixed = pa.FixedSizeListArray.from_arrays(values, 2)
+    table = pa.table({"features": fixed})
+
+    ds = ray.data.from_arrow(table)
+    ds = ds.with_column("features_list", col("features").arr.to_list())
+    ds.show()
+
+.. testoutput::
+
+    {'features': [1, 2], 'features_list': [1, 2]}
+    {'features': [3, 4], 'features_list': [3, 4]}
+
+The following example shows how to use the `datetime` namespace to extract components:
+
+.. testcode::
+
+    import datetime
+    import pandas as pd
+    import ray
+    from ray.data.expressions import col
+
+    ds = ray.data.from_items([
+        {"ts": pd.Timestamp("2024-01-02 03:04:05")},
+        {"ts": pd.Timestamp("2024-02-03 04:05:06")}
+    ])
+
+    ds = ds.with_column("year", col("ts").dt.year())
+    ds.show()
+
+.. testoutput::
+
+    {'ts': datetime.datetime(2024, 1, 2, 3, 4, 5), 'year': 2024}
+    {'ts': datetime.datetime(2024, 2, 3, 4, 5, 6), 'year': 2024}
+
+.. autoclass:: _ListNamespace
+    :members:
+    :exclude-members: _expr
+
+.. autoclass:: _StringNamespace
+    :members:
+    :exclude-members: _expr
+
+.. autoclass:: _StructNamespace
+    :members:
+    :exclude-members: _expr
+
+.. autoclass:: _ArrayNamespace
+    :members:
+    :exclude-members: _expr
+
+.. autoclass:: _DatetimeNamespace
+    :members:
+    :exclude-members: _expr

doc/source/data/api/from_other_data_libs.rst

@@ -0,0 +1,93 @@
+.. _api-guide-for-users-from-other-data-libs:
+
+API Guide for Users from Other Data Libraries
+=============================================
+
+Ray Data is a data loading and preprocessing library for ML. It shares certain
+similarities with other ETL data processing libraries, but also has its own focus.
+This guide provides API mappings for users who come from those data
+libraries, so you can quickly map what you may already know to Ray Data APIs.
+
+.. note::
+
+  - This is meant to map APIs that perform comparable but not necessarily identical operations.
+    Select the API reference for exact semantics and usage.
+  - This list may not be exhaustive: It focuses on common APIs or APIs that are less obvious to see a connection.
+
+.. _api-guide-for-pandas-users:
+
+For Pandas Users
+----------------
+
+.. list-table:: Pandas DataFrame vs. Ray Data APIs
+   :header-rows: 1
+
+   * - Pandas DataFrame API
+     - Ray Data API
+   * - df.head()
+     - :meth:`ds.show() <ray.data.Dataset.show>`, :meth:`ds.take() <ray.data.Dataset.take>`, or :meth:`ds.take_batch() <ray.data.Dataset.take_batch>`
+   * - df.dtypes
+     - :meth:`ds.schema() <ray.data.Dataset.schema>`
+   * - len(df) or df.shape[0]
+     - :meth:`ds.count() <ray.data.Dataset.count>`
+   * - df.truncate()
+     - :meth:`ds.limit() <ray.data.Dataset.limit>`
+   * - df.iterrows()
+     - :meth:`ds.iter_rows() <ray.data.Dataset.iter_rows>`
+   * - df.drop()
+     - :meth:`ds.drop_columns() <ray.data.Dataset.drop_columns>`
+   * - df.transform()
+     - :meth:`ds.map_batches() <ray.data.Dataset.map_batches>` or :meth:`ds.map() <ray.data.Dataset.map>`
+   * - df.groupby()
+     - :meth:`ds.groupby() <ray.data.Dataset.groupby>`
+   * - df.groupby().apply()
+     - :meth:`ds.groupby().map_groups() <ray.data.grouped_data.GroupedData.map_groups>`
+   * - df.sample()
+     - :meth:`ds.random_sample() <ray.data.Dataset.random_sample>`
+   * - df.sort_values()
+     - :meth:`ds.sort() <ray.data.Dataset.sort>`
+   * - df.append()
+     - :meth:`ds.union() <ray.data.Dataset.union>`
+   * - df.aggregate()
+     - :meth:`ds.aggregate() <ray.data.Dataset.aggregate>`
+   * - df.min()
+     - :meth:`ds.min() <ray.data.Dataset.min>`
+   * - df.max()
+     - :meth:`ds.max() <ray.data.Dataset.max>`
+   * - df.sum()
+     - :meth:`ds.sum() <ray.data.Dataset.sum>`
+   * - df.mean()
+     - :meth:`ds.mean() <ray.data.Dataset.mean>`
+   * - df.std()
+     - :meth:`ds.std() <ray.data.Dataset.std>`
+
+.. _api-guide-for-pyarrow-users:
+
+For PyArrow Users
+-----------------
+
+.. list-table:: PyArrow Table vs. Ray Data APIs
+   :header-rows: 1
+
+   * - PyArrow Table API
+     - Ray Data API
+   * - ``pa.Table.schema``
+     - :meth:`ds.schema() <ray.data.Dataset.schema>`
+   * - ``pa.Table.num_rows``
+     - :meth:`ds.count() <ray.data.Dataset.count>`
+   * - ``pa.Table.filter()``
+     - :meth:`ds.filter() <ray.data.Dataset.filter>`
+   * - ``pa.Table.drop()``
+     - :meth:`ds.drop_columns() <ray.data.Dataset.drop_columns>`
+   * - ``pa.Table.add_column()``
+     - :meth:`ds.with_column() <ray.data.Dataset.with_column>`
+   * - ``pa.Table.groupby()``
+     - :meth:`ds.groupby() <ray.data.Dataset.groupby>`
+   * - ``pa.Table.sort_by()``
+     - :meth:`ds.sort() <ray.data.Dataset.sort>`
+
+
+For PyTorch Dataset & DataLoader Users
+--------------------------------------
+
+For more details, see the :ref:`Migrating from PyTorch to Ray Data <migrate_pytorch>`.

doc/source/data/api/grouped_data.rst

@@ -0,0 +1,11 @@
+.. _grouped-dataset-api:
+
+GroupedData API
+===============
+
+.. currentmodule:: ray.data
+
+The groupby call returns GroupedData objects:
+:meth:`Dataset.groupby() <ray.data.Dataset.groupby>`.
+
+.. include:: ray.data.grouped_data.GroupedData.rst

doc/source/data/api/llm.rst

@@ -0,0 +1,52 @@
+.. _llm-ref:
+
+Large Language Model (LLM) API
+==============================
+
+.. currentmodule:: ray.data.llm
+
+LLM processor builder
+---------------------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~build_processor
+
+Processor
+---------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~Processor
+
+Processor configs
+-----------------
+
+.. autosummary::
+    :nosignatures:
+    :template: autosummary/class_without_autosummary_noinheritance.rst
+    :toctree: doc/
+
+    ~ProcessorConfig
+    ~HttpRequestProcessorConfig
+    ~vLLMEngineProcessorConfig
+    ~SGLangEngineProcessorConfig
+
+.. _stage-configs-ref:
+
+Stage configs
+-------------
+
+.. autosummary::
+    :nosignatures:
+    :template: autosummary/class_without_autosummary_noinheritance.rst
+    :toctree: doc/
+
+    ~ChatTemplateStageConfig
+    ~TokenizerStageConfig
+    ~DetokenizeStageConfig
+    ~PrepareMultimodalStageConfig

doc/source/data/api/loading_data.rst

@@ -0,0 +1,467 @@
+.. _loading-data-api:
+
+Loading Data API
+================
+
+.. currentmodule:: ray.data
+
+Public APIs
+-----------
+
+Arrow
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_arrow
+
+Audio
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_audio
+
+Avro
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_avro
+
+BigQuery
+^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_bigquery
+
+Binary
+^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_binary_files
+
+Catalog
+^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Catalog
+
+   DatabricksUnityCatalog
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+   :template: autosummary/class_without_autosummary.rst
+
+   ReaderFormat
+
+CSV
+^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_csv
+
+ClickHouse
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_clickhouse
+
+Daft
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_daft
+
+Dask
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_dask
+
+Databricks
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_databricks_tables
+
+Delta Lake
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_delta
+
+Delta Sharing
+^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_delta_sharing_tables
+
+Hudi
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_hudi
+
+Hugging Face
+^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_huggingface
+
+Iceberg
+^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_iceberg
+
+Images
+^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_images
+
+JSON
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_json
+
+Kafka
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_kafka
+
+Lance
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_lance
+
+MCAP (Message Capture)
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_mcap
+
+Mars
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_mars
+
+Modin
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_modin
+
+MongoDB
+^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_mongo
+
+NumPy
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_numpy
+   read_numpy
+
+Pandas
+^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_pandas
+
+Parquet
+^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_parquet
+
+Python Objects
+^^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_items
+
+SQL Databases
+^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_sql
+
+Snowflake
+^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_snowflake
+
+Spark
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_spark
+
+Synthetic Data
+^^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   range
+   range_tensor
+
+TFRecords
+^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_tfrecords
+
+TensorFlow
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_tf
+
+Text
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_text
+
+Torch
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_torch
+
+Unity Catalog
+^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_unity_catalog
+
+Video
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_videos
+
+WebDataset
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_webdataset
+
+Zarr
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   read_zarr
+
+Partitioning API
+^^^^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   datasource.Partitioning
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+   :template: autosummary/class_without_autosummary.rst
+
+   datasource.PartitionStyle
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   datasource.PathPartitionFilter
+   datasource.PathPartitionParser
+
+Shuffling API
+^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   FileShuffleConfig
+
+Developer APIs
+--------------
+
+Arrow refs
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_arrow_refs
+
+Datasource API
+^^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Datasource
+   datasource.FileBasedDatasource
+   read_datasource
+   ReadTask
+
+.. _metadata_provider:
+
+MetadataProvider API
+^^^^^^^^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   datasource.BaseFileMetadataProvider
+   datasource.DefaultFileMetadataProvider
+   datasource.FileMetadataProvider
+
+NumPy refs
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_numpy_refs
+
+Pandas refs
+^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   from_pandas_refs

doc/source/data/api/preprocessor.rst

@@ -0,0 +1,100 @@
+.. _preprocessor-ref:
+
+Preprocessor
+============
+
+Preprocessor Interface
+------------------------
+
+.. currentmodule:: ray.data
+
+Constructor
+~~~~~~~~~~~
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~preprocessor.Preprocessor
+
+Fit/Transform APIs
+~~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~preprocessor.Preprocessor.fit
+    ~preprocessor.Preprocessor.fit_transform
+    ~preprocessor.Preprocessor.transform
+    ~preprocessor.Preprocessor.transform_batch
+    ~preprocessor.PreprocessorNotFittedException
+
+
+Generic Preprocessors
+---------------------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~preprocessors.Concatenator
+    ~preprocessors.SimpleImputer
+    ~preprocessors.Chain
+
+Categorical Encoders
+--------------------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~preprocessors.Categorizer
+    ~preprocessors.LabelEncoder
+    ~preprocessors.MultiHotEncoder
+    ~preprocessors.OneHotEncoder
+    ~preprocessors.OrdinalEncoder
+
+Feature Scalers
+---------------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~preprocessors.MaxAbsScaler
+    ~preprocessors.MinMaxScaler
+    ~preprocessors.Normalizer
+    ~preprocessors.PowerTransformer
+    ~preprocessors.RobustScaler
+    ~preprocessors.StandardScaler
+
+K-Bins Discretizers
+-------------------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~preprocessors.CustomKBinsDiscretizer
+    ~preprocessors.UniformKBinsDiscretizer
+
+Feature Hashers and Vectorizers
+-------------------------------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~preprocessors.FeatureHasher
+    ~preprocessors.CountVectorizer
+    ~preprocessors.HashingVectorizer
+
+Specialized Preprocessors
+-------------------------
+
+.. autosummary::
+    :nosignatures:
+    :toctree: doc/
+
+    ~preprocessors.TorchVisionPreprocessor

doc/source/data/api/saving_data.rst

@@ -0,0 +1,234 @@
+.. _saving-data-api:
+
+Saving Data API
+===============
+
+.. currentmodule:: ray.data
+
+Public APIs
+-----------
+
+BigQuery
+^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_bigquery
+
+CSV
+^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_csv
+
+ClickHouse
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_clickhouse
+
+Daft
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_daft
+
+Dask
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_dask
+
+Iceberg
+^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_iceberg
+
+Images
+^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_images
+
+JSON
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_json
+
+Lance
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_lance
+
+Mars
+^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_mars
+
+Modin
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_modin
+
+MongoDB
+^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_mongo
+
+NumPy
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_numpy
+
+Pandas
+^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_pandas
+
+Parquet
+^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_parquet
+
+SQL Databases
+^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_sql
+
+Snowflake
+^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_snowflake
+
+Spark
+^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_spark
+
+TFRecords
+^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.write_tfrecords
+
+Developer APIs
+--------------
+
+Arrow refs
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_arrow_refs
+
+Datasink API
+^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Datasink
+   Dataset.write_datasink
+   datasource.RowBasedFileDatasink
+   datasource.BlockBasedFileDatasink
+   datasource.WriteResult
+   datasource.WriteReturnType
+
+FilenameProvider
+^^^^^^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   datasource.FilenameProvider
+
+NumPy refs
+^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_numpy_refs
+
+Pandas refs
+^^^^^^^^^^^
+
+.. autosummary::
+   :nosignatures:
+   :toctree: doc/
+
+   Dataset.to_pandas_refs
+