chore: import upstream snapshot with attribution

python/ray/data/tests/preprocessors/test_backwards_compatibility.py

@@ -0,0 +1,764 @@
+"""
+Backwards compatibility tests for Preprocessor private field renaming.
+
+These tests verify that preprocessors pickled with old public field names
+(e.g., 'columns') can be deserialized correctly after fields were renamed
+to private (e.g., '_columns').
+
+The __setstate__ method in each preprocessor handles migration automatically.
+"""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+import ray
+from ray.data.preprocessors import (
+    Categorizer,
+    Chain,
+    Concatenator,
+    CountVectorizer,
+    CustomKBinsDiscretizer,
+    FeatureHasher,
+    HashingVectorizer,
+    LabelEncoder,
+    MaxAbsScaler,
+    MinMaxScaler,
+    MultiHotEncoder,
+    Normalizer,
+    OneHotEncoder,
+    OrdinalEncoder,
+    PowerTransformer,
+    RobustScaler,
+    SimpleImputer,
+    StandardScaler,
+    Tokenizer,
+    TorchVisionPreprocessor,
+    UniformKBinsDiscretizer,
+)
+
+# =============================================================================
+# Field Migration Tests
+# =============================================================================
+
+
+@pytest.mark.parametrize(
+    "preprocessor_class,old_state,expected_attrs",
+    [
+        (
+            Concatenator,
+            {
+                "columns": ["A", "B"],
+                "output_column_name": "concat",
+                "dtype": np.float32,
+                "raise_if_missing": True,
+                "flatten": True,
+            },
+            {
+                "columns": ["A", "B"],
+                "output_column_name": "concat",
+                "dtype": np.float32,
+                "raise_if_missing": True,
+                "flatten": True,
+            },
+        ),
+        (
+            Normalizer,
+            {
+                "columns": ["A", "B"],
+                "norm": "l1",
+                "output_columns": ["A_norm", "B_norm"],
+            },
+            {
+                "columns": ["A", "B"],
+                "norm": "l1",
+                "output_columns": ["A_norm", "B_norm"],
+            },
+        ),
+        (
+            Tokenizer,
+            {
+                "columns": ["text"],
+                "tokenization_fn": lambda s: s.split(),
+                "output_columns": ["text_tokens"],
+            },
+            {
+                "columns": ["text"],
+                "tokenization_fn": "callable",  # Special marker
+                "output_columns": ["text_tokens"],
+            },
+        ),
+        (
+            PowerTransformer,
+            {
+                "columns": ["A", "B"],
+                "power": 3,
+                "method": "box-cox",
+                "output_columns": ["A_pow", "B_pow"],
+            },
+            {
+                "columns": ["A", "B"],
+                "power": 3,
+                "method": "box-cox",
+                "output_columns": ["A_pow", "B_pow"],
+            },
+        ),
+        (
+            HashingVectorizer,
+            {
+                "columns": ["text"],
+                "num_features": 200,
+                "tokenization_fn": lambda s: s.split(),
+                "output_columns": ["text_vec"],
+            },
+            {
+                "columns": ["text"],
+                "num_features": 200,
+                "tokenization_fn": "callable",
+                "output_columns": ["text_vec"],
+            },
+        ),
+        (
+            CountVectorizer,
+            {
+                "columns": ["text"],
+                "tokenization_fn": lambda s: s.split(),
+                "max_features": 100,
+                "output_columns": ["text_count"],
+            },
+            {
+                "columns": ["text"],
+                "tokenization_fn": "callable",
+                "max_features": 100,
+                "output_columns": ["text_count"],
+            },
+        ),
+        (
+            FeatureHasher,
+            {"columns": ["A", "B"], "num_features": 20, "output_column": "features"},
+            {"columns": ["A", "B"], "num_features": 20, "output_column": "features"},
+        ),
+        (
+            OrdinalEncoder,
+            {
+                "columns": ["color"],
+                "output_columns": ["color_encoded"],
+                "encode_lists": False,
+            },
+            {
+                "columns": ["color"],
+                "output_columns": ["color_encoded"],
+                "encode_lists": False,
+            },
+        ),
+        (
+            OneHotEncoder,
+            {
+                "columns": ["color"],
+                "output_columns": ["color_encoded"],
+                "max_categories": {"color": 5},
+            },
+            {
+                "columns": ["color"],
+                "output_columns": ["color_encoded"],
+                "max_categories": {"color": 5},
+            },
+        ),
+        (
+            MultiHotEncoder,
+            {
+                "columns": ["tags"],
+                "output_columns": ["tags_encoded"],
+                "max_categories": {},
+            },
+            {"columns": ["tags"], "output_columns": ["tags_encoded"]},
+        ),
+        (
+            LabelEncoder,
+            {"label_column": "label", "output_column": "label_id"},
+            {"label_column": "label", "output_column": "label_id"},
+        ),
+        (
+            Categorizer,
+            {"columns": ["sex"], "output_columns": ["sex_cat"], "dtypes": {}},
+            {"columns": ["sex"], "output_columns": ["sex_cat"]},
+        ),
+        (
+            StandardScaler,
+            {"columns": ["A", "B"], "output_columns": ["A_scaled", "B_scaled"]},
+            {"columns": ["A", "B"], "output_columns": ["A_scaled", "B_scaled"]},
+        ),
+        (
+            MinMaxScaler,
+            {"columns": ["A"], "output_columns": ["A_scaled"]},
+            {"columns": ["A"], "output_columns": ["A_scaled"]},
+        ),
+        (
+            MaxAbsScaler,
+            {"columns": ["A"], "output_columns": ["A_scaled"]},
+            {"columns": ["A"], "output_columns": ["A_scaled"]},
+        ),
+        (
+            RobustScaler,
+            {
+                "columns": ["A"],
+                "output_columns": ["A_scaled"],
+                "quantile_range": (0.1, 0.9),
+                "quantile_precision": 1000,
+            },
+            {
+                "columns": ["A"],
+                "output_columns": ["A_scaled"],
+                "quantile_range": (0.1, 0.9),
+                "quantile_precision": 1000,
+            },
+        ),
+        (
+            SimpleImputer,
+            {
+                "columns": ["A", "B"],
+                "output_columns": ["A_imputed", "B_imputed"],
+                "strategy": "median",
+                "fill_value": 99.0,
+            },
+            {
+                "columns": ["A", "B"],
+                "output_columns": ["A_imputed", "B_imputed"],
+                "strategy": "median",
+                "fill_value": 99.0,
+            },
+        ),
+        (
+            CustomKBinsDiscretizer,
+            {
+                "columns": ["A", "B"],
+                "bins": [0, 1, 2, 3],
+                "right": False,
+                "include_lowest": True,
+                "duplicates": "drop",
+                "dtypes": None,
+                "output_columns": ["A_binned", "B_binned"],
+            },
+            {
+                "columns": ["A", "B"],
+                "bins": [0, 1, 2, 3],
+                "right": False,
+                "include_lowest": True,
+                "duplicates": "drop",
+                "dtypes": None,
+                "output_columns": ["A_binned", "B_binned"],
+            },
+        ),
+        (
+            UniformKBinsDiscretizer,
+            {
+                "columns": ["A", "B"],
+                "bins": 4,
+                "right": False,
+                "include_lowest": True,
+                "duplicates": "drop",
+                "dtypes": None,
+                "output_columns": ["A_binned", "B_binned"],
+            },
+            {
+                "columns": ["A", "B"],
+                "bins": 4,
+                "right": False,
+                "include_lowest": True,
+                "duplicates": "drop",
+                "dtypes": None,
+                "output_columns": ["A_binned", "B_binned"],
+            },
+        ),
+    ],
+    ids=lambda x: x.__name__ if hasattr(x, "__name__") else str(x)[:20],
+)
+def test_field_migration_from_old_public_names(
+    preprocessor_class, old_state, expected_attrs
+):
+    """Verify old public field names are migrated to new private fields."""
+    preprocessor = preprocessor_class.__new__(preprocessor_class)
+    preprocessor.__setstate__(old_state)
+
+    for attr_name, expected_value in expected_attrs.items():
+        actual_value = getattr(preprocessor, attr_name)
+        if expected_value == "callable":
+            assert callable(actual_value), f"{attr_name} should be callable"
+        else:
+            assert actual_value == expected_value, f"Mismatch in {attr_name}"
+
+
+@pytest.mark.parametrize(
+    "preprocessor_class,minimal_state,expected_defaults",
+    [
+        # Callable default: tokenization_fn must be stored as the function itself,
+        # not called. This would have failed with the old callable() check.
+        (
+            Tokenizer,
+            {"columns": ["text"]},
+            {"tokenization_fn": "callable", "output_columns": ["text"]},
+        ),
+        (
+            HashingVectorizer,
+            {"columns": ["text"], "num_features": 100},
+            {"tokenization_fn": "callable", "output_columns": ["text"]},
+        ),
+        (
+            CountVectorizer,
+            {"columns": ["text"]},
+            {"tokenization_fn": "callable", "output_columns": ["text"]},
+        ),
+        # _Computed default: output_columns derives from _columns.
+        (
+            StandardScaler,
+            {"columns": ["A", "B"]},
+            {"output_columns": ["A", "B"]},
+        ),
+        # _Computed default deriving from a different source field.
+        (
+            LabelEncoder,
+            {"label_column": "label"},
+            {"output_column": "label"},
+        ),
+        # Plain value default alongside a _Computed default.
+        (
+            Normalizer,
+            {"columns": ["A", "B"]},
+            {"norm": "l2", "output_columns": ["A", "B"]},
+        ),
+    ],
+    ids=[
+        "Tokenizer",
+        "HashingVectorizer",
+        "CountVectorizer",
+        "StandardScaler",
+        "LabelEncoder",
+        "Normalizer",
+    ],
+)
+def test_missing_optional_fields_use_defaults(
+    preprocessor_class, minimal_state, expected_defaults
+):
+    """
+    Verify that absent optional fields are filled with their correct defaults.
+
+    This exercises the default-fallback branch of migrate_private_fields. The minimal_state
+    deliberately omits optional fields to force the default path.
+    """
+    preprocessor = preprocessor_class.__new__(preprocessor_class)
+    preprocessor.__setstate__(minimal_state)
+
+    for attr_name, expected_value in expected_defaults.items():
+        actual_value = getattr(preprocessor, attr_name)
+        if expected_value == "callable":
+            assert callable(
+                actual_value
+            ), f"{attr_name} should be a stored callable, not the result of calling it"
+        else:
+            assert (
+                actual_value == expected_value
+            ), f"Mismatch in {attr_name}: {actual_value!r} != {expected_value!r}"
+
+
+def test_torchvision_preprocessor_field_migration():
+    try:
+        from torchvision import transforms
+    except ImportError:
+        pytest.skip("torchvision not installed")
+
+    transform = transforms.Lambda(lambda x: x)
+    preprocessor = TorchVisionPreprocessor.__new__(TorchVisionPreprocessor)
+    state = {
+        "columns": ["image"],
+        "output_columns": ["image_out"],
+        "torchvision_transform": transform,
+        "batched": True,
+    }
+    preprocessor.__setstate__(state)
+
+    assert preprocessor.columns == ["image"]
+    assert preprocessor.output_columns == ["image_out"]
+    assert preprocessor.torchvision_transform == transform
+    assert preprocessor.batched is True
+
+
+def test_chain_field_migration():
+    scaler1 = StandardScaler(columns=["A"])
+    scaler2 = StandardScaler(columns=["B"])
+    chain = Chain.__new__(Chain)
+
+    state = {"preprocessors": (scaler1, scaler2)}
+    chain.__setstate__(state)
+
+    assert len(chain.preprocessors) == 2
+    assert chain.preprocessors[0] == scaler1
+    assert chain.preprocessors[1] == scaler2
+
+
+# =============================================================================
+# Functional Test Helpers
+# =============================================================================
+
+
+def _simulate_old_format_deserialization(preprocessor, field_mapping):
+    """Simulate deserialization from old format by renaming private->public fields."""
+    state = preprocessor.__dict__.copy()
+    for public_name, private_name in field_mapping.items():
+        if private_name in state:
+            state[public_name] = state.pop(private_name)
+
+    new_preprocessor = preprocessor.__class__.__new__(preprocessor.__class__)
+    new_preprocessor.__setstate__(state)
+    return new_preprocessor
+
+
+def _test_functional_backwards_compat(preprocessor, test_ds, field_mapping):
+    """Generic functional test: verify deserialized preprocessor produces same output."""
+    expected_result = preprocessor.transform(test_ds).to_pandas()
+    new_preprocessor = _simulate_old_format_deserialization(preprocessor, field_mapping)
+    result = new_preprocessor.transform(test_ds).to_pandas()
+    pd.testing.assert_frame_equal(result, expected_result)
+
+
+# =============================================================================
+# Functional Tests - Simple Preprocessors (No Fitting Required)
+# =============================================================================
+
+
+@pytest.mark.parametrize(
+    "setup_func,field_mapping",
+    [
+        (
+            lambda: (
+                Concatenator(columns=["A", "B"], output_column_name="C"),
+                pd.DataFrame({"A": [1, 2], "B": [3, 4]}),
+                {
+                    "columns": "_columns",
+                    "output_column_name": "_output_column_name",
+                    "dtype": "_dtype",
+                    "raise_if_missing": "_raise_if_missing",
+                    "flatten": "_flatten",
+                },
+            ),
+            None,
+        ),
+        (
+            lambda: (
+                Normalizer(columns=["A", "B"], norm="l2"),
+                pd.DataFrame({"A": [1.0, 2.0], "B": [3.0, 4.0]}),
+                {
+                    "columns": "_columns",
+                    "norm": "_norm",
+                    "output_columns": "_output_columns",
+                },
+            ),
+            None,
+        ),
+        (
+            lambda: (
+                Tokenizer(columns=["text"]),
+                pd.DataFrame({"text": ["hello world", "foo bar"]}),
+                {
+                    "columns": "_columns",
+                    "tokenization_fn": "_tokenization_fn",
+                    "output_columns": "_output_columns",
+                },
+            ),
+            None,
+        ),
+        (
+            lambda: (
+                PowerTransformer(columns=["A", "B"], power=2),
+                pd.DataFrame({"A": [1.0, 2.0, 3.0], "B": [4.0, 5.0, 6.0]}),
+                {
+                    "columns": "_columns",
+                    "power": "_power",
+                    "method": "_method",
+                    "output_columns": "_output_columns",
+                },
+            ),
+            None,
+        ),
+        (
+            lambda: (
+                HashingVectorizer(columns=["text"], num_features=10),
+                pd.DataFrame({"text": ["hello world", "foo bar"]}),
+                {
+                    "columns": "_columns",
+                    "num_features": "_num_features",
+                    "tokenization_fn": "_tokenization_fn",
+                    "output_columns": "_output_columns",
+                },
+            ),
+            None,
+        ),
+        (
+            lambda: (
+                FeatureHasher(
+                    columns=["token_a", "token_b"],
+                    num_features=5,
+                    output_column="hashed",
+                ),
+                pd.DataFrame({"token_a": [1, 2], "token_b": [3, 4]}),
+                {
+                    "columns": "_columns",
+                    "num_features": "_num_features",
+                    "output_column": "_output_column",
+                },
+            ),
+            None,
+        ),
+        (
+            lambda: (
+                CustomKBinsDiscretizer(
+                    columns=["A", "B"],
+                    bins=[0, 1, 2, 3, 4],
+                    output_columns=["A_binned", "B_binned"],
+                ),
+                pd.DataFrame({"A": [0.5, 1.5, 2.5, 3.5], "B": [0.2, 1.2, 2.2, 3.2]}),
+                {
+                    "columns": "_columns",
+                    "bins": "_bins",
+                    "right": "_right",
+                    "include_lowest": "_include_lowest",
+                    "duplicates": "_duplicates",
+                    "dtypes": "_dtypes",
+                    "output_columns": "_output_columns",
+                },
+            ),
+            None,
+        ),
+    ],
+    ids=[
+        "Concatenator",
+        "Normalizer",
+        "Tokenizer",
+        "PowerTransformer",
+        "HashingVectorizer",
+        "FeatureHasher",
+        "CustomKBinsDiscretizer",
+    ],
+)
+def test_simple_functional_backwards_compat(setup_func, field_mapping):
+    """Verify preprocessors that don't need fitting work after deserialization."""
+    preprocessor, test_data, field_mapping = setup_func()
+    test_ds = ray.data.from_pandas(test_data)
+    _test_functional_backwards_compat(preprocessor, test_ds, field_mapping)
+
+
+# =============================================================================
+# Functional Tests - Stateful Preprocessors (Require Fitting)
+# =============================================================================
+
+
+@pytest.mark.parametrize(
+    "setup_func",
+    [
+        lambda: (
+            OrdinalEncoder(columns=["color"]),
+            pd.DataFrame({"color": ["red", "green", "blue", "red", "green"]}),
+            {
+                "columns": "_columns",
+                "output_columns": "_output_columns",
+                "encode_lists": "_encode_lists",
+            },
+        ),
+        lambda: (
+            OneHotEncoder(columns=["color"]),
+            pd.DataFrame({"color": ["red", "green", "blue", "red", "green", "blue"]}),
+            {
+                "columns": "_columns",
+                "output_columns": "_output_columns",
+                "max_categories": "_max_categories",
+            },
+        ),
+        lambda: (
+            LabelEncoder(label_column="label"),
+            pd.DataFrame(
+                {
+                    "feature": [1.0, 2.0, 3.0, 4.0],
+                    "label": ["cat", "dog", "cat", "bird"],
+                }
+            ),
+            {"label_column": "_label_column", "output_column": "_output_column"},
+        ),
+        lambda: (
+            StandardScaler(columns=["A", "B"]),
+            pd.DataFrame(
+                {"A": [1.0, 2.0, 3.0, 4.0, 5.0], "B": [10.0, 20.0, 30.0, 40.0, 50.0]}
+            ),
+            {"columns": "_columns", "output_columns": "_output_columns"},
+        ),
+        lambda: (
+            MinMaxScaler(columns=["A", "B"]),
+            pd.DataFrame(
+                {"A": [1.0, 2.0, 3.0, 4.0, 5.0], "B": [10.0, 20.0, 30.0, 40.0, 50.0]}
+            ),
+            {"columns": "_columns", "output_columns": "_output_columns"},
+        ),
+        lambda: (
+            RobustScaler(columns=["A"]),
+            pd.DataFrame({"A": [1.0, 2.0, 3.0, 4.0, 5.0, 100.0]}),
+            {
+                "columns": "_columns",
+                "output_columns": "_output_columns",
+                "quantile_range": "_quantile_range",
+                "quantile_precision": "_quantile_precision",
+            },
+        ),
+        lambda: (
+            SimpleImputer(columns=["A", "B"], strategy="mean"),
+            pd.DataFrame(
+                {"A": [1.0, 2.0, None, 4.0, 5.0], "B": [10.0, None, 30.0, 40.0, 50.0]}
+            ),
+            {
+                "columns": "_columns",
+                "output_columns": "_output_columns",
+                "strategy": "_strategy",
+                "fill_value": "_fill_value",
+            },
+        ),
+        lambda: (
+            CountVectorizer(columns=["text"]),
+            pd.DataFrame({"text": ["hello world", "foo bar", "hello foo"]}),
+            {
+                "columns": "_columns",
+                "tokenization_fn": "_tokenization_fn",
+                "max_features": "_max_features",
+                "output_columns": "_output_columns",
+            },
+        ),
+        lambda: (
+            UniformKBinsDiscretizer(
+                columns=["A", "B"], bins=3, output_columns=["A_binned", "B_binned"]
+            ),
+            pd.DataFrame(
+                {"A": [1.0, 2.0, 3.0, 4.0, 5.0], "B": [10.0, 20.0, 30.0, 40.0, 50.0]}
+            ),
+            {
+                "columns": "_columns",
+                "bins": "_bins",
+                "right": "_right",
+                "include_lowest": "_include_lowest",
+                "duplicates": "_duplicates",
+                "dtypes": "_dtypes",
+                "output_columns": "_output_columns",
+            },
+        ),
+        lambda: (
+            MultiHotEncoder(columns=["genre"]),
+            pd.DataFrame(
+                {
+                    "genre": [
+                        ["comedy", "action"],
+                        ["drama", "action"],
+                        ["comedy", "drama"],
+                    ]
+                }
+            ),
+            {
+                "columns": "_columns",
+                "output_columns": "_output_columns",
+                "max_categories": "_max_categories",
+            },
+        ),
+        lambda: (
+            MaxAbsScaler(columns=["A", "B"]),
+            pd.DataFrame({"A": [-6.0, 3.0, -3.0], "B": [2.0, -4.0, 1.0]}),
+            {"columns": "_columns", "output_columns": "_output_columns"},
+        ),
+        lambda: (
+            Categorizer(columns=["color"]),
+            pd.DataFrame({"color": ["red", "green", "blue", "red", "green"]}),
+            {
+                "columns": "_columns",
+                "output_columns": "_output_columns",
+                "dtypes": "_dtypes",
+            },
+        ),
+    ],
+    ids=[
+        "OrdinalEncoder",
+        "OneHotEncoder",
+        "LabelEncoder",
+        "StandardScaler",
+        "MinMaxScaler",
+        "RobustScaler",
+        "SimpleImputer",
+        "CountVectorizer",
+        "UniformKBinsDiscretizer",
+        "MultiHotEncoder",
+        "MaxAbsScaler",
+        "Categorizer",
+    ],
+)
+def test_stateful_functional_backwards_compat(setup_func):
+    """Verify fitted preprocessors work after deserialization."""
+    preprocessor, test_data, field_mapping = setup_func()
+    test_ds = ray.data.from_pandas(test_data)
+    preprocessor = preprocessor.fit(test_ds)
+    _test_functional_backwards_compat(preprocessor, test_ds, field_mapping)
+
+
+def test_chain_functional_backwards_compat():
+    df = pd.DataFrame({"A": [1.0, 2.0, 3.0]})
+    ds = ray.data.from_pandas(df)
+
+    scaler = StandardScaler(columns=["A"])
+    normalizer = Normalizer(columns=["A"])
+    chain = Chain(scaler, normalizer)
+    chain = chain.fit(ds)
+
+    expected_result = chain.transform(ds).to_pandas()
+
+    state = chain.__dict__.copy()
+    state["preprocessors"] = state.pop("_preprocessors")
+
+    new_chain = Chain.__new__(Chain)
+    new_chain.__setstate__(state)
+
+    result = new_chain.transform(ds).to_pandas()
+    pd.testing.assert_frame_equal(result, expected_result)
+
+
+def test_torchvision_functional_backwards_compat():
+    try:
+        import torch
+        from torchvision import transforms
+    except ImportError:
+        pytest.skip("torchvision not installed")
+
+    transform = transforms.Lambda(lambda x: torch.as_tensor(x, dtype=torch.float32))
+    df = pd.DataFrame(
+        {
+            "image": [
+                np.array([[1, 2], [3, 4]], dtype=np.uint8),
+                np.array([[5, 6], [7, 8]], dtype=np.uint8),
+            ]
+        }
+    )
+    ds = ray.data.from_pandas(df)
+
+    preprocessor = TorchVisionPreprocessor(
+        columns=["image"], transform=transform, batched=False
+    )
+    expected_result = preprocessor.transform(ds).to_pandas()
+
+    state = preprocessor.__dict__.copy()
+    state["columns"] = state.pop("_columns")
+    state["output_columns"] = state.pop("_output_columns")
+    state["torchvision_transform"] = state.pop("_torchvision_transform")
+    state["batched"] = state.pop("_batched")
+
+    new_preprocessor = TorchVisionPreprocessor.__new__(TorchVisionPreprocessor)
+    new_preprocessor.__setstate__(state)
+
+    result = new_preprocessor.transform(ds).to_pandas()
+    assert len(result) == len(expected_result)
+    assert "image" in result.columns
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_chain.py

@@ -0,0 +1,243 @@
+import pandas as pd
+import pytest
+
+import ray
+from ray.data.preprocessor import Preprocessor
+from ray.data.preprocessors import Chain, LabelEncoder, SimpleImputer, StandardScaler
+from ray.data.util.data_batch_conversion import BatchFormat
+
+
+def test_chain():
+    """Tests basic Chain functionality."""
+    col_a = [-1, -1, 1, 1]
+    col_b = [1, 1, 1, None]
+    col_c = ["sunday", "monday", "tuesday", "tuesday"]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    imputer = SimpleImputer(["B"])
+    scaler = StandardScaler(["A", "B"])
+    encoder = LabelEncoder("C")
+    chain = Chain(scaler, imputer, encoder)
+
+    # Fit data.
+    chain.fit(ds)
+    # Transform data.
+    transformed = chain.transform(ds)
+    out_df = transformed.to_pandas()
+
+    assert imputer.stats_ == {
+        "mean(B)": 0.0,
+    }
+    assert scaler.stats_ == {
+        "mean(A)": 0.0,
+        "mean(B)": 1.0,
+        "std(A)": 1.0,
+        "std(B)": 0.0,
+    }
+    assert encoder.stats_ == {
+        "unique_values(C)": {"monday": 0, "sunday": 1, "tuesday": 2}
+    }
+
+    processed_col_a = [-1.0, -1.0, 1.0, 1.0]
+    processed_col_b = [0.0, 0.0, 0.0, 0.0]
+    processed_col_c = [1, 0, 2, 2]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Transform batch.
+    pred_col_a = [1, 2, None]
+    pred_col_b = [0, None, 2]
+    pred_col_c = ["monday", "tuesday", "wednesday"]
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+
+    pred_out_df = chain.transform_batch(pred_in_df)
+
+    pred_processed_col_a = [1, 2, None]
+    pred_processed_col_b = [-1.0, 0.0, 1.0]
+    pred_processed_col_c = [0, 2, None]
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_processed_col_a,
+            "B": pred_processed_col_b,
+            "C": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+
+def test_nested_chain_state():
+    col_a = [-1, -1, 1, 1]
+    col_b = [1, 1, 1, None]
+    col_c = ["sunday", "monday", "tuesday", "tuesday"]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    def create_chain():
+        imputer = SimpleImputer(["B"])
+        scaler = StandardScaler(["A", "B"])
+        encoder = LabelEncoder("C")
+        return Chain(Chain(scaler, imputer), encoder)
+
+    chain = create_chain()
+    assert chain.fit_status() == Preprocessor.FitStatus.NOT_FITTED
+
+    chain = create_chain()
+    chain.preprocessors[1].fit(ds)
+    assert chain.fit_status() == Preprocessor.FitStatus.PARTIALLY_FITTED
+
+    chain = create_chain()
+    chain.preprocessors[0].fit(ds)
+    assert chain.fit_status() == Preprocessor.FitStatus.PARTIALLY_FITTED
+
+    chain.preprocessors[1].fit(ds)
+    assert chain.fit_status() == Preprocessor.FitStatus.FITTED
+
+    chain = create_chain()
+    chain.fit(ds)
+    assert chain.fit_status() == Preprocessor.FitStatus.FITTED
+
+
+def test_nested_chain():
+    """Tests Chain-inside-Chain functionality."""
+    col_a = [-1, -1, 1, 1]
+    col_b = [1, 1, 1, None]
+    col_c = ["sunday", "monday", "tuesday", "tuesday"]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    imputer = SimpleImputer(["B"])
+    scaler = StandardScaler(["A", "B"])
+    encoder = LabelEncoder("C")
+    chain = Chain(Chain(scaler, imputer), encoder)
+
+    # Fit data.
+    chain.fit(ds)
+    # Transform data.
+    transformed = chain.transform(ds)
+    out_df = transformed.to_pandas()
+
+    assert imputer.stats_ == {
+        "mean(B)": 0.0,
+    }
+    assert scaler.stats_ == {
+        "mean(A)": 0.0,
+        "mean(B)": 1.0,
+        "std(A)": 1.0,
+        "std(B)": 0.0,
+    }
+    assert encoder.stats_ == {
+        "unique_values(C)": {"monday": 0, "sunday": 1, "tuesday": 2}
+    }
+
+    processed_col_a = [-1.0, -1.0, 1.0, 1.0]
+    processed_col_b = [0.0, 0.0, 0.0, 0.0]
+    processed_col_c = [1, 0, 2, 2]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Transform batch.
+    pred_col_a = [1, 2, None]
+    pred_col_b = [0, None, 2]
+    pred_col_c = ["monday", "tuesday", "wednesday"]
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+
+    pred_out_df = chain.transform_batch(pred_in_df)
+
+    pred_processed_col_a = [1, 2, None]
+    pred_processed_col_b = [-1.0, 0.0, 1.0]
+    pred_processed_col_c = [0, 2, None]
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_processed_col_a,
+            "B": pred_processed_col_b,
+            "C": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+
+class PreprocessorWithoutTransform(Preprocessor):
+    pass
+
+
+def test_determine_transform_to_use():
+    # Test that _determine_transform_to_use doesn't throw any exceptions
+    # and selects the transform function of the underlying preprocessor
+    # while dealing with the nested Chain case.
+
+    # Check that error is propagated correctly
+    with pytest.raises(NotImplementedError):
+        chain = Chain(PreprocessorWithoutTransform())
+        chain._determine_transform_to_use()
+
+    # Should have no errors from here on
+    preprocessor = SimpleImputer(["A"])
+    chain1 = Chain(preprocessor)
+    format1 = chain1._determine_transform_to_use()
+    assert format1 == BatchFormat.PANDAS
+
+    chain2 = Chain(chain1)
+    format2 = chain2._determine_transform_to_use()
+
+    assert format1 == format2
+
+
+def test_chain_serialization():
+    """Test Chain serialization and deserialization functionality."""
+    import ray
+    from ray.data.preprocessor import SerializablePreprocessorBase
+    from ray.data.preprocessors import Normalizer, StandardScaler
+
+    # Create and fit chain
+    scaler = StandardScaler(columns=["A"])
+    normalizer = Normalizer(columns=["A"])
+    chain = Chain(scaler, normalizer)
+
+    df = pd.DataFrame({"A": [1.0, 2.0, 3.0]})
+    ds = ray.data.from_pandas(df)
+    fitted_chain = chain.fit(ds)
+
+    # Serialize using CloudPickle
+    serialized = fitted_chain.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = Chain.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, Chain)
+    assert len(deserialized._preprocessors) == 2
+    assert isinstance(deserialized._preprocessors[0], StandardScaler)
+    assert isinstance(deserialized._preprocessors[1], Normalizer)
+    # Verify the StandardScaler is fitted (Normalizer is stateless)
+    assert deserialized._preprocessors[0]._fitted
+
+    # Verify it works correctly
+    test_df = pd.DataFrame({"A": [1.5, 2.5]})
+    result = deserialized.transform_batch(test_df)
+
+    # Result should have been transformed by both preprocessors
+    assert "A" in result.columns
+    assert len(result) == 2
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_concatenator.py

@@ -0,0 +1,227 @@
+import numpy as np
+import pandas as pd
+import pytest
+from pandas.testing import assert_frame_equal
+
+import ray
+from ray.data.exceptions import UserCodeException
+from ray.data.preprocessors import Concatenator, OneHotEncoder
+
+
+class TestConcatenator:
+    def test_basic(self):
+        df = pd.DataFrame(
+            {
+                "a": [1, 2, 3, 4],
+                "b": [5, 6, 7, 8],
+            }
+        )
+        ds = ray.data.from_pandas(df)
+        prep = Concatenator(columns=["a", "b"], output_column_name="c")
+        new_ds = prep.transform(ds)
+        for i, row in enumerate(new_ds.take()):
+            assert np.array_equal(row["c"], np.array([i + 1, i + 5]))
+
+    def test_raise_if_missing(self):
+        df = pd.DataFrame({"a": [1, 2, 3, 4]})
+        ds = ray.data.from_pandas(df)
+        prep = Concatenator(
+            columns=["a", "b"], output_column_name="c", raise_if_missing=True
+        )
+
+        with pytest.raises(UserCodeException):
+            with pytest.raises(ValueError, match="'b'"):
+                prep.transform(ds).materialize()
+
+    def test_exclude_column(self):
+        df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [2, 3, 4, 5], "c": [3, 4, 5, 6]})
+        ds = ray.data.from_pandas(df)
+        prep = Concatenator(columns=["a", "c"])
+        new_ds = prep.transform(ds)
+        for _, row in enumerate(new_ds.take()):
+            assert set(row) == {"concat_out", "b"}
+
+    def test_include_columns(self):
+        df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [2, 3, 4, 5], "c": [3, 4, 5, 6]})
+        ds = ray.data.from_pandas(df)
+        prep = Concatenator(columns=["a", "b"])
+        new_ds = prep.transform(ds)
+        for _, row in enumerate(new_ds.take()):
+            assert set(row) == {"concat_out", "c"}
+
+    def test_change_column_order(self):
+        df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [2, 3, 4, 5]})
+        ds = ray.data.from_pandas(df)
+        prep = Concatenator(columns=["b", "a"])
+        new_ds = prep.transform(ds)
+        expected_df = pd.DataFrame({"concat_out": [[2, 1], [3, 2], [4, 3], [5, 4]]})
+        print(new_ds.to_pandas())
+        assert_frame_equal(new_ds.to_pandas(), expected_df)
+
+    def test_strings(self):
+        df = pd.DataFrame({"a": ["string", "string2", "string3"]})
+        ds = ray.data.from_pandas(df)
+        prep = Concatenator(columns=["a"], output_column_name="huh")
+        new_ds = prep.transform(ds)
+        assert "huh" in set(new_ds.schema().names)
+
+    def test_preserves_order(self):
+        df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [2, 3, 4, 5]})
+        ds = ray.data.from_pandas(df)
+        prep = Concatenator(columns=["a", "b"], output_column_name="c")
+        prep = prep.fit(ds)
+
+        df = pd.DataFrame({"a": [5, 6, 7, 8], "b": [6, 7, 8, 9]})
+        concatenated_df = prep.transform_batch(df)
+        expected_df = pd.DataFrame({"c": [[5, 6], [6, 7], [7, 8], [8, 9]]})
+        assert_frame_equal(concatenated_df, expected_df)
+
+        other_df = pd.DataFrame({"a": [9, 10, 11, 12], "b": [10, 11, 12, 13]})
+        concatenated_other_df = prep.transform_batch(other_df)
+        expected_df = pd.DataFrame(
+            {
+                "c": [
+                    [9, 10],
+                    [10, 11],
+                    [11, 12],
+                    [12, 13],
+                ]
+            }
+        )
+        assert_frame_equal(concatenated_other_df, expected_df)
+
+    @pytest.mark.parametrize("col_b", [[[2, 3], [3, 4], [4, 5], [5, 6]], [2, 3, 4, 5]])
+    @pytest.mark.parametrize("flatten", [True, False])
+    def test_flatten(self, col_b, flatten):
+        col_a = [1, 2, 3, 4]
+        col_b = [np.array(v) for v in col_b] if isinstance(col_b[0], list) else col_b
+        df = pd.DataFrame({"a": col_a, "b": col_b})
+        ds = ray.data.from_pandas(df)
+        prep = Concatenator(columns=["a", "b"], flatten=flatten)
+        new_ds = prep.transform(ds)
+
+        for i, row in enumerate(new_ds.take()):
+            if flatten or not isinstance(col_b[i], np.ndarray):
+                # When flatten=True or when col_b contains simple values
+                if isinstance(col_b[i], np.ndarray):
+                    expected = np.concatenate([np.array([col_a[i]]), col_b[i]])
+                else:
+                    expected = np.array([col_a[i], col_b[i]])
+                assert np.array_equal(row["concat_out"], expected)
+            else:
+                # When flatten=False and col_b contains numpy arrays
+                # The output should be a list containing the scalar and the array
+                assert len(row["concat_out"]) == 2
+                assert row["concat_out"][0] == col_a[i]
+                assert np.array_equal(row["concat_out"][1], col_b[i])
+
+    @pytest.mark.parametrize("flatten", [True, False])
+    def test_concatenate_with_onehotencoder(self, flatten):
+        df = pd.DataFrame(
+            {
+                "color": ["red", "green", "blue", "red"],
+                "value": [1, 2, 3, 4],
+            }
+        )
+        ds = ray.data.from_pandas(df)
+        # OneHot encode the color column
+        encoder = OneHotEncoder(columns=["color"], output_columns=["color_encoded"])
+        encoder = encoder.fit(ds)
+        encoded_ds = encoder.transform(ds)
+        # Concatenate the one-hot encoded column with the value column
+        prep = Concatenator(
+            columns=["color_encoded", "value"],
+            output_column_name="features",
+            flatten=flatten,
+        )
+        new_ds = prep.transform(encoded_ds)
+        # Get the expected one-hot vectors
+        color_map = {"blue": [1, 0, 0], "green": [0, 1, 0], "red": [0, 0, 1]}
+        for i, row in enumerate(new_ds.take()):
+            if flatten:
+                expected = color_map[df["color"][i]] + [df["value"][i]]
+                assert np.array_equal(row["features"], np.array(expected))
+            else:
+                expected = [np.array(color_map[df["color"][i]]), df["value"][i]]
+                assert np.array_equal(row["features"][0], expected[0])
+                assert row["features"][1] == expected[1]
+
+    @pytest.mark.parametrize("flatten", [True, False])
+    def test_nested_list_with_dtype(self, flatten: bool):
+        # Tests Concatenator with nested lists and dtype: flattens and coerces when flatten=True,
+        # raises ValueError when flatten=False.
+        output_column = "c"
+        df = pd.DataFrame(
+            {
+                "a": [12.0],
+                "b": [[1, 0, 0, 0]],
+            }
+        )
+        prep = Concatenator(
+            columns=["a", "b"],
+            output_column_name=output_column,
+            dtype=np.float32,
+            flatten=flatten,
+        )
+
+        if flatten:
+            pd_ds = prep._transform_pandas(df)
+            expected_pd = pd.DataFrame(
+                {output_column: pd.Series([[12.0, 1.0, 0.0, 0.0, 0.0]])}
+            )
+            assert_frame_equal(pd_ds, expected_pd)
+        else:
+            # Only for flattened output do we expect the dtype coercion to apply
+            with pytest.raises(ValueError):
+                pd_ds = prep._transform_pandas(df)
+
+    def test_concatenator_deserialize_backward_compat(self):
+        p1 = Concatenator(columns=["A"], flatten=True)
+        delattr(p1, "_flatten")
+        data = p1.serialize()
+        p2 = Concatenator.deserialize(data)
+        assert isinstance(p2, Concatenator)
+        assert p2.flatten is False
+
+    def test_concatenator_serialization(self):
+        """Test Concatenator serialization and deserialization functionality."""
+        from ray.data.preprocessor import SerializablePreprocessorBase
+
+        # Create concatenator
+        concatenator = Concatenator(
+            columns=["A", "B"],
+            output_column_name="combined",
+            dtype=np.float32,
+            flatten=True,
+        )
+
+        # Serialize using CloudPickle
+        serialized = concatenator.serialize()
+
+        # Verify it's binary CloudPickle format
+        assert isinstance(serialized, bytes)
+        assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+        # Deserialize
+        deserialized = Concatenator.deserialize(serialized)
+
+        # Verify type and field values
+        assert isinstance(deserialized, Concatenator)
+        assert deserialized.columns == ["A", "B"]
+        assert deserialized.output_column_name == "combined"
+        assert deserialized.dtype == np.float32
+        assert deserialized.flatten is True
+
+        # Verify it works correctly
+        df = pd.DataFrame({"A": [[1, 2]], "B": [[3, 4]]})
+        result = deserialized.transform_batch(df)
+
+        # Verify concatenation was applied correctly
+        assert "combined" in result.columns
+        assert len(result["combined"][0]) == 4
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_discretizer.py

@@ -0,0 +1,312 @@
+import pandas as pd
+import pytest
+
+import ray
+from ray.data._internal.util import rows_same
+from ray.data.preprocessors import CustomKBinsDiscretizer, UniformKBinsDiscretizer
+
+
+@pytest.mark.parametrize("bins", (3, {"A": 4, "B": 3}))
+@pytest.mark.parametrize(
+    "dtypes",
+    (
+        None,
+        {"A": int, "B": int},
+        {"A": int, "B": pd.CategoricalDtype(["cat1", "cat2", "cat3"], ordered=True)},
+    ),
+)
+@pytest.mark.parametrize("right", (True, False))
+@pytest.mark.parametrize("include_lowest", (True, False))
+def test_uniform_kbins_discretizer(
+    bins,
+    dtypes,
+    right,
+    include_lowest,
+):
+    """Tests basic UniformKBinsDiscretizer functionality."""
+
+    col_a = [0.2, 1.4, 2.5, 6.2, 9.7, 2.1]
+    col_b = col_a.copy()
+    col_c = col_a.copy()
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df).repartition(2)
+
+    discretizer = UniformKBinsDiscretizer(
+        ["A", "B"], bins=bins, dtypes=dtypes, right=right, include_lowest=include_lowest
+    )
+
+    transformed = discretizer.fit_transform(ds)
+    out_df = transformed.to_pandas()
+
+    if isinstance(bins, dict):
+        bins_A = bins["A"]
+        bins_B = bins["B"]
+    else:
+        bins_A = bins_B = bins
+
+    labels_A = False
+    ordered_A = True
+    labels_B = False
+    ordered_B = True
+    if isinstance(dtypes, dict):
+        if isinstance(dtypes.get("A"), pd.CategoricalDtype):
+            labels_A = dtypes.get("A").categories
+            ordered_A = dtypes.get("A").ordered
+        if isinstance(dtypes.get("B"), pd.CategoricalDtype):
+            labels_B = dtypes.get("B").categories
+            ordered_B = dtypes.get("B").ordered
+
+    # Create expected dataframe with transformed columns
+    expected_df = in_df.copy()
+    expected_df["A"] = pd.cut(
+        in_df["A"],
+        bins_A,
+        labels=labels_A,
+        ordered=ordered_A,
+        right=right,
+        include_lowest=include_lowest,
+    )
+    expected_df["B"] = pd.cut(
+        in_df["B"],
+        bins_B,
+        labels=labels_B,
+        ordered=ordered_B,
+        right=right,
+        include_lowest=include_lowest,
+    )
+
+    # Use rows_same to compare regardless of row ordering
+    assert rows_same(out_df, expected_df)
+
+    # append mode
+    expected_message = "The length of columns and output_columns must match."
+    with pytest.raises(ValueError, match=expected_message):
+        UniformKBinsDiscretizer(["A", "B"], bins=bins, output_columns=["A_discretized"])
+
+    discretizer = UniformKBinsDiscretizer(
+        ["A", "B"],
+        bins=bins,
+        dtypes=dtypes,
+        right=right,
+        include_lowest=include_lowest,
+        output_columns=["A_discretized", "B_discretized"],
+    )
+
+    transformed = discretizer.fit_transform(ds)
+    out_df = transformed.to_pandas()
+
+    # Create expected dataframe with appended columns
+    expected_df = in_df.copy()
+    expected_df["A_discretized"] = pd.cut(
+        in_df["A"],
+        bins_A,
+        labels=labels_A,
+        ordered=ordered_A,
+        right=right,
+        include_lowest=include_lowest,
+    )
+    expected_df["B_discretized"] = pd.cut(
+        in_df["B"],
+        bins_B,
+        labels=labels_B,
+        ordered=ordered_B,
+        right=right,
+        include_lowest=include_lowest,
+    )
+
+    # Use rows_same to compare regardless of row ordering
+    assert rows_same(out_df, expected_df)
+
+
+@pytest.mark.parametrize(
+    "bins", ([3, 4, 6, 9], {"A": [3, 4, 6, 8, 9], "B": [3, 4, 6, 9]})
+)
+@pytest.mark.parametrize(
+    "dtypes",
+    (
+        None,
+        {"A": int, "B": int},
+        {"A": int, "B": pd.CategoricalDtype(["cat1", "cat2", "cat3"], ordered=True)},
+    ),
+)
+@pytest.mark.parametrize("right", (True, False))
+@pytest.mark.parametrize("include_lowest", (True, False))
+def test_custom_kbins_discretizer(
+    bins,
+    dtypes,
+    right,
+    include_lowest,
+):
+    """Tests basic CustomKBinsDiscretizer functionality."""
+
+    col_a = [0.2, 1.4, 2.5, 6.2, 9.7, 2.1]
+    col_b = col_a.copy()
+    col_c = col_a.copy()
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df).repartition(2)
+
+    discretizer = CustomKBinsDiscretizer(
+        ["A", "B"], bins=bins, dtypes=dtypes, right=right, include_lowest=include_lowest
+    )
+
+    transformed = discretizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    if isinstance(bins, dict):
+        bins_A = bins["A"]
+        bins_B = bins["B"]
+    else:
+        bins_A = bins_B = bins
+
+    labels_A = False
+    ordered_A = True
+    labels_B = False
+    ordered_B = True
+    if isinstance(dtypes, dict):
+        if isinstance(dtypes.get("A"), pd.CategoricalDtype):
+            labels_A = dtypes.get("A").categories
+            ordered_A = dtypes.get("A").ordered
+        if isinstance(dtypes.get("B"), pd.CategoricalDtype):
+            labels_B = dtypes.get("B").categories
+            ordered_B = dtypes.get("B").ordered
+
+    # Create expected dataframe with transformed columns
+    expected_df = in_df.copy()
+    expected_df["A"] = pd.cut(
+        in_df["A"],
+        bins_A,
+        labels=labels_A,
+        ordered=ordered_A,
+        right=right,
+        include_lowest=include_lowest,
+    )
+    expected_df["B"] = pd.cut(
+        in_df["B"],
+        bins_B,
+        labels=labels_B,
+        ordered=ordered_B,
+        right=right,
+        include_lowest=include_lowest,
+    )
+
+    # Use rows_same to compare regardless of row ordering
+    assert rows_same(out_df, expected_df)
+
+    # append mode
+    expected_message = "The length of columns and output_columns must match."
+    with pytest.raises(ValueError, match=expected_message):
+        CustomKBinsDiscretizer(["A", "B"], bins=bins, output_columns=["A_discretized"])
+
+    discretizer = CustomKBinsDiscretizer(
+        ["A", "B"],
+        bins=bins,
+        dtypes=dtypes,
+        right=right,
+        include_lowest=include_lowest,
+        output_columns=["A_discretized", "B_discretized"],
+    )
+
+    transformed = discretizer.fit_transform(ds)
+    out_df = transformed.to_pandas()
+
+    # Create expected dataframe with appended columns
+    expected_df = in_df.copy()
+    expected_df["A_discretized"] = pd.cut(
+        in_df["A"],
+        bins_A,
+        labels=labels_A,
+        ordered=ordered_A,
+        right=right,
+        include_lowest=include_lowest,
+    )
+    expected_df["B_discretized"] = pd.cut(
+        in_df["B"],
+        bins_B,
+        labels=labels_B,
+        ordered=ordered_B,
+        right=right,
+        include_lowest=include_lowest,
+    )
+
+    # Use rows_same to compare regardless of row ordering
+    assert rows_same(out_df, expected_df)
+
+
+def test_custom_kbins_discretizer_serialization():
+    """Test CustomKBinsDiscretizer serialization and deserialization functionality."""
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create discretizer
+    discretizer = CustomKBinsDiscretizer(
+        columns=["A"], bins={"A": [0, 1, 2, 3]}, right=True
+    )
+
+    # Serialize using CloudPickle
+    serialized = discretizer.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = CustomKBinsDiscretizer.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, CustomKBinsDiscretizer)
+    assert deserialized.columns == ["A"]
+    assert deserialized.bins == {"A": [0, 1, 2, 3]}
+    assert deserialized.right is True
+
+    # Verify it works correctly
+    df = pd.DataFrame({"A": [0.5, 1.5, 2.5]})
+    result = deserialized.transform_batch(df)
+
+    # Verify discretization was applied correctly
+    assert "A" in result.columns
+    assert len(result) == 3
+
+
+def test_uniform_kbins_discretizer_serialization():
+    """Test UniformKBinsDiscretizer serialization and deserialization functionality."""
+    import ray
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create and fit discretizer
+    discretizer = UniformKBinsDiscretizer(columns=["A"], bins=3)
+    df = pd.DataFrame({"A": [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]})
+    ds = ray.data.from_pandas(df)
+    fitted_discretizer = discretizer.fit(ds)
+
+    # Serialize using CloudPickle
+    serialized = fitted_discretizer.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = UniformKBinsDiscretizer.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, UniformKBinsDiscretizer)
+    assert deserialized._fitted
+    assert deserialized.columns == ["A"]
+    assert deserialized.bins == 3
+
+    # Verify stats are preserved: bin edges for 3 bins = 4 edge values
+    assert "A" in deserialized.stats_
+    assert len(deserialized.stats_["A"]) == 4
+
+    # Verify it works correctly
+    test_df = pd.DataFrame({"A": [1.5, 3.5, 5.5]})
+    result = deserialized.transform_batch(test_df)
+
+    # Verify discretization was applied correctly
+    assert "A" in result.columns
+    assert len(result) == 3
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_hasher.py

@@ -0,0 +1,80 @@
+import pandas as pd
+import pytest
+
+import ray
+from ray.data.preprocessors import FeatureHasher
+
+
+def test_feature_hasher():
+    """Tests basic FeatureHasher functionality."""
+    # This dataframe represents the counts from the documents "I like Python" and "I
+    # dislike Python".
+    token_counts = pd.DataFrame(
+        {"I": [1, 1], "like": [1, 0], "dislike": [0, 1], "Python": [1, 1]}
+    )
+
+    hasher = FeatureHasher(
+        ["I", "like", "dislike", "Python"],
+        num_features=256,
+        output_column="hashed_features",
+    )
+    document_term_matrix = hasher.fit_transform(
+        ray.data.from_pandas(token_counts)
+    ).to_pandas()
+
+    hashed_features = document_term_matrix["hashed_features"]
+    # Document-term matrix should have shape (# documents, # features)
+    assert hashed_features.shape == (2,)
+
+    # The tokens tokens "I", "like", and "Python" should be hashed to distinct indices
+    # for adequately large `num_features`.
+    assert len(hashed_features.iloc[0]) == 256
+    assert hashed_features.iloc[0].sum() == 3
+    assert all(hashed_features.iloc[0] <= 1)
+
+    # The tokens tokens "I", "dislike", and "Python" should be hashed to distinct
+    # indices for adequately large `num_features`.
+    assert len(hashed_features.iloc[1]) == 256
+    assert hashed_features.iloc[1].sum() == 3
+    assert all(hashed_features.iloc[1] <= 1)
+
+
+def test_feature_hasher_serialization():
+    """Test FeatureHasher serialization and deserialization functionality."""
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create hasher
+    hasher = FeatureHasher(
+        columns=["I", "like", "Python"], num_features=8, output_column="hashed"
+    )
+
+    # Serialize using CloudPickle
+    serialized = hasher.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = FeatureHasher.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, FeatureHasher)
+    assert deserialized.columns == ["I", "like", "Python"]
+    assert deserialized.num_features == 8
+    assert deserialized.output_column == "hashed"
+
+    # Verify it works correctly
+    df = pd.DataFrame({"I": [1, 1], "like": [1, 0], "Python": [1, 1]})
+    result = deserialized.transform_batch(df)
+
+    # Verify hashing was applied correctly
+    assert "hashed" in result.columns
+    assert len(result["hashed"][0]) == 8
+    assert len(result["hashed"][1]) == 8
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_imputer.py

@@ -0,0 +1,629 @@
+"""
+Tests for SimpleImputer functionality and serialization.
+
+This file contains:
+1. Basic functional tests for SimpleImputer operations
+2. Comprehensive serialization/deserialization tests
+"""
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import pytest
+
+import ray
+from ray.data._internal.util import rows_same
+from ray.data.preprocessor import (
+    PreprocessorNotFittedException,
+    SerializablePreprocessorBase,
+)
+from ray.data.preprocessors import SimpleImputer
+from ray.data.preprocessors.version_support import UnknownPreprocessorError
+
+
+def test_simple_imputer():
+    col_a = [1, 1, 1, np.nan]
+    col_b = [1, 3, None, np.nan]
+    col_c = [1, 1, 1, 1]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+
+    ds = ray.data.from_pandas(in_df)
+
+    imputer = SimpleImputer(["B", "C"])
+
+    # Transform with unfitted preprocessor.
+    with pytest.raises(PreprocessorNotFittedException):
+        imputer.transform(ds)
+
+    # Fit data.
+    imputer.fit(ds)
+    assert imputer.stats_ == {"mean(B)": 2.0, "mean(C)": 1.0}
+
+    # Transform data.
+    transformed = imputer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [1.0, 3.0, 2.0, 2.0]
+    processed_col_c = [1, 1, 1, 1]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    )
+    expected_df = expected_df.astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df)
+
+    # Transform batch.
+    pred_col_a = [1, 2, np.nan]
+    pred_col_b = [1, 2, np.nan]
+    pred_col_c = [None, None, None]
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+
+    pred_out_df = imputer.transform_batch(pred_in_df)
+
+    pred_processed_col_a = pred_col_a
+    pred_processed_col_b = [1.0, 2.0, 2.0]
+    pred_processed_col_c = [1.0, 1.0, 1.0]
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_processed_col_a,
+            "B": pred_processed_col_b,
+            "C": pred_processed_col_c,
+        }
+    )
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+    # with missing column
+    pred_in_df = pd.DataFrame.from_dict({"A": pred_col_a, "B": pred_col_b})
+    pred_out_df = imputer.transform_batch(pred_in_df)
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_processed_col_a,
+            "B": pred_processed_col_b,
+            "C": pred_processed_col_c,
+        }
+    )
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+    # append mode
+    with pytest.raises(ValueError):
+        SimpleImputer(columns=["B", "C"], output_columns=["B_encoded"])
+
+    imputer = SimpleImputer(
+        columns=["B", "C"],
+        output_columns=["B_imputed", "C_imputed"],
+    )
+    imputer.fit(ds)
+
+    pred_col_a = [1, 2, np.nan]
+    pred_col_b = [1, 2, np.nan]
+    pred_col_c = [None, None, None]
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+    pred_out_df = imputer.transform_batch(pred_in_df)
+
+    pred_processed_col_b = [1.0, 2.0, 2.0]
+    pred_processed_col_c = [1.0, 1.0, 1.0]
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_col_a,
+            "B": pred_col_b,
+            "C": pred_col_c,
+            "B_imputed": pred_processed_col_b,
+            "C_imputed": pred_processed_col_c,
+        }
+    )
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+    # Test "most_frequent" strategy.
+    most_frequent_col_a = [1, 2, 2, None, None, None]
+    # Use 3 "c"s to ensure it's clearly the most frequent (no tie with "b")
+    most_frequent_col_b = [None, "c", "c", "c", "b", "a"]
+    most_frequent_df = pd.DataFrame.from_dict(
+        {"A": most_frequent_col_a, "B": most_frequent_col_b}
+    )
+    most_frequent_ds = ray.data.from_pandas(most_frequent_df).repartition(3)
+
+    most_frequent_imputer = SimpleImputer(["A", "B"], strategy="most_frequent")
+    most_frequent_imputer.fit(most_frequent_ds)
+    assert most_frequent_imputer.stats_ == {
+        "most_frequent(A)": 2.0,
+        "most_frequent(B)": "c",
+    }
+
+    most_frequent_transformed = most_frequent_imputer.transform(most_frequent_ds)
+    most_frequent_out_df = most_frequent_transformed.to_pandas()
+
+    most_frequent_processed_col_a = [1.0, 2.0, 2.0, 2.0, 2.0, 2.0]
+    most_frequent_processed_col_b = ["c", "c", "c", "c", "b", "a"]
+    most_frequent_expected_df = pd.DataFrame.from_dict(
+        {"A": most_frequent_processed_col_a, "B": most_frequent_processed_col_b}
+    )
+
+    assert rows_same(most_frequent_out_df, most_frequent_expected_df)
+
+    # Test "constant" strategy.
+    constant_col_a = ["apple", None]
+    constant_col_b = constant_col_a.copy()
+    constant_df = pd.DataFrame.from_dict({"A": constant_col_a, "B": constant_col_b})
+    # category dtype requires special handling
+    constant_df["B"] = constant_df["B"].astype("category")
+    constant_ds = ray.data.from_pandas(constant_df)
+
+    with pytest.raises(ValueError):
+        SimpleImputer(["A", "B"], strategy="constant")
+
+    constant_imputer = SimpleImputer(
+        ["A", "B"], strategy="constant", fill_value="missing"
+    )
+    constant_transformed = constant_imputer.transform(constant_ds)
+    constant_out_df = constant_transformed.to_pandas()
+
+    constant_processed_col_a = ["apple", "missing"]
+    constant_processed_col_b = constant_processed_col_a.copy()
+    constant_expected_df = pd.DataFrame.from_dict(
+        {"A": constant_processed_col_a, "B": constant_processed_col_b}
+    )
+    constant_expected_df["B"] = constant_expected_df["B"].astype("category")
+    constant_expected_df = constant_expected_df.astype(constant_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(
+        constant_out_df, constant_expected_df, check_like=True
+    )
+
+
+def test_imputer_all_nan_raise_error():
+    data = {
+        "A": [np.nan, np.nan, np.nan, np.nan],
+    }
+    df = pd.DataFrame(data)
+    dataset = ray.data.from_pandas(df)
+
+    imputer = SimpleImputer(columns=["A"], strategy="mean")
+    imputer.fit(dataset)
+
+    with pytest.raises(ValueError):
+        imputer.transform_batch(df)
+
+
+def test_imputer_constant_categorical():
+    data = {
+        "A_cat": ["one", "two", None, "four"],
+    }
+    df = pd.DataFrame(data)
+    df["A_cat"] = df["A_cat"].astype("category")
+    dataset = ray.data.from_pandas(df)
+
+    imputer = SimpleImputer(columns=["A_cat"], strategy="constant", fill_value="three")
+    imputer.fit(dataset)
+
+    transformed_df = imputer.transform_batch(df)
+
+    expected = {
+        "A_cat": ["one", "two", "three", "four"],
+    }
+
+    for column in data.keys():
+        np.testing.assert_array_equal(transformed_df[column].values, expected[column])
+
+    df = pd.DataFrame({"A": [1, 2, 3, 4]})
+    transformed_df = imputer.transform_batch(df)
+
+    expected = {
+        "A": [1, 2, 3, 4],
+        "A_cat": ["three", "three", "three", "three"],
+    }
+
+    for column in df:
+        np.testing.assert_array_equal(transformed_df[column].values, expected[column])
+
+
+class TestSimpleImputerSerialization:
+    """Test CloudPickle-based serialization/deserialization functionality for SimpleImputer."""
+
+    def setup_method(self):
+        """Set up test data."""
+        self.df_numeric = pd.DataFrame(
+            {
+                "temp": [20.0, 25.0, None, 30.0, None],
+                "humidity": [60.0, None, 70.0, 80.0, 65.0],
+                "other": ["a", "b", "c", "d", "e"],  # Non-processed column
+            }
+        )
+
+    def test_basic_serialization(self):
+        """Test basic serialization and deserialization functionality."""
+        # Create and fit a simple imputer
+        imputer = SimpleImputer(columns=["temp", "humidity"], strategy="mean")
+
+        # Create test data
+        df = pd.DataFrame(
+            {
+                "temp": [1.0, 2.0, None, 4.0],
+                "humidity": [None, 2.0, 3.0, 4.0],
+                "other": [1, 2, 3, 4],
+            }
+        )
+
+        # Fit the imputer
+        dataset = ray.data.from_pandas(df)
+        fitted_imputer = imputer.fit(dataset)
+
+        # Serialize using CloudPickle (primary format)
+        serialized = fitted_imputer.serialize()
+
+        # Verify it's binary CloudPickle format
+        assert isinstance(serialized, bytes)
+        assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+        # Deserialize
+        deserialized = SimpleImputer.deserialize(serialized)
+
+        # Verify type and state
+        assert isinstance(deserialized, SimpleImputer)
+        assert deserialized._fitted
+        assert deserialized.columns == ["temp", "humidity"]
+        assert deserialized.strategy == "mean"
+
+        # Verify stats are preserved
+        assert "mean(temp)" in deserialized.stats_
+        assert "mean(humidity)" in deserialized.stats_
+        assert abs(deserialized.stats_["mean(temp)"] - 2.333333) < 0.001
+        assert abs(deserialized.stats_["mean(humidity)"] - 3.0) < 0.001
+
+    def test_serialization_formats(self):
+        """Test serialization and deserialization."""
+        imputer = SimpleImputer(columns=["temp"], strategy="mean")
+        dataset = ray.data.from_pandas(self.df_numeric)
+        fitted_imputer = imputer.fit(dataset)
+
+        # Test CloudPickle format (default)
+        serialized = fitted_imputer.serialize()
+        assert isinstance(serialized, bytes)
+        assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+        # Deserialize and verify it works
+        deserialized = SimpleImputer.deserialize(serialized)
+
+        # Verify it works correctly
+        test_df = pd.DataFrame({"temp": [None, 35.0], "other": [1, 2]})
+        result = deserialized.transform_batch(test_df.copy())
+
+        # Verify the result has the expected structure
+        assert "temp" in result.columns
+        assert "other" in result.columns
+
+    def test_functional_equivalence(self):
+        """Test that deserialized SimpleImputer works identically to original."""
+        # Create and fit original
+        imputer = SimpleImputer(columns=["value"], strategy="mean")
+        train_df = pd.DataFrame({"value": [10, 20, None, 40], "id": [1, 2, 3, 4]})
+        train_dataset = ray.data.from_pandas(train_df)
+        fitted_imputer = imputer.fit(train_dataset)
+
+        # Test data
+        test_df = pd.DataFrame({"value": [None, 50, None], "id": [5, 6, 7]})
+
+        # Transform with original
+        original_result = fitted_imputer.transform_batch(test_df.copy())
+
+        # Serialize, deserialize, and transform (using CloudPickle)
+        serialized = fitted_imputer.serialize()
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+        deserialized_result = deserialized.transform_batch(test_df.copy())
+
+        # Results should be identical
+        pd.testing.assert_frame_equal(original_result, deserialized_result)
+
+        # Verify specific values
+        expected_mean = (10 + 20 + 40) / 3  # 23.333...
+        assert abs(original_result.iloc[0]["value"] - expected_mean) < 1e-10
+        assert abs(deserialized_result.iloc[0]["value"] - expected_mean) < 1e-10
+
+    def test_complex_stats_preservation(self):
+        """Test that CloudPickle perfectly preserves complex stats with various key types."""
+        imputer = SimpleImputer(columns=["A"], strategy="mean")
+
+        # Manually set complex stats that would be problematic for other formats
+        imputer.stats_ = {
+            # Simple stats
+            "mean(A)": 5.0,
+            "count(A)": 100,
+            # Complex key types that CloudPickle handles natively
+            "unique_values(ints)": {1: 0, 2: 1, 3: 2, 4: 3, 5: 4},  # int keys
+            "unique_values(floats)": {1.1: 0, 2.2: 1, 3.3: 2},  # float keys
+            "unique_values(bools)": {True: 0, False: 1},  # bool keys
+            "unique_values(none)": {None: 0},  # None keys
+            "unique_values(tuples)": {
+                ("red", "car"): 0,
+                ("blue", "bike"): 1,
+                (1, 2, 3): 2,
+                ("nested", ("inner", "tuple")): 3,
+            },
+            "unique_values(sets)": {
+                frozenset([1, 2, 3]): 0,
+                frozenset(["a", "b"]): 1,
+            },
+            "unique_values(mixed)": {
+                "string": 0,
+                42: 1,
+                (1, 2): 2,
+                frozenset([3, 4]): 3,
+                None: 4,
+                True: 5,
+            },
+        }
+        imputer._fitted = True
+
+        # Serialize and deserialize (using CloudPickle)
+        serialized = imputer.serialize()
+        deserialized = SimpleImputer.deserialize(serialized)
+
+        # Verify ALL stats are perfectly preserved
+        assert deserialized.stats_ == imputer.stats_
+
+        # Verify specific complex key preservation
+        for stat_name, stat_dict in imputer.stats_.items():
+            if isinstance(stat_dict, dict):
+                original_keys = set(stat_dict.keys())
+                restored_keys = set(deserialized.stats_[stat_name].keys())
+
+                # Keys should be identical (including types)
+                assert original_keys == restored_keys
+
+                # Values should be identical
+                for key in original_keys:
+                    assert stat_dict[key] == deserialized.stats_[stat_name][key]
+
+                # Key types should be preserved
+                for orig_key, rest_key in zip(original_keys, restored_keys):
+                    if orig_key == rest_key:  # Same key
+                        assert type(orig_key) is type(rest_key)
+
+    def test_performance_comparison(self):
+        """Test CloudPickle performance and simplicity."""
+        # Create a large imputer with many stats
+        imputer = SimpleImputer(
+            columns=[f"col_{i}" for i in range(10)], strategy="mean"
+        )
+
+        # Create large stats dictionary
+        large_stats = {}
+        for i in range(10):
+            large_stats[f"mean(col_{i})"] = float(i)
+            large_stats[f"count(col_{i})"] = 1000 + i
+
+            # Add complex key stats that CloudPickle handles natively
+            large_stats[f"unique_values(col_{i})"] = {
+                (f"key_{j}", j): j for j in range(100)  # 100 tuple keys per column
+            }
+
+        imputer.stats_ = large_stats
+        imputer._fitted = True
+
+        # Test serialization performance and correctness (using CloudPickle)
+        start_time = time.time()
+        serialized = imputer.serialize()
+        serialize_time = time.time() - start_time
+
+        start_time = time.time()
+        deserialized = SimpleImputer.deserialize(serialized)
+        deserialize_time = time.time() - start_time
+
+        # Verify correctness
+        assert deserialized.stats_ == imputer.stats_
+        assert len(deserialized.stats_) == len(imputer.stats_)
+
+        # Performance should be reasonable (less than 1 second for this size)
+        assert serialize_time < 1.0
+        assert deserialize_time < 1.0
+
+        # Verify no data loss with complex keys
+        for stat_name in large_stats:
+            if "unique_values" in stat_name:
+                original_keys = set(large_stats[stat_name].keys())
+                restored_keys = set(deserialized.stats_[stat_name].keys())
+                assert original_keys == restored_keys
+
+    def test_cloudpickle_native_support(self):
+        """Test that CloudPickle handles all Python types natively without transformation."""
+        imputer = SimpleImputer(columns=["A"], strategy="mean")
+
+        # Test all the key types that used to require custom transformation
+        test_keys = [
+            # Basic types
+            "string_key",
+            42,  # int
+            3.14,  # float
+            True,  # bool
+            False,  # bool
+            None,  # None
+            # Complex types that CloudPickle handles natively
+            (1, 2, 3),  # tuple
+            ("nested", ("inner", "tuple")),  # nested tuple
+            frozenset([1, 2, 3]),  # frozenset
+            frozenset(["a", "b"]),  # frozenset with strings
+        ]
+
+        # Create stats with all these key types
+        imputer.stats_ = {
+            "test_dict": {key: f"value_{i}" for i, key in enumerate(test_keys)}
+        }
+        imputer._fitted = True
+
+        # Serialize and deserialize (using CloudPickle)
+        serialized = imputer.serialize()
+        deserialized = SimpleImputer.deserialize(serialized)
+
+        # Verify perfect preservation
+        original_dict = imputer.stats_["test_dict"]
+        restored_dict = deserialized.stats_["test_dict"]
+
+        assert len(original_dict) == len(restored_dict)
+
+        # Check each key-value pair and key type preservation
+        for orig_key, orig_value in original_dict.items():
+            # Key should exist and have same value
+            assert orig_key in restored_dict
+            assert restored_dict[orig_key] == orig_value
+
+            # Find the corresponding restored key to check type
+            for rest_key in restored_dict.keys():
+                if rest_key == orig_key:
+                    assert type(orig_key) is type(rest_key)
+                    break
+
+    def test_edge_case_empty_stats(self):
+        """Test serialization with empty stats."""
+        imputer = SimpleImputer(columns=["A"], strategy="constant", fill_value=0)
+        # Constant strategy doesn't need fitting, so stats will be empty
+
+        serialized = imputer.serialize()
+        deserialized = SimpleImputer.deserialize(serialized)
+
+        assert deserialized.stats_ == {}
+        assert deserialized.strategy == "constant"
+        assert deserialized.fill_value == 0
+        assert deserialized._is_fittable is False
+
+    def test_edge_case_none_values(self):
+        """Test serialization with None values in stats."""
+        imputer = SimpleImputer(columns=["A"], strategy="mean")
+        imputer._fitted = True
+        imputer.stats_ = {
+            "mean(A)": None,
+            "count(A)": 0,
+            "complex_dict": {
+                None: "none_key",
+                "none_value": None,
+                (None, "tuple"): "tuple_with_none",
+            },
+        }
+
+        serialized = imputer.serialize()
+        deserialized = SimpleImputer.deserialize(serialized)
+
+        assert deserialized.stats_ == imputer.stats_
+        assert deserialized.stats_["mean(A)"] is None
+        assert None in deserialized.stats_["complex_dict"]
+
+    def test_nested_complex_structures(self):
+        """Test deeply nested complex data structures."""
+        imputer = SimpleImputer(columns=["A"], strategy="mean")
+        imputer._fitted = True
+
+        # Create deeply nested structure with various key types
+        imputer.stats_ = {
+            "nested_structure": {
+                ("level1", "tuple"): {
+                    frozenset([1, 2]): "frozenset_key",
+                    42: {"nested_dict": "value"},
+                    None: [1, 2, 3],
+                    True: {"another": {"level": "deep"}},
+                }
+            }
+        }
+
+        serialized = imputer.serialize()
+        deserialized = SimpleImputer.deserialize(serialized)
+
+        assert deserialized.stats_ == imputer.stats_
+
+        # Verify specific nested access works
+        nested = deserialized.stats_["nested_structure"]
+        tuple_key = ("level1", "tuple")
+        assert tuple_key in nested
+        assert frozenset([1, 2]) in nested[tuple_key]
+
+    def test_unknown_preprocessor_type(self):
+        """Test error when trying to deserialize unknown preprocessor type."""
+        import cloudpickle
+
+        # Create fake serialized data with unknown type
+        unknown_data = {
+            "type": "NonExistentPreprocessor",
+            "version": 1,
+            "fields": {"columns": ["test"]},
+            "stats": {},
+            "stats_type": "cloudpickle",
+        }
+
+        fake_serialized = (
+            SerializablePreprocessorBase.MAGIC_CLOUDPICKLE
+            + cloudpickle.dumps(unknown_data)
+        )
+
+        with pytest.raises(UnknownPreprocessorError) as exc_info:
+            SerializablePreprocessorBase.deserialize(fake_serialized)
+
+        # Verify the exception contains the correct preprocessor type
+        assert exc_info.value.preprocessor_type == "NonExistentPreprocessor"
+        assert "Unknown preprocessor type: NonExistentPreprocessor" in str(
+            exc_info.value
+        )
+
+    def test_file_system_integration(self):
+        """Test integration with file system operations."""
+        imputer = SimpleImputer(columns=["value"], strategy="mean")
+        df = pd.DataFrame({"value": [1, 2, None, 4]})
+        dataset = ray.data.from_pandas(df)
+        fitted = imputer.fit(dataset)
+
+        # Test with binary files (CloudPickle)
+        with tempfile.NamedTemporaryFile(mode="wb", suffix=".cloudpickle") as f:
+            # Save as CloudPickle
+            serialized = fitted.serialize()
+            f.write(serialized)
+            f.flush()
+
+            # Load from file
+            with open(f.name, "rb") as read_f:
+                loaded_data = read_f.read()
+
+            deserialized = SerializablePreprocessorBase.deserialize(loaded_data)
+            assert isinstance(deserialized, SimpleImputer)
+            assert abs(deserialized.stats_["mean(value)"] - 2.333333333333333) < 1e-10
+
+    def test_special_numeric_values(self):
+        """Test serialization with inf, -inf, and NaN values."""
+        # Test with inf fill_value
+        imputer1 = SimpleImputer(columns=["col"], strategy="mean")
+        imputer1.stats_ = {"mean(col)": float("inf")}
+        imputer1._fitted = True
+
+        serialized = imputer1.serialize()
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+        assert np.isinf(deserialized.stats_["mean(col)"])
+
+        # Test with -inf fill_value
+        imputer2 = SimpleImputer(columns=["col"], strategy="mean")
+        imputer2.stats_ = {"mean(col)": float("-inf")}
+        imputer2._fitted = True
+
+        serialized = imputer2.serialize()
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+        assert (
+            np.isinf(deserialized.stats_["mean(col)"])
+            and deserialized.stats_["mean(col)"] < 0
+        )
+
+        # Test with NaN fill_value
+        imputer3 = SimpleImputer(columns=["col"], strategy="mean")
+        imputer3.stats_ = {"mean(col)": float("nan")}
+        imputer3._fitted = True
+
+        serialized = imputer3.serialize()
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+        assert np.isnan(deserialized.stats_["mean(col)"])
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_normalizer.py

@@ -0,0 +1,121 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+import ray
+from ray.data.preprocessors import Normalizer
+
+
+def test_normalizer():
+    """Tests basic Normalizer functionality."""
+
+    col_a = [10, 10, 10]
+    col_b = [1, 3, 3]
+    col_c = [2, 4, -4]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    # l2 norm
+    normalizer = Normalizer(["B", "C"])
+    transformed = normalizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [1 / np.sqrt(5), 0.6, 0.6]
+    processed_col_c = [2 / np.sqrt(5), 0.8, -0.8]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # l1 norm
+    normalizer = Normalizer(["B", "C"], norm="l1")
+
+    transformed = normalizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [1 / 3, 3 / 7, 3 / 7]
+    processed_col_c = [2 / 3, 4 / 7, -4 / 7]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # max norm
+    normalizer = Normalizer(["B", "C"], norm="max")
+
+    transformed = normalizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [0.5, 0.75, 0.75]
+    processed_col_c = [1.0, 1.0, -1.0]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # append mode
+    with pytest.raises(ValueError):
+        Normalizer(columns=["B", "C"], output_columns=["B_encoded"])
+
+    normalizer = Normalizer(["B", "C"], output_columns=["B_normalized", "C_normalized"])
+    transformed = normalizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [1 / np.sqrt(5), 0.6, 0.6]
+    processed_col_c = [2 / np.sqrt(5), 0.8, -0.8]
+    expected_df = pd.DataFrame.from_dict(
+        {
+            "A": col_a,
+            "B": col_b,
+            "C": col_c,
+            "B_normalized": processed_col_b,
+            "C_normalized": processed_col_c,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+
+def test_normalizer_serialization():
+    """Test Normalizer serialization and deserialization functionality."""
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create normalizer with test data
+    normalizer = Normalizer(columns=["A", "B"], norm="l1")
+
+    # Serialize using CloudPickle
+    serialized = normalizer.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = Normalizer.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, Normalizer)
+    assert deserialized.columns == ["A", "B"]
+    assert deserialized.norm == "l1"
+    assert deserialized.output_columns == ["A", "B"]
+
+    # Verify it works correctly
+    df = pd.DataFrame({"A": [3.0, 4.0], "B": [4.0, 3.0]})
+    result = deserialized.transform_batch(df)
+
+    # For l1 norm, values should sum to 1 for each row
+    assert abs(result["A"][0] + result["B"][0] - 1.0) < 1e-10
+    assert abs(result["A"][1] + result["B"][1] - 1.0) < 1e-10
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_preprocessors.py

@@ -0,0 +1,570 @@
+import re
+import warnings
+from typing import Dict, Union
+from unittest.mock import patch
+
+import numpy as np
+import pandas as pd
+import pyarrow
+import pytest
+
+import ray
+from ray.data.aggregate import Mean
+from ray.data.constants import MAX_REPR_LENGTH
+from ray.data.preprocessor import Preprocessor
+from ray.data.preprocessors import (
+    Categorizer,
+    Chain,
+    Concatenator,
+    CountVectorizer,
+    FeatureHasher,
+    HashingVectorizer,
+    LabelEncoder,
+    MaxAbsScaler,
+    MinMaxScaler,
+    MultiHotEncoder,
+    Normalizer,
+    OneHotEncoder,
+    OrdinalEncoder,
+    PowerTransformer,
+    RobustScaler,
+    SimpleImputer,
+    StandardScaler,
+    Tokenizer,
+    TorchVisionPreprocessor,
+)
+from ray.data.util.data_batch_conversion import BatchFormat
+
+
+@pytest.fixture
+def create_dummy_preprocessors():
+    class DummyPreprocessorWithNothing(Preprocessor):
+        _is_fittable = False
+
+    class DummyPreprocessorWithPandas(DummyPreprocessorWithNothing):
+        def _transform_pandas(self, df: "pd.DataFrame") -> "pd.DataFrame":
+            return df
+
+    class DummyPreprocessorWithNumpy(DummyPreprocessorWithNothing):
+        batch_format = "numpy"
+
+        def _transform_numpy(
+            self, np_data: Union[np.ndarray, Dict[str, np.ndarray]]
+        ) -> Union[np.ndarray, Dict[str, np.ndarray]]:
+            return np_data
+
+    class DummyPreprocessorWithPandasAndNumpy(DummyPreprocessorWithNothing):
+        def _transform_pandas(self, df: "pd.DataFrame") -> "pd.DataFrame":
+            return df
+
+        def _transform_numpy(
+            self, np_data: Union[np.ndarray, Dict[str, np.ndarray]]
+        ) -> Union[np.ndarray, Dict[str, np.ndarray]]:
+            return np_data
+
+    class DummyPreprocessorWithPandasAndNumpyPreferred(DummyPreprocessorWithNothing):
+        def _transform_pandas(self, df: "pd.DataFrame") -> "pd.DataFrame":
+            return df
+
+        def _transform_numpy(
+            self, np_data: Union[np.ndarray, Dict[str, np.ndarray]]
+        ) -> Union[np.ndarray, Dict[str, np.ndarray]]:
+            return np_data
+
+        def preferred_batch_format(cls) -> BatchFormat:
+            return BatchFormat.NUMPY
+
+    yield (
+        DummyPreprocessorWithNothing(),
+        DummyPreprocessorWithPandas(),
+        DummyPreprocessorWithNumpy(),
+        DummyPreprocessorWithPandasAndNumpy(),
+        DummyPreprocessorWithPandasAndNumpyPreferred(),
+    )
+
+
+@pytest.mark.parametrize(
+    "preprocessor",
+    [
+        Categorizer(columns=["X"]),
+        CountVectorizer(columns=["X"]),
+        Chain(StandardScaler(columns=["X"]), MinMaxScaler(columns=["X"])),
+        FeatureHasher(columns=["X"], num_features=1, output_column="X_transformed"),
+        HashingVectorizer(columns=["X"], num_features=1),
+        LabelEncoder(label_column="X"),
+        MaxAbsScaler(columns=["X"]),
+        MinMaxScaler(columns=["X"]),
+        MultiHotEncoder(columns=["X"]),
+        Normalizer(columns=["X"]),
+        OneHotEncoder(columns=["X"]),
+        OrdinalEncoder(columns=["X"]),
+        PowerTransformer(columns=["X"], power=1),
+        RobustScaler(columns=["X"]),
+        SimpleImputer(columns=["X"]),
+        StandardScaler(columns=["X"]),
+        Concatenator(columns=["X"]),
+        Tokenizer(columns=["X"]),
+    ],
+)
+def test_repr(preprocessor):
+    representation = repr(preprocessor)
+
+    assert len(representation) < MAX_REPR_LENGTH
+    pattern = re.compile(f"^{preprocessor.__class__.__name__}\\((.*)\\)$")
+    assert pattern.match(representation)
+
+
+def test_fitted_preprocessor_without_stats():
+    """Tests that Preprocessors can be fitted without needing to set self.stats_."""
+
+    class FittablePreprocessor(Preprocessor):
+        def _fit(self, ds):
+            return self
+
+    preprocessor = FittablePreprocessor()
+    ds = ray.data.from_items([1])
+    _ = preprocessor.fit(ds)
+    assert preprocessor.fit_status() == Preprocessor.FitStatus.FITTED
+
+
+def test_fitted_preprocessor_with_stats():
+    """Tests that Preprocessors can be fitted by setting an attribute that ends
+    with _."""
+
+    class FittablePreprocessor(Preprocessor):
+        ...
+
+    preprocessor = FittablePreprocessor()
+    preprocessor.stats_ = True
+    assert preprocessor.fit_status() == Preprocessor.FitStatus.FITTED
+
+
+@patch.object(warnings, "warn")
+def test_fit_twice(mocked_warn):
+    """Tests that a warning msg should be printed."""
+    col_a = [-1, 0, 1]
+    col_b = [1, 3, 5]
+    col_c = [1, 1, None]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    scaler = MinMaxScaler(["B", "C"])
+
+    # Fit data.
+    scaler.fit(ds)
+    assert scaler.stats_ == {"min(B)": 1, "max(B)": 5, "min(C)": 1, "max(C)": 1}
+
+    ds = ds.map_batches(lambda x: {k: v * 2 for k, v in x.items()})
+    # Fit again
+    scaler.fit(ds)
+    # Assert that the fitted state is corresponding to the second ds.
+    assert scaler.stats_ == {"min(B)": 2, "max(B)": 10, "min(C)": 2, "max(C)": 2}
+    msg = (
+        "`fit` has already been called on the preprocessor (or at least one "
+        "contained preprocessors if this is a chain). "
+        "All previously fitted state will be overwritten!"
+    )
+    mocked_warn.assert_called_once_with(msg)
+
+
+def test_fit_twice_clears_stale_stats():
+    """Tests that fit() clears stale stats when stat keys are data-dependent.
+
+    When a preprocessor's stat keys depend on the data (e.g., auto-detected columns),
+    calling fit() again on a different dataset should not retain stale stats from
+    the previous fit. This ensures that fit(A).fit(B) is equivalent to fit(B).
+    """
+
+    class DataDependentPreprocessor(Preprocessor):
+        """A preprocessor whose stat keys depend on the data columns present."""
+
+        _is_fittable = True
+
+        def _fit(self, ds):
+            # Dynamically detect columns from the dataset schema
+            schema = ds.schema()
+            column_names = list(schema.names)
+            self.stat_computation_plan.add_aggregator(
+                aggregator_fn=Mean,
+                columns=column_names,
+            )
+            return self
+
+        def _transform_pandas(self, df):
+            return df
+
+    # Dataset A has columns: "a", "b"
+    dataset_a = ray.data.from_items(
+        [
+            {"a": 1.0, "b": 10.0},
+            {"a": 2.0, "b": 20.0},
+            {"a": 3.0, "b": 30.0},
+        ]
+    )
+
+    # Dataset B has columns: "b", "c" (note: "a" is missing, "c" is new)
+    dataset_b = ray.data.from_items(
+        [
+            {"b": 100.0, "c": 1000.0},
+            {"b": 200.0, "c": 2000.0},
+            {"b": 300.0, "c": 3000.0},
+        ]
+    )
+
+    preprocessor = DataDependentPreprocessor()
+
+    # First fit on dataset A
+    preprocessor.fit(dataset_a)
+    assert preprocessor.stats_ == {"mean(a)": 2.0, "mean(b)": 20.0}
+
+    # Second fit on dataset B - stale stats should be cleared
+    preprocessor.fit(dataset_b)
+
+    # Verify stale stat "mean(a)" is NOT present
+    # Verify stats are correct after refit, and stale stats are cleared.
+    expected_stats = {"mean(b)": 200.0, "mean(c)": 2000.0}
+    assert preprocessor.stats_ == expected_stats, (
+        f"Stats after refit are incorrect. "
+        f"Expected: {expected_stats}, Got: {preprocessor.stats_}"
+    )
+
+
+def test_transform_all_configs():
+    batch_size = 2
+    num_cpus = 2
+    concurrency = 2
+    memory = 1024
+
+    class DummyPreprocessor(Preprocessor):
+        _is_fittable = False
+
+        def _get_transform_config(self):
+            return {"batch_size": batch_size}
+
+        def _transform_numpy(self, data):
+            assert ray.get_runtime_context().get_assigned_resources()["CPU"] == num_cpus
+            assert (
+                ray.get_runtime_context().get_assigned_resources()["memory"] == memory
+            )
+            # Read(10 rows) → Limit(5) → Transform(batch_size=2)
+            assert (
+                len(data["value"]) <= batch_size
+            )  # The last batch is size 1, and limit pushdown resulted in the transform occurring for fewer rows.
+            return data
+
+        def _transform_pandas(self, data):
+            raise RuntimeError(
+                "Pandas transform should not be called with numpy batch format."
+            )
+
+        def _determine_transform_to_use(self):
+            return "numpy"
+
+    prep = DummyPreprocessor()
+    ds = ray.data.from_pandas(pd.DataFrame({"value": list(range(10))}))
+    ds = prep.transform(
+        ds,
+        num_cpus=num_cpus,
+        memory=memory,
+        concurrency=concurrency,
+    )
+    assert [x["value"] for x in ds.take(5)] == [0, 1, 2, 3, 4]
+
+
+@pytest.mark.parametrize("dataset_format", ["simple", "pandas", "arrow"])
+def test_transform_all_formats(create_dummy_preprocessors, dataset_format):
+    (
+        with_nothing,
+        with_pandas,
+        with_numpy,
+        with_pandas_and_numpy,
+        with_pandas_and_numpy_preferred,
+    ) = create_dummy_preprocessors
+
+    if dataset_format == "simple":
+        ds = ray.data.range(10)
+    elif dataset_format == "pandas":
+        df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=["A", "B", "C"])
+        ds = ray.data.from_pandas(df)
+    elif dataset_format == "arrow":
+        df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=["A", "B", "C"])
+        ds = ray.data.from_arrow(pyarrow.Table.from_pandas(df))
+    else:
+        raise ValueError(f"Untested dataset_format configuration: {dataset_format}.")
+
+    with pytest.raises(NotImplementedError):
+        with_nothing.transform(ds)
+
+    patcher = patch.object(ray.data.dataset.Dataset, "map_batches")
+
+    with patcher as mock_map_batches:
+        with_pandas.transform(ds)
+        mock_map_batches.assert_called_once_with(
+            with_pandas._transform_pandas,
+            batch_format=BatchFormat.PANDAS,
+            zero_copy_batch=True,
+        )
+
+    with patcher as mock_map_batches:
+        with_numpy.transform(ds)
+        mock_map_batches.assert_called_once_with(
+            with_numpy._transform_numpy,
+            batch_format=BatchFormat.NUMPY,
+            zero_copy_batch=True,
+        )
+
+    # Pandas preferred by default.
+    with patcher as mock_map_batches:
+        with_pandas_and_numpy.transform(ds)
+    mock_map_batches.assert_called_once_with(
+        with_pandas_and_numpy._transform_pandas,
+        batch_format=BatchFormat.PANDAS,
+        zero_copy_batch=True,
+    )
+
+    with patcher as mock_map_batches:
+        with_pandas_and_numpy_preferred.transform(ds)
+    mock_map_batches.assert_called_once_with(
+        with_pandas_and_numpy_preferred._transform_numpy,
+        batch_format=BatchFormat.NUMPY,
+        zero_copy_batch=True,
+    )
+
+
+def test_numpy_pandas_support_transform_batch_wrong_format(create_dummy_preprocessors):
+    # Case 1: simple dataset. No support
+    (
+        with_nothing,
+        with_pandas,
+        with_numpy,
+        with_pandas_and_numpy,
+        with_pandas_and_numpy_preferred,
+    ) = create_dummy_preprocessors
+
+    batch = [1, 2, 3]
+    with pytest.raises(ValueError):
+        with_nothing.transform_batch(batch)
+
+    with pytest.raises(ValueError):
+        with_pandas.transform_batch(batch)
+
+    with pytest.raises(ValueError):
+        with_numpy.transform_batch(batch)
+
+    with pytest.raises(ValueError):
+        with_pandas_and_numpy.transform_batch(batch)
+
+    with pytest.raises(ValueError):
+        with_pandas_and_numpy_preferred.transform_batch(batch)
+
+
+def test_numpy_pandas_support_transform_batch_pandas(create_dummy_preprocessors):
+    # Case 2: pandas dataset
+    (
+        with_nothing,
+        with_pandas,
+        with_numpy,
+        with_pandas_and_numpy,
+        with_pandas_and_numpy_preferred,
+    ) = create_dummy_preprocessors
+
+    df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=["A", "B", "C"])
+    df_single_column = pd.DataFrame([1, 2, 3], columns=["A"])
+    with pytest.raises(NotImplementedError):
+        with_nothing.transform_batch(df)
+    with pytest.raises(NotImplementedError):
+        with_nothing.transform_batch(df_single_column)
+
+    assert isinstance(with_pandas.transform_batch(df), pd.DataFrame)
+    assert isinstance(with_pandas.transform_batch(df_single_column), pd.DataFrame)
+
+    assert isinstance(with_numpy.transform_batch(df), (np.ndarray, dict))
+    # We can get pd.DataFrame after returning numpy data from UDF
+    assert isinstance(with_numpy.transform_batch(df_single_column), (np.ndarray, dict))
+
+    assert isinstance(with_pandas_and_numpy.transform_batch(df), pd.DataFrame)
+    assert isinstance(
+        with_pandas_and_numpy.transform_batch(df_single_column), pd.DataFrame
+    )
+
+    assert isinstance(
+        with_pandas_and_numpy_preferred.transform_batch(df), (np.ndarray, dict)
+    )
+    assert isinstance(
+        with_pandas_and_numpy_preferred.transform_batch(df_single_column),
+        (np.ndarray, dict),
+    )
+
+
+def test_numpy_pandas_support_transform_batch_arrow(create_dummy_preprocessors):
+    # Case 3: arrow dataset
+    (
+        with_nothing,
+        with_pandas,
+        with_numpy,
+        with_pandas_and_numpy,
+        with_pandas_and_numpy_preferred,
+    ) = create_dummy_preprocessors
+
+    df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=["A", "B", "C"])
+    df_single_column = pd.DataFrame([1, 2, 3], columns=["A"])
+
+    table = pyarrow.Table.from_pandas(df)
+    table_single_column = pyarrow.Table.from_pandas(df_single_column)
+    with pytest.raises(NotImplementedError):
+        with_nothing.transform_batch(table)
+    with pytest.raises(NotImplementedError):
+        with_nothing.transform_batch(table_single_column)
+
+    assert isinstance(with_pandas.transform_batch(table), pd.DataFrame)
+    assert isinstance(with_pandas.transform_batch(table_single_column), pd.DataFrame)
+
+    assert isinstance(with_numpy.transform_batch(table), (np.ndarray, dict))
+    # We can get pyarrow.Table after returning numpy data from UDF
+    assert isinstance(
+        with_numpy.transform_batch(table_single_column), (np.ndarray, dict)
+    )
+
+    assert isinstance(with_pandas_and_numpy.transform_batch(table), pd.DataFrame)
+    assert isinstance(
+        with_pandas_and_numpy.transform_batch(table_single_column), pd.DataFrame
+    )
+
+    assert isinstance(
+        with_pandas_and_numpy_preferred.transform_batch(table), (np.ndarray, dict)
+    )
+    assert isinstance(
+        with_pandas_and_numpy_preferred.transform_batch(table_single_column),
+        (np.ndarray, dict),
+    )
+
+
+def test_numpy_pandas_support_transform_batch_tensor(create_dummy_preprocessors):
+    # Case 4: tensor dataset created by from numpy data directly
+    (
+        with_nothing,
+        with_pandas,
+        with_numpy,
+        with_pandas_and_numpy,
+        with_pandas_and_numpy_preferred,
+    ) = create_dummy_preprocessors
+    np_data = np.arange(12).reshape(3, 2, 2)
+    np_single_column = {"A": np.arange(12).reshape(3, 2, 2)}
+    np_multi_column = {
+        "A": np.arange(12).reshape(3, 2, 2),
+        "B": np.arange(12, 24).reshape(3, 2, 2),
+    }
+
+    with pytest.raises(NotImplementedError):
+        with_nothing.transform_batch(np_data)
+    with pytest.raises(NotImplementedError):
+        with_nothing.transform_batch(np_single_column)
+    with pytest.raises(NotImplementedError):
+        with_nothing.transform_batch(np_multi_column)
+
+    assert isinstance(with_pandas.transform_batch(np_data), pd.DataFrame)
+    assert isinstance(with_pandas.transform_batch(np_single_column), pd.DataFrame)
+    assert isinstance(with_pandas.transform_batch(np_multi_column), pd.DataFrame)
+
+    assert isinstance(with_numpy.transform_batch(np_data), np.ndarray)
+    assert isinstance(with_numpy.transform_batch(np_single_column), dict)
+    assert isinstance(with_numpy.transform_batch(np_multi_column), dict)
+
+    assert isinstance(with_pandas_and_numpy.transform_batch(np_data), pd.DataFrame)
+    assert isinstance(
+        with_pandas_and_numpy.transform_batch(np_single_column), pd.DataFrame
+    )
+    assert isinstance(
+        with_pandas_and_numpy.transform_batch(np_multi_column), pd.DataFrame
+    )
+
+    assert isinstance(
+        with_pandas_and_numpy_preferred.transform_batch(np_data), np.ndarray
+    )
+    assert isinstance(
+        with_pandas_and_numpy_preferred.transform_batch(np_single_column), dict
+    )
+    assert isinstance(
+        with_pandas_and_numpy_preferred.transform_batch(np_multi_column), dict
+    )
+
+
+def test_get_input_output_columns():
+    """Tests get_input_columns() and get_output_columns() methods."""
+    # Test with preprocessors that have columns attribute
+    scaler = StandardScaler(columns=["A", "B"])
+    assert scaler.get_input_columns() == ["A", "B"]
+    assert scaler.get_output_columns() == ["A", "B"]
+
+    # Test with output_columns specified
+    scaler_with_output = StandardScaler(
+        columns=["A", "B"], output_columns=["A_scaled", "B_scaled"]
+    )
+    assert scaler_with_output.get_input_columns() == ["A", "B"]
+    assert scaler_with_output.get_output_columns() == ["A_scaled", "B_scaled"]
+
+    # Test with encoders
+    encoder = OneHotEncoder(columns=["X", "Y"])
+    assert encoder.get_input_columns() == ["X", "Y"]
+    assert encoder.get_output_columns() == ["X", "Y"]
+
+    encoder_with_output = OneHotEncoder(
+        columns=["X", "Y"], output_columns=["X_encoded", "Y_encoded"]
+    )
+    assert encoder_with_output.get_input_columns() == ["X", "Y"]
+    assert encoder_with_output.get_output_columns() == ["X_encoded", "Y_encoded"]
+
+    # Test LabelEncoder without output_column (in-place transformation)
+    label_encoder = LabelEncoder(label_column="target")
+    assert label_encoder.get_input_columns() == ["target"]
+    assert label_encoder.get_output_columns() == ["target"]
+
+    # Test LabelEncoder with output_column (append mode)
+    label_encoder = LabelEncoder(label_column="target", output_column="target_encoded")
+    assert label_encoder.get_input_columns() == ["target"]
+    assert label_encoder.get_output_columns() == ["target_encoded"]
+
+    # Test Concatenator (uses output_column_name instead of output_columns)
+    concatenator = Concatenator(columns=["A", "B"])
+    assert concatenator.get_input_columns() == ["A", "B"]
+    assert concatenator.get_output_columns() == ["concat_out"]
+
+    concatenator_with_output = Concatenator(
+        columns=["A", "B"], output_column_name="AB_concat"
+    )
+    assert concatenator_with_output.get_input_columns() == ["A", "B"]
+    assert concatenator_with_output.get_output_columns() == ["AB_concat"]
+
+    # Test FeatureHasher (uses output_column instead of output_columns)
+    feature_hasher = FeatureHasher(
+        columns=["token1", "token2"], num_features=8, output_column="hashed"
+    )
+    assert feature_hasher.get_input_columns() == ["token1", "token2"]
+    assert feature_hasher.get_output_columns() == ["hashed"]
+
+    # Test TorchVisionPreprocessor (uses _columns and _output_columns)
+    torch_preprocessor = TorchVisionPreprocessor(
+        columns=["image"], transform=lambda x: x
+    )
+    assert torch_preprocessor.get_input_columns() == ["image"]
+    assert torch_preprocessor.get_output_columns() == ["image"]
+
+    torch_preprocessor_with_output = TorchVisionPreprocessor(
+        columns=["image"], transform=lambda x: x, output_columns=["image_transformed"]
+    )
+    assert torch_preprocessor_with_output.get_input_columns() == ["image"]
+    assert torch_preprocessor_with_output.get_output_columns() == ["image_transformed"]
+
+    # Test with preprocessor without columns attribute
+    class CustomPreprocessor(Preprocessor):
+        _is_fittable = False
+
+    custom = CustomPreprocessor()
+    assert custom.get_input_columns() == []
+    assert custom.get_output_columns() == []
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_scaler.py

@@ -0,0 +1,891 @@
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import pytest
+
+import ray
+from ray.data.preprocessor import (
+    PreprocessorNotFittedException,
+    SerializablePreprocessorBase,
+)
+from ray.data.preprocessors import (
+    MaxAbsScaler,
+    MinMaxScaler,
+    RobustScaler,
+    StandardScaler,
+)
+
+
+def test_min_max_scaler():
+    """Tests basic MinMaxScaler functionality."""
+    col_a = [-1, 0, 1]
+    col_b = [1, 3, 5]
+    col_c = [1, 1, None]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    scaler = MinMaxScaler(["B", "C"])
+
+    # Transform with unfitted preprocessor.
+    with pytest.raises(PreprocessorNotFittedException):
+        scaler.transform(ds)
+
+    # Fit data.
+    scaler.fit(ds)
+    assert scaler.stats_ == {"min(B)": 1, "max(B)": 5, "min(C)": 1, "max(C)": 1}
+
+    transformed = scaler.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [0.0, 0.5, 1.0]
+    processed_col_c = [0.0, 0.0, None]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df)
+
+    # Transform batch.
+    pred_col_a = [1, 2, 3]
+    pred_col_b = [3, 5, 7]
+    pred_col_c = [0, 1, 2]
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+
+    pred_out_df = scaler.transform_batch(pred_in_df)
+
+    pred_processed_col_a = pred_col_a
+    pred_processed_col_b = [0.5, 1.0, 1.5]
+    pred_processed_col_c = [-1.0, 0.0, 1.0]
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_processed_col_a,
+            "B": pred_processed_col_b,
+            "C": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df)
+
+    # append mode
+    with pytest.raises(ValueError):
+        MinMaxScaler(columns=["B", "C"], output_columns=["B_mm_scaled"])
+
+    scaler = MinMaxScaler(
+        columns=["B", "C"], output_columns=["B_mm_scaled", "C_mm_scaled"]
+    )
+    scaler.fit(ds)
+
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+    pred_out_df = scaler.transform_batch(pred_in_df)
+
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_col_a,
+            "B": pred_col_b,
+            "C": pred_col_c,
+            "B_mm_scaled": pred_processed_col_b,
+            "C_mm_scaled": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+
+def test_max_abs_scaler():
+    """Tests basic MaxAbsScaler functionality."""
+    col_a = [-1, 0, 1]
+    col_b = [1, 3, -5]
+    col_c = [1, 1, None]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    scaler = MaxAbsScaler(["B", "C"])
+
+    # Transform with unfitted preprocessor.
+    with pytest.raises(PreprocessorNotFittedException):
+        scaler.transform(ds)
+
+    # Fit data.
+    scaler.fit(ds)
+    assert scaler.stats_ == {"abs_max(B)": 5, "abs_max(C)": 1}
+
+    transformed = scaler.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [0.2, 0.6, -1.0]
+    processed_col_c = [1.0, 1.0, None]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Transform batch.
+    pred_col_a = [1, 2, 3]
+    pred_col_b = [3, 5, 7]
+    pred_col_c = [0, 1, -2]
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+
+    pred_out_df = scaler.transform_batch(pred_in_df)
+
+    pred_processed_col_a = pred_col_a
+    pred_processed_col_b = [0.6, 1.0, 1.4]
+    pred_processed_col_c = [0.0, 1.0, -2.0]
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_processed_col_a,
+            "B": pred_processed_col_b,
+            "C": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+    # append mode
+    with pytest.raises(ValueError):
+        MaxAbsScaler(columns=["B", "C"], output_columns=["B_ma_scaled"])
+
+    scaler = MaxAbsScaler(
+        columns=["B", "C"], output_columns=["B_ma_scaled", "C_ma_scaled"]
+    )
+    scaler.fit(ds)
+
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+    pred_out_df = scaler.transform_batch(pred_in_df)
+
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_col_a,
+            "B": pred_col_b,
+            "C": pred_col_c,
+            "B_ma_scaled": pred_processed_col_b,
+            "C_ma_scaled": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+
+def test_robust_scaler():
+    """Tests basic RobustScaler functionality."""
+    col_a = [-2, -1, 0, 1, 2]
+    col_b = [-2, -1, 0, 1, 2]
+    col_c = [-10, 1, 2, 3, 10]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    scaler = RobustScaler(["B", "C"])
+
+    # Transform with unfitted preprocessor.
+    with pytest.raises(PreprocessorNotFittedException):
+        scaler.transform(ds)
+
+    # Fit data.
+    scaler.fit(ds)
+    assert scaler.stats_ == {
+        "low_quantile(B)": -1,
+        "median(B)": 0,
+        "high_quantile(B)": 1,
+        "low_quantile(C)": 1,
+        "median(C)": 2,
+        "high_quantile(C)": 3,
+    }
+
+    transformed = scaler.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [-1.0, -0.5, 0, 0.5, 1.0]
+    processed_col_c = [-6, -0.5, 0, 0.5, 4]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b, "C": processed_col_c}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Transform batch.
+    pred_col_a = [1, 2, 3]
+    pred_col_b = [3, 5, 7]
+    pred_col_c = [0, 1, 2]
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+
+    pred_out_df = scaler.transform_batch(pred_in_df)
+
+    pred_processed_col_a = pred_col_a
+    pred_processed_col_b = [1.5, 2.5, 3.5]
+    pred_processed_col_c = [-1.0, -0.5, 0.0]
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_processed_col_a,
+            "B": pred_processed_col_b,
+            "C": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+    # append mode
+    with pytest.raises(ValueError):
+        RobustScaler(columns=["B", "C"], output_columns=["B_r_scaled"])
+
+    scaler = RobustScaler(
+        columns=["B", "C"], output_columns=["B_r_scaled", "C_r_scaled"]
+    )
+    scaler.fit(ds)
+
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+    pred_out_df = scaler.transform_batch(pred_in_df)
+
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_col_a,
+            "B": pred_col_b,
+            "C": pred_col_c,
+            "B_r_scaled": pred_processed_col_b,
+            "C_r_scaled": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+
+def test_standard_scaler():
+    """Tests basic StandardScaler functionality."""
+    col_a = [-1, 0, 1, 2]
+    col_b = [1, 1, 5, 5]
+    col_c = [1, 1, 1, None]
+    col_d = [None, None, None, None]
+    sample_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c, "D": col_d})
+    ds = ray.data.from_pandas(sample_df)
+
+    scaler = StandardScaler(["B", "C", "D"])
+
+    # Transform with unfitted preprocessor.
+    with pytest.raises(PreprocessorNotFittedException):
+        scaler.transform(ds)
+
+    # Fit data.
+    scaler = scaler.fit(ds)
+    assert scaler.stats_ == {
+        "mean(B)": 3.0,
+        "mean(C)": 1.0,
+        "mean(D)": None,
+        "std(B)": 2.0,
+        "std(C)": 0.0,
+        "std(D)": None,
+    }
+
+    # Transform data.
+    in_col_a = [-1, 0, 1, 2]
+    in_col_b = [1, 1, 5, 5]
+    in_col_c = [1, 1, 1, None]
+    in_col_d = [0, None, None, None]
+    in_df = pd.DataFrame.from_dict(
+        {"A": in_col_a, "B": in_col_b, "C": in_col_c, "D": in_col_d}
+    )
+    in_ds = ray.data.from_pandas(in_df)
+    transformed = scaler.transform(in_ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = col_a
+    processed_col_b = [-1.0, -1.0, 1.0, 1.0]
+    processed_col_c = [0.0, 0.0, 0.0, None]
+    processed_col_d = [np.nan, np.nan, np.nan, np.nan]
+    expected_df = pd.DataFrame.from_dict(
+        {
+            "A": processed_col_a,
+            "B": processed_col_b,
+            "C": processed_col_c,
+            "D": processed_col_d,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Transform batch.
+    pred_col_a = [1, 2, 3]
+    pred_col_b = [3, 5, 7]
+    pred_col_c = [0, 1, 2]
+    pred_col_d = [None, None, None]
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c, "D": pred_col_d}
+    )
+
+    pred_out_df = scaler.transform_batch(pred_in_df)
+
+    pred_processed_col_a = pred_col_a
+    pred_processed_col_b = [0.0, 1.0, 2.0]
+    pred_processed_col_c = [-1.0, 0.0, 1.0]
+    pred_processed_col_d = [None, None, None]
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_processed_col_a,
+            "B": pred_processed_col_b,
+            "C": pred_processed_col_c,
+            "D": pred_processed_col_d,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+    # append mode
+    with pytest.raises(ValueError):
+        StandardScaler(columns=["B", "C"], output_columns=["B_s_scaled"])
+
+    scaler = StandardScaler(
+        columns=["B", "C"], output_columns=["B_s_scaled", "C_s_scaled"]
+    )
+    scaler.fit(ds)
+
+    pred_in_df = pd.DataFrame.from_dict(
+        {"A": pred_col_a, "B": pred_col_b, "C": pred_col_c}
+    )
+    pred_out_df = scaler.transform_batch(pred_in_df)
+
+    pred_expected_df = pd.DataFrame.from_dict(
+        {
+            "A": pred_col_a,
+            "B": pred_col_b,
+            "C": pred_col_c,
+            "B_s_scaled": pred_processed_col_b,
+            "C_s_scaled": pred_processed_col_c,
+        }
+    ).astype(pred_out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(pred_out_df, pred_expected_df, check_like=True)
+
+
+def test_standard_scaler_arrow_transform():
+    """Test the StandardScaler _transform_arrow method directly."""
+    # Create test data
+    col_a = ["red", "green", "blue", "red"]
+    col_b = [1.0, 3.0, 5.0, 7.0]  # mean=4, std=2.236
+    col_c = [10.0, 10.0, 10.0, 10.0]  # constant column, std=0
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b, "C": col_c})
+
+    scaler = StandardScaler(["B", "C"])
+    scaler.fit(ray.data.from_pandas(in_df))
+
+    # Create Arrow table for transformation
+    table = pa.Table.from_pandas(in_df)
+
+    # Transform using Arrow
+    result_table = scaler._transform_arrow(table)
+
+    # Verify result is an Arrow table
+    assert isinstance(result_table, pa.Table)
+
+    # Convert to pandas for easier comparison
+    result_df = result_table.to_pandas()
+
+    # Expected encoding:
+    # B: (x - mean(B)) / std(B)
+    # C: std(C)=0 -> std becomes 1 -> (x - mean(C)) / 1 = 0 for all
+    b_mean = scaler.stats_["mean(B)"]
+    b_std = scaler.stats_["std(B)"] or 0.0
+    if b_std == 0:
+        b_std = 1
+    expected_col_b = [(x - b_mean) / b_std for x in col_b]
+
+    c_mean = scaler.stats_["mean(C)"]
+    c_std = scaler.stats_["std(C)"] or 0.0
+    if c_std == 0:
+        c_std = 1
+    expected_col_c = [(x - c_mean) / c_std for x in col_c]
+
+    assert result_df["A"].tolist() == col_a, "Column A should be unchanged"
+    assert np.allclose(
+        result_df["B"].tolist(), expected_col_b
+    ), f"Column B mismatch: {result_df['B'].tolist()}"
+    assert np.allclose(
+        result_df["C"].tolist(), expected_col_c
+    ), f"Column C mismatch: {result_df['C'].tolist()}"
+
+
+def test_standard_scaler_arrow_transform_append_mode():
+    """Test the StandardScaler _transform_arrow method in append mode."""
+    col_a = ["red", "green", "blue"]
+    col_b = [1.0, 3.0, 5.0]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+
+    scaler = StandardScaler(["B"], output_columns=["B_scaled"])
+    scaler.fit(ray.data.from_pandas(in_df))
+
+    table = pa.Table.from_pandas(in_df)
+    result_table = scaler._transform_arrow(table)
+    result_df = result_table.to_pandas()
+
+    # Original columns should be unchanged
+    assert result_df["A"].tolist() == col_a
+    assert result_df["B"].tolist() == col_b
+
+    # New column should have scaled values: (x - 3) / 2
+    b_mean = scaler.stats_["mean(B)"]
+    b_std = scaler.stats_["std(B)"] or 0.0
+    if b_std == 0:
+        b_std = 1
+    expected_b_scaled = [(x - b_mean) / b_std for x in col_b]
+    assert np.allclose(result_df["B_scaled"].tolist(), expected_b_scaled)
+
+
+def test_standard_scaler_arrow_transform_null_stats():
+    """Test the StandardScaler _transform_arrow method with null mean/std."""
+    # Use an all-null column to produce null mean/std during fit.
+    in_df = pd.DataFrame.from_dict({"A": [None, None, None]})
+
+    scaler = StandardScaler(["A"])
+    scaler.fit(ray.data.from_pandas(in_df))
+
+    table = pa.Table.from_pandas(in_df)
+    result_table = scaler._transform_arrow(table)
+    result_df = result_table.to_pandas()
+
+    # All values should be null when mean/std is None
+    assert result_df["A"].isna().all(), "All values should be null when stats are None"
+
+
+def test_standard_scaler_arrow_transform_overlapping_columns():
+    """Test StandardScaler _transform_arrow with overlapping input/output columns.
+
+    This tests the case where output_columns[i] == columns[j] for i < j.
+    The Arrow implementation must read all input columns before writing any output
+    to avoid corrupting data that will be read later.
+    """
+    # columns=['A', 'B'], output_columns=['B', 'C']
+    # Without the fix, B would be overwritten before being read as input
+    col_a = [2.0, 4.0, 6.0]  # mean=4, std=2 -> scaled: [-1, 0, 1]
+    col_b = [10.0, 20.0, 30.0]  # mean=20, std=10 -> scaled: [-1, 0, 1]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+
+    scaler = StandardScaler(["A", "B"], output_columns=["B", "C"])
+    scaler.fit(ray.data.from_pandas(in_df))
+
+    # Test Arrow transform
+    table = pa.Table.from_pandas(in_df)
+    result_table = scaler._transform_arrow(table)
+    result_df = result_table.to_pandas()
+
+    # Test pandas transform for comparison
+    pandas_result = scaler._transform_pandas(in_df.copy())
+
+    # Column A should be unchanged (not in output_columns with same index)
+    assert result_df["A"].tolist() == col_a, "Column A should be unchanged"
+
+    # Column B should contain scaled A: (A - 4) / 2 = [-1, 0, 1]
+    a_mean = scaler.stats_["mean(A)"]
+    a_std = scaler.stats_["std(A)"] or 0.0
+    if a_std == 0:
+        a_std = 1
+    expected_b = [(x - a_mean) / a_std for x in col_a]
+    assert np.allclose(result_df["B"].tolist(), expected_b), (
+        f"Column B should contain scaled A. Expected {expected_b}, "
+        f"got {result_df['B'].tolist()}"
+    )
+
+    # Column C should contain scaled B: (B - 20) / 10 = [-1, 0, 1]
+    b_mean = scaler.stats_["mean(B)"]
+    b_std = scaler.stats_["std(B)"] or 0.0
+    if b_std == 0:
+        b_std = 1
+    expected_c = [(x - b_mean) / b_std for x in col_b]
+    assert np.allclose(result_df["C"].tolist(), expected_c), (
+        f"Column C should contain scaled B. Expected {expected_c}, "
+        f"got {result_df['C'].tolist()}"
+    )
+
+    # Arrow and pandas results should match
+    pd.testing.assert_frame_equal(
+        result_df,
+        pandas_result,
+        check_like=True,
+        obj="Arrow vs Pandas transform results should match",
+    )
+
+
+class TestScalerSerialization:
+    """Test serialization/deserialization functionality for scaler preprocessors."""
+
+    def setup_method(self):
+        """Set up test data."""
+        self.test_df = pd.DataFrame(
+            {
+                "feature1": [1, 2, 3, 4, 5],
+                "feature2": [10, 20, 30, 40, 50],
+                "feature3": [100, 200, 300, 400, 500],
+                "other": ["a", "b", "c", "d", "e"],
+            }
+        )
+        self.test_dataset = ray.data.from_pandas(self.test_df)
+
+    @pytest.mark.parametrize(
+        "scaler_class,fit_data,expected_stats,transform_data",
+        [
+            (
+                StandardScaler,
+                None,  # Use default self.test_df
+                {
+                    "mean(feature1)": 3.0,
+                    "mean(feature2)": 30.0,
+                    "std(feature1)": np.sqrt(2.0),
+                    "std(feature2)": np.sqrt(200.0),
+                },
+                pd.DataFrame(
+                    {
+                        "feature1": [6, 7, 8],
+                        "feature2": [60, 70, 80],
+                        "other": ["f", "g", "h"],
+                    }
+                ),
+            ),
+            (
+                MinMaxScaler,
+                None,  # Use default self.test_df
+                {
+                    "min(feature1)": 1,
+                    "min(feature2)": 10,
+                    "max(feature1)": 5,
+                    "max(feature2)": 50,
+                },
+                pd.DataFrame(
+                    {
+                        "feature1": [6, 7, 8],
+                        "feature2": [60, 70, 80],
+                        "other": ["f", "g", "h"],
+                    }
+                ),
+            ),
+            (
+                MaxAbsScaler,
+                pd.DataFrame(
+                    {
+                        "feature1": [-5, -2, 0, 2, 5],
+                        "feature2": [-50, -20, 0, 20, 50],
+                        "other": ["a", "b", "c", "d", "e"],
+                    }
+                ),
+                {
+                    "abs_max(feature1)": 5,
+                    "abs_max(feature2)": 50,
+                },
+                pd.DataFrame(
+                    {
+                        "feature1": [-6, 0, 6],
+                        "feature2": [-60, 0, 60],
+                        "other": ["f", "g", "h"],
+                    }
+                ),
+            ),
+            (
+                RobustScaler,
+                None,  # Use default self.test_df
+                {
+                    "low_quantile(feature1)": 2.0,
+                    "median(feature1)": 3.0,
+                    "high_quantile(feature1)": 4.0,
+                    "low_quantile(feature2)": 20.0,
+                    "median(feature2)": 30.0,
+                    "high_quantile(feature2)": 40.0,
+                },
+                pd.DataFrame(
+                    {
+                        "feature1": [6, 7, 8],
+                        "feature2": [60, 70, 80],
+                        "other": ["f", "g", "h"],
+                    }
+                ),
+            ),
+        ],
+        ids=["StandardScaler", "MinMaxScaler", "MaxAbsScaler", "RobustScaler"],
+    )
+    def test_scaler_serialization(
+        self, scaler_class, fit_data, expected_stats, transform_data
+    ):
+        """Test scaler serialization for all scaler types."""
+        # Use custom fit data if provided, otherwise use default test dataset
+        if fit_data is not None:
+            fit_dataset = ray.data.from_pandas(fit_data)
+        else:
+            fit_dataset = self.test_dataset
+
+        # Create and fit scaler
+        scaler = scaler_class(columns=["feature1", "feature2"])
+        fitted_scaler = scaler.fit(fit_dataset)
+
+        # Verify fitted stats match expected values
+        assert fitted_scaler.stats_ == expected_stats, (
+            f"Stats mismatch for {scaler_class.__name__}:\n"
+            f"Expected: {expected_stats}\n"
+            f"Got: {fitted_scaler.stats_}"
+        )
+
+        # Test CloudPickle serialization
+        serialized = fitted_scaler.serialize()
+        assert isinstance(serialized, bytes)
+        assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+        # Test deserialization
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+        assert deserialized.__class__.__name__ == scaler_class.__name__
+        assert deserialized.columns == ["feature1", "feature2"]
+        assert deserialized._fitted
+
+        # Verify stats are preserved after deserialization
+        assert deserialized.stats_ == expected_stats, (
+            f"Deserialized stats mismatch for {scaler_class.__name__}:\n"
+            f"Expected: {expected_stats}\n"
+            f"Got: {deserialized.stats_}"
+        )
+
+        # Verify each stat key exists and has correct value
+        for stat_key, stat_value in expected_stats.items():
+            assert stat_key in deserialized.stats_
+            if isinstance(stat_value, float):
+                assert np.isclose(deserialized.stats_[stat_key], stat_value)
+            else:
+                assert deserialized.stats_[stat_key] == stat_value
+
+        # Test functional equivalence
+        original_result = fitted_scaler.transform_batch(transform_data.copy())
+        deserialized_result = deserialized.transform_batch(transform_data.copy())
+
+        pd.testing.assert_frame_equal(original_result, deserialized_result)
+
+    def test_scaler_with_output_columns_serialization(self):
+        """Test scaler serialization with custom output columns."""
+        # Test with StandardScaler and output columns
+        scaler = StandardScaler(
+            columns=["feature1", "feature2"],
+            output_columns=["scaled_feature1", "scaled_feature2"],
+        )
+        fitted_scaler = scaler.fit(self.test_dataset)
+
+        # Serialize and deserialize
+        serialized = fitted_scaler.serialize()
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+
+        # Verify output columns are preserved
+        assert deserialized.output_columns == ["scaled_feature1", "scaled_feature2"]
+
+        # Test functional equivalence
+        test_df = pd.DataFrame(
+            {"feature1": [6, 7, 8], "feature2": [60, 70, 80], "other": ["f", "g", "h"]}
+        )
+
+        original_result = fitted_scaler.transform_batch(test_df.copy())
+        deserialized_result = deserialized.transform_batch(test_df.copy())
+
+        pd.testing.assert_frame_equal(original_result, deserialized_result)
+
+    @pytest.mark.parametrize(
+        "scaler_class",
+        [StandardScaler, MinMaxScaler, MaxAbsScaler, RobustScaler],
+        ids=["StandardScaler", "MinMaxScaler", "MaxAbsScaler", "RobustScaler"],
+    )
+    def test_unfitted_scaler_serialization(self, scaler_class):
+        """Test serialization of unfitted scalers."""
+        # Test unfitted scaler
+        scaler = scaler_class(columns=["feature1", "feature2"])
+
+        # Serialize unfitted scaler
+        serialized = scaler.serialize()
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+
+        # Verify it's still unfitted
+        assert not deserialized._fitted
+        assert deserialized.columns == ["feature1", "feature2"]
+        assert deserialized.__class__.__name__ == scaler_class.__name__
+
+        # Should raise error when trying to transform
+        test_df = pd.DataFrame({"feature1": [1, 2, 3], "feature2": [10, 20, 30]})
+        with pytest.raises(PreprocessorNotFittedException):
+            deserialized.transform_batch(test_df)
+
+    @pytest.mark.parametrize(
+        "scaler_class,expected_stats",
+        [
+            (
+                StandardScaler,
+                {
+                    "mean(feature1)": 3.0,
+                    "std(feature1)": np.sqrt(2.0),
+                },
+            ),
+            (
+                MinMaxScaler,
+                {
+                    "min(feature1)": 1,
+                    "max(feature1)": 5,
+                },
+            ),
+            (
+                MaxAbsScaler,
+                {
+                    "abs_max(feature1)": 5,
+                },
+            ),
+            (
+                RobustScaler,
+                {
+                    "low_quantile(feature1)": 2.0,
+                    "median(feature1)": 3.0,
+                    "high_quantile(feature1)": 4.0,
+                },
+            ),
+        ],
+        ids=["StandardScaler", "MinMaxScaler", "MaxAbsScaler", "RobustScaler"],
+    )
+    def test_scaler_stats_preservation(self, scaler_class, expected_stats):
+        """Test that scaler statistics are perfectly preserved during serialization."""
+        # Create scaler with known stats
+        scaler = scaler_class(columns=["feature1"])
+        fitted_scaler = scaler.fit(self.test_dataset)
+
+        # Verify fitted stats match expected values
+        for stat_key, stat_value in expected_stats.items():
+            assert stat_key in fitted_scaler.stats_
+            if isinstance(stat_value, float):
+                assert np.isclose(fitted_scaler.stats_[stat_key], stat_value)
+            else:
+                assert fitted_scaler.stats_[stat_key] == stat_value
+
+        # Get original stats
+        original_stats = fitted_scaler.stats_.copy()
+
+        # Serialize and deserialize
+        serialized = fitted_scaler.serialize()
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+
+        # Verify stats are identical
+        assert deserialized.stats_ == original_stats
+
+        # Verify expected stat values are preserved
+        for stat_key, stat_value in expected_stats.items():
+            assert stat_key in deserialized.stats_
+            if isinstance(stat_value, float):
+                assert np.isclose(deserialized.stats_[stat_key], stat_value)
+            else:
+                assert deserialized.stats_[stat_key] == stat_value
+
+    @pytest.mark.parametrize(
+        "scaler_class",
+        [StandardScaler, MinMaxScaler, MaxAbsScaler, RobustScaler],
+        ids=["StandardScaler", "MinMaxScaler", "MaxAbsScaler", "RobustScaler"],
+    )
+    def test_scaler_version_compatibility(self, scaler_class):
+        """Test that scalers can be deserialized with version support."""
+        # Create and fit scaler
+        scaler = scaler_class(columns=["feature1", "feature2"])
+        fitted_scaler = scaler.fit(self.test_dataset)
+
+        # Serialize
+        serialized = fitted_scaler.serialize()
+
+        # Deserialize and verify version handling
+        deserialized = SerializablePreprocessorBase.deserialize(serialized)
+        assert deserialized.__class__.__name__ == scaler_class.__name__
+        assert deserialized._fitted
+
+        # Test that it works correctly
+        test_df = pd.DataFrame({"feature1": [6, 7, 8], "feature2": [60, 70, 80]})
+
+        result = deserialized.transform_batch(test_df)
+        assert len(result.columns) == 2  # Should have the scaled columns
+        assert "feature1" in result.columns
+        assert "feature2" in result.columns
+
+
+def test_standard_scaler_near_zero_std():
+    """Test StandardScaler handles near-zero standard deviation correctly."""
+    # Create data with very small standard deviation (near-constant values)
+    col_a = [1.0, 1.0 + 1e-10, 1.0]
+    col_b = [5, 10, 15]  # Normal column for comparison
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+    ds = ray.data.from_pandas(in_df)
+
+    scaler = StandardScaler(["A", "B"])
+    scaler.fit(ds)
+    transformed = scaler.transform(ds)
+    out_df = transformed.to_pandas()
+
+    # Column A should be scaled to zeros (near-constant)
+    # Instead of NaN or inf values
+    assert np.allclose(
+        out_df["A"], 0.0, atol=1e-6
+    ), "Near-constant column should be scaled to zeros"
+
+    # Column B should be normally scaled
+    assert not np.allclose(out_df["B"], 0.0), "Normal column should not be all zeros"
+
+    # No NaN or inf values should be present
+    assert not out_df["A"].isna().any(), "Should not contain NaN values"
+    assert not np.isinf(out_df["A"]).any(), "Should not contain inf values"
+
+
+def test_min_max_scaler_near_zero_range():
+    """Test MinMaxScaler handles near-zero range correctly."""
+    # Create data with very small range (near-constant values)
+    col_a = [2.0, 2.0 + 1e-10, 2.0]
+    col_b = [1, 5, 10]  # Normal column for comparison
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+    ds = ray.data.from_pandas(in_df)
+
+    scaler = MinMaxScaler(["A", "B"])
+    scaler.fit(ds)
+    transformed = scaler.transform(ds)
+    out_df = transformed.to_pandas()
+
+    # Column A should be scaled to zeros (near-constant)
+    # Instead of NaN or inf values
+    assert np.allclose(
+        out_df["A"], 0.0, atol=1e-6
+    ), "Near-constant column should be scaled to zeros"
+
+    # Column B should be normally scaled
+    expected_b = [0.0, 4 / 9, 1.0]
+    assert np.allclose(
+        out_df["B"], expected_b, atol=1e-6
+    ), "Normal column should be scaled correctly"
+
+    # No NaN or inf values should be present
+    assert not out_df["A"].isna().any(), "Should not contain NaN values"
+    assert not np.isinf(out_df["A"]).any(), "Should not contain inf values"
+
+
+def test_standard_scaler_exact_zero_std():
+    """Test StandardScaler still handles exact zero standard deviation.
+
+    This is a regression test to ensure the epsilon-based handling
+    doesn't break the existing behavior for exact zero std.
+    """
+    # Create constant column (exact zero std)
+    col_c = [5, 5, 5]
+    in_df = pd.DataFrame.from_dict({"C": col_c})
+    ds = ray.data.from_pandas(in_df)
+
+    scaler = StandardScaler(["C"])
+    scaler.fit(ds)
+    transformed = scaler.transform(ds)
+    out_df = transformed.to_pandas()
+
+    # Should be all zeros
+    assert np.allclose(out_df["C"], 0.0), "Constant column should be scaled to zeros"
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_tokenizer.py

@@ -0,0 +1,118 @@
+import pandas as pd
+import pytest
+
+import ray
+from ray.data.preprocessors import Tokenizer
+
+
+def test_tokenizer():
+    """Tests basic Tokenizer functionality."""
+
+    col_a = ["this is a test", "apple"]
+    col_b = ["the quick brown fox jumps over the lazy dog", "banana banana"]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+    ds = ray.data.from_pandas(in_df)
+
+    tokenizer = Tokenizer(["A", "B"])
+    transformed = tokenizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = [["this", "is", "a", "test"], ["apple"]]
+    processed_col_b = [
+        ["the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog"],
+        ["banana", "banana"],
+    ]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Test append mode
+    with pytest.raises(
+        ValueError, match="The length of columns and output_columns must match."
+    ):
+        Tokenizer(columns=["A", "B"], output_columns=["A_tokenized"])
+
+    tokenizer = Tokenizer(
+        columns=["A", "B"], output_columns=["A_tokenized", "B_tokenized"]
+    )
+    transformed = tokenizer.transform(ds)
+    out_df = transformed.to_pandas()
+    print(out_df)
+    expected_df = pd.DataFrame.from_dict(
+        {
+            "A": col_a,
+            "B": col_b,
+            "A_tokenized": processed_col_a,
+            "B_tokenized": processed_col_b,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Test custom tokenization function
+    def custom_tokenizer(s: str) -> list:
+        return s.replace("banana", "fruit").split()
+
+    tokenizer = Tokenizer(
+        columns=["A", "B"],
+        tokenization_fn=custom_tokenizer,
+        output_columns=["A_custom", "B_custom"],
+    )
+    transformed = tokenizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    custom_processed_col_a = [["this", "is", "a", "test"], ["apple"]]
+    custom_processed_col_b = [
+        ["the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog"],
+        ["fruit", "fruit"],
+    ]
+    expected_df = pd.DataFrame.from_dict(
+        {
+            "A": col_a,
+            "B": col_b,
+            "A_custom": custom_processed_col_a,
+            "B_custom": custom_processed_col_b,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+
+def test_tokenizer_serialization():
+    """Test Tokenizer serialization and deserialization functionality."""
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create tokenizer
+    tokenizer = Tokenizer(columns=["text"])
+
+    # Serialize using CloudPickle
+    serialized = tokenizer.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = Tokenizer.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, Tokenizer)
+    assert deserialized.columns == ["text"]
+    assert callable(deserialized.tokenization_fn)
+    assert deserialized.output_columns == ["text"]
+
+    # Verify it works correctly
+    df = pd.DataFrame({"text": ["hello world", "foo bar"]})
+    result = deserialized.transform_batch(df)
+
+    # Verify tokenization was applied correctly
+    assert result["text"][0] == ["hello", "world"]
+    assert result["text"][1] == ["foo", "bar"]
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_torch.py

@@ -0,0 +1,161 @@
+import numpy as np
+import pytest
+import torch
+from torchvision import transforms
+
+import ray
+from ray.data.exceptions import UserCodeException
+from ray.data.preprocessors import TorchVisionPreprocessor
+
+
+class TestTorchVisionPreprocessor:
+    def test_repr(self):
+        class StubTransform:
+            def __call__(self, tensor):
+                return tensor
+
+            def __repr__(self):
+                return "StubTransform()"
+
+        preprocessor = TorchVisionPreprocessor(
+            columns=["spam"], transform=StubTransform()
+        )
+        assert repr(preprocessor) == (
+            "TorchVisionPreprocessor(columns=['spam'], "
+            "output_columns=['spam'], transform=StubTransform())"
+        )
+
+    @pytest.mark.parametrize(
+        "transform",
+        [
+            transforms.ToTensor(),  # `ToTensor` accepts an `np.ndarray` as input
+            transforms.Lambda(lambda tensor: tensor.permute(2, 0, 1)),
+        ],
+    )
+    def test_transform_images(self, transform):
+        dataset = ray.data.from_items(
+            [
+                {"image": np.zeros((32, 32, 3)), "label": 0},
+                {"image": np.zeros((32, 32, 3)), "label": 1},
+            ]
+        )
+        preprocessor = TorchVisionPreprocessor(columns=["image"], transform=transform)
+
+        transformed_dataset = preprocessor.transform(dataset)
+
+        assert transformed_dataset.schema().names == ["image", "label"]
+        transformed_images = [
+            record["image"] for record in transformed_dataset.take_all()
+        ]
+        assert all(image.shape == (3, 32, 32) for image in transformed_images)
+        assert all(image.dtype == np.double for image in transformed_images)
+        labels = {record["label"] for record in transformed_dataset.take_all()}
+        assert labels == {0, 1}
+
+    def test_batch_transform_images(self):
+        dataset = ray.data.from_items(
+            [
+                {"image": np.zeros((32, 32, 3)), "label": 0},
+                {"image": np.zeros((32, 32, 3)), "label": 1},
+            ]
+        )
+        transform = transforms.Compose(
+            [
+                transforms.Lambda(
+                    lambda batch: torch.as_tensor(batch).permute(0, 3, 1, 2)
+                ),
+                transforms.Resize(64),
+            ]
+        )
+        preprocessor = TorchVisionPreprocessor(
+            columns=["image"], transform=transform, batched=True
+        )
+
+        transformed_dataset = preprocessor.transform(dataset)
+
+        assert transformed_dataset.schema().names == ["image", "label"]
+        transformed_images = [
+            record["image"] for record in transformed_dataset.take_all()
+        ]
+        assert all(image.shape == (3, 64, 64) for image in transformed_images)
+        assert all(image.dtype == np.double for image in transformed_images)
+        labels = {record["label"] for record in transformed_dataset.take_all()}
+        assert labels == {0, 1}
+
+    def test_transform_ragged_images(self):
+        dataset = ray.data.from_items(
+            [
+                {"image": np.zeros((16, 16, 3)), "label": 0},
+                {"image": np.zeros((32, 32, 3)), "label": 1},
+            ]
+        )
+        transform = transforms.ToTensor()
+        preprocessor = TorchVisionPreprocessor(columns=["image"], transform=transform)
+
+        transformed_dataset = preprocessor.transform(dataset)
+
+        assert transformed_dataset.schema().names == ["image", "label"]
+        transformed_images = [
+            record["image"] for record in transformed_dataset.take_all()
+        ]
+        assert sorted(image.shape for image in transformed_images) == [
+            (3, 16, 16),
+            (3, 32, 32),
+        ]
+        assert all(image.dtype == np.double for image in transformed_images)
+        labels = {record["label"] for record in transformed_dataset.take_all()}
+        assert labels == {0, 1}
+
+    def test_invalid_transform_raises_value_error(self):
+        dataset = ray.data.from_items(
+            [
+                {"image": np.zeros((32, 32, 3)), "label": 0},
+                {"image": np.zeros((32, 32, 3)), "label": 1},
+            ]
+        )
+        transform = transforms.Lambda(lambda tensor: "BLAH BLAH INVALID")
+        preprocessor = TorchVisionPreprocessor(columns=["image"], transform=transform)
+
+        with pytest.raises((UserCodeException, ValueError)):
+            preprocessor.transform(dataset).materialize()
+
+
+def test_torchvision_preprocessor_serialization():
+    """Test TorchVisionPreprocessor serialization and deserialization functionality."""
+    from torchvision import transforms
+
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create preprocessor
+    transform = transforms.Compose([transforms.ToTensor()])
+    preprocessor = TorchVisionPreprocessor(columns=["image"], transform=transform)
+
+    # Serialize using CloudPickle
+    serialized = preprocessor.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = TorchVisionPreprocessor.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, TorchVisionPreprocessor)
+    assert deserialized.columns == ["image"]
+    assert isinstance(deserialized.torchvision_transform, type(transform))
+
+    # Verify it works correctly
+    test_data = {"image": np.zeros((32, 32, 3), dtype=np.uint8)}
+    result = deserialized.transform_batch(test_data)
+
+    # Verify transformation was applied - ToTensor converts uint8 [0,255] to float [0.0, 1.0]
+    assert "image" in result
+    assert result["image"].dtype in (np.float32, np.float64)
+    assert result["image"].min() >= 0.0 and result["image"].max() <= 1.0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_transformer.py

@@ -0,0 +1,146 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+import ray
+from ray.data.preprocessors import PowerTransformer
+
+
+def test_power_transformer():
+    """Tests basic PowerTransformer functionality."""
+
+    # yeo-johnson
+    col_a = [-1, 0]
+    col_b = [0, 1]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+    ds = ray.data.from_pandas(in_df)
+
+    # yeo-johnson power=0
+    transformer = PowerTransformer(["A", "B"], power=0)
+    transformed = transformer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = [-1.5, 0]
+    processed_col_b = [0, np.log(2)]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # yeo-johnson power=2
+    transformer = PowerTransformer(["A", "B"], power=2)
+    transformed = transformer.transform(ds)
+    out_df = transformed.to_pandas()
+    processed_col_a = [-np.log(2), 0]
+    processed_col_b = [0, 1.5]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # box-cox
+    col_a = [1, 2]
+    col_b = [3, 4]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+    ds = ray.data.from_pandas(in_df)
+
+    # box-cox power=0
+    transformer = PowerTransformer(["A", "B"], power=0, method="box-cox")
+    transformed = transformer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = [0, np.log(2)]
+    processed_col_b = [np.log(3), np.log(4)]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # box-cox power=2
+    transformer = PowerTransformer(["A", "B"], power=2, method="box-cox")
+    transformed = transformer.transform(ds)
+    out_df = transformed.to_pandas()
+    processed_col_a = [0, 1.5]
+    processed_col_b = [4, 7.5]
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Test append mode
+    # First test that providing wrong number of output columns raises error
+    with pytest.raises(
+        ValueError, match="The length of columns and output_columns must match."
+    ):
+        PowerTransformer(columns=["A", "B"], power=2, output_columns=["A_transformed"])
+
+    # Test append mode with correct output columns
+    transformer = PowerTransformer(
+        columns=["A", "B"],
+        power=2,
+        method="box-cox",
+        output_columns=["A_transformed", "B_transformed"],
+    )
+    transformed = transformer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    # Transformed columns should have the expected values
+    processed_col_a = [0, 1.5]
+    processed_col_b = [4, 7.5]
+
+    expected_df = pd.DataFrame(
+        {
+            "A": col_a,
+            "B": col_b,
+            "A_transformed": processed_col_a,
+            "B_transformed": processed_col_b,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+
+def test_power_transformer_serialization():
+    """Test PowerTransformer serialization and deserialization functionality."""
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create transformer with test data
+    transformer = PowerTransformer(columns=["A", "B"], power=2.0, method="yeo-johnson")
+
+    # Serialize using CloudPickle
+    serialized = transformer.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = PowerTransformer.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, PowerTransformer)
+    assert deserialized.columns == ["A", "B"]
+    assert deserialized.power == 2.0
+    assert deserialized.method == "yeo-johnson"
+    assert deserialized.output_columns == ["A", "B"]
+
+    # Verify it works correctly
+    df = pd.DataFrame({"A": [1.0, 2.0, 3.0], "B": [4.0, 5.0, 6.0]})
+    result = deserialized.transform_batch(df.copy())
+
+    # Verify transformation was applied
+    # For power=2, yeo-johnson on positive values: ((x+1)^2 - 1) / 2
+    expected_a_0 = ((1.0 + 1) ** 2.0 - 1) / 2.0
+    assert abs(result["A"][0] - expected_a_0) < 1e-10
+    assert "A" in result.columns
+    assert "B" in result.columns
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_utils.py

@@ -0,0 +1,29 @@
+import pytest
+
+from ray.data.preprocessors.utils import simple_hash, simple_split_tokenizer
+
+
+def test_simple_split_tokenizer():
+    # Tests simple_split_tokenizer.
+    assert simple_split_tokenizer("one_word") == ["one_word"]
+    assert simple_split_tokenizer("two words") == ["two", "words"]
+    assert simple_split_tokenizer("One fish. Two fish.") == [
+        "One",
+        "fish.",
+        "Two",
+        "fish.",
+    ]
+
+
+def test_simple_hash():
+    # Tests simple_hash determinism.
+    assert simple_hash(1, 100) == 15
+    assert simple_hash("a", 100) == 99
+    assert simple_hash("banana", 100) == 10
+    assert simple_hash([1, 2, "apple"], 100) == 58
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/ray/data/tests/preprocessors/test_vectorizer.py

@@ -0,0 +1,234 @@
+from collections import Counter
+
+import pandas as pd
+import pytest
+
+import ray
+from ray.data.preprocessors import CountVectorizer, HashingVectorizer
+
+
+def test_count_vectorizer():
+    """Tests basic CountVectorizer functionality."""
+
+    # Increase data size & repartition to test for
+    # discuss.ray.io/t/xgboost-ray-crashes-when-used-for-multiclass-text-classification
+    row_multiplier = 100000
+
+    col_a = ["a b b c c c", "a a a a c"] * row_multiplier
+    col_b = ["apple", "banana banana banana"] * row_multiplier
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+    ds = ray.data.from_pandas(in_df).repartition(10)
+
+    vectorizer = CountVectorizer(["A", "B"])
+    vectorizer.fit(ds)
+    assert vectorizer.stats_ == {
+        "token_counts(A)": Counter(
+            {"a": 5 * row_multiplier, "c": 4 * row_multiplier, "b": 2 * row_multiplier}
+        ),
+        "token_counts(B)": Counter(
+            {"banana": 3 * row_multiplier, "apple": 1 * row_multiplier}
+        ),
+    }
+
+    transformed = vectorizer.transform(ds)
+    out_df = transformed.to_pandas(limit=float("inf"))
+
+    processed_col_a = [[1, 3, 2], [4, 1, 0]] * row_multiplier
+    processed_col_b = [[0, 1], [3, 0]] * row_multiplier
+
+    expected_df = pd.DataFrame.from_dict(
+        {
+            "A": processed_col_a,
+            "B": processed_col_b,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # max_features
+    vectorizer = CountVectorizer(["A", "B"], max_features=2)
+    vectorizer.fit(ds)
+
+    assert vectorizer.stats_ == {
+        "token_counts(A)": Counter({"a": 5 * row_multiplier, "c": 4 * row_multiplier}),
+        "token_counts(B)": Counter(
+            {"banana": 3 * row_multiplier, "apple": 1 * row_multiplier}
+        ),
+    }
+
+    transformed = vectorizer.transform(ds)
+    out_df = transformed.to_pandas(limit=float("inf"))
+
+    processed_col_a = [[1, 3], [4, 1]] * row_multiplier
+    processed_col_b = [[0, 1], [3, 0]] * row_multiplier
+
+    expected_df = pd.DataFrame.from_dict(
+        {
+            "A": processed_col_a,
+            "B": processed_col_b,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Test append mode
+    with pytest.raises(
+        ValueError, match="The length of columns and output_columns must match."
+    ):
+        CountVectorizer(
+            columns=["A", "B"],
+            output_columns=[
+                "A_counts"
+            ],  # Should provide same number of output columns as input
+        )
+
+    vectorizer = CountVectorizer(["A", "B"], output_columns=["A_counts", "B_counts"])
+    vectorizer.fit(ds)
+
+    transformed = vectorizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = [[1, 3, 2], [4, 1, 0]] * row_multiplier
+    processed_col_b = [[0, 1], [3, 0]] * row_multiplier
+
+    expected_df = pd.DataFrame.from_dict(
+        {
+            "A": col_a,
+            "B": col_b,
+            "A_counts": processed_col_a,
+            "B_counts": processed_col_b,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+
+def test_hashing_vectorizer():
+    """Tests basic HashingVectorizer functionality."""
+
+    col_a = ["a b b c c c", "a a a a c"]
+    col_b = ["apple", "banana banana banana"]
+    in_df = pd.DataFrame.from_dict({"A": col_a, "B": col_b})
+    ds = ray.data.from_pandas(in_df)
+
+    vectorizer = HashingVectorizer(["A", "B"], num_features=3)
+
+    transformed = vectorizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    processed_col_a = [[0, 4, 2], [0, 5, 0]]
+    processed_col_b = [[0, 0, 1], [3, 0, 0]]
+
+    expected_df = pd.DataFrame.from_dict(
+        {"A": processed_col_a, "B": processed_col_b}
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+    # Test append mode
+    with pytest.raises(
+        ValueError, match="The length of columns and output_columns must match."
+    ):
+        HashingVectorizer(
+            columns=["A", "B"],
+            num_features=3,
+            output_columns=[
+                "A_hashed"
+            ],  # Should provide same number of output columns as input
+        )
+
+    vectorizer = HashingVectorizer(
+        ["A", "B"], num_features=3, output_columns=["A_hashed", "B_hashed"]
+    )
+
+    transformed = vectorizer.transform(ds)
+    out_df = transformed.to_pandas()
+
+    expected_df = pd.DataFrame.from_dict(
+        {
+            "A": col_a,
+            "B": col_b,
+            "A_hashed": processed_col_a,
+            "B_hashed": processed_col_b,
+        }
+    ).astype(out_df.dtypes.to_dict())
+
+    pd.testing.assert_frame_equal(out_df, expected_df, check_like=True)
+
+
+def test_hashing_vectorizer_serialization():
+    """Test HashingVectorizer serialization and deserialization functionality."""
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create vectorizer
+    vectorizer = HashingVectorizer(columns=["text"], num_features=16)
+
+    # Serialize using CloudPickle
+    serialized = vectorizer.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = HashingVectorizer.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, HashingVectorizer)
+    assert deserialized.columns == ["text"]
+    assert deserialized.num_features == 16
+    assert callable(deserialized.tokenization_fn)
+    assert deserialized.output_columns == ["text"]
+
+    # Verify it works correctly
+    df = pd.DataFrame({"text": ["hello world", "foo bar"]})
+    result = deserialized.transform_batch(df)
+
+    # Verify vectorization was applied correctly
+    assert "text" in result.columns
+    assert len(result["text"][0]) == 16
+    assert len(result["text"][1]) == 16
+
+
+def test_count_vectorizer_serialization():
+    """Test CountVectorizer serialization and deserialization functionality."""
+    import ray
+    from ray.data.preprocessor import SerializablePreprocessorBase
+
+    # Create and fit vectorizer
+    vectorizer = CountVectorizer(columns=["text"], max_features=5)
+    df = pd.DataFrame({"text": ["hello world", "foo bar", "hello foo"]})
+    ds = ray.data.from_pandas(df)
+    fitted_vectorizer = vectorizer.fit(ds)
+
+    # Serialize using CloudPickle
+    serialized = fitted_vectorizer.serialize()
+
+    # Verify it's binary CloudPickle format
+    assert isinstance(serialized, bytes)
+    assert serialized.startswith(SerializablePreprocessorBase.MAGIC_CLOUDPICKLE)
+
+    # Deserialize
+    deserialized = CountVectorizer.deserialize(serialized)
+
+    # Verify type and field values
+    assert isinstance(deserialized, CountVectorizer)
+    assert deserialized._fitted
+    assert deserialized.columns == ["text"]
+    assert deserialized.max_features == 5
+
+    # Verify stats are preserved
+    assert "token_counts(text)" in deserialized.stats_
+
+    # Verify it works correctly
+    test_df = pd.DataFrame({"text": ["hello world"]})
+    result = deserialized.transform_batch(test_df)
+
+    # Verify vectorization was applied correctly
+    assert "text" in result.columns
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))