chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,50 @@
|
||||
from ray.data.preprocessors.chain import Chain
|
||||
from ray.data.preprocessors.concatenator import Concatenator
|
||||
from ray.data.preprocessors.discretizer import (
|
||||
CustomKBinsDiscretizer,
|
||||
UniformKBinsDiscretizer,
|
||||
)
|
||||
from ray.data.preprocessors.encoder import (
|
||||
Categorizer,
|
||||
LabelEncoder,
|
||||
MultiHotEncoder,
|
||||
OneHotEncoder,
|
||||
OrdinalEncoder,
|
||||
)
|
||||
from ray.data.preprocessors.hasher import FeatureHasher
|
||||
from ray.data.preprocessors.imputer import SimpleImputer
|
||||
from ray.data.preprocessors.normalizer import Normalizer
|
||||
from ray.data.preprocessors.scaler import (
|
||||
MaxAbsScaler,
|
||||
MinMaxScaler,
|
||||
RobustScaler,
|
||||
StandardScaler,
|
||||
)
|
||||
from ray.data.preprocessors.tokenizer import Tokenizer
|
||||
from ray.data.preprocessors.torch import TorchVisionPreprocessor
|
||||
from ray.data.preprocessors.transformer import PowerTransformer
|
||||
from ray.data.preprocessors.vectorizer import CountVectorizer, HashingVectorizer
|
||||
|
||||
__all__ = [
|
||||
"Categorizer",
|
||||
"CountVectorizer",
|
||||
"Chain",
|
||||
"FeatureHasher",
|
||||
"HashingVectorizer",
|
||||
"LabelEncoder",
|
||||
"MaxAbsScaler",
|
||||
"MinMaxScaler",
|
||||
"MultiHotEncoder",
|
||||
"Normalizer",
|
||||
"OneHotEncoder",
|
||||
"OrdinalEncoder",
|
||||
"PowerTransformer",
|
||||
"RobustScaler",
|
||||
"SimpleImputer",
|
||||
"StandardScaler",
|
||||
"Concatenator",
|
||||
"Tokenizer",
|
||||
"TorchVisionPreprocessor",
|
||||
"CustomKBinsDiscretizer",
|
||||
"UniformKBinsDiscretizer",
|
||||
]
|
||||
@@ -0,0 +1,147 @@
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
|
||||
|
||||
from ray.data.preprocessor import Preprocessor, SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import (
|
||||
_PublicField,
|
||||
migrate_private_fields,
|
||||
)
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.data.util.data_batch_conversion import BatchFormat
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.air.data_batch_type import DataBatchType
|
||||
from ray.data.dataset import Dataset
|
||||
|
||||
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.chain")
|
||||
class Chain(SerializablePreprocessorBase):
|
||||
"""Combine multiple preprocessors into a single :py:class:`Preprocessor`.
|
||||
|
||||
When you call ``fit``, each preprocessor is fit on the dataset produced by the
|
||||
preceeding preprocessor's ``fit_transform``.
|
||||
|
||||
Example:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import *
|
||||
>>>
|
||||
>>> df = pd.DataFrame({
|
||||
... "X0": [0, 1, 2],
|
||||
... "X1": [3, 4, 5],
|
||||
... "Y": ["orange", "blue", "orange"],
|
||||
... })
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>>
|
||||
>>> preprocessor = Chain(
|
||||
... StandardScaler(columns=["X0", "X1"]),
|
||||
... Concatenator(columns=["X0", "X1"], output_column_name="X"),
|
||||
... LabelEncoder(label_column="Y")
|
||||
... )
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
Y X
|
||||
0 1 [-1.224744871391589, -1.224744871391589]
|
||||
1 0 [0.0, 0.0]
|
||||
2 1 [1.224744871391589, 1.224744871391589]
|
||||
|
||||
Args:
|
||||
*preprocessors: The preprocessors to sequentially compose.
|
||||
"""
|
||||
|
||||
def fit_status(self):
|
||||
fittable_count = 0
|
||||
fitted_count = 0
|
||||
|
||||
for p in self._preprocessors:
|
||||
if p.fit_status() == Preprocessor.FitStatus.FITTED:
|
||||
fittable_count += 1
|
||||
fitted_count += 1
|
||||
elif p.fit_status() in (
|
||||
Preprocessor.FitStatus.NOT_FITTED,
|
||||
Preprocessor.FitStatus.PARTIALLY_FITTED,
|
||||
):
|
||||
fittable_count += 1
|
||||
else:
|
||||
assert p.fit_status() == Preprocessor.FitStatus.NOT_FITTABLE
|
||||
if fittable_count > 0:
|
||||
if fitted_count == fittable_count:
|
||||
return Preprocessor.FitStatus.FITTED
|
||||
elif fitted_count > 0:
|
||||
return Preprocessor.FitStatus.PARTIALLY_FITTED
|
||||
else:
|
||||
return Preprocessor.FitStatus.NOT_FITTED
|
||||
else:
|
||||
return Preprocessor.FitStatus.NOT_FITTABLE
|
||||
|
||||
def __init__(self, *preprocessors: SerializablePreprocessorBase):
|
||||
super().__init__()
|
||||
self._preprocessors = preprocessors
|
||||
|
||||
@property
|
||||
def preprocessors(self) -> Tuple[SerializablePreprocessorBase, ...]:
|
||||
return self._preprocessors
|
||||
|
||||
def _fit(self, ds: "Dataset") -> SerializablePreprocessorBase:
|
||||
for preprocessor in self._preprocessors[:-1]:
|
||||
ds = preprocessor.fit_transform(ds)
|
||||
self._preprocessors[-1].fit(ds)
|
||||
return self
|
||||
|
||||
def fit_transform(self, ds: "Dataset") -> "Dataset":
|
||||
for preprocessor in self._preprocessors:
|
||||
ds = preprocessor.fit_transform(ds)
|
||||
return ds
|
||||
|
||||
def _transform(
|
||||
self,
|
||||
ds: "Dataset",
|
||||
batch_size: Optional[int],
|
||||
num_cpus: Optional[float] = None,
|
||||
memory: Optional[float] = None,
|
||||
concurrency: Optional[int] = None,
|
||||
) -> "Dataset":
|
||||
for preprocessor in self._preprocessors:
|
||||
ds = preprocessor.transform(
|
||||
ds,
|
||||
batch_size=batch_size,
|
||||
num_cpus=num_cpus,
|
||||
memory=memory,
|
||||
concurrency=concurrency,
|
||||
)
|
||||
return ds
|
||||
|
||||
def _transform_batch(self, df: "DataBatchType") -> "DataBatchType":
|
||||
for preprocessor in self._preprocessors:
|
||||
df = preprocessor.transform_batch(df)
|
||||
return df
|
||||
|
||||
def __repr__(self):
|
||||
arguments = ", ".join(
|
||||
repr(preprocessor) for preprocessor in self._preprocessors
|
||||
)
|
||||
return f"{self.__class__.__name__}({arguments})"
|
||||
|
||||
def _determine_transform_to_use(self) -> BatchFormat:
|
||||
# This is relevant for BatchPrediction.
|
||||
# For Chain preprocessor, we picked the first one as entry point.
|
||||
# TODO (jiaodong): We should revisit if our Chain preprocessor is
|
||||
# still optimal with context of lazy execution.
|
||||
return self._preprocessors[0]._determine_transform_to_use()
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"preprocessors": self._preprocessors,
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._preprocessors = fields["preprocessors"]
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
"""Handle backwards compatibility for old pickled objects."""
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_preprocessors": _PublicField(public_field="preprocessors"),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,222 @@
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import (
|
||||
_PublicField,
|
||||
migrate_private_fields,
|
||||
)
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.concatenator")
|
||||
class Concatenator(SerializablePreprocessorBase):
|
||||
"""Combine numeric columns into a column of type
|
||||
:class:`~ray.data._internal.tensor_extensions.pandas.TensorDtype`. Only columns
|
||||
specified in ``columns`` will be concatenated.
|
||||
|
||||
This preprocessor concatenates numeric columns and stores the result in a new
|
||||
column. The new column contains
|
||||
:class:`~ray.data._internal.tensor_extensions.pandas.TensorArrayElement` objects of
|
||||
shape :math:`(m,)`, where :math:`m` is the number of columns concatenated.
|
||||
The :math:`m` concatenated columns are dropped after concatenation.
|
||||
The preprocessor preserves the order of the columns provided in the ``colummns``
|
||||
argument and will use that order when calling ``transform()`` and ``transform_batch()``.
|
||||
|
||||
Examples:
|
||||
>>> import numpy as np
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import Concatenator
|
||||
|
||||
:py:class:`Concatenator` combines numeric columns into a column of
|
||||
:py:class:`~ray.data._internal.tensor_extensions.pandas.TensorDtype`.
|
||||
|
||||
>>> df = pd.DataFrame({"X0": [0, 3, 1], "X1": [0.5, 0.2, 0.9]})
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>> concatenator = Concatenator(columns=["X0", "X1"])
|
||||
>>> concatenator.transform(ds).to_pandas() # doctest: +SKIP
|
||||
concat_out
|
||||
0 [0.0, 0.5]
|
||||
1 [3.0, 0.2]
|
||||
2 [1.0, 0.9]
|
||||
|
||||
By default, the created column is called `"concat_out"`, but you can specify
|
||||
a different name.
|
||||
|
||||
>>> concatenator = Concatenator(columns=["X0", "X1"], output_column_name="tensor")
|
||||
>>> concatenator.transform(ds).to_pandas() # doctest: +SKIP
|
||||
tensor
|
||||
0 [0.0, 0.5]
|
||||
1 [3.0, 0.2]
|
||||
2 [1.0, 0.9]
|
||||
|
||||
>>> concatenator = Concatenator(columns=["X0", "X1"], dtype=np.float32)
|
||||
>>> concatenator.transform(ds) # doctest: +SKIP
|
||||
Dataset(num_rows=3, schema={Y: object, concat_out: TensorDtype(shape=(2,), dtype=float32)})
|
||||
|
||||
When ``flatten=True``, nested vectors in the columns will be flattened during concatenation:
|
||||
|
||||
>>> df = pd.DataFrame({"X0": [[1, 2], [3, 4]], "X1": [0.5, 0.2]})
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>> concatenator = Concatenator(columns=["X0", "X1"], flatten=True)
|
||||
>>> concatenator.transform(ds).to_pandas() # doctest: +SKIP
|
||||
concat_out
|
||||
0 [1.0, 2.0, 0.5]
|
||||
1 [3.0, 4.0, 0.2]
|
||||
|
||||
Args:
|
||||
columns: A list of columns to concatenate. The provided order of the columns
|
||||
will be retained during concatenation.
|
||||
output_column_name: The desired name for the new column.
|
||||
Defaults to ``"concat_out"``.
|
||||
dtype: The ``dtype`` to convert the output tensors to. If unspecified,
|
||||
the ``dtype`` is determined by standard coercion rules.
|
||||
raise_if_missing: If ``True``, an error is raised if any
|
||||
of the columns in ``columns`` don't exist.
|
||||
Defaults to ``False``.
|
||||
flatten: If ``True``, nested vectors in the columns will be flattened during
|
||||
concatenation. Defaults to ``False``.
|
||||
|
||||
Raises:
|
||||
ValueError: if `raise_if_missing` is `True` and a column in `columns` or
|
||||
doesn't exist in the dataset.
|
||||
""" # noqa: E501
|
||||
|
||||
_is_fittable = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
output_column_name: str = "concat_out",
|
||||
dtype: Optional[np.dtype] = None,
|
||||
raise_if_missing: bool = False,
|
||||
flatten: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._output_column_name = output_column_name
|
||||
self._dtype = dtype
|
||||
self._raise_if_missing = raise_if_missing
|
||||
self._flatten = flatten
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def output_column_name(self) -> str:
|
||||
return self._output_column_name
|
||||
|
||||
@property
|
||||
def dtype(self) -> Optional[np.dtype]:
|
||||
return self._dtype
|
||||
|
||||
@property
|
||||
def raise_if_missing(self) -> bool:
|
||||
return self._raise_if_missing
|
||||
|
||||
@property
|
||||
def flatten(self) -> bool:
|
||||
return self._flatten
|
||||
|
||||
def _validate(self, df: pd.DataFrame) -> None:
|
||||
missing_columns = set(self._columns) - set(df)
|
||||
if missing_columns:
|
||||
message = (
|
||||
f"Missing columns specified in '{self._columns}': {missing_columns}"
|
||||
)
|
||||
if self._raise_if_missing:
|
||||
raise ValueError(message)
|
||||
else:
|
||||
logger.warning(message)
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
self._validate(df)
|
||||
|
||||
if self._flatten:
|
||||
concatenated = df[self._columns].to_numpy()
|
||||
concatenated = [
|
||||
np.concatenate(
|
||||
[
|
||||
np.atleast_1d(elem)
|
||||
if self._dtype is None
|
||||
else np.atleast_1d(elem).astype(self._dtype)
|
||||
for elem in row
|
||||
]
|
||||
)
|
||||
for row in concatenated
|
||||
]
|
||||
else:
|
||||
concatenated = df[self._columns].to_numpy(dtype=self._dtype)
|
||||
|
||||
df = df.drop(columns=self._columns)
|
||||
# Use a Pandas Series for column assignment to get more consistent
|
||||
# behavior across Pandas versions.
|
||||
df.loc[:, self._output_column_name] = pd.Series(list(concatenated))
|
||||
return df
|
||||
|
||||
def get_input_columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
def get_output_columns(self) -> List[str]:
|
||||
return [self._output_column_name]
|
||||
|
||||
def __repr__(self):
|
||||
default_values = {
|
||||
"output_column_name": "concat_out",
|
||||
"columns": None,
|
||||
"dtype": None,
|
||||
"raise_if_missing": False,
|
||||
"flatten": False,
|
||||
}
|
||||
|
||||
non_default_arguments = []
|
||||
for parameter, default_value in default_values.items():
|
||||
value = getattr(self, parameter)
|
||||
if value != default_value:
|
||||
non_default_arguments.append(f"{parameter}={value}")
|
||||
|
||||
return f"{self.__class__.__name__}({', '.join(non_default_arguments)})"
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"output_column_name": self._output_column_name,
|
||||
"dtype": self._dtype,
|
||||
"raise_if_missing": self._raise_if_missing,
|
||||
"flatten": getattr(self, "_flatten", False),
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._output_column_name = fields["output_column_name"]
|
||||
self._dtype = fields["dtype"]
|
||||
self._raise_if_missing = fields["raise_if_missing"]
|
||||
# optional fields (flatten was added later)
|
||||
self._flatten = fields.get("flatten", False)
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_output_column_name": _PublicField(
|
||||
public_field="output_column_name", default="concat_out"
|
||||
),
|
||||
"_dtype": _PublicField(public_field="dtype", default=None),
|
||||
"_raise_if_missing": _PublicField(
|
||||
public_field="raise_if_missing", default=False
|
||||
),
|
||||
"_flatten": _PublicField(public_field="flatten", default=False),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,596 @@
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Type, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from ray.data.aggregate import Max, Min
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import (
|
||||
_Computed,
|
||||
_PublicField,
|
||||
migrate_private_fields,
|
||||
)
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data.dataset import Dataset
|
||||
|
||||
|
||||
class _AbstractKBinsDiscretizer(SerializablePreprocessorBase):
|
||||
"""Abstract base class for all KBinsDiscretizers.
|
||||
|
||||
Essentially a thin wraper around ``pd.cut``.
|
||||
|
||||
Expects either ``self.stats_`` or ``self.bins`` to be set and
|
||||
contain {column:list_of_bin_intervals}.
|
||||
"""
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
def bin_values(s: pd.Series) -> pd.Series:
|
||||
if s.name not in self.columns:
|
||||
return s
|
||||
labels = self.dtypes.get(s.name) if self.dtypes else False
|
||||
ordered = True
|
||||
if labels:
|
||||
if isinstance(labels, pd.CategoricalDtype):
|
||||
ordered = labels.ordered
|
||||
labels = list(labels.categories)
|
||||
else:
|
||||
labels = False
|
||||
|
||||
bins = self.stats_ if self._is_fittable else self.bins
|
||||
return pd.cut(
|
||||
s,
|
||||
bins[s.name] if isinstance(bins, dict) else bins,
|
||||
right=self.right,
|
||||
labels=labels,
|
||||
ordered=ordered,
|
||||
retbins=False,
|
||||
include_lowest=self.include_lowest,
|
||||
duplicates=self.duplicates,
|
||||
)
|
||||
|
||||
binned_df = df.apply(bin_values, axis=0)
|
||||
df[self.output_columns] = binned_df[self.columns]
|
||||
return df
|
||||
|
||||
def _validate_bins_columns(self):
|
||||
if isinstance(self.bins, dict) and not all(
|
||||
col in self.bins for col in self.columns
|
||||
):
|
||||
raise ValueError(
|
||||
"If `bins` is a dictionary, all elements of `columns` must be present "
|
||||
"in it."
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"{self.__class__.__name__}("
|
||||
f"columns={self.columns!r}, "
|
||||
f"bins={self.bins!r}, "
|
||||
f"right={self.right!r}, "
|
||||
f"include_lowest={self.include_lowest!r}, "
|
||||
f"duplicates={self.duplicates!r}, "
|
||||
f"dtypes={self.dtypes!r}, "
|
||||
f"output_columns={self.output_columns!r})"
|
||||
)
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(
|
||||
version=1, identifier="io.ray.preprocessors.custom_kbins_discretizer"
|
||||
)
|
||||
class CustomKBinsDiscretizer(_AbstractKBinsDiscretizer):
|
||||
"""Bin values into discrete intervals using custom bin edges.
|
||||
|
||||
Columns must contain numerical values.
|
||||
|
||||
Examples:
|
||||
Use :class:`CustomKBinsDiscretizer` to bin continuous features.
|
||||
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import CustomKBinsDiscretizer
|
||||
>>> df = pd.DataFrame({
|
||||
... "value_1": [0.2, 1.4, 2.5, 6.2, 9.7, 2.1],
|
||||
... "value_2": [10, 15, 13, 12, 23, 25],
|
||||
... })
|
||||
>>> ds = ray.data.from_pandas(df)
|
||||
>>> discretizer = CustomKBinsDiscretizer(
|
||||
... columns=["value_1", "value_2"],
|
||||
... bins=[0, 1, 4, 10, 25]
|
||||
... )
|
||||
>>> discretizer.transform(ds).to_pandas()
|
||||
value_1 value_2
|
||||
0 0 2
|
||||
1 1 3
|
||||
2 1 3
|
||||
3 2 3
|
||||
4 2 3
|
||||
5 1 3
|
||||
|
||||
:class:`CustomKBinsDiscretizer` can also be used in append mode by providing the
|
||||
name of the output_columns that should hold the encoded values.
|
||||
|
||||
>>> discretizer = CustomKBinsDiscretizer(
|
||||
... columns=["value_1", "value_2"],
|
||||
... bins=[0, 1, 4, 10, 25],
|
||||
... output_columns=["value_1_discretized", "value_2_discretized"]
|
||||
... )
|
||||
>>> discretizer.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
value_1 value_2 value_1_discretized value_2_discretized
|
||||
0 0.2 10 0 2
|
||||
1 1.4 15 1 3
|
||||
2 2.5 13 1 3
|
||||
3 6.2 12 2 3
|
||||
4 9.7 23 2 3
|
||||
5 2.1 25 1 3
|
||||
|
||||
You can also specify different bin edges per column.
|
||||
|
||||
>>> discretizer = CustomKBinsDiscretizer(
|
||||
... columns=["value_1", "value_2"],
|
||||
... bins={"value_1": [0, 1, 4], "value_2": [0, 18, 35, 70]},
|
||||
... )
|
||||
>>> discretizer.transform(ds).to_pandas()
|
||||
value_1 value_2
|
||||
0 0.0 0
|
||||
1 1.0 0
|
||||
2 1.0 0
|
||||
3 <NA> 0
|
||||
4 <NA> 1
|
||||
5 1.0 1
|
||||
|
||||
|
||||
Args:
|
||||
columns: The columns to discretize.
|
||||
bins: Defines custom bin edges. Can be an iterable of numbers,
|
||||
a ``pd.IntervalIndex``, or a dict mapping columns to either of them.
|
||||
Note that ``pd.IntervalIndex`` for bins must be non-overlapping.
|
||||
right: Indicates whether bins include the rightmost edge.
|
||||
include_lowest: Indicates whether the first interval should be left-inclusive.
|
||||
duplicates: Can be either 'raise' or 'drop'. If bin edges are not unique,
|
||||
raise ``ValueError`` or drop non-uniques.
|
||||
dtypes: An optional dictionary that maps columns to ``pd.CategoricalDtype``
|
||||
objects or ``np.integer`` types. If you don't include a column in ``dtypes``
|
||||
or specify it as an integer dtype, the outputted column will consist of
|
||||
ordered integers corresponding to bins. If you use a
|
||||
``pd.CategoricalDtype``, the outputted column will be a
|
||||
``pd.CategoricalDtype`` with the categories being mapped to bins.
|
||||
You can use ``pd.CategoricalDtype(categories, ordered=True)`` to
|
||||
preserve information about bin order.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
|
||||
.. seealso::
|
||||
|
||||
:class:`UniformKBinsDiscretizer`
|
||||
If you want to bin data into uniform width bins.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
bins: Union[
|
||||
Iterable[float],
|
||||
pd.IntervalIndex,
|
||||
Dict[str, Union[Iterable[float], pd.IntervalIndex]],
|
||||
],
|
||||
*,
|
||||
right: bool = True,
|
||||
include_lowest: bool = False,
|
||||
duplicates: str = "raise",
|
||||
dtypes: Optional[
|
||||
Dict[str, Union[pd.CategoricalDtype, Type[np.integer]]]
|
||||
] = None,
|
||||
output_columns: Optional[List[str]] = None,
|
||||
):
|
||||
self._columns = columns
|
||||
self._bins = bins
|
||||
self._right = right
|
||||
self._include_lowest = include_lowest
|
||||
self._duplicates = duplicates
|
||||
self._dtypes = dtypes
|
||||
self._output_columns = (
|
||||
SerializablePreprocessorBase._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
)
|
||||
|
||||
self._validate_bins_columns()
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def bins(
|
||||
self,
|
||||
) -> Union[
|
||||
Iterable[float],
|
||||
pd.IntervalIndex,
|
||||
Dict[str, Union[Iterable[float], pd.IntervalIndex]],
|
||||
]:
|
||||
return self._bins
|
||||
|
||||
@property
|
||||
def right(self) -> bool:
|
||||
return self._right
|
||||
|
||||
@property
|
||||
def include_lowest(self) -> bool:
|
||||
return self._include_lowest
|
||||
|
||||
@property
|
||||
def duplicates(self) -> str:
|
||||
return self._duplicates
|
||||
|
||||
@property
|
||||
def dtypes(
|
||||
self,
|
||||
) -> Optional[Dict[str, Union[pd.CategoricalDtype, Type[np.integer]]]]:
|
||||
return self._dtypes
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
_is_fittable = False
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"bins": self._bins,
|
||||
"right": self._right,
|
||||
"include_lowest": self._include_lowest,
|
||||
"duplicates": self._duplicates,
|
||||
"dtypes": self._dtypes,
|
||||
"output_columns": self._output_columns,
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._bins = fields["bins"]
|
||||
self._right = fields["right"]
|
||||
self._include_lowest = fields["include_lowest"]
|
||||
self._duplicates = fields["duplicates"]
|
||||
self._dtypes = fields["dtypes"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_bins": _PublicField(public_field="bins"),
|
||||
"_right": _PublicField(public_field="right", default=True),
|
||||
"_include_lowest": _PublicField(
|
||||
public_field="include_lowest", default=False
|
||||
),
|
||||
"_duplicates": _PublicField(public_field="duplicates", default="raise"),
|
||||
"_dtypes": _PublicField(public_field="dtypes", default=None),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(
|
||||
version=1, identifier="io.ray.preprocessors.uniform_kbins_discretizer"
|
||||
)
|
||||
class UniformKBinsDiscretizer(_AbstractKBinsDiscretizer):
|
||||
"""Bin values into discrete intervals (bins) of uniform width.
|
||||
|
||||
Columns must contain numerical values.
|
||||
|
||||
Examples:
|
||||
Use :class:`UniformKBinsDiscretizer` to bin continuous features.
|
||||
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import UniformKBinsDiscretizer
|
||||
>>> df = pd.DataFrame({
|
||||
... "value_1": [0.2, 1.4, 2.5, 6.2, 9.7, 2.1],
|
||||
... "value_2": [10, 15, 13, 12, 23, 25],
|
||||
... })
|
||||
>>> ds = ray.data.from_pandas(df)
|
||||
>>> discretizer = UniformKBinsDiscretizer(
|
||||
... columns=["value_1", "value_2"], bins=4
|
||||
... )
|
||||
>>> discretizer.fit_transform(ds).to_pandas()
|
||||
value_1 value_2
|
||||
0 0 0
|
||||
1 0 1
|
||||
2 0 0
|
||||
3 2 0
|
||||
4 3 3
|
||||
5 0 3
|
||||
|
||||
:class:`UniformKBinsDiscretizer` can also be used in append mode by providing the
|
||||
name of the output_columns that should hold the encoded values.
|
||||
|
||||
>>> discretizer = UniformKBinsDiscretizer(
|
||||
... columns=["value_1", "value_2"],
|
||||
... bins=4,
|
||||
... output_columns=["value_1_discretized", "value_2_discretized"]
|
||||
... )
|
||||
>>> discretizer.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
value_1 value_2 value_1_discretized value_2_discretized
|
||||
0 0.2 10 0 0
|
||||
1 1.4 15 0 1
|
||||
2 2.5 13 0 0
|
||||
3 6.2 12 2 0
|
||||
4 9.7 23 3 3
|
||||
5 2.1 25 0 3
|
||||
|
||||
You can also specify different number of bins per column.
|
||||
|
||||
>>> discretizer = UniformKBinsDiscretizer(
|
||||
... columns=["value_1", "value_2"], bins={"value_1": 4, "value_2": 3}
|
||||
... )
|
||||
>>> discretizer.fit_transform(ds).to_pandas()
|
||||
value_1 value_2
|
||||
0 0 0
|
||||
1 0 0
|
||||
2 0 0
|
||||
3 2 0
|
||||
4 3 2
|
||||
5 0 2
|
||||
|
||||
|
||||
Args:
|
||||
columns: The columns to discretize.
|
||||
bins: Defines the number of equal-width bins.
|
||||
Can be either an integer (which will be applied to all columns),
|
||||
or a dict that maps columns to integers.
|
||||
The range is extended by .1% on each side to include
|
||||
the minimum and maximum values.
|
||||
right: Indicates whether bins includes the rightmost edge or not.
|
||||
include_lowest: Whether the first interval should be left-inclusive
|
||||
or not.
|
||||
duplicates: Can be either 'raise' or 'drop'. If bin edges are not unique,
|
||||
raise ``ValueError`` or drop non-uniques.
|
||||
dtypes: An optional dictionary that maps columns to ``pd.CategoricalDtype``
|
||||
objects or ``np.integer`` types. If you don't include a column in ``dtypes``
|
||||
or specify it as an integer dtype, the outputted column will consist of
|
||||
ordered integers corresponding to bins. If you use a
|
||||
``pd.CategoricalDtype``, the outputted column will be a
|
||||
``pd.CategoricalDtype`` with the categories being mapped to bins.
|
||||
You can use ``pd.CategoricalDtype(categories, ordered=True)`` to
|
||||
preserve information about bin order.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
|
||||
.. seealso::
|
||||
|
||||
:class:`CustomKBinsDiscretizer`
|
||||
If you want to specify your own bin edges.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
bins: Union[int, Dict[str, int]],
|
||||
*,
|
||||
right: bool = True,
|
||||
include_lowest: bool = False,
|
||||
duplicates: str = "raise",
|
||||
dtypes: Optional[
|
||||
Dict[str, Union[pd.CategoricalDtype, Type[np.integer]]]
|
||||
] = None,
|
||||
output_columns: Optional[List[str]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._bins = bins
|
||||
self._right = right
|
||||
self._include_lowest = include_lowest
|
||||
self._duplicates = duplicates
|
||||
self._dtypes = dtypes
|
||||
self._output_columns = (
|
||||
SerializablePreprocessorBase._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def bins(self) -> Union[int, Dict[str, int]]:
|
||||
return self._bins
|
||||
|
||||
@property
|
||||
def right(self) -> bool:
|
||||
return self._right
|
||||
|
||||
@property
|
||||
def include_lowest(self) -> bool:
|
||||
return self._include_lowest
|
||||
|
||||
@property
|
||||
def duplicates(self) -> str:
|
||||
return self._duplicates
|
||||
|
||||
@property
|
||||
def dtypes(
|
||||
self,
|
||||
) -> Optional[Dict[str, Union[pd.CategoricalDtype, Type[np.integer]]]]:
|
||||
return self._dtypes
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _fit(self, dataset: "Dataset") -> SerializablePreprocessorBase:
|
||||
self._validate_on_fit()
|
||||
|
||||
if isinstance(self.bins, dict):
|
||||
columns = self.bins.keys()
|
||||
else:
|
||||
columns = self.columns
|
||||
|
||||
for column in columns:
|
||||
bins = self.bins[column] if isinstance(self.bins, dict) else self.bins
|
||||
if not isinstance(bins, int):
|
||||
raise TypeError(
|
||||
f"`bins` must be an integer or a dict of integers, got {bins}"
|
||||
)
|
||||
|
||||
self._stat_computation_plan.add_aggregator(
|
||||
aggregator_fn=Min,
|
||||
columns=columns,
|
||||
)
|
||||
self._stat_computation_plan.add_aggregator(
|
||||
aggregator_fn=Max,
|
||||
columns=columns,
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def _validate_on_fit(self):
|
||||
self._validate_bins_columns()
|
||||
|
||||
def _fit_execute(self, dataset: "Dataset"):
|
||||
stats = self._stat_computation_plan.compute(dataset)
|
||||
self.stats_ = post_fit_processor(stats, self.bins, self.right)
|
||||
return self
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"bins": self._bins,
|
||||
"right": self._right,
|
||||
"include_lowest": self._include_lowest,
|
||||
"duplicates": self._duplicates,
|
||||
"dtypes": self._dtypes,
|
||||
"output_columns": self._output_columns,
|
||||
"_fitted": getattr(self, "_fitted", None),
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._bins = fields["bins"]
|
||||
self._right = fields["right"]
|
||||
self._include_lowest = fields["include_lowest"]
|
||||
self._duplicates = fields["duplicates"]
|
||||
self._dtypes = fields["dtypes"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
# optional fields
|
||||
self._fitted = fields.get("_fitted")
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_bins": _PublicField(public_field="bins"),
|
||||
"_right": _PublicField(public_field="right", default=True),
|
||||
"_include_lowest": _PublicField(
|
||||
public_field="include_lowest", default=False
|
||||
),
|
||||
"_duplicates": _PublicField(public_field="duplicates", default="raise"),
|
||||
"_dtypes": _PublicField(public_field="dtypes", default=None),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def post_fit_processor(aggregate_stats: dict, bins: Union[str, Dict], right: bool):
|
||||
mins, maxes, stats = {}, {}, {}
|
||||
for key, value in aggregate_stats.items():
|
||||
column_name = key[4:-1] # min(column) -> column
|
||||
if key.startswith("min"):
|
||||
mins[column_name] = value
|
||||
if key.startswith("max"):
|
||||
maxes[column_name] = value
|
||||
|
||||
for column in mins.keys():
|
||||
stats[column] = _translate_min_max_number_of_bins_to_bin_edges(
|
||||
mn=mins[column],
|
||||
mx=maxes[column],
|
||||
bins=bins[column] if isinstance(bins, dict) else bins,
|
||||
right=right,
|
||||
)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
# Copied from
|
||||
# https://github.com/pandas-dev/pandas/blob/v1.4.4/pandas/core/reshape/tile.py#L257
|
||||
# under
|
||||
# BSD 3-Clause License
|
||||
#
|
||||
# Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc.
|
||||
# and PyData Development Team
|
||||
# All rights reserved.
|
||||
#
|
||||
# Copyright (c) 2011-2022, Open source contributors.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# * Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# * Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
def _translate_min_max_number_of_bins_to_bin_edges(
|
||||
mn: float, mx: float, bins: int, right: bool
|
||||
) -> List[float]:
|
||||
"""Translates a range and desired number of bins into list of bin edges."""
|
||||
rng = (mn, mx)
|
||||
mn, mx = (mi + 0.0 for mi in rng)
|
||||
|
||||
if np.isinf(mn) or np.isinf(mx):
|
||||
raise ValueError(
|
||||
"Cannot specify integer `bins` when input data contains infinity."
|
||||
)
|
||||
elif mn == mx: # adjust end points before binning
|
||||
mn -= 0.001 * abs(mn) if mn != 0 else 0.001
|
||||
mx += 0.001 * abs(mx) if mx != 0 else 0.001
|
||||
bins = np.linspace(mn, mx, bins + 1, endpoint=True)
|
||||
else: # adjust end points after binning
|
||||
bins = np.linspace(mn, mx, bins + 1, endpoint=True)
|
||||
adj = (mx - mn) * 0.001 # 0.1% of the range
|
||||
if right:
|
||||
bins[0] -= adj
|
||||
else:
|
||||
bins[-1] += adj
|
||||
return bins
|
||||
|
||||
|
||||
# TODO(ml-team)
|
||||
# Add QuantileKBinsDiscretizer
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,167 @@
|
||||
import collections
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import (
|
||||
_PublicField,
|
||||
migrate_private_fields,
|
||||
simple_hash,
|
||||
)
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.feature_hasher")
|
||||
class FeatureHasher(SerializablePreprocessorBase):
|
||||
r"""Apply the `hashing trick <https://en.wikipedia.org/wiki/Feature_hashing>`_ to a
|
||||
table that describes token frequencies.
|
||||
|
||||
:class:`FeatureHasher` creates ``num_features`` columns named ``hash_{index}``,
|
||||
where ``index`` ranges from :math:`0` to ``num_features``:math:`- 1`. The column
|
||||
``hash_{index}`` describes the frequency of tokens that hash to ``index``.
|
||||
|
||||
Distinct tokens can correspond to the same index. However, if ``num_features`` is
|
||||
large enough, then columns probably correspond to a unique token.
|
||||
|
||||
This preprocessor is memory efficient and quick to pickle. However, given a
|
||||
transformed column, you can't know which tokens correspond to it. This might make it
|
||||
hard to determine which tokens are important to your model.
|
||||
|
||||
.. warning::
|
||||
Sparse matrices aren't supported. If you use a large ``num_features``, this
|
||||
preprocessor might behave poorly.
|
||||
|
||||
Examples:
|
||||
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import FeatureHasher
|
||||
|
||||
The data below describes the frequencies of tokens in ``"I like Python"`` and
|
||||
``"I dislike Python"``.
|
||||
|
||||
>>> df = pd.DataFrame({
|
||||
... "I": [1, 1],
|
||||
... "like": [1, 0],
|
||||
... "dislike": [0, 1],
|
||||
... "Python": [1, 1]
|
||||
... })
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
|
||||
:class:`FeatureHasher` hashes each token to determine its index. For example,
|
||||
the index of ``"I"`` is :math:`hash(\\texttt{"I"}) \pmod 8 = 5`.
|
||||
|
||||
>>> hasher = FeatureHasher(columns=["I", "like", "dislike", "Python"], num_features=8, output_column = "hashed")
|
||||
>>> hasher.fit_transform(ds)["hashed"].to_pandas().to_numpy() # doctest: +SKIP
|
||||
array([[0, 0, 0, 2, 0, 1, 0, 0],
|
||||
[0, 0, 0, 1, 0, 1, 1, 0]])
|
||||
|
||||
Notice the hash collision: both ``"like"`` and ``"Python"`` correspond to index
|
||||
:math:`3`. You can avoid hash collisions like these by increasing
|
||||
``num_features``.
|
||||
|
||||
Args:
|
||||
columns: The columns to apply the hashing trick to. Each column should describe
|
||||
the frequency of a token.
|
||||
num_features: The number of features used to represent the vocabulary. You
|
||||
should choose a value large enough to prevent hash collisions between
|
||||
distinct tokens.
|
||||
output_column: The name of the column that contains the hashed features.
|
||||
|
||||
.. seealso::
|
||||
:class:`~ray.data.preprocessors.CountVectorizer`
|
||||
Use this preprocessor to generate inputs for :class:`FeatureHasher`.
|
||||
|
||||
:class:`ray.data.preprocessors.HashingVectorizer`
|
||||
If your input data describes documents rather than token frequencies,
|
||||
use :class:`~ray.data.preprocessors.HashingVectorizer`.
|
||||
""" # noqa: E501
|
||||
|
||||
_is_fittable = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
num_features: int,
|
||||
output_column: str,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
# TODO(matt): Set default number of features.
|
||||
# This likely requires sparse matrix support to avoid explosion of columns.
|
||||
self._num_features = num_features
|
||||
self._output_column = output_column
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def num_features(self) -> int:
|
||||
return self._num_features
|
||||
|
||||
@property
|
||||
def output_column(self) -> str:
|
||||
return self._output_column
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
# TODO(matt): Use sparse matrix for efficiency.
|
||||
def row_feature_hasher(row):
|
||||
hash_counts = collections.defaultdict(int)
|
||||
for column in self._columns:
|
||||
hashed_value = simple_hash(column, self._num_features)
|
||||
hash_counts[hashed_value] += row[column]
|
||||
return {f"hash_{i}": hash_counts[i] for i in range(self._num_features)}
|
||||
|
||||
feature_columns = df.loc[:, self._columns].apply(
|
||||
row_feature_hasher, axis=1, result_type="expand"
|
||||
)
|
||||
|
||||
# Concatenate the hash columns
|
||||
hash_columns = [f"hash_{i}" for i in range(self._num_features)]
|
||||
concatenated = feature_columns[hash_columns].to_numpy()
|
||||
# Use a Pandas Series for column assignment to get more consistent
|
||||
# behavior across Pandas versions.
|
||||
df.loc[:, self._output_column] = pd.Series(list(concatenated))
|
||||
|
||||
return df
|
||||
|
||||
def get_input_columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
def get_output_columns(self) -> List[str]:
|
||||
return [self._output_column]
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"{self.__class__.__name__}(columns={self._columns!r}, "
|
||||
f"num_features={self._num_features!r}, "
|
||||
f"output_column={self._output_column!r})"
|
||||
)
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"num_features": self._num_features,
|
||||
"output_column": self._output_column,
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._num_features = fields["num_features"]
|
||||
self._output_column = fields["output_column"]
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_num_features": _PublicField(public_field="num_features"),
|
||||
"_output_column": _PublicField(public_field="output_column"),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,293 @@
|
||||
import logging
|
||||
from collections import Counter
|
||||
from numbers import Number
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pandas.api.types import is_categorical_dtype
|
||||
|
||||
from ray.data.aggregate import Mean
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import _Computed, _PublicField, migrate_private_fields
|
||||
from ray.data.preprocessors.version_support import (
|
||||
SerializablePreprocessor as Serializable,
|
||||
)
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data.dataset import Dataset
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@Serializable(version=1, identifier="io.ray.preprocessors.simple_imputer")
|
||||
class SimpleImputer(SerializablePreprocessorBase):
|
||||
"""Replace missing values with imputed values. If the column is missing from a
|
||||
batch, it will be filled with the imputed value.
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import SimpleImputer
|
||||
>>> df = pd.DataFrame({"X": [0, None, 3, 3], "Y": [None, "b", "c", "c"]})
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>> ds.to_pandas() # doctest: +SKIP
|
||||
X Y
|
||||
0 0.0 None
|
||||
1 NaN b
|
||||
2 3.0 c
|
||||
3 3.0 c
|
||||
|
||||
The `"mean"` strategy imputes missing values with the mean of non-missing
|
||||
values. This strategy doesn't work with categorical data.
|
||||
|
||||
>>> preprocessor = SimpleImputer(columns=["X"], strategy="mean")
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X Y
|
||||
0 0.0 None
|
||||
1 2.0 b
|
||||
2 3.0 c
|
||||
3 3.0 c
|
||||
|
||||
The `"most_frequent"` strategy imputes missing values with the most frequent
|
||||
value in each column.
|
||||
|
||||
>>> preprocessor = SimpleImputer(columns=["X", "Y"], strategy="most_frequent")
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X Y
|
||||
0 0.0 c
|
||||
1 3.0 b
|
||||
2 3.0 c
|
||||
3 3.0 c
|
||||
|
||||
The `"constant"` strategy imputes missing values with the value specified by
|
||||
`fill_value`.
|
||||
|
||||
>>> preprocessor = SimpleImputer(
|
||||
... columns=["Y"],
|
||||
... strategy="constant",
|
||||
... fill_value="?",
|
||||
... )
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X Y
|
||||
0 0.0 ?
|
||||
1 NaN b
|
||||
2 3.0 c
|
||||
3 3.0 c
|
||||
|
||||
:class:`SimpleImputer` can also be used in append mode by providing the
|
||||
name of the output_columns that should hold the imputed values.
|
||||
|
||||
>>> preprocessor = SimpleImputer(columns=["X"], output_columns=["X_imputed"], strategy="mean")
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X Y X_imputed
|
||||
0 0.0 None 0.0
|
||||
1 NaN b 2.0
|
||||
2 3.0 c 3.0
|
||||
3 3.0 c 3.0
|
||||
|
||||
Args:
|
||||
columns: The columns to apply imputation to.
|
||||
strategy: How imputed values are chosen.
|
||||
|
||||
* ``"mean"``: The mean of non-missing values. This strategy only works with numeric columns.
|
||||
* ``"most_frequent"``: The most common value.
|
||||
* ``"constant"``: The value passed to ``fill_value``.
|
||||
|
||||
fill_value: The value to use when ``strategy`` is ``"constant"``.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
|
||||
Raises:
|
||||
ValueError: if ``strategy`` is not ``"mean"``, ``"most_frequent"``, or
|
||||
``"constant"``.
|
||||
""" # noqa: E501
|
||||
|
||||
_valid_strategies = ["mean", "most_frequent", "constant"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
strategy: str = "mean",
|
||||
fill_value: Optional[Union[str, Number]] = None,
|
||||
*,
|
||||
output_columns: Optional[List[str]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._strategy = strategy
|
||||
self._fill_value = fill_value
|
||||
|
||||
if strategy not in self._valid_strategies:
|
||||
raise ValueError(
|
||||
f"Strategy {strategy} is not supported."
|
||||
f"Supported values are: {self._valid_strategies}"
|
||||
)
|
||||
|
||||
if strategy == "constant":
|
||||
# There is no information to be fitted.
|
||||
self._is_fittable = False
|
||||
if fill_value is None:
|
||||
raise ValueError(
|
||||
'`fill_value` must be set when using "constant" strategy.'
|
||||
)
|
||||
|
||||
self._output_columns = (
|
||||
SerializablePreprocessorBase._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def strategy(self) -> str:
|
||||
return self._strategy
|
||||
|
||||
@property
|
||||
def fill_value(self) -> Optional[Union[str, Number]]:
|
||||
return self._fill_value
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _fit(self, dataset: "Dataset") -> SerializablePreprocessorBase:
|
||||
if self._strategy == "mean":
|
||||
self._stat_computation_plan.add_aggregator(
|
||||
aggregator_fn=Mean, columns=self._columns
|
||||
)
|
||||
elif self._strategy == "most_frequent":
|
||||
self._stat_computation_plan.add_callable_stat(
|
||||
stat_fn=lambda key_gen: _get_most_frequent_values(
|
||||
dataset=dataset,
|
||||
columns=self._columns,
|
||||
key_gen=key_gen,
|
||||
),
|
||||
stat_key_fn=lambda col: f"most_frequent({col})",
|
||||
columns=self._columns,
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
for column, output_column in zip(self._columns, self._output_columns):
|
||||
value = self._get_fill_value(column)
|
||||
|
||||
if value is None:
|
||||
raise ValueError(
|
||||
f"Column {column} has no fill value. "
|
||||
"Check the data used to fit the SimpleImputer."
|
||||
)
|
||||
|
||||
if column not in df.columns:
|
||||
# Create the column with the fill_value if it doesn't exist
|
||||
df[output_column] = value
|
||||
else:
|
||||
if is_categorical_dtype(df.dtypes[column]):
|
||||
df[output_column] = df[column].cat.add_categories([value])
|
||||
|
||||
if (
|
||||
output_column != column
|
||||
# If the backing array is memory-mapped from shared memory, then the
|
||||
# array won't be writeable.
|
||||
or (
|
||||
isinstance(df[output_column].values, np.ndarray)
|
||||
and not df[output_column].values.flags.writeable
|
||||
)
|
||||
):
|
||||
df[output_column] = df[column].copy(deep=True)
|
||||
|
||||
df.fillna({output_column: value}, inplace=True)
|
||||
|
||||
return df
|
||||
|
||||
def _get_fill_value(self, column):
|
||||
if self._strategy == "mean":
|
||||
return self.stats_[f"mean({column})"]
|
||||
elif self._strategy == "most_frequent":
|
||||
return self.stats_[f"most_frequent({column})"]
|
||||
elif self._strategy == "constant":
|
||||
return self._fill_value
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Strategy {self._strategy} is not supported. "
|
||||
f"Supported values are: {self._valid_strategies}"
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"{self.__class__.__name__}(columns={self._columns!r}, "
|
||||
f"strategy={self._strategy!r}, fill_value={self._fill_value!r}, "
|
||||
f"output_columns={self._output_columns!r})"
|
||||
)
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"output_columns": self._output_columns,
|
||||
"_fitted": getattr(self, "_fitted", None),
|
||||
"strategy": self._strategy,
|
||||
"fill_value": getattr(
|
||||
self, "_fill_value", getattr(self, "fill_value", None)
|
||||
),
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
self._strategy = fields["strategy"]
|
||||
# optional fields
|
||||
self._fitted = fields.get("_fitted")
|
||||
self._fill_value = fields.get("fill_value")
|
||||
|
||||
if self._strategy == "constant":
|
||||
self._is_fittable = False
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
"""Handle backwards compatibility for old pickled objects."""
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
"_strategy": _PublicField(public_field="strategy"),
|
||||
"_fill_value": _PublicField(
|
||||
public_field="fill_value",
|
||||
default=None,
|
||||
), # _fill_value is optional
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _get_most_frequent_values(
|
||||
dataset: "Dataset",
|
||||
columns: List[str],
|
||||
key_gen: Callable[[str], str],
|
||||
) -> Dict[str, Union[str, Number]]:
|
||||
def get_pd_value_counts(df: pd.DataFrame) -> Dict[str, List[Counter]]:
|
||||
return {col: [Counter(df[col].value_counts().to_dict())] for col in columns}
|
||||
|
||||
value_counts = dataset.map_batches(get_pd_value_counts, batch_format="pandas")
|
||||
final_counters = {col: Counter() for col in columns}
|
||||
for batch in value_counts.iter_batches(batch_size=None):
|
||||
for col, counters in batch.items():
|
||||
for counter in counters:
|
||||
final_counters[col] += counter
|
||||
|
||||
return {
|
||||
key_gen(column): final_counters[column].most_common(1)[0][0] # noqa
|
||||
for column in columns
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import _Computed, _PublicField, migrate_private_fields
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.normalizer")
|
||||
class Normalizer(SerializablePreprocessorBase):
|
||||
r"""Scales each sample to have unit norm.
|
||||
|
||||
This preprocessor works by dividing each sample (i.e., row) by the sample's norm.
|
||||
The general formula is given by
|
||||
|
||||
.. math::
|
||||
|
||||
s' = \frac{s}{\lVert s \rVert_p}
|
||||
|
||||
where :math:`s` is the sample, :math:`s'` is the transformed sample,
|
||||
:math:\lVert s \rVert`, and :math:`p` is the norm type.
|
||||
|
||||
The following norms are supported:
|
||||
|
||||
* `"l1"` (:math:`L^1`): Sum of the absolute values.
|
||||
* `"l2"` (:math:`L^2`): Square root of the sum of the squared values.
|
||||
* `"max"` (:math:`L^\infty`): Maximum value.
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import Normalizer
|
||||
>>>
|
||||
>>> df = pd.DataFrame({"X1": [1, 1], "X2": [1, 0], "X3": [0, 1]})
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>> ds.to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 1 1 0
|
||||
1 1 0 1
|
||||
|
||||
The :math:`L^2`-norm of the first sample is :math:`\sqrt{2}`, and the
|
||||
:math:`L^2`-norm of the second sample is :math:`1`.
|
||||
|
||||
>>> preprocessor = Normalizer(columns=["X1", "X2"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 0.707107 0.707107 0
|
||||
1 1.000000 0.000000 1
|
||||
|
||||
The :math:`L^1`-norm of the first sample is :math:`2`, and the
|
||||
:math:`L^1`-norm of the second sample is :math:`1`.
|
||||
|
||||
>>> preprocessor = Normalizer(columns=["X1", "X2"], norm="l1")
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 0.5 0.5 0
|
||||
1 1.0 0.0 1
|
||||
|
||||
The :math:`L^\infty`-norm of the both samples is :math:`1`.
|
||||
|
||||
>>> preprocessor = Normalizer(columns=["X1", "X2"], norm="max")
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 1.0 1.0 0
|
||||
1 1.0 0.0 1
|
||||
|
||||
:class:`Normalizer` can also be used in append mode by providing the
|
||||
name of the output_columns that should hold the normalized values.
|
||||
|
||||
>>> preprocessor = Normalizer(columns=["X1", "X2"], output_columns=["X1_normalized", "X2_normalized"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3 X1_normalized X2_normalized
|
||||
0 1 1 0 0.707107 0.707107
|
||||
1 1 0 1 1.000000 0.000000
|
||||
|
||||
Args:
|
||||
columns: The columns to scale. For each row, these colmumns are scaled to
|
||||
unit-norm.
|
||||
norm: The norm to use. The supported values are ``"l1"``, ``"l2"``, or
|
||||
``"max"``. Defaults to ``"l2"``.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
|
||||
Raises:
|
||||
ValueError: if ``norm`` is not ``"l1"``, ``"l2"``, or ``"max"``.
|
||||
"""
|
||||
|
||||
_norm_fns = {
|
||||
"l1": lambda cols: np.abs(cols).sum(axis=1),
|
||||
"l2": lambda cols: np.sqrt(np.power(cols, 2).sum(axis=1)),
|
||||
"max": lambda cols: np.max(abs(cols), axis=1),
|
||||
}
|
||||
|
||||
_is_fittable = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
norm: str = "l2",
|
||||
*,
|
||||
output_columns: Optional[List[str]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._norm = norm
|
||||
|
||||
if norm not in self._norm_fns:
|
||||
raise ValueError(
|
||||
f"Norm {norm} is not supported."
|
||||
f"Supported values are: {self._norm_fns.keys()}"
|
||||
)
|
||||
|
||||
self._output_columns = (
|
||||
SerializablePreprocessorBase._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def norm(self) -> str:
|
||||
return self._norm
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
columns = df.loc[:, self._columns]
|
||||
column_norms = self._norm_fns[self._norm](columns)
|
||||
|
||||
df[self._output_columns] = columns.div(column_norms, axis=0)
|
||||
return df
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"{self.__class__.__name__}(columns={self._columns!r}, "
|
||||
f"norm={self._norm!r}, "
|
||||
f"output_columns={self._output_columns!r})"
|
||||
)
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"norm": self._norm,
|
||||
"output_columns": self._output_columns,
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._norm = fields["norm"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_norm": _PublicField(public_field="norm", default="l2"),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,677 @@
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pyarrow as pa
|
||||
import pyarrow.compute as pc
|
||||
|
||||
from ray.data.aggregate import AbsMax, ApproximateQuantile, Max, Mean, Min, Std
|
||||
from ray.data.block import BlockAccessor
|
||||
from ray.data.preprocessor import Preprocessor, SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import _Computed, _PublicField, migrate_private_fields
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.data.util.data_batch_conversion import BatchFormat
|
||||
from ray.util.annotations import DeveloperAPI, PublicAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data.dataset import Dataset
|
||||
|
||||
# Small epsilon value to handle near-zero values in division operations.
|
||||
# This prevents numerical instability when scaling columns with very small
|
||||
# variance or range. Similar to sklearn's approach.
|
||||
_EPSILON = 1e-8
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.standard_scaler")
|
||||
class StandardScaler(SerializablePreprocessorBase):
|
||||
r"""Translate and scale each column by its mean and standard deviation,
|
||||
respectively.
|
||||
|
||||
The general formula is given by
|
||||
|
||||
.. math::
|
||||
|
||||
x' = \frac{x - \bar{x}}{s}
|
||||
|
||||
where :math:`x` is the column, :math:`x'` is the transformed column,
|
||||
:math:`\bar{x}` is the column average, and :math:`s` is the column's sample
|
||||
standard deviation. If :math:`s = 0` (i.e., the column is constant-valued),
|
||||
then the transformed column will contain zeros.
|
||||
|
||||
.. warning::
|
||||
:class:`StandardScaler` works best when your data is normal. If your data isn't
|
||||
approximately normal, then the transformed features won't be meaningful.
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import StandardScaler
|
||||
>>>
|
||||
>>> df = pd.DataFrame({"X1": [-2, 0, 2], "X2": [-3, -3, 3], "X3": [1, 1, 1]})
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>> ds.to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -2 -3 1
|
||||
1 0 -3 1
|
||||
2 2 3 1
|
||||
|
||||
Columns are scaled separately.
|
||||
|
||||
>>> preprocessor = StandardScaler(columns=["X1", "X2"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -1.224745 -0.707107 1
|
||||
1 0.000000 -0.707107 1
|
||||
2 1.224745 1.414214 1
|
||||
|
||||
Constant-valued columns get filled with zeros.
|
||||
|
||||
>>> preprocessor = StandardScaler(columns=["X3"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -2 -3 0.0
|
||||
1 0 -3 0.0
|
||||
2 2 3 0.0
|
||||
|
||||
>>> preprocessor = StandardScaler(
|
||||
... columns=["X1", "X2"],
|
||||
... output_columns=["X1_scaled", "X2_scaled"]
|
||||
... )
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3 X1_scaled X2_scaled
|
||||
0 -2 -3 1 -1.224745 -0.707107
|
||||
1 0 -3 1 0.000000 -0.707107
|
||||
2 2 3 1 1.224745 1.414214
|
||||
|
||||
Args:
|
||||
columns: The columns to separately scale.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
"""
|
||||
|
||||
def __init__(self, columns: List[str], output_columns: Optional[List[str]] = None):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._output_columns = Preprocessor._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _fit(self, dataset: "Dataset") -> Preprocessor:
|
||||
self._stat_computation_plan.add_aggregator(
|
||||
aggregator_fn=Mean,
|
||||
columns=self._columns,
|
||||
)
|
||||
self._stat_computation_plan.add_aggregator(
|
||||
aggregator_fn=lambda col: Std(col, ddof=0),
|
||||
columns=self._columns,
|
||||
)
|
||||
return self
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
def column_standard_scaler(s: pd.Series):
|
||||
s_mean = self.stats_[f"mean({s.name})"]
|
||||
s_std = self.stats_[f"std({s.name})"]
|
||||
|
||||
if s_std is None or s_mean is None:
|
||||
s[:] = np.nan
|
||||
return s
|
||||
|
||||
# Handle division by zero and near-zero values for numerical stability.
|
||||
# If standard deviation is very small (constant or near-constant column),
|
||||
# treat it as 1 to avoid numerical instability.
|
||||
if s_std < _EPSILON:
|
||||
s_std = 1
|
||||
|
||||
return (s - s_mean) / s_std
|
||||
|
||||
df[self._output_columns] = df[self._columns].transform(column_standard_scaler)
|
||||
return df
|
||||
|
||||
@staticmethod
|
||||
def _scale_column(column: pa.Array, mean: float, std: float) -> pa.Array:
|
||||
# Handle division by zero and near-zero values for numerical stability.
|
||||
if std < _EPSILON:
|
||||
std = 1
|
||||
|
||||
return pc.divide(
|
||||
pc.subtract(column, pa.scalar(float(mean))), pa.scalar(float(std))
|
||||
)
|
||||
|
||||
def _transform_arrow(self, table: pa.Table) -> pa.Table:
|
||||
"""Transform using fast native PyArrow operations."""
|
||||
# Read all input columns first to avoid reading modified data when
|
||||
# output_columns[i] == columns[j] for i < j
|
||||
input_columns = [table.column(input_col) for input_col in self._columns]
|
||||
|
||||
for input_col, output_col, column in zip(
|
||||
self._columns, self._output_columns, input_columns
|
||||
):
|
||||
s_mean = self.stats_[f"mean({input_col})"]
|
||||
s_std = self.stats_[f"std({input_col})"]
|
||||
|
||||
if s_std is None or s_mean is None:
|
||||
# Return column filled with nulls, preserving original column type
|
||||
null_array = pa.nulls(len(column), type=column.type)
|
||||
table = BlockAccessor.for_block(table).upsert_column(
|
||||
output_col, null_array
|
||||
)
|
||||
continue
|
||||
|
||||
scaled_column = self._scale_column(column, s_mean, s_std)
|
||||
|
||||
table = BlockAccessor.for_block(table).upsert_column(
|
||||
output_col, scaled_column
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
@classmethod
|
||||
@DeveloperAPI
|
||||
def preferred_batch_format(cls) -> BatchFormat:
|
||||
return BatchFormat.ARROW
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"output_columns": self._output_columns,
|
||||
"_fitted": getattr(self, "_fitted", None),
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
# optional fields
|
||||
self._fitted = fields.get("_fitted")
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
"""Handle backwards compatibility for old pickled objects."""
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.__class__.__name__}(columns={self._columns!r}, output_columns={self._output_columns!r})"
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.min_max_scaler")
|
||||
class MinMaxScaler(SerializablePreprocessorBase):
|
||||
r"""Scale each column by its range.
|
||||
|
||||
The general formula is given by
|
||||
|
||||
.. math::
|
||||
|
||||
x' = \frac{x - \min(x)}{\max{x} - \min{x}}
|
||||
|
||||
where :math:`x` is the column and :math:`x'` is the transformed column. If
|
||||
:math:`\max{x} - \min{x} = 0` (i.e., the column is constant-valued), then the
|
||||
transformed column will get filled with zeros.
|
||||
|
||||
Transformed values are always in the range :math:`[0, 1]`.
|
||||
|
||||
.. tip::
|
||||
This can be used as an alternative to :py:class:`StandardScaler`.
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import MinMaxScaler
|
||||
>>>
|
||||
>>> df = pd.DataFrame({"X1": [-2, 0, 2], "X2": [-3, -3, 3], "X3": [1, 1, 1]}) # noqa: E501
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>> ds.to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -2 -3 1
|
||||
1 0 -3 1
|
||||
2 2 3 1
|
||||
|
||||
Columns are scaled separately.
|
||||
|
||||
>>> preprocessor = MinMaxScaler(columns=["X1", "X2"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 0.0 0.0 1
|
||||
1 0.5 0.0 1
|
||||
2 1.0 1.0 1
|
||||
|
||||
Constant-valued columns get filled with zeros.
|
||||
|
||||
>>> preprocessor = MinMaxScaler(columns=["X3"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -2 -3 0.0
|
||||
1 0 -3 0.0
|
||||
2 2 3 0.0
|
||||
|
||||
>>> preprocessor = MinMaxScaler(columns=["X1", "X2"], output_columns=["X1_scaled", "X2_scaled"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3 X1_scaled X2_scaled
|
||||
0 -2 -3 1 0.0 0.0
|
||||
1 0 -3 1 0.5 0.0
|
||||
2 2 3 1 1.0 1.0
|
||||
|
||||
Args:
|
||||
columns: The columns to separately scale.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
"""
|
||||
|
||||
def __init__(self, columns: List[str], output_columns: Optional[List[str]] = None):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._output_columns = Preprocessor._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _fit(self, dataset: "Dataset") -> Preprocessor:
|
||||
aggregates = [Agg(col) for Agg in [Min, Max] for col in self._columns]
|
||||
self.stats_ = dataset.aggregate(*aggregates)
|
||||
return self
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
def column_min_max_scaler(s: pd.Series):
|
||||
s_min = self.stats_[f"min({s.name})"]
|
||||
s_max = self.stats_[f"max({s.name})"]
|
||||
diff = s_max - s_min
|
||||
|
||||
# Handle division by zero and near-zero values for numerical stability.
|
||||
# If range is very small (constant or near-constant column),
|
||||
# treat it as 1 to avoid numerical instability.
|
||||
if diff < _EPSILON:
|
||||
diff = 1
|
||||
|
||||
return (s - s_min) / diff
|
||||
|
||||
df[self._output_columns] = df[self._columns].transform(column_min_max_scaler)
|
||||
return df
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"output_columns": self._output_columns,
|
||||
"_fitted": getattr(self, "_fitted", None),
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
# optional fields
|
||||
self._fitted = fields.get("_fitted")
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
"""Handle backwards compatibility for old pickled objects."""
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.__class__.__name__}(columns={self._columns!r}, output_columns={self._output_columns!r})"
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.max_abs_scaler")
|
||||
class MaxAbsScaler(SerializablePreprocessorBase):
|
||||
r"""Scale each column by its absolute max value.
|
||||
|
||||
The general formula is given by
|
||||
|
||||
.. math::
|
||||
|
||||
x' = \frac{x}{\max{\vert x \vert}}
|
||||
|
||||
where :math:`x` is the column and :math:`x'` is the transformed column. If
|
||||
:math:`\max{\vert x \vert} = 0` (i.e., the column contains all zeros), then the
|
||||
column is unmodified.
|
||||
|
||||
.. tip::
|
||||
This is the recommended way to scale sparse data. If you data isn't sparse,
|
||||
you can use :class:`MinMaxScaler` or :class:`StandardScaler` instead.
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import MaxAbsScaler
|
||||
>>>
|
||||
>>> df = pd.DataFrame({"X1": [-6, 3], "X2": [2, -4], "X3": [0, 0]}) # noqa: E501
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>> ds.to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -6 2 0
|
||||
1 3 -4 0
|
||||
|
||||
Columns are scaled separately.
|
||||
|
||||
>>> preprocessor = MaxAbsScaler(columns=["X1", "X2"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -1.0 0.5 0
|
||||
1 0.5 -1.0 0
|
||||
|
||||
Zero-valued columns aren't scaled.
|
||||
|
||||
>>> preprocessor = MaxAbsScaler(columns=["X3"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -6 2 0.0
|
||||
1 3 -4 0.0
|
||||
|
||||
>>> preprocessor = MaxAbsScaler(columns=["X1", "X2"], output_columns=["X1_scaled", "X2_scaled"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3 X1_scaled X2_scaled
|
||||
0 -2 -3 1 -1.0 -1.0
|
||||
1 0 -3 1 0.0 -1.0
|
||||
2 2 3 1 1.0 1.0
|
||||
|
||||
Args:
|
||||
columns: The columns to separately scale.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
"""
|
||||
|
||||
def __init__(self, columns: List[str], output_columns: Optional[List[str]] = None):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._output_columns = Preprocessor._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _fit(self, dataset: "Dataset") -> Preprocessor:
|
||||
aggregates = [AbsMax(col) for col in self._columns]
|
||||
self.stats_ = dataset.aggregate(*aggregates)
|
||||
return self
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
def column_abs_max_scaler(s: pd.Series):
|
||||
s_abs_max = self.stats_[f"abs_max({s.name})"]
|
||||
|
||||
# Handle division by zero.
|
||||
# All values are 0.
|
||||
if s_abs_max == 0:
|
||||
s_abs_max = 1
|
||||
|
||||
return s / s_abs_max
|
||||
|
||||
df[self._output_columns] = df[self._columns].transform(column_abs_max_scaler)
|
||||
return df
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"output_columns": self._output_columns,
|
||||
"_fitted": getattr(self, "_fitted", None),
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
# optional fields
|
||||
self._fitted = fields.get("_fitted")
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
"""Handle backwards compatibility for old pickled objects."""
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self.__class__.__name__}(columns={self._columns!r}, output_columns={self._output_columns!r})"
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.robust_scaler")
|
||||
class RobustScaler(SerializablePreprocessorBase):
|
||||
r"""Scale and translate each column using approximate quantiles.
|
||||
|
||||
The general formula is given by
|
||||
|
||||
.. math::
|
||||
x' = \frac{x - \mu_{1/2}}{\mu_h - \mu_l}
|
||||
|
||||
where :math:`x` is the column, :math:`x'` is the transformed column,
|
||||
:math:`\mu_{1/2}` is the column median. :math:`\mu_{h}` and :math:`\mu_{l}` are the
|
||||
high and low quantiles, respectively. By default, :math:`\mu_{h}` is the third
|
||||
quartile and :math:`\mu_{l}` is the first quartile.
|
||||
|
||||
Internally, the `ApproximateQuantile` aggregator is used to calculate the
|
||||
approximate quantiles.
|
||||
|
||||
.. tip::
|
||||
This scaler works well when your data contains many outliers.
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import RobustScaler
|
||||
>>>
|
||||
>>> df = pd.DataFrame({
|
||||
... "X1": [1, 2, 3, 4, 5],
|
||||
... "X2": [13, 5, 14, 2, 8],
|
||||
... "X3": [1, 2, 2, 2, 3],
|
||||
... })
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>> ds.to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 1 13 1
|
||||
1 2 5 2
|
||||
2 3 14 2
|
||||
3 4 2 2
|
||||
4 5 8 3
|
||||
|
||||
:class:`RobustScaler` separately scales each column.
|
||||
|
||||
>>> preprocessor = RobustScaler(columns=["X1", "X2"])
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3
|
||||
0 -1.0 0.625 1
|
||||
1 -0.5 -0.375 2
|
||||
2 0.0 0.750 2
|
||||
3 0.5 -0.750 2
|
||||
4 1.0 0.000 3
|
||||
|
||||
>>> preprocessor = RobustScaler(
|
||||
... columns=["X1", "X2"],
|
||||
... output_columns=["X1_scaled", "X2_scaled"]
|
||||
... )
|
||||
>>> preprocessor.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
X1 X2 X3 X1_scaled X2_scaled
|
||||
0 1 13 1 -1.0 0.625
|
||||
1 2 5 2 -0.5 -0.375
|
||||
2 3 14 2 0.0 0.750
|
||||
3 4 2 2 0.5 -0.750
|
||||
4 5 8 3 1.0 0.000
|
||||
|
||||
Args:
|
||||
columns: The columns to separately scale.
|
||||
quantile_range: A tuple that defines the lower and upper quantiles. Values
|
||||
must be between 0 and 1. Defaults to the 1st and 3rd quartiles:
|
||||
``(0.25, 0.75)``.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
quantile_precision: Controls the accuracy and memory footprint of the sketch (K in KLL);
|
||||
higher values yield lower error but use more memory. Defaults to 800. See
|
||||
https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html
|
||||
for details on accuracy and size.
|
||||
"""
|
||||
|
||||
DEFAULT_QUANTILE_PRECISION = 800
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
quantile_range: Tuple[float, float] = (0.25, 0.75),
|
||||
output_columns: Optional[List[str]] = None,
|
||||
quantile_precision: int = DEFAULT_QUANTILE_PRECISION,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._quantile_range = quantile_range
|
||||
self._quantile_precision = quantile_precision
|
||||
|
||||
self._output_columns = Preprocessor._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def quantile_range(self) -> Tuple[float, float]:
|
||||
return self._quantile_range
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
@property
|
||||
def quantile_precision(self) -> int:
|
||||
return self._quantile_precision
|
||||
|
||||
def _fit(self, dataset: "Dataset") -> Preprocessor:
|
||||
quantiles = [
|
||||
self._quantile_range[0],
|
||||
0.50,
|
||||
self._quantile_range[1],
|
||||
]
|
||||
aggregates = [
|
||||
ApproximateQuantile(
|
||||
on=col,
|
||||
quantiles=quantiles,
|
||||
quantile_precision=self._quantile_precision,
|
||||
)
|
||||
for col in self._columns
|
||||
]
|
||||
aggregated = dataset.aggregate(*aggregates)
|
||||
|
||||
self.stats_ = {}
|
||||
for col in self._columns:
|
||||
low_q, med_q, high_q = aggregated[f"approx_quantile({col})"]
|
||||
self.stats_[f"low_quantile({col})"] = low_q
|
||||
self.stats_[f"median({col})"] = med_q
|
||||
self.stats_[f"high_quantile({col})"] = high_q
|
||||
|
||||
return self
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
def column_robust_scaler(s: pd.Series):
|
||||
s_low_q = self.stats_[f"low_quantile({s.name})"]
|
||||
s_median = self.stats_[f"median({s.name})"]
|
||||
s_high_q = self.stats_[f"high_quantile({s.name})"]
|
||||
diff = s_high_q - s_low_q
|
||||
|
||||
# Handle division by zero.
|
||||
# Return all zeros.
|
||||
if diff == 0:
|
||||
return np.zeros_like(s)
|
||||
|
||||
return (s - s_median) / diff
|
||||
|
||||
df[self._output_columns] = df[self._columns].transform(column_robust_scaler)
|
||||
return df
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"output_columns": self._output_columns,
|
||||
"quantile_range": self._quantile_range,
|
||||
"quantile_precision": self._quantile_precision,
|
||||
"_fitted": getattr(self, "_fitted", None),
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
self._quantile_range = fields["quantile_range"]
|
||||
self._quantile_precision = fields["quantile_precision"]
|
||||
# optional fields
|
||||
self._fitted = fields.get("_fitted")
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
"""Handle backwards compatibility for old pickled objects."""
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
"_quantile_range": _PublicField(
|
||||
public_field="quantile_range", default=(0.25, 0.75)
|
||||
),
|
||||
"_quantile_precision": _PublicField(
|
||||
public_field="quantile_precision",
|
||||
default=self.DEFAULT_QUANTILE_PRECISION,
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"{self.__class__.__name__}(columns={self._columns!r}, "
|
||||
f"quantile_range={self._quantile_range!r}, "
|
||||
f"output_columns={self._output_columns!r})"
|
||||
)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""
|
||||
Serialization handlers for preprocessor save/load functionality.
|
||||
|
||||
This module implements a factory pattern to abstract different serialization formats,
|
||||
making it easier to add new formats and maintain existing ones.
|
||||
"""
|
||||
|
||||
import abc
|
||||
import base64
|
||||
import pickle
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
from ray.cloudpickle import cloudpickle
|
||||
from ray.util.annotations import DeveloperAPI
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
class HandlerFormatName(Enum):
|
||||
"""Enum for consistent format naming in the factory."""
|
||||
|
||||
CLOUDPICKLE = "cloudpickle"
|
||||
PICKLE = "pickle"
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
class SerializationHandler(abc.ABC):
|
||||
"""Abstract base class for handling preprocessor serialization formats."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def serialize(
|
||||
self, data: Union["Preprocessor", Dict[str, Any]] # noqa: F821
|
||||
) -> Union[str, bytes]:
|
||||
"""Serialize preprocessor data to the specific format.
|
||||
|
||||
Args:
|
||||
data: Dictionary containing preprocessor metadata and stats
|
||||
|
||||
Returns:
|
||||
Serialized data in format-specific representation
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def deserialize(self, serialized: Union[str, bytes]) -> Any:
|
||||
"""Deserialize data from the specific format.
|
||||
|
||||
Args:
|
||||
serialized: Serialized data in format-specific representation
|
||||
|
||||
Returns:
|
||||
For structured formats (CloudPickle/JSON/MessagePack): Dictionary containing preprocessor metadata and stats
|
||||
For pickle format: The actual deserialized object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_magic_bytes(self) -> Union[str, bytes]:
|
||||
"""Get the magic bytes/prefix for this format."""
|
||||
pass
|
||||
|
||||
def strip_magic_bytes(self, serialized: Union[str, bytes]) -> Union[str, bytes]:
|
||||
"""Remove magic bytes from serialized data."""
|
||||
magic = self.get_magic_bytes()
|
||||
if isinstance(serialized, (str, bytes)) and serialized.startswith(magic):
|
||||
return serialized[len(magic) :]
|
||||
return serialized
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
class CloudPickleSerializationHandler(SerializationHandler):
|
||||
"""Handler for CloudPickle serialization format."""
|
||||
|
||||
MAGIC_CLOUDPICKLE = b"CPKL:"
|
||||
|
||||
def serialize(
|
||||
self, data: Union["Preprocessor", Dict[str, Any]] # noqa: F821
|
||||
) -> bytes:
|
||||
"""Serialize to CloudPickle format with magic prefix."""
|
||||
return self.MAGIC_CLOUDPICKLE + cloudpickle.dumps(data)
|
||||
|
||||
def deserialize(self, serialized: bytes) -> Dict[str, Any]:
|
||||
"""Deserialize from CloudPickle format."""
|
||||
if not isinstance(serialized, bytes):
|
||||
raise ValueError(
|
||||
f"Expected bytes for CloudPickle deserialization, got {type(serialized)}"
|
||||
)
|
||||
|
||||
if not serialized.startswith(self.MAGIC_CLOUDPICKLE):
|
||||
raise ValueError(f"Invalid CloudPickle magic bytes: {serialized[:10]}")
|
||||
|
||||
cloudpickle_data = self.strip_magic_bytes(serialized)
|
||||
return cloudpickle.loads(cloudpickle_data)
|
||||
|
||||
def get_magic_bytes(self) -> bytes:
|
||||
return self.MAGIC_CLOUDPICKLE
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
class PickleSerializationHandler(SerializationHandler):
|
||||
"""Handler for legacy Pickle serialization format."""
|
||||
|
||||
def serialize(
|
||||
self, data: Union["Preprocessor", Dict[str, Any]] # noqa: F821
|
||||
) -> str:
|
||||
"""
|
||||
Serialize using pickle format (for backward compatibility).
|
||||
data is ignored, but kept for consistency
|
||||
|
||||
"""
|
||||
return base64.b64encode(pickle.dumps(data)).decode("ascii")
|
||||
|
||||
def deserialize(
|
||||
self, serialized: str
|
||||
) -> Any: # Returns the actual object, not metadata
|
||||
"""Deserialize from pickle format (legacy support)."""
|
||||
# For pickle, we return the actual deserialized object directly
|
||||
return pickle.loads(base64.b64decode(serialized))
|
||||
|
||||
def get_magic_bytes(self) -> str:
|
||||
return "" # Pickle format doesn't use magic bytes
|
||||
|
||||
|
||||
class SerializationHandlerFactory:
|
||||
"""Factory class for creating appropriate serialization handlers."""
|
||||
|
||||
_handlers = {
|
||||
HandlerFormatName.CLOUDPICKLE: CloudPickleSerializationHandler,
|
||||
HandlerFormatName.PICKLE: PickleSerializationHandler,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def register_handler(cls, format_name: HandlerFormatName, handler_class: type):
|
||||
"""Register a new serialization handler."""
|
||||
cls._handlers[format_name] = handler_class
|
||||
|
||||
@classmethod
|
||||
def get_handler(
|
||||
cls,
|
||||
format_identifier: Optional[HandlerFormatName] = None,
|
||||
data: Optional[Union[str, bytes]] = None,
|
||||
**kwargs,
|
||||
) -> SerializationHandler:
|
||||
"""Get the appropriate serialization handler for a format or serialized data.
|
||||
|
||||
Args:
|
||||
format_identifier: The format to use for serialization. If None, will detect from data.
|
||||
data: Serialized data to detect format from (used when format_identifier is None).
|
||||
**kwargs: Additional keyword arguments (currently unused).
|
||||
|
||||
Returns:
|
||||
SerializationHandler instance for the format
|
||||
|
||||
Raises:
|
||||
ValueError: If format is not supported or cannot be detected
|
||||
"""
|
||||
# If it's already a format enum, use it directly
|
||||
if not format_identifier:
|
||||
format_identifier = cls.detect_format(data)
|
||||
|
||||
if format_identifier not in cls._handlers:
|
||||
raise ValueError(
|
||||
f"Unsupported serialization format: {format_identifier.value}. "
|
||||
f"Supported formats: {list(cls._handlers.keys())}"
|
||||
)
|
||||
|
||||
handler_class = cls._handlers[format_identifier]
|
||||
return handler_class()
|
||||
|
||||
@classmethod
|
||||
def detect_format(cls, serialized: Union[str, bytes]) -> HandlerFormatName:
|
||||
"""Detect the serialization format from the magic bytes.
|
||||
|
||||
Args:
|
||||
serialized: Serialized data
|
||||
|
||||
Returns:
|
||||
Format name enum
|
||||
|
||||
Raises:
|
||||
ValueError: If format cannot be detected
|
||||
"""
|
||||
# Check for CloudPickle first (binary format)
|
||||
if isinstance(serialized, bytes) and serialized.startswith(
|
||||
CloudPickleSerializationHandler.MAGIC_CLOUDPICKLE
|
||||
):
|
||||
return HandlerFormatName.CLOUDPICKLE
|
||||
|
||||
# Check for legacy pickle format (no magic bytes, should be base64 encoded)
|
||||
if isinstance(serialized, str):
|
||||
return HandlerFormatName.PICKLE
|
||||
|
||||
raise ValueError(
|
||||
f"Cannot detect serialization format from: {serialized[:20]}..."
|
||||
)
|
||||
@@ -0,0 +1,142 @@
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import (
|
||||
_Computed,
|
||||
_PublicField,
|
||||
migrate_private_fields,
|
||||
simple_split_tokenizer,
|
||||
)
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.tokenizer")
|
||||
class Tokenizer(SerializablePreprocessorBase):
|
||||
"""Replace each string with a list of tokens.
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> df = pd.DataFrame({"text": ["Hello, world!", "foo bar\\nbaz"]})
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
|
||||
The default ``tokenization_fn`` delimits strings using the space character.
|
||||
|
||||
>>> from ray.data.preprocessors import Tokenizer
|
||||
>>> tokenizer = Tokenizer(columns=["text"])
|
||||
>>> tokenizer.transform(ds).to_pandas() # doctest: +SKIP
|
||||
text
|
||||
0 [Hello,, world!]
|
||||
1 [foo, bar\\nbaz]
|
||||
|
||||
If the default logic isn't adequate for your use case, you can specify a
|
||||
custom ``tokenization_fn``.
|
||||
|
||||
>>> import string
|
||||
>>> def tokenization_fn(s):
|
||||
... for character in string.punctuation:
|
||||
... s = s.replace(character, "")
|
||||
... return s.split()
|
||||
>>> tokenizer = Tokenizer(columns=["text"], tokenization_fn=tokenization_fn)
|
||||
>>> tokenizer.transform(ds).to_pandas() # doctest: +SKIP
|
||||
text
|
||||
0 [Hello, world]
|
||||
1 [foo, bar, baz]
|
||||
|
||||
:class:`Tokenizer` can also be used in append mode by providing the
|
||||
name of the output_columns that should hold the tokenized values.
|
||||
|
||||
>>> tokenizer = Tokenizer(columns=["text"], output_columns=["text_tokenized"])
|
||||
>>> tokenizer.transform(ds).to_pandas() # doctest: +SKIP
|
||||
text text_tokenized
|
||||
0 Hello, world! [Hello,, world!]
|
||||
1 foo bar\\nbaz [foo, bar\\nbaz]
|
||||
|
||||
Args:
|
||||
columns: The columns to tokenize.
|
||||
tokenization_fn: The function used to generate tokens. This function
|
||||
should accept a string as input and return a list of tokens as
|
||||
output. If unspecified, the tokenizer uses a function equivalent to
|
||||
``lambda s: s.split(" ")``.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
"""
|
||||
|
||||
_is_fittable = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
tokenization_fn: Optional[Callable[[str], List[str]]] = None,
|
||||
output_columns: Optional[List[str]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
# TODO(matt): Add a more robust default tokenizer.
|
||||
self._tokenization_fn = tokenization_fn or simple_split_tokenizer
|
||||
self._output_columns = (
|
||||
SerializablePreprocessorBase._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def tokenization_fn(self) -> Callable[[str], List[str]]:
|
||||
return self._tokenization_fn
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
def column_tokenizer(s: pd.Series):
|
||||
return s.map(self._tokenization_fn)
|
||||
|
||||
df[self._output_columns] = df.loc[:, self._columns].transform(column_tokenizer)
|
||||
return df
|
||||
|
||||
def __repr__(self):
|
||||
name = getattr(self._tokenization_fn, "__name__", self._tokenization_fn)
|
||||
return (
|
||||
f"{self.__class__.__name__}(columns={self._columns!r}, "
|
||||
f"tokenization_fn={name}, output_columns={self._output_columns!r})"
|
||||
)
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"tokenization_fn": self._tokenization_fn,
|
||||
"output_columns": self._output_columns,
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._tokenization_fn = fields["tokenization_fn"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_tokenization_fn": _PublicField(
|
||||
public_field="tokenization_fn", default=simple_split_tokenizer
|
||||
),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,211 @@
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ray.data._internal.tensor_extensions.utils import _create_possibly_ragged_ndarray
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import _Computed, _PublicField, migrate_private_fields
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.data.util.data_batch_conversion import BatchFormat
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(
|
||||
version=1, identifier="io.ray.preprocessors.torchvision_preprocessor"
|
||||
)
|
||||
class TorchVisionPreprocessor(SerializablePreprocessorBase):
|
||||
"""Apply a `TorchVision transform <https://pytorch.org/vision/stable/transforms.html>`_
|
||||
to image columns.
|
||||
|
||||
Examples:
|
||||
|
||||
Torch models expect inputs of shape :math:`(B, C, H, W)` in the range
|
||||
:math:`[0.0, 1.0]`. To convert images to this format, add ``ToTensor`` to your
|
||||
preprocessing pipeline.
|
||||
|
||||
.. testcode::
|
||||
|
||||
from torchvision import transforms
|
||||
|
||||
import ray
|
||||
from ray.data.preprocessors import TorchVisionPreprocessor
|
||||
|
||||
transform = transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Resize((224, 224)),
|
||||
])
|
||||
preprocessor = TorchVisionPreprocessor(["image"], transform=transform)
|
||||
|
||||
dataset = ray.data.read_images("s3://anonymous@air-example-data-2/imagenet-sample-images")
|
||||
dataset = preprocessor.transform(dataset)
|
||||
|
||||
|
||||
For better performance, set ``batched`` to ``True`` and replace ``ToTensor``
|
||||
with a batch-supporting ``Lambda``.
|
||||
|
||||
.. testcode::
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
def to_tensor(batch: np.ndarray) -> torch.Tensor:
|
||||
tensor = torch.as_tensor(batch, dtype=torch.float)
|
||||
# (B, H, W, C) -> (B, C, H, W)
|
||||
tensor = tensor.permute(0, 3, 1, 2).contiguous()
|
||||
# [0., 255.] -> [0., 1.]
|
||||
tensor = tensor.div(255)
|
||||
return tensor
|
||||
|
||||
transform = transforms.Compose([
|
||||
transforms.Lambda(to_tensor),
|
||||
transforms.Resize((224, 224))
|
||||
])
|
||||
preprocessor = TorchVisionPreprocessor(["image"], transform=transform, batched=True)
|
||||
|
||||
dataset = ray.data.read_images("s3://anonymous@air-example-data-2/imagenet-sample-images")
|
||||
dataset = preprocessor.transform(dataset)
|
||||
|
||||
Args:
|
||||
columns: The columns to apply the TorchVision transform to.
|
||||
transform: The TorchVision transform you want to apply. This transform should
|
||||
accept a ``np.ndarray`` or ``torch.Tensor`` as input and return a
|
||||
``torch.Tensor`` as output.
|
||||
output_columns: The output name for each input column. If not specified, this
|
||||
defaults to the same set of columns as the columns.
|
||||
batched: If ``True``, apply ``transform`` to batches of shape
|
||||
:math:`(B, H, W, C)`. Otherwise, apply ``transform`` to individual images.
|
||||
""" # noqa: E501
|
||||
|
||||
_is_fittable = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
transform: Callable[[Union["np.ndarray", "torch.Tensor"]], "torch.Tensor"],
|
||||
output_columns: Optional[List[str]] = None,
|
||||
batched: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
if not output_columns:
|
||||
output_columns = columns
|
||||
if len(columns) != len(output_columns):
|
||||
raise ValueError(
|
||||
"The length of columns should match the "
|
||||
f"length of output_columns: {columns} vs {output_columns}."
|
||||
)
|
||||
self._columns = columns
|
||||
self._output_columns = output_columns
|
||||
self._torchvision_transform = transform
|
||||
self._batched = batched
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def torchvision_transform(
|
||||
self,
|
||||
) -> Callable[[Union["np.ndarray", "torch.Tensor"]], "torch.Tensor"]:
|
||||
return self._torchvision_transform
|
||||
|
||||
@property
|
||||
def batched(self) -> bool:
|
||||
return self._batched
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"{self.__class__.__name__}("
|
||||
f"columns={self._columns}, "
|
||||
f"output_columns={self._output_columns}, "
|
||||
f"transform={self._torchvision_transform!r})"
|
||||
)
|
||||
|
||||
def _transform_numpy(
|
||||
self, data_batch: Dict[str, "np.ndarray"]
|
||||
) -> Dict[str, "np.ndarray"]:
|
||||
import torch
|
||||
|
||||
from ray.data.util.torch_utils import convert_ndarray_to_torch_tensor
|
||||
|
||||
def apply_torchvision_transform(array: np.ndarray) -> np.ndarray:
|
||||
try:
|
||||
tensor = convert_ndarray_to_torch_tensor(array)
|
||||
output = self._torchvision_transform(tensor)
|
||||
except TypeError:
|
||||
# Transforms like `ToTensor` expect a `np.ndarray` as input.
|
||||
output = self._torchvision_transform(array)
|
||||
if isinstance(output, torch.Tensor):
|
||||
output = output.numpy()
|
||||
if not isinstance(output, np.ndarray):
|
||||
raise ValueError(
|
||||
"`TorchVisionPreprocessor` expected your transform to return a "
|
||||
"`torch.Tensor` or `np.ndarray`, but your transform returned a "
|
||||
f"`{type(output).__name__}` instead."
|
||||
)
|
||||
return output
|
||||
|
||||
def transform_batch(batch: np.ndarray) -> np.ndarray:
|
||||
if self._batched:
|
||||
return apply_torchvision_transform(batch)
|
||||
return _create_possibly_ragged_ndarray(
|
||||
[apply_torchvision_transform(array) for array in batch]
|
||||
)
|
||||
|
||||
if isinstance(data_batch, Mapping):
|
||||
for input_col, output_col in zip(self._columns, self._output_columns):
|
||||
data_batch[output_col] = transform_batch(data_batch[input_col])
|
||||
else:
|
||||
# TODO(ekl) deprecate this code path. Unfortunately, predictors are still
|
||||
# sending schemaless arrays to preprocessors.
|
||||
data_batch = transform_batch(data_batch)
|
||||
|
||||
return data_batch
|
||||
|
||||
def get_input_columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
def get_output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def preferred_batch_format(cls) -> BatchFormat:
|
||||
return BatchFormat.NUMPY
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"output_columns": self._output_columns,
|
||||
"torchvision_transform": self._torchvision_transform,
|
||||
"batched": self._batched,
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
self._torchvision_transform = fields["torchvision_transform"]
|
||||
self._batched = fields["batched"]
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_torchvision_transform": _PublicField(
|
||||
public_field="torchvision_transform"
|
||||
),
|
||||
"_batched": _PublicField(public_field="batched", default=False),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,155 @@
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import _Computed, _PublicField, migrate_private_fields
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(
|
||||
version=1, identifier="io.ray.preprocessors.power_transformer"
|
||||
)
|
||||
class PowerTransformer(SerializablePreprocessorBase):
|
||||
"""Apply a `power transform <https://en.wikipedia.org/wiki/Power_transform>`_ to
|
||||
make your data more normally distributed.
|
||||
|
||||
Some models expect data to be normally distributed. By making your data more
|
||||
Gaussian-like, you might be able to improve your model's performance.
|
||||
|
||||
This preprocessor supports the following transformations:
|
||||
|
||||
* `Yeo-Johnson <https://en.wikipedia.org/wiki/Power_transform#Yeo%E2%80%93Johnson_transformation>`_
|
||||
* `Box-Cox <https://en.wikipedia.org/wiki/Power_transform#Box%E2%80%93Cox_transformation>`_
|
||||
|
||||
Box-Cox requires all data to be positive.
|
||||
|
||||
.. warning::
|
||||
|
||||
You need to manually specify the transform's power parameter. If you
|
||||
choose a bad value, the transformation might not work well.
|
||||
|
||||
Args:
|
||||
columns: The columns to separately transform.
|
||||
power: A parameter that determines how your data is transformed. Practioners
|
||||
typically set ``power`` between :math:`-2.5` and :math:`2.5`, although you
|
||||
may need to try different values to find one that works well.
|
||||
method: A string representing which transformation to apply. Supports
|
||||
``"yeo-johnson"`` and ``"box-cox"``. If you choose ``"box-cox"``, your data
|
||||
needs to be positive. Defaults to ``"yeo-johnson"``.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
""" # noqa: E501
|
||||
|
||||
_valid_methods = ["yeo-johnson", "box-cox"]
|
||||
_is_fittable = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
power: float,
|
||||
method: str = "yeo-johnson",
|
||||
*,
|
||||
output_columns: Optional[List[str]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._method = method
|
||||
self._power = power
|
||||
self._output_columns = (
|
||||
SerializablePreprocessorBase._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
)
|
||||
|
||||
if method not in self._valid_methods:
|
||||
raise ValueError(
|
||||
f"Method {method} is not supported."
|
||||
f"Supported values are: {self._valid_methods}"
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def method(self) -> str:
|
||||
return self._method
|
||||
|
||||
@property
|
||||
def power(self) -> float:
|
||||
return self._power
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
def column_power_transformer(s: pd.Series):
|
||||
if self._method == "yeo-johnson":
|
||||
result = np.zeros_like(s, dtype=np.float64)
|
||||
pos = s >= 0 # binary mask
|
||||
|
||||
if self._power != 0:
|
||||
result[pos] = (np.power(s[pos] + 1, self._power) - 1) / self._power
|
||||
else:
|
||||
result[pos] = np.log(s[pos] + 1)
|
||||
|
||||
if self._power != 2:
|
||||
result[~pos] = -(np.power(-s[~pos] + 1, 2 - self._power) - 1) / (
|
||||
2 - self._power
|
||||
)
|
||||
else:
|
||||
result[~pos] = -np.log(-s[~pos] + 1)
|
||||
return result
|
||||
|
||||
else: # box-cox
|
||||
if self._power != 0:
|
||||
return (np.power(s, self._power) - 1) / self._power
|
||||
else:
|
||||
return np.log(s)
|
||||
|
||||
df[self._output_columns] = df[self._columns].transform(column_power_transformer)
|
||||
return df
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"{self.__class__.__name__}(columns={self._columns!r}, "
|
||||
f"power={self._power!r}, method={self._method!r}, "
|
||||
f"output_columns={self._output_columns!r})"
|
||||
)
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"power": self._power,
|
||||
"method": self._method,
|
||||
"output_columns": self._output_columns,
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._power = fields["power"]
|
||||
self._method = fields["method"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_power": _PublicField(public_field="power"),
|
||||
"_method": _PublicField(public_field="method", default="yeo-johnson"),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,318 @@
|
||||
import hashlib
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Deque,
|
||||
Dict,
|
||||
List,
|
||||
Optional,
|
||||
Union,
|
||||
)
|
||||
|
||||
import ray
|
||||
from ray.air.util.data_batch_conversion import BatchFormat
|
||||
from ray.data.aggregate import AggregateFnV2
|
||||
from ray.util.annotations import DeveloperAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data.dataset import Dataset
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
def simple_split_tokenizer(value: str) -> List[str]:
|
||||
"""Tokenize a string using a split on spaces."""
|
||||
return value.split(" ")
|
||||
|
||||
|
||||
@DeveloperAPI
|
||||
def simple_hash(value: object, num_features: int) -> int:
|
||||
"""Deterministically hash a value into the integer space."""
|
||||
encoded_value = str(value).encode()
|
||||
hashed_value = hashlib.sha256(encoded_value)
|
||||
hashed_value_int = int(hashed_value.hexdigest(), 16)
|
||||
return hashed_value_int % num_features
|
||||
|
||||
|
||||
class BaseStatSpec:
|
||||
"""Encapsulates a statistical computation with optional post-processing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
stat_fn: Union[AggregateFnV2, Callable],
|
||||
post_process_fn: Callable = lambda x: x,
|
||||
):
|
||||
self.stat_fn = stat_fn
|
||||
self.post_process_fn = post_process_fn
|
||||
|
||||
|
||||
class AggregateStatSpec(BaseStatSpec):
|
||||
"""Represents an AggregateFnV2 spec for a single column."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
aggregator_fn: Union[AggregateFnV2, Callable[[str], AggregateFnV2]],
|
||||
post_process_fn: Callable = lambda x: x,
|
||||
column: Optional[str] = None,
|
||||
batch_format: Optional[BatchFormat] = None,
|
||||
):
|
||||
super().__init__(
|
||||
stat_fn=aggregator_fn,
|
||||
post_process_fn=post_process_fn,
|
||||
)
|
||||
self.column = column
|
||||
self.batch_format = batch_format
|
||||
|
||||
|
||||
class CallableStatSpec(BaseStatSpec):
|
||||
"""Represents a user-defined stat function that operates outside Dataset.aggregate."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
stat_fn: Callable,
|
||||
stat_key_fn: Optional[Callable[[str], str]],
|
||||
post_key_fn: Optional[Callable[[str], str]],
|
||||
post_process_fn: Callable = lambda x: x,
|
||||
columns: List[str],
|
||||
):
|
||||
super().__init__(
|
||||
stat_fn=stat_fn,
|
||||
post_process_fn=post_process_fn,
|
||||
)
|
||||
self.columns = columns
|
||||
self.stat_key_fn = stat_key_fn
|
||||
self.post_key_fn = post_key_fn
|
||||
|
||||
|
||||
class StatComputationPlan:
|
||||
"""
|
||||
Encapsulates a set of aggregators (AggregateFnV2) and legacy stat functions
|
||||
to compute statistics over a Ray dataset.
|
||||
|
||||
Supports two types of aggregations:
|
||||
1. AggregateFnV2-based aggregators, which are batch-executed using `Dataset.aggregate(...)`.
|
||||
2. Callable-based stat functions, executed sequentially (legacy use case).
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._aggregators: Deque[BaseStatSpec] = deque()
|
||||
|
||||
def reset(self):
|
||||
self._aggregators.clear()
|
||||
|
||||
def add_aggregator(
|
||||
self,
|
||||
*,
|
||||
aggregator_fn: Callable[[str], AggregateFnV2],
|
||||
post_process_fn: Callable = lambda x: x,
|
||||
columns: List[str],
|
||||
batch_format: Optional[BatchFormat] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Registers an AggregateFnV2 factory for one or more columns.
|
||||
|
||||
Args:
|
||||
aggregator_fn: A callable (typically a lambda or class) that accepts a column name and returns an instance of AggregateFnV2.
|
||||
The aggregator should set its name using alias_name parameter to control the output key.
|
||||
post_process_fn: Function to post-process the aggregated result.
|
||||
columns: List of column names to aggregate.
|
||||
batch_format: The batch format for aggregation results. If ARROW, results
|
||||
are kept in Arrow format for post_process_fn. Otherwise,
|
||||
results are converted to Python/pandas format.
|
||||
"""
|
||||
for column in columns:
|
||||
agg_instance = aggregator_fn(column)
|
||||
self._aggregators.append(
|
||||
AggregateStatSpec(
|
||||
aggregator_fn=agg_instance,
|
||||
post_process_fn=post_process_fn,
|
||||
column=column,
|
||||
batch_format=batch_format,
|
||||
)
|
||||
)
|
||||
|
||||
def add_callable_stat(
|
||||
self,
|
||||
*,
|
||||
stat_fn: Callable[[], Any],
|
||||
stat_key_fn: Callable[[str], str],
|
||||
post_key_fn: Optional[Callable[[str], str]] = None,
|
||||
post_process_fn: Callable = lambda x: x,
|
||||
columns: List[str],
|
||||
) -> None:
|
||||
"""
|
||||
Registers a custom stat function to be run sequentially.
|
||||
|
||||
This supports legacy use cases where arbitrary callables are needed
|
||||
and cannot be run via Dataset.aggregate().
|
||||
|
||||
Args:
|
||||
stat_fn: A zero-argument callable that returns the stat.
|
||||
stat_key_fn: A callable that takes a column name and returns the key for the stat.
|
||||
post_key_fn: Optional; a callable to post-process the key. If not provided, stat_key_fn is used.
|
||||
post_process_fn: Function to post-process the result.
|
||||
columns: List of column names to compute the stat for.
|
||||
"""
|
||||
self._aggregators.append(
|
||||
CallableStatSpec(
|
||||
stat_fn=stat_fn,
|
||||
post_process_fn=post_process_fn,
|
||||
columns=columns,
|
||||
stat_key_fn=stat_key_fn,
|
||||
post_key_fn=post_key_fn or stat_key_fn,
|
||||
)
|
||||
)
|
||||
|
||||
def compute(self, dataset: "Dataset") -> Dict[str, Any]:
|
||||
"""
|
||||
Executes all registered aggregators and stat functions.
|
||||
|
||||
AggregateFnV2-based aggregators are batched and executed via Dataset.aggregate().
|
||||
Callable-based stat functions are run sequentially.
|
||||
|
||||
Args:
|
||||
dataset: The Ray Dataset to compute statistics on.
|
||||
|
||||
Returns:
|
||||
A dictionary of computed statistics.
|
||||
"""
|
||||
stats = {}
|
||||
# Run batched aggregators (AggregateFnV2)
|
||||
aggregators = self._get_aggregate_fn_list()
|
||||
if aggregators:
|
||||
agg_ds = dataset.groupby(None).aggregate(*aggregators)
|
||||
arrow_refs = agg_ds.to_arrow_refs()
|
||||
if not arrow_refs:
|
||||
raise ValueError("Aggregation returned no results")
|
||||
arrow_table = ray.get(arrow_refs[0])
|
||||
for spec in self._get_aggregate_specs():
|
||||
stat_key = spec.stat_fn.name
|
||||
# Aggregation returns single row - extract the scalar value
|
||||
# ChunkedArray[0] handles multi-chunk arrays automatically
|
||||
agg_result = arrow_table.column(stat_key)[0]
|
||||
# Convert to appropriate format based on batch_format
|
||||
if spec.batch_format == BatchFormat.ARROW:
|
||||
# Pass Arrow scalar (e.g., ListScalar) for Arrow-optimized post-processing
|
||||
stats[stat_key] = spec.post_process_fn(agg_result)
|
||||
else:
|
||||
# Convert to Python for pandas-style post-processing
|
||||
stats[stat_key] = spec.post_process_fn(agg_result.as_py())
|
||||
|
||||
# Run sequential stat functions
|
||||
for spec in self._get_custom_stat_fn_specs():
|
||||
result = spec.stat_fn(spec.stat_key_fn)
|
||||
for col in spec.columns:
|
||||
stat_key = spec.stat_key_fn(col)
|
||||
post_key = spec.post_key_fn(col)
|
||||
stats[post_key] = spec.post_process_fn(result[stat_key])
|
||||
|
||||
return stats
|
||||
|
||||
def _get_aggregate_fn_list(self) -> List[AggregateFnV2]:
|
||||
return [
|
||||
spec.stat_fn
|
||||
for spec in self._aggregators
|
||||
if isinstance(spec, AggregateStatSpec)
|
||||
]
|
||||
|
||||
def _get_aggregate_specs(self) -> List[AggregateStatSpec]:
|
||||
return [
|
||||
spec for spec in self._aggregators if isinstance(spec, AggregateStatSpec)
|
||||
]
|
||||
|
||||
def _get_custom_stat_fn_specs(self) -> List[CallableStatSpec]:
|
||||
return [
|
||||
spec for spec in self._aggregators if isinstance(spec, CallableStatSpec)
|
||||
]
|
||||
|
||||
def has_custom_stat_fn(self):
|
||||
return len(self._get_custom_stat_fn_specs()) > 0
|
||||
|
||||
def __iter__(self):
|
||||
"""
|
||||
Iterates over all AggregatorSpecs.
|
||||
"""
|
||||
return iter(self._get_aggregate_specs())
|
||||
|
||||
|
||||
def make_post_processor(base_fn, callbacks: List[Callable]):
|
||||
"""
|
||||
Wraps a base post-processing function with a sequence of callback functions.
|
||||
Useful when multiple post-processing steps need to be applied in order.
|
||||
"""
|
||||
|
||||
def wrapper(result):
|
||||
processed = base_fn(result)
|
||||
for cb in callbacks:
|
||||
processed = cb(processed)
|
||||
return processed
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class _Computed:
|
||||
"""
|
||||
Wraps a factory callable for defaults that must be computed from the object.
|
||||
|
||||
Plain callable values (e.g. a tokenizer function stored as an attribute)
|
||||
must NOT be wrapped — they will be stored as-is.
|
||||
"""
|
||||
|
||||
def __init__(self, factory: Callable[[Any], Any]) -> None:
|
||||
self._factory = factory
|
||||
|
||||
def __call__(self, obj: Any) -> Any:
|
||||
return self._factory(obj)
|
||||
|
||||
|
||||
_REQUIRED_FIELD = object() # Sentinel for required fields with no default value
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PublicField:
|
||||
"""
|
||||
Represents a public field that may have been used in older versions of the code.
|
||||
If the field's default value is not _REQUIRED_FIELD, it will be used if neither the private field nor the public field is present during unpickling.
|
||||
Otherwise, the field is required and must be present as either the private or public field during unpickling, or a ValueError will be raised.
|
||||
Used for backwards compatibility during unpickling.
|
||||
"""
|
||||
|
||||
public_field: str
|
||||
default: Any = _REQUIRED_FIELD
|
||||
|
||||
|
||||
def migrate_private_fields(
|
||||
obj: Any,
|
||||
*,
|
||||
fields: Dict[str, _PublicField],
|
||||
) -> None:
|
||||
"""
|
||||
Migrates old public field names to new private field names during unpickling for backwards compatibility.
|
||||
"""
|
||||
for private_field, public_field_obj in fields.items():
|
||||
if private_field not in obj.__dict__:
|
||||
if public_field_obj.public_field in obj.__dict__:
|
||||
# Migrate from old public field names to new private field names
|
||||
setattr(
|
||||
obj, private_field, obj.__dict__.pop(public_field_obj.public_field)
|
||||
)
|
||||
elif public_field_obj.default is _REQUIRED_FIELD:
|
||||
raise ValueError(
|
||||
f"Invalid serialized {type(obj).__name__}: missing required field '{private_field}'."
|
||||
)
|
||||
else:
|
||||
# Set defaults for missing fields.
|
||||
# _Computed defaults are called with obj; all other values are stored as-is,
|
||||
# including callable objects like tokenizer functions.
|
||||
setattr(
|
||||
obj,
|
||||
private_field,
|
||||
public_field_obj.default(obj)
|
||||
if isinstance(public_field_obj.default, _Computed)
|
||||
else public_field_obj.default,
|
||||
)
|
||||
@@ -0,0 +1,446 @@
|
||||
from collections import Counter
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
from ray.data.preprocessors.utils import (
|
||||
_Computed,
|
||||
_PublicField,
|
||||
migrate_private_fields,
|
||||
simple_hash,
|
||||
simple_split_tokenizer,
|
||||
)
|
||||
from ray.data.preprocessors.version_support import SerializablePreprocessor
|
||||
from ray.util.annotations import PublicAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.data.dataset import Dataset
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(
|
||||
version=1, identifier="io.ray.preprocessors.hashing_vectorizer"
|
||||
)
|
||||
class HashingVectorizer(SerializablePreprocessorBase):
|
||||
"""Count the frequency of tokens using the
|
||||
`hashing trick <https://en.wikipedia.org/wiki/Feature_hashing>`_.
|
||||
|
||||
This preprocessors creates a list column for each input column. For each row,
|
||||
the list contains the frequency counts of tokens (for CountVectorizer) or hash values
|
||||
(for HashingVectorizer). For HashingVectorizer, the list will have length
|
||||
``num_features``. If ``num_features`` is large enough relative to the size of your
|
||||
vocabulary, then each index approximately corresponds to the frequency of a unique
|
||||
token.
|
||||
|
||||
:class:`HashingVectorizer` is memory efficient and quick to pickle. However, given a
|
||||
transformed column, you can't know which tokens correspond to it. This might make it
|
||||
hard to determine which tokens are important to your model.
|
||||
|
||||
.. note::
|
||||
|
||||
This preprocessor transforms each input column to a
|
||||
`document-term matrix <https://en.wikipedia.org/wiki/Document-term_matrix>`_.
|
||||
|
||||
A document-term matrix is a table that describes the frequency of tokens in a
|
||||
collection of documents. For example, the strings `"I like Python"` and `"I
|
||||
dislike Python"` might have the document-term matrix below:
|
||||
|
||||
.. code-block::
|
||||
|
||||
corpus_I corpus_Python corpus_dislike corpus_like
|
||||
0 1 1 1 0
|
||||
1 1 1 0 1
|
||||
|
||||
To generate the matrix, you typically map each token to a unique index. For
|
||||
example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
token index
|
||||
0 I 0
|
||||
1 Python 1
|
||||
2 dislike 2
|
||||
3 like 3
|
||||
|
||||
The problem with this approach is that memory use scales linearly with the size
|
||||
of your vocabulary. :class:`HashingVectorizer` circumvents this problem by
|
||||
computing indices with a hash function:
|
||||
:math:`\\texttt{index} = hash(\\texttt{token})`.
|
||||
|
||||
.. warning::
|
||||
Sparse matrices aren't currently supported. If you use a large ``num_features``,
|
||||
this preprocessor might behave poorly.
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import HashingVectorizer
|
||||
>>>
|
||||
>>> df = pd.DataFrame({
|
||||
... "corpus": [
|
||||
... "Jimmy likes volleyball",
|
||||
... "Bob likes volleyball too",
|
||||
... "Bob also likes fruit jerky"
|
||||
... ]
|
||||
... })
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>>
|
||||
>>> vectorizer = HashingVectorizer(["corpus"], num_features=8)
|
||||
>>> vectorizer.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
corpus
|
||||
0 [1, 0, 1, 0, 0, 0, 0, 1]
|
||||
1 [1, 0, 1, 0, 0, 0, 1, 1]
|
||||
2 [0, 0, 1, 1, 0, 2, 1, 0]
|
||||
|
||||
:class:`HashingVectorizer` can also be used in append mode by providing the
|
||||
name of the output_columns that should hold the encoded values.
|
||||
|
||||
>>> vectorizer = HashingVectorizer(["corpus"], num_features=8, output_columns=["corpus_hashed"])
|
||||
>>> vectorizer.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
corpus corpus_hashed
|
||||
0 Jimmy likes volleyball [1, 0, 1, 0, 0, 0, 0, 1]
|
||||
1 Bob likes volleyball too [1, 0, 1, 0, 0, 0, 1, 1]
|
||||
2 Bob also likes fruit jerky [0, 0, 1, 1, 0, 2, 1, 0]
|
||||
|
||||
Args:
|
||||
columns: The columns to separately tokenize and count.
|
||||
num_features: The number of features used to represent the vocabulary. You
|
||||
should choose a value large enough to prevent hash collisions between
|
||||
distinct tokens.
|
||||
tokenization_fn: The function used to generate tokens. This function
|
||||
should accept a string as input and return a list of tokens as
|
||||
output. If unspecified, the tokenizer uses a function equivalent to
|
||||
``lambda s: s.split(" ")``.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
|
||||
.. seealso::
|
||||
|
||||
:class:`CountVectorizer`
|
||||
Another method for counting token frequencies. Unlike :class:`HashingVectorizer`,
|
||||
:class:`CountVectorizer` creates a feature for each unique token. This
|
||||
enables you to compute the inverse transformation.
|
||||
|
||||
:class:`FeatureHasher`
|
||||
This preprocessor is similar to :class:`HashingVectorizer`, except it expects
|
||||
a table describing token frequencies. In contrast,
|
||||
:class:`FeatureHasher` expects a column containing documents.
|
||||
""" # noqa: E501
|
||||
|
||||
_is_fittable = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
num_features: int,
|
||||
tokenization_fn: Optional[Callable[[str], List[str]]] = None,
|
||||
*,
|
||||
output_columns: Optional[List[str]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._num_features = num_features
|
||||
self._tokenization_fn = tokenization_fn or simple_split_tokenizer
|
||||
self._output_columns = (
|
||||
SerializablePreprocessorBase._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def num_features(self) -> int:
|
||||
return self._num_features
|
||||
|
||||
@property
|
||||
def tokenization_fn(self) -> Callable[[str], List[str]]:
|
||||
return self._tokenization_fn
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
def hash_count(tokens: List[str]) -> Counter:
|
||||
hashed_tokens = [simple_hash(token, self._num_features) for token in tokens]
|
||||
return Counter(hashed_tokens)
|
||||
|
||||
for col, output_col in zip(self._columns, self._output_columns):
|
||||
tokenized = df[col].map(self._tokenization_fn)
|
||||
hashed = tokenized.map(hash_count)
|
||||
# Create a list to store the hash columns
|
||||
hash_columns = []
|
||||
for i in range(self._num_features):
|
||||
series = hashed.map(lambda counts: counts[i])
|
||||
series.name = f"hash_{i}"
|
||||
hash_columns.append(series)
|
||||
# Concatenate all hash columns into a single list column
|
||||
df[output_col] = pd.concat(hash_columns, axis=1).values.tolist()
|
||||
|
||||
return df
|
||||
|
||||
def __repr__(self):
|
||||
fn_name = getattr(self._tokenization_fn, "__name__", self._tokenization_fn)
|
||||
return (
|
||||
f"{self.__class__.__name__}(columns={self._columns!r}, "
|
||||
f"num_features={self._num_features!r}, tokenization_fn={fn_name}, "
|
||||
f"output_columns={self._output_columns!r})"
|
||||
)
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"num_features": self._num_features,
|
||||
"tokenization_fn": self._tokenization_fn,
|
||||
"output_columns": self._output_columns,
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._num_features = fields["num_features"]
|
||||
self._tokenization_fn = fields["tokenization_fn"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
"""Handle backwards compatibility for old pickled objects."""
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_num_features": _PublicField(public_field="num_features"),
|
||||
"_tokenization_fn": _PublicField(
|
||||
public_field="tokenization_fn", default=simple_split_tokenizer
|
||||
),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
@SerializablePreprocessor(version=1, identifier="io.ray.preprocessors.count_vectorizer")
|
||||
class CountVectorizer(SerializablePreprocessorBase):
|
||||
"""Count the frequency of tokens in a column of strings.
|
||||
|
||||
:class:`CountVectorizer` operates on columns that contain strings. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
corpus
|
||||
0 I dislike Python
|
||||
1 I like Python
|
||||
|
||||
This preprocessor creates a list column for each input column. Each list contains
|
||||
the frequency counts of tokens in order of their first appearance. For example:
|
||||
|
||||
.. code-block::
|
||||
|
||||
corpus
|
||||
0 [1, 1, 1, 0] # Counts for [I, dislike, Python, like]
|
||||
1 [1, 0, 1, 1] # Counts for [I, dislike, Python, like]
|
||||
|
||||
Examples:
|
||||
>>> import pandas as pd
|
||||
>>> import ray
|
||||
>>> from ray.data.preprocessors import CountVectorizer
|
||||
>>>
|
||||
>>> df = pd.DataFrame({
|
||||
... "corpus": [
|
||||
... "Jimmy likes volleyball",
|
||||
... "Bob likes volleyball too",
|
||||
... "Bob also likes fruit jerky"
|
||||
... ]
|
||||
... })
|
||||
>>> ds = ray.data.from_pandas(df) # doctest: +SKIP
|
||||
>>>
|
||||
>>> vectorizer = CountVectorizer(["corpus"])
|
||||
>>> vectorizer.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
corpus
|
||||
0 [1, 0, 1, 1, 0, 0, 0, 0]
|
||||
1 [1, 1, 1, 0, 0, 0, 0, 1]
|
||||
2 [1, 1, 0, 0, 1, 1, 1, 0]
|
||||
|
||||
You can limit the number of tokens in the vocabulary with ``max_features``.
|
||||
|
||||
>>> vectorizer = CountVectorizer(["corpus"], max_features=3)
|
||||
>>> vectorizer.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
corpus
|
||||
0 [1, 0, 1]
|
||||
1 [1, 1, 1]
|
||||
2 [1, 1, 0]
|
||||
|
||||
:class:`CountVectorizer` can also be used in append mode by providing the
|
||||
name of the output_columns that should hold the encoded values.
|
||||
|
||||
>>> vectorizer = CountVectorizer(["corpus"], output_columns=["corpus_counts"])
|
||||
>>> vectorizer.fit_transform(ds).to_pandas() # doctest: +SKIP
|
||||
corpus corpus_counts
|
||||
0 Jimmy likes volleyball [1, 0, 1, 1, 0, 0, 0, 0]
|
||||
1 Bob likes volleyball too [1, 1, 1, 0, 0, 0, 0, 1]
|
||||
2 Bob also likes fruit jerky [1, 1, 0, 0, 1, 1, 1, 0]
|
||||
|
||||
Args:
|
||||
columns: The columns to separately tokenize and count.
|
||||
tokenization_fn: The function used to generate tokens. This function
|
||||
should accept a string as input and return a list of tokens as
|
||||
output. If unspecified, the tokenizer uses a function equivalent to
|
||||
``lambda s: s.split(" ")``.
|
||||
max_features: The maximum number of tokens to encode in the transformed
|
||||
dataset. If specified, only the most frequent tokens are encoded.
|
||||
output_columns: The names of the transformed columns. If None, the transformed
|
||||
columns will be the same as the input columns. If not None, the length of
|
||||
``output_columns`` must match the length of ``columns``, othwerwise an error
|
||||
will be raised.
|
||||
""" # noqa: E501
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
columns: List[str],
|
||||
tokenization_fn: Optional[Callable[[str], List[str]]] = None,
|
||||
max_features: Optional[int] = None,
|
||||
*,
|
||||
output_columns: Optional[List[str]] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self._columns = columns
|
||||
self._tokenization_fn = tokenization_fn or simple_split_tokenizer
|
||||
self._max_features = max_features
|
||||
self._output_columns = (
|
||||
SerializablePreprocessorBase._derive_and_validate_output_columns(
|
||||
columns, output_columns
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def columns(self) -> List[str]:
|
||||
return self._columns
|
||||
|
||||
@property
|
||||
def tokenization_fn(self) -> Callable[[str], List[str]]:
|
||||
return self._tokenization_fn
|
||||
|
||||
@property
|
||||
def max_features(self) -> Optional[int]:
|
||||
return self._max_features
|
||||
|
||||
@property
|
||||
def output_columns(self) -> List[str]:
|
||||
return self._output_columns
|
||||
|
||||
def _fit(self, dataset: "Dataset") -> SerializablePreprocessorBase:
|
||||
def stat_fn(key_gen):
|
||||
def get_pd_value_counts(df: pd.DataFrame) -> List[Counter]:
|
||||
def get_token_counts(col):
|
||||
token_series = df[col].apply(self._tokenization_fn)
|
||||
tokens = token_series.sum()
|
||||
return Counter(tokens)
|
||||
|
||||
return {col: [get_token_counts(col)] for col in self._columns}
|
||||
|
||||
value_counts = dataset.map_batches(
|
||||
get_pd_value_counts, batch_format="pandas"
|
||||
)
|
||||
total_counts = {col: Counter() for col in self._columns}
|
||||
for batch in value_counts.iter_batches(batch_size=None):
|
||||
for col, counters in batch.items():
|
||||
for counter in counters:
|
||||
total_counts[col].update(counter)
|
||||
|
||||
def most_common(counter: Counter, n: int):
|
||||
return Counter(dict(counter.most_common(n)))
|
||||
|
||||
top_counts = [
|
||||
most_common(counter, self._max_features)
|
||||
for counter in total_counts.values()
|
||||
]
|
||||
|
||||
return {
|
||||
key_gen(col): counts # noqa
|
||||
for (col, counts) in zip(self._columns, top_counts)
|
||||
}
|
||||
|
||||
self._stat_computation_plan.add_callable_stat(
|
||||
stat_fn=lambda key_gen: stat_fn(key_gen),
|
||||
stat_key_fn=lambda col: f"token_counts({col})",
|
||||
columns=self._columns,
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def _transform_pandas(self, df: pd.DataFrame):
|
||||
result_columns = []
|
||||
for col, output_col in zip(self._columns, self._output_columns):
|
||||
token_counts = self.stats_[f"token_counts({col})"]
|
||||
sorted_tokens = [token for (token, count) in token_counts.most_common()]
|
||||
tokenized = df[col].map(self._tokenization_fn).map(Counter)
|
||||
|
||||
# Create a list to store token frequencies
|
||||
token_columns = []
|
||||
for token in sorted_tokens:
|
||||
series = tokenized.map(lambda val: val[token])
|
||||
series.name = token
|
||||
token_columns.append(series)
|
||||
|
||||
# Concatenate all token columns into a single list column
|
||||
if token_columns:
|
||||
df[output_col] = pd.concat(token_columns, axis=1).values.tolist()
|
||||
else:
|
||||
df[output_col] = [[]] * len(df)
|
||||
result_columns.append(output_col)
|
||||
|
||||
return df
|
||||
|
||||
def __repr__(self):
|
||||
fn_name = getattr(self._tokenization_fn, "__name__", self._tokenization_fn)
|
||||
return (
|
||||
f"{self.__class__.__name__}(columns={self._columns!r}, "
|
||||
f"tokenization_fn={fn_name}, max_features={self._max_features!r}, "
|
||||
f"output_columns={self._output_columns!r})"
|
||||
)
|
||||
|
||||
def _get_serializable_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"columns": self._columns,
|
||||
"tokenization_fn": self._tokenization_fn,
|
||||
"max_features": self._max_features,
|
||||
"output_columns": self._output_columns,
|
||||
"_fitted": getattr(self, "_fitted", None),
|
||||
}
|
||||
|
||||
def _set_serializable_fields(self, fields: Dict[str, Any], version: int):
|
||||
# required fields
|
||||
self._columns = fields["columns"]
|
||||
self._tokenization_fn = fields["tokenization_fn"]
|
||||
self._max_features = fields["max_features"]
|
||||
self._output_columns = fields["output_columns"]
|
||||
# optional fields
|
||||
self._fitted = fields.get("_fitted")
|
||||
|
||||
def __setstate__(self, state: Dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
migrate_private_fields(
|
||||
self,
|
||||
fields={
|
||||
"_columns": _PublicField(public_field="columns"),
|
||||
"_tokenization_fn": _PublicField(
|
||||
public_field="tokenization_fn", default=simple_split_tokenizer
|
||||
),
|
||||
"_max_features": _PublicField(
|
||||
public_field="max_features", default=None
|
||||
),
|
||||
"_output_columns": _PublicField(
|
||||
public_field="output_columns",
|
||||
default=_Computed(lambda obj: obj._columns),
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,86 @@
|
||||
class UnknownPreprocessorError(ValueError):
|
||||
"""Raised when attempting to deserialize an unknown/unregistered preprocessor type."""
|
||||
|
||||
def __init__(self, preprocessor_type: str):
|
||||
self.preprocessor_type = preprocessor_type
|
||||
super().__init__(f"Unknown preprocessor type: {preprocessor_type}")
|
||||
|
||||
|
||||
_PREPROCESSOR_REGISTRY = {}
|
||||
|
||||
|
||||
def SerializablePreprocessor(version: int, identifier: str):
|
||||
"""Register a preprocessor class for serialization.
|
||||
|
||||
This decorator registers a preprocessor class in the serialization registry,
|
||||
enabling it to be serialized and deserialized. The decorated class MUST inherit
|
||||
from SerializablePreprocessor.
|
||||
|
||||
Args:
|
||||
version: Version number for this preprocessor's serialization format
|
||||
identifier: Stable identifier for serialization. This identifier will be used
|
||||
in serialized data. Using an explicit identifier allows classes to be
|
||||
renamed without breaking compatibility with existing serialized data.
|
||||
|
||||
Returns:
|
||||
A decorator function that registers the class and returns it unchanged.
|
||||
|
||||
Raises:
|
||||
TypeError: If the decorated class does not inherit from SerializablePreprocessor
|
||||
|
||||
Note:
|
||||
If a class with the same identifier is already registered, logs a debug message
|
||||
and overwrites the previous registration.
|
||||
|
||||
Examples:
|
||||
@SerializablePreprocessor(version=1, identifier="my_preprocessor_v1")
|
||||
class MyPreprocessor(SerializablePreprocessor):
|
||||
pass
|
||||
"""
|
||||
|
||||
def decorator(cls):
|
||||
import logging
|
||||
|
||||
from ray.data.preprocessor import SerializablePreprocessorBase
|
||||
|
||||
# Verify that the class inherits from SerializablePreprocessor
|
||||
if not issubclass(cls, SerializablePreprocessorBase):
|
||||
raise TypeError(
|
||||
f"Class {cls.__module__}.{cls.__qualname__} must inherit from "
|
||||
f"SerializablePreprocessor to use @SerializablePreprocessor decorator."
|
||||
)
|
||||
|
||||
cls.set_version(version)
|
||||
cls.set_preprocessor_class_id(identifier)
|
||||
|
||||
# Check for collisions and log debug message
|
||||
if identifier in _PREPROCESSOR_REGISTRY:
|
||||
existing = _PREPROCESSOR_REGISTRY[identifier]
|
||||
if existing != cls:
|
||||
logging.debug(
|
||||
f"Preprocessor id collision: '{identifier}' was already registered "
|
||||
f"by {existing.__module__}.{existing.__qualname__}. "
|
||||
f"Overwriting with {cls.__module__}.{cls.__qualname__}."
|
||||
)
|
||||
|
||||
_PREPROCESSOR_REGISTRY[identifier] = cls
|
||||
return cls
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def _lookup_class(serialization_id: str):
|
||||
"""Look up a preprocessor class by its serialization ID.
|
||||
|
||||
Args:
|
||||
serialization_id: The serialization ID of the preprocessor (either explicit or class name)
|
||||
|
||||
Returns:
|
||||
The registered preprocessor class
|
||||
|
||||
Raises:
|
||||
UnknownPreprocessorError: If the serialization ID is not registered
|
||||
"""
|
||||
if serialization_id not in _PREPROCESSOR_REGISTRY:
|
||||
raise UnknownPreprocessorError(serialization_id)
|
||||
return _PREPROCESSOR_REGISTRY[serialization_id]
|
||||
Reference in New Issue
Block a user