chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,215 @@
|
||||
import warnings
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
List,
|
||||
Protocol,
|
||||
Sequence,
|
||||
Union,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ray.data.constants import TENSOR_COLUMN_NAME
|
||||
from ray.util import PublicAPI
|
||||
from ray.util.annotations import DeveloperAPI
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pandas.core.dtypes.generic import ABCSeries
|
||||
|
||||
|
||||
@DeveloperAPI(stability="beta")
|
||||
class ArrayLike(Protocol):
|
||||
"""Protocol matching ndarray-like objects (like torch.Tensor)"""
|
||||
|
||||
def __array__(self):
|
||||
...
|
||||
|
||||
def __len__(self):
|
||||
...
|
||||
|
||||
|
||||
@DeveloperAPI(stability="beta")
|
||||
def is_ndarray_like(value: ArrayLike) -> bool:
|
||||
"""Checks whether objects are ndarray-like (for ex, torch.Tensor)
|
||||
but NOT and ndarray itself"""
|
||||
return (
|
||||
hasattr(value, "__array__")
|
||||
and hasattr(value, "__len__")
|
||||
and not isinstance(value, np.ndarray)
|
||||
)
|
||||
|
||||
|
||||
def _is_arrow_array(value: ArrayLike) -> bool:
|
||||
import pyarrow as pa
|
||||
|
||||
return isinstance(value, (pa.Array, pa.ChunkedArray))
|
||||
|
||||
|
||||
def _should_convert_to_tensor(
|
||||
column_values: Union[List[Any], np.ndarray, ArrayLike], column_name: str
|
||||
) -> bool:
|
||||
assert len(column_values) > 0
|
||||
|
||||
# We convert passed in column values into a tensor representation (involving
|
||||
# Arrow/Pandas extension types) in either of the following cases:
|
||||
return (
|
||||
# - Column name is `TENSOR_COLUMN_NAME` (for compatibility)
|
||||
column_name == TENSOR_COLUMN_NAME
|
||||
# - Provided column values are already represented by a Numpy tensor (ie
|
||||
# ndarray with ndim > 1)
|
||||
or _is_ndarray_tensor(column_values)
|
||||
# - Provided collection is already implementing Ndarray protocol like
|
||||
# `torch.Tensor`, `pd.Series`, etc (but *excluding* `pyarrow.Array`,
|
||||
# `pyarrow.ChunkedArray`)
|
||||
or _is_ndarray_like_not_pyarrow_array(column_values)
|
||||
# - Provided column values is a list of a) ndarrays or b) ndarray-like objects
|
||||
# (excluding Pyarrow arrays). This is done for compatibility with previous
|
||||
# existing behavior where all column values were blindly converted to Numpy
|
||||
# leading to list of ndarrays being converted a tensor):
|
||||
or isinstance(column_values[0], np.ndarray)
|
||||
or _is_ndarray_like_not_pyarrow_array(column_values[0])
|
||||
)
|
||||
|
||||
|
||||
def _is_ndarray_like_not_pyarrow_array(column_values):
|
||||
return is_ndarray_like(column_values) and not _is_arrow_array(column_values)
|
||||
|
||||
|
||||
def _is_ndarray_tensor(t: Any) -> bool:
|
||||
"""Return whether provided ndarray is a tensor (ie ndim > 1).
|
||||
|
||||
NOTE: Tensor is defined as a NumPy array such that `len(arr.shape) > 1`
|
||||
"""
|
||||
|
||||
if not isinstance(t, np.ndarray):
|
||||
return False
|
||||
|
||||
# Case of uniform-shaped (ie non-ragged) tensor
|
||||
if t.ndim > 1:
|
||||
return True
|
||||
|
||||
# Case of ragged tensor (as produced by `create_ragged_ndarray` utility)
|
||||
elif t.dtype.type is np.object_ and len(t) > 0 and isinstance(t[0], np.ndarray):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _is_ndarray_variable_shaped_tensor(arr: np.ndarray) -> bool:
|
||||
"""Return whether the provided NumPy ndarray is comprised of variable-shaped
|
||||
tensors.
|
||||
|
||||
NOTE: This is an O(rows) check.
|
||||
"""
|
||||
if arr.dtype.type is not np.object_:
|
||||
return False
|
||||
if len(arr) == 0:
|
||||
return False
|
||||
if not isinstance(arr[0], np.ndarray):
|
||||
return False
|
||||
shape = arr[0].shape
|
||||
for a in arr[1:]:
|
||||
if not isinstance(a, np.ndarray):
|
||||
return False
|
||||
if a.shape != shape:
|
||||
return True
|
||||
|
||||
# All shapes are identical
|
||||
return False
|
||||
|
||||
|
||||
def _create_possibly_ragged_ndarray(
|
||||
values: Union[np.ndarray, "ABCSeries", Sequence[Any]]
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Create a possibly ragged ndarray.
|
||||
Using the np.array() constructor will fail to construct a ragged ndarray that has a
|
||||
uniform first dimension (e.g. uniform channel dimension in imagery). This function
|
||||
catches this failure and tries a create-and-fill method to construct the ragged
|
||||
ndarray.
|
||||
"""
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
# For NumPy < 1.24, constructing a ragged ndarray directly via
|
||||
# `np.array(...)` without the `dtype=object` parameter will raise a
|
||||
# VisibleDeprecationWarning which we suppress.
|
||||
# More details: https://stackoverflow.com/q/63097829
|
||||
if np.lib.NumpyVersion(np.__version__) >= "2.0.0":
|
||||
copy_if_needed = None
|
||||
warning_type = np.exceptions.VisibleDeprecationWarning
|
||||
else:
|
||||
copy_if_needed = False
|
||||
warning_type = np.VisibleDeprecationWarning
|
||||
|
||||
warnings.simplefilter("ignore", category=warning_type)
|
||||
arr = np.array(values, copy=copy_if_needed)
|
||||
return arr
|
||||
except ValueError as e:
|
||||
# Constructing a ragged ndarray directly via `np.array(...)`
|
||||
# without the `dtype=object` parameter will raise a ValueError.
|
||||
# For NumPy < 1.24, the message is of the form:
|
||||
# "could not broadcast input array from shape..."
|
||||
# For NumPy >= 1.24, the message is of the form:
|
||||
# "The requested array has an inhomogeneous shape..."
|
||||
# More details: https://github.com/numpy/numpy/pull/22004
|
||||
error_str = str(e)
|
||||
if (
|
||||
"could not broadcast input array from shape" in error_str
|
||||
or "The requested array has an inhomogeneous shape" in error_str
|
||||
):
|
||||
# Fall back to strictly creating a ragged ndarray.
|
||||
return create_ragged_ndarray(values)
|
||||
else:
|
||||
# Re-raise original error if the failure wasn't a broadcast error.
|
||||
raise e from None
|
||||
|
||||
|
||||
@PublicAPI(stability="alpha")
|
||||
def create_ragged_ndarray(values: Sequence[Any]) -> np.ndarray:
|
||||
"""Create an array that contains arrays of different length
|
||||
|
||||
If you're working with variable-length arrays like images, use this function to
|
||||
create ragged arrays instead of ``np.array``.
|
||||
|
||||
.. note::
|
||||
``np.array`` fails to construct ragged arrays if the input arrays have a uniform
|
||||
first dimension:
|
||||
|
||||
.. testsetup::
|
||||
|
||||
import numpy as np
|
||||
from ray.data._internal.tensor_extensions.utils import create_ragged_ndarray
|
||||
|
||||
.. doctest::
|
||||
|
||||
>>> values = [np.zeros((3, 1)), np.zeros((3, 2))]
|
||||
>>> np.array(values, dtype=object)
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
ValueError: could not broadcast input array from shape (3,1) into shape (3,)
|
||||
>>> create_ragged_ndarray(values)
|
||||
array([array([[0.],
|
||||
[0.],
|
||||
[0.]]), array([[0., 0.],
|
||||
[0., 0.],
|
||||
[0., 0.]])], dtype=object)
|
||||
|
||||
Or if you're creating a ragged array from a single array:
|
||||
|
||||
.. doctest::
|
||||
|
||||
>>> values = [np.zeros((3, 1))]
|
||||
>>> np.array(values, dtype=object)[0].dtype
|
||||
dtype('O')
|
||||
>>> create_ragged_ndarray(values)[0].dtype
|
||||
dtype('float64')
|
||||
|
||||
``create_ragged_ndarray`` avoids the limitations of ``np.array`` by creating an
|
||||
empty array and filling it with pointers to the variable-length arrays.
|
||||
""" # noqa: E501
|
||||
# Create an empty object-dtyped 1D array.
|
||||
arr = np.empty(len(values), dtype=object)
|
||||
# Try to fill the 1D array of pointers with the (ragged) tensors.
|
||||
arr[:] = list(values)
|
||||
return arr
|
||||
Reference in New Issue
Block a user