chore: import upstream snapshot with attribution

easygraph/datapipe/__init__.py

@@ -0,0 +1,29 @@
+try:
+    from .common import compose_pipes
+    from .common import to_bool_tensor
+    from .common import to_long_tensor
+    from .common import to_tensor
+    from .normalize import min_max_scaler
+    from .normalize import norm_ft
+except:
+    print(
+        "Warning raise in module:datapipe. Please install Pytorch before you use"
+        " functions related to nueral network"
+    )
+
+from .loader import load_from_json
+from .loader import load_from_pickle
+from .loader import load_from_txt
+
+
+# __all__ = [
+#     "compose_pipes",
+#     "norm_ft",
+#     "min_max_scaler",
+#     "to_tensor",
+#     "to_bool_tensor",
+#     "to_long_tensor",
+#     "load_from_pickle",
+#     "load_from_json",
+#     "load_from_txt",
+# ]

easygraph/datapipe/common.py

@@ -0,0 +1,106 @@
+from typing import Any
+from typing import Callable
+from typing import List
+from typing import Union
+
+import numpy as np
+import scipy.sparse
+import torch
+
+
+def to_tensor(
+    X: Union[list, np.ndarray, torch.Tensor, scipy.sparse.csr_matrix]
+) -> torch.Tensor:
+    r"""Convert ``List``, ``numpy.ndarray``, ``scipy.sparse.csr_matrix`` to ``torch.Tensor``.
+
+    Args:
+        ``X`` (``Union[List, np.ndarray, torch.Tensor, scipy.sparse.csr_matrix]``): Input.
+
+    Examples:
+        >>> import easygraph.datapipe as dd
+        >>> X = [[0.1, 0.2, 0.5],
+                 [0.5, 0.2, 0.3],
+                 [0.3, 0.2, 0]]
+        >>> dd.to_tensor(X)
+        tensor([[0.1000, 0.2000, 0.5000],
+                [0.5000, 0.2000, 0.3000],
+                [0.3000, 0.2000, 0.0000]])
+    """
+    if isinstance(X, list):
+        X = torch.tensor(X)
+    elif isinstance(X, scipy.sparse.csr_matrix):
+        X = X.todense()
+        X = torch.tensor(X)
+    elif isinstance(X, scipy.sparse.coo_matrix):
+        X = X.todense()
+        X = torch.tensor(X)
+    elif isinstance(X, np.ndarray):
+        X = torch.tensor(X)
+    else:
+        X = torch.tensor(X)
+    return X.float()
+
+
+def to_bool_tensor(X: Union[List, np.ndarray, torch.Tensor]) -> torch.BoolTensor:
+    r"""Convert ``List``, ``numpy.ndarray``, ``torch.Tensor`` to ``torch.BoolTensor``.
+
+    Args:
+        ``X`` (``Union[List, np.ndarray, torch.Tensor]``): Input.
+
+    Examples:
+        >>> import easygraph.datapipe as dd
+        >>> X = [[0.1, 0.2, 0.5],
+                 [0.5, 0.2, 0.3],
+                 [0.3, 0.2, 0]]
+        >>> dd.to_bool_tensor(X)
+        tensor([[ True,  True,  True],
+                [ True,  True,  True],
+                [ True,  True, False]])
+    """
+    if isinstance(X, list):
+        X = torch.tensor(X)
+    elif isinstance(X, np.ndarray):
+        X = torch.tensor(X)
+    else:
+        X = torch.tensor(X)
+    return X.bool()
+
+
+def to_long_tensor(X: Union[List, np.ndarray, torch.Tensor]) -> torch.LongTensor:
+    r"""Convert ``List``, ``numpy.ndarray``, ``torch.Tensor`` to ``torch.LongTensor``.
+
+    Args:
+        ``X`` (``Union[List, np.ndarray, torch.Tensor]``): Input.
+
+    Examples:
+        >>> import easygraph.datapipe as dd
+        >>> X = [[1, 2, 5],
+                 [5, 2, 3],
+                 [3, 2, 0]]
+        >>> dd.to_long_tensor(X)
+        tensor([[1, 2, 5],
+                [5, 2, 3],
+                [3, 2, 0]])
+    """
+    if isinstance(X, list):
+        X = torch.tensor(X)
+    elif isinstance(X, np.ndarray):
+        X = torch.tensor(X)
+    else:
+        X = torch.tensor(X)
+    return X.long()
+
+
+def compose_pipes(*pipes: Callable) -> Callable:
+    r"""Compose datapipe functions.
+
+    Args:
+        ``pipes`` (``Callable``): Datapipe functions to compose.
+    """
+
+    def composed_pipes(X: Any) -> torch.Tensor:
+        for pipe in pipes:
+            X = pipe(X)
+        return X
+
+    return composed_pipes

easygraph/datapipe/loader.py

@@ -0,0 +1,90 @@
+import json
+import pickle as pkl
+import re
+
+from pathlib import Path
+from typing import Callable
+from typing import List
+from typing import Optional
+from typing import Union
+
+
+def load_from_pickle(
+    file_path: Path, keys: Optional[Union[str, List[str]]] = None, **kwargs
+):
+    r"""Load data from a pickle file.
+
+    Args:
+        ``file_path`` (``Path``): The local path of the file.
+        ``keys`` (``Union[str, List[str]]``, optional): The keys of the data. Defaults to ``None``.
+    """
+    if isinstance(file_path, list):
+        raise ValueError("This function only support loading data from a single file.")
+    with open(file_path, "rb") as f:
+        data = pkl.load(f, **kwargs)
+    if keys is None:
+        return data
+    elif isinstance(keys, str):
+        return data[keys]
+    else:
+        return {key: data[key] for key in keys}
+
+
+def load_from_json(file_path: Path, **kwargs):
+    r"""Load data from a json file.
+
+    Args:
+        ``file_path`` (``Path``): The local path of the file.
+    """
+    with open(file_path, "r") as f:
+        data = json.load(f, **kwargs)
+    return data
+
+
+def load_from_txt(
+    file_path: Path,
+    dtype: Union[str, Callable],
+    sep: str = ",| |\t",
+    ignore_header: int = 0,
+):
+    r"""Load data from a txt file.
+
+    .. note::
+        The separator is a regular expression of ``re`` module. Multiple separators can be separated by ``|``. More details can refer to `re.split <https://docs.python.org/3/library/re.html#re.split>`_.
+
+    Args:
+        ``file_path`` (``Path``): The local path of the file.
+        ``dtype`` (``Union[str, Callable]``): The data type of the data can be either a string or a callable function.
+        ``sep`` (``str``, optional): The separator of each line in the file. Defaults to ``",| |\t"``.
+        ``ignore_header`` (``int``, optional): The number of lines to ignore in the header of the file. Defaults to ``0``.
+    """
+    cast_fun = ret_cast_fun(dtype)
+    file_path = Path(file_path)
+    assert file_path.exists(), f"{file_path} does not exist."
+    data = []
+    with open(file_path, "r") as f:
+        for _ in range(ignore_header):
+            f.readline()
+        data = [
+            list(map(cast_fun, re.split(sep, line.strip()))) for line in f.readlines()
+        ]
+    return data
+
+
+def ret_cast_fun(dtype: Union[str, Callable]):
+    r"""Return the cast function of the data type. The supported data types are: ``int``, ``float``, ``str``.
+
+    Args:
+        ``dtype`` (``Union[str, Callable]``): The data type of the data can be either a string or a callable function.
+    """
+    if isinstance(dtype, str):
+        if dtype == "int":
+            return int
+        elif dtype == "float":
+            return float
+        elif dtype == "str":
+            return str
+        else:
+            raise ValueError("dtype must be one of 'int', 'float', 'str'.")
+    else:
+        return dtype

easygraph/datapipe/normalize.py

@@ -0,0 +1,74 @@
+from typing import Optional
+from typing import Union
+
+import torch
+
+
+def norm_ft(X: torch.Tensor, ord: Optional[Union[int, float]] = None) -> torch.Tensor:
+    r"""Normalize the input feature matrix with specified ``ord`` refer to pytorch's `torch.linalg.norm <https://pytorch.org/docs/stable/generated/torch.linalg.norm.html#torch.linalg.norm>`_ function.
+
+    .. note::
+        The input feature matrix is expected to be a 1D vector or a 2D tensor with shape (num_samples, num_features).
+
+    Args:
+        ``X`` (``torch.Tensor``): The input feature.
+        ``ord`` (``Union[int, float]``, optional): The order of the norm can be either an ``int``, ``float``. If ``ord`` is ``None``, the norm is computed with the 2-norm. Defaults to ``None``.
+
+    Examples:
+        >>> import easygraph.datapipe as dd
+        >>> import torch
+        >>> X = torch.tensor([
+                    [0.1, 0.2, 0.5],
+                    [0.5, 0.2, 0.3],
+                    [0.3, 0.2, 0]
+                ])
+        >>> dd.norm_ft(X)
+        tensor([[0.1826, 0.3651, 0.9129],
+                [0.8111, 0.3244, 0.4867],
+                [0.8321, 0.5547, 0.0000]])
+    """
+    if X.dim() == 1:
+        X_norm = 1 / torch.linalg.norm(X, ord=ord)
+        X_norm[torch.isinf(X_norm)] = 0
+        return X * X_norm
+    elif X.dim() == 2:
+        X_norm = 1 / torch.linalg.norm(X, ord=ord, dim=1, keepdim=True)
+        X_norm[torch.isinf(X_norm)] = 0
+        return X * X_norm
+    else:
+        raise ValueError(
+            "The input feature matrix is expected to be a 1D verter or a 2D tensor with"
+            " shape (num_samples, num_features)."
+        )
+
+
+def min_max_scaler(X: torch.Tensor, ft_min: float, ft_max: float) -> torch.Tensor:
+    r"""Normalize the input feature matrix with min-max scaling.
+
+    Args:
+        ``X`` (``torch.Tensor``): The input feature.
+        ``ft_min`` (``float``): The minimum value of the output feature.
+        ``ft_max`` (``float``): The maximum value of the output feature.
+
+    Examples:
+        >>> import easygraph.datapipe as dd
+        >>> import torch
+        >>> X = torch.tensor([
+                    [0.1, 0.2, 0.5],
+                    [0.5, 0.2, 0.3],
+                    [0.3, 0.2, 0.0]
+                ])
+        >>> dd.min_max_scaler(X, -1, 1)
+        tensor([[-0.6000, -0.2000,  1.0000],
+                [ 1.0000, -0.2000,  0.2000],
+                [ 0.2000, -0.2000, -1.0000]])
+    """
+    assert (
+        ft_min < ft_max
+    ), "The minimum value of the feature should be less than the maximum value."
+    X_min, X_max = X.min().item(), X.max().item()
+    X_range = X_max - X_min
+    scale_ = (ft_max - ft_min) / X_range
+    min_ = ft_min - X_min * scale_
+    X = X * scale_ + min_
+    return X