wehub/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/paddle/sparse/__init__.py
+95
View File
@@ -0,0 +1,95 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import nn # noqa: F401
from .binary import (
add,
divide,
is_same_shape,
mask_as,
masked_matmul,
matmul,
multiply,
mv,
subtract,
)
from .creation import sparse_coo_tensor, sparse_csr_tensor
from .multiary import addmm
from .unary import (
abs,
asin,
asinh,
atan,
atanh,
cast,
coalesce,
deg2rad,
expm1,
isnan,
log1p,
neg,
pca_lowrank,
pow,
rad2deg,
reshape,
sin,
sinh,
slice,
sqrt,
square,
sum,
tan,
tanh,
transpose,
)
__all__ = [
'sparse_coo_tensor',
'sparse_csr_tensor',
'sin',
'tan',
'asin',
'atan',
'sinh',
'tanh',
'asinh',
'atanh',
'sqrt',
'square',
'log1p',
'abs',
'pow',
'pca_lowrank',
'cast',
'neg',
'deg2rad',
'rad2deg',
'expm1',
'mv',
'matmul',
'mask_as',
'masked_matmul',
'addmm',
'add',
'subtract',
'transpose',
'sum',
'multiply',
'divide',
'coalesce',
'is_same_shape',
'reshape',
'isnan',
'slice',
]
python/paddle/sparse/binary.py
+548
View File
@@ -0,0 +1,548 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from paddle import _C_ops
from paddle.base.framework import (
core,
dygraph_only,
in_dygraph_mode,
in_dynamic_or_pir_mode,
in_pir_mode,
)
from paddle.base.layer_helper import LayerHelper
if TYPE_CHECKING:
from paddle import Tensor
__all__ = []
_int_dtype_ = [
core.VarDesc.VarType.UINT8,
core.VarDesc.VarType.INT8,
core.VarDesc.VarType.INT16,
core.VarDesc.VarType.INT32,
core.VarDesc.VarType.INT64,
core.VarDesc.VarType.BOOL,
core.DataType.UINT8,
core.DataType.INT8,
core.DataType.INT16,
core.DataType.INT32,
core.DataType.INT64,
core.DataType.BOOL,
]
_pir_int_dtype_ = {
core.DataType.UINT8: 1,
core.DataType.INT8: 1,
core.DataType.INT16: 2,
core.DataType.INT32: 4,
core.DataType.INT64: 8,
core.DataType.BOOL: 1,
}
def matmul(x: Tensor, y: Tensor, name: str | None = None) -> Tensor:
"""
Applies matrix multiplication of two Tensors.
The supported input/output Tensor type are as follows:
Note:
x[SparseCsrTensor] @ y[SparseCsrTensor] -> out[SparseCsrTensor]
x[SparseCsrTensor] @ y[DenseTensor] -> out[DenseTensor]
x[SparseCooTensor] @ y[SparseCooTensor] -> out[SparseCooTensor]
x[SparseCooTensor] @ y[DenseTensor] -> out[DenseTensor]
It supports backward propagation.
Dimensions `x` and `y` must be >= 2D. Automatic broadcasting of Tensor is not supported.
the shape of `x` should be `[*, M, K]` , and the shape of `y` should be `[*, K, N]` , where `*`
is zero or more batch dimensions.
Args:
x (SparseTensor): The input tensor. It can be SparseCooTensor/SparseCsrTensor. The data type can be float32 or float64.
y (SparseTensor|DenseTensor): The input tensor. It can be SparseCooTensor/SparseCsrTensor/DenseTensor. The data type can be float32 or float64.
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
SparseTensor|DenseTensor: Determined by `x` and `y` .
Examples:
.. code-block:: pycon
>>> # doctest: +REQUIRES(env:GPU)
>>> import paddle
>>> paddle.device.set_device('gpu')
>>> # csr @ dense -> dense
>>> crows = [0, 1, 2, 3]
>>> cols = [1, 2, 0]
>>> values = [1.0, 2.0, 3.0]
>>> csr = paddle.sparse.sparse_csr_tensor(crows, cols, values, [3, 3])
>>> print(csr)
Tensor(shape=[3, 3], dtype=paddle.float32, place=Place(gpu:0), stop_gradient=True,
crows=[0, 1, 2, 3],
cols=[1, 2, 0],
values=[1., 2., 3.])
>>> dense = paddle.ones([3, 2])
>>> out = paddle.sparse.matmul(csr, dense)
>>> print(out)
Tensor(shape=[3, 2], dtype=float32, place=Place(gpu:0), stop_gradient=True,
[[1., 1.],
[2., 2.],
[3., 3.]])
>>> # coo @ dense -> dense
>>> indices = [[0, 1, 2], [1, 2, 0]]
>>> values = [1.0, 2.0, 3.0]
>>> coo = paddle.sparse.sparse_coo_tensor(indices, values, [3, 3])
>>> print(coo)
Tensor(shape=[3, 3], dtype=paddle.float32, place=Place(gpu:0), stop_gradient=True,
indices=[[0, 1, 2],
[1, 2, 0]],
values=[1., 2., 3.])
>>> dense = paddle.ones([3, 2])
>>> out = paddle.sparse.matmul(coo, dense)
>>> print(out)
Tensor(shape=[3, 2], dtype=float32, place=Place(gpu:0), stop_gradient=True,
[[1., 1.],
[2., 2.],
[3., 3.]])
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
return _C_ops.sparse_matmul(x, y)
def masked_matmul(
x: Tensor, y: Tensor, mask: Tensor, name: str | None = None
) -> Tensor:
"""
Applies matrix multiplication of two Dense Tensors.
The supported input/output Tensor layout are as follows:
Note:
x[DenseTensor] @ y[DenseTensor] * mask[SparseCooTensor] -> out[SparseCooTensor]
x[DenseTensor] @ y[DenseTensor] * mask[SparseCsrTensor] -> out[SparseCsrTensor]
It supports backward propagation.
Dimensions `x` and `y` must be >= 2D. Automatic broadcasting of Tensor is not supported.
the shape of `x` should be `[*, M, K]` , and the shape of `y` should be `[*, K, N]` , and the shape of `mask` should be `[*, M, N]` ,
where `*` is zero or more batch dimensions.
Args:
x (DenseTensor): The input tensor. It is DenseTensor. The data type can be float32 or float64.
y (DenseTensor): The input tensor. It is DenseTensor. The data type can be float32 or float64.
mask (SparseTensor): The mask tensor, which can be SparseCooTensor/SparseCsrTensor. It specify sparse coordinates. The data type can be float32 or float64.
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
SparseTensor: SparseCooTensor or SparseCsrTensor, which is same with `mask` .
Examples:
.. code-block:: pycon
>>> # doctest: +REQUIRES(env:GPU)
>>> import paddle
>>> paddle.device.set_device('gpu')
>>> paddle.seed(100)
>>> # dense @ dense * csr_mask -> csr
>>> crows = [0, 2, 3, 5]
>>> cols = [1, 3, 2, 0, 1]
>>> values = [1.0, 2.0, 3.0, 4.0, 5.0]
>>> dense_shape = [3, 4]
>>> mask = paddle.sparse.sparse_csr_tensor(crows, cols, values, dense_shape)
>>> print(mask)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(gpu:0), stop_gradient=True,
crows=[0, 2, 3, 5],
cols=[1, 3, 2, 0, 1],
values=[1., 2., 3., 4., 5.])
>>> x = paddle.rand([3, 5])
>>> y = paddle.rand([5, 4])
>>> out = paddle.sparse.masked_matmul(x, y, mask)
>>> print(out)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(gpu:0), stop_gradient=True,
crows=[0, 2, 3, 5],
cols=[1, 3, 2, 0, 1],
values=[0.98986477, 0.97800624, 1.14591956, 0.68561077, 0.94714981])
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
return _C_ops.sparse_masked_matmul(x, y, mask)
def mv(x: Tensor, vec: Tensor, name: str | None = None) -> Tensor:
"""
Applies matrix-vector product of Sparse Matrix 'x' and Dense vector 'vec' .
The supported input/output Tensor layout are as follows:
Note:
x[SparseCsrTensor] @ vec[DenseTensor] -> out[DenseTensor]
x[SparseCooTensor] @ vec[DenseTensor] -> out[DenseTensor]
It supports backward propagation.
The shape of `x` should be `[M, N]` , and the shape of `vec` should be `[N]` ,
and the shape of `out` will be `[M]` .
Args:
x (SparseTensor): The input 2D tensor. It must be SparseCooTensor/SparseCsrTensor. The data type can be float32 or float64.
vec (DenseTensor): The input 1D tensor. It must be DenseTensor vector. The data type can be float32 or float64.
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
DenseTensor: 1D DenseTensor whose dtype is same with input.
Examples:
.. code-block:: pycon
>>> # doctest: +REQUIRES(env:GPU)
>>> import paddle
>>> paddle.device.set_device('gpu')
>>> paddle.seed(100)
>>> # csr @ dense -> dense
>>> crows = [0, 2, 3, 5]
>>> cols = [1, 3, 2, 0, 1]
>>> values = [1.0, 2.0, 3.0, 4.0, 5.0]
>>> dense_shape = [3, 4]
>>> csr = paddle.sparse.sparse_csr_tensor(crows, cols, values, dense_shape)
>>> print(csr)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(gpu:0), stop_gradient=True,
crows=[0, 2, 3, 5],
cols=[1, 3, 2, 0, 1],
values=[1., 2., 3., 4., 5.])
>>> vec = paddle.randn([4])
>>> out = paddle.sparse.mv(csr, vec)
>>> print(out)
Tensor(shape=[3], dtype=float32, place=Place(gpu:0), stop_gradient=True,
[-3.85499096, -2.42975140, -1.75087738])
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
return _C_ops.sparse_mv(x, vec)
def add(x: Tensor, y: Tensor, name: str | None = None) -> Tensor:
"""
Add two sparse tensors element-wise. Input x and y's shape should be identical and have same sparse
type（SparseCooTensor or SparseCsrTensor）.If input is SparseCooTensor, x and y's sparse_dim should be identical.
The equation is:
.. math::
out = x + y
Args:
x (Tensor): the input tensor, it's data type should be float32, float64, int32, int64, complex64, complex128.
y (Tensor): the input tensor, it's data type should be float32, float64, int32, int64, complex64, complex128.
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
Tensor: the result tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.device.set_device("cpu")
>>> x = paddle.to_tensor([[0, -1, 0, 2], [0, 0, -3, 0], [4, 5, 0, 0]], 'float32')
>>> y = paddle.to_tensor([[0, 0, 0, -2], [0, 2, -3, 0], [2, 3, 4, 8]], 'float32')
>>> sparse_x = x.to_sparse_csr()
>>> sparse_y = y.to_sparse_csr()
>>> sparse_z = paddle.sparse.add(sparse_x, sparse_y)
>>> print(sparse_z.to_dense())
Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
[[ 0., -1., 0., 0.],
[ 0., 2., -6., 0.],
[ 6., 8., 4., 8.]])
"""
if in_dynamic_or_pir_mode():
return _C_ops.sparse_add(x, y)
else:
op_type = 'sparse_add'
inputs = {'x': x, 'y': y}
helper = LayerHelper(op_type)
out = helper.create_sparse_variable_for_type_inference(x.dtype)
helper.append_op(
type=op_type, inputs=inputs, outputs={'out': out}, attrs={}
)
return out
def subtract(x: Tensor, y: Tensor, name: str | None = None) -> Tensor:
"""
Subtract two sparse tensors element-wise. Input x and y's shape should be identical and have same sparse
type（SparseCooTensor or SparseCsrTensor）.If input is SparseCooTensor, x and y's sparse_dim should be identical.
The equation is:
.. math::
out = x - y
Args:
x (Tensor): the input tensor, it's data type should be float32, float64, int32, int64, complex64, complex128.
y (Tensor): the input tensor, it's data type should be float32, float64, int32, int64, complex64, complex128.
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
Tensor: the result tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.device.set_device("cpu")
>>> x = paddle.to_tensor([[0, -1, 0, 2], [0, 0, -3, 0], [4, 5, 0, 0]], 'float32')
>>> y = paddle.to_tensor([[0, 0, 0, -2], [0, 2, -3, 0], [2, 3, 4, 8]], 'float32')
>>> sparse_x = x.to_sparse_csr()
>>> sparse_y = y.to_sparse_csr()
>>> sparse_z = paddle.sparse.subtract(sparse_x, sparse_y)
>>> print(sparse_z.to_dense())
Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
[[ 0., -1., 0., 4.],
[ 0., -2., 0., 0.],
[ 2., 2., -4., -8.]])
"""
if in_dygraph_mode():
return _C_ops.sparse_subtract(x, y)
elif in_pir_mode():
return _C_ops.sparse_subtract(x, y)
else:
raise RuntimeError(
"We currently only support dynamic graph mode or the new IR mode."
)
def multiply(x: Tensor, y: Tensor, name: str | None = None) -> Tensor:
"""
Multiply two sparse tensors element-wise. Input x and y's shape should be identical and have same sparse
type（SparseCooTensor or SparseCsrTensor）.If input is SparseCooTensor, x and y's sparse_dim should be identical.
The equation is:
.. math::
out = x * y
Args:
x (Tensor): the input tensor, it's data type should be float32, float64, int32, int64, complex64, complex128.
y (Tensor): the input tensor, it's data type should be float32, float64, int32, int64, complex64, complex128.
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
Tensor: the result tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.device.set_device("cpu")
>>> x = paddle.to_tensor([[0, -1, 0, 2], [0, 0, -3, 0], [4, 5, 0, 0]], 'float32')
>>> y = paddle.to_tensor([[0, 0, 0, -2], [0, 2, -3, 0], [2, 3, 4, 8]], 'float32')
>>> sparse_x = x.to_sparse_csr()
>>> sparse_y = y.to_sparse_csr()
>>> sparse_z = paddle.sparse.multiply(sparse_x, sparse_y)
>>> print(sparse_z.to_dense())
Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
[[ 0., -0., 0., -4.],
[ 0., 0., 9., 0.],
[ 8., 15., 0., 0.]])
"""
if isinstance(y, (int, float)):
return _C_ops.sparse_scale(x, float(y), 0.0, True)
else:
if in_dygraph_mode():
return _C_ops.sparse_multiply(x, y)
elif in_pir_mode():
return _C_ops.sparse_multiply(x, y)
else:
raise RuntimeError(
"We currently only support dynamic graph mode or the new IR mode."
)
def divide(x: Tensor, y: Tensor, name: str | None = None) -> Tensor:
"""
Divide two sparse tensors element-wise. Input x and y's shape should be identical and have same sparse
type（SparseCooTensor or SparseCsrTensor）.If input is SparseCooTensor, x and y's sparse_dim should be identical.
The equation is:
.. math::
out = x / y
Args:
x (Tensor): the input tensor, it's data type should be float32, float64, int32, int64, complex64, complex128.
y (Tensor): the input tensor, it's data type should be float32, float64, int32, int64, complex64, complex128.
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
Tensor: the result tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.device.set_device("cpu")
>>> x = paddle.to_tensor([[0, -1, 0, 2], [0, 0, -3, 0], [4, 5, 0, 0]], 'float32')
>>> y = paddle.to_tensor([[0, 0, 0, -2], [0, 2, -3, 0], [2, 3, 4, 8]], 'float32')
>>> sparse_x = x.to_sparse_csr()
>>> sparse_y = y.to_sparse_csr()
>>> sparse_z = paddle.sparse.divide(sparse_x, sparse_y)
>>> print(sparse_z.to_dense())
Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
[[ nan , -inf. , nan , -1. ],
[ nan , 0. , 1. , nan ],
[ 2. , 1.66666663, 0. , 0. ]])
"""
if isinstance(y, (int, float)):
return _C_ops.sparse_divide_scalar(x, float(y))
else:
if in_dygraph_mode():
return _C_ops.sparse_divide(x, y)
elif in_pir_mode():
return _C_ops.sparse_divide(x, y)
else:
raise RuntimeError(
"We currently only support dynamic graph mode or the new IR mode."
)
def is_same_shape(x: Tensor, y: Tensor) -> bool:
"""
Return the results of shape comparison between two Tensors, check whether x.shape equal to y.shape.
Any two type Tensor among DenseTensor/SparseCooTensor/SparseCsrTensor are supported.
Args:
x (Tensor): The input tensor. It can be DenseTensor/SparseCooTensor/SparseCsrTensor.
y (Tensor): The input tensor. It can be DenseTensor/SparseCooTensor/SparseCsrTensor.
Returns:
bool: True for same shape and False for different shape.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.rand([2, 3, 8])
>>> y = paddle.rand([2, 3, 8])
>>> y = y.to_sparse_csr()
>>> z = paddle.rand([2, 5])
>>> paddle.sparse.is_same_shape(x, y)
True
>>> paddle.sparse.is_same_shape(x, z)
False
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
return x.is_same_shape(y)
@dygraph_only
def mask_as(x: Tensor, mask: Tensor, name: str | None = None) -> Tensor:
r"""
Filter the input dense tensor `x` using the `indices` of the sparse matrix `mask`,
which in turn generates a sparse matrix of the corresponding format.
The input `x` and `mask` must have the same shape, and the sparse tensor returned has the same indices as `mask`
even `zero` values exist in the corresponding indices.
Args:
x (Tensor): The input tensor. It should be a DenseTensor.
The data type can be float32, float64, int32, int64, complex64, complex128, int8, int16, float16.
mask (Tensor): The input tensor. It can be SparseCooTensor or SparseCsrTensor.
It should be 2D or 3D when the mask is SparseCsrTensor.
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
Tensor: A sparse tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.set_device('cpu')
>>> # csr sparse tensor
>>> crows = [0, 2, 3, 5]
>>> cols = [1, 3, 2, 0, 1]
>>> values = [1.0, 2.0, 3.0, 4.0, 5.0]
>>> dense_shape = [3, 4]
>>> csr = paddle.sparse.sparse_csr_tensor(crows, cols, values, dense_shape)
>>> paddle.seed(2024)
>>> x = paddle.rand(dense_shape).astype(csr.dtype)
>>> out = paddle.sparse.mask_as(x, csr)
>>> print(out)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
crows=[0, 2, 3, 5],
cols=[1, 3, 2, 0, 1],
values=[0.23659813, 0.08467803, 0.64152628, 0.66596609, 0.90394485])
>>> # coo sparse tensor
>>> indices = [[0, 1, 2], [1, 2, 0]]
>>> values = [1.0, 2.0, 3.0]
>>> dense_shape = [3, 3]
>>> coo = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape)
>>> paddle.seed(2024)
>>> x = paddle.rand(dense_shape).astype(coo.dtype)
>>> out = paddle.sparse.mask_as(x, coo)
>>> print(out)
Tensor(shape=[3, 3], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
indices=[[0, 1, 2],
[1, 2, 0]],
values=[0.23659813, 0.40340215, 0.64152628])
"""
return _C_ops.sparse_mask_as(x, mask)
python/paddle/sparse/creation.py
+335
View File
@@ -0,0 +1,335 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import numpy as np
import paddle
from paddle import _C_ops, in_dynamic_mode
from paddle.base.data_feeder import convert_dtype
from paddle.base.framework import (
_current_expected_place,
_get_paddle_place,
core,
dygraph_only,
in_pir_mode,
)
from paddle.base.layer_helper import LayerHelper
from paddle.tensor import max, to_tensor
if TYPE_CHECKING:
import numpy.typing as npt
from paddle import CPUPlace, CUDAPinnedPlace, CUDAPlace, Tensor
from paddle._typing import DTypeLike, NumericSequence, ShapeLike
__all__ = [
'sparse_coo_tensor',
'sparse_csr_tensor',
]
def _handle_dtype(data, dtype):
if dtype:
if convert_dtype(dtype) != convert_dtype(data.dtype):
return data.astype(convert_dtype(dtype))
return data
def _infer_dense_shape(indices, values):
assert len(indices.shape) == 2
lens = max(indices, axis=1)
lens = lens + 1
lens = lens.numpy()
if len(values.shape) > 1:
lens = np.append(lens, values.shape[1:])
return list(lens)
def _get_place(place):
place = _get_paddle_place(place)
if place is None:
place = _current_expected_place()
elif not isinstance(
place, (core.Place, core.CPUPlace, core.CUDAPinnedPlace, core.CUDAPlace)
):
raise ValueError(
"'place' must be any of paddle.Place, paddle.CPUPlace, paddle.CUDAPinnedPlace, paddle.CUDAPlace"
)
return place
def _check_indices_dtype(dtype):
if dtype not in [paddle.int8, paddle.int16, paddle.int32, paddle.int64]:
raise TypeError(
"the dtype of indices must be 'int8' or 'int16' or 'int32' or 'int64'"
)
def sparse_coo_tensor(
indices: (
list[list[int]]
| tuple[tuple[int, ...], ...]
| npt.NDArray[np.int_]
| Tensor
),
values: NumericSequence | npt.NDArray[Any] | Tensor,
shape: ShapeLike | None = None,
dtype: DTypeLike | None = None,
place: CPUPlace | CUDAPinnedPlace | CUDAPlace | str | None = None,
stop_gradient: bool = True,
) -> Tensor:
r"""
Constructs a sparse ``paddle.Tensor`` in coordinate format according to the indices
and values of the specified non-zero elements.
Args:
indices(list|tuple|ndarray|Tensor): the indices of non-zero elements.
Can be a list, tuple, numpy\.ndarray, paddle\.Tensor. The indices must be 2-D.
values(list|tuple|ndarray|Tensor): Initial values for the tensor.
Can be a scalar, list, tuple, numpy\.ndarray, paddle\.Tensor.
shape(list|tuple|None, optional): The shape of the sparse tensor also represents the shape of
original dense tensor. If not provided the smallest shape will be inferred to
hold all elements.
dtype(str|paddle.dtype|np.dtype, optional): The desired data type of returned tensor. Can be 'bool' , 'float16' ,
'float32' , 'float64' , 'int8' , 'int16' , 'int32' , 'int64' , 'uint8',
'complex64' , 'complex128'. Default: None, infers dtype from ``data``
except for python float number which gets dtype from ``get_default_type`` .
place(CPUPlace|CUDAPinnedPlace|CUDAPlace|str|None, optional): The place to allocate Tensor. Can be
CPUPlace, CUDAPinnedPlace, CUDAPlace. Default: None, means global place. If ``place`` is
string, It can be ``cpu``, ``gpu:x`` and ``gpu_pinned``, where ``x`` is the index of the GPUs.
stop_gradient(bool, optional): Whether to block the gradient propagation of Autograd. Default: True.
Returns:
Tensor: A Tensor constructed from ``indices`` and ``values`` .
Examples:
.. code-block:: pycon
>>> import paddle
>>> indices = [[0, 1, 2], [1, 2, 0]]
>>> values = [1.0, 2.0, 3.0]
>>> dense_shape = [3, 3]
>>> coo = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape)
>>> print(coo)
Tensor(shape=[3, 3], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
indices=[[0, 1, 2],
[1, 2, 0]],
values=[1., 2., 3.])
"""
if in_dynamic_mode():
place = _get_place(place)
if not isinstance(indices, core.eager.Tensor):
indices = to_tensor(
indices, dtype=None, place=place, stop_gradient=True
)
if not isinstance(values, core.eager.Tensor):
values = to_tensor(values, dtype, place, stop_gradient)
if len(indices.shape) != 2:
raise ValueError("'indices' must be 2-D.")
nnz = indices.shape[1]
sparse_dim = indices.shape[0]
_check_indices_dtype(indices.dtype)
if nnz != values.shape[0]:
raise ValueError(
f"the indices and values must have same number of non-zero, but get {nnz} and {values.shape[0]}"
)
dense_dim = len(values.shape) - 1
if not indices.place._equals(place):
indices = indices._copy_to(place, False)
if not values.place._equals(place):
values = values._copy_to(place, False)
values = _handle_dtype(values, dtype)
values.stop_gradient = stop_gradient
min_shape = _infer_dense_shape(indices, values)
if shape is None:
shape = min_shape
else:
shape = list(shape)
if shape < min_shape:
raise ValueError(
f"the minimum shape required is {min_shape}, but get {shape}"
)
if len(shape) != sparse_dim + dense_dim:
raise ValueError(
f"the number of dimensions(len(shape) must be sparse_dim({sparse_dim}) + dense_dim({dense_dim}), but get {len(shape)}"
)
return _C_ops.sparse_sparse_coo_tensor(values, indices, shape)
elif in_pir_mode():
return _C_ops.sparse_sparse_coo_tensor(values, indices, shape)
if shape[0] is None:
shape[0] = -1
else:
op_type = 'sparse_sparse_coo_tensor'
inputs = {'values': values, 'indices': indices}
if shape[0] is None:
shape[0] = -1
attrs = {'shape': shape}
helper = LayerHelper(op_type)
out = helper.create_sparse_variable_for_type_inference(dtype)
helper.append_op(
type=op_type, inputs=inputs, outputs={'out': out}, attrs=attrs
)
return out
def _infer_dense_csr_shape(crows, cols):
crows_numpy = crows.numpy()
cols_numpy = cols.numpy()
batchs = np.sum(crows_numpy[:-1] > crows_numpy[1:]) + 1
if (int(len(crows_numpy) / batchs) * batchs) != len(crows_numpy):
raise ValueError(
f"The calculated original matrix batch size is {batchs}, but it cannot correctly split the row data. Please carefully check the data or the input shape."
)
col = cols_numpy.max() + 1
row = int(len(crows_numpy) / batchs) - 1
if batchs == 1:
return [row, col]
else:
return [batchs, row, col]
# TODO: need to support shape is None
@dygraph_only
def sparse_csr_tensor(
crows: list[int] | tuple[int, ...] | npt.NDArray[np.int_] | Tensor,
cols: list[int] | tuple[int, ...] | npt.NDArray[np.int_] | Tensor,
values: NumericSequence | npt.NDArray[Any] | Tensor,
shape: ShapeLike | None = None,
dtype: DTypeLike | None = None,
place: CPUPlace | CUDAPinnedPlace | CUDAPlace | str | None = None,
stop_gradient: bool = True,
) -> Tensor:
r"""
Constructs a sparse ``paddle.Tensor`` in CSR(Compressed Sparse Row) format according to the
``crows``, ``cols`` and ``values``.
Currently, the crows and cols of each batch must be incremented.
Args:
crows(list|tuple|ndarray|Tensor): 1-D array, each element in the rows represents the
starting position of the first non-zero element of each row in values.
Can be a list, tuple, numpy\.ndarray, paddle\.Tensor.
cols(list|tuple|ndarray|Tensor): 1-D array, the column of non-zero elements.
Can be a list, tuple, numpy\.ndarray, paddle\.Tensor.
values(list|tuple|ndarray|Tensor): 1-D array, the non-zero elements.
Can be a scalar, list, tuple, numpy\.ndarray, paddle\.Tensor.
shape(list|tuple, optional): The shape of the sparse tensor also represents the shape of
original dense tensor.
hold all elements.
dtype(str|paddle.dtype|np.dtype, optional): The desired data type of returned tensor. Can be 'bool' , 'float16' ,
'float32' , 'float64' , 'int8' , 'int16' , 'int32' , 'int64' , 'uint8',
'complex64' , 'complex128'. Default: None, infers dtype from ``data``
except for python float number which gets dtype from ``get_default_type`` .
place(CPUPlace|CUDAPinnedPlace|CUDAPlace|str|None, optional): The place to allocate Tensor. Can be
CPUPlace, CUDAPinnedPlace, CUDAPlace. Default: None, means global place. If ``place`` is
string, It can be ``cpu``, ``gpu:x`` and ``gpu_pinned``, where ``x`` is the index of the GPUs.
stop_gradient(bool, optional): Whether to block the gradient propagation of Autograd. Default: True.
Returns:
Tensor: A Tensor constructed from ``crows``, ``cols`` and ``values`` .
Examples:
.. code-block:: pycon
>>> import paddle
>>> crows = [0, 2, 3, 5]
>>> cols = [1, 3, 2, 0, 1]
>>> values = [1, 2, 3, 4, 5]
>>> dense_shape = [3, 4]
>>> csr = paddle.sparse.sparse_csr_tensor(crows, cols, values, dense_shape)
>>> print(csr)
Tensor(shape=[3, 4], dtype=paddle.int64, place=Place(cpu), stop_gradient=True,
crows=[0, 2, 3, 5],
cols=[1, 3, 2, 0, 1],
values=[1, 2, 3, 4, 5])
"""
place = _get_place(place)
if not isinstance(crows, core.eager.Tensor):
crows = to_tensor(crows, dtype=None, place=place, stop_gradient=True)
if not isinstance(cols, core.eager.Tensor):
cols = to_tensor(cols, dtype=None, place=place, stop_gradient=True)
if not isinstance(values, core.eager.Tensor):
values = to_tensor(values, dtype, place, stop_gradient)
_check_indices_dtype(crows.dtype)
_check_indices_dtype(cols.dtype)
if shape is not None:
if len(shape) != 2 and len(shape) != 3:
raise ValueError(
f"SparseCsrTensor only support 2-D or 3-D matrix. but get shape {shape}"
)
else:
shape = _infer_dense_csr_shape(crows, cols)
rows = shape[len(shape) - 2]
if not crows.place._equals(place):
crows = crows._copy_to(place, False)
if not cols.place._equals(place):
cols = cols._copy_to(place, False)
if not values.place._equals(place):
values = values._copy_to(place, False)
values = _handle_dtype(values, dtype)
values.stop_gradient = stop_gradient
if len(crows.shape) != 1 or len(cols.shape) != 1 or len(values.shape) != 1:
raise ValueError("The 'crows', 'cols' and 'values' must be 1-D.")
if len(cols) != len(values):
raise ValueError("the length of cols must be same as length of values")
if len(shape) == 2:
if crows.shape[0] != rows + 1:
raise ValueError(
f"The length({crows.shape[0]}) of crows must be equal to the rows({rows})+1 of matrix."
)
if crows[0] != 0:
raise ValueError("the 0th value of crows must be 0")
if crows[-1] != values.shape[0]:
raise ValueError(
"the last value of crows must be equal the number of non-zero"
)
else:
if crows.shape[0] % (rows + 1) != 0:
raise ValueError(
f"The length({crows.shape[0]}) of crows must be divisible the rows({rows})+1 of matrix."
)
# TODO(zkh2016): check whether the value in crows and cols is legal
return core.eager.sparse_csr_tensor(
crows, cols, values, shape, stop_gradient
)
python/paddle/sparse/multiary.py
+96
View File
@@ -0,0 +1,96 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from paddle import _C_ops
from paddle.base.framework import in_dynamic_or_pir_mode
if TYPE_CHECKING:
from paddle import Tensor
__all__ = []
def addmm(
input: Tensor,
x: Tensor,
y: Tensor,
beta: float = 1.0,
alpha: float = 1.0,
name: str | None = None,
) -> Tensor:
"""
Applies matrix multiplication for `x` and `y` , `input` is added to
the final result. The equation is:
.. math::
out = alpha * x * y + beta * input
The supported input/output Tensor layout are as follows:
Note:
input[SparseCsrTensor] + x[SparseCsrTensor] @ y[SparseCsrTensor] -> out[SparseCsrTensor]
input[DenseTensor] + x[SparseCsrTensor] @ y[DenseTensor] -> out[DenseTensor]
input[SparseCooTensor] + x[SparseCooTensor] @ y[SparseCooTensor] -> out[SparseCooTensor]
input[DenseTensor] + x[SparseCooTensor] @ y[DenseTensor] -> out[DenseTensor]
It supports backward propagation.
Dimensions `input` , `x` , `y` must be same and >= 2D. Automatic broadcasting of Tensor is not supported.
Args:
input (SparseTensor|DenseTensor): The input tensor. Shape is [*, M, N]. The data type can be float32 or float64.
x (SparseTensor): The input SparseTensor. Shape is [*, M, K]. The data type can be float32 or float64.
y (SparseTensor|DenseTensor): The input tensor. Shape is [*, K, N]. The data type can be float32 or float64.
beta (float, optional): Coefficient of `input` . Default: 1.0
alpha (float, optional): Coefficient of `x * y` . Default: 1.0
name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Returns:
SparseTensor|DenseTensor: Tensor type, date type and shape is the same with `input` .
Examples:
.. code-block:: pycon
>>> # doctest: +REQUIRES(env:GPU)
>>> import paddle
>>> paddle.device.set_device('gpu')
>>> # dense + csr @ dense -> dense
>>> input = paddle.rand([3, 2])
>>> crows = [0, 1, 2, 3]
>>> cols = [1, 2, 0]
>>> values = [1.0, 2.0, 3.0]
>>> x = paddle.sparse.sparse_csr_tensor(crows, cols, values, [3, 3])
>>> y = paddle.rand([3, 2])
>>> out = paddle.sparse.addmm(input, x, y, 3.0, 2.0)
>>> # dense + coo @ dense -> dense
>>> input = paddle.rand([3, 2])
>>> indices = [[0, 1, 2], [1, 2, 0]]
>>> values = [1.0, 2.0, 3.0]
>>> x = paddle.sparse.sparse_coo_tensor(indices, values, [3, 3])
>>> y = paddle.rand([3, 2])
>>> out = paddle.sparse.addmm(input, x, y, 3.0, 2.0)
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
return _C_ops.sparse_addmm(input, x, y, beta, alpha)
python/paddle/sparse/nn/__init__.py
+33
View File
@@ -0,0 +1,33 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import functional # noqa: F401
from .layer.activation import LeakyReLU, ReLU, ReLU6, Softmax
from .layer.conv import Conv2D, Conv3D, SubmConv2D, SubmConv3D
from .layer.norm import BatchNorm, SyncBatchNorm
from .layer.pooling import MaxPool3D
__all__ = [
'ReLU',
'ReLU6',
'LeakyReLU',
'Softmax',
'BatchNorm',
'SyncBatchNorm',
'Conv2D',
'Conv3D',
'SubmConv2D',
'SubmConv3D',
'MaxPool3D',
]
python/paddle/sparse/nn/functional/__init__.py
+40
View File
@@ -0,0 +1,40 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .activation import leaky_relu, relu, relu6, softmax
from .conv import (
conv2d,
conv3d,
subm_conv2d,
subm_conv2d_igemm,
subm_conv3d,
subm_conv3d_igemm,
)
from .pooling import max_pool3d
from .transformer import attention
__all__ = [
'conv2d',
'conv3d',
'subm_conv2d',
'subm_conv2d_igemm',
'subm_conv3d',
'subm_conv3d_igemm',
'max_pool3d',
'relu',
'relu6',
'leaky_relu',
'softmax',
'attention',
]
python/paddle/sparse/nn/functional/activation.py
+223
View File
@@ -0,0 +1,223 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
__all__ = []
from paddle import _C_ops
from paddle.base.framework import in_dynamic_or_pir_mode
from paddle.base.layer_helper import LayerHelper
if TYPE_CHECKING:
from paddle import Tensor
def relu(x: Tensor, name: str | None = None) -> Tensor:
"""
sparse relu activation, requiring x to be a SparseCooTensor or SparseCsrTensor.
.. math::
out = max(x, 0)
Parameters:
x (Tensor): The input Sparse Tensor with data type float32, float64.
name (str|None, optional): Name for the operation (optional, default is None).
For more information, please refer to :ref:`api_guide_Name`.
Returns:
A Sparse Tensor with the same data type and shape as ``x`` .
Examples:
.. code-block:: pycon
>>> import paddle
>>> dense_x = paddle.to_tensor([-2.0, 0.0, 1.0])
>>> sparse_x = dense_x.to_sparse_coo(1)
>>> out = paddle.sparse.nn.functional.relu(sparse_x)
>>> print(out)
Tensor(shape=[3], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
indices=[[0, 2]],
values=[0., 1.])
"""
if in_dynamic_or_pir_mode():
return _C_ops.sparse_relu(x)
else:
op_type = 'sparse_relu'
helper = LayerHelper(op_type)
out = helper.create_sparse_variable_for_type_inference(x.dtype)
helper.append_op(
type=op_type, inputs={'x': x}, outputs={'out': out}, attrs={}
)
return out
def softmax(x: Tensor, axis: int = -1, name: str | None = None) -> Tensor:
r"""
sparse softmax activation, requiring x to be a SparseCooTensor or SparseCsrTensor.
Note:
Only support axis=-1 for SparseCsrTensor, which is faster when read data
by row (axis=-1).
From the point of view of dense matrix, for each row :math:`i` and each column :math:`j`
in the matrix, we have:
.. math::
softmax_ij = \frac{\exp(x_ij - max_j(x_ij))}{\sum_j(exp(x_ij - max_j(x_ij))}
Parameters:
x (Tensor): The input tensor. It can be SparseCooTensor/SparseCsrTensor. The data type can be float32 or float64.
axis (int, optional): The axis along which to perform softmax calculations. Only support -1 for SparseCsrTensor.
name (str|None, optional): Name for the operation (optional, default is None).
For more information, please refer to :ref:`api_guide_Name`.
Returns:
Tensor: SparseCoo or SparseCsr, whose layout is the same with `x` .
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(100)
>>> mask = paddle.rand((3, 4)) < 0.5
>>> x = paddle.rand((3, 4)) * mask.astype('float32')
>>> print(x)
Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
[[0. , 0.95717543, 0.43864486, 0. ],
[0.84765935, 0.45680618, 0.39412445, 0. ],
[0.59444654, 0. , 0.78364515, 0. ]])
>>> csr = x.to_sparse_csr()
>>> print(csr)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
crows=[0, 2, 5, 7],
cols=[1, 2, 0, 1, 2, 0, 2],
values=[0.95717543, 0.43864486, 0.84765935, 0.45680618, 0.39412445,
0.59444654, 0.78364515])
>>> out = paddle.sparse.nn.functional.softmax(csr)
>>> print(out)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
crows=[0, 2, 5, 7],
cols=[1, 2, 0, 1, 2, 0, 2],
values=[0.62680405, 0.37319586, 0.43255258, 0.29261294, 0.27483448,
0.45284089, 0.54715902])
>>> coo = x.to_sparse_coo(sparse_dim=2)
>>> print(coo)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
indices=[[0, 0, 1, 1, 1, 2, 2],
[1, 2, 0, 1, 2, 0, 2]],
values=[0.95717543, 0.43864486, 0.84765935, 0.45680618, 0.39412445,
0.59444654, 0.78364515])
>>> out = paddle.sparse.nn.functional.softmax(coo)
>>> print(out)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
indices=[[0, 0, 1, 1, 1, 2, 2],
[1, 2, 0, 1, 2, 0, 2]],
values=[0.62680405, 0.37319589, 0.43255258, 0.29261294, 0.27483445,
0.45284092, 0.54715902])
"""
if in_dynamic_or_pir_mode():
return _C_ops.sparse_softmax(x, axis)
else:
op_type = 'sparse_softmax'
helper = LayerHelper(op_type)
out = helper.create_sparse_variable_for_type_inference(x.dtype)
helper.append_op(
type=op_type,
inputs={'x': x},
outputs={'out': out},
attrs={'axis': axis},
)
return out
def relu6(x: Tensor, name: str | None = None) -> Tensor:
"""
sparse relu6 activation, requiring x to be a SparseCooTensor or SparseCsrTensor.
.. math::
relu6(x) = min(max(0, x), 6)
Parameters:
x (Tensor): The input Sparse Tensor with data type float32, float64.
name (str|None, optional): Name for the operation (optional, default is None).
For more information, please refer to :ref:`api_guide_Name`.
Returns:
A Sparse Tensor with the same data type and shape as ``x`` .
Examples:
.. code-block:: pycon
>>> import paddle
>>> dense_x = paddle.to_tensor([-2.0, 0.0, 8.0])
>>> sparse_x = dense_x.to_sparse_coo(1)
>>> out = paddle.sparse.nn.functional.relu6(sparse_x)
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
return _C_ops.sparse_relu6(x)
def leaky_relu(
x: Tensor, negative_slope: float = 0.01, name: str | None = None
) -> Tensor:
r"""
sparse leaky_relu activation, requiring x to be a SparseCooTensor or SparseCsrTensor.
.. math::
leaky\_relu(x)=
\left\{
\begin{array}{rcl}
x, & & if \ x >= 0 \\
negative\_slope * x, & & otherwise \\
\end{array}
\right.
Parameters:
x (Tensor): The input Sparse Tensor with data type float32, float64.
negative_slope (float, optional): Slope of the activation function at
:math:`x < 0` . Default is 0.01.
name (str|None, optional): Name for the operation (optional, default is None).
For more information, please refer to :ref:`api_guide_Name`.
Returns:
A Sparse Tensor with the same data type and shape as ``x`` .
Examples:
.. code-block:: pycon
>>> import paddle
>>> dense_x = paddle.to_tensor([-2., 0., 5.])
>>> sparse_x = dense_x.to_sparse_coo(1)
>>> out = paddle.sparse.nn.functional.leaky_relu(sparse_x, 0.5)
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
return _C_ops.sparse_leaky_relu(x, negative_slope)
python/paddle/sparse/nn/functional/conv.py
+1025
View File
File diff suppressed because it is too large Load Diff
python/paddle/sparse/nn/functional/pooling.py
+117
View File
@@ -0,0 +1,117 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING, Literal
from paddle import _C_ops
from paddle.framework import in_dynamic_or_pir_mode
from paddle.nn.functional.pooling import _update_padding_nd
from paddle.utils import convert_to_list
if TYPE_CHECKING:
from paddle import Tensor
from paddle._typing import (
Size3,
Size6,
)
from paddle.nn.functional.common import _PaddingSizeMode
__all__ = []
def max_pool3d(
x: Tensor,
kernel_size: Size3,
stride: Size3 | None = None,
padding: _PaddingSizeMode | Size3 | Size6 = 0,
ceil_mode: bool = False,
data_format: Literal['NDHWC'] = "NDHWC",
name: str | None = None,
) -> Tensor:
"""
Implements sparse max pooling 3d operation.
See more details in :ref:`api_paddle_sparse_nn_MaxPool3D` .
Args:
x (Tensor): The input SparseCooTensor of pooling operator, which is a 5-D tensor with
shape [N, D, H, W, C]. The format of input tensor `"NDHWC"`, where N represents batch size, C represents the number of channels, D, H and W represent the depth, height and width of the feature respectively.
kernel_size (int|list|tuple): The pool kernel size. If the kernel size
is a tuple or list, it must contain three integers,
(kernel_size_Depth, kernel_size_Height, kernel_size_Width).
Otherwise, the pool kernel size will be the cube of an int.
stride (int|list|tuple, optional): The pool stride size. If pool stride size is a tuple or list,
it must contain three integers, [stride_Depth, stride_Height, stride_Width).
Otherwise, the pool stride size will be a cube of an int.
padding (string|int|list|tuple, optional): The padding size. Padding could be in one of the following forms.
1. A string in ['valid', 'same'].
2. An int, which means the feature map is zero padded by size of `padding` on every sides.
3. A list[int] or tuple(int) whose length is 3, [pad_depth, pad_height, pad_weight] whose value means the padding size of each dimension.
4. A list[int] or tuple(int) whose length is 6. [pad_depth_front, pad_depth_back, pad_height_top, pad_height_bottom, pad_width_left, pad_width_right] whose value means the padding size of each side.
5. A list or tuple of pairs of integers. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension should be [0,0] or (0,0).
The default value is 0.
ceil_mode (bool, optional): ${ceil_mode_comment}
data_format (string, optional): The data format of the input and output data. An optional string from: `"NCDHW"`, `"NDHWC"`.
The default is `"NCDHW"`. When it is `"NCDHW"`, the data is stored in the order of:
`[batch_size, input_channels, input_depth, input_height, input_width]`. Currently only support `"NDHWC"` .
name(str|None, optional): For detailed information, please refer
to :ref:`api_guide_Name`. Usually name is no need to set and
None by default.
Returns:
Tensor: The output tensor of pooling result. The data type is same as input tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> dense_x = paddle.randn((1, 4, 4, 4, 3))
>>> sparse_x = dense_x.to_sparse_coo(4)
>>> kernel_sizes = [3, 3, 3]
>>> paddings = [0, 0, 0]
>>> strides = [1, 1, 1]
>>> out = paddle.sparse.nn.functional.max_pool3d(sparse_x, kernel_sizes, stride=strides, padding=paddings)
>>> print(out.shape)
paddle.Size([1, 2, 2, 2, 3])
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
assert x.is_sparse_coo(), (
"Currently, sparse.relu only support the input of SparseCooTensor"
)
assert data_format == 'NDHWC', (
"Currently, sparse.max_pool3d only support data format of 'NDHWC'"
)
kernel_size = convert_to_list(kernel_size, 3, 'pool_size')
if stride is None:
stride = kernel_size
else:
stride = convert_to_list(stride, 3, 'pool_stride')
channel_last = True
padding, padding_algorithm = _update_padding_nd(
padding, 3, channel_last=channel_last, ceil_mode=ceil_mode
)
# TODO(zkh2016): remove the dependency on dilation from the backend
dilation = [1, 1, 1]
return _C_ops.sparse_maxpool(x, kernel_size, padding, dilation, stride)
python/paddle/sparse/nn/functional/transformer.py
+105
View File
@@ -0,0 +1,105 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
__all__ = []
from paddle import _C_ops
from paddle.base.framework import in_dynamic_or_pir_mode
if TYPE_CHECKING:
from paddle import Tensor
def attention(
query: Tensor,
key: Tensor,
value: Tensor,
sparse_mask: Tensor,
key_padding_mask: Tensor | None = None,
attn_mask: Tensor | None = None,
name: str | None = None,
) -> Tensor:
r"""
Note:
This API is only used from ``CUDA 11.8`` .
SparseCsrTensor is used to store the intermediate result of Attention matrix
in Transformer module, which can reduce memory usage and improve performance.
``sparse_mask`` express the sparse layout in CSR format.
The calculation equation is:
.. math::
result = softmax(\frac{ Q * K^T }{\sqrt{d}}) * V
where : ``Q``, ``K``, and ``V`` represent the three input parameters of the attention module.
The shape of the three parameters are: `[batch_size, num_heads, seq_len, head_dim]`, and
``d`` represents ``head_dim`` .
Args:
query (DenseTensor): `query` in the Attention module. 4D Tensor with float32 or float64.
key (DenseTensor): `key` in the Attention module. 4D Tensor with float32 or float64.
value (DenseTensor): `value` in the Attention module. 4D Tensor with float32 or float64.
sparse_mask (SparseCsrTensor): The sparse layout in the Attention module. Its dense shape
is `[batch_size*num_heads, seq_len, seq_len]`. `nnz` of each batch must be the same.
dtype of `crows` and `cols` must be int64, dtype of `values` can be float32 or float64.
key_padding_mask (DenseTensor|None, optional): The key padding mask tensor in the Attention module.
2D tensor with shape: [batch_size, seq_len]. dtype can be float32 or float64. Default: None.
attn_mask (DenseTensor|None, optional): The attention mask tensor in the Attention module.
2D tensor with shape: [seq_len, seq_len]. dtype can be float32 or float64. Default: None.
name (str|None, optional): The default value is None. Normally there is no need for user
to set this property. For more information, please refer to :ref:`api_guide_Name`.
Returns:
4D tensor with shape: [batch_size, num_heads, seq_len, head_dim]. dtype is same with input.
Examples:
.. code-block:: pycon
>>> # doctest: +REQUIRES(env:GPU)
>>> import paddle
>>> paddle.device.set_device('gpu')
>>> batch_size = 16
>>> num_heads = 16
>>> seq_len = 512
>>> head_dim = 32
>>> query = paddle.rand([batch_size, num_heads, seq_len, head_dim])
>>> key = paddle.rand([batch_size, num_heads, seq_len, head_dim])
>>> value = paddle.rand([batch_size, num_heads, seq_len, head_dim])
>>> query.stop_gradient = False
>>> key.stop_gradient = False
>>> value.stop_gradient = False
>>> mask = paddle.nn.functional.dropout(paddle.ones([seq_len, seq_len])).expand([batch_size, num_heads, seq_len, seq_len])
>>> sp_mask = mask.reshape([-1, seq_len, seq_len]).to_sparse_csr()
>>> kp_mask = paddle.randint(0, 2, [batch_size, seq_len]).astype(paddle.float32)
>>> attn_mask = paddle.randint(0, 2, [seq_len, seq_len]).astype(paddle.float32)
>>> output = paddle.sparse.nn.functional.attention(query, key, value, sp_mask, kp_mask, attn_mask)
>>> output.backward()
"""
assert in_dynamic_or_pir_mode(), (
"Currently, Sparse API only support dynamic mode or pir mode."
)
return _C_ops.sparse_fused_attention(
query, key, value, sparse_mask, key_padding_mask, attn_mask
)
python/paddle/sparse/nn/layer/activation.py
+248
View File
@@ -0,0 +1,248 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from paddle.nn import Layer
from .. import functional as F
if TYPE_CHECKING:
from paddle import Tensor
__all__ = []
class ReLU(Layer):
"""
Sparse ReLU Activation, requiring x to be a SparseCooTensor or SparseCsrTensor.
.. math::
ReLU(x) = max(x, 0)
Parameters:
name (str|None, optional): Name for the operation (optional, default is None).
For more information, please refer to :ref:`api_guide_Name`.
Shape:
- input: Sparse Tensor with any shape.
- output: Sparse Tensor with the same shape as input.
Examples:
.. code-block:: pycon
>>> import paddle
>>> dense_x = paddle.to_tensor([-2.0, 0.0, 1.0])
>>> sparse_x = dense_x.to_sparse_coo(1)
>>> relu = paddle.sparse.nn.ReLU()
>>> out = relu(sparse_x)
>>> print(out)
Tensor(shape=[3], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
indices=[[0, 2]],
values=[0., 1.])
"""
def __init__(self, name: str | None = None) -> None:
super().__init__()
self._name = name
def forward(self, x: Tensor) -> Tensor:
return F.relu(x, self._name)
def extra_repr(self) -> str:
name_str = f'name={self._name}' if self._name else ''
return name_str
class Softmax(Layer):
r"""
Sparse Softmax Activation, requiring x to be a SparseCooTensor or SparseCsrTensor.
Note:
Only support axis=-1 for SparseCsrTensor, which is faster when read data
by row (axis=-1).
Transform x to dense matrix, and :math:`i` is row index, :math:`j` is column index.
If axis=-1, We have:
.. math::
softmax_ij = \frac{\exp(x_ij - max_j(x_ij))}{\sum_j(exp(x_ij - max_j(x_ij))}
Parameters:
axis (int, optional): The axis along which to perform softmax calculations. Only support -1 for SparseCsrTensor.
name (str|None, optional): Name for the operation (optional, default is None).
For more information, please refer to :ref:`api_guide_Name`.
Shape:
- input: SparseCooTensor / SparseCsrTensor with any shape.
- output: Sparse Tensor with the same shape as input.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(2022)
>>> mask = paddle.rand((3, 4)) < 0.7
>>> x = paddle.rand((3, 4)) * mask.astype('float32')
>>> print(x)
Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
[[0.88156885, 0.14463395, 0.17831714, 0.43818203],
[0.07617740, 0.75576496, 0. , 0.61921930],
[0. , 0. , 0.42460245, 0.03001321]])
>>> csr = x.to_sparse_csr()
>>> print(csr)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
crows=[0, 4, 7, 9],
cols=[0, 1, 2, 3, 0, 1, 3, 2, 3],
values=[0.88156885, 0.14463395, 0.17831714, 0.43818203, 0.07617740,
0.75576496, 0.61921930, 0.42460245, 0.03001321])
>>> softmax = paddle.sparse.nn.Softmax()
>>> out = softmax(csr)
>>> print(out)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
crows=[0, 4, 7, 9],
cols=[0, 1, 2, 3, 0, 1, 3, 2, 3],
values=[0.38234913, 0.18298410, 0.18925257, 0.24541418, 0.21302439,
0.42031071, 0.36666498, 0.59738696, 0.40261301])
>>> coo = x.to_sparse_coo(sparse_dim=2)
>>> print(coo)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
indices=[[0, 0, 0, 0, 1, 1, 1, 2, 2],
[0, 1, 2, 3, 0, 1, 3, 2, 3]],
values=[0.88156885, 0.14463395, 0.17831714, 0.43818203, 0.07617740,
0.75576496, 0.61921930, 0.42460245, 0.03001321])
>>> out = softmax(coo)
>>> print(out)
Tensor(shape=[3, 4], dtype=paddle.float32, place=Place(cpu), stop_gradient=True,
indices=[[0, 0, 0, 0, 1, 1, 1, 2, 2],
[0, 1, 2, 3, 0, 1, 3, 2, 3]],
values=[0.38234913, 0.18298411, 0.18925257, 0.24541420, 0.21302438,
0.42031071, 0.36666498, 0.59738696, 0.40261301])
"""
def __init__(self, axis: int = -1, name: str | None = None) -> None:
super().__init__()
self._axis = axis
self._name = name
def forward(self, x: Tensor) -> Tensor:
return F.softmax(x, self._axis, self._name)
def extra_repr(self) -> str:
name_str = f'name={self._name}' if self._name else ''
return name_str
class ReLU6(Layer):
"""
Sparse ReLU6 Activation, requiring x to be a SparseCooTensor or SparseCsrTensor.
.. math::
ReLU6(x) = min(max(0,x), 6)
Parameters:
name (str|None, optional): Name for the operation (optional, default is None).
For more information, please refer to :ref:`api_guide_Name`.
Shape:
- input: Sparse Tensor with any shape.
- output: Sparse Tensor with the same shape as input.
Examples:
.. code-block:: pycon
>>> import paddle
>>> dense_x = paddle.to_tensor([-2.0, 0.0, 8.0])
>>> sparse_x = dense_x.to_sparse_coo(1)
>>> relu6 = paddle.sparse.nn.ReLU6()
>>> out = relu6(sparse_x)
"""
def __init__(self, name: str | None = None) -> None:
super().__init__()
self._name = name
def forward(self, x: Tensor) -> Tensor:
return F.relu6(x, self._name)
def extra_repr(self) -> str:
name_str = f'name={self._name}' if self._name else ''
return name_str
class LeakyReLU(Layer):
r"""
Sparse Leaky ReLU Activation, requiring x to be a SparseCooTensor or SparseCsrTensor.
.. math::
LeakyReLU(x)=
\left\{
\begin{array}{rcl}
x, & & if \ x >= 0 \\
negative\_slope * x, & & otherwise \\
\end{array}
\right.
Parameters:
negative_slope (float, optional): Slope of the activation function at
:math:`x < 0` . Default is 0.01.
name (str|None, optional): Name for the operation (optional, default is None).
For more information, please refer to :ref:`api_guide_Name`.
Shape:
- input: Sparse Tensor with any shape.
- output: Sparse Tensor with the same shape as input.
Examples:
.. code-block:: pycon
>>> import paddle
>>> dense_x = paddle.to_tensor([-2., 0., 5.])
>>> sparse_x = dense_x.to_sparse_coo(1)
>>> leaky_relu = paddle.sparse.nn.LeakyReLU(0.5)
>>> out = leaky_relu(sparse_x)
"""
def __init__(
self, negative_slope: float = 0.01, name: str | None = None
) -> None:
super().__init__()
self._negative_slope = negative_slope
self._name = name
def forward(self, x: Tensor) -> Tensor:
return F.leaky_relu(x, self._negative_slope, self._name)
def extra_repr(self) -> str:
name_str = f'name={self._name}' if self._name else ''
return name_str
python/paddle/sparse/nn/layer/conv.py
+869
View File
@@ -0,0 +1,869 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING, Literal
import numpy as np
from paddle.nn import Layer
from paddle.nn.functional.conv import _update_padding_nd
from paddle.nn.initializer import Normal
from paddle.utils import convert_to_list
from .. import functional as F
if TYPE_CHECKING:
from collections.abc import Sequence
from paddle import Tensor
from paddle._typing import (
ParamAttrLike,
Size2,
Size3,
Size4,
Size6,
)
from paddle.nn.functional.common import _PaddingSizeMode
__all__ = []
class _Conv3D(Layer):
weight: Tensor
bias: Tensor
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: Size3,
stride: Size3 = 1,
padding: _PaddingSizeMode | Size3 | Size6 | Sequence[Size2] = 0,
dilation: Size3 = 1,
groups: Literal[1] = 1,
subm: bool = False,
key: str | None = None,
padding_mode: Literal['zeros'] = 'zeros',
weight_attr: ParamAttrLike | None = None,
bias_attr: ParamAttrLike | None = None,
data_format: Literal['NDHWC'] = "NDHWC",
backend: Literal['igemm'] | None = None,
) -> None:
super().__init__()
assert weight_attr is not False, (
"weight_attr should not be False in Conv."
)
self._param_attr = weight_attr
self._bias_attr = bias_attr
self._groups = groups
self._in_channels = in_channels
self._out_channels = out_channels
self._data_format = data_format
self._subm = subm
self._key = key
self._backend = backend
assert padding_mode == 'zeros', (
"Currently, only support padding_mode='zeros'"
)
assert groups == 1, "Currently, only support groups=1"
assert backend in [
None,
'igemm',
], "The value of 'backend' in Conv3D should be None or 'igemm'."
valid_format = {'NDHWC'}
if data_format not in valid_format:
raise ValueError(
f"data_format must be one of {valid_format}, but got data_format='{data_format}'"
)
channel_last = data_format == "NDHWC"
dims = 3
self._stride = convert_to_list(stride, dims, 'stride')
self._dilation = convert_to_list(dilation, dims, 'dilation')
self._kernel_size = convert_to_list(kernel_size, dims, 'kernel_size')
self._padding = padding
self._padding_mode = padding_mode
self._updated_padding, self._padding_algorithm = _update_padding_nd(
padding, channel_last, dims
)
# the sparse conv restricts the shape is [D, H, W, in_channels, out_channels]
filter_shape = [
*self._kernel_size,
self._in_channels,
self._out_channels,
]
def _get_default_param_initializer():
filter_elem_num = np.prod(self._kernel_size) * self._in_channels
std = (2.0 / filter_elem_num) ** 0.5
return Normal(0.0, std)
self.weight = self.create_parameter(
shape=filter_shape,
attr=self._param_attr,
default_initializer=_get_default_param_initializer(),
)
self.bias = self.create_parameter(
attr=self._bias_attr, shape=[self._out_channels], is_bias=True
)
def forward(self, x: Tensor) -> Tensor:
if self._backend is None:
out = F.conv._conv3d(
x,
self.weight,
bias=self.bias,
stride=self._stride,
padding=self._updated_padding,
dilation=self._dilation,
groups=self._groups,
subm=self._subm,
key=self._key,
data_format=self._data_format,
)
elif self._backend == 'igemm':
out = F.conv._conv3d_igemm(
x,
self.weight,
bias=self.bias,
stride=self._stride,
padding=self._updated_padding,
dilation=self._dilation,
groups=self._groups,
subm=self._subm,
key=self._key,
data_format=self._data_format,
)
else:
raise ValueError(
f"The value of 'backend' in Conv3D should be None or 'igemm', but got {self._backend}."
)
return out
def extra_repr(self) -> str:
main_str = '{_in_channels}, {_out_channels}, kernel_size={_kernel_size}'
if self._stride != [1] * len(self._stride):
main_str += ', stride={_stride}'
if self._padding != 0:
main_str += ', padding={_padding}'
if self._padding_mode != 'zeros':
main_str += ', padding_mode={_padding_mode}'
if self._dilation != [1] * len(self._dilation):
main_str += ', dilation={_dilation}'
if self._groups != 1:
main_str += ', groups={_groups}'
main_str += ', data_format={_data_format}'
return main_str.format(**self.__dict__)
class _Conv2D(Layer):
weight: Tensor
bias: Tensor
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: Size2,
stride: Size2 = 1,
padding: _PaddingSizeMode | Size2 | Size4 | Sequence[Size2] = 0,
dilation: Size2 = 1,
groups: Literal[1] = 1,
subm: bool = False,
key: str | None = None,
padding_mode: Literal['zeros'] = 'zeros',
weight_attr: ParamAttrLike | None = None,
bias_attr: ParamAttrLike | None = None,
data_format: Literal["NHWC"] = "NHWC",
backend: Literal['igemm'] | None = None,
) -> None:
super().__init__()
assert weight_attr is not False, (
"weight_attr should not be False in Conv."
)
self._param_attr = weight_attr
self._bias_attr = bias_attr
self._groups = groups
self._in_channels = in_channels
self._out_channels = out_channels
self._data_format = data_format
self._subm = subm
self._key = key
self._backend = backend
assert padding_mode == 'zeros', (
"Currently, only support padding_mode='zeros'"
)
assert groups == 1, "Currently, only support groups=1"
assert backend in [
None,
'igemm',
], "The value of 'backend' in Conv3D should be None or 'igemm'."
valid_format = {'NHWC'}
if data_format not in valid_format:
raise ValueError(
f"data_format must be one of {valid_format}, but got data_format='{data_format}'"
)
channel_last = data_format == "NHWC"
dims = 2
self._stride = convert_to_list(stride, dims, 'stride')
self._dilation = convert_to_list(dilation, dims, 'dilation')
self._kernel_size = convert_to_list(kernel_size, dims, 'kernel_size')
self._padding = padding
self._padding_mode = padding_mode
self._updated_padding, self._padding_algorithm = _update_padding_nd(
padding, channel_last, dims
)
# the sparse conv restricts the shape is [H, W, in_channels, out_channels]
filter_shape = [
*self._kernel_size,
self._in_channels,
self._out_channels,
]
def _get_default_param_initializer():
filter_elem_num = np.prod(self._kernel_size) * self._in_channels
std = (2.0 / filter_elem_num) ** 0.5
return Normal(0.0, std)
self.weight = self.create_parameter(
shape=filter_shape,
attr=self._param_attr,
default_initializer=_get_default_param_initializer(),
)
self.bias = self.create_parameter(
attr=self._bias_attr, shape=[self._out_channels], is_bias=True
)
def forward(self, x: Tensor) -> Tensor:
if self._backend is None:
out = F.conv._conv2d(
x,
self.weight,
bias=self.bias,
stride=self._stride,
padding=self._updated_padding,
dilation=self._dilation,
groups=self._groups,
subm=self._subm,
key=self._key,
data_format=self._data_format,
)
elif self._backend == 'igemm':
out = F.conv._conv2d_igemm(
x,
self.weight,
bias=self.bias,
stride=self._stride,
padding=self._updated_padding,
dilation=self._dilation,
groups=self._groups,
subm=self._subm,
key=self._key,
data_format=self._data_format,
)
else:
raise ValueError(
f"The value of 'backend' in Conv2D should be None or 'igemm', but got {self._backend}."
)
return out
def extra_repr(self) -> str:
main_str = '{_in_channels}, {_out_channels}, kernel_size={_kernel_size}'
if self._stride != [1] * len(self._stride):
main_str += ', stride={_stride}'
if self._padding != 0:
main_str += ', padding={_padding}'
if self._padding_mode != 'zeros':
main_str += ', padding_mode={_padding_mode}'
if self._dilation != [1] * len(self._dilation):
main_str += ', dilation={_dilation}'
if self._groups != 1:
main_str += ', groups={_groups}'
main_str += ', data_format={_data_format}'
return main_str.format(**self.__dict__)
class Conv3D(_Conv3D):
r"""
**Sparse Convolution3d Layer**
The Sparse convolution3d layer calculates the output based on the input, filter
and strides, paddings, dilations, groups parameters. Input(Input) and
Output(Output) are multidimensional SparseCooTensors with a shape of
:math:`[N, D, H, W, C]` . Where N is batch size, C is the number of
channels, D is the depth of the feature, H is the height of the feature,
and W is the width of the feature. If bias attribution is provided,
bias is added to the output of the convolution.
For each input :math:`X`, the equation is:
.. math::
Out = W \ast X + b
In the above equation:
* :math:`X`: Input value, a tensor with NDHWC format.
* :math:`W`: Filter value, a tensor with DHWCM format.
* :math:`\\ast`: Convolution operation.
* :math:`b`: Bias value, a 1-D tensor with shape [M].
* :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
Parameters:
in_channels(int): The number of input channels in the input image.
out_channels(int): The number of output channels produced by the convolution.
kernel_size(int|list|tuple): The size of the convolving kernel.
stride(int|list|tuple, optional): The stride size. If stride is a list/tuple, it must
contain three integers, (stride_D, stride_H, stride_W). Otherwise, the
stride_D = stride_H = stride_W = stride. The default value is 1.
padding(int|str|tuple|list, optional): The padding size. Padding couple be in one of the following forms.
1. a string in ['valid', 'same'].
2. an int, which means each spatial dimension(depth, height, width) is zero padded by size of `padding`
3. a list[int] or tuple[int] whose length is the number of spatial dimensions, which contains the amount of padding on each side for each spatial dimension. It has the form [pad_d1, pad_d2, ...].
4. a list[int] or tuple[int] whose length is 2 * number of spatial dimensions. It has the form [pad_before, pad_after, pad_before, pad_after, ...] for all spatial dimensions.
5. a list or tuple of pairs of ints. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension are also included. Each pair of integers correspond to the amount of padding for a dimension of the input. Padding in batch dimension and channel dimension should be [0, 0] or (0, 0).
The default value is 0.
dilation(int|list|tuple, optional): The dilation size. If dilation is a list/tuple, it must
contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the
dilation_D = dilation_H = dilation_W = dilation. The default value is 1.
groups(int, optional): The groups number of the Conv3D Layer. According to grouped
convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
the first half of the filters is only connected to the first half
of the input channels, while the second half of the filters is only
connected to the second half of the input channels. The default value is 1, currently, only support groups=1.
padding_mode(str, optional): ``'zeros'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. Currently only support ``'zeros'``.
weight_attr(ParamAttr, optional): The parameter attribute for learnable parameters/weights
of conv3d. If it is set to None or one attribute of ParamAttr, conv3d
will create ParamAttr as param_attr. If it is set to None, the parameter
is initialized with :math:`Normal(0.0, std)`, and the :math:`std` is
:math:`(\frac{2.0 }{filter\_elem\_num})^{0.5}`. The default value is None.
bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of conv3d.
If it is set to False, no bias will be added to the output units.
If it is set to None or one attribute of ParamAttr, conv3d
will create ParamAttr as bias_attr. If the Initializer of the bias_attr
is not set, the bias is initialized zero. The default value is None.
data_format(str, optional): Data format that specifies the layout of input.
It can be "NCDHW" or "NDHWC". Currently, only support "NCDHW".
Attribute:
**weight** (Parameter): the learnable weights of filters of this layer.
**bias** (Parameter): the learnable bias of this layer.
Shape:
- x: :math:`(N, D_{in}, H_{in}, W_{in}, C_{in})`
- weight: :math:`(K_{d}, K_{h}, K_{w}, C_{in}, C_{out})`
- bias: :math:`(C_{out})`
- output: :math:`(N, D_{out}, H_{out}, W_{out}, C_{out})`
Where
.. math::
D_{out}&= \frac{(D_{in} + 2 * paddings[0] - (dilations[0] * (kernel\_size[0] - 1) + 1))}{strides[0]} + 1
H_{out}&= \frac{(H_{in} + 2 * paddings[1] - (dilations[1] * (kernel\_size[1] - 1) + 1))}{strides[1]} + 1
W_{out}&= \frac{(W_{in} + 2 * paddings[2] - (dilations[2] * (kernel\_size[2] - 1) + 1))}{strides[2]} + 1
Examples:
.. code-block:: pycon
>>> import paddle
>>> indices = [
... [0, 0, 0, 0],
... [0, 0, 0, 0],
... [0, 0, 1, 2],
... [1, 3, 2, 3],
... ]
>>> values = [[1], [2], [3], [4]]
>>> indices = paddle.to_tensor(indices, dtype='int32')
>>> values = paddle.to_tensor(values, dtype='float32')
>>> dense_shape = [1, 1, 3, 4, 1]
>>> sparse_x = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape, stop_gradient=True)
>>> conv = paddle.sparse.nn.Conv3D(1, 1, (1, 3, 3))
>>> y = conv(sparse_x)
>>> print(y.shape)
paddle.Size([1, 1, 1, 2, 1])
"""
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: Size3,
stride: Size3 = 1,
padding: _PaddingSizeMode | Size3 | Size6 | Sequence[Size2] = 0,
dilation: Size3 = 1,
groups: Literal[1] = 1,
padding_mode: Literal['zeros'] = 'zeros',
weight_attr: ParamAttrLike | None = None,
bias_attr: ParamAttrLike | None = None,
data_format: Literal["NDHWC"] = "NDHWC",
) -> None:
super().__init__(
in_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding,
dilation=dilation,
groups=groups,
subm=False,
key=None,
padding_mode=padding_mode,
weight_attr=weight_attr,
bias_attr=bias_attr,
data_format=data_format,
)
class Conv2D(_Conv2D):
r"""
**Sparse Convolution2d Layer**
The Sparse convolution2d layer calculates the output based on the input, filter
and strides, paddings, dilations, groups parameters. Input(Input) and
Output(Output) are multidimensional SparseCooTensors with a shape of
:math:`[N, H, W, C]` . Where N is batch size, C is the number of
channels, H is the height of the feature,
and W is the width of the feature. If bias attribution is provided,
bias is added to the output of the convolution.
For each input :math:`X`, the equation is:
.. math::
Out = W \ast X + b
In the above equation:
* :math:`X`: Input value, a tensor with NHWC format.
* :math:`W`: Filter value, a tensor with HWCM format.
* :math:`\\ast`: Convolution operation.
* :math:`b`: Bias value, a 1-D tensor with shape [M].
* :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
Parameters:
in_channels(int): The number of input channels in the input image.
out_channels(int): The number of output channels produced by the convolution.
kernel_size(int|list|tuple): The size of the convolving kernel.
stride(int|list|tuple, optional): The stride size. If stride is a list/tuple, it must
contain three integers, (stride_H, stride_W). Otherwise, the
stride_H = stride_W = stride. The default value is 1.
padding(int|str|tuple|list, optional): The padding size. Padding couple be in one of the following forms.
1. a string in ['valid', 'same'].
2. an int, which means each spatial dimension(height, width) is zero padded by size of `padding`
3. a list[int] or tuple[int] whose length is the number of spatial dimensions, which contains the amount of padding on each side for each spatial dimension. It has the form [pad_d1, pad_d2, ...].
4. a list[int] or tuple[int] whose length is 2 * number of spatial dimensions. It has the form [pad_before, pad_after, pad_before, pad_after, ...] for all spatial dimensions.
5. a list or tuple of pairs of ints. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...].
Note that, the batch dimension and channel dimension are also included. Each pair of integers correspond to the amount of padding for a dimension of the input. Padding in batch dimension and channel dimension should be [0, 0] or (0, 0).
The default value is 0.
dilation(int|list|tuple, optional): The dilation size. If dilation is a list/tuple, it must
contain three integers, (dilation_H, dilation_W). Otherwise, the
dilation_H = dilation_W = dilation. The default value is 1.
groups(int, optional): The groups number of the Conv2D Layer. According to grouped
convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
the first half of the filters is only connected to the first half
of the input channels, while the second half of the filters is only
connected to the second half of the input channels. The default value is 1, currently, only support groups=1.
padding_mode(str, optional): ``'zeros'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. Currently only support ``'zeros'``.
weight_attr(ParamAttr, optional): The parameter attribute for learnable parameters/weights
of conv2d. If it is set to None or one attribute of ParamAttr, conv2d
will create ParamAttr as param_attr. If it is set to None, the parameter
is initialized with :math:`Normal(0.0, std)`, and the :math:`std` is
:math:`(\frac{2.0 }{filter\_elem\_num})^{0.5}`. The default value is None.
bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of conv2d.
If it is set to False, no bias will be added to the output units.
If it is set to None or one attribute of ParamAttr, conv2d
will create ParamAttr as bias_attr. If the Initializer of the bias_attr
is not set, the bias is initialized zero. The default value is None.
data_format(str, optional): Data format that specifies the layout of input.
It can be "NCHW" or "NHWC". Currently, only support "NHWC".
Attribute:
**weight** (Parameter): the learnable weights of filters of this layer.
**bias** (Parameter): the learnable bias of this layer.
Shape:
- x: :math:`(N, H_{in}, W_{in}, C_{in})`
- weight: :math:`(K_{h}, K_{w}, C_{in}, C_{out})`
- bias: :math:`(C_{out})`
- output: :math:`(N, H_{out}, W_{out}, C_{out})`
Where
.. math::
H_{out}&= \frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (kernel\_size[0] - 1) + 1))}{strides[0]} + 1
W_{out}&= \frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (kernel\_size[1] - 1) + 1))}{strides[1]} + 1
Examples:
.. code-block:: pycon
>>> import paddle
>>> indices = [[0, 0, 0, 0], [0, 0, 1, 2], [1, 3, 2, 3]]
>>> values = [[1], [2], [3], [4]]
>>> indices = paddle.to_tensor(indices, dtype='int32')
>>> values = paddle.to_tensor(values, dtype='float32')
>>> dense_shape = [1, 3, 4, 1]
>>> sparse_x = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape, stop_gradient=True)
>>> conv = paddle.sparse.nn.Conv2D(1, 1, (3, 3))
>>> y = conv(sparse_x)
>>> print(y.shape)
paddle.Size([1, 1, 2, 1])
"""
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: Size2,
stride: Size2 = 1,
padding: _PaddingSizeMode | Size2 | Size4 | Sequence[Size2] = 0,
dilation: Size2 = 1,
groups: Literal[1] = 1,
padding_mode: Literal['zeros'] = 'zeros',
weight_attr: ParamAttrLike | None = None,
bias_attr: ParamAttrLike | None = None,
data_format: Literal["NHWC"] = "NHWC",
) -> None:
super().__init__(
in_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding,
dilation=dilation,
groups=groups,
subm=False,
key=None,
padding_mode=padding_mode,
weight_attr=weight_attr,
bias_attr=bias_attr,
data_format=data_format,
)
class SubmConv3D(_Conv3D):
r"""
**Submanifold Sparse Convolution3d Layer**
The submanifold sparse convolution3d layer calculates the output based on the input, filter
and strides, paddings, dilations, groups parameters. Input(Input) and
Output(Output) are multidimensional SparseCooTensors with a shape of
:math:`[N, D, H, W, C]` . Where N is batch size, C is the number of
channels, D is the depth of the feature, H is the height of the feature,
and W is the width of the feature. If bias attribution is provided,
bias is added to the output of the convolution.
For each input :math:`X`, the equation is:
.. math::
Out = W \ast X + b
In the above equation:
* :math:`X`: Input value, a tensor with NDHWC format.
* :math:`W`: Filter value, a tensor with DHWCM format.
* :math:`\\ast`: Submanifold Convolution operation, refer to the paper: https://arxiv.org/abs/1706.01307.
* :math:`b`: Bias value, a 1-D tensor with shape [M].
* :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
Parameters:
in_channels(int): The number of input channels in the input image.
out_channels(int): The number of output channels produced by the convolution.
kernel_size(int|list|tuple): The size of the convolving kernel.
stride(int|list|tuple, optional): The stride size. If stride is a list/tuple, it must
contain three integers, (stride_D, stride_H, stride_W). Otherwise, the
stride_D = stride_H = stride_W = stride. The default value is 1.
padding(int|str|tuple|list, optional): The padding size. Padding couple be in one of the following forms.
1. a string in ['valid', 'same'].
2. an int, which means each spatial dimension(depth, height, width) is zero padded by size of `padding`
3. a list[int] or tuple[int] whose length is the number of spatial dimensions, which contains the amount of padding on each side for each spatial dimension. It has the form [pad_d1, pad_d2, ...].
4. a list[int] or tuple[int] whose length is 2 * number of spatial dimensions. It has the form [pad_before, pad_after, pad_before, pad_after, ...] for all spatial dimensions.
5. a list or tuple of pairs of ints. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension are also included. Each pair of integers correspond to the amount of padding for a dimension of the input. Padding in batch dimension and channel dimension should be [0, 0] or (0, 0).
The default value is 0.
dilation(int|list|tuple, optional): The dilation size. If dilation is a list/tuple, it must
contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the
dilation_D = dilation_H = dilation_W = dilation. The default value is 1.
groups(int, optional): The groups number of the Conv3D Layer. According to grouped
convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
the first half of the filters is only connected to the first half
of the input channels, while the second half of the filters is only
connected to the second half of the input channels. The default value is 1.
padding_mode(str, optional): ``'zeros'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. Currently only support ``'zeros'``.
key(str, optional): the key is used to save or use the same rulebook,
the definition and role of rulebook refers to
https://pdfs.semanticscholar.org/5125/a16039cabc6320c908a4764f32596e018ad3.pdf. The
default value is None.
weight_attr(ParamAttr, optional): The parameter attribute for learnable parameters/weights
of conv3d. If it is set to None or one attribute of ParamAttr, conv3d
will create ParamAttr as param_attr. If it is set to None, the parameter
is initialized with :math:`Normal(0.0, std)`, and the :math:`std` is
:math:`(\frac{2.0 }{filter\_elem\_num})^{0.5}`. The default value is None.
bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of conv3d.
If it is set to False, no bias will be added to the output units.
If it is set to None or one attribute of ParamAttr, conv3d
will create ParamAttr as bias_attr. If the Initializer of the bias_attr
is not set, the bias is initialized zero. The default value is None.
data_format(str, optional): Data format that specifies the layout of input.
It can be "NCDHW" or "NDHWC". Currently, only support "NCDHW".
Attribute:
**weight** (Parameter): the learnable weights of filters of this layer.
**bias** (Parameter): the learnable bias of this layer.
Shape:
- x: :math:`(N, D_{in}, H_{in}, W_{in}, C_{in})`
- weight: :math:`(K_{d}, K_{h}, K_{w}, C_{in}, C_{out})`
- bias: :math:`(C_{out})`
- output: :math:`(N, D_{out}, H_{out}, W_{out}, C_{out})`
Where
.. math::
D_{out}&= \frac{(D_{in} + 2 * paddings[0] - (dilations[0] * (kernel\_size[0] - 1) + 1))}{strides[0]} + 1
H_{out}&= \frac{(H_{in} + 2 * paddings[1] - (dilations[1] * (kernel\_size[1] - 1) + 1))}{strides[1]} + 1
W_{out}&= \frac{(W_{in} + 2 * paddings[2] - (dilations[2] * (kernel\_size[2] - 1) + 1))}{strides[2]} + 1
Examples:
.. code-block:: pycon
>>> import paddle
>>> indices = [
... [0, 0, 0, 0],
... [0, 0, 0, 0],
... [0, 0, 1, 2],
... [1, 3, 2, 3],
... ]
>>> values = [[1], [2], [3], [4]]
>>> dense_shape = [1, 1, 3, 4, 1]
>>> indices = paddle.to_tensor(indices, dtype='int32')
>>> values = paddle.to_tensor(values, dtype='float32')
>>> sparse_x = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape, stop_gradient=True)
>>> subm_conv = paddle.sparse.nn.SubmConv3D(1, 1, (1, 3, 3))
>>> y = subm_conv(sparse_x)
>>> print(y.shape)
paddle.Size([1, 1, 3, 4, 1])
"""
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: Size3,
stride: Size3 = 1,
padding: _PaddingSizeMode | Size3 | Size6 | Sequence[Size2] = 0,
dilation: Size3 = 1,
groups: Literal[1] = 1,
padding_mode: Literal['zeros'] = 'zeros',
key: str | None = None,
weight_attr: ParamAttrLike | None = None,
bias_attr: ParamAttrLike | None = None,
data_format: Literal["NDHWC"] = "NDHWC",
backend: Literal['igemm'] | None = None,
) -> None:
super().__init__(
in_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding,
dilation=dilation,
groups=groups,
subm=True,
key=key,
padding_mode=padding_mode,
weight_attr=weight_attr,
bias_attr=bias_attr,
data_format=data_format,
backend=backend,
)
class SubmConv2D(_Conv2D):
r"""
**Submanifold Sparse Convolution2d Layer**
The submanifold sparse convolution2d layer calculates the output based on the input, filter
and strides, paddings, dilations, groups parameters. Input(Input) and
Output(Output) are multidimensional SparseCooTensors with a shape of
:math:`[N, H, W, C]` . Where N is batch size, C is the number of
channels, H is the height of the feature,
and W is the width of the feature. If bias attribution is provided,
bias is added to the output of the convolution.
For each input :math:`X`, the equation is:
.. math::
Out = W \ast X + b
In the above equation:
* :math:`X`: Input value, a tensor with NDHWC format.
* :math:`W`: Filter value, a tensor with DHWCM format.
* :math:`\\ast`: Submanifold Convolution operation, refer to the paper: https://arxiv.org/abs/1706.01307.
* :math:`b`: Bias value, a 1-D tensor with shape [M].
* :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different.
Parameters:
in_channels(int): The number of input channels in the input image.
out_channels(int): The number of output channels produced by the convolution.
kernel_size(int|list|tuple): The size of the convolving kernel.
stride(int|list|tuple, optional): The stride size. If stride is a list/tuple, it must
contain two integers, (stride_H, stride_W). Otherwise, the
stride_H = stride_W = stride. The default value is 1.
padding(int|str|tuple|list, optional): The padding size. Padding couple be in one of the following forms.
1. a string in ['valid', 'same'].
2. an int, which means each spatial dimension(depth, height, width) is zero padded by size of `padding`
3. a list[int] or tuple[int] whose length is the number of spatial dimensions, which contains the amount of padding on each side for each spatial dimension. It has the form [pad_d1, pad_d2, ...].
4. a list[int] or tuple[int] whose length is 2 * number of spatial dimensions. It has the form [pad_before, pad_after, pad_before, pad_after, ...] for all spatial dimensions.
5. a list or tuple of pairs of ints. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...].
Note that, the batch dimension and channel dimension are also included. Each pair of integers correspond to the amount of padding for a dimension of the input. Padding in batch dimension and channel dimension should be [0, 0] or (0, 0).
The default value is 0.
dilation(int|list|tuple, optional): The dilation size. If dilation is a list/tuple, it must
contain two integers, (dilation_H, dilation_W). Otherwise, the
dilation_H = dilation_W = dilation. The default value is 1.
groups(int, optional): The groups number of the Conv2D Layer. According to grouped
convolution in Alex Krizhevsky's Deep CNN paper: when group=2,
the first half of the filters is only connected to the first half
of the input channels, while the second half of the filters is only
connected to the second half of the input channels. The default value is 1.
padding_mode(str, optional): ``'zeros'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. Currently only support ``'zeros'``.
key(str, optional): the key is used to save or use the same rulebook,
the definition and role of rulebook refers to
https://pdfs.semanticscholar.org/5125/a16039cabc6320c908a4764f32596e018ad3.pdf. The
default value is None.
weight_attr(ParamAttr, optional): The parameter attribute for learnable parameters/weights
of conv2d. If it is set to None or one attribute of ParamAttr, conv2d
will create ParamAttr as param_attr. If it is set to None, the parameter
is initialized with :math:`Normal(0.0, std)`, and the :math:`std` is
:math:`(\frac{2.0 }{filter\_elem\_num})^{0.5}`. The default value is None.
bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of conv2d.
If it is set to False, no bias will be added to the output units.
If it is set to None or one attribute of ParamAttr, conv2d
will create ParamAttr as bias_attr. If the Initializer of the bias_attr
is not set, the bias is initialized zero. The default value is None.
data_format(str, optional): Data format that specifies the layout of input.
It can be "NCHW" or "NHWC". Currently, only support "NHWC".
Attribute:
**weight** (Parameter): the learnable weights of filters of this layer.
**bias** (Parameter): the learnable bias of this layer.
Shape:
- x: :math:`(N, H_{in}, W_{in}, C_{in})`
- weight: :math:`(K_{h}, K_{w}, C_{in}, C_{out})`
- bias: :math:`(C_{out})`
- output: :math:`(N, H_{out}, W_{out}, C_{out})`
Where
.. math::
H_{out}&= \frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (kernel\_size[0] - 1) + 1))}{strides[0]} + 1
W_{out}&= \frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (kernel\_size[1] - 1) + 1))}{strides[1]} + 1
Examples:
.. code-block:: pycon
>>> import paddle
>>> indices = [[0, 0, 0, 0], [0, 0, 1, 2], [1, 3, 2, 3]]
>>> values = [[1], [2], [3], [4]]
>>> dense_shape = [1, 3, 4, 1]
>>> indices = paddle.to_tensor(indices, dtype='int32')
>>> values = paddle.to_tensor(values, dtype='float32')
>>> sparse_x = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape, stop_gradient=True)
>>> subm_conv = paddle.sparse.nn.SubmConv2D(1, 1, (3, 3))
>>> y = subm_conv(sparse_x)
>>> print(y.shape)
paddle.Size([1, 3, 4, 1])
"""
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: Size2,
stride: Size2 = 1,
padding: _PaddingSizeMode | Size2 | Size4 | Sequence[Size2] = 0,
dilation: Size2 = 1,
groups: Literal[1] = 1,
padding_mode: Literal['zeros'] = 'zeros',
key: str | None = None,
weight_attr: ParamAttrLike | None = None,
bias_attr: ParamAttrLike | None = None,
data_format: Literal["NHWC"] = "NHWC",
backend: Literal['igemm'] | None = None,
) -> None:
super().__init__(
in_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding,
dilation=dilation,
groups=groups,
subm=True,
key=key,
padding_mode=padding_mode,
weight_attr=weight_attr,
bias_attr=bias_attr,
data_format=data_format,
backend=backend,
)
python/paddle/sparse/nn/layer/norm.py
+428
View File
@@ -0,0 +1,428 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Literal
import paddle
from paddle import _C_ops
from paddle.base.layer_helper import LayerHelper
from paddle.framework import in_dynamic_or_pir_mode, no_grad
from paddle.nn.layer.norm import _BatchNormBase
if TYPE_CHECKING:
from paddle import Tensor
from paddle._typing import (
DataLayoutND,
ParamAttrLike,
)
from paddle.nn import Layer
class BatchNorm(paddle.nn.BatchNorm1D):
r"""
Applies Batch Normalization over a SparseCooTensor as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .
When use_global_stats = False, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch.
Calculated as follows:
.. math::
\mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\
\ mini-batch\ mean \\
\sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \
\mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\
When use_global_stats = True, the :math:`\mu_{\beta}`
and :math:`\sigma_{\beta}^{2}` are not the statistics of one mini-batch.
They are global or running statistics (moving_mean and moving_variance). It usually got from the
pre-trained model. Calculated as follows:
.. math::
moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\
The normalization function formula is as follows:
.. math::
\hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
- :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
- :math:`\gamma` : trainable proportional parameter
- :math:`\beta` : trainable deviation parameter
Parameters:
num_features(int): Indicate the number of channels of the input ``Tensor``.
momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm
will create ParamAttr as weight_attr. If it is set to False, the weight is not learnable.
If the Initializer of the weight_attr is not set, the parameter is initialized with Xavier. Default: None.
bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of batch_norm.
If it is set to None or one attribute of ParamAttr, batch_norm
will create ParamAttr as bias_attr. If it is set to False, the weight is not learnable.
If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None.
data_format(str, optional): Specify the input data format, may be "NDHWC" or "NHWC". Default "NDHWC".
use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None.
name(str, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..
Shape:
- x: A SparseCooTensor with layout = 'NDHWC' or 'NHWC'.
- output: SparseCooTensor with same shape as input x.
Returns:
None.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(123)
>>> channels = 3
>>> x_data = paddle.randn((1, 6, 6, 6, channels)).astype('float32')
>>> dense_x = paddle.to_tensor(x_data)
>>> sparse_x = dense_x.to_sparse_coo(4)
>>> batch_norm = paddle.sparse.nn.BatchNorm(channels)
>>> batch_norm_out = batch_norm(sparse_x)
>>> print(batch_norm_out.shape)
paddle.Size([1, 6, 6, 6, 3])
"""
def __init__(
self,
num_features: int,
momentum: float = 0.9,
epsilon: float = 1e-05,
weight_attr: ParamAttrLike | None = None,
bias_attr: ParamAttrLike | None = None,
data_format: Literal["NDHWC", "NHWC"] = "NDHWC",
use_global_stats: bool | None = None,
name: str | None = None,
) -> None:
super().__init__(
num_features,
momentum=momentum,
epsilon=epsilon,
weight_attr=weight_attr,
bias_attr=bias_attr,
data_format=data_format,
use_global_stats=use_global_stats,
name=name,
)
def _check_data_format(self, input: Literal["NDHWC", "NHWC"]) -> None:
if input not in ["NDHWC", "NHWC"]:
raise ValueError(
'sparse BatchNorm only support layout of "NDHWC" and "NHWC"'
)
def forward(self, input: Tensor) -> Tensor:
self._check_data_format(self._data_format)
if self.training:
warnings.warn(
"When training, we now always track global mean and variance."
)
if self._use_global_stats is None:
self._use_global_stats = not self.training
trainable_statistics = False
else:
trainable_statistics = not self._use_global_stats
data_format = 'NCHW' if self._data_format[1] == 'C' else 'NHWC'
if in_dynamic_or_pir_mode():
batch_norm_out, _, _, _, _, _ = _C_ops.sparse_batch_norm_(
input,
self._mean,
self._variance,
self.weight,
self.bias,
not self.training,
self._momentum,
self._epsilon,
data_format,
self._use_global_stats,
trainable_statistics,
)
return batch_norm_out
else:
inputs = {
'x': input,
'scale': self.weight,
'bias': self.bias,
'mean': self._mean,
'variance': self._variance,
}
attrs = {
'momentum': self._momentum,
'epsilon': self._epsilon,
'data_layout': data_format,
'is_test': not self.training,
'use_global_stats': self._use_global_stats,
'trainable_statistics': trainable_statistics,
'fuse_with_relu': False,
}
op_type = 'sparse_batch_norm'
helper = LayerHelper(op_type)
dtype = input.dtype
mean_out = helper.create_variable_for_type_inference(
dtype=dtype, stop_gradient=True
)
variance_out = helper.create_variable_for_type_inference(
dtype=dtype, stop_gradient=True
)
saved_mean = helper.create_variable_for_type_inference(
dtype=dtype, stop_gradient=True
)
saved_variance = helper.create_variable_for_type_inference(
dtype=dtype, stop_gradient=True
)
reserve_space = helper.create_variable_for_type_inference(
dtype=dtype, stop_gradient=True
)
out = helper.create_sparse_variable_for_type_inference(dtype)
outputs = {
"out": out,
"mean_out": mean_out,
"variance_out": variance_out,
"saved_mean": saved_mean,
"saved_variance": saved_variance,
"reserve_space": reserve_space,
}
helper.append_op(
type=op_type, inputs=inputs, outputs=outputs, attrs=attrs
)
return out
class SyncBatchNorm(paddle.nn.SyncBatchNorm):
r"""
This interface is used to construct a callable object of the ``SyncBatchNorm`` class.
It implements the function of the Cross-GPU Synchronized Batch Normalization Layer, and can
be used as a normalizer function for other operations, such as conv2d and fully connected
operations.
The data is normalized by the mean and variance of the channel based on whole mini-batch
, which including data in all gpus.
Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing
Internal Covariate Shift <https://arxiv.org/pdf/1502.03167.pdf>`_
for more details.
When model in training mode, the :math:`\\mu_{\\beta}`
and :math:`\\sigma_{\\beta}^{2}` are the statistics of whole mini-batch data in all gpus.
Calculated as follows:
.. math::
\mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\
\ mini-batch\ mean \\
\sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \
\mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\
- :math:`x` : whole mini-batch data in all gpus
- :math:`m` : the size of the whole mini-batch data
When model in evaluation mode, the :math:`\\mu_{\\beta}`
and :math:`\sigma_{\beta}^{2}` are global statistics (moving_mean and moving_variance,
which usually got from the pre-trained model). Global statistics calculated as follows:
.. math::
moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\
moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\
The formula of normalization is as follows:
.. math::
\hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\
\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\
y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift
- :math:`\epsilon` : add a smaller value to the variance to prevent division by zero
- :math:`\gamma` : trainable scale parameter vector
- :math:`\beta` : trainable shift parameter vector
Note:
If you want to use container to pack your model and has ``SyncBatchNorm`` in the
evaluation phase, please use ``nn.LayerList`` or ``nn.Sequential`` instead of
``list`` to pack the model.
Parameters:
num_features(int): Indicate the number of channels of the input ``Tensor``.
epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale`
of this layer. If it is set to None or one attribute of ParamAttr, this layer
will create ParamAttr as param_attr. If the Initializer of the param_attr
is not set, the parameter is initialized with Xavier. If it is set to False,
this layer will not have trainable scale parameter. Default: None.
bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of this layer.
If it is set to None or one attribute of ParamAttr, this layer
will create ParamAttr as bias_attr. If the Initializer of the bias_attr
is not set, the bias is initialized zero. If it is set to False, this layer will not
have trainable bias parameter. Default: None.
data_format(str, optional): Specify the input data format, may be "NCHW". Default "NCHW".
name(str, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..
Shapes:
input: Tensor that the dimension from 2 to 5.
output: Tensor with the same shape as input.
Examples:
.. code-block:: pycon
>>> # doctest: +REQUIRES(env:GPU)
>>> import paddle
>>> import paddle.sparse.nn as nn
>>> paddle.device.set_device('gpu')
>>> x = paddle.to_tensor([[[[0.3, 0.4], [0.3, 0.07]], [[0.83, 0.37], [0.18, 0.93]]]], dtype='float32')
>>> x = x.to_sparse_coo(len(x.shape)-1)
>>> if paddle.is_compiled_with_cuda():
... sync_batch_norm = nn.SyncBatchNorm(2)
... hidden1 = sync_batch_norm(x)
... print(hidden1)
Tensor(shape=[1, 2, 2, 2], dtype=paddle.float32, place=Place(gpu:0), stop_gradient=False,
indices=[[0, 0, 0, 0],
[0, 0, 1, 1],
[0, 1, 0, 1]],
values=[[-0.40730840, -0.13725480],
[-0.40730840, -1.20299828],
[ 1.69877410, -0.23414057],
[-0.88415730, 1.57439375]])
"""
def __init__(
self,
num_features: int,
momentum: float = 0.9,
epsilon: float = 1e-05,
weight_attr: ParamAttrLike | None = None,
bias_attr: ParamAttrLike | None = None,
data_format: DataLayoutND = 'NCHW',
name: str | None = None,
) -> None:
super().__init__(
num_features,
momentum,
epsilon,
weight_attr,
bias_attr,
data_format,
name,
)
def forward(self, x: Tensor) -> Tensor:
self._check_data_format()
sync_batch_norm_out, _, _, _, _, _ = _C_ops.sparse_sync_batch_norm_(
x,
self._mean,
self._variance,
self.weight,
self.bias,
not self.training,
self._momentum,
self._epsilon,
self._data_format,
False,
False,
)
return sync_batch_norm_out
@classmethod
def convert_sync_batchnorm(cls, layer: Layer) -> Layer:
r"""
Helper function to convert :class: `paddle.sparse.nn.BatchNorm` layers in the model to :class: `paddle.sparse.nn.SyncBatchNorm` layers.
Parameters:
layer(paddle.nn.Layer): model containing one or more `BatchNorm` layers.
Returns:
The original model with converted SyncBatchNorm layers. If BatchNorm layer in the model, use SyncBatchNorm layer instead.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.sparse.nn as nn
>>> model = paddle.nn.Sequential(nn.Conv3D(3, 5, 3), nn.BatchNorm(5))
>>> sync_model = nn.SyncBatchNorm.convert_sync_batchnorm(model)
"""
layer_output = layer
if isinstance(layer, _BatchNormBase):
if (
layer._weight_attr is not None
and not isinstance(layer._weight_attr, bool)
and layer._weight_attr.name is not None
):
layer._weight_attr.name = layer._weight_attr.name + '_sync'
if (
layer._bias_attr is not None
and not isinstance(layer._bias_attr, bool)
and layer._bias_attr.name is not None
):
layer._bias_attr.name = layer._bias_attr.name + '_sync'
# convert sparse BatchNorm
if isinstance(layer, BatchNorm):
layer_output = SyncBatchNorm(
layer._num_features,
layer._momentum,
layer._epsilon,
layer._weight_attr,
layer._bias_attr,
layer._data_format,
layer._name,
)
# convert dense BatchNorm
else:
layer_output = paddle.nn.SyncBatchNorm(
layer._num_features,
layer._momentum,
layer._epsilon,
layer._weight_attr,
layer._bias_attr,
layer._data_format,
layer._name,
)
if (
layer._weight_attr is not False
and layer._bias_attr is not False
):
with no_grad():
layer_output.weight = layer.weight
layer_output.bias = layer.bias
layer_output._mean = layer._mean
layer_output._variance = layer._variance
for name, sublayer in layer.named_children():
layer_output.add_sublayer(
name, cls.convert_sync_batchnorm(sublayer)
)
del layer
return layer_output
python/paddle/sparse/nn/layer/pooling.py
+128
View File
@@ -0,0 +1,128 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from paddle.nn import Layer
from .. import functional as F
if TYPE_CHECKING:
from paddle import Tensor
from paddle._typing import (
DataLayout3D,
Size3,
Size6,
)
from paddle.nn.functional.common import _PaddingSizeMode
class MaxPool3D(Layer):
"""
This operation applies 3D max pooling over input features based on the sparse input,
and kernel_size, stride, padding parameters. Input(X) and Output(Out) are
in NDHWC format, where N is batch size, C is the number of channels,
H is the height of the feature, D is the depth of the feature, and W is the width of the feature.
Parameters:
kernel_size(int|list|tuple): The pool kernel size. If the kernel size
is a tuple or list, it must contain three integers,
(kernel_size_Depth, kernel_size_Height, kernel_size_Width).
Otherwise, the pool kernel size will be the cube of an int.
stride(int|list|tuple, optional): The pool stride size. If pool stride size is a tuple or list,
it must contain three integers, [stride_Depth, stride_Height, stride_Width).
Otherwise, the pool stride size will be a cube of an int.
Default None, then stride will be equal to the kernel_size.
padding(str|int|list|tuple, optional): The padding size. Padding could be in one of the following forms.
1. A string in ['valid', 'same'].
2. An int, which means the feature map is zero padded by size of `padding` on every sides.
3. A list[int] or tuple(int) whose length is 3, [pad_depth, pad_height, pad_weight] whose value means the padding size of each dimension.
4. A list[int] or tuple(int) whose length is \6. [pad_depth_front, pad_depth_back, pad_height_top, pad_height_bottom, pad_width_left, pad_width_right] whose value means the padding size of each side.
5. A list or tuple of pairs of integers. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension should be [0,0] or (0,0).
The default value is 0.
ceil_mode(bool, optional): ${ceil_mode_comment}
return_mask(bool, optional): Whether to return the max indices along with the outputs.
data_format(str, optional): The data format of the input and output data. An optional string from: `"NCDHW"`,
`"NDHWC"`. The default is `"NCDHW"`. When it is `"NCDHW"`, the data is stored in the order of:
`[batch_size, input_channels, input_depth, input_height, input_width]`. Currently, only support "NDHWC".
name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`.
Usually name is no need to set and None by default.
Returns:
A callable object of MaxPool3D.
Shape:
- x(Tensor): The input SparseCooTensor of max pool3d operator, which is a 5-D tensor.
The data type can be float32, float64.
- output(Tensor): The output tensor of max pool3d operator, which is a 5-D tensor.
The data type is same as input x.
Examples:
.. code-block:: pycon
>>> import paddle
>>> dense_x = paddle.randn((2, 3, 6, 6, 3))
>>> sparse_x = dense_x.to_sparse_coo(4)
>>> max_pool3d = paddle.sparse.nn.MaxPool3D(kernel_size=3, data_format='NDHWC')
>>> out = max_pool3d(sparse_x)
>>> print(out.shape)
paddle.Size([2, 1, 2, 2, 3])
"""
kernel_size: Size3
stride: Size3 | None
padding: _PaddingSizeMode | Size3 | Size6
return_mask: bool
ceil_mode: bool
data_format: DataLayout3D
name: str | None
def __init__(
self,
kernel_size: Size3,
stride: Size3 | None = None,
padding: _PaddingSizeMode | Size3 | Size6 = 0,
return_mask: bool = False,
ceil_mode: bool = False,
data_format: DataLayout3D = "NDHWC",
name: str | None = None,
) -> None:
super().__init__()
self.ksize = kernel_size
self.stride = stride
self.padding = padding
self.return_mask = return_mask
self.ceil_mode = ceil_mode
self.data_format = data_format
self.name = name
def forward(self, x: Tensor) -> Tensor:
return F.max_pool3d(
x,
kernel_size=self.ksize,
stride=self.stride,
padding=self.padding,
ceil_mode=self.ceil_mode,
data_format=self.data_format,
name=self.name,
)
def extra_repr(self) -> str:
return 'kernel_size={ksize}, stride={stride}, padding={padding}'.format(
**self.__dict__
)
python/paddle/sparse/unary.py
+1327
View File
File diff suppressed because it is too large Load Diff