chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,920 @@
|
||||
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from math import sqrt
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import paddle
|
||||
from paddle import nn
|
||||
from paddle.nn.modules.utils import _single
|
||||
from paddle.utils.decorator_utils import ForbidKeywordsDecorator
|
||||
|
||||
from . import functional
|
||||
from .transformer import MultiheadAttention
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from paddle import Tensor
|
||||
from paddle._typing import (
|
||||
DTypeLike,
|
||||
PlaceLike,
|
||||
Size1,
|
||||
Size2,
|
||||
Size3,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'Unfold',
|
||||
'Linear',
|
||||
'Softmax',
|
||||
'AvgPool1D',
|
||||
'AvgPool2D',
|
||||
'AvgPool3D',
|
||||
'AvgPool1d',
|
||||
'AvgPool2d',
|
||||
'AvgPool3d',
|
||||
'BatchNorm1D',
|
||||
'BatchNorm2D',
|
||||
'BatchNorm3D',
|
||||
'BatchNorm1d',
|
||||
'BatchNorm2d',
|
||||
'BatchNorm3d',
|
||||
'MultiheadAttention',
|
||||
'SmoothL1Loss',
|
||||
]
|
||||
|
||||
|
||||
class BatchNorm1D(nn.BatchNorm1D):
|
||||
def __init__(
|
||||
self,
|
||||
num_features: int,
|
||||
eps: float = 1e-5,
|
||||
momentum: float | None = 0.1,
|
||||
affine: bool = True,
|
||||
track_running_stats: bool = True,
|
||||
device: PlaceLike | None = None,
|
||||
dtype: DTypeLike | None = None,
|
||||
) -> None:
|
||||
if momentum is None:
|
||||
paddle_momentum = None
|
||||
else:
|
||||
paddle_momentum = 1.0 - momentum
|
||||
super().__init__(
|
||||
num_features=num_features,
|
||||
momentum=paddle_momentum,
|
||||
epsilon=eps,
|
||||
use_global_stats=None if track_running_stats else False,
|
||||
affine=affine,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.momentum = momentum
|
||||
|
||||
|
||||
class BatchNorm2D(nn.BatchNorm2D):
|
||||
def __init__(
|
||||
self,
|
||||
num_features: int,
|
||||
eps: float = 1e-5,
|
||||
momentum: float | None = 0.1,
|
||||
affine: bool = True,
|
||||
track_running_stats: bool = True,
|
||||
device: PlaceLike | None = None,
|
||||
dtype: DTypeLike | None = None,
|
||||
) -> None:
|
||||
if momentum is None:
|
||||
paddle_momentum = None
|
||||
else:
|
||||
paddle_momentum = 1.0 - momentum
|
||||
super().__init__(
|
||||
num_features=num_features,
|
||||
momentum=paddle_momentum,
|
||||
epsilon=eps,
|
||||
use_global_stats=None if track_running_stats else False,
|
||||
affine=affine,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.momentum = momentum
|
||||
|
||||
|
||||
class BatchNorm3D(nn.BatchNorm3D):
|
||||
def __init__(
|
||||
self,
|
||||
num_features: int,
|
||||
eps: float = 1e-5,
|
||||
momentum: float | None = 0.1,
|
||||
affine: bool = True,
|
||||
track_running_stats: bool = True,
|
||||
device: PlaceLike | None = None,
|
||||
dtype: DTypeLike | None = None,
|
||||
) -> None:
|
||||
if momentum is None:
|
||||
paddle_momentum = None
|
||||
else:
|
||||
paddle_momentum = 1.0 - momentum
|
||||
super().__init__(
|
||||
num_features=num_features,
|
||||
momentum=paddle_momentum,
|
||||
epsilon=eps,
|
||||
use_global_stats=None if track_running_stats else False,
|
||||
affine=affine,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.momentum = momentum
|
||||
|
||||
|
||||
BatchNorm1d = BatchNorm1D
|
||||
BatchNorm2d = BatchNorm2D
|
||||
BatchNorm3d = BatchNorm3D
|
||||
|
||||
|
||||
class AvgPool1D(nn.Layer):
|
||||
r"""
|
||||
This operation applies a 1D average pooling over an input signal composed
|
||||
of several input planes, based on the input, output_size, return_mask parameters.
|
||||
Input(X) and output(Out) are in NCL format, where N is batch
|
||||
size, C is the number of channels, L is the length of the feature.
|
||||
The output tensor shape will be [N, C, output_size].
|
||||
|
||||
The output value of the layer with input size (N, C, L),
|
||||
output (N, C, :math:`L_{out}`) and kernel_size ksize can be precisely described as
|
||||
For average pool1d:
|
||||
|
||||
.. math::
|
||||
|
||||
Output(N_i, C_i, l) = \frac{Input[N_i, C_i, stride \times l:stride \times l+k]}{ksize}
|
||||
|
||||
Parameters:
|
||||
kernel_size(int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list,
|
||||
it must contain an integer.
|
||||
stride(int|list|tuple|None, optional): The pool stride size. If pool stride size is a tuple or list,
|
||||
it must contain an integer. Default None, then stride will be equal to the kernel_size.
|
||||
padding(str|int|list|tuple, optional): The padding size. Padding could be in one of the following forms.
|
||||
1. A string in ['valid', 'same'].
|
||||
2. An int, which means the feature map is zero padded by size of `padding` on every sides.
|
||||
3. A list[int] or tuple(int) whose length is 1, which means the feature map is zero padded by the size of `padding[0]` on every sides.
|
||||
4. A list[int] or tuple(int) whose length is 2. It has the form [pad_before, pad_after].
|
||||
5. A list or tuple of pairs of integers. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension should be [0,0] or (0,0).
|
||||
The default value is 0.
|
||||
ceil_mode(bool, optional): ${ceil_mode_comment}Whether to use the ceil function to calculate output height
|
||||
and width. If it is set to False, the floor function will be used. The default value is False.
|
||||
count_include_pad(bool, optional): Whether to include padding points in average pooling mode, default is `False`.
|
||||
|
||||
Shape:
|
||||
- x(Tensor): The input tensor of avg pool1d operator, which is a 3-D tensor.
|
||||
The data type can be float32, float64.
|
||||
- output(Tensor): The output tensor of avg pool1d operator, which is a 3-D tensor.
|
||||
The data type is same as input x.
|
||||
|
||||
Returns:
|
||||
A callable object of AvgPool1D.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> import paddle.compat.nn as nn
|
||||
|
||||
>>> data = paddle.uniform([1, 3, 32], dtype="float32", min=-1, max=1)
|
||||
>>> AvgPool1D = nn.AvgPool1D(kernel_size=2, stride=2, padding=0)
|
||||
>>> pool_out = AvgPool1D(data)
|
||||
>>> print(pool_out.shape)
|
||||
paddle.Size([1, 3, 16])
|
||||
|
||||
"""
|
||||
|
||||
__constants__ = [
|
||||
"kernel_size",
|
||||
"stride",
|
||||
"padding",
|
||||
"ceil_mode",
|
||||
"count_include_pad",
|
||||
]
|
||||
|
||||
kernel_size: Size1
|
||||
stride: Size1
|
||||
padding: Size1
|
||||
ceil_mode: bool
|
||||
count_include_pad: bool
|
||||
|
||||
@ForbidKeywordsDecorator(
|
||||
illegal_keys={"exclusive", "name"},
|
||||
func_name="paddle.compat.nn.AvgPool1D",
|
||||
correct_name="paddle.nn.AvgPool1D",
|
||||
)
|
||||
def __init__(
|
||||
self,
|
||||
kernel_size: Size1,
|
||||
stride: Size1 | None = None,
|
||||
padding: Size1 = 0,
|
||||
ceil_mode: bool = False,
|
||||
count_include_pad: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.kernel_size = _single(kernel_size)
|
||||
self.stride = _single(stride if stride is not None else kernel_size)
|
||||
self.padding = _single(padding)
|
||||
self.ceil_mode = ceil_mode
|
||||
self.count_include_pad = count_include_pad
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return nn.functional.avg_pool1d(
|
||||
input,
|
||||
self.kernel_size,
|
||||
self.stride,
|
||||
self.padding,
|
||||
not self.count_include_pad,
|
||||
self.ceil_mode,
|
||||
)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"kernel_size={self.kernel_size}, stride={self.stride}, padding={self.padding}"
|
||||
|
||||
|
||||
class AvgPool2D(nn.Layer):
|
||||
r"""
|
||||
This operation applies 2D average pooling over input features based on the input,
|
||||
and kernel_size, stride, padding parameters. Input(X) and Output(Out) are
|
||||
in NCHW format, where N is batch size, C is the number of channels,
|
||||
H is the height of the feature, and W is the width of the feature.
|
||||
|
||||
Example:
|
||||
Input:
|
||||
X shape: :math:`(N, C, :math:`H_{in}`, :math:`W_{in}`)`
|
||||
Attr:
|
||||
kernel_size: ksize
|
||||
|
||||
Output:
|
||||
Out shape: :math:`(N, C, :math:`H_{out}`, :math:`W_{out}`)`
|
||||
|
||||
.. math::
|
||||
|
||||
Output(N_i, C_j, h, w) = \frac{\sum_{m=0}^{ksize[0]-1} \sum_{n=0}^{ksize[1]-1}
|
||||
Input(N_i, C_j, stride[0] \times h + m, stride[1] \times w + n)}{ksize[0] * ksize[1]}
|
||||
|
||||
|
||||
Parameters:
|
||||
kernel_size(int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list,
|
||||
it must contain two integers, (pool_size_Height, pool_size_Width).
|
||||
Otherwise, the pool kernel size will be a square of an int.
|
||||
stride(int|list|tuple|None, optional): The pool stride size. If pool stride size is a tuple or list,
|
||||
it must contain two integers, (pool_stride_Height, pool_stride_Width).
|
||||
Otherwise, the pool stride size will be a square of an int.
|
||||
Default None, then stride will be equal to the kernel_size.
|
||||
padding(str|int|list|tuple, optional): The padding size. Padding could be in one of the following forms.
|
||||
1. A string in ['valid', 'same'].
|
||||
2. An int, which means the feature map is zero padded by size of `padding` on every sides.
|
||||
3. A list[int] or tuple(int) whose length is 2, [pad_height, pad_weight] whose value means the padding size of each dimension.
|
||||
4. A list[int] or tuple(int) whose length is 4. [pad_height_top, pad_height_bottom, pad_width_left, pad_width_right] whose value means the padding size of each side.
|
||||
5. A list or tuple of pairs of integers. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension should be [0,0] or (0,0).
|
||||
The default value is 0.
|
||||
ceil_mode(bool, optional): When True, will use `ceil` instead of `floor` to compute the output shape.
|
||||
count_include_pad(bool, optional): Whether to include padding points in average pooling
|
||||
mode, default is `False`.
|
||||
divisor_override(float, optional): If specified, it will be used as divisor, otherwise kernel_size will be
|
||||
used. Default None.
|
||||
|
||||
Shape:
|
||||
- x(Tensor): The input tensor of avg pool2d operator, which is a 4-D tensor.
|
||||
The data type can be float32, float64.
|
||||
- output(Tensor): The output tensor of avg pool2d operator, which is a 4-D tensor.
|
||||
The data type is same as input x.
|
||||
|
||||
Returns:
|
||||
A callable object of AvgPool2D.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> import paddle.compat.nn as nn
|
||||
|
||||
>>> # max pool2d
|
||||
>>> input = paddle.uniform([1, 3, 32, 32], dtype="float32", min=-1, max=1)
|
||||
>>> AvgPool2D = nn.AvgPool2D(kernel_size=2, stride=2, padding=0)
|
||||
>>> output = AvgPool2D(input)
|
||||
>>> print(output.shape)
|
||||
paddle.Size([1, 3, 16, 16])
|
||||
|
||||
"""
|
||||
|
||||
__constants__ = [
|
||||
"kernel_size",
|
||||
"stride",
|
||||
"padding",
|
||||
"ceil_mode",
|
||||
"count_include_pad",
|
||||
"divisor_override",
|
||||
]
|
||||
|
||||
kernel_size: Size2
|
||||
stride: Size2
|
||||
padding: Size2
|
||||
ceil_mode: bool
|
||||
count_include_pad: bool
|
||||
divisor_override: int | None
|
||||
|
||||
@ForbidKeywordsDecorator(
|
||||
illegal_keys={"exclusive", "data_format", "name"},
|
||||
func_name="paddle.compat.nn.AvgPool2D",
|
||||
correct_name="paddle.nn.AvgPool2D",
|
||||
)
|
||||
def __init__(
|
||||
self,
|
||||
kernel_size: Size2,
|
||||
stride: Size2 | None = None,
|
||||
padding: Size2 = 0,
|
||||
ceil_mode: bool = False,
|
||||
count_include_pad: bool = True,
|
||||
divisor_override: int | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.kernel_size = kernel_size
|
||||
self.stride = stride if (stride is not None) else kernel_size
|
||||
self.padding = padding
|
||||
self.ceil_mode = ceil_mode
|
||||
self.count_include_pad = count_include_pad
|
||||
self.divisor_override = divisor_override
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return nn.functional.avg_pool2d(
|
||||
input,
|
||||
self.kernel_size,
|
||||
self.stride,
|
||||
self.padding,
|
||||
self.ceil_mode,
|
||||
not self.count_include_pad,
|
||||
self.divisor_override,
|
||||
)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"kernel_size={self.kernel_size}, stride={self.stride}, padding={self.padding}"
|
||||
|
||||
|
||||
class AvgPool3D(nn.Layer):
|
||||
"""
|
||||
|
||||
This operation applies 3D max pooling over input features based on the input,
|
||||
and kernel_size, stride, padding parameters. Input(X) and Output(Out) are
|
||||
in NCDHW format, where N is batch size, C is the number of channels,
|
||||
H is the height of the feature, D is the depth of the feature, and W is the width of the feature.
|
||||
|
||||
Parameters:
|
||||
kernel_size(int|list|tuple): The pool kernel size. If pool kernel size
|
||||
is a tuple or list, it must contain three integers,
|
||||
(kernel_size_Depth, kernel_size_Height, kernel_size_Width).
|
||||
Otherwise, the pool kernel size will be the cube of an int.
|
||||
stride(int|list|tuple|None, optional): The pool stride size. If pool stride size is a tuple or list,
|
||||
it must contain three integers, [stride_Depth, stride_Height, stride_Width).
|
||||
Otherwise, the pool stride size will be a cube of an int.
|
||||
Default None, then stride will be equal to the kernel_size.
|
||||
padding(str|int|list|tuple, optional): The padding size. Padding could be in one of the following forms.
|
||||
|
||||
1. A string in ['valid', 'same'].
|
||||
2. An int, which means the feature map is zero padded by size of `padding` on every sides.
|
||||
3. A list[int] or tuple(int) whose length is 3, [pad_depth, pad_height, pad_weight] whose value means the padding size of each dimension.
|
||||
4. A list[int] or tuple(int) whose length is 6. [pad_depth_front, pad_depth_back, pad_height_top, pad_height_bottom, pad_width_left, pad_width_right] whose value means the padding size of each side.
|
||||
5. A list or tuple of pairs of integers. It has the form [[pad_before, pad_after], [pad_before, pad_after], ...]. Note that, the batch dimension and channel dimension should be [0,0] or (0,0).
|
||||
|
||||
The default value is 0.
|
||||
ceil_mode(bool, optional): ${ceil_mode_comment}
|
||||
count_include_pad(bool, optional): Whether to include padding points in average pooling mode, default is True.
|
||||
divisor_override(int|float, optional): if specified, it will be used as divisor, otherwise kernel_size will
|
||||
be used. Default None.
|
||||
|
||||
Returns:
|
||||
A callable object of AvgPool3D.
|
||||
|
||||
Shape:
|
||||
- x(Tensor): The input tensor of avg pool3d operator, which is a 5-D tensor.
|
||||
The data type can be float16, float32, float64.
|
||||
- output(Tensor): The output tensor of avg pool3d operator, which is a 5-D tensor.
|
||||
The data type is same as input x.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> import paddle.compat.nn as nn
|
||||
|
||||
>>> # avg pool3d
|
||||
>>> input = paddle.uniform([1, 2, 3, 32, 32], dtype="float32", min=-1, max=1)
|
||||
>>> AvgPool3D = nn.AvgPool3D(kernel_size=2, stride=2, padding=0)
|
||||
>>> output = AvgPool3D(input)
|
||||
>>> print(output.shape)
|
||||
paddle.Size([1, 2, 1, 16, 16])
|
||||
|
||||
"""
|
||||
|
||||
__constants__ = [
|
||||
"kernel_size",
|
||||
"stride",
|
||||
"padding",
|
||||
"ceil_mode",
|
||||
"count_include_pad",
|
||||
"divisor_override",
|
||||
]
|
||||
kernel_size: Size3
|
||||
stride: Size3
|
||||
padding: Size3
|
||||
ceil_mode: bool
|
||||
count_include_pad: bool
|
||||
divisor_override: int | None
|
||||
|
||||
@ForbidKeywordsDecorator(
|
||||
illegal_keys={"exclusive", "data_format", "name"},
|
||||
func_name="paddle.compat.nn.AvgPool3D",
|
||||
correct_name="paddle.nn.AvgPool3D",
|
||||
)
|
||||
def __init__(
|
||||
self,
|
||||
kernel_size: Size3,
|
||||
stride: Size3 | None = None,
|
||||
padding: Size3 = 0,
|
||||
ceil_mode: bool = False,
|
||||
count_include_pad: bool = True,
|
||||
divisor_override: int | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.kernel_size = kernel_size
|
||||
self.stride = stride if (stride is not None) else kernel_size
|
||||
self.padding = padding
|
||||
self.ceil_mode = ceil_mode
|
||||
self.count_include_pad = count_include_pad
|
||||
self.divisor_override = divisor_override
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return nn.functional.avg_pool3d(
|
||||
input,
|
||||
self.kernel_size,
|
||||
self.stride,
|
||||
self.padding,
|
||||
self.ceil_mode,
|
||||
not self.count_include_pad,
|
||||
self.divisor_override,
|
||||
)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"kernel_size={self.kernel_size}, stride={self.stride}, padding={self.padding}"
|
||||
|
||||
def __setstate__(self, state):
|
||||
super().__setstate__(state)
|
||||
self.__dict__.setdefault("padding", 0)
|
||||
self.__dict__.setdefault("ceil_mode", False)
|
||||
self.__dict__.setdefault("count_include_pad", True)
|
||||
|
||||
|
||||
class Unfold(nn.Unfold):
|
||||
"""
|
||||
A compatible version of paddle.nn.Unfold:
|
||||
|
||||
The keyword arguments are in non-plural forms, example: `kernel_size` instead of `kernel_sizes`. `padding` restricts the size of the input to be 1(int) or 2, Size4 is not allowed.
|
||||
|
||||
All the input parameters allow `Tensor` or `pir.Value` as inputs, and will be converted to lists. Other aspects are the same. To use a more input-flexible version of Unfold, please refer to `paddle.nn.Unfold`.
|
||||
|
||||
Args:
|
||||
kernel_size(int|list|tuple|Tensor): The size of convolution kernel, should be [k_h, k_w]
|
||||
or an integer k treated as [k, k].
|
||||
stride(int|list|tuple|Tensor, optional): The strides, should be [stride_h, stride_w]
|
||||
or an integer stride treated as [sride, stride]. For default, strides will be [1, 1].
|
||||
padding(int|list|tuple|Tensor, optional): The paddings of each dimension, should be
|
||||
a single integer or [padding_h, padding_w]. If [padding_h, padding_w] was given, it will expanded to
|
||||
[padding_h, padding_w, padding_h, padding_w]. If an integer padding was given,
|
||||
[padding, padding, padding, padding] will be used. By default, paddings will be 0.
|
||||
dilation(int|list|tuple|Tensor, optional): The dilations of convolution kernel, should be
|
||||
[dilation_h, dilation_w], or an integer dilation treated as [dilation, dilation].
|
||||
For default, it will be [1, 1].
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> x = paddle.randn((100, 3, 224, 224))
|
||||
>>> unfold = paddle.compat.nn.Unfold(kernel_size=[3, 3])
|
||||
>>> result = unfold(x)
|
||||
>>> print(result.shape)
|
||||
paddle.Size([100, 27, 49284])
|
||||
"""
|
||||
|
||||
kernel_sizes: Size2
|
||||
dilations: Size2
|
||||
paddings: Size2
|
||||
strides: Size2
|
||||
|
||||
@ForbidKeywordsDecorator(
|
||||
illegal_keys={"kernel_sizes", "dilations", "paddings", "strides"},
|
||||
func_name="paddle.compat.nn.Unfold",
|
||||
correct_name="paddle.nn.Unfold",
|
||||
)
|
||||
def __init__(
|
||||
self,
|
||||
kernel_size: Size2,
|
||||
dilation: Size2 = 1,
|
||||
padding: Size2 = 0,
|
||||
stride: Size2 = 1,
|
||||
) -> None:
|
||||
super().__init__(kernel_size, dilation, padding, stride)
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
def to_list_if_necessary(x):
|
||||
if isinstance(x, (paddle.pir.Value, paddle.Tensor)):
|
||||
x = x.tolist()
|
||||
return x
|
||||
|
||||
return nn.functional.unfold(
|
||||
input,
|
||||
kernel_sizes=to_list_if_necessary(self.kernel_sizes),
|
||||
strides=to_list_if_necessary(self.strides),
|
||||
paddings=to_list_if_necessary(self.paddings),
|
||||
dilations=to_list_if_necessary(self.dilations),
|
||||
)
|
||||
|
||||
|
||||
class Linear(nn.Layer):
|
||||
r"""
|
||||
|
||||
Python compatible fully-connected linear transformation layer. For each input :math:`X` ,
|
||||
the equation is:
|
||||
|
||||
.. math::
|
||||
|
||||
Out = XW^T + b
|
||||
|
||||
where :math:`W` is the weight and :math:`b` is the bias.
|
||||
|
||||
Linear layer takes only one multi-dimensional tensor as input with the
|
||||
shape :math:`[*, in\_features]` , where :math:`*` means any
|
||||
number of additional dimensions. It multiplies input tensor with the transpose
|
||||
of weight (a 2-D tensor of shape :math:`[out\_features, in\_features]` ) and
|
||||
produces an output tensor of shape :math:`[*, out\_features]` .
|
||||
If ``bias`` is not False, the bias (a 1-D tensor of
|
||||
shape :math:`[out\_features]` ) will be created and added to the output. At the
|
||||
end of the initialization, ``reset_parameters`` will be called to initialize
|
||||
the ``weight`` and ``bias`` (if available) randomly.
|
||||
|
||||
Parameters:
|
||||
in_features (int):
|
||||
The number of input units.
|
||||
out_features (int):
|
||||
The number of output units.
|
||||
bias (bool): If True, the bias (a 1-D tensor of shape :math:`[out\_features]` ) will be created and
|
||||
added to the output. Default: True.
|
||||
device (PlaceLike): The device of the parameters created. Default: None,
|
||||
representing the default paddle device.
|
||||
dtype (DTypeLike): The dtype of the parameters created. Default: None, and is set by
|
||||
the default dtype of Linear (float32).
|
||||
|
||||
Variables:
|
||||
weight (paddle.Tensor): learnable parameters of the module of shape :math:`[out\_features, in\_features]`.
|
||||
The values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`, where :math:`k` is :math:`\frac{1}{in\_features}`.
|
||||
bias (paddle.Tensor): learnable parameters of the module of shape :math:`[out\_features]`. If ``bias`` is True,
|
||||
the values are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`, where :math:`k` is :math:`\frac{1}{in\_features}`.
|
||||
|
||||
Shape:
|
||||
- input: Multi-dimensional tensor with shape :math:`[*, in\_features]` . Its data types are float16, float32, float64 ,The default is float32 .
|
||||
- output: Multi-dimensional tensor with shape :math:`[*, out\_features]` . The data type is the same as the input .
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> paddle.seed(100)
|
||||
|
||||
>>> # Define the linear layer.
|
||||
>>> linear = paddle.compat.nn.Linear(2, 4, bias=True)
|
||||
>>> print(linear.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[4, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[-0.49191639, 0.28120756],
|
||||
[-0.17887023, 0.40572405],
|
||||
[ 0.35139430, 0.45717543],
|
||||
[-0.06135514, -0.21088189]])
|
||||
|
||||
>>> print(linear.bias)
|
||||
Parameter containing:
|
||||
Tensor(shape=[4], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[ 0.49166456, -0.06108528, -0.14973064, 0.31168410])
|
||||
|
||||
>>> x = paddle.arange(6, dtype="float32").reshape([3, 2])
|
||||
>>> y = linear(x)
|
||||
>>> print(y)
|
||||
Tensor(shape=[3, 4], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[ 0.77287209, 0.34463876, 0.30744481, 0.10080221],
|
||||
[ 0.35145447, 0.79834640, 1.92458415, -0.44367185],
|
||||
[-0.06996319, 1.25205410, 3.54172373, -0.98814595]])
|
||||
"""
|
||||
|
||||
__constants__ = ["in_features", "out_features"]
|
||||
in_features: int
|
||||
out_features: int
|
||||
weight: Tensor
|
||||
|
||||
@ForbidKeywordsDecorator(
|
||||
illegal_keys={"weight_attr", "bias_attr", "name"},
|
||||
func_name="paddle.compat.nn.Linear",
|
||||
correct_name="paddle.nn.Linear",
|
||||
)
|
||||
def __init__(
|
||||
self,
|
||||
in_features: int,
|
||||
out_features: int,
|
||||
bias: bool = True,
|
||||
device: PlaceLike | None = None,
|
||||
dtype: DTypeLike | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._dtype = (
|
||||
self._helper.get_default_dtype() if dtype is None else dtype
|
||||
)
|
||||
self.in_features = in_features
|
||||
self.out_features = out_features
|
||||
self.weight = self.create_parameter(
|
||||
shape=[out_features, in_features],
|
||||
attr=None,
|
||||
dtype=self._dtype,
|
||||
is_bias=False,
|
||||
device=device,
|
||||
)
|
||||
self.bias = None
|
||||
if bias:
|
||||
self.bias = self.create_parameter(
|
||||
shape=[out_features],
|
||||
attr=None,
|
||||
dtype=self._dtype,
|
||||
is_bias=True,
|
||||
device=device,
|
||||
)
|
||||
# The same parameter initialization as PyTorch
|
||||
self.reset_parameters()
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return functional.linear.__wrapped__( # bypass ForbidKeywordsDecorator
|
||||
input=input, weight=self.weight, bias=self.bias
|
||||
)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
"""
|
||||
Return the extra representation of the module.
|
||||
"""
|
||||
return f"in_features={self.in_features}, out_features={self.out_features}, bias={self.bias is not None}"
|
||||
|
||||
def reset_parameters(self) -> None:
|
||||
"""
|
||||
Resets parameters based on their initialization used in ``__init__``.
|
||||
"""
|
||||
|
||||
bound = 1 / sqrt(self.in_features) if self.in_features > 0 else 0
|
||||
if self.in_features > 0 and self.out_features > 0:
|
||||
nn.init.uniform_(self.weight, -bound, bound)
|
||||
if self.bias is not None and self.out_features > 0:
|
||||
nn.init.uniform_(self.bias, -bound, bound)
|
||||
|
||||
|
||||
class Softmax(nn.Layer):
|
||||
r"""
|
||||
Softmax Activation.
|
||||
|
||||
This operator implements the softmax layer. The calculation process is as follows:
|
||||
|
||||
1. The dimension :attr:`dim` of ``input`` will be permuted to the last.
|
||||
|
||||
2. Then ``input`` will be logically flattened to a 2-D matrix. The matrix's second
|
||||
dimension(row length) is the same as the dimension :attr:`dim` of ``input``,
|
||||
and the first dimension(column length) is the product of all other dimensions
|
||||
of ``input``. For each row of the matrix, the softmax operator squashes the
|
||||
K-dimensional(K is the width of the matrix, which is also the size of ``input``'s
|
||||
dimension :attr:`dim`) vector of arbitrary real values to a K-dimensional
|
||||
vector of real values in the range [0, 1] that add up to 1.
|
||||
|
||||
3. After the softmax operation is completed, the inverse operations of steps 1 and 2
|
||||
are performed to restore the two-dimensional matrix to the same dimension as the ``input`` .
|
||||
|
||||
It computes the exponential of the given dimension and the sum of exponential
|
||||
values of all the other dimensions in the K-dimensional vector input.
|
||||
Then the ratio of the exponential of the given dimension and the sum of
|
||||
exponential values of all the other dimensions is the output of the softmax
|
||||
operator.
|
||||
|
||||
For each row :math:`i` and each column :math:`j` in the matrix, we have:
|
||||
|
||||
.. math::
|
||||
|
||||
Softmax[i, j] = \frac{\exp(x[i, j])}{\sum_j(exp(x[i, j])}
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
Case 1:
|
||||
Input:
|
||||
x.shape = [2, 3, 4]
|
||||
x.data = [[[2.0, 3.0, 4.0, 5.0],
|
||||
[3.0, 4.0, 5.0, 6.0],
|
||||
[7.0, 8.0, 8.0, 9.0]],
|
||||
[[1.0, 2.0, 3.0, 4.0],
|
||||
[5.0, 6.0, 7.0, 8.0],
|
||||
[6.0, 7.0, 8.0, 9.0]]]
|
||||
|
||||
Attrs:
|
||||
dim = -1
|
||||
|
||||
Output:
|
||||
out.shape = [2, 3, 4]
|
||||
out.data = [[[0.0320586 , 0.08714432, 0.23688282, 0.64391426],
|
||||
[0.0320586 , 0.08714432, 0.23688282, 0.64391426],
|
||||
[0.07232949, 0.19661193, 0.19661193, 0.53444665]],
|
||||
[[0.0320586 , 0.08714432, 0.23688282, 0.64391426],
|
||||
[0.0320586 , 0.08714432, 0.23688282, 0.64391426],
|
||||
[0.0320586 , 0.08714432, 0.23688282, 0.64391426]]]
|
||||
|
||||
Case 2:
|
||||
Input:
|
||||
x.shape = [2, 3, 4]
|
||||
x.data = [[[2.0, 3.0, 4.0, 5.0],
|
||||
[3.0, 4.0, 5.0, 6.0],
|
||||
[7.0, 8.0, 8.0, 9.0]],
|
||||
[[1.0, 2.0, 3.0, 4.0],
|
||||
[5.0, 6.0, 7.0, 8.0],
|
||||
[6.0, 7.0, 8.0, 9.0]]]
|
||||
Attrs:
|
||||
dim = 1
|
||||
|
||||
Output:
|
||||
out.shape = [2, 3, 4]
|
||||
out.data = [[[0.00657326, 0.00657326, 0.01714783, 0.01714783],
|
||||
[0.01786798, 0.01786798, 0.04661262, 0.04661262],
|
||||
[0.97555875, 0.97555875, 0.93623955, 0.93623955]],
|
||||
[[0.00490169, 0.00490169, 0.00490169, 0.00490169],
|
||||
[0.26762315, 0.26762315, 0.26762315, 0.26762315],
|
||||
[0.72747516, 0.72747516, 0.72747516, 0.72747516]]]
|
||||
|
||||
Parameters:
|
||||
dim (int, optional): The dim along which to perform log_softmax
|
||||
calculations. It should be in range [-D, D), where D is the
|
||||
dimensions of ``input`` . If ``dim`` < 0, it works the same way as
|
||||
:math:`dim + D` . Default is None.
|
||||
|
||||
Shape:
|
||||
- input: Tensor with any shape.
|
||||
- output: Tensor with the same shape as input.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
|
||||
>>> x = paddle.to_tensor(
|
||||
... [
|
||||
... [
|
||||
... [2.0, 3.0, 4.0, 5.0],
|
||||
... [3.0, 4.0, 5.0, 6.0],
|
||||
... [7.0, 8.0, 8.0, 9.0],
|
||||
... ],
|
||||
... [
|
||||
... [1.0, 2.0, 3.0, 4.0],
|
||||
... [5.0, 6.0, 7.0, 8.0],
|
||||
... [6.0, 7.0, 8.0, 9.0],
|
||||
... ],
|
||||
... ],
|
||||
... dtype='float32',
|
||||
... )
|
||||
>>> m = paddle.compat.nn.Softmax()
|
||||
>>> out = m(x)
|
||||
>>> print(out)
|
||||
Tensor(shape=[2, 3, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
|
||||
[[[0.73105854, 0.73105854, 0.73105854, 0.73105854],
|
||||
[0.11920292, 0.11920292, 0.11920292, 0.11920292],
|
||||
[0.73105854, 0.73105854, 0.50000000, 0.50000000]],
|
||||
[[0.26894143, 0.26894143, 0.26894143, 0.26894143],
|
||||
[0.88079703, 0.88079703, 0.88079703, 0.88079703],
|
||||
[0.26894143, 0.26894143, 0.50000000, 0.50000000]]])
|
||||
|
||||
"""
|
||||
|
||||
@ForbidKeywordsDecorator(
|
||||
illegal_keys={"axis"},
|
||||
func_name="paddle.compat.nn.Softmax",
|
||||
correct_name="paddle.nn.Softmax",
|
||||
)
|
||||
def __init__(self, dim: int | None = None) -> None:
|
||||
super().__init__()
|
||||
self._dim = dim
|
||||
self._dtype = None
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return functional.softmax(input, self._dim)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"dim={self._dim}"
|
||||
|
||||
|
||||
class SmoothL1Loss(nn.Layer):
|
||||
r"""
|
||||
|
||||
PyTorch compatible version of :ref:`api_paddle_nn_SmoothL1Loss`, aligned with
|
||||
``torch.nn.SmoothL1Loss``. The per-element loss is
|
||||
|
||||
.. math::
|
||||
|
||||
z_i = \left\{\begin{array}{rcl}
|
||||
0.5 (x_i - y_i)^2 / beta & & {if |x_i - y_i| < beta} \\
|
||||
|x_i - y_i| - 0.5 * beta & & {otherwise}
|
||||
\end{array} \right.
|
||||
|
||||
which equals Paddle's Huber loss divided by ``beta``. This differs from
|
||||
:ref:`api_paddle_nn_SmoothL1Loss` whose default ``is_huber=True`` returns the
|
||||
raw Huber loss.
|
||||
|
||||
Parameters:
|
||||
size_average (bool|None, optional): Deprecated (see ``reduction``). When
|
||||
``size_average`` or ``reduce`` is not ``None``, it is translated into
|
||||
``reduction`` with a ``DeprecationWarning``. Default is ``None``.
|
||||
reduce (bool|None, optional): Deprecated (see ``reduction``). Default is ``None``.
|
||||
reduction (str, optional): Indicate how to calculate the loss, the candidates
|
||||
are ``'none'`` | ``'mean'`` | ``'sum'``. Default is ``'mean'``.
|
||||
beta (float, optional): Non-negative threshold at which to change between L1
|
||||
and L2 loss. When ``beta == 0`` the loss degrades to the L1 loss, matching
|
||||
PyTorch. Default is ``1.0``.
|
||||
|
||||
Call Parameters:
|
||||
input (Tensor): Input tensor, the data type is float32 or float64.
|
||||
target (Tensor): Label tensor with the same shape as ``input``.
|
||||
|
||||
Returns:
|
||||
Tensor, The tensor storing the smooth L1 loss of ``input`` and ``target``.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
|
||||
>>> input = paddle.to_tensor([[0.5, 1.5], [2.0, 0.0]], dtype='float32')
|
||||
>>> target = paddle.to_tensor([[1.0, 1.0], [1.0, 0.5]], dtype='float32')
|
||||
>>> loss = paddle.compat.nn.SmoothL1Loss(beta=1.0)
|
||||
>>> output = loss(input, target)
|
||||
>>> print(output)
|
||||
Tensor(shape=[], dtype=float32, place=Place(cpu), stop_gradient=True,
|
||||
0.21875000)
|
||||
"""
|
||||
|
||||
__constants__ = ["reduction", "beta"]
|
||||
reduction: str
|
||||
beta: float
|
||||
|
||||
@ForbidKeywordsDecorator(
|
||||
illegal_keys={"delta", "is_huber", "name", "label"},
|
||||
func_name="paddle.compat.nn.SmoothL1Loss",
|
||||
correct_name="paddle.nn.SmoothL1Loss",
|
||||
)
|
||||
def __init__(
|
||||
self,
|
||||
size_average: bool | None = None,
|
||||
reduce: bool | None = None,
|
||||
reduction: str = 'mean',
|
||||
beta: float = 1.0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if size_average is not None or reduce is not None:
|
||||
reduction = (
|
||||
'none'
|
||||
if reduce is False
|
||||
else ('sum' if size_average is False else 'mean')
|
||||
)
|
||||
warnings.warn(
|
||||
"'size_average' and 'reduce' args of 'SmoothL1Loss' will be "
|
||||
f"deprecated, please use reduction='{reduction}' instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self.reduction = reduction
|
||||
self.beta = beta
|
||||
|
||||
def forward(self, input: Tensor, target: Tensor) -> Tensor:
|
||||
return functional.smooth_l1_loss.__wrapped__(
|
||||
input, target, reduction=self.reduction, beta=self.beta
|
||||
)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"reduction={self.reduction}, beta={self.beta}"
|
||||
|
||||
|
||||
AvgPool1d = AvgPool1D
|
||||
AvgPool2d = AvgPool2D
|
||||
AvgPool3d = AvgPool3D
|
||||
Reference in New Issue
Block a user