wehub/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/paddle/nn/__init__.py
+439
View File
@@ -0,0 +1,439 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import ( # noqa: F401
attention,
functional,
init,
initializer,
quant,
utils,
)
from .clip import ClipGradByGlobalNorm, ClipGradByNorm, ClipGradByValue
from .decode import BeamSearchDecoder, dynamic_decode
# TODO: remove loss, keep it for too many used in unittests
from .layer import loss # noqa: F401
from .layer.activation import (
CELU,
ELU,
GELU,
GLU,
SELU,
Hardshrink,
Hardsigmoid,
Hardswish,
Hardtanh,
LeakyReLU,
LogSigmoid,
LogSoftmax,
Maxout,
Mish,
PReLU,
ReLU,
ReLU6,
RReLU,
Sigmoid,
Silu,
Softmax,
Softmax2D,
Softplus,
Softshrink,
Softsign,
Swish,
Tanh,
Tanhshrink,
ThresholdedReLU,
)
from .layer.common import (
AlphaDropout,
Bilinear,
CircularPad1D,
CircularPad2D,
CircularPad3D,
ConstantPad1D,
ConstantPad2D,
ConstantPad3D,
CosineSimilarity,
Dropout,
Dropout2D,
Dropout3D,
Embedding,
FeatureAlphaDropout,
Flatten,
Fold,
Identity,
Linear,
Pad1D,
Pad2D,
Pad3D,
ReflectionPad1D,
ReflectionPad2D,
ReflectionPad3D,
ReplicationPad1D,
ReplicationPad2D,
ReplicationPad3D,
Unflatten,
Unfold,
Upsample,
UpsamplingBilinear2D,
UpsamplingNearest2D,
ZeroPad1D,
ZeroPad2D,
ZeroPad3D,
)
# TODO: import all neural network related api under this directory,
# including layers, linear, conv, rnn etc.
from .layer.container import (
LayerDict,
LayerList,
ParameterDict,
ParameterList,
Sequential,
)
from .layer.conv import (
Conv1D,
Conv1DTranspose,
Conv2D,
Conv2DTranspose,
Conv3D,
Conv3DTranspose,
)
from .layer.distance import PairwiseDistance
from .layer.layers import Layer
from .layer.loss import (
AdaptiveLogSoftmaxWithLoss,
BCELoss,
BCEWithLogitsLoss,
CosineEmbeddingLoss,
CrossEntropyLoss,
CTCLoss,
GaussianNLLLoss,
HingeEmbeddingLoss,
HSigmoidLoss,
KLDivLoss,
L1Loss,
MarginRankingLoss,
MSELoss,
MultiLabelMarginLoss,
MultiLabelSoftMarginLoss,
MultiMarginLoss,
NLLLoss,
PoissonNLLLoss,
RNNTLoss,
SmoothL1Loss,
SoftMarginLoss,
TripletMarginLoss,
TripletMarginWithDistanceLoss,
)
from .layer.norm import (
BatchNorm,
BatchNorm1D,
BatchNorm2D,
BatchNorm3D,
GroupNorm,
InstanceNorm1D,
InstanceNorm2D,
InstanceNorm3D,
LayerNorm,
LocalResponseNorm,
SpectralNorm,
SyncBatchNorm,
)
from .layer.pooling import (
AdaptiveAvgPool1D,
AdaptiveAvgPool2D,
AdaptiveAvgPool3D,
AdaptiveMaxPool1D,
AdaptiveMaxPool2D,
AdaptiveMaxPool3D,
AvgPool1D,
AvgPool2D,
AvgPool3D,
FractionalMaxPool2D,
FractionalMaxPool3D,
LPPool1D,
LPPool2D,
MaxPool1D,
MaxPool2D,
MaxPool3D,
MaxUnPool1D,
MaxUnPool2D,
MaxUnPool3D,
)
from .layer.rnn import (
GRU,
LSTM,
RNN,
BiRNN,
GRUCell,
LSTMCell,
RNNCellBase,
SimpleRNN,
SimpleRNNCell,
)
from .layer.transformer import (
MultiHeadAttention,
Transformer,
TransformerDecoder,
TransformerDecoderLayer,
TransformerEncoder,
TransformerEncoderLayer,
)
from .layer.vision import ChannelShuffle, PixelShuffle, PixelUnshuffle
from .modules.container import (
ModuleDict,
ModuleList,
)
from .modules.module import Module
from .parameter import Parameter
from .utils.spectral_norm_hook import spectral_norm # noqa: F401
SiLU = Silu
AdaptiveAvgPool1d = AdaptiveAvgPool1D
AdaptiveAvgPool2d = AdaptiveAvgPool2D
AdaptiveAvgPool3d = AdaptiveAvgPool3D
HuberLoss = SmoothL1Loss
MultilabelMarginLoss = MultiLabelMarginLoss
MultilabelSoftMarginLoss = MultiLabelSoftMarginLoss
MaxUnpool1d = MaxUnPool1D
MaxUnpool2d = MaxUnPool2D
MaxUnpool3d = MaxUnPool3D
UpsamplingBilinear2d = UpsamplingBilinear2D
UpsamplingNearest2d = UpsamplingNearest2D
ZeroPad1d = ZeroPad1D
ZeroPad2d = ZeroPad2D
ZeroPad3d = ZeroPad3D
ReflectionPad1d = ReflectionPad1D
ReflectionPad2d = ReflectionPad2D
ReflectionPad3d = ReflectionPad3D
ConstantPad1d = ConstantPad1D
ConstantPad2d = ConstantPad2D
ConstantPad3d = ConstantPad3D
ReplicationPad1d = ReplicationPad1D
ReplicationPad2d = ReplicationPad2D
ReplicationPad3d = ReplicationPad3D
CircularPad1d = CircularPad1D
CircularPad2d = CircularPad2D
CircularPad3d = CircularPad3D
Conv1d = Conv1D
Conv2d = Conv2D
Conv3d = Conv3D
ConvTranspose1d = Conv1DTranspose
ConvTranspose2d = Conv2DTranspose
ConvTranspose3d = Conv3DTranspose
AdaptiveMaxPool1d = AdaptiveMaxPool1D
AdaptiveMaxPool2d = AdaptiveMaxPool2D
AdaptiveMaxPool3d = AdaptiveMaxPool3D
LPPool2d = LPPool2D
LPPool1d = LPPool1D
MaxPool1d = MaxPool1D
MaxPool2d = MaxPool2D
MaxPool3d = MaxPool3D
FractionalMaxPool2d = FractionalMaxPool2D
FractionalMaxPool3d = FractionalMaxPool3D
__all__ = [
'BatchNorm',
'CELU',
'GroupNorm',
'LayerNorm',
'SpectralNorm',
'BatchNorm1D',
'BatchNorm2D',
'BatchNorm3D',
'InstanceNorm1D',
'InstanceNorm2D',
'InstanceNorm3D',
'SyncBatchNorm',
'LocalResponseNorm',
'Embedding',
'Linear',
'Upsample',
'UpsamplingNearest2D',
'UpsamplingBilinear2D',
'Pad1D',
'Pad2D',
'Pad3D',
'ConstantPad1D',
'ConstantPad2D',
'ConstantPad3D',
'CircularPad1D',
'CircularPad2D',
'CircularPad3D',
'ReplicationPad1D',
'ReplicationPad2D',
'ReplicationPad3D',
'ReflectionPad1D',
'ReflectionPad2D',
'ReflectionPad3D',
'CircularPad1d',
'CircularPad2d',
'CircularPad3d',
'ConstantPad1d',
'ConstantPad2d',
'ConstantPad3d',
'ReplicationPad1d',
'ReplicationPad2d',
'ReplicationPad3d',
'ReflectionPad1d',
'ReflectionPad2d',
'ReflectionPad3d',
'CosineSimilarity',
'Dropout',
'Dropout2D',
'Dropout3D',
'Bilinear',
'AlphaDropout',
'FeatureAlphaDropout',
'Unfold',
'Fold',
'RNNCellBase',
'SimpleRNNCell',
'LSTMCell',
'GRUCell',
'RNN',
'BiRNN',
'SimpleRNN',
'LSTM',
'GRU',
'dynamic_decode',
'MultiHeadAttention',
'Maxout',
'Softsign',
'Transformer',
'MSELoss',
'LogSigmoid',
'BeamSearchDecoder',
'ClipGradByNorm',
'ReLU',
'PairwiseDistance',
'BCEWithLogitsLoss',
'SmoothL1Loss',
'MaxPool3D',
'AdaptiveMaxPool2D',
'Hardshrink',
'Softplus',
'KLDivLoss',
'AvgPool2D',
'L1Loss',
'LeakyReLU',
'AvgPool1D',
'AdaptiveAvgPool3D',
'AdaptiveMaxPool3D',
'NLLLoss',
'PoissonNLLLoss',
'Conv1D',
'Conv1d',
'Sequential',
'Hardswish',
'Conv1DTranspose',
'ConvTranspose1d',
'AdaptiveMaxPool1D',
'TransformerEncoder',
'Softmax',
'Softmax2D',
'ParameterDict',
'ParameterList',
'Conv2D',
'Conv2d',
'Softshrink',
'Hardtanh',
'TransformerDecoderLayer',
'CrossEntropyLoss',
'GELU',
'GLU',
'SELU',
'Silu',
'SiLU',
'Conv2DTranspose',
'ConvTranspose2d',
'CTCLoss',
'RNNTLoss',
'ThresholdedReLU',
'AdaptiveAvgPool2D',
'MaxPool1D',
'Layer',
'TransformerDecoder',
'Conv3D',
'Conv3d',
'Tanh',
'Conv3DTranspose',
'ConvTranspose3d',
'Flatten',
'AdaptiveAvgPool1D',
'Tanhshrink',
'HSigmoidLoss',
'PReLU',
'TransformerEncoderLayer',
'AvgPool3D',
'MaxPool2D',
'MarginRankingLoss',
'LayerList',
'ClipGradByValue',
'BCELoss',
'Hardsigmoid',
'ClipGradByGlobalNorm',
'LogSoftmax',
'Sigmoid',
'Swish',
'Mish',
'PixelShuffle',
'PixelUnshuffle',
'ChannelShuffle',
'ELU',
'ReLU6',
'LayerDict',
'ZeroPad2D',
'MaxUnPool1D',
'MaxUnPool2D',
'MaxUnPool3D',
'MultiLabelSoftMarginLoss',
'MultilabelSoftMarginLoss',
'HingeEmbeddingLoss',
'Identity',
'CosineEmbeddingLoss',
'RReLU',
'MultiMarginLoss',
'MultiLabelMarginLoss',
'MultilabelMarginLoss',
'TripletMarginWithDistanceLoss',
'TripletMarginLoss',
'SoftMarginLoss',
'GaussianNLLLoss',
'AdaptiveLogSoftmaxWithLoss',
'Unflatten',
'FractionalMaxPool2D',
'FractionalMaxPool3D',
'LPPool1D',
'LPPool2D',
'ZeroPad1D',
'ZeroPad3D',
'Parameter',
'AdaptiveMaxPool1d',
'AdaptiveMaxPool2d',
'AdaptiveMaxPool3d',
'LPPool2d',
'LPPool1d',
'Module',
'ModuleDict',
'ModuleList',
'MaxPool1d',
'MaxPool2d',
'MaxPool3d',
'FractionalMaxPool2d',
'FractionalMaxPool3d',
]
python/paddle/nn/attention/__init__.py
+22
View File
@@ -0,0 +1,22 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import flex_attention # noqa: F401
from .sdpa import ( # noqa: F401
SDPBackend,
_cur_sdpa_kernel_backends,
sdpa_kernel,
)
__all__ = ["SDPBackend", "sdpa_kernel"]
python/paddle/nn/attention/flex_attention.py
+121
View File
@@ -0,0 +1,121 @@
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Callable
from typing import TypeAlias
from paddle import Tensor
MaskModSignature: TypeAlias = Callable[
[Tensor, Tensor, Tensor, Tensor], Tensor
]
__all__ = ["or_masks", "and_masks"]
def or_masks(*mask_mods: MaskModSignature) -> MaskModSignature:
"""
Return a mask function that computes the union of provided mask functions.
Args:
*mask_mods (Callable): Mask functions with signature
``mask_mod(b, h, q_idx, kv_idx)``.
Returns:
Callable: A mask function that applies logical OR to all mask results.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.nn.attention.flex_attention import or_masks
>>> def mask_a(b, h, q_idx, kv_idx):
... return q_idx >= kv_idx
>>> def mask_b(b, h, q_idx, kv_idx):
... return h == 0
>>> b = paddle.to_tensor([0])
>>> h = paddle.to_tensor([1])
>>> q_idx = paddle.to_tensor([2])
>>> kv_idx = paddle.to_tensor([3])
>>> mask = or_masks(mask_a, mask_b)
>>> print(mask(b, h, q_idx, kv_idx))
Tensor(shape=[1], dtype=bool, place=Place(cpu), stop_gradient=True,
[False])
"""
if not all(callable(arg) for arg in mask_mods):
raise RuntimeError(
f"All inputs should be callable mask_mods: {mask_mods}"
)
def or_mask(b: Tensor, h: Tensor, q_idx: Tensor, kv_idx: Tensor) -> Tensor:
result = b.new_zeros((), dtype='bool')
for mask in mask_mods:
result = result | mask(b, h, q_idx, kv_idx)
return result
return or_mask
def and_masks(*mask_mods: MaskModSignature) -> MaskModSignature:
"""
Return a mask function that computes the intersection of provided mask functions.
Args:
*mask_mods (Callable): Mask functions with signature
``mask_mod(b, h, q_idx, kv_idx)``.
Returns:
Callable: A mask function that applies logical AND to all mask results.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.nn.attention.flex_attention import and_masks
>>> def mask_a(b, h, q_idx, kv_idx):
... return q_idx >= kv_idx
>>> def mask_b(b, h, q_idx, kv_idx):
... return h == 0
>>> b = paddle.to_tensor([0])
>>> h = paddle.to_tensor([0])
>>> q_idx = paddle.to_tensor([2])
>>> kv_idx = paddle.to_tensor([1])
>>> mask = and_masks(mask_a, mask_b)
>>> print(mask(b, h, q_idx, kv_idx))
Tensor(shape=[1], dtype=bool, place=Place(cpu), stop_gradient=True,
[True])
"""
if not all(callable(arg) for arg in mask_mods):
raise RuntimeError(
f"All inputs should be callable mask_mods: {mask_mods}"
)
def and_mask(b: Tensor, h: Tensor, q_idx: Tensor, kv_idx: Tensor) -> Tensor:
result = b.new_ones((), dtype='bool')
for mask in mask_mods:
result = result & mask(b, h, q_idx, kv_idx)
return result
return and_mask
python/paddle/nn/attention/sdpa.py
+205
View File
@@ -0,0 +1,205 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from enum import IntEnum
from typing import TYPE_CHECKING
import paddle
from paddle.base.wrapped_decorator import signature_safe_contextmanager
if TYPE_CHECKING:
from collections.abc import Iterable
class SDPBackend(IntEnum):
"""
An enum-like class that contains the different backends for scaled dot product attention.
This backend class is designed to be used with the sdpa_kernel context manager.
The following Enums are available:
- ERROR: An error occurred when trying to determine the backend.
- MATH: The math backend for scaled dot product attention.
- FLASH_ATTENTION: The flash attention backend for scaled dot product attention.
- EFFICIENT_ATTENTION: The efficient attention backend for scaled dot product attention.
See :func:`paddle.nn.attention.sdpa_kernel` for more details.
.. warning:: This class is in beta and subject to change.
"""
ERROR = -1
MATH = 0
FLASH_ATTENTION = 1
EFFICIENT_ATTENTION = 2
_backend_enabled = {
SDPBackend.MATH: True,
SDPBackend.FLASH_ATTENTION: paddle.framework._global_flags().get(
"FLAGS_flash_attn_available", False
),
SDPBackend.EFFICIENT_ATTENTION: paddle.framework._global_flags().get(
"FLAGS_mem_efficient_attn_available", False
),
}
_current_priority = [
SDPBackend.FLASH_ATTENTION,
SDPBackend.EFFICIENT_ATTENTION,
SDPBackend.MATH,
]
def _get_enabled_backends():
global _backend_enabled
return [backend for backend, enabled in _backend_enabled.items() if enabled]
def _set_enabled_backends(backends: list[SDPBackend]):
global _backend_enabled
for backend in _backend_enabled:
_backend_enabled[backend] = False
for backend in backends:
if backend in _backend_enabled:
_backend_enabled[backend] = True
def _get_backend_priority():
global _current_priority
return _current_priority.copy()
def _set_backend_priority(priority: list[SDPBackend]):
global _current_priority
_current_priority = priority.copy()
def _validate_backends(backends):
if isinstance(backends, SDPBackend):
backends = [backends]
if not isinstance(backends, (list, tuple)):
raise TypeError(
"backends must be an instance of SDPBackend or a list of SDPBackend instances"
)
for backend in backends:
if not isinstance(backend, SDPBackend):
raise TypeError(
f"All backends must be SDPBackend instances, got {type(backend)}"
)
return list(dict.fromkeys(backends))
def _cur_sdpa_kernel_backends(with_priority: bool = False):
backends = _get_enabled_backends()
if with_priority:
curr_priority = _get_backend_priority()
backends = sorted(
backends,
key=lambda backend: (
curr_priority.index(backend)
if backend in curr_priority
else float('inf')
),
)
return backends
def _sdpa_kernel(backends: Iterable[SDPBackend], set_priority: bool = False):
_set_enabled_backends(list(backends))
if set_priority:
user_priority = list(backends)
previous_priority = _get_backend_priority()
for backend in previous_priority:
if backend not in user_priority:
user_priority.append(backend)
_set_backend_priority(user_priority)
@signature_safe_contextmanager
def sdpa_kernel(
backends: list[SDPBackend] | SDPBackend, set_priority: bool = False
):
"""
Context manager to select which backend to use for scaled dot product attention.
.. warning:: This function is beta and subject to change.
Args:
backends (Union[list[SDPBackend], SDPBackend]): A backend or list of backends
for scaled dot product attention.
set_priority (bool, optional): Whether the ordering of the backends is
interpreted as their priority order. Default: False.
Example:
>>> import paddle
>>> from paddle.nn.functional import scaled_dot_product_attention
>>> from paddle.nn.attention import SDPBackend, sdpa_kernel
>>> # Create dummy tensors
>>> query = paddle.rand(shape=[2, 4, 8, 16])
>>> key = paddle.rand(shape=[2, 4, 8, 16])
>>> value = paddle.rand(shape=[2, 4, 8, 16])
>>> # Example 1: Only enable math backend
>>> with sdpa_kernel(SDPBackend.MATH):
... out = scaled_dot_product_attention(query, key, value)
>>> print(out.shape)
[2, 4, 8, 16]
>>> # Example 2: Enable multiple backends
>>> with sdpa_kernel([SDPBackend.MATH, SDPBackend.EFFICIENT_ATTENTION]):
... out = scaled_dot_product_attention(query, key, value)
>>> print(out.shape)
[2, 4, 8, 16]
>>> # Example 3: Set priority order for multiple backends
>>> with sdpa_kernel(
... [SDPBackend.MATH, SDPBackend.EFFICIENT_ATTENTION],
... set_priority=True,
... ):
... out = scaled_dot_product_attention(query, key, value)
>>> print(out.shape)
[2, 4, 8, 16]
>>> # doctest: +SKIP('FlashAttention may not be available in all environments')
>>> # Example 4: Flash attention (skipped due to environment requirements)
>>> with sdpa_kernel(SDPBackend.FLASH_ATTENTION):
... out = scaled_dot_product_attention(query, key, value)
>>> # doctest: -SKIP
This context manager can be used to select which backend to use for scaled dot product attention.
Upon exiting the context manager, the previous state of the flags will be restored.
"""
assert isinstance(backends, (list, SDPBackend)), (
"Backend must be an instance of SDPBackend or a list of SDPBackend instances"
)
backends = _validate_backends(backends)
if not backends:
raise ValueError("At least one backend must be specified")
previous_backends = _cur_sdpa_kernel_backends(with_priority=set_priority)
try:
_sdpa_kernel(backends, set_priority)
yield {}
finally:
_sdpa_kernel(previous_backends, set_priority)
python/paddle/nn/clip.py
+1616
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/decode.py
+1393
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/functional/__init__.py
+329
View File
@@ -0,0 +1,329 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO: import all neural network related api under this directory,
# including layers, linear, conv, rnn etc.
from .activation import (
celu,
elu,
elu_,
gelu,
glu,
gumbel_softmax,
hardshrink,
hardsigmoid,
hardswish,
hardtanh,
hardtanh_,
leaky_relu,
leaky_relu_,
log_sigmoid,
log_softmax,
maxout,
mish,
prelu,
relu,
relu6,
relu_,
rrelu,
selu,
sigmoid,
silu,
softmax,
softmax_,
softplus,
softshrink,
softsign,
swiglu,
swish,
tanh,
tanh_,
tanhshrink,
thresholded_relu,
thresholded_relu_,
)
from .common import (
alpha_dropout,
bilinear,
class_center_sample,
cosine_similarity,
dropout,
dropout1d,
dropout2d,
dropout3d,
feature_alpha_dropout,
fold,
interpolate,
label_smooth,
linear,
pad,
unfold,
upsample,
zeropad2d,
)
from .conv import (
conv1d,
conv1d_transpose,
conv2d,
conv2d_transpose,
conv3d,
conv3d_transpose,
)
from .distance import pairwise_distance, pdist # noqa: F401
from .extension import (
diag_embed, # noqa: F401
gather_tree,
sequence_mask,
temporal_shift,
)
from .flash_attention import (
flash_attention_v3_varlen,
flash_attn_qkvpacked,
flash_attn_varlen_qkvpacked,
flashmask_attention,
flashmask_get_unique_id,
sdp_kernel, # noqa: F401
)
from .input import (
embedding,
embedding_renorm_, # noqa: F401
one_hot,
)
from .loss import (
adaptive_log_softmax_with_loss,
binary_cross_entropy,
binary_cross_entropy_with_logits,
cosine_embedding_loss,
cross_entropy,
ctc_loss,
dice_loss,
gaussian_nll_loss,
hinge_embedding_loss,
hsigmoid_loss,
kl_div,
l1_loss,
log_loss,
margin_cross_entropy,
margin_ranking_loss,
mse_loss,
multi_label_margin_loss,
multi_label_soft_margin_loss,
multi_margin_loss,
nll_loss,
npair_loss,
poisson_nll_loss,
rnnt_loss,
sigmoid_focal_loss,
smooth_l1_loss,
soft_margin_loss,
softmax_with_cross_entropy,
square_error_cost,
triplet_margin_loss,
triplet_margin_with_distance_loss,
)
from .moe_permute import moe_permute
from .moe_unpermute import moe_unpermute
from .norm import (
batch_norm,
group_norm,
instance_norm,
layer_norm,
local_response_norm,
normalize,
rms_norm,
)
from .pooling import (
adaptive_avg_pool1d,
adaptive_avg_pool2d,
adaptive_avg_pool3d,
adaptive_max_pool1d,
adaptive_max_pool2d,
adaptive_max_pool3d,
avg_pool1d,
avg_pool2d,
avg_pool3d,
fractional_max_pool2d,
fractional_max_pool3d,
lp_pool1d,
lp_pool2d,
max_pool1d,
max_pool2d,
max_pool3d,
max_unpool1d,
max_unpool2d,
max_unpool3d,
)
from .sdpa import scaled_dot_product_attention
from .sparse_attention import sparse_attention
from .vision import (
affine_grid,
channel_shuffle,
grid_sample,
pixel_shuffle,
pixel_unshuffle,
)
logsigmoid = log_sigmoid
conv_transpose1d = conv1d_transpose
conv_transpose2d = conv2d_transpose
conv_transpose3d = conv3d_transpose
huber_loss = smooth_l1_loss
multilabel_margin_loss = multi_label_margin_loss
multilabel_soft_margin_loss = multi_label_soft_margin_loss
__all__ = [
'celu',
'conv1d',
'conv1d_transpose',
'conv2d',
'conv2d_transpose',
'conv3d',
'conv3d_transpose',
'conv_transpose1d',
'conv_transpose2d',
'conv_transpose3d',
'pairwise_distance',
'elu',
'elu_',
'gelu',
'hardshrink',
'hardtanh',
'hardtanh_',
'hardsigmoid',
'hardswish',
'leaky_relu',
'leaky_relu_',
'log_sigmoid',
'logsigmoid',
'maxout',
'prelu',
'relu',
'relu_',
'relu6',
'selu',
'softmax',
'softmax_',
'softplus',
'softshrink',
'softsign',
'sigmoid',
'silu',
'swiglu',
'swish',
'mish',
'tanh',
'tanh_',
'tanhshrink',
'thresholded_relu',
'thresholded_relu_',
'log_softmax',
'glu',
'gumbel_softmax',
'sequence_mask',
'dropout',
'dropout1d',
'dropout2d',
'dropout3d',
'alpha_dropout',
'feature_alpha_dropout',
'label_smooth',
'linear',
'pad',
'zeropad2d',
'unfold',
'interpolate',
'upsample',
'bilinear',
'cosine_similarity',
'avg_pool1d',
'avg_pool2d',
'avg_pool3d',
'lp_pool1d',
'lp_pool2d',
'max_pool1d',
'max_pool2d',
'max_pool3d',
'max_unpool1d',
'max_unpool2d',
'max_unpool3d',
'moe_permute',
'moe_unpermute',
'adaptive_avg_pool1d',
'adaptive_avg_pool2d',
'adaptive_avg_pool3d',
'adaptive_max_pool1d',
'adaptive_max_pool2d',
'adaptive_max_pool3d',
'fractional_max_pool2d',
'fractional_max_pool3d',
'binary_cross_entropy',
'binary_cross_entropy_with_logits',
'cross_entropy',
'dice_loss',
'hsigmoid_loss',
'kl_div',
'l1_loss',
'log_loss',
'mse_loss',
'margin_ranking_loss',
'multi_label_soft_margin_loss',
'nll_loss',
'poisson_nll_loss',
'npair_loss',
'sigmoid_focal_loss',
'smooth_l1_loss',
'softmax_with_cross_entropy',
'margin_cross_entropy',
'square_error_cost',
'ctc_loss',
'rnnt_loss',
'hinge_embedding_loss',
'affine_grid',
'grid_sample',
'local_response_norm',
'pixel_shuffle',
'pixel_unshuffle',
'channel_shuffle',
'embedding',
'gather_tree',
'one_hot',
'normalize',
'temporal_shift',
'batch_norm',
'layer_norm',
'rms_norm',
'instance_norm',
'class_center_sample',
'sparse_attention',
'fold',
'cosine_embedding_loss',
'rrelu',
'triplet_margin_with_distance_loss',
'triplet_margin_loss',
'adaptive_log_softmax_with_loss',
'multi_margin_loss',
'multi_label_margin_loss',
'multilabel_margin_loss',
'multilabel_soft_margin_loss',
'soft_margin_loss',
'gaussian_nll_loss',
'scaled_dot_product_attention',
'flashmask_attention',
'flashmask_get_unique_id',
'flash_attn_qkvpacked',
"flash_attention_v3_varlen",
'flash_attn_varlen_qkvpacked',
'group_norm',
'huber_loss',
]
python/paddle/nn/functional/activation.py
+2048
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/functional/common.py
+3053
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/functional/conv.py
+1981
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/functional/distance.py
+163
View File
@@ -0,0 +1,163 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import paddle
from paddle import _C_ops
from paddle.framework import in_dynamic_or_pir_mode
from paddle.utils.decorator_utils import ParamAliasDecorator
from ...base.data_feeder import check_type, check_variable_and_dtype
from ...base.layer_helper import LayerHelper
__all__ = []
@ParamAliasDecorator(
{
"x": ["x1"],
"y": ["x2"],
"epsilon": ["eps"],
}
)
def pairwise_distance(
x: paddle.Tensor,
y: paddle.Tensor,
p: float = 2.0,
epsilon: float = 1e-6,
keepdim: bool = False,
name: str | None = None,
) -> paddle.Tensor:
r"""
It computes the pairwise distance between two vectors. The
distance is calculated by p-order norm:
.. math::
\Vert x \Vert _p = \left( \sum_{i=1}^n \vert x_i \vert ^ p \right) ^ {1/p}.
Parameters:
x (Tensor): Tensor, shape is :math:`[N, D]` or :math:`[D]`, where :math:`N`
is batch size, :math:`D` is the dimension of vector. Available dtype is
float16, float32, float64.
y (Tensor): Tensor, shape is :math:`[N, D]` or :math:`[D]`, where :math:`N`
is batch size, :math:`D` is the dimension of vector. Available dtype is
float16, float32, float64.
p (float, optional): The order of norm. Default: :math:`2.0`.
epsilon (float, optional): Add small value to avoid division by zero.
Default: :math:`1e-6`.
keepdim (bool, optional): Whether to reserve the reduced dimension
in the output Tensor. The result tensor is one dimension less than
the result of ``|x-y|`` unless :attr:`keepdim` is True. Default: False.
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`.
Generally, no setting is required. Default: None.
Returns:
Tensor, the dtype is same as input tensor.
- If :attr:`keepdim` is True, the output shape is :math:`[N, 1]` or :math:`[1]`,
depending on whether the input has data shaped as :math:`[N, D]`.
- If :attr:`keepdim` is False, the output shape is :math:`[N]` or :math:`[]`,
depending on whether the input has data shaped as :math:`[N, D]`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.to_tensor([[1.0, 3.0], [3.0, 5.0]], dtype=paddle.float64)
>>> y = paddle.to_tensor([[5.0, 6.0], [7.0, 8.0]], dtype=paddle.float64)
>>> distance = paddle.nn.functional.pairwise_distance(x, y)
>>> print(distance)
Tensor(shape=[2], dtype=float64, place=Place(cpu), stop_gradient=True,
[4.99999860, 4.99999860])
"""
if in_dynamic_or_pir_mode():
sub = _C_ops.subtract(x, y)
# p_norm op has not used epsilon, so change it to the following.
if epsilon != 0.0:
epsilon = paddle.to_tensor([epsilon], dtype=sub.dtype)
sub = _C_ops.add(sub, epsilon)
return _C_ops.p_norm(sub, p, -1, 0.0, keepdim, False)
else:
check_type(p, 'porder', (float, int), 'PairwiseDistance')
check_type(epsilon, 'epsilon', (float), 'PairwiseDistance')
check_type(keepdim, 'keepdim', (bool), 'PairwiseDistance')
check_variable_and_dtype(
x, 'x', ['float16', 'float32', 'float64'], 'PairwiseDistance'
)
check_variable_and_dtype(
y, 'y', ['float16', 'float32', 'float64'], 'PairwiseDistance'
)
sub = paddle.subtract(x, y)
if epsilon != 0.0:
epsilon_var = sub.block.create_var(dtype=sub.dtype)
epsilon_var = paddle.full(
shape=[1], fill_value=epsilon, dtype=sub.dtype
)
sub = paddle.add(sub, epsilon_var)
helper = LayerHelper("PairwiseDistance", name=name)
attrs = {
'axis': -1,
'porder': p,
'keepdim': keepdim,
'epsilon': 0.0,
}
out = helper.create_variable_for_type_inference(dtype=x.dtype)
helper.append_op(
type='p_norm', inputs={'X': sub}, outputs={'Out': out}, attrs=attrs
)
return out
def pdist(
x: paddle.Tensor, p: float = 2.0, name: str | None = None
) -> paddle.Tensor:
r'''
Computes the p-norm distance between every pair of row vectors in the input.
Args:
x (Tensor): The input tensor with shape :math:`N \times M`.
p (float, optional): The value for the p-norm distance to calculate between each vector pair. Default: :math:`2.0`.
name (str, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
Returns:
Tensor with shape :math:`N(N-1)/2` , the dtype is same as input tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(2023)
>>> a = paddle.randn([4, 5])
>>> print(a)
Tensor(shape=[4, 5], dtype=float32, place=Place(cpu), stop_gradient=True,
[[ 0.06132207, 1.11349595, 0.41906244, -0.24858207, -1.85169315],
[-1.50370061, 1.73954511, 0.13331604, 1.66359663, -0.55764782],
[-0.59911072, -0.57773495, -1.03176904, -0.33741450, -0.29695082],
[-1.50258386, 0.67233968, -1.07747352, 0.80170447, -0.06695852]])
>>> pdist_out = paddle.pdist(a)
>>> print(pdist_out)
Tensor(shape=[6], dtype=float32, place=Place(cpu), stop_gradient=True,
[2.87295413, 2.79758120, 3.02793980, 3.40844536, 1.89435327, 1.93171620])
'''
x_shape = list(x.shape)
assert len(x_shape) == 2, "The x must be 2-dimensional"
d = paddle.linalg.norm(x[..., None, :] - x[..., None, :, :], p=p, axis=-1)
mask = ~paddle.tril(paddle.ones(d.shape, dtype='bool'))
return paddle.masked_select(d, mask)
python/paddle/nn/functional/extension.py
+348
View File
@@ -0,0 +1,348 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import (
TYPE_CHECKING,
)
from paddle import _C_ops, tensor
from paddle.utils import deprecated
from ...base.data_feeder import (
check_dtype,
check_type,
check_variable_and_dtype,
)
from ...base.layer_helper import LayerHelper
from ...common_ops_import import Variable
from ...framework import (
convert_nptype_to_datatype_or_vartype,
core,
in_dynamic_or_pir_mode,
)
if TYPE_CHECKING:
from paddle import Tensor
from paddle._typing import DataLayout2D, DTypeLike
__all__ = []
@deprecated(
since="2.5.2",
update_to="paddle.diag_embed",
level=1,
reason="diag_embed in paddle.nn.functional will be removed in future",
)
def diag_embed(
input: Tensor, offset: int = 0, dim1: int = -2, dim2: int = -1
) -> Tensor:
return tensor.diag_embed(input, offset, dim1, dim2)
def sequence_mask(
x: Tensor,
maxlen: int | None = None,
dtype: DTypeLike = 'int64',
name: str | None = None,
) -> Tensor:
r"""
**SequenceMask Layer**
This layer outputs a mask according to the input :code:`x` and
:code:`maxlen` with data type of :code:`dtype`.
Supposing :code:`x` is a Tensor with shape [d_1, d_2, ..., d_n], the
:code:`y` is a mask with shape [d_1, d_2, ..., d_n, maxlen], where:
.. math::
y(i_1, i_2,..., i_n, j) = (j < x(i_1, i_2,..., i_n))
.. code-block:: text
Case:
Consider input:
x = [3, 1, 1, 0] max_len = 4
then we get out:
mask = [[1, 1, 1, 0],
[1, 0, 0, 0],
[1, 0, 0, 0],
[0, 0, 0, 0]]
Args:
x (Variable): Input tensor of sequence_mask layer, \
whose elements are integers less than :code:`maxlen`. \
Tensor with shape [d_1, d_2, ..., d_n].
maxlen (int|None, optional): Maximum length of the sequence. If :code:`maxlen` \
is None, it would be replace with :math:`max(x)`.
dtype (np.dtype|paddle.dtype|str, optional): Data type of the output, \
``int64`` by default.
name(str|None, optional): For detailed information, please refer \
to :ref:`api_guide_Name`. Usually name is no need to set and \
None by default.
Returns:
Tensor, The output sequence mask. Tensor with shape [d_1, d_2, ..., d_n, maxlen] \
and data type of :code:`dtype`. The data type should be bool, float32, float64, int8, \
int32 or int64.
Examples:
.. code-block:: pycon
>>> import paddle
>>> lengths = paddle.to_tensor([10, 9, 8])
>>> mask = paddle.nn.functional.sequence_mask(lengths)
>>> print(mask)
Tensor(shape=[3, 10], dtype=int64, place=Place(cpu), stop_gradient=True,
[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 0, 0]])
"""
if in_dynamic_or_pir_mode():
if not isinstance(dtype, (core.VarDesc.VarType, core.DataType)):
dtype = convert_nptype_to_datatype_or_vartype(dtype)
if maxlen is None:
maxlen = -1
out = _C_ops.sequence_mask(x, maxlen, dtype)
out.stop_gradient = True
return out
helper = LayerHelper('sequence_mask', **locals())
out = helper.create_variable_for_type_inference(dtype=dtype)
inputs = {'X': [x]}
attrs = {'out_dtype': out.dtype}
if maxlen is not None:
if isinstance(maxlen, Variable):
inputs['MaxLenTensor'] = maxlen
else:
attrs['maxlen'] = maxlen
helper.append_op(
type='sequence_mask', inputs=inputs, outputs={'Y': out}, attrs=attrs
)
out.stop_gradient = True
return out
def gather_tree(ids: Tensor, parents: Tensor) -> Tensor:
r"""
To be used after beam search. After beam search, we get selected ids at
each time step and the corresponding parents in the search tree. Both ids
and parents have the layout :attr:`[max_time, batch_size, beam_size]`. Then
:attr:`gather_tree` is used to backtrace from the last time step and
generate the full sequences by collecting selected ids.
Here is an example:
.. code-block:: text
Given:
ids = [[[2 2]
[6 1]]
[[3 9]
[6 1]]
[[0 1]
[9 0]]]
parents = [[[0 0]
[1 1]]
[[1 0]
[1 0]]
[[0 0]
[0 1]]]
Then:
gather_tree(ids, parents)
= [[[2 2]
[1 6]]
[[3 3]
[6 1]]
[[0 1]
[9 0]]]
Args:
ids(Tensor): A Tensor with shape :attr:`[length, batch_size, beam_size]`
and data type :attr:`int32` or :attr:`int64`. It contains the selected
ids of all time steps.
parents(Tensor): A Tensor with the same shape and data type as :attr:`ids`,
It contains the parents corresponding to selected ids when searching
among beams.
Returns:
A Tensor with the same shape and data type as :attr:`ids`. \
It contains the full sequences. The sequences are collected from \
:attr:`ids` by backtracing according to :attr:`parents`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> ids = paddle.to_tensor([[[2, 2], [6, 1]], [[3, 9], [6, 1]], [[0, 1], [9, 0]]])
>>> parents = paddle.to_tensor([[[0, 0], [1, 1]], [[1, 0], [1, 0]], [[0, 0], [0, 1]]])
>>> final_sequences = paddle.nn.functional.gather_tree(ids, parents)
>>> [[[2, 2], [1, 6]], [[3, 3], [6, 1]], [[0, 1], [9, 0]]]
>>> final_sequences = paddle.nn.functional.gather_tree(ids, parents)
>>> print(final_sequences)
Tensor(shape=[3, 2, 2], dtype=int64, place=Place(cpu), stop_gradient=True,
[[[2, 2],
[1, 6]],
[[3, 3],
[6, 1]],
[[0, 1],
[9, 0]]])
"""
if ids.ndim != 3:
raise ValueError(
"The input ids must be a 3D tensor with shape [length, batch_size, beam_size]"
)
if ids.ndim != parents.ndim:
raise ValueError("The ids's shape must be the same as parents' shape. ")
if in_dynamic_or_pir_mode():
check_dtype(parents.dtype, "parents", ['int32', 'int64'], 'gather_tree')
return _C_ops.gather_tree(ids, parents)
else:
helper = LayerHelper('gather_tree', **locals())
check_variable_and_dtype(ids, 'ids', ['int32', 'int64'], 'gather_tree')
check_variable_and_dtype(
parents, 'parents', ['int32', 'int64'], 'gather_tree'
)
out = helper.create_variable_for_type_inference(dtype=ids.dtype)
helper.append_op(
type="gather_tree",
inputs={"Ids": ids, "Parents": parents},
outputs={"Out": out},
)
return out
def temporal_shift(
x: Tensor,
seg_num: int,
shift_ratio: float = 0.25,
name: str | None = None,
data_format: DataLayout2D | str = 'NCHW',
) -> Tensor:
"""
**Temporal Shift Operator**
Calculate the temporal shifting features for Input(X).
Input(X) should be in shape of [N*T, C, H, W] or [N*T, H, W, C], while
N is the batch size, T is the temporal segment number specified by
:attr:`seg_num`, C is the channel number, H and W is the height and
width of features.
Temporal Shifting is calculated as follows when data format is NCHW:
Step 1: Reshape Input(X) to [N, T, C, H, W].
Step 2: Pad 0 to reshaping result in the 2nd(T) dimension with
padding width as 1 on each side, padding result will be in shape
of [N, T+2, C, H, W].
Step 3: Assume :attr:`shift_ratio` is :math:`1/4`, slice padding
result as follows:
$$
slice1 = x[:, :T, :C/4, :, :]
$$
$$
slice2 = x[:, 2:T+2, C/4:C/2, :, :]
$$
$$
slice3 = x[:, 1:T+1, C/2:, :, :]
$$
Step 4: Concatenate three slices along the 3rd(C) dimension and
reshape result to [N*T, C, H, W].
For details of temporal shifting, please refer to paper:
`Temporal Shift Module <http://arxiv.org/abs/1811.08383>`_ .
Args:
x(Tensor): ${x_comment}
seg_num(int): ${seg_num_comment}
shift_ratio(float): ${shift_ratio_comment}
name(str|None, optional): For detailed information, please refer
to :ref:`api_guide_Name`. Usually name is no need to set and
None by default.
data_format(str, optional): Data format that specifies the layout of input.
It can be "NCHW" or "NHWC". Default: "NCHW".
Returns:
out(Tensor): The temporal shifting result is a tensor with the
same shape and same data type as the input.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn.functional as F
>>> input = paddle.randn([6, 4, 2, 2])
>>> out = F.temporal_shift(x=input, seg_num=2, shift_ratio=0.2)
"""
if data_format not in ["NCHW", "NHWC"]:
raise ValueError(
"Attr(data_format) should be 'NCHW' or 'NHWC'. "
f"Received Attr(data_format): {data_format}."
)
if in_dynamic_or_pir_mode():
return _C_ops.temporal_shift(x, seg_num, shift_ratio, data_format)
else:
helper = LayerHelper("temporal_shift", **locals())
check_variable_and_dtype(
x,
'x',
['float16', 'uint16', 'float32', 'float64'],
'temporal_shift',
)
check_type(seg_num, 'seg_num', int, 'temporal_shift')
check_type(shift_ratio, 'shift_ratio', float, 'temporal_shift')
out = helper.create_variable_for_type_inference(dtype=x.dtype)
if not isinstance(seg_num, int):
raise TypeError("seg_num must be int type.")
helper.append_op(
type="temporal_shift",
inputs={"X": x},
outputs={"Out": out},
attrs={
"seg_num": seg_num,
"shift_ratio": shift_ratio,
"data_format": data_format,
},
)
return out
python/paddle/nn/functional/flash_attention.py
+2304
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/functional/input.py
+363
View File
@@ -0,0 +1,363 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
import paddle
from paddle import _C_ops
from paddle.utils.decorator_utils import param_one_alias
from ...base.data_feeder import check_variable_and_dtype
from ...base.layer_helper import LayerHelper
from ...common_ops_import import Variable
from ...framework import in_dynamic_or_pir_mode
if TYPE_CHECKING:
from paddle import Tensor
__all__ = []
@param_one_alias(["x", "input"])
def one_hot(
x: Tensor,
num_classes: int = -1,
name: str | None = None,
) -> Tensor:
"""
The operator converts each id in the input `x` to an one-hot vector with a
`num_classes` length. The value in the vector dimension corresponding to the id
is 1, and the value in the remaining dimension is 0.
The shape of output Tensor is generated by appending `num_classes` dimension
behind the last dimension of the `x` shape.
.. code-block:: text
Example 1:
input:
x.shape = [4]
x.data = [1, 1, 3, 0]
num_classes = 4
output:
Out.shape = [4, 4]
Out.data = [[0., 1., 0., 0.],
[0., 1., 0., 0.],
[0., 0., 0., 1.],
[1., 0., 0., 0.]]
Example 2:
input:
x.shape = [4]
x.data = [1, 1, 5, 0]
num_classes = 4
output: Throw an exception for Illegal value
The second dimension in X is 5, which is greater than num_classes,
so it throws an exception.
.. note::
Alias Support: The parameter name ``input`` can be used as an alias for ``x``.
For example, ``one_hot(input=tensor_x, ...)`` is equivalent to ``one_hot(x=tensor_x, ...)``.
Args:
x (Tensor): Tensor with shape :math:`[N_1, N_2, ..., N_k]` ,
which contains at least one dimension. The data type is int32 or int64.
Alias: ``input``.
num_classes (int): An integer defining the `num_classes` of the one hot dimension. If input `x`
is word id, `num_classes` is generally the dictionary size. Default value: -1.
name (str|None, optional): For detailed information, please refer
to :ref:`api_guide_Name`. Usually name is no need to set and
None by default.
Returns:
Tensor, The one-hot representations of `x`. A Tensor with type float32.
Examples:
.. code-block:: pycon
>>> import paddle
>>> # Correspond to the first example above, where label.shape is 4 and one_hot_label.shape is [4, 4].
>>> label = paddle.to_tensor([1, 1, 3, 0], dtype='int64')
>>> print(label.shape)
paddle.Size([4])
>>> one_hot_label = paddle.nn.functional.one_hot(label, num_classes=4)
>>> print(one_hot_label.shape)
paddle.Size([4, 4])
>>> print(one_hot_label)
Tensor(shape=[4, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
[[0., 1., 0., 0.],
[0., 1., 0., 0.],
[0., 0., 0., 1.],
[1., 0., 0., 0.]])
"""
if not isinstance(num_classes, paddle.pir.Value) and num_classes == -1:
num_classes = x.max() + 1
if in_dynamic_or_pir_mode():
return _C_ops.one_hot(x, num_classes)
else:
check_variable_and_dtype(x, 'input', ['int32', 'int64'], 'one_hot_v2')
helper = LayerHelper("one_hot_v2", **locals())
one_hot_out = helper.create_variable_for_type_inference(dtype='float32')
if not isinstance(num_classes, Variable):
# user attribute
inputs = {'X': x}
attrs = {'depth': num_classes, 'allow_out_of_range': False}
else:
num_classes.stop_gradient = True
inputs = {'X': x, 'depth_tensor': num_classes}
attrs = {'allow_out_of_range': False}
helper.append_op(
type="one_hot_v2",
inputs=inputs,
attrs=attrs,
outputs={'Out': one_hot_out},
stop_gradient=True,
)
return one_hot_out
def embedding_renorm_(
x: Tensor, weight: Tensor, max_norm: float, norm_type: float = 2.0
) -> Tensor:
r"""
Renorm the weight of embedding with respect to the provided :attr:`max_norm` and :attr:`norm_type` .
Note:
In the dynamic graph mode, the input weight will be updated in-place, and the return value will be the changed weight.
Args:
x (Tensor): A Tensor with type int32/int64, which contains the id information. The value of the input id should
satisfy :math:`0<= id < weight.shape[0]` .
weight (Tensor): The weight. A Tensor with shape of lookup table parameter. It should have two elements which
indicates the size of the dictionary of embeddings and the size of each embedding vector respectively.
max_norm (float): The maximum norm for each embedding vector.
norm_type (float, optional): The p of the p-norm to compute for the max_norm option. Default: 2.0.
Returns:
Tensor, The updated weight. The data type is the same as :attr:`weight`.
"""
with paddle.set_grad_enabled(False):
unique_x = paddle.unique(x)
selected_rows = paddle.index_select(weight, unique_x)
norm = paddle.norm(selected_rows, p=norm_type, axis=1, keepdim=True)
mask = norm > max_norm
scale = max_norm / (norm + 1e-7)
scale = paddle.where(mask, scale, paddle.ones_like(scale))
scale = paddle.expand_as(scale, selected_rows)
updated_rows = selected_rows * scale
paddle.scatter_(weight, unique_x, updated_rows, overwrite=True)
return weight
@param_one_alias(["x", "input"])
def embedding(
x: Tensor,
weight: Tensor,
padding_idx: int | None = None,
max_norm: float | None = None,
norm_type: float = 2.0,
sparse: bool = False,
scale_grad_by_freq: bool = False,
name: str | None = None,
) -> Tensor:
r"""
Used to lookup embeddings vector of ids provided by :attr:`x` .
The shape of output Tensor is generated by appending the last dimension of the input Tensor shape
with embedding size.
Note:
The id in :attr:`x` must satisfy :math:`0 <= id < weight.shape[0]` ,
otherwise the program will throw an exception and exit.
.. code-block:: text
x is a Tensor.
padding_idx = -1
x.data = [[1, 3], [2, 4], [4, 127]]
x.shape = [3, 2]
weight.shape = [128, 16]
output is a Tensor:
out.shape = [3, 2, 16]
out.data = [[[0.129435295, 0.244512452, ..., 0.436322452],
[0.345421456, 0.524563927, ..., 0.144534654]],
[[0.345249859, 0.124939536, ..., 0.194353745],
[0.945345345, 0.435394634, ..., 0.435345365]],
[[0.945345345, 0.435394634, ..., 0.435345365],
[0.0, 0.0, ..., 0.0 ]]] # padding data
The input padding_idx is less than 0, it is automatically converted to padding_idx = -1 + 128 = 127
It will pad all-zero data when id is 127.
.. note::
Alias Support: The parameter name ``input`` can be used as an alias for ``x``.
For example, ``embedding(input=tensor_x, ...)`` is equivalent to ``embedding(x=tensor_x, ...)``.
Args:
x (Tensor): A Tensor with type int32/int64, which contains the id information. The value of the input id should
satisfy :math:`0 <= id < weight.shape[0]` .
Alias: ``input``.
weight (Tensor): The weight. A Tensor with shape of lookup table parameter. It should have two elements which
indicates the size of the dictionary of embeddings and the size of each embedding vector respectively.
sparse (bool, optional): The flag indicating whether to use sparse update. This parameter only
affects the performance of the backwards gradient update. It is recommended to set
True because sparse update is faster. But some optimizers does not support sparse update,
such as :ref:`api_paddle_optimizer_adadelta_Adadelta` , :ref:`api_paddle_optimizer_adamax_Adamax` , :ref:`api_paddle_optimizer_lamb_Lamb`.
In these cases, sparse must be False. Default: False.
padding_idx (int|None, optional): padding_idx needs to be in the interval [-weight.shape[0], weight.shape[0]).
If :math:`padding\_idx < 0`, the :math:`padding\_idx` will automatically be converted
to :math:`weight.shape[0] + padding\_idx` . It will output all-zero padding data whenever lookup
encounters :math:`padding\_idx` in id. And the padding data will not be updated while training.
If set None, it makes no effect to output. Default: None.
max_norm (float, optional): If provided, will renormalize the embedding vectors to have a norm larger than
:attr:`max\_norm` . It will inplace update the input embedding weight in dynamic graph mode. Default: None.
norm_type (float, optional): The p of the p-norm to compute for the max_norm option. Default: 2.0.
scale_grad_by_freq (bool, optional): Indicating whether to scale the gradients by the inverse frequency of the
word ids in input `x`. Default: False.
name (str|None, optional): For detailed information, please refer
to :ref:`api_guide_Name`. Usually name is no need to set and
None by default.
Returns:
Tensor, Embedding Tensor mapped by x. The data type is the same as :attr:`weight`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn as nn
>>> x0 = paddle.arange(3, 6).reshape((3, 1)).astype(paddle.int64)
>>> w0 = paddle.full(shape=(10, 3), fill_value=2).astype(paddle.float32)
>>> x = paddle.to_tensor(x0, stop_gradient=False)
>>> print(x.numpy())
[[3]
[4]
[5]]
>>> print(x.shape)
paddle.Size([3, 1])
>>> w = paddle.to_tensor(w0, stop_gradient=False)
>>> print(w.numpy())
[[2. 2. 2.]
[2. 2. 2.]
[2. 2. 2.]
[2. 2. 2.]
[2. 2. 2.]
[2. 2. 2.]
[2. 2. 2.]
[2. 2. 2.]
[2. 2. 2.]
[2. 2. 2.]]
>>> print(w.shape)
paddle.Size([10, 3])
>>> emb = nn.functional.embedding(x=x, weight=w, sparse=True, name="embedding")
>>> print(emb.numpy())
[[[2. 2. 2.]]
[[2. 2. 2.]]
[[2. 2. 2.]]]
>>> print(emb.shape)
paddle.Size([3, 1, 3])
"""
padding_idx = (
-1
if padding_idx is None
else (
padding_idx if padding_idx >= 0 else (weight.shape[0] + padding_idx)
)
)
if weight.shape[0] != 0 and (
padding_idx >= weight.shape[0] or padding_idx < -weight.shape[0]
):
raise ValueError(
f"padding_idx must be within [-{weight.shape[0]}, {weight.shape[0]})"
)
if max_norm and weight.size != 0:
weight = embedding_renorm_(
x, weight, max_norm=max_norm, norm_type=norm_type
)
if scale_grad_by_freq:
if sparse:
raise AttributeError(
"scale_grad_by_freq = True is not supported with sparse update."
)
if in_dynamic_or_pir_mode():
return _C_ops.embedding_with_scaled_gradient(x, weight, padding_idx)
else:
helper = LayerHelper('embedding_with_scaled_gradient', **locals())
dtype = helper.input_dtype(input_param_name='weight')
check_variable_and_dtype(
x,
'input',
['uint8', 'int8', 'int16', 'int32', 'int64'],
'embedding_with_scaled_gradient',
)
tmp = helper.create_variable_for_type_inference(dtype)
helper.append_op(
type='embedding_with_scaled_gradient',
inputs={'x': x, 'weight': weight},
outputs={'out': tmp},
attrs={'padding_idx': padding_idx},
)
return tmp
else:
if in_dynamic_or_pir_mode():
return _C_ops.embedding(x, weight, padding_idx, sparse)
else:
helper = LayerHelper('embedding', **locals())
dtype = helper.input_dtype(input_param_name='weight')
check_variable_and_dtype(
x,
'input',
['uint8', 'int8', 'int16', 'int32', 'int64'],
'embedding',
)
is_distributed = False
remote_prefetch = sparse and (not is_distributed)
tmp = helper.create_variable_for_type_inference(dtype)
helper.append_op(
type='lookup_table_v2',
inputs={'Ids': x, 'W': weight},
outputs={'Out': tmp},
attrs={
'is_sparse': sparse,
'is_distributed': is_distributed,
'remote_prefetch': remote_prefetch,
'padding_idx': padding_idx,
},
)
return tmp
python/paddle/nn/functional/loss.py
+5146
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/functional/moe_permute.py
+177
View File
@@ -0,0 +1,177 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from paddle import _C_ops
from paddle.base.framework import in_dynamic_or_pir_mode
if TYPE_CHECKING:
from paddle import Tensor
def moe_permute(
hidden_states: Tensor,
scale: Tensor | None,
expert_routemap_topk: Tensor,
expert_prob_topk: Tensor,
num_experts: int,
tokens_per_expert: list,
padding_alignment: int,
do_gather: bool = True,
using_ue8m0_scale: bool = False,
return_expert_indices: bool = False,
override_buffer_size: int = -1,
name: str | None = None,
) -> tuple[Tensor, Tensor, Tensor, Tensor]:
r"""
Permute tokens for Mixture of Experts (MoE) computation in distributed training scenarios.
Note:
This function reorganizes input tokens based on expert assignments to prepare for expert computation.
It handles both bfloat16 and float8_e4m3fn data types with proper scaling for float8 inputs.
1. This function is typically used in pair of moe_unpermute to provide complete MoE functionality.
2. For float8 inputs, proper scaling must be provided via the scale parameter.
3. The padding_alignment parameter affects memory efficiency but not correctness.
4. Any output tokens can find an exact-match in the original input tokens.
5. This permute function has overcomed the aadiff issue, is deterministic.
6. If using_ue8m0_scale is True, then the data type of scale must be int32, and each int32 is packaged from 4 ue8m0 scaling factors.
Args:
hidden_states (Tensor): The input tensor containing tokens to be permuted, stored in row-major layout.
Supported data types: bfloat16 or float8_e4m3fn.
Shape: [sequence_length, token_dimension]
scale (Tensor|None): Scaling factors required when hidden_states is of float8 type.
For float8 inputs, this tensor provides the scaling factors for dequantization.
Shape: [sequence_length, ceil(token_dimension / 128)]. If using_ue8m0_scale is True, the shape is [sequence_length, ceil(ceil(token_dimension / 128)/4)].
Data type: float32 or int32(Only when using_ue8m0_scale is True). If using_ue8m0_scale is True, the data type of scale is int32 which is packed of four ue8m0 scaling factors.
expert_routemap_topk (Tensor): Tensor indicating expert assignments for each token (top-k experts).
Each value represents the expert index the token is assigned to (-1 indicates not assigned).
Shape: [sequence_length, top_k_experts]
Data type: int32
Value range: [-1, num_experts)
expert_prob_topk (Tensor): Tensor containing routing probabilities for top-k experts.
Shape: [sequence_length, top_k_experts]
Data type: float32
num_experts (int): Total number of experts in the MoE layer, limited between 1 and 64.
tokens_per_expert (list[int]): List where each element indicates the number of tokens
assigned to the corresponding expert.
padding_alignment (int): Tokens alignment requirement for expert buffers (in bytes).
Must be a power of 2. Typical values are 16, 32 or 64 for optimal memory access.
do_gather(bool): Decide whether do actual tokens gather operation or not, default is True.
using_ue8m0_scale (bool): Whether to use the ue8m0 scaling for float8 inputs. Default is False.
return_expert_indices(bool): Whether to return an 1D tensor of expert indices for each token, with -1 representing padding. Default is False.
override_buffer_size(bool): Whether to override the buffer size using the given CPU integer, default is -1.
name (str|None, optional): Name prefix for the operation (optional).
Default: None
Returns:
tuple[Tensor, Tensor, Tensor, Tensor]:
- hidden_states_unzipped (Tensor): The permuted and broadcasted input tensor.
Shape: [total_tokens_after_broadcast, token_dimension]
Data type: same as input hidden_states
- zipped_expertwise_rowmap (Tensor): Mapping tensor used to restore original order (unpermute).
Shape: [sequence_length, num_experts]
Data type: int32
- token_prob_unzipped (Tensor): Flattened expert probabilities aligned with permuted tokens.
Shape: [total_tokens_after_broadcast, 1]
Data type: float32
- scale_unzipped (Tensor): Broadcasted scale tensor (only valid for float8 inputs).
Shape: [total_tokens_after_broadcast, scale.shape[-1]]
Data type: float32 or int32. It is same as scale.
Examples:
.. code-block:: pycon
>>> # doctest: +REQUIRES(env:GPU)
>>> # doctest: +SKIP('This is only support in cuda 12.0+')
>>> import paddle
>>> import numpy as np
>>> import paddle.nn.functional as F
>>> hidden_states = paddle.randn([3, 128], dtype='bfloat16')
>>> expert_routemap_topk = paddle.to_tensor(
... [
... [-1, 0, -1, -1, 2, -1, -1, -1],
... [1, -1, -1, -1, -1, -1, -1, -1],
... [-1, -1, -1, -1, -1, -1, 1, -1],
... ],
... dtype='int32',
... )
>>> expert_prob_topk = paddle.to_tensor(
... [
... [0.0, 0.6, 0.0, 0.0, 0.4, 0.0, 0.0, 0.0],
... [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0],
... ],
... dtype='float32',
... )
>>> num_experts = 3
>>> tokens_per_expert = [1, 2, 1]
>>> padding_alignment = 2
>>> hidden_states_unzipped, zipped_expertwise_rowmap, token_prob_unzipped, scale_unzipped = F.moe_permute(
... hidden_states,
... None,
... expert_routemap_topk,
... expert_prob_topk,
... num_experts,
... tokens_per_expert,
... padding_alignment,
... )
>>> # weighted by probs.
>>> hidden_states_unzipped = (
... hidden_states_unzipped.astype("float32") * token_prob_unzipped.astype("float32").unsqueeze(-1)
... ).astype("bfloat16")
>>> zipped_tokens, zipped_probs = F.moe_unpermute(
... hidden_states_unzipped, zipped_expertwise_rowmap, expert_routemap_topk, token_prob_unzipped, 3, 3
... )
>>> np.testing.assert_allclose(zipped_tokens.numpy(), hidden_states.numpy(), rtol=1e-05, atol=1e-06)
"""
if in_dynamic_or_pir_mode():
(
hidden_states_unzipped,
zipped_expertwise_rowmap,
token_prob_unzipped,
scale_unzipped,
expert_indices,
) = _C_ops.moe_permute(
hidden_states,
scale,
expert_routemap_topk,
expert_prob_topk,
num_experts,
tokens_per_expert,
padding_alignment,
do_gather,
using_ue8m0_scale,
return_expert_indices,
override_buffer_size,
)
if return_expert_indices:
return (
hidden_states_unzipped,
zipped_expertwise_rowmap,
token_prob_unzipped,
scale_unzipped,
expert_indices,
)
else:
return (
hidden_states_unzipped,
zipped_expertwise_rowmap,
token_prob_unzipped,
scale_unzipped,
)
python/paddle/nn/functional/moe_unpermute.py
+119
View File
@@ -0,0 +1,119 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from paddle import _C_ops
from paddle.base.framework import in_dynamic_or_pir_mode
if TYPE_CHECKING:
from paddle import Tensor
def moe_unpermute(
hidden_states_unzipped: Tensor,
zipped_expertwise_rowmap: Tensor,
expert_routemap_topk: Tensor,
token_prob_unzipped: Tensor,
total_zipped_tokens: int,
num_experts: int,
using_mix_precision: bool = True,
using_weighted_combine: bool = False,
name: str | None = None,
) -> tuple[Tensor, Tensor]:
r"""
Args:
hidden_states_unzipped (Tensor): The input Tensor containing broadcasted and permuted hidden states.
Shape: (seqlen_broadcasted, token_len). Dtype: bfloat16.
zipped_expertwise_rowmap (Tensor): The input Tensor recording the mapping relationship for unpermute operation.
Shape: (seqlen, num_experts). Dtype: int32.
expert_routemap_topk (Tensor): The input Tensor indicating which expert each token is assigned to.
Shape: (seqlen, 8). Value range: [-1, num_experts]. Dtype: int32.
token_prob_unzipped (Tensor): The input Tensor containing flattened expert probabilities corresponding to hidden_states_unzipped.
Shape: (seqlen_broadcasted, 1). Dtype: float32.
total_zipped_tokens_num (int): The total number of tokens before permutation for output buffer allocation. Dtype: int32.
num_experts (int): The number of experts. Dtype: int32.
using_mix_precision (bool, optional): Whether to use mixed precision during accumulation.
This option significantly improves precision when number of experts > 4. Default: True.
using_weighted_combine (bool, optional): Whether to use weighted token accumulation during unpermute.
Which utilize probs as weights to accumulate tokens. Default: False.
name (str|None, optional): Name for the operation. Default: None.
Returns:
tuple[Tensor, Tensor]: A tuple containing:
- hidden_states (Tensor): The output Tensor with unpermuted tokens.
Shape: (seqlen, token_len). Dtype: bfloat16.
- expert_prob_topk (Tensor): The output Tensor with unpermuted probabilities.
Shape: (seqlen, topk). Dtype: float32.
Examples:
.. code-block:: pycon
>>> # doctest: +REQUIRES(env:GPU)
>>> # doctest: +SKIP('This is only support in cuda 12.0+')
>>> import paddle
>>> import numpy as np
>>> import paddle.nn.functional as F
>>> hidden_states = paddle.randn([3, 128], dtype='bfloat16')
>>> expert_routemap_topk = paddle.to_tensor(
... [
... [-1, 0, -1, -1, 2, -1, -1, -1],
... [1, -1, -1, -1, -1, -1, -1, -1],
... [-1, -1, -1, -1, -1, -1, 1, -1],
... ],
... dtype='int32',
... )
>>> expert_prob_topk = paddle.to_tensor(
... [
... [0.0, 0.6, 0.0, 0.0, 0.4, 0.0, 0.0, 0.0],
... [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
... [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0],
... ],
... dtype='float32',
... )
>>> num_experts = 3
>>> tokens_per_expert = [1, 2, 1]
>>> padding_alignment = 2
>>> hidden_states_unzipped, zipped_expertwise_rowmap, token_prob_unzipped, scale_unzipped = F.moe_permute(
... hidden_states,
... None,
... expert_routemap_topk,
... expert_prob_topk,
... num_experts,
... tokens_per_expert,
... padding_alignment,
... )
>>> # weighted by probs.
>>> hidden_states_unzipped = (
... hidden_states_unzipped.astype("float32") * token_prob_unzipped.astype("float32").unsqueeze(-1)
... ).astype("bfloat16")
>>> zipped_tokens, zipped_probs = F.moe_unpermute(
... hidden_states_unzipped, zipped_expertwise_rowmap, expert_routemap_topk, token_prob_unzipped, 3, 3
... )
>>> np.testing.assert_allclose(zipped_tokens.numpy(), hidden_states.numpy(), rtol=1e-05, atol=1e-06)
"""
if in_dynamic_or_pir_mode():
zipped_tokens, zipped_probs_topk = _C_ops.moe_unpermute(
hidden_states_unzipped,
zipped_expertwise_rowmap,
expert_routemap_topk,
token_prob_unzipped,
total_zipped_tokens,
num_experts,
using_mix_precision,
using_weighted_combine,
)
return (zipped_tokens, zipped_probs_topk)
python/paddle/nn/functional/norm.py
+921
View File
@@ -0,0 +1,921 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import inspect
import numbers
from typing import TYPE_CHECKING, Any
import numpy as np
from typing_extensions import overload
import paddle
from paddle import _C_ops, in_dynamic_mode
from paddle.base.framework import (
in_dygraph_mode,
in_dynamic_or_pir_mode,
in_pir_mode,
)
from paddle.utils.decorator_utils import (
ParamAliasDecorator,
param_one_alias,
param_two_alias,
)
from ...base.data_feeder import check_type, check_variable_and_dtype
from ...base.layer_helper import LayerHelper
if TYPE_CHECKING:
from collections.abc import Sequence
from typing import Literal
from paddle import Tensor
from paddle._typing import (
DataLayout1D,
DataLayout2D,
DataLayout3D,
DataLayoutND,
)
__all__ = []
@ParamAliasDecorator({"x": ["input"], "axis": ["dim"], "epsilon": ["eps"]})
def normalize(
x: Tensor,
p: float = 2,
axis: int = 1,
epsilon: float = 1e-12,
out: Tensor | None = None,
name: str | None = None,
) -> Tensor:
r"""
Normalize ``x`` along dimension ``axis`` using :math:`L_p` norm. This layer computes
.. math::
y = \frac{x}{ \max\left( \lvert \lvert x \rvert \rvert_p, epsilon\right) }
.. math::
\lvert \lvert x \rvert \rvert_p = \left( \sum_i {\lvert x_i \rvert^p} \right)^{1/p}
where, :math:`\sum_i{\lvert x_i \rvert^p}` is calculated along the ``axis`` dimension.
Parameters:
x (Tensor): The input tensor could be N-D tensor, and the input data type could be float32 or float64.
Alias: ``input``.
p (float|int, optional): The exponent value in the norm formulation. Default: 2.
axis (int, optional): The axis on which to apply normalization. If `axis < 0`, the dimension to normalization is `x.ndim + axis`. -1 is the last dimension.
Alias: ``dim``.
epsilon (float, optional): Small float added to denominator to avoid dividing by zero. Default is 1e-12.
Alias: ``esp``.
name (str|None, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
out (Tensor|None, optional): The output tensor. Default: None.
Returns:
Tensor, the output has the same shape and data type with ``x``.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn.functional as F
>>> paddle.disable_static()
>>> x = paddle.arange(6, dtype="float32").reshape([2, 3])
>>> y = F.normalize(x)
>>> print(y)
Tensor(shape=[2, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
[[0. , 0.44721359, 0.89442718],
[0.42426407, 0.56568545, 0.70710677]])
>>> y = F.normalize(x, p=1.5)
>>> print(y)
Tensor(shape=[2, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
[[0. , 0.40862012, 0.81724024],
[0.35684019, 0.47578692, 0.59473366]])
>>> y = F.normalize(x, axis=0)
>>> print(y)
Tensor(shape=[2, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
[[0. , 0.24253564, 0.37139067],
[1. , 0.97014254, 0.92847669]])
"""
if in_dygraph_mode():
eps = paddle.full(shape=[1], fill_value=epsilon, dtype=x.dtype)
ret = _C_ops.p_norm(x, float(p), axis, epsilon, True, False)
ret = x / _C_ops.maximum(ret, eps)
elif in_pir_mode():
eps = paddle.full(shape=[1], fill_value=epsilon, dtype=x.dtype)
ret = _C_ops.p_norm(x, float(p), axis, epsilon, True, False)
ret = paddle.divide(x, _C_ops.maximum(ret, eps), name=name)
else:
check_type(p, 'p', (float, int), 'normalize')
check_type(axis, 'axis', (int), 'normalize')
check_variable_and_dtype(
x, 'x', ['float16', 'float32', 'float64'], 'normalize'
)
if len(x.shape) == 1 and axis != 0 and axis != -1:
raise ValueError(
f"Axis must be 0 or -1 when x is a 1-D tensor, but received axis = {axis}"
)
attrs = {
'axis': axis,
'porder': float(p),
'keepdim': True,
'epsilon': epsilon,
}
helper = LayerHelper('p_norm', **locals())
out = helper.create_variable_for_type_inference(dtype=x.dtype)
helper.append_op(
type='p_norm', inputs={'X': x}, outputs={'Out': out}, attrs=attrs
)
eps = out.block.create_var(dtype=out.dtype)
eps = paddle.full(shape=[1], fill_value=epsilon, dtype=out.dtype)
out = paddle.divide(x, paddle.maximum(out, eps), name=name)
if out is not None:
paddle.assign(ret, out)
return out
return ret
@param_two_alias(["x", "input"], ["epsilon", "eps"])
def batch_norm(
x,
running_mean: Tensor,
running_var: Tensor,
weight: Tensor | None = None,
bias: Tensor | None = None,
training: bool = False,
momentum: float = 0.9,
epsilon: float = 1e-05,
data_format: DataLayoutND = 'NCHW',
use_global_stats: bool | None = None,
name: str | None = None,
) -> Tensor:
"""
Applies Batch Normalization as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift .
nn.functional.batch_norm is used for nn.BatchNorm1D, nn.BatchNorm2D, nn.BatchNorm3D. Please use above API for BatchNorm.
Parameters:
x(Tensor): input value. It's data type should be float32, float64.
running_mean(Tensor): running mean.
running_var(Tensor): running variance.
weight(Tensor, optional): The weight tensor of batch_norm. Default: None.
bias(Tensor, optional): The bias tensor of batch_norm. Default: None.
epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5.
training(bool, optional): True means train mode which compute by batch data and track global mean and var during train period. False means inference mode which compute by global mean and var which calculated by train period. Default False.
momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
data_format(str, optional): Specify the input data format, may be "NC", "NCL", "NCHW", "NCDHW", "NLC", "NHWC" or "NDHWC", where `N` is batch size, `C` is the number of the feature map, `D` is the depth of the feature, `H` is the height of the feature map, `W` is the width of the feature map, `L` is the length of the feature map. Default "NCHW".
use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None.
name(str|None, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..
Returns:
None
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.arange(12, dtype="float32").reshape([2, 1, 2, 3])
>>> print(x)
Tensor(shape=[2, 1, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[0. , 1. , 2. ],
[3. , 4. , 5. ]]],
[[[6. , 7. , 8. ],
[9. , 10., 11.]]]])
>>> running_mean = paddle.to_tensor([0], dtype="float32")
>>> running_variance = paddle.to_tensor([1], dtype="float32")
>>> weight = paddle.to_tensor([2], dtype="float32")
>>> bias = paddle.to_tensor([1], dtype="float32")
>>> batch_norm_out = paddle.nn.functional.batch_norm(
... x,
... running_mean,
... running_variance,
... weight,
... bias,
... )
>>> print(batch_norm_out)
Tensor(shape=[2, 1, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[1. , 2.99998999 , 4.99997997 ],
[6.99996996 , 8.99995995 , 10.99995041]]],
[[[12.99993992, 14.99992943, 16.99991989],
[18.99991035, 20.99990082, 22.99988937]]]])
"""
assert len(x.shape) >= 2, "input dim must be larger than 1"
# input ad out must share the memory
mean_out = running_mean
variance_out = running_var
true_data_format = ['NC', 'NCL', 'NCHW', 'NCDHW', 'NLC', 'NHWC', 'NDHWC']
if data_format not in true_data_format:
raise ValueError(
"data_format must be one of 'NC', 'NCL', 'NCHW', 'NCDHW', "
f"'NLC', 'NHWC', 'NDHWC' but receive {data_format}"
)
data_format = 'NCHW' if data_format[1] == 'C' else 'NHWC'
if use_global_stats is None:
use_global_stats = not training
trainable_statistics = False
else:
trainable_statistics = not use_global_stats
if in_dynamic_mode():
batch_norm_out, _, _, _, _, _ = _C_ops.batch_norm(
x,
running_mean,
running_var,
weight,
bias,
not training,
momentum,
epsilon,
data_format,
use_global_stats,
trainable_statistics,
)
return batch_norm_out
elif in_pir_mode():
batch_norm_out, t1, t2, t3, t4, _ = _C_ops.batch_norm_(
x,
running_mean,
running_var,
weight,
bias,
not training,
momentum,
epsilon,
data_format,
use_global_stats,
trainable_statistics,
)
return batch_norm_out
else:
check_variable_and_dtype(
x, 'input', ['float16', 'uint16', 'float32', 'float64'], 'BatchNorm'
)
# for static need dict
attrs = {
"momentum": momentum,
"epsilon": epsilon,
"is_test": not training,
"data_layout": data_format,
"fuse_with_relu": False,
"use_global_stats": use_global_stats,
"trainable_statistics": trainable_statistics,
}
inputs = {
"X": [x],
"Mean": [running_mean],
"Variance": [running_var],
}
if weight:
inputs['Scale'] = [weight]
if bias:
inputs['Bias'] = [bias]
helper = LayerHelper('batch_norm', **locals())
from paddle.base.data_feeder import convert_dtype
param_dtype = (
x.dtype
if convert_dtype(x.dtype) not in ['float16', 'uint16']
else 'float32'
)
saved_mean = helper.create_variable_for_type_inference(
dtype=param_dtype, stop_gradient=True
)
saved_variance = helper.create_variable_for_type_inference(
dtype=param_dtype, stop_gradient=True
)
batch_norm_out = helper.create_variable_for_type_inference(x.dtype)
outputs = {
"Y": [batch_norm_out],
"MeanOut": [running_mean],
"VarianceOut": [running_var],
"SavedMean": [saved_mean],
"SavedVariance": [saved_variance],
}
if training or trainable_statistics:
# reserve_space is only used for training.
reserve_space = helper.create_variable_for_type_inference(
dtype=x.dtype, stop_gradient=True
)
outputs["ReserveSpace"] = [reserve_space]
helper.append_op(
type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs
)
return helper.append_activation(batch_norm_out)
@param_two_alias(["x", "input"], ["epsilon", "eps"])
def layer_norm(
x: Tensor,
normalized_shape: int | Sequence[int],
weight: Tensor | None = None,
bias: Tensor | None = None,
epsilon: float = 1e-05,
name: str | None = None,
) -> Tensor:
"""
nn.LayerNorm is recommended.
For more information, please refer to :ref:`api_paddle_nn_LayerNorm` .
.. note::
Alias Support: The parameter name ``input`` can be used as an alias for ``x`` and the parameter name ``eps`` can be used as an alias for ``epsilon``.
For example, ``layer_norm(input=tensor_x, eps=1e-5)`` is equivalent to ``layer_norm(x=tensor_x, epsilon=1e-5)``.
Parameters:
x(Tensor): Input Tensor. It's data type should be bfloat16, float16, float32, float64.
alias: ``input``.
normalized_shape(int|list|tuple): Input shape from an expected input of
size :math:`[*, normalized_shape[0], normalized_shape[1], ..., normalized_shape[-1]]`.
If it is a single integer, this module will normalize over the last dimension
which is expected to be of that specific size.
weight(Tensor, optional): The weight tensor of layer_norm. Default: None.
bias(Tensor, optional): The bias tensor of layer_norm. Default: None.
epsilon(float, optional): The small value added to the variance to prevent
division by zero. Default: 1e-05.
alias: ``eps``.
name(str, optional): Name for the LayerNorm, default is None. For more information, please refer to :ref:`api_guide_Name` .
Returns:
None
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(2023)
>>> x = paddle.rand((2, 2, 2, 3))
>>> layer_norm_out = paddle.nn.functional.layer_norm(x, x.shape[1:])
>>> print(layer_norm_out)
Tensor(shape=[2, 2, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[ 0.87799621, -0.32706589, -1.23529351],
[ 1.01540303, -0.66222930, -0.72354060]],
[[ 1.24183691, 0.45458117, -0.33506936],
[ 0.41468447, 1.26852846, -1.98983335]]],
[[[ 0.02837802, 1.27684653, -0.90110677],
[-0.94709361, -0.15110941, -1.16546965]],
[[-0.82010192, 0.11218391, -0.86506510],
[ 1.09489346, 0.19107464, 2.14656830]]]])
"""
input_shape = list(x.shape)
input_ndim = len(input_shape)
if isinstance(normalized_shape, numbers.Integral):
normalized_shape = [normalized_shape]
elif isinstance(normalized_shape, tuple):
normalized_shape = list(normalized_shape)
elif not isinstance(normalized_shape, list):
raise ValueError(
"`normalized_shape` should be int, list of ints or tuple of ints."
)
normalized_ndim = len(normalized_shape)
begin_norm_axis = input_ndim - normalized_ndim
if input_ndim < normalized_ndim or (
not paddle.utils.is_same_shape(
input_shape[begin_norm_axis:], normalized_shape
)
):
str_normalized_shape = str(normalized_shape)
raise ValueError(
'Given normalized_shape is '
+ str_normalized_shape
+ ', expected input with shape [*, '
+ str_normalized_shape[1:]
+ ', but got input shape '
+ str(input_shape)
)
if in_dynamic_or_pir_mode():
out = _C_ops.layer_norm(x, weight, bias, epsilon, begin_norm_axis)
return out
else:
check_variable_and_dtype(
x, 'input', ['uint16', 'float16', 'float32', 'float64'], 'LayerNorm'
)
inputs = {}
inputs['X'] = [x]
if weight:
inputs['Scale'] = [weight]
if bias:
inputs['Bias'] = [bias]
attrs = {"epsilon": epsilon, "begin_norm_axis": begin_norm_axis}
# create output
helper = LayerHelper('layer_norm', **locals())
from paddle.base.data_feeder import convert_dtype
param_dtype = (
x.dtype if convert_dtype(x.dtype) != 'float16' else 'float32'
)
mean_out = helper.create_variable_for_type_inference(
dtype=param_dtype, stop_gradient=True
)
variance_out = helper.create_variable_for_type_inference(
dtype=param_dtype, stop_gradient=True
)
layer_norm_out = helper.create_variable_for_type_inference(x.dtype)
helper.append_op(
type="layer_norm",
inputs=inputs,
outputs={
"Y": layer_norm_out,
"Mean": mean_out,
"Variance": variance_out,
},
attrs={"epsilon": epsilon, "begin_norm_axis": begin_norm_axis},
)
return helper.append_activation(layer_norm_out)
def rms_norm(
input: Tensor,
normalized_shape: Sequence[int],
weight: Tensor | None = None,
eps: float | None = None,
name: str | None = None,
) -> Tensor:
"""
Applies Layer Normalization over the last dimension of the input tensor using CUDA implementation.
Args:
input (Tensor): Input tensor of shape [rows, cols] or higher dimensions (flattened to 2D).
normalized_shape(list|tuple): Input shape from an expected input of
size :math:`[*, normalized_shape[0], normalized_shape[1], ..., normalized_shape[-1]]`.
If it is a single integer, this module will normalize over the last dimension
which is expected to be of that specific size.
weight(Tensor, optional): The weight tensor of rms_norm. Default: None.
eps(float|None, optional): The small value added to the variance to prevent division by zero.
If None, uses machine epsilon for the compute dtype: ``float64`` inputs use
``np.finfo(np.float64).eps`` (double epsilon), all other dtypes use
``np.finfo(np.float32).eps`` (float epsilon). Default: None.
name (str, optional): Name of the operator.
Returns:
out (Tensor): Normalized tensor of same shape as input.
"""
if eps is None:
if input.dtype == paddle.float64:
eps = np.finfo(np.float64).eps # ~2.22e-16, double machine epsilon
else:
eps = np.finfo(np.float32).eps # ~1.19e-7, float machine epsilon
return _C_ops.rms_norm(input, weight, normalized_shape, eps)[0]
@param_one_alias(["x", "input"])
def instance_norm(
x: Tensor,
running_mean: Tensor | None = None,
running_var: Tensor | None = None,
weight: Tensor | None = None,
bias: Tensor | None = None,
use_input_stats: bool = True,
momentum: float = 0.9,
eps: float = 1e-05,
data_format: Literal['NC', 'NCL', 'NCHW', 'NCDHW'] = 'NCHW',
name: str | None = None,
) -> Tensor:
"""
It is recommended to use :ref:`api_paddle_nn_InstanceNorm1D` , :ref:`api_paddle_nn_InstanceNorm2D` , :ref:`api_paddle_nn_InstanceNorm3D` to call this method internally.
Parameters:
x (Tensor): Input Tensor. It's data type should be float32, float64.
running_mean (Tensor, optional): running mean. Default None. Obsolete (that is, no longer usable).
running_var (Tensor, optional): running variance. Default None. Obsolete (that is, no longer usable).
weight (Tensor, optional): The weight tensor of instance_norm. Default: None.
If its value is None, this parameter will be initialized by one.
bias (Tensor, optional): The bias tensor of instance_norm. Default: None.
If its value is None, this parameter will be initialized by zero.
eps (float, optional): A value added to the denominator for numerical stability. Default is 1e-5.
momentum (float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9.
use_input_stats (bool, optional): Default True. Obsolete (that is, no longer usable).
data_format (str, optional): Specify the input data format, may be "NC", "NCL", "NCHW" or "NCDHW". Default "NCHW".
name (str|None, optional): Name for the InstanceNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..
Returns:
None.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(2023)
>>> x = paddle.rand((2, 2, 2, 3))
>>> instance_norm_out = paddle.nn.functional.instance_norm(x)
>>> print(instance_norm_out)
Tensor(shape=[2, 2, 2, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[ 1.25768495, -0.18054862, -1.26451230],
[ 1.42167914, -0.58056390, -0.65373862]],
[[ 0.95882601, 0.25075224, -0.45947552],
[ 0.21486834, 0.98283297, -1.94780385]]],
[[[ 0.40697321, 1.90885782, -0.71117985],
[-0.76650119, 0.19105314, -1.02920341]],
[[-1.06926346, -0.18710862, -1.11180890],
[ 0.74275863, -0.11246002, 1.73788261]]]])
"""
if in_dynamic_or_pir_mode():
out = _C_ops.instance_norm(x, weight, bias, eps)
return out
else:
check_variable_and_dtype(
x,
'input',
['float32', 'float64', 'float16', 'uint16'],
"InstanceNorm",
)
attrs = {
"epsilon": eps,
"momentum": momentum,
"data_format": data_format,
}
if weight and bias:
inputs = {"X": [x], "Scale": [weight], "Bias": [bias]}
else:
inputs = {"X": [x]}
helper = LayerHelper('instance_norm', **locals())
saved_mean = helper.create_variable_for_type_inference(
dtype=x.dtype, stop_gradient=True
)
saved_variance = helper.create_variable_for_type_inference(
dtype=x.dtype, stop_gradient=True
)
instance_norm_out = helper.create_variable_for_type_inference(x.dtype)
outputs = {
"Y": [instance_norm_out],
"SavedMean": [saved_mean],
"SavedVariance": [saved_variance],
}
helper.append_op(
type="instance_norm", inputs=inputs, outputs=outputs, attrs=attrs
)
return instance_norm_out
def local_response_norm(
x: Tensor,
size: int,
alpha: float = 0.0001,
beta: float = 0.75,
k: float = 1.0,
data_format: DataLayout1D | DataLayout2D | DataLayout3D = 'NCHW',
name: str | None = None,
) -> Tensor:
r"""
Local Response Normalization performs a type of "lateral inhibition" by normalizing over local input regions.
For more information, please refer to `ImageNet Classification with Deep Convolutional Neural Networks <https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf>`_
The formula is as follows:
.. math::
Output(i, x, y) = Input(i, x, y) / \left(k + \alpha \sum\limits^{\min(C-1, i + size/2)}_{j = \max(0, i - size/2)}(Input(j, x, y))^2\right)^{\beta}
In the above equation:
- :math:`size` : The number of channels to sum over.
- :math:`k` : The offset (avoid being divided by 0).
- :math:`\\alpha` : The scaling parameter.
- :math:`\\beta` : The exponent parameter.
Args:
x (Tensor): The input 3-D/4-D/5-D tensor. The data type is float16 or float32.
size (int): The number of channels to sum over.
alpha (float, optional): The scaling parameter, positive. Default:1e-4
beta (float, optional): The exponent, positive. Default:0.75
k (float, optional): An offset, positive. Default: 1.0
data_format (str, optional): Specify the data format of the input, and the data format of the output
will be consistent with that of the input. An optional string from:
If x is 3-D Tensor, the string could be `"NCL"` or `"NLC"` . When it is `"NCL"`,
the data is stored in the order of: `[batch_size, input_channels, feature_length]`.
If x is 4-D Tensor, the string could be `"NCHW"`, `"NHWC"`. When it is `"NCHW"`,
the data is stored in the order of: `[batch_size, input_channels, input_height, input_width]`.
If x is 5-D Tensor, the string could be `"NCDHW"`, `"NDHWC"` . When it is `"NCDHW"`,
the data is stored in the order of: `[batch_size, input_channels, input_depth, input_height, input_width]`.
name (str|None, optional): Name for the operation (optional, default is None). For more information,
please refer to :ref:`api_guide_Name`.
Returns:
A tensor storing the transformation result with the same shape and data type as input.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.rand(shape=(3, 3, 112, 112), dtype="float32")
>>> y = paddle.nn.functional.local_response_norm(x, size=5)
>>> print(y.shape)
paddle.Size([3, 3, 112, 112])
"""
if not in_dynamic_mode():
check_variable_and_dtype(
x, 'x', ['float16', 'float32'], 'local_response_norm'
)
if data_format not in ['NCL', 'NLC', 'NCHW', 'NHWC', 'NCDHW', 'NDHWC']:
raise ValueError(
"data_format should be in one of [NCL, NCHW, NCDHW, NLC, NHWC, NDHWC], "
f"but got {data_format}"
)
sizes = x.shape
dim = len(sizes)
if dim < 3:
raise ValueError(
f'Expected 3D or higher dimensionality input, but got {dim} dimensions'
)
for i, sz in enumerate(sizes):
if not sz > 0 and i > 0:
raise ValueError(
"Expected every dim's size to be larger than 0, "
f"but the size of the {i}-th dim is {sz}"
)
channel_last = True if data_format[-1] == "C" else False
from functools import reduce
sum_sizes = reduce(lambda x, y: x * y, sizes[1:], 1)
div = paddle.unsqueeze(paddle.multiply(x, x), axis=1)
if not channel_last:
pad4d_shape = [0, 0, size // 2, (size - 1) // 2]
pool2d_shape = (size, 1)
reshape_shape = [
sizes[0],
1,
sizes[1],
sizes[2],
int(sum_sizes / (sizes[1] * sizes[2])),
]
pad5d_shape = [0, 0, 0, 0, size // 2, (size - 1) // 2]
pool3d_shape = (size, 1, 1)
else:
pad4d_shape = [size // 2, (size - 1) // 2, 0, 0]
pool2d_shape = (1, size)
reshape_shape = [
sizes[0],
1,
sizes[1],
int(sum_sizes / (sizes[1] * sizes[-1])),
sizes[-1],
]
pad5d_shape = [size // 2, (size - 1) // 2, 0, 0, 0, 0]
pool3d_shape = (1, 1, size)
if dim == 3:
div = paddle.nn.functional.pad(div, pad=pad4d_shape)
div = paddle.nn.functional.avg_pool2d(
div, kernel_size=pool2d_shape, stride=1
)
div = paddle.squeeze(div, axis=1)
else:
div = paddle.reshape(div, shape=reshape_shape)
div = paddle.nn.functional.pad(
div, pad=pad5d_shape, data_format='NCDHW'
)
div = paddle.nn.functional.avg_pool3d(
div, kernel_size=pool3d_shape, stride=1
)
div = paddle.reshape(paddle.squeeze(div, axis=1), sizes)
div = paddle.scale(div, scale=alpha, bias=k)
div = paddle.pow(div, beta)
res = paddle.divide(x, div, name=name)
return res
@overload
def group_norm(
x: Tensor,
num_groups: int,
epsilon: float = 1e-05,
weight: Tensor | None = None,
bias: Tensor | None = None,
data_format: DataLayout1D | DataLayout2D | DataLayout3D = 'NCHW',
name: str | None = None,
) -> Tensor: ...
@overload
def group_norm(
input: Tensor,
num_groups: int,
weight: Tensor | None = None,
bias: Tensor | None = None,
eps: float = 1e-05,
) -> Tensor: ...
def group_norm(*args: Any, **kwargs: Any) -> Tensor:
"""
nn.GroupNorm is recommended.
For more information, please refer to :ref:`api_paddle_nn_GroupNorm` .
This function has two functionalities, depending on the parameters passed:
1. ``group_norm(Tensor input, int num_groups, Tensor weight = None, Tensor bias = None, float eps = 1e-05)``:
PyTorch compatible group_norm.
2. ``group_norm(Tensor x, int num_groups, float epsilon = 1e-05, Tensor weight = None, Tensor bias = None,
DataLayout1D | DataLayout2D | DataLayout3D data_format = 'NCHW', str | None name = None)``:
The original paddle.nn.functional.group_norm, see the following docs.
Parameters:
x(Tensor): Input Tensor with shape: attr:`(batch, num_features, *)`.
alias: ``input``.
num_groups(int): The number of groups that divided from channels.
epsilon(float, optional): The small value added to the variance to prevent
division by zero. Default: 1e-05.
alias: ``eps``.
weight(Tensor, optional): The weight Tensor of group_norm, with shape: attr:`[num_channels]`.
Default: None.
bias(Tensor, optional): The bias Tensor of group_norm, with shape: attr:`[num_channels]`.
Default: None.
data_format(str, optional): Specify the input data format. Support "NCL", "NCHW", "NCDHW", "NLC", "NHWC" or "NDHWC". Default: "NCHW".
name(str|None, optional): Name for the GroupNorm, default is None. For more information, please refer to :ref:`api_guide_Name`..
Returns:
Tensor, the output has the same shape with ``x``.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(100)
>>> x = paddle.arange(48, dtype="float32").reshape((2, 6, 2, 2))
>>> group_norm_out = paddle.nn.functional.group_norm(x, num_groups=6)
>>> print(group_norm_out)
Tensor(shape=[2, 6, 2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]]],
[[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]],
[[-1.34163547, -0.44721183],
[ 0.44721183, 1.34163547]]]])
"""
len_args = len(args)
if len_args + len(kwargs) < 2:
raise TypeError(
f"Too few arguments in the function call: {len_args}, {len(kwargs)}. Expect one of: \n"
" - (Tensor input, int num_groups, Tensor weight = None, Tensor bias = None, float eps = 1e-05)\n"
" - (Tensor x, int num_groups, float epsilon = 1e-05, Tensor weight = None, Tensor bias = None, "
"DataLayout1D | DataLayout2D | DataLayout3D data_format = 'NCHW', str | None name = None)"
)
def safe_set_param(key: str, value: Any):
if key in kwargs:
raise TypeError(f"got multiple values for argument '{key}'")
kwargs[key] = value
if 'input' in kwargs:
safe_set_param('x', kwargs.pop('input'))
if 'eps' in kwargs:
safe_set_param('epsilon', kwargs.pop('eps'))
if len_args >= 3 and not isinstance(args[2], float):
param_keys = ["weight", "bias", "epsilon"]
for idx in range(min(len_args - 2, len(param_keys))):
safe_set_param(param_keys[idx], args[idx + 2])
args = args[:2]
return _group_norm_wrapper(*args, **kwargs)
def _group_norm_wrapper(
x: Tensor,
num_groups: int,
epsilon: float = 1e-05,
weight: Tensor | None = None,
bias: Tensor | None = None,
data_format: DataLayout1D | DataLayout2D | DataLayout3D = 'NCHW',
name: str | None = None,
) -> Tensor:
if data_format not in ['NCL', 'NCHW', 'NCDHW', 'NLC', 'NHWC', 'NDHWC']:
raise ValueError("unsupported data layout:" + data_format)
data_format = 'NCHW' if data_format[1] == 'C' else 'NHWC'
if in_dynamic_or_pir_mode():
return _C_ops.group_norm(
x,
weight,
bias,
epsilon,
num_groups,
data_format,
)
else:
helper = LayerHelper('group_norm', **locals())
mean_out = helper.create_variable_for_type_inference(
dtype=x.dtype, stop_gradient=True
)
variance_out = helper.create_variable_for_type_inference(
dtype=x.dtype, stop_gradient=True
)
inputs = {'X': x}
if bias is not None:
inputs['Bias'] = bias
if weight is not None:
inputs['Scale'] = weight
# create output
group_norm_out = helper.create_variable_for_type_inference(
dtype=x.dtype
)
helper.append_op(
type="group_norm",
inputs=inputs,
outputs={
"Y": group_norm_out,
"Mean": mean_out,
"Variance": variance_out,
},
attrs={
"epsilon": epsilon,
"groups": num_groups,
"data_layout": data_format,
},
)
return helper.append_activation(group_norm_out)
group_norm.__signature__ = inspect.signature(_group_norm_wrapper)
python/paddle/nn/functional/pooling.py
Executable
+2822
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/functional/sdpa.py
+681
View File
@@ -0,0 +1,681 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from dataclasses import dataclass
from functools import cached_property, lru_cache
from typing import TYPE_CHECKING
import paddle
from paddle import _C_ops
from paddle.base.log_helper import get_logger
from paddle.nn.attention.sdpa import (
SDPBackend,
_get_backend_priority,
_get_enabled_backends,
)
_logger = get_logger(
__name__, "INFO", fmt='%(asctime)s-%(levelname)s: %(message)s'
)
from paddle.nn.functional.flash_attention import _math_attention
if TYPE_CHECKING:
from paddle import Tensor, dtype
from paddle.base.core import Place
_config = {}
def init_config():
global _config
_config = {
"flash_attn": {
"MINIMUM_SM_VERSION": (8, 0),
"MAXIMUM_SM_VERSION": (12, 1),
"support_dtypes": (paddle.float16, paddle.bfloat16)
if paddle.device.is_bf16_supported(including_emulation=False)
else (paddle.float16,),
},
"mem_efficient_attn": {
"MINIMUM_SM_VERSION": (5, 0),
"MAXIMUM_SM_VERSION": (12, 1),
"support_dtypes": (
paddle.float16,
paddle.bfloat16,
paddle.float,
)
if paddle.device.is_bf16_supported(including_emulation=False)
else (paddle.float16, paddle.float),
},
}
def _repeat_kv(key: Tensor, value: Tensor, num_repeats: int):
"""
Repeat key and value tensors along the num_heads(3) dimension. The layout
of key and value should be [batch_size, seq_len, num_heads, head_dim].
"""
if num_repeats == 1:
return key, value
# repeat_interleave does not support float16 on GPU, so we manually expand the tensor
key, value = key.unsqueeze(3), value.unsqueeze(3)
key, value = (
key.expand([-1, -1, -1, num_repeats, -1]),
value.expand([-1, -1, -1, num_repeats, -1]),
)
key, value = (
key.flatten(2, 3).contiguous(),
value.flatten(2, 3).contiguous(),
)
return key, value
@dataclass
class SDPParams:
query_shape: paddle.Size
key_shape: paddle.Size
value_shape: paddle.Size
attn_mask_shape: paddle.Size | None
dropout: float
is_causal: bool
scale: float | None
query_stop_gradient: bool
dtype: tuple[dtype, dtype, dtype]
place: tuple[Place, Place, Place]
@cached_property
def batch_size(self) -> tuple[int, int, int]:
return self.query_shape[0], self.key_shape[0], self.value_shape[0]
@cached_property
def seq_len(self) -> tuple[int, int, int]:
return self.query_shape[1], self.key_shape[1], self.value_shape[1]
@cached_property
def num_heads(self) -> tuple[int, int, int]:
return self.query_shape[2], self.key_shape[2], self.value_shape[2]
@cached_property
def head_dim(self) -> tuple[int, int, int]:
return self.query_shape[-1], self.key_shape[-1], self.value_shape[-1]
@cached_property
def device_id(self) -> tuple[int, ...]:
ret = tuple(
pl.gpu_device_id() if pl.is_gpu_place() else -1 for pl in self.place
)
return ret
@lru_cache(maxsize=8)
def get_device_capability(device_id: int) -> tuple[int, int]:
if device_id < 0:
return (0, 0)
return paddle.device.cuda.get_device_capability(device_id)
@lru_cache(maxsize=32)
def check_sm_version(
min_sm: tuple[int, int], max_sm: tuple[int, int], device_id: int = 0
) -> bool:
major, minor = get_device_capability(device_id)
current = (major, minor)
return min_sm <= current <= max_sm
@lru_cache(maxsize=1)
def check_cuda_is_available() -> bool:
return paddle.is_compiled_with_cuda() and paddle.cuda.is_available()
def check_all_tensors_on_device(params: SDPParams):
"""
Check all input tensors are placed on the GPU device.
"""
if not (
params.place[0].is_gpu_place() or params.place[0].is_custom_place()
):
_logger.debug(
"All input tensors should be placed on GPU or custom place, but "
f"query place: {params.place[0]}, key place: "
f"{params.place[1]}, value place: {params.place[2]}"
)
return False
return True
def check_head_dim_size_flash(params: SDPParams):
"""
Check the dimension of head in query, key, and value should be equal and all less than 256.
"""
q_head_dim, k_head_dim, v_head_dim = params.head_dim
if q_head_dim > 256 or q_head_dim != k_head_dim or k_head_dim != v_head_dim:
_logger.debug(
"The dimension of head in query, key, and value should be equal and all less than 256, "
f"but q_head_dim: {q_head_dim}, k_head_dim: {k_head_dim}, v_head_dim: {v_head_dim}"
)
return False
if q_head_dim % 8 != 0:
_logger.debug(
"The dimension of head in query, key, and value should be a multiple of 8, "
f"but q_head_dim: {q_head_dim}"
)
return False
return True
@lru_cache(maxsize=8)
def check_flash_attention_hardware_support(device_id: int):
"""
Check flash attention requires CUDA support and SM between 8.0 and 12.1.
"""
if SDPBackend.FLASH_ATTENTION and paddle.is_compiled_with_custom_device(
paddle.device.get_all_device_type()[0]
):
return True
if not check_cuda_is_available():
_logger.debug("Flash attention requires CUDA support.")
return False
if not check_sm_version(
_config["flash_attn"]["MINIMUM_SM_VERSION"],
_config["flash_attn"]["MAXIMUM_SM_VERSION"],
device_id,
):
_logger.debug(
f"Flash attention requires SM between {_config['flash_attn']['MINIMUM_SM_VERSION']}"
f"and {_config['flash_attn']['MAXIMUM_SM_VERSION']}, but found SM "
f"{get_device_capability(device_id)}"
)
return False
return True
def check_flash_causal_non_square_seqlens(params: SDPParams):
"""
Check flash attention only supports causal attention when the sequence length of query and key are equal.
"""
if not params.is_causal:
return True
q_len, k_len, _ = params.seq_len
if q_len == k_len:
return True
_logger.debug(
f"Flash attention only supports causal attention when the sequence"
f"length of query and key are equal, but got query shape: "
f"{params.query_shape}, key shape: {params.key_shape}"
)
return False
def check_dtypes_low_precision_fa(params: SDPParams):
"""
check QKV share the same dtype and are supported dtype.
"""
q_dtype, k_dtype, v_dtype = params.dtype
if (
q_dtype != k_dtype
or v_dtype != k_dtype
or q_dtype not in _config["flash_attn"]["support_dtypes"]
):
_logger.debug(
f"Flash attention requires query, key, and value "
f"to be of the same dtype and support dtype, but "
f"got query dtype: {q_dtype}, key dtype: {k_dtype}"
f", value dtype: {v_dtype}. Supported dtypes are: "
f"{_config['flash_attn']['support_dtypes']}"
)
return False
return True
def check_dtypes_low_precision_mem_efficient_attn(params: SDPParams):
"""
check QKV share the same dtype and are supported dtype.
"""
q_dtype, k_dtype, v_dtype = params.dtype
if (
q_dtype != k_dtype
or v_dtype != k_dtype
or q_dtype not in _config["mem_efficient_attn"]["support_dtypes"]
):
_logger.debug(
f"Mem_efficient_attn requires query, key, and value "
f"to be of the same dtype and support dtype, but "
f"got query dtype: {q_dtype}, key dtype: {k_dtype}"
f", value dtype: {v_dtype}. Supported dtypes are: "
f"{_config['mem_efficient_attn']['support_dtypes']}"
)
return False
return True
@lru_cache(maxsize=2)
def use_tensor_cores(is_half: bool, device_id: int) -> bool:
major, _ = get_device_capability(device_id)
if major >= 8:
return True
if major == 7:
return is_half
return False
@lru_cache(maxsize=32)
def minimum_gemm_alignment(dtype: dtype, device_id: int):
is_half = dtype in (paddle.float16, paddle.bfloat16)
use_tc = use_tensor_cores(is_half, device_id)
major, _ = get_device_capability(device_id)
matmul_alignment_mn = 4 if major > 8 else 1
bits_per_scalar = 16 if is_half else 32
if use_tc:
matmul_alignment_mn = max(matmul_alignment_mn, 128 / bits_per_scalar)
return matmul_alignment_mn
@lru_cache(maxsize=8)
def check_mem_efficient_hardware_support(device_id: int):
"""
Check mem_efficient attention requires CUDA support and SM between 5.0 and 12.1.
"""
if not check_cuda_is_available():
_logger.debug("Mem efficient attention requires CUDA support.")
return False
if not check_sm_version(
_config["mem_efficient_attn"]["MINIMUM_SM_VERSION"],
_config["mem_efficient_attn"]["MAXIMUM_SM_VERSION"],
device_id,
):
_logger.debug(
f"Mem efficient attention requires SM between {_config['mem_efficient_attn']['MINIMUM_SM_VERSION']}"
f"and {_config['mem_efficient_attn']['MAXIMUM_SM_VERSION']}, but found SM "
f"{get_device_capability(device_id)}"
)
return False
return True
def check_head_dim_size_mem_efficient(params: SDPParams):
q_head_dim, k_head_dim, v_head_dim = (
params.query_shape[-1],
params.key_shape[-1],
params.value_shape[-1],
)
alignment = minimum_gemm_alignment(params.dtype[0], params.device_id[0])
if (
q_head_dim % alignment != 0
or k_head_dim % alignment != 0
or v_head_dim % alignment != 0
):
_logger.debug(
f"Mem efficient attention requires head dim size aligned to {alignment}, "
f"but found q_head_dim: {q_head_dim}, k_head_dim: {k_head_dim}, v_head_dim: {v_head_dim}"
)
return False
return True
def check_attn_mask_alignment(params: SDPParams) -> bool:
if params.is_causal:
return True
if params.attn_mask_shape is None:
return True
last_dim = params.attn_mask_shape[-1]
if last_dim % 8 != 0:
_logger.debug(
f"Mem efficient attention requires attn_mask last dimension to be divisible by 8 "
f"to satisfy vector alignment, but got {last_dim}. "
"Falling back to other backends."
)
return False
return True
def check_scale_is_None(params: SDPParams) -> bool:
if params.scale is None:
return True
_logger.debug("Paddle's FAV2 does not support scale parameter.")
return False
def can_use_flash_attention(params: SDPParams = False) -> bool:
general_constraints = [
check_all_tensors_on_device,
check_head_dim_size_flash,
check_flash_causal_non_square_seqlens,
check_dtypes_low_precision_fa,
check_scale_is_None,
]
for constraint in general_constraints:
if not constraint(params):
return False
if not check_flash_attention_hardware_support(params.device_id[0]):
return False
return True
def can_use_mem_efficient_attention(params: SDPParams = False) -> bool:
constraints = [
check_all_tensors_on_device,
check_head_dim_size_mem_efficient,
check_attn_mask_alignment,
check_dtypes_low_precision_mem_efficient_attn,
]
for constraint in constraints:
if not constraint(params):
return False
if not check_mem_efficient_hardware_support(params.device_id[0]):
return False
return True
def select_sdp_for_sdpa(param: SDPParams) -> str:
# Note: This API is designed for nn.functional.scaled_dot_product_attention,
# and is **NOT** expected to be called by others. Some promises should be guaranteed
# by caller to skip some rarely unmet constraints:
# 1. The input dim is 4, layout is (batch, seq_len, num_heads, head_dim)
# 2. The batch_size and num_heads of each input should be the same
place = paddle.get_device()
if "xpu" in place:
return "flash_attn"
enabled_backends = _get_enabled_backends()
priority_order = _get_backend_priority()
for backend in priority_order:
if backend not in enabled_backends:
continue
if backend == SDPBackend.FLASH_ATTENTION:
if can_use_flash_attention(param):
return "flash_attn"
elif backend == SDPBackend.EFFICIENT_ATTENTION:
if can_use_mem_efficient_attention(param):
return "mem_efficient"
elif backend == SDPBackend.MATH:
return "math"
raise RuntimeError(
"No available backend for scaled_dot_product_attention was found."
)
def scaled_dot_product_attention(
query: Tensor,
key: Tensor,
value: Tensor,
attn_mask: Tensor | None = None,
dropout_p: float = 0.0,
is_causal: bool = False,
training: bool = True,
backend: str | None = None,
scale: float | None = None,
enable_gqa: bool = True,
name: str | None = None,
) -> Tensor:
r"""
The equation is:
.. math::
result=softmax(\frac{ Q * K^T }{\sqrt{d}}) * V
where : ``Q``, ``K``, and ``V`` represent the three input parameters of the attention module.
The dimensions of the three parameters are the same.
``d`` represents the size of the last dimension of the three parameters.
Warning:
This API only verifies inputs with dtype float16 and bfloat16, other dtypes may fall back to math
implementation, which is less optimized.
Warning:
If is_causal is set to True, the causal mask should not be provided, otherwise
the provided mask will be ignored.
Note:
This API differs from :ref:`api_paddle_compat_nn_functional_scaled_dot_product_attention` in that:
1. The QKV layout of this API is [batch_size, seq_len, num_heads, head_dim] or [seq_len, num_heads, head_dim].
If you need num_heads before seq_len layout, please use ``paddle.compat.nn.functional.scaled_dot_product_attention``.
Args:
query(Tensor): The query tensor in the Attention module.
4-D tensor with shape:
[batch_size, seq_len_key, num_heads, head_dim].
3-D tensor with shape:
[seq_len_key, num_heads, head_dim].
The dtype can be float16 or bfloat16.
key(Tensor): The key tensor in the Attention module.
4-D tensor with shape:
[batch_size, seq_len_key, num_heads, head_dim].
3-D tensor with shape:
[seq_len_key, num_heads, head_dim].
The dtype can be float16 or bfloat16.
value(Tensor): The value tensor in the Attention module.
4-D tensor with shape:
[batch_size, seq_len_value, num_heads, head_dim].
3-D tensor with shape:
[seq_len_value, num_heads, head_dim].
The dtype can be float16 or bfloat16.
attn_mask(Tensor, optional): The attention mask tensor. The shape should be broadcastable to
[batch_size, num_heads, seq_len_key, seq_len_query]. The dtype can be bool
or same type of query. The bool mask indicates the positions should take part
in attention. The non-bool mask will be added to attention score.
dropout_p(float, optional): The dropout ratio.
is_causal(bool, optional): Whether enable causal mode.
training(bool, optional): Whether it is in the training phase.
backend(str, optional): Specify which backend to compute scaled dot product attention.
Currently only support "p2p" for distribution usage.
scale(float, optional): The scaling factor used in the calculation of attention weights.
If None, scale = 1 / sqrt(head_dim).
enable_gqa(bool, optional): Whether enable GQA(Group Query Attention) mode. Default is True.
name(str|None, optional): The default value is None. Normally there is no need for user
to set this property. For more information, please refer to
:ref:`api_guide_Name`.
Returns:
out(Tensor): The attention tensor.
4-D tensor with shape: [batch_size, seq_len, num_heads, head_dim].
3-D tensor with shape: [seq_len, num_heads, head_dim].
The dtype can be float16 or bfloat16.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP('bfloat need V100 compile')
>>> import paddle
>>> q = paddle.rand((1, 128, 2, 16), dtype=paddle.bfloat16)
>>> output = paddle.nn.functional.scaled_dot_product_attention(q, q, q, None, 0.9, False)
>>> print(output)
>>> # doctest: -SKIP
"""
is_batched = query.dim() == 4
if not is_batched:
# FlashAttention backend does not support unbatched input,
# we add batch dim here and will skip check input dim when selecting FA backend.
query = query.unsqueeze(0)
key = key.unsqueeze(0)
value = value.unsqueeze(0)
k_heads, q_heads, v_heads = (
key.shape[2],
query.shape[2],
value.shape[2],
)
if enable_gqa:
assert k_heads == 0 or q_heads % k_heads == 0, (
f"The number of groups in query({q_heads}) must be divisible by the number of groups in key({k_heads}) if GQA enabled."
)
assert k_heads == v_heads, (
f"The number of groups in key({k_heads}) must be equal to the number of groups in value({v_heads}) if GQA enabled."
)
else:
assert q_heads == k_heads == v_heads, (
f"The number of groups in query({q_heads}) must be equal to the number of groups in key({k_heads}) "
f"and the number of groups in value({v_heads}) if GQA disabled."
)
bs, seq_len_q, num_heads_q, head_dim_q = query.shape
_, seq_len_k, num_heads_k, head_dim_k = key.shape
if (
backend == 'p2p'
and query.is_dist()
and key.is_dist()
and value.is_dist()
):
# ring attention for auto_parallel mode
assert scale is None, f"Backend {backend} not support scale parameter."
out = paddle.distributed.auto_parallel.ring_attention.RingFlashAttention.apply(
query,
key,
value,
attn_mask,
dropout_p,
is_causal,
)
return out
if not paddle.base.in_dygraph_mode():
qkv_place = (paddle.framework._current_expected_place_(),) * 3
else:
qkv_place = (query.place, key.place, value.place)
param = SDPParams(
query_shape=query.shape,
key_shape=key.shape,
value_shape=value.shape,
attn_mask_shape=attn_mask.shape if attn_mask is not None else None,
dropout=dropout_p,
is_causal=is_causal,
scale=scale,
query_stop_gradient=query.stop_gradient,
dtype=(query.dtype, key.dtype, value.dtype),
place=qkv_place,
)
if len(_config) == 0:
init_config()
is_zero_size = (
query.shape.numel() == 0
or key.shape.numel() == 0
or value.shape.numel() == 0
)
if attn_mask is not None:
if attn_mask.dtype == paddle.bool:
attn_mask = paddle.where(
attn_mask,
paddle.to_tensor(0.0, dtype=query.dtype),
paddle.to_tensor(-float('inf'), dtype=query.dtype),
)
if is_zero_size:
sdp_func_name = "math"
else:
sdp_func_name = select_sdp_for_sdpa(param)
_logger.debug("Selected backend:" + sdp_func_name)
if sdp_func_name == "flash_attn":
fixed_seed_offset = None
return_softmax = False
rng_name = ""
if attn_mask is not None:
if attn_mask.ndim == 2:
attn_mask = attn_mask.expand([bs, 1, *attn_mask.shape])
elif attn_mask.ndim == 3:
attn_mask = paddle.unsqueeze(attn_mask, axis=1)
out, _, _, _ = _C_ops.flash_attn(
query,
key,
value,
fixed_seed_offset,
attn_mask,
dropout_p,
is_causal,
return_softmax,
not training,
rng_name,
)
elif sdp_func_name == "mem_efficient":
from paddle.incubate.nn.memory_efficient_attention import (
LowerTriangularMask,
memory_efficient_attention,
)
repeats = q_heads // k_heads
key, value = _repeat_kv(key, value, repeats)
if is_causal:
attn_mask = LowerTriangularMask()
elif attn_mask is not None:
# if need broadcast, memory_efficient_attention requires to
# broadcast first two dim simultaneously
if attn_mask.dim() == 3:
attn_mask = attn_mask.unsqueeze(axis=1)
if attn_mask.dim() == 4 and (
attn_mask.shape[0] != bs ^ attn_mask.shape[1] != num_heads_q
):
attn_mask = attn_mask.expand(
[
bs,
num_heads_q,
attn_mask.shape[2],
attn_mask.shape[3],
]
)
out = memory_efficient_attention(
query,
key,
value,
attn_bias=attn_mask,
p=dropout_p,
scale=scale,
training=training,
)
elif sdp_func_name == "math":
repeats = q_heads // k_heads if k_heads != 0 else 1
key, value = _repeat_kv(key, value, repeats)
if attn_mask is not None and attn_mask.dim() == 3:
attn_mask = attn_mask.unsqueeze(axis=1)
out = _math_attention(
query,
key,
value,
attn_mask,
dropout_p,
is_causal,
False,
training,
scale,
)[0]
else:
raise ValueError(f"Invalid backend {backend}")
if not is_batched:
out = paddle.squeeze(out, axis=0)
return out
python/paddle/nn/functional/sparse_attention.py
+179
View File
@@ -0,0 +1,179 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import paddle
from paddle import _C_ops
from paddle.base.layer_helper import LayerHelper
def sparse_attention(
query: paddle.Tensor,
key: paddle.Tensor,
value: paddle.Tensor,
sparse_csr_offset: paddle.Tensor,
sparse_csr_columns: paddle.Tensor,
key_padding_mask: paddle.Tensor | None = None,
attn_mask: paddle.Tensor | None = None,
name: str | None = None,
) -> paddle.Tensor:
r"""
This operator sparsify the Attention matrix in Transformer module
to achieve the effect of reducing memory consumption and computation.
The sparse layout is expressed in CSR format and contains two parameters,
``offset`` and ``columns``. The equation is:
.. math::
result=softmax(\frac{ Q * K^T }{\sqrt{d}}) * V
where : ``Q``, ``K``, and ``V`` represent the three input parameters of the attention module.
The dimensions of the three parameters are the same.
``d`` represents the size of the last dimension of the three parameters.
Warning:
This API is only used in ``CUDA 11.3`` and above versions.
Args:
query(Tensor): The query tensor in the Attention module.
4-D tensor with shape:
[batch_size, num_heads, seq_len, head_dim].
The dtype can be float32 and float64.
key(Tensor): The key tensor in the Attention module.
4-D tensor with shape:
[batch_size, num_heads, seq_len, head_dim].
The dtype can be float32 and float64.
value(Tensor): The value tensor in the Attention module.
4-D tensor with shape:
[batch_size, num_heads, seq_len, head_dim].
The dtype can be float32 and float64.
sparse_csr_offset(Tensor): The sparsity feature in the Attention module
is expressed in the CSR format, and the offset represents
the number of non-zero elements in each row of the matrix.
3-D tensor with shape:
[batch_size, num_heads, seq_len + 1].
The dtype should be int32.
sparse_csr_columns(Tensor): The sparsity feature in the Attention module
is expressed in the CSR format, and the columns represent
the column index values of non-zero elements in the matrix.
3-D tensor with shape:
[batch_size, num_heads, sparse_nnz].
The dtype should be int32.
key_padding_mask(Tensor|None, optional):The key padding mask tensor in the Attention module.
2-D tensor with shape: [batch_size, seq_len].
The dtype can be float32 and float64.
A value of 0 means that the position is masked.
attn_mask(Tensor|None, optional):The attention mask tensor in the Attention module.
2-D tensor with shape: [seq_len, seq_len].
The dtype can be float32 and float64.
A value of 0 means that the position is masked.
name(str|None, optional): The default value is None. Normally there is no need for user
to set this property. For more information, please refer to
:ref:`api_guide_Name`.
Returns:
Tensor, 4-D tensor with shape:
[batch_size, num_heads, seq_len, head_dim].
The dtype can be float32 or float64.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP('This API is only used in CUDA11.3 and above.')
>>> import paddle
>>> paddle.disable_static()
>>> # `query`, `key` and `value` all have shape [1, 1, 4, 2]
>>> query = paddle.to_tensor(
... [[[[0, 1], [2, 3], [0, 1], [2, 3]]]],
... dtype="float32",
... )
>>> key = paddle.to_tensor([[[[0, 1], [2, 3], [0, 1], [2, 3]]]], dtype="float32")
>>> value = paddle.to_tensor([[[[0, 1], [2, 3], [0, 1], [2, 3]]]], dtype="float32")
>>> offset = paddle.to_tensor([[[0, 2, 4, 6, 8]]], dtype="int32")
>>> columns = paddle.to_tensor([[[0, 1, 0, 1, 2, 3, 2, 3]]], dtype="int32")
>>> print(offset.shape)
paddle.Size([1, 1, 5])
>>> print(columns.shape)
paddle.Size([1, 1, 8])
...
>>> key_padding_mask = paddle.to_tensor([[1, 1, 1, 0]], dtype="float32")
>>> attention_mask = paddle.to_tensor(
... [
... [1, 0, 1, 1],
... [1, 1, 1, 1],
... [1, 1, 1, 1],
... [1, 1, 1, 1],
... ],
... dtype="float32",
... )
>>> output_mask = paddle.nn.functional.sparse_attention(
... query,
... key,
... value,
... offset,
... columns,
... key_padding_mask=key_padding_mask,
... attn_mask=attention_mask,
... )
>>> print(output_mask)
Tensor(shape=[1, 1, 4, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[[[0. , 1. ],
[1.99830270, 2.99830270],
[0. , 1. ],
[0. , 1. ]]]])
>>> output = paddle.nn.functional.sparse_attention(query, key, value, offset, columns)
>>> print(output)
Tensor(shape=[1, 1, 4, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[[[1.60885942, 2.60885954],
[1.99830270, 2.99830270],
[1.60885942, 2.60885954],
[1.99830270, 2.99830270]]]])
"""
if paddle.framework.in_dynamic_or_pir_mode():
res = _C_ops.sparse_attention(
query,
key,
value,
sparse_csr_offset,
sparse_csr_columns,
key_padding_mask,
attn_mask,
)
return res
helper = LayerHelper('sparse_attention', **locals())
dtype = helper.input_dtype(input_param_name='Q')
out = helper.create_variable_for_type_inference(dtype)
result_sdd = helper.create_variable_for_type_inference(dtype)
result_softmax = helper.create_variable_for_type_inference(dtype)
inputs = {
'Q': query,
'K': key,
'V': value,
'Offset': sparse_csr_offset,
'Columns': sparse_csr_columns,
'KeyPaddingMask': key_padding_mask,
'AttnMask': attn_mask,
}
outputs = {
'Out': out,
'SparseDotSdd': result_sdd,
'Softmax': result_softmax,
}
helper.append_op(type='sparse_attention', inputs=inputs, outputs=outputs)
return out
python/paddle/nn/functional/transformer.py
+16
View File
@@ -0,0 +1,16 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO: define the classes of Transformer neural network
# __all__ = [ ]
python/paddle/nn/functional/vision.py
+300
View File
@@ -0,0 +1,300 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
import paddle
from paddle import _C_ops, in_dynamic_mode
from paddle._C_ops import (
grid_sample, # noqa: F401
pixel_shuffle, # noqa: F401
)
from paddle.base.framework import (
in_dynamic_or_pir_mode,
in_pir_mode,
)
from paddle.utils.decorator_utils import param_one_alias
from ...base.data_feeder import check_variable_and_dtype
from ...base.layer_helper import LayerHelper
from ...common_ops_import import Variable
from ...device import get_cudnn_version, is_compiled_with_rocm
if TYPE_CHECKING:
from paddle import Tensor
from paddle._typing import DataLayout2D, ShapeLike
__all__ = []
def affine_grid(
theta: Tensor,
out_shape: ShapeLike,
align_corners: bool = True,
name: str | None = None,
) -> Tensor:
"""
It generates a grid of (x,y) or (x,y,z) coordinates using the parameters of
the affine transformation that correspond to a set of points where
the input feature map should be sampled to produce the transformed
output feature map.
Args:
theta (Tensor): A tensor with shape [N, 2, 3] or [N, 3, 4]. It contains a batch of affine transform parameters.
The data type can be float32 or float64.
out_shape (Tensor | list | tuple): Type can be a 1-D Tensor, list, or tuple. It is used to represent the shape of the output in an affine transformation, in the format ``[N, C, H, W]`` or ``[N, C, D, H, W]``.
When the format is ``[N, C, H, W]``, it represents the batch size, number of channels, height and width. When the format is ``[N, C, D, H, W]``, it represents the batch size, number of channels, depth, height and width.
The data type must be int32.
align_corners(bool, optional): if True, aligns the centers of the 4 (4D) or 8 (5D) corner pixels of the input and output tensors, and preserves the value of the corner pixels. Default: True
name(str|None, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`.
Returns:
Tensor, A Tensor with shape [batch_size, H, W, 2] or [batch, D, H, W, 3] while ('D')'H', 'W' are the (depth)height, width of feature map in affine transformation. The data type is the same as `theta`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn.functional as F
>>> # theta.shape = [1, 2, 3]
>>> theta = paddle.to_tensor(
... [
... [
... [-0.7, -0.4, 0.3],
... [0.6, 0.5, 1.5],
... ]
... ],
... dtype="float32",
... )
>>> y_t = F.affine_grid(
... theta,
... [1, 2, 3, 3],
... align_corners=False,
... )
>>> print(y_t)
Tensor(shape=[1, 3, 3, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[ 1.03333330, 0.76666665],
[ 0.56666672, 1.16666663],
[ 0.10000002, 1.56666672]],
[[ 0.76666665, 1.09999990],
[ 0.30000001, 1.50000000],
[-0.16666666, 1.90000010]],
[[ 0.50000000, 1.43333328],
[ 0.03333333, 1.83333337],
[-0.43333334, 2.23333335]]]])
"""
if not isinstance(theta, (Variable, paddle.pir.Value)):
raise TypeError("The theta should be a Tensor.")
cudnn_version = get_cudnn_version()
if cudnn_version is not None and cudnn_version >= 6000 and align_corners:
use_cudnn = True
else:
use_cudnn = False
if theta.shape[1] == 3:
use_cudnn = False
if is_compiled_with_rocm():
use_cudnn = (
False # ROCM platform do not have MIOPEN kernel for affine_grid
)
if paddle.get_flags(["FLAGS_use_accuracy_compatible_kernel"]).get(
"FLAGS_use_accuracy_compatible_kernel", False
):
use_cudnn = False
if in_dynamic_mode():
_out_shape = (
out_shape.tolist() if isinstance(out_shape, Variable) else out_shape
)
if isinstance(_out_shape, paddle.Tensor) and _out_shape.size == 0:
raise ValueError("The out_shape cannot be empty.")
theta = theta._use_gpudnn(use_cudnn)
return _C_ops.affine_grid(theta, _out_shape, align_corners)
elif in_pir_mode():
return _C_ops.affine_grid(
theta,
out_shape,
align_corners,
)
else:
helper = LayerHelper('affine_grid', **locals())
check_variable_and_dtype(
theta, 'theta', ['float32', 'float64'], 'affine_grid'
)
out = helper.create_variable_for_type_inference(dtype=theta.dtype)
inputs = {'Theta': theta}
attrs = {"align_corners": align_corners, "use_cudnn": use_cudnn}
if isinstance(out_shape, Variable):
inputs['OutputShape'] = out_shape
check_variable_and_dtype(
out_shape, 'out_shape', ['int32'], 'affine_grid'
)
else:
attrs['output_shape'] = out_shape
helper.append_op(
type='affine_grid',
inputs=inputs,
outputs={'Output': out},
attrs=None if len(attrs) == 0 else attrs,
)
return out
@param_one_alias(["x", "input"])
def pixel_unshuffle(
x: Tensor,
downscale_factor: int,
data_format: DataLayout2D = 'NCHW',
name: str | None = None,
) -> Tensor:
"""
This API implements pixel unshuffle operation.
See more details in :ref:`PixelUnShuffle <api_paddle_nn_PixelUnshuffle>` .
Parameters:
x (Tensor): 4-D tensor, the data type should be float32 or float64.
Alias: ``input``.
downscale_factor (int): Factor to decrease spatial resolution.
data_format (str, optional): The data format of the input and output data. An optional string of ``'NCHW'`` or ``'NHWC'``. When it is ``'NCHW'``, the data is stored in the order of [batch_size, input_channels, input_height, input_width]. Default: ``'NCHW'``.
name (str|None, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`.
Returns:
Out (Tensor): Reshaped tensor according to the new dimension.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn.functional as F
>>> x = paddle.randn([2, 1, 12, 12])
>>> out = F.pixel_unshuffle(x, 3)
>>> print(out.shape)
paddle.Size([2, 9, 4, 4])
"""
if len(x.shape) != 4:
raise ValueError(
f"Input x should be 4D tensor, but received x with the shape of {x.shape}"
)
if not isinstance(downscale_factor, int):
raise TypeError("Downscale factor must be int type")
if downscale_factor <= 0:
raise ValueError("Downscale factor must be positive")
if data_format not in ["NCHW", "NHWC"]:
raise ValueError(
"Attr(data_format) should be 'NCHW' or 'NHWC'."
f"But receive Attr(data_format): {data_format} "
)
if in_dynamic_or_pir_mode():
return _C_ops.pixel_unshuffle(x, downscale_factor, data_format)
helper = LayerHelper("pixel_unshuffle", **locals())
check_variable_and_dtype(
x, 'x', ['float16', 'float32', 'float64', 'uint16'], 'pixel_unshuffle'
)
out = helper.create_variable_for_type_inference(dtype=x.dtype)
helper.append_op(
type="pixel_unshuffle",
inputs={"X": x},
outputs={"Out": out},
attrs={
"downscale_factor": downscale_factor,
"data_format": data_format,
},
)
return out
def channel_shuffle(
x: Tensor,
groups: int,
data_format: DataLayout2D = 'NCHW',
name: str | None = None,
) -> Tensor:
"""
This API implements channel shuffle operation.
See more details in :ref:`api_paddle_nn_ChannelShuffle`.
Parameters:
x (Tensor): 4-D tensor, the data type should be float32 or float64.
groups (int): Number of groups to divide channels in.
data_format (str, optional): The data format of the input and output data. An optional string of NCHW or NHWC. The default is NCHW. When it is NCHW, the data is stored in the order of [batch_size, input_channels, input_height, input_width].
name (str|None, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`.
Returns:
Out (Tensor): Rearranged tensor keeping the original tensor shape.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn.functional as F
>>> x = paddle.arange(0, 0.6, 0.1, 'float32')
>>> x = paddle.reshape(x, [1, 6, 1, 1])
>>> print(x)
Tensor(shape=[1, 6, 1, 1], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[0. ]],
[[0.10000000]],
[[0.20000000]],
[[0.30000001]],
[[0.40000001]],
[[0.50000000]]]])
>>> y = F.channel_shuffle(x, 3)
>>> print(y)
Tensor(shape=[1, 6, 1, 1], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[0. ]],
[[0.20000000]],
[[0.40000001]],
[[0.10000000]],
[[0.30000001]],
[[0.50000000]]]])
"""
if len(x.shape) != 4:
raise ValueError(
f"Input x should be 4D tensor, but received x with the shape of {x.shape}"
)
if not isinstance(groups, int):
raise TypeError("groups must be int type")
if groups <= 0:
raise ValueError("groups must be positive")
if data_format not in ["NCHW", "NHWC"]:
raise ValueError(
"Attr(data_format) should be 'NCHW' or 'NHWC'."
f"But receive Attr(data_format): {data_format} "
)
if in_dynamic_or_pir_mode():
return _C_ops.channel_shuffle(x, groups, data_format)
helper = LayerHelper("channel_shuffle", **locals())
check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'channel_shuffle')
out = helper.create_variable_for_type_inference(dtype=x.dtype)
helper.append_op(
type="channel_shuffle",
inputs={"X": x},
outputs={"Out": out},
attrs={"groups": groups, "data_format": data_format},
)
return out
python/paddle/nn/init.py
+472
View File
@@ -0,0 +1,472 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Callable
from typing import TypeVar
from typing_extensions import ParamSpec
_InputT = ParamSpec("_InputT")
_RetT = TypeVar("_RetT")
import math
import warnings
import numpy as np
import paddle
from ..base.framework import in_dygraph_mode, in_pir_mode
from .initializer.constant import Constant
from .initializer.dirac import Dirac
from .initializer.initializer import calculate_gain # noqa: F401
from .initializer.kaiming import KaimingNormal, KaimingUniform
from .initializer.normal import Normal, TruncatedNormal
from .initializer.orthogonal import Orthogonal
from .initializer.uniform import Uniform
from .initializer.xavier import XavierNormal, XavierUniform
def _calculate_fan_in_and_fan_out(var: paddle.Tensor) -> tuple[int, int]:
"""Compute the fan_in and the fan_out for layers
This method computes the fan_in and the fan_out
for neural network layers, if not specified. It is
not possible to perfectly estimate fan_in and fan_out.
This method will estimate it correctly for matrix multiply and
convolutions.
Args:
var: variable for which fan_in and fan_out have to be computed.
Returns:
tuple of two integers (fan_in, fan_out).
"""
shape = var.shape
if not shape or len(shape) == 0:
fan_in = fan_out = 1
elif len(shape) == 1:
fan_in = fan_out = shape[0]
elif len(shape) == 2:
# This is the case for simple matrix multiply
fan_in = shape[0]
fan_out = shape[1]
else:
# Assume this to be a convolutional kernel
# In PaddlePaddle, the shape of the kernel is like:
# [num_filters, num_filter_channels, ...] where the remaining
# dimensions are the filter_size
receptive_field_size = np.prod(shape[2:])
fan_in = int(shape[1] * receptive_field_size)
fan_out = int(shape[0] * receptive_field_size)
return (fan_in, fan_out)
def kaiming_uniform_(
tensor: paddle.Tensor,
a: float = 0,
mode: str = "fan_in",
nonlinearity: str = "leaky_relu",
) -> paddle.Tensor:
"""Modify tensor inplace using Kaiming uniform method.
Args:
tensor (Tensor): Paddle Tensor.
a (float, optional): The negative slope of the rectifier used after this layer.
Defaults to 0.
mode (str, optional): Mode to compute the fan. Choose from ["fan_in", "fan_out"].
When set to 'fan_in', the fan_in parameter is used for initialization.
When set to 'fan_out', the out_features of trainable Tensor will be used.
Default is 'fan_in'.
nonlinearity (str, optional): Nonlinearity method name. Defaults to "leaky_relu".
Returns:
Tensor: Initialized tensor.
"""
init = KaimingUniform(
negative_slope=a, nonlinearity=nonlinearity, mode=mode
)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def kaiming_normal_(
tensor: paddle.Tensor,
a: float = 0,
mode: str = "fan_in",
nonlinearity: str = "leaky_relu",
) -> paddle.Tensor:
"""Modify tensor inplace using Kaiming normal method.
Args:
tensor (Tensor): Paddle Tensor.
a (float, optional): The negative slope of the rectifier used after this layer.
Defaults to 0.
mode (str, optional): Mode to compute the fan. Choose from ["fan_in", "fan_out"].
When set to 'fan_in', the fan_in parameter is used for initialization.
When set to 'fan_out', the out_features of trainable Tensor will be used.
Default is 'fan_in'.
nonlinearity (str, optional): Nonlinearity method name. Defaults to "leaky_relu".
Returns:
Tensor: Initialized tensor.
"""
init = KaimingNormal(negative_slope=a, nonlinearity=nonlinearity, mode=mode)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def xavier_uniform_(
tensor: paddle.Tensor,
gain: float = 1.0,
fan_in: float | None = None,
fan_out: float | None = None,
) -> paddle.Tensor:
"""Modify tensor inplace using Xavier uniform method.
Args:
tensor (Tensor): Paddle Tensor.
gain (float, optional): Scaling Tensor. Default is 1.0.
fan_in (float|None, optional): fan_in for Xavier initialization, which is
inferred from the Tensor. Default is None.
fan_out (float|None, optional): fan_out for Xavier initialization, which is
inferred from the Tensor. Default is None.
Returns:
Tensor: Initialized tensor.
"""
init = XavierUniform(
gain=gain,
fan_in=fan_in,
fan_out=fan_out,
)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def xavier_normal_(
tensor: paddle.Tensor,
gain: float = 1.0,
fan_in: float | None = None,
fan_out: float | None = None,
) -> paddle.Tensor:
"""Modify tensor inplace using Xavier normal method.
Args:
tensor (Tensor): Paddle Tensor.
gain (float, optional): Scaling Tensor. Default is 1.0.
fan_in (float|None, optional): fan_in for Xavier initialization, which is
inferred from the Tensor. Default is None.
fan_out (float|None, optional): fan_out for Xavier initialization, which is
inferred from the Tensor. Default is None.
Returns:
Tensor: Initialized tensor.
"""
init = XavierNormal(
gain=gain,
fan_in=fan_in,
fan_out=fan_out,
)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def uniform_(
tensor: paddle.Tensor,
a: float = 0.0,
b: float = 1.0,
) -> paddle.Tensor:
"""Modify tensor inplace using uniform method.
Args:
tensor (Tensor): Paddle Tensor.
low (float, optional): Lower boundary of the uniform distribution. Default is :math:`-1.0`.
high (float, optional): Upper boundary of the uniform distribution. Default is :math:`1.0`.
Returns:
Tensor: Initialized tensor.
"""
init = Uniform(low=a, high=b)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def normal_(
tensor: paddle.Tensor,
mean: float = 0.0,
std: float = 1.0,
) -> paddle.Tensor:
"""Modify tensor inplace using normal method.
Args:
tensor (Tensor): Paddle Tensor.
mean (float|complex, optional): mean of the normal distribution. Default is 0.0.
std (float, optional): standard deviation of the normal distribution. Default is 1.0.
Returns:
Tensor: Initialized tensor.
"""
init = Normal(mean=mean, std=std)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def trunc_normal_(
tensor: paddle.Tensor,
mean: float = 0.0,
std: float = 1.0,
a: float = -2.0,
b: float = 2.0,
) -> paddle.Tensor:
"""Modify tensor inplace using truncated normal method.
Args:
tensor (Tensor): Paddle Tensor.
mean (float|complex, optional): mean of the normal distribution. Default is 0.0.
std (float, optional): standard deviation of the normal distribution. Default is 1.0.
a (float, optional): The minimum cutoff value. Default is -2.0.
b (float, optional): The maximum cutoff value. Default is 2.0.
Returns:
Tensor: Initialized tensor.
"""
init = TruncatedNormal(mean=mean, std=std, a=a, b=b)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def constant_(
tensor: paddle.Tensor,
val: float,
) -> paddle.Tensor:
"""Modify tensor inplace using constant method.
Args:
tensor (Tensor): Paddle Tensor.
value (float32|float64, optional): constant value to initialize the parameter.
Returns:
Tensor: Initialized tensor.
"""
init = Constant(value=val)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def ones_(
tensor: paddle.Tensor,
) -> paddle.Tensor:
"""Fill the input Tensor with the scalar value 1.
Args:
tensor (Tensor): Paddle Tensor.
Returns:
Tensor: Initialized tensor.
"""
init = Constant(value=1.0)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def zeros_(
tensor: paddle.Tensor,
) -> paddle.Tensor:
"""Fill the input Tensor with the scalar value 0.
Args:
tensor (Tensor): Paddle Tensor.
Returns:
Tensor: Initialized tensor.
"""
init = Constant(value=0.0)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def dirac_(
tensor: paddle.Tensor,
groups: int = 1,
) -> paddle.Tensor:
"""Initialize the 3D/4D/5D Tensor with Dirac delta function.
Args:
tensor (Tensor): Paddle Tensor.
groups (int|None, optional): 0-dimension of the Tensor will be divided by groups,
each group has the same value. Default: 1.
Returns:
Tensor: Initialized tensor.
"""
init = Dirac(groups=groups)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def eye_(
tensor: paddle.Tensor,
) -> paddle.Tensor:
"""Fill the 2-dimensional input Tensor with the identity matrix.
Args:
tensor (Tensor): Paddle Tensor.
Returns:
Tensor: Initialized tensor.
"""
if len(tensor.shape) != 2:
raise AssertionError(
f"Only support 2 dimensional tensor, but got {len(tensor.shape)}."
)
if in_dygraph_mode():
new_tensor = paddle.eye(
tensor.shape[0], tensor.shape[1], dtype=tensor.dtype
)
new_tensor._share_underline_tensor_to(tensor)
return tensor
elif in_pir_mode():
new_tensor = paddle.eye(
tensor.shape[0], tensor.shape[1], dtype=tensor.dtype
)
return new_tensor
else:
raise NotImplementedError(
'Only support run in dygraph mode or PIR mode.'
)
def orthogonal_(
tensor: paddle.Tensor,
gain: float = 1,
) -> paddle.Tensor:
"""Fill the input Tensor with a (semi) orthogonal matrix.
Args:
tensor (Tensor): Paddle Tensor.
gain(float, optional): The multiplication coefficient for initialized tensor. Default: 1.0.
Returns:
Tensor: Initialized tensor.
"""
init = Orthogonal(gain=gain)
if in_dygraph_mode():
init(tensor)
return tensor
return init(tensor)
def sparse_(
tensor: paddle.Tensor, sparsity: float, std: float = 0.01
) -> paddle.Tensor:
"""Fill the 2D input Tensor as a sparse matrix.
The non-zero elements will be drawn from the normal distribution.
Args:
tensor (Tensor): Paddle Tensor with 2 dimensions.
sparsity (float): The fraction of elements in each column to be set to zero.
std (float): the standard deviation of the normal distribution used to generate
the non-zero values. Default is 0.01.
Examples:
>>> tensor = paddle.empty(3, 5)
>>> result = paddle.nn.init.sparse_(tensor, sparsity=0.1)
"""
if tensor.ndimension() != 2:
raise ValueError("Only tensors with 2 dimensions are supported")
rows, cols = tensor.shape
num_zeros = math.ceil(sparsity * rows)
with paddle.no_grad():
tensor = normal_(tensor, mean=0, std=std)
for col_idx in range(cols):
row_indices = paddle.randperm(rows)
zero_indices = row_indices[:num_zeros]
tensor[zero_indices, col_idx] = 0
return tensor
def _make_deprecate(func: Callable[_InputT, _RetT]) -> Callable[_InputT, _RetT]:
new_name = func.__name__
old_name = new_name[:-1]
def deprecated_init(*args: _InputT.args, **kwargs: _InputT.kwargs) -> _RetT:
warnings.warn(
f"`nn.init.{old_name}` is now deprecated in favor of `nn.init.{new_name}`.",
FutureWarning,
stacklevel=2,
)
return func(*args, **kwargs)
deprecated_init.__doc__ = rf"""
{old_name}(...)
.. warning::
This method is now deprecated in favor of :func:`paddle.nn.init.{new_name}`.
See :func:`~paddle.nn.init.{new_name}` for details."""
deprecated_init.__name__ = old_name
return deprecated_init
uniform = _make_deprecate(uniform_)
normal = _make_deprecate(normal_)
constant = _make_deprecate(constant_)
eye = _make_deprecate(eye_)
dirac = _make_deprecate(dirac_)
xavier_uniform = _make_deprecate(xavier_uniform_)
xavier_normal = _make_deprecate(xavier_normal_)
kaiming_uniform = _make_deprecate(kaiming_uniform_)
kaiming_normal = _make_deprecate(kaiming_normal_)
orthogonal = _make_deprecate(orthogonal_)
sparse = _make_deprecate(sparse_)
python/paddle/nn/initializer/__init__.py
+68
View File
@@ -0,0 +1,68 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ...base.initializer import set_global_initializer
from .assign import (
Assign,
NumpyArrayInitializer, # noqa: F401
)
from .bilinear import Bilinear
from .constant import (
Constant,
ConstantInitializer, # noqa: F401
)
from .dirac import Dirac
from .initializer import (
Initializer, # noqa: F401
calculate_gain,
)
from .kaiming import (
KaimingNormal,
KaimingUniform,
MSRAInitializer, # noqa: F401
)
from .normal import (
Normal,
NormalInitializer, # noqa: F401
TruncatedNormal,
TruncatedNormalInitializer, # noqa: F401
)
from .orthogonal import Orthogonal
from .uniform import (
Uniform,
UniformInitializer, # noqa: F401
)
from .xavier import (
XavierInitializer, # noqa: F401
XavierNormal,
XavierUniform,
)
__all__ = [
'Bilinear',
'Constant',
'KaimingUniform',
'KaimingNormal',
'XavierNormal',
'XavierUniform',
'Assign',
'Normal',
'TruncatedNormal',
'Uniform',
'Orthogonal',
'Dirac',
'set_global_initializer',
'calculate_gain',
]
python/paddle/nn/initializer/assign.py
+297
View File
@@ -0,0 +1,297 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import paddle
from paddle import _C_ops
from ...base import core, framework, unique_name
from ...base.data_feeder import check_type
from ...base.framework import (
_current_expected_place,
in_dygraph_mode,
in_pir_mode,
)
from .initializer import Initializer
if TYPE_CHECKING:
from collections.abc import Sequence
import numpy.typing as npt
from paddle._typing import NestedSequence
__all__ = []
class NumpyArrayInitializer(Initializer):
"""Init an parameter with an numpy array
This api initialize the tensor by numpy array.
Args:
value (numpy): numpy array to initialize the tensor
Returns:
A Tensor initialized by numpy.
"""
def __init__(self, value: npt.NDArray[Any]) -> None:
import numpy
assert isinstance(value, numpy.ndarray)
super().__init__()
self._value = value
def forward(
self, var: paddle.Tensor, block: paddle.pir.Block | None = None
) -> paddle.Tensor | None:
"""Initialize the input tensor with Numpy array.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The initialization op
"""
assert not (
isinstance(var, framework.EagerParamBase) and var.is_dist()
), "Currently, assign initializer not support lazy init for dist param."
block = self._check_block(block)
assert isinstance(
var, (framework.Variable, paddle.pir.core.ParameterMeta)
)
assert isinstance(block, (framework.Block, paddle.pir.Block))
# to be compatible of fp16 initializers
origin_dtype = var.dtype
if origin_dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
]:
out_dtype = core.VarDesc.VarType.FP32
np_value = self._value.astype("float32")
out_var = block.create_var(
name=unique_name.generate(
".".join(['numpy_array_init', var.name, 'tmp'])
),
shape=var.shape,
dtype=out_dtype,
type=core.VarDesc.VarType.DENSE_TENSOR,
persistable=False,
)
elif origin_dtype in [core.DataType.FLOAT16, core.DataType.BFLOAT16]:
out_var = var
out_dtype = core.DataType.FLOAT32
np_value = self._value.astype("float32")
else:
out_var = var
out_dtype = origin_dtype
np_value = self._value
if out_dtype in (core.VarDesc.VarType.FP32, core.DataType.FLOAT32):
value_name = "values"
values = [float(v) for v in np_value.flat]
elif out_dtype in (core.VarDesc.VarType.FP64, core.DataType.FLOAT64):
value_name = "values"
values = [float(v) for v in np_value.flat]
elif out_dtype in (core.VarDesc.VarType.INT32, core.DataType.INT32):
value_name = "values"
values = [int(v) for v in np_value.flat]
elif out_dtype in (
core.VarDesc.VarType.INT8,
core.VarDesc.VarType.UINT8,
core.DataType.INT8,
core.DataType.UINT8,
):
value_name = "int8_values"
values = [int(v) for v in np_value.flat]
else:
raise ValueError(f"Unsupported dtype {self._value.dtype}")
if self._value.size > 1024 * 1024 * 1024:
raise ValueError(
"The size of input is too big. Please consider "
"saving it to file and 'load_op' to load it"
)
if in_dygraph_mode():
_C_ops.assign_value_(
out_var,
list(self._value.shape),
out_dtype,
values,
_current_expected_place(),
)
if origin_dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
core.DataType.FLOAT16,
core.DataType.BFLOAT16,
]:
var_tmp = _C_ops.cast(out_var, origin_dtype)
var_tmp._share_underline_tensor_to(var)
else:
out_var._share_underline_tensor_to(var)
return None
elif in_pir_mode():
out_var = _C_ops.assign_value(
list(self._value.shape),
out_dtype,
values,
_current_expected_place(),
)
if origin_dtype in [core.DataType.FLOAT16, core.DataType.BFLOAT16]:
out_var = _C_ops.cast(out_var, origin_dtype)
return out_var
else:
op = block.append_op(
type='assign_value',
outputs={'Out': out_var},
attrs={
'dtype': out_dtype,
'shape': list(self._value.shape),
value_name: values,
},
stop_gradient=True,
)
if origin_dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
]:
block.append_op(
type="cast",
inputs={"X": out_var},
outputs={"Out": var},
attrs={
"in_dtype": out_var.dtype,
"out_dtype": origin_dtype,
},
)
var.op = op
return op
class Assign(NumpyArrayInitializer):
"""Init an parameter with a numpy array, list, or tensor.
Args:
value (Tensor|numpy.ndarray|list|tuple): numpy array, list, tuple, or tensor to initialize the parameter.
name(str|None, optional): Normally there is no need for user to set this
property. For more information, please refer to :ref:`api_guide_Name`. Default is None.
Returns:
A parameter initialized by the input numpy array, list, or tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import numpy as np
>>> # numpy array
>>> data_1 = paddle.ones(shape=[1, 2], dtype='float32')
>>> weight_attr_1 = paddle.ParamAttr(
... name="linear_weight_1",
... initializer=paddle.nn.initializer.Assign(np.array([[2, 2], [2, 2]])),
... )
>>> bias_attr_1 = paddle.ParamAttr(
... name="linear_bias_1",
... initializer=paddle.nn.initializer.Assign(np.array([2, 2])),
... )
>>> linear_1 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_1, bias_attr=bias_attr_1)
>>> print(linear_1.weight.numpy())
[[2. 2.]
[2. 2.]]
>>> print(linear_1.bias.numpy())
[2. 2.]
>>> res_1 = linear_1(data_1)
>>> print(res_1.numpy())
[[6. 6.]]
>>> # python list
>>> data_2 = paddle.ones(shape=[1, 2], dtype='float32')
>>> weight_attr_2 = paddle.ParamAttr(
... name="linear_weight_2",
... initializer=paddle.nn.initializer.Assign([[2, 2], [2, 2]]),
... )
>>> bias_attr_2 = paddle.ParamAttr(
... name="linear_bias_2",
... initializer=paddle.nn.initializer.Assign([2, 2]),
... )
>>> linear_2 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_2, bias_attr=bias_attr_2)
>>> print(linear_2.weight.numpy())
[[2. 2.]
[2. 2.]]
>>> print(linear_2.bias.numpy())
[2. 2.]
>>> res_2 = linear_2(data_2)
>>> print(res_2.numpy())
[[6. 6.]]
>>> # tensor
>>> data_3 = paddle.ones(shape=[1, 2], dtype='float32')
>>> weight_attr_3 = paddle.ParamAttr(
... name="linear_weight_3",
... initializer=paddle.nn.initializer.Assign(paddle.full([2, 2], 2)),
... )
>>> bias_attr_3 = paddle.ParamAttr(
... name="linear_bias_3",
... initializer=paddle.nn.initializer.Assign(paddle.full([2], 2)),
... )
>>> linear_3 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_3, bias_attr=bias_attr_3)
>>> print(linear_3.weight.numpy())
[[2. 2.]
[2. 2.]]
>>> print(linear_3.bias.numpy())
[2. 2.]
>>> res_3 = linear_3(data_3)
>>> print(res_3.numpy())
[[6. 6.]]
"""
def __init__(
self,
value: npt.NDArray[Any]
| Sequence[NestedSequence[int | float | bool | complex]]
| paddle.Tensor,
name: str | None = None,
) -> None:
import numpy
check_type(
value,
'value',
(numpy.ndarray, list, tuple, paddle.static.Variable),
'Assign',
)
if isinstance(value, (list, tuple)):
value = numpy.array(value)
# TODO: value is already is a tensor, accounting efficiency maybe it does not need to convert tensor to numpy data and then initialized.
if isinstance(value, paddle.static.Variable):
value = value.numpy(False)
super().__init__(value)
python/paddle/nn/initializer/bilinear.py
+225
View File
@@ -0,0 +1,225 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import numpy as np
import paddle
from paddle import _C_ops, pir
from ...base import core, framework, unique_name
from ...base.framework import (
_current_expected_place,
in_dygraph_mode,
in_pir_mode,
)
from .initializer import Initializer
__all__ = []
class Bilinear(Initializer):
"""
This initializer can be used in transposed convolution operator to
act as upsampling. Users can upsample a feature map with shape of
(B, C, H, W) by any integer factor.
Returns:
Bilinear initializer instance objects.
Examples:
.. code-block:: pycon
>>> import math
>>> import paddle
>>> import paddle.nn as nn
>>> from paddle.regularizer import L2Decay
>>> factor = 2
>>> C = 2
>>> B = 8
>>> H = W = 32
>>> w_attr = paddle.ParamAttr(
... learning_rate=0.0,
... regularizer=L2Decay(0.0),
... initializer=nn.initializer.Bilinear(),
... )
>>> data = paddle.rand([B, 3, H, W], dtype='float32')
>>> conv_up = nn.Conv2DTranspose(
... 3,
... out_channels=C,
... kernel_size=2 * factor - factor % 2,
... padding=int(math.ceil((factor - 1) / 2.0)),
... stride=factor,
... weight_attr=w_attr,
... bias_attr=False,
... )
>>> x = conv_up(data)
Where, `out_channels=C` and `groups=C` means this is channel-wise transposed
convolution. The filter shape will be (C, 1, K, K) where K is `kernel_size`,
This initializer will set a (K, K) interpolation kernel for every channel
of the filter identically. The resulting shape of the output feature map
will be (B, C, factor * H, factor * W). Note that the learning rate and the
weight decay are set to 0 in order to keep coefficient values of bilinear
interpolation unchanged during training.
"""
def __init__(self) -> None:
"""Constructor for BilinearInitializer."""
super().__init__()
def forward(
self, var: paddle.Tensor, block: pir.Block | None = None
) -> paddle.Tensor | None:
"""Initialize the input tensor with Bilinear initialization.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The initialization op
"""
assert not (
isinstance(var, framework.EagerParamBase) and var.is_dist()
), (
"Currently, Bilinear initializer not support lazy init for dist param."
)
block = self._check_block(block)
if not isinstance(var, (framework.Variable, pir.core.ParameterMeta)):
raise ValueError(
"var must be framework.Variable or pir.core.ParameterMeta."
)
if not isinstance(block, (framework.Block, pir.Block)):
raise ValueError("block must be framework.Block or pir.Block.")
shape = var.shape
if len(shape) != 4:
raise ValueError("the length of shape must be 4.")
if shape[2] != shape[3]:
raise ValueError("shape[2] must be equal to shape[3].")
weight = np.zeros(np.prod(var.shape), dtype='float32')
size = shape[3]
# factor
f = np.ceil(size / 2.0)
# center
c = (2 * f - 1 - f % 2) / (2.0 * f)
for i in range(np.prod(shape)):
x = i % size
y = (i / size) % size
weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))
weight = np.reshape(weight, shape)
# to be compatible of fp16 initializers
if var.dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
core.VarDesc.VarType.FP64,
]:
out_dtype = core.VarDesc.VarType.FP32
out_var = block.create_var(
name=unique_name.generate(
".".join(['bilinear_init', var.name, 'tmp'])
),
shape=var.shape,
dtype=out_dtype,
type=core.VarDesc.VarType.DENSE_TENSOR,
persistable=False,
)
elif var.dtype in [
core.DataType.FLOAT16,
core.DataType.BFLOAT16,
core.DataType.FLOAT64,
]:
out_dtype = core.DataType.FLOAT32
out_var = var
else:
out_dtype = var.dtype
out_var = var
if out_dtype in (core.VarDesc.VarType.FP32, core.DataType.FLOAT32):
value_name = "values"
values = [float(v) for v in weight.flat]
else:
raise TypeError(f"Unsupported dtype {var.dtype}")
if np.prod(shape) > 1024 * 1024:
raise ValueError("The size of input is too big. ")
if in_dygraph_mode():
_C_ops.assign_value_(
out_var,
list(shape),
out_dtype,
values,
_current_expected_place(),
)
if var.dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
core.VarDesc.VarType.FP64,
]:
var_tmp = _C_ops.cast(out_var, var.dtype)
var_tmp._share_underline_tensor_to(var)
else:
out_var._share_underline_tensor_to(var)
return None
elif in_pir_mode():
out_var = _C_ops.assign_value(
list(shape),
out_dtype,
values,
_current_expected_place(),
)
if var.dtype in [
core.DataType.FLOAT16,
core.DataType.BFLOAT16,
core.DataType.FLOAT64,
]:
out_var = _C_ops.cast(out_var, var.dtype)
return out_var
else:
op = block.append_op(
type='assign_value',
outputs={'Out': [out_var]},
attrs={
'dtype': out_dtype,
'shape': list(shape),
value_name: values,
},
)
if var.dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
core.VarDesc.VarType.FP64,
]:
block.append_op(
type="cast",
inputs={"X": out_var},
outputs={"Out": var},
attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
)
var.op = op
return op
python/paddle/nn/initializer/constant.py
+144
View File
@@ -0,0 +1,144 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import paddle
from paddle import _C_ops
from ...base import core, framework
from ...base.framework import (
_current_expected_place,
in_dygraph_mode,
in_dynamic_or_pir_mode,
)
# TODO: define the initializers of Constant in neural network
from .initializer import Initializer
__all__ = []
class ConstantInitializer(Initializer):
"""Implements the constant initializer
Args:
value (float32, optional): constant value to initialize the variable. Default: 0.0.
"""
def __init__(self, value: float = 0.0, force_cpu: bool = False) -> None:
assert value is not None
super().__init__()
self._value = value
self._force_cpu = force_cpu
def forward(
self,
var: paddle.Tensor,
block: paddle.pir.Block | None = None,
) -> paddle.Tensor | None:
"""Initialize the input tensor with constant.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The initialization op
"""
block = self._check_block(block)
assert isinstance(
var,
(
framework.Variable,
framework.EagerParamBase,
paddle.pir.Value,
paddle.pir.core.ParameterMeta,
),
)
assert isinstance(block, (framework.Block, paddle.pir.Block))
if in_dynamic_or_pir_mode():
place = _current_expected_place()
if self._force_cpu:
place = core.CPUPlace()
if in_dygraph_mode():
if isinstance(var, framework.EagerParamBase) and var.is_dist():
out_var = _C_ops.full(
var._local_shape, float(self._value), var.dtype, place
)
out_var = (
paddle.distributed.auto_parallel.api.dtensor_from_local(
out_var, var.process_mesh, var.placements
)
)
out_var._share_underline_tensor_to(var)
else:
_C_ops.full_(
var, var.shape, float(self._value), var.dtype, place
)
return None
else:
return _C_ops.full(
var.shape, float(self._value), var.dtype, place
)
else:
op = block.append_op(
type="fill_constant",
outputs={"Out": var},
attrs={
"shape": var.shape,
"dtype": int(var.dtype),
"value": float(self._value),
'str_value': str(float(self._value)),
'force_cpu': self._force_cpu,
},
stop_gradient=True,
)
var.op = op
return op
class Constant(ConstantInitializer):
"""Implement the constant initializer.
Args:
value (float32|float64, optional): constant value to initialize the parameter. Default: 0.0.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn as nn
>>> paddle.seed(2023)
>>> data = paddle.rand([30, 10, 2], dtype='float32')
>>> linear = nn.Linear(2, 4, weight_attr=nn.initializer.Constant(value=2.0))
>>> res = linear(data)
>>> print(linear.weight)
Parameter containing:
Tensor(shape=[2, 4], dtype=float32, place=Place(cpu), stop_gradient=False,
[[2., 2., 2., 2.],
[2., 2., 2., 2.]])
"""
def __init__(self, value: float = 0.0) -> None:
if value is None:
raise ValueError("value must not be none.")
super().__init__(value=value, force_cpu=False)
python/paddle/nn/initializer/dirac.py
+366
View File
@@ -0,0 +1,366 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import paddle
from paddle import _C_ops, in_dynamic_mode, pir
from paddle.utils import unique_name
from ... import base
from ...base import core, framework
from ...base.core import VarDesc
from ...base.data_feeder import check_variable_and_dtype
from ...base.framework import _current_expected_place
from .initializer import Initializer
__all__ = []
class Dirac(Initializer):
r"""Initialize the 3D/4D/5D Tensor with Dirac delta function.
It can reserve the feature of convolution layer input, which means that
as many channels are reserved as possible.
In this initialize method, elements in the middle of convolution kernels will
be set to 1 . The formula can be described as follow.
.. math::
X[d, d, shape[2]//2, shape[3]//2, ...]=1, \ d=0,1...N
where, ``N`` is the minimum value of ``in_channels`` and ``out_channels``
Args:
groups(int|None, optional): 0-dimension of the Tensor will be divided by groups,
each group has the same value. Default: 1.
name(str|None, optional): The default value is None. Normally there is no need for user to set this
property. For more information, please refer to :ref:`api_guide_Name`.
Returns:
Dirac initializer instance objects.
Examples:
.. code-block:: pycon
>>> import paddle
>>> # 1. For kernel_size is uneven number:
>>> attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Dirac())
>>> conv = paddle.nn.Conv1D(3, 2, 3, weight_attr=attr)
>>> print(conv.weight)
Parameter containing:
Tensor(shape=[2, 3, 3], dtype=float32, place=CPUPlace, stop_gradient=False,
[[[0., 1., 0.],
[0., 0., 0.],
[0., 0., 0.]],
[[0., 0., 0.],
[0., 1., 0.],
[0., 0., 0.]]])
>>> input = paddle.rand([8, 3, 10])
>>> output = conv(input)
>>> output == input[:, 0:2, 1:9]
>>> print(output.shape)
paddle.Size([8, 2, 8])
>>> # It means output is almost the same with input, 2 channels are reserved
>>> # 2. For kernel_size is even number:
>>> attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Dirac())
>>> conv = paddle.nn.Conv1D(3, 2, 4, weight_attr=attr)
>>> print(conv.weight)
Parameter containing:
Tensor(shape=[2, 3, 4], dtype=float32, place=CPUPlace, stop_gradient=False,
[[[0., 0., 1., 0.],
[0., 0., 0., 0.],
[0., 0., 0., 0.]],
[[0., 0., 0., 0.],
[0., 0., 1., 0.],
[0., 0., 0., 0.]]])
"""
def __init__(self, groups: int = 1, name: str | None = None) -> None:
assert groups > 0 and isinstance(groups, int), (
" 'groups' must be a positive integer. "
)
super().__init__()
self._groups = groups
def __call__(
self, var: paddle.Tensor, block: pir.Block | None = None
) -> paddle.Tensor:
"""Initialize the input tensor with dirac initializer.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The most critical OP(scatter) in this initializer, which contains 7~8 ops in total.
"""
assert not (
isinstance(var, framework.EagerParamBase) and var.is_dist()
), "Currently, dirac initializer not support lazy init for dist param."
block = self._check_block(block)
assert isinstance(
var, (framework.Variable, paddle.pir.Value, pir.core.ParameterMeta)
)
assert isinstance(block, (framework.Block, pir.Block))
check_variable_and_dtype(
var, "Out", ['float16', 'bfloat16', 'float32', 'float64'], 'Dirac'
)
assert len(var.shape) in [
3,
4,
5,
], "Only Tensor with 3/4/5 dimensions can be initialized by Dirac"
assert (var.shape[0] % self._groups) == 0, (
"Tensor 0-dimension must be divisible by groups"
)
if framework.in_pir_mode():
if var.dtype != core.DataType.FLOAT32:
out_dtype = core.DataType.FLOAT32
out_var = var
else:
out_dtype = var.dtype
out_var = var
else:
if var.dtype != VarDesc.VarType.FP32:
out_dtype = VarDesc.VarType.FP32
out_var = block.create_var(
name=unique_name.generate(
".".join(['dirac', var.name, 'tmp'])
),
shape=var.shape,
dtype=out_dtype,
type=VarDesc.VarType.DENSE_TENSOR,
persistable=False,
)
else:
out_dtype = var.dtype
out_var = var
op = None
if framework.in_dygraph_mode():
with base.dygraph.no_grad():
place = _current_expected_place()
_C_ops.full_(
out_var, out_var.shape, str(float(0)), out_dtype, place
)
elif framework.in_pir_mode():
place = _current_expected_place()
out_var = _C_ops.full(out_var.shape, float(0), out_dtype, place)
else:
block.append_op(
type='fill_constant',
inputs={},
outputs={'Out': out_var},
attrs={
'value': float(0),
'dtype': out_var.dtype,
'shape': out_var.shape,
},
stop_gradient=True,
)
origin_shape = var.shape
num_per_group = origin_shape[0] // self._groups
min_shape = min(num_per_group, origin_shape[1])
idx_list = []
value_list = []
strides = []
prod = 1
for dim in reversed(origin_shape):
strides.insert(0, prod)
prod *= dim
for i in range(self._groups):
for j in range(min_shape):
value_list.append(1.0)
offset = 0
for k, stride in enumerate(strides):
if k == 0:
offset += (j + i * num_per_group) * stride
elif k == 1:
offset += j * stride
else:
offset += origin_shape[k] // 2 * stride
idx_list.append(offset)
if framework.in_dygraph_mode():
with base.dygraph.no_grad():
tmp_out = _C_ops.reshape(out_var, [-1])
tmp_out._share_underline_tensor_to(out_var)
elif framework.in_pir_mode():
out_var = _C_ops.reshape(out_var, [-1])
else:
x_shape = block.create_var(
name=unique_name.generate(".".join([out_var.name, "XShape"])),
dtype=out_dtype,
shape=out_var.shape,
type=VarDesc.VarType.DENSE_TENSOR,
persistable=False,
stop_gradient=True,
)
block.append_op(
type="reshape2",
inputs={"X": out_var},
attrs={'shape': [-1]},
outputs={"Out": out_var, "XShape": x_shape},
stop_gradient=True,
)
if framework.in_pir_mode():
index_tensor = paddle.zeros(
[len(idx_list)], dtype=core.DataType.INT64
)
index_tensor.stop_gradient = True
else:
index_tensor = block.create_var(
name=unique_name.generate('scatter_index'),
persistable=False,
stop_gradient=True,
)
if framework.in_dygraph_mode():
with base.dygraph.no_grad():
tmp_tensor = framework._create_tensor()
_C_ops.assign_value_(
tmp_tensor,
[len(idx_list)],
VarDesc.VarType.INT64,
idx_list,
_current_expected_place(),
)
tmp_tensor._share_underline_tensor_to(index_tensor)
elif framework.in_pir_mode():
_C_ops.assign_value_(
index_tensor,
[len(idx_list)],
core.DataType.INT64,
idx_list,
_current_expected_place(),
)
else:
block.append_op(
type='assign_value',
outputs={'Out': index_tensor},
attrs={
'dtype': VarDesc.VarType.INT64,
'shape': [len(idx_list)],
'values': idx_list,
},
stop_gradient=True,
)
if framework.in_pir_mode():
value_tensor = paddle.zeros(
[len(value_list)], dtype=core.DataType.FLOAT32
)
value_tensor.stop_gradient = True
else:
value_tensor = block.create_var(
name=unique_name.generate('scatter_value'),
persistable=False,
stop_gradient=True,
)
if framework.in_dygraph_mode():
with base.dygraph.no_grad():
tmp_tensor = framework._create_tensor()
_C_ops.assign_value_(
tmp_tensor,
[len(value_list)],
VarDesc.VarType.FP32,
value_list,
_current_expected_place(),
)
tmp_tensor._share_underline_tensor_to(value_tensor)
elif framework.in_pir_mode():
_C_ops.assign_value_(
value_tensor,
[len(value_list)],
core.DataType.FLOAT32,
value_list,
_current_expected_place(),
)
else:
block.append_op(
type='assign_value',
outputs={'Out': value_tensor},
attrs={
'dtype': VarDesc.VarType.FP32,
'shape': [len(value_list)],
'values': value_list,
},
stop_gradient=True,
)
if framework.in_dygraph_mode():
with base.dygraph.no_grad():
tmp_out = _C_ops.scatter(
out_var, index_tensor, value_tensor, True
)
tmp_out._share_underline_tensor_to(out_var)
tmp_reshape_out = _C_ops.reshape(out_var, origin_shape)
tmp_reshape_out._share_underline_tensor_to(out_var)
if var.dtype != VarDesc.VarType.FP32:
tmp_cast_out = _C_ops.cast(out_var, var.dtype)
tmp_cast_out._share_underline_tensor_to(var)
elif framework.in_pir_mode():
out_var = _C_ops.scatter(out_var, index_tensor, value_tensor, True)
out_var = _C_ops.reshape(out_var, origin_shape)
if var.dtype != core.DataType.FLOAT32:
return _C_ops.cast(out_var, var.dtype)
return out_var
else:
op = block.append_op(
type="scatter",
inputs={
"X": out_var,
"Ids": index_tensor,
"Updates": value_tensor,
},
attrs={'overwrite': True},
outputs={"Out": out_var},
stop_gradient=True,
)
x_shape = block.create_var(
name=unique_name.generate(".".join([out_var.name, "XShape"])),
dtype=out_dtype,
shape=out_var.shape,
type=VarDesc.VarType.DENSE_TENSOR,
persistable=False,
stop_gradient=True,
)
block.append_op(
type="reshape2",
inputs={"X": out_var},
attrs={'shape': origin_shape},
outputs={"Out": out_var, "XShape": x_shape},
stop_gradient=True,
)
if var.dtype != VarDesc.VarType.FP32:
block.append_op(
type="cast",
inputs={"X": out_var},
outputs={"Out": var},
attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
stop_gradient=True,
)
if not in_dynamic_mode():
var.op = op
return op
python/paddle/nn/initializer/initializer.py
+213
View File
@@ -0,0 +1,213 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import functools
import math
from typing import TYPE_CHECKING, Literal, TypeAlias
import numpy as np
from ...base.framework import (
EagerParamBase,
default_main_program,
in_dygraph_mode,
use_pir_api,
)
from .lazy_init import lazy_init_helper
if TYPE_CHECKING:
import paddle
_NonLinearity: TypeAlias = Literal[ # noqa: PYI047
"sigmoid",
"linear",
"conv1d",
"conv2d",
"conv3d",
"conv1d_transpose",
"conv_transpose1d",
"conv2d_transpose",
"conv_transpose2d",
"conv3d_transpose",
"conv_transpose3d",
"tanh",
"relu",
"leaky_relu",
"selu",
]
__all__ = []
class Initializer:
"""Base class for parameter initializers
Defines the common interface of parameter initializers.
They add operations to the init program that are used
to initialize parameter. Users should not use this class
directly, but need to use one of its implementations.
"""
def __init__(self) -> None:
pass
def __call__(
self, param: paddle.Tensor, block: paddle.pir.Block | None = None
):
if not lazy_init_helper().state:
return self.forward(param, block)
return self._lazy_init(param, block)
def forward(
self, param: paddle.Tensor, block: paddle.pir.Block | None = None
) -> paddle.Tensor | None:
"""Add corresponding initialization operations to the network."""
raise NotImplementedError
def _lazy_init(
self, param: paddle.Tensor, block: paddle.pir.Block | None = None
):
"""
Apply lazy initialization
"""
assert in_dygraph_mode()
def init_op_creator(
forward, param: paddle.Tensor, block: paddle.pir.Block | None
):
if use_pir_api():
new_var = param
else:
new_var = param._to_static_var(True, block=block)
# Record initializer operator
with lazy_init_helper():
forward(new_var, block)
# Add hook function for initializing param in dygraph mode
param.set_init_func(functools.partial(self.forward))
param._init_op_creator = functools.partial(
init_op_creator, self.forward
)
return param
def _check_block(self, block: paddle.pir.Block | None) -> paddle.pir.Block:
if block is None:
block = default_main_program().global_block()
return block
def _compute_fans(self, var: paddle.Tensor) -> tuple[int, int]:
"""Compute the fan_in and the fan_out for layers
This method computes the fan_in and the fan_out
for neural network layers, if not specified. It is
not possible to perfectly estimate fan_in and fan_out.
This method will estimate it correctly for matrix multiply and
convolutions.
Args:
var: variable for which fan_in and fan_out have to be computed.
Returns:
tuple of two integers (fan_in, fan_out).
"""
shape = (
var._local_shape
if (isinstance(var, EagerParamBase) and var.is_dist())
else var.shape
)
if not shape or len(shape) == 0:
fan_in = fan_out = 1
elif len(shape) == 1:
fan_in = fan_out = shape[0]
elif len(shape) == 2:
# This is the case for simple matrix multiply
fan_in = shape[0]
fan_out = shape[1]
else:
# Assume this to be a convolutional kernel
# In PaddlePaddle, the shape of the kernel is like:
# [num_filters, num_filter_channels, ...] where the remaining
# dimensions are the filter_size
receptive_field_size = np.prod(shape[2:])
fan_in = shape[1] * receptive_field_size
fan_out = shape[0] * receptive_field_size
return (fan_in, fan_out)
def calculate_gain(
nonlinearity: str, param: bool | float | None = None
) -> float:
"""
Get the recommended ``gain`` value of some nonlinearity function. ``gain`` value can be used in some
``paddle.nn.initializer`` api to adjust the initialization value.
Args:
nonlinearity(str): name of nonlinearity activation function. If it is a linear function, such as:
`linear/conv1d/conv2d/conv3d/conv1d_transpose/conv2d_transpose/conv3d_transpose` , 1.0 will be returned.
param(bool|int|float|None, optional): optional parameter for some nonlinearity function. Now, it only applies to
'leaky_relu'. Default: None, it will be calculated as 0.01 in the formula.
Returns:
A float value, which is the recommended gain for this nonlinearity function.
Examples:
.. code-block:: pycon
>>> import paddle
>>> gain = paddle.nn.initializer.calculate_gain('tanh')
>>> print(gain)
1.6666666666666667
>>> # 5.0 / 3
>>> gain = paddle.nn.initializer.calculate_gain('leaky_relu', param=1.0)
>>> print(gain)
1.0
>>> # math.sqrt(2.0 / (1+param^2))
>>> initializer = paddle.nn.initializer.Orthogonal(gain)
"""
if param is None:
param = 0.01
else:
assert isinstance(param, (bool, int, float))
param = float(param)
recommended_gain = {
'sigmoid': 1,
'linear': 1,
'conv1d': 1,
'conv2d': 1,
'conv3d': 1,
'conv1d_transpose': 1,
'conv_transpose1d': 1,
'conv2d_transpose': 1,
'conv_transpose2d': 1,
'conv3d_transpose': 1,
'conv_transpose3d': 1,
'tanh': 5.0 / 3,
'relu': math.sqrt(2.0),
'leaky_relu': math.sqrt(2.0 / (1 + param**2)),
'selu': 3.0 / 4,
}
if nonlinearity in recommended_gain.keys():
return recommended_gain[nonlinearity]
else:
raise ValueError(
f"nonlinearity function {nonlinearity} is not supported now."
)
python/paddle/nn/initializer/kaiming.py
+394
View File
@@ -0,0 +1,394 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
# TODO: define the initializers of Kaiming functions in neural network
import math
from typing import TYPE_CHECKING
import paddle
from paddle import _C_ops
from ...base import core, framework, unique_name
from ...base.framework import (
_current_expected_place,
in_dygraph_mode,
in_pir_mode,
)
from .initializer import Initializer, calculate_gain
if TYPE_CHECKING:
from .initializer import _NonLinearity
__all__ = []
class MSRAInitializer(Initializer):
r"""Implements the MSRA initializer a.k.a. Kaiming Initializer
This class implements the weight initialization from the paper
`Delving Deep into Rectifiers: Surpassing Human-Level Performance on
ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
robust initialization method that particularly considers the rectifier
nonlinearities. In case of Uniform distribution, the range is [-x, x], where
.. math::
x = gain \times \sqrt{\frac{3}{fan\_in}}
In case of Normal distribution, the mean is 0 and the standard deviation
is
.. math::
\frac{gain}{\sqrt{{fan\_in}}}
Args:
uniform (bool, optional): whether to use uniform or normal distribution. Default is True.
fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be inferred automatically. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None.
seed (int32, optional): random seed. Default is 0.
negative_slope (float, optional): negative_slope (only used with leaky_relu). Default is 0.0.
nonlinearity(str, optional): the non-linear function. Default is relu.
mode(str, optional): the mode of initialization, can be 'fan_in' or 'fan_out'. When set to 'fan_in', the fan_in parameter is used for initialization. When set to 'fan_out', the out_features of trainable Tensor will be used. Default is 'fan_in'.
Note:
It is recommended to set fan_in to None for most cases.
"""
def __init__(
self,
uniform: bool = True,
fan_in: float | None = None,
seed: int = 0,
negative_slope: float = 0,
nonlinearity: _NonLinearity = 'relu',
mode: str = 'fan_in',
) -> None:
"""Constructor for MSRAInitializer"""
assert uniform is not None
assert seed is not None
super().__init__()
self._uniform = uniform
self._fan_in = fan_in
self._seed = seed
self._negative_slope = negative_slope
self._nonlinearity = nonlinearity
self._mode = mode
if self._mode not in ['fan_in', 'fan_out']:
raise ValueError(
"The mode of KaimingNormal/KaimingUniform should be 'fan_in' or 'fan_out', "
f"but received {self._mode}."
)
if self._mode == 'fan_out' and self._fan_in is not None:
raise ValueError(
"The mode of KaimingNormal/KaimingUniform is 'fan_out', "
"but fan_in is set. Please set fan_in to None."
)
def forward(
self, var: paddle.Tensor, block: paddle.pir.Block | None = None
) -> paddle.Tensor | None:
"""Initialize the input tensor with MSRA initialization.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The initialization op.
"""
assert not (
isinstance(var, framework.EagerParamBase) and var.is_dist()
), (
"Currently, kaiming initializer not support lazy init for dist param."
)
block = self._check_block(block)
assert isinstance(
var,
(
framework.Variable,
paddle.pir.Value,
paddle.pir.core.ParameterMeta,
),
)
assert isinstance(block, (framework.Block, paddle.pir.Block))
f_in, f_out = self._compute_fans(var)
# If fan_in is passed, use it
if self._mode == 'fan_in':
fan_in = f_in if self._fan_in is None else self._fan_in
if self._mode == 'fan_out':
fan_in = f_out
if self._seed == 0:
self._seed = block.program.random_seed
# to be compatible of fp16 initializers
origin_dtype = var.dtype
if origin_dtype == core.VarDesc.VarType.FP16 or (
origin_dtype == core.VarDesc.VarType.BF16 and not self._uniform
):
out_dtype = core.VarDesc.VarType.FP32
out_var = block.create_var(
name=unique_name.generate(
".".join(['masra_init', var.name, 'tmp'])
),
shape=var.shape,
dtype=out_dtype,
type=core.VarDesc.VarType.DENSE_TENSOR,
persistable=False,
)
elif (
origin_dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
and not self._uniform
):
out_dtype = core.DataType.FLOAT32
out_var = var
else:
out_dtype = origin_dtype
out_var = var
if in_dygraph_mode():
if self._uniform:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
std = gain / math.sqrt(float(fan_in))
limit = math.sqrt(3.0) * std
out_var = _C_ops.uniform(
var.shape,
out_dtype,
-limit,
limit,
self._seed,
var.place
if var.place._type()
else _current_expected_place(),
)
else:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
std = gain / math.sqrt(float(fan_in))
# var.place._type() means undefined, happens when initializer is specified in ParamAttr
place = (
var.place
if var.place._type()
else _current_expected_place()
)
out_var = _C_ops.gaussian(
out_var.shape, 0.0, std, self._seed, out_dtype, place
)
if origin_dtype == core.VarDesc.VarType.FP16 or (
origin_dtype
in [
core.VarDesc.VarType.BF16,
core.DataType.FLOAT16,
core.DataType.BFLOAT16,
]
and not self._uniform
):
var_tmp = _C_ops.cast(out_var, origin_dtype)
var_tmp._share_underline_tensor_to(var)
else:
out_var._share_underline_tensor_to(var)
return None
elif in_pir_mode():
if self._uniform:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
std = gain / math.sqrt(float(fan_in))
limit = math.sqrt(3.0) * std
out_var = _C_ops.uniform(
var.shape,
out_dtype,
-limit,
limit,
self._seed,
_current_expected_place(),
)
else:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
std = gain / math.sqrt(float(fan_in))
place = _current_expected_place()
out_var = _C_ops.gaussian(
out_var.shape, 0.0, std, self._seed, out_dtype, place
)
if (
origin_dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
and not self._uniform
):
return _C_ops.cast(out_var, origin_dtype)
return out_var
else:
if self._uniform:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
std = gain / math.sqrt(float(fan_in))
limit = math.sqrt(3.0) * std
op = block.append_op(
type="uniform_random",
inputs={},
outputs={"Out": out_var},
attrs={
"shape": out_var.shape,
"dtype": int(out_dtype),
"min": -limit,
"max": limit,
"seed": self._seed,
},
stop_gradient=True,
)
else:
gain = calculate_gain(self._nonlinearity, self._negative_slope)
std = gain / math.sqrt(float(fan_in))
op = block.append_op(
type="gaussian_random",
outputs={"Out": out_var},
attrs={
"shape": out_var.shape,
"dtype": int(out_dtype),
"mean": 0.0,
"std": std,
"seed": self._seed,
},
stop_gradient=True,
)
if origin_dtype == core.VarDesc.VarType.FP16 or (
origin_dtype == core.VarDesc.VarType.BF16 and not self._uniform
):
block.append_op(
type="cast",
inputs={"X": out_var},
outputs={"Out": var},
attrs={
"in_dtype": out_var.dtype,
"out_dtype": origin_dtype,
},
)
var.op = op
return op
class KaimingNormal(MSRAInitializer):
r"""Implements the Kaiming Normal initializer
This class implements the weight initialization from the paper
`Delving Deep into Rectifiers: Surpassing Human-Level Performance on
ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
robust initialization method that particularly considers the rectifier
nonlinearities.
In case of Normal distribution, the mean is 0 and the standard deviation
is
.. math::
\frac{gain}{\sqrt{{fan\_in}}}
Args:
fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be inferred automatically. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None.
negative_slope (float, optional): negative_slope (only used with leaky_relu). Default is 0.0.
nonlinearity(str, optional): the non-linear function. Default is relu.
mode(str, optional): the mode of initialization, can be 'fan_in' or 'fan_out'. When set to 'fan_in', the fan_in parameter is used for initialization. When set to 'fan_out', the out_features of trainable Tensor will be used. Default is 'fan_in'.
Note:
It is recommended to set fan_in to None for most cases.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn as nn
>>> linear = nn.Linear(2, 4, weight_attr=nn.initializer.KaimingNormal())
>>> data = paddle.rand([30, 10, 2], dtype='float32')
>>> res = linear(data)
"""
def __init__(
self,
fan_in: float | None = None,
negative_slope: float = 0.0,
nonlinearity: str = 'relu',
mode: str = 'fan_in',
) -> None:
super().__init__(
uniform=False,
fan_in=fan_in,
seed=0,
negative_slope=negative_slope,
nonlinearity=nonlinearity,
mode=mode,
)
class KaimingUniform(MSRAInitializer):
r"""Implements the Kaiming Uniform initializer
This class implements the weight initialization from the paper
`Delving Deep into Rectifiers: Surpassing Human-Level Performance on
ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
robust initialization method that particularly considers the rectifier
nonlinearities.
In case of Uniform distribution, the range is [-x, x], where
.. math::
x = gain \times \sqrt{\frac{3}{fan\_in}}
Args:
fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be inferred automatically. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None.
negative_slope (float, optional): negative_slope (only used with leaky_relu). Default is 0.0.
nonlinearity(str, optional): the non-linear function. Default is relu.
mode(str, optional): the mode of initialization, can be 'fan_in' or 'fan_out'. When set to 'fan_in', the fan_in parameter is used for initialization. When set to 'fan_out', the out_features of trainable Tensor will be used. Default is 'fan_in'.
Note:
It is recommended to set fan_in to None for most cases.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn as nn
>>> linear = nn.Linear(2, 4, weight_attr=nn.initializer.KaimingUniform())
>>> data = paddle.rand([30, 10, 2], dtype='float32')
>>> res = linear(data)
"""
def __init__(
self,
fan_in: float | None = None,
negative_slope: float = 0.0,
nonlinearity: str = 'relu',
mode: str = 'fan_in',
) -> None:
super().__init__(
uniform=True,
fan_in=fan_in,
seed=0,
negative_slope=negative_slope,
nonlinearity=nonlinearity,
mode=mode,
)
python/paddle/nn/initializer/lazy_init.py
+144
View File
@@ -0,0 +1,144 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from ...base import framework
if TYPE_CHECKING:
from types import TracebackType
__all__ = ["LazyGuard"]
class LazyInitHelper:
"""
A Helper Context to trigger switching mode between dygraph and static graph mode,
and holds the startup program resource.
"""
def __init__(self):
self._state = False
self._tracer = None
self._in_guard = False
def enable(self):
"""
Switch into lazy mode.
NOTE(dev): This is a very low level API and not exposed for user.
"""
if self._state:
return
assert framework.in_dygraph_mode(), (
"LazyInit.enable() is only available in dygraph mode."
)
self._state = True
def disable(self):
"""
Exit from lazy mode.
NOTE(dev): This is a very low level API and not exposed for user.
"""
if not self._state:
return
self._state = False
def __enter__(self):
"""
Switch into lazy mode and set _dygraph_tracer_ with None to convert
dygraph mode into static graph mode.
"""
self.enable()
if self._in_guard:
return
self._tracer = framework.global_var._dygraph_tracer_
framework.global_var._dygraph_tracer_ = None
self._in_guard = True
def __exit__(self, *args, **kwargs):
"""
Exit from lazy mode and recover _dygraph_tracer_.
"""
self.disable()
if not self._in_guard:
return
assert self._tracer is not None
framework.global_var._dygraph_tracer_ = self._tracer
self._tracer = None
self._in_guard = False
@property
def state(self):
return self._state
_lazy_init_helper = LazyInitHelper()
def lazy_init_helper():
global _lazy_init_helper
return _lazy_init_helper
class LazyGuard:
"""
LazyGuard is a wrapper interface for nn.Layer, it forwards the construct
process of user defined Layer. Meanwhile, it provides necessary API to
trigger EagerParamBase Lazy Initialization and get startup Program.
Examples:
.. code-block:: pycon
>>> from paddle import LazyGuard
>>> from paddle.nn import Linear
>>> with LazyGuard():
... # w and b are initialized lazily and have no memory.
... net = Linear(10, 10)
>>> for param in net.parameters():
... # Initialize param and allocate memory explicitly.
... param.initialize()
"""
def __enter__(self) -> None:
"""
Construct instance from class_obj by Lazy Initializing parameters.
Examples:
.. code-block:: pycon
>>> from paddle import LazyGuard
>>> from paddle.nn import Linear
>>> with LazyGuard():
... fc = LazyInit(Linear)(10, 10)
>>> for param in fc.parameters():
... param.initialize()
"""
lazy_init_helper().enable()
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> None:
lazy_init_helper().disable()
python/paddle/nn/initializer/normal.py
+408
View File
@@ -0,0 +1,408 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import paddle
from paddle import _C_ops, pir
from ...base import core, framework, unique_name
from ...base.data_feeder import check_variable_and_dtype
from ...base.framework import (
_current_expected_place,
in_dygraph_mode,
in_pir_mode,
)
from .initializer import Initializer
from .lazy_init import lazy_init_helper
__all__ = []
class NormalInitializer(Initializer):
"""Implements the Random Normal(Gaussian) distribution initializer
Args:
loc (float|complex, optional): mean of the normal distribution. Default is 0.0.
scale (float, optional): standard deviation of the normal distribution. Default is 1.0.
seed (int, optional): random seed. Default is 0.
"""
def __init__(
self, loc: float = 0.0, scale: float = 1.0, seed: int = 0
) -> None:
assert loc is not None
assert scale is not None
assert seed is not None
super().__init__()
self._mean = loc
self._std_dev = scale
self._seed = seed
if isinstance(self._mean, complex):
if self._mean.real != self._mean.imag:
raise ValueError(
"if mean is a complex number, its real part should equal imag part, "
f"but got real part: {self._mean.real} != imag part: {self._mean.imag}"
)
self._mean = self._mean.real
def forward(
self, var: paddle.Tensor, block: pir.Block | None = None
) -> paddle.Tensor | None:
"""Initialize the input tensor with Normal distribution.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The initialization op.
"""
assert not (
isinstance(var, framework.EagerParamBase) and var.is_dist()
), "Currently, normal initializer not support lazy init for dist param."
block = self._check_block(block)
assert isinstance(block, (framework.Block, pir.Block))
check_variable_and_dtype(
var,
"Out",
[
"uint16",
"float16",
"float32",
"float64",
"complex64",
"complex128",
],
"gaussian_random",
)
if self._seed == 0:
self._seed = block.program.random_seed
if in_dygraph_mode():
place = _current_expected_place()
out_var = _C_ops.gaussian(
var.shape,
self._mean,
self._std_dev,
self._seed,
var.dtype,
place,
)
out_var._share_underline_tensor_to(var)
return None
elif in_pir_mode():
place = _current_expected_place()
out_var = _C_ops.gaussian(
var.shape,
self._mean,
self._std_dev,
self._seed,
var.dtype,
place,
)
return out_var
else:
op = block.append_op(
type="gaussian_random",
outputs={"Out": var},
attrs={
"shape": var.shape,
"dtype": var.dtype,
"mean": self._mean,
"std": self._std_dev,
"seed": self._seed,
},
stop_gradient=True,
)
var.op = op
return op
class Normal(NormalInitializer):
"""The Random Normal (Gaussian) distribution initializer.
Args:
mean (float|complex, optional): mean of the normal distribution. Default is 0.0.
std (float, optional): standard deviation of the normal distribution. Default is 1.0.
name(str|None, optional): The default value is None. Normally there is no need for user to set this
property. For more information, please refer to :ref:`api_guide_Name`. Default: None.
Returns:
A parameter initialized by Random Normal (Gaussian) distribution.
Examples:
.. code-block:: pycon
>>> import paddle
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
>>> weight_attr = paddle.framework.ParamAttr(
... name="linear_weight",
... initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0),
... )
>>> bias_attr = paddle.framework.ParamAttr(
... name="linear_bias",
... initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0),
... )
>>> # doctest: +SKIP('name has been used')
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
>>> print(linear.weight)
Parameter containing:
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[ 2.1973135 -2.2697184],
[-1.9104223 -1.0541488]])
>>> print(linear.bias)
Parameter containing:
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
[ 0.7885926 -0.74719954])
>>> res = linear(data)
>>> print(res)
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[[ 1.0754838 -4.071067 ]],
[[ 1.0754838 -4.071067 ]],
[[ 1.0754838 -4.071067 ]]])
"""
def __init__(
self, mean: float = 0.0, std: float = 1.0, name: str | None = None
) -> None:
assert mean is not None, 'mean should not be None'
assert std is not None, 'std should not be None'
super().__init__(loc=mean, scale=std, seed=0)
class TruncatedNormalInitializer(Initializer):
"""Implements the Random TruncatedNormal(Gaussian) distribution initializer
Note:
It is better to set `a <= mean <= b`.
If `mean < a - 2*std` or `mean > b + 2*std`, the distribution of values may be incorrect.
Args:
loc (float, optional): Mean of the normal distribution. Default is :math:`0.0`.
scale (float, optional): Standard deviation of the normal distribution. Default is :math:`1.0`.
seed (int, optional): random seed. Default is 0.
a (float, optional): The minimum cutoff value. Default is -2.0.
b (float, optional): The maximum cutoff value. Default is 2.0.
"""
def __init__(
self,
loc: float = 0.0,
scale: float = 1.0,
seed: int = 0,
a: float = -2.0,
b: float = 2.0,
) -> None:
assert loc is not None
assert scale is not None
assert seed is not None
assert a is not None
assert b is not None
super().__init__()
self._mean = loc
self._std_dev = scale
self._seed = seed
self._a = a
self._b = b
def forward(
self, var: paddle.Tensor, block: pir.Block | None = None
) -> paddle.Tensor | None:
"""Initialize the input tensor with TruncatedNormal distribution.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The initialization op
"""
block = self._check_block(block)
if lazy_init_helper().state:
expected = (
framework.Variable,
paddle.pir.core.ParameterMeta,
core.eager.Tensor,
)
else:
expected = (
framework.Variable,
paddle.pir.Value,
paddle.pir.core.ParameterMeta,
)
assert isinstance(var, expected)
assert isinstance(block, (framework.Block, pir.Block))
if self._seed == 0:
self._seed = block.program.random_seed
# to be compatible of fp16 initializers
if var.dtype in [core.VarDesc.VarType.FP16, core.VarDesc.VarType.BF16]:
out_dtype = core.VarDesc.VarType.FP32
out_var = block.create_var(
name=unique_name.generate(
".".join(['truncated_gaussian_random', var.name, 'tmp'])
),
shape=var.shape,
dtype=out_dtype,
type=core.VarDesc.VarType.DENSE_TENSOR,
persistable=False,
)
else:
out_dtype = var.dtype
out_var = var
if in_dygraph_mode():
out_var = _C_ops.truncated_gaussian_random(
var.shape,
self._mean,
self._std_dev,
self._seed,
self._a,
self._b,
out_dtype,
_current_expected_place(),
)
if var.dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
]:
var_tmp = _C_ops.cast(out_var, var.dtype)
var_tmp._share_underline_tensor_to(var)
else:
out_var._share_underline_tensor_to(var)
return None
elif in_pir_mode():
out_var = _C_ops.truncated_gaussian_random(
var.shape,
self._mean,
self._std_dev,
self._seed,
self._a,
self._b,
out_dtype,
_current_expected_place(),
)
if var.dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
]:
var_tmp = _C_ops.cast(out_var, var.dtype)
var_tmp._share_underline_tensor_to(var)
return out_var
else:
op = block.append_op(
type="truncated_gaussian_random",
outputs={"Out": out_var},
attrs={
"shape": var.shape,
"dtype": out_dtype,
"mean": self._mean,
"std": self._std_dev,
"seed": self._seed,
"a": self._a,
"b": self._b,
},
stop_gradient=True,
)
if var.dtype in [
core.VarDesc.VarType.FP16,
core.VarDesc.VarType.BF16,
]:
block.append_op(
type="cast",
inputs={"X": out_var},
outputs={"Out": var},
attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
)
var.op = op
return op
class TruncatedNormal(TruncatedNormalInitializer):
"""The truncated normal distribution (Gaussian distribution) initializer.
Note:
It is better to set `a <= mean <= b`.
If `mean < a - 2*std` or `mean > b + 2*std`, the distribution of values may be incorrect.
Args:
mean (float, optional): Mean of the normal distribution. Default is :math:`0.0`.
std (float, optional): Standard deviation of the normal distribution. Default is :math:`1.0`.
a (float, optional): The minimum cutoff value. Default is -2.0.
b (float, optional): The maximum cutoff value. Default is 2.0.
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
Returns:
A parameter initialized by truncated normal distribution (Gaussian distribution).
Examples:
.. code-block:: pycon
>>> import paddle
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
>>> weight_attr = paddle.framework.ParamAttr(
... name="linear_weight",
... initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, std=2.0),
... )
>>> bias_attr = paddle.framework.ParamAttr(
... name="linear_bias",
... initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, std=2.0),
... )
>>> # doctest: +SKIP('name has been used')
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
>>> print(linear.weight)
Parameter containing:
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[-1.0981836 1.4140984],
[ 3.1390522 -2.8266568]])
>>> print(linear.bias)
Parameter containing:
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
[ -2.1546738 -1.6570673])
>>> res = linear(data)
>>> print(res)
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[[-0.11380529 -3.0696259 ]],
[[-0.11380529 -3.0696259 ]],
[[-0.11380529 -3.0696259 ]]])
"""
def __init__(
self,
mean: float = 0.0,
std: float = 1.0,
a: float = -2.0,
b: float = 2.0,
name: str | None = None,
) -> None:
assert mean is not None, 'mean should not be None'
assert std is not None, 'std should not be None'
assert a is not None, 'a should not be None'
assert b is not None, 'b should not be None'
super().__init__(loc=mean, scale=std, seed=0, a=a, b=b)
python/paddle/nn/initializer/orthogonal.py
+271
View File
@@ -0,0 +1,271 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import paddle
from paddle import _C_ops, pir
from paddle.utils import unique_name
from ...base import framework
from ...base.data_feeder import check_variable_and_dtype
from ...base.dygraph import no_grad
from .initializer import Initializer
__all__ = []
class Orthogonal(Initializer):
"""The orthogonal initializer. The initialized tensor is (semi) orthogonal.
It's only applied to Tensor whose dimension is greater than or equal to 2.
For the Tensor whose dimension is greater than 2, the 0 dimension is seen as ``rows`` ,
and the >=1 dimension are flattened as ``cols`` .
Which can be describe as:
.. code-block:: text
rows = shape[0]
cols = shape[1]·shape[2]···shape[N]
if rows < cols:
The rows are orthogonal vectors
elif rows > cols:
The columns are orthogonal vectors
else rows = cols:
Both rows and columns are orthogonal vectors
Args:
gain(float, optional): The multiplication coefficient for initialized tensor. Default: 1.0.
name(str|None, optional): The default value is None. Normally there is no need for user to set this
property. For more information, please refer to :ref:`api_guide_Name`.
Returns:
A parameter initialized by orthogonal initialized.
Examples:
.. code-block:: pycon
>>> import paddle
>>> weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Orthogonal())
>>> linear = paddle.nn.Linear(10, 15, weight_attr=weight_attr)
>>> # linear.weight: X * X' = I
>>> linear = paddle.nn.Linear(15, 10, weight_attr=weight_attr)
>>> # linear.weight: X' * X = I
"""
def __init__(self, gain: float = 1.0, name: str | None = None) -> None:
assert gain is not None, 'gain should not be None'
super().__init__()
self._gain = gain
def __call__(self, var: paddle.Tensor, block: pir.Block | None = None):
"""Initialize the input tensor with orthogonal initializer.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The last initialization op, it contain 8 ops in orthogonal initializer.
"""
assert not (
isinstance(var, framework.EagerParamBase) and var.is_dist()
), (
"Currently, orthogonal initializer not support lazy init for dist param."
)
block = self._check_block(block)
assert isinstance(
var, (framework.Variable, paddle.pir.Value, pir.core.ParameterMeta)
)
assert isinstance(block, (framework.Block, pir.Block))
self._seed = block.program.random_seed
shape = var.shape
assert len(shape) >= 2, (
"Only Tensor with 2 or more dimensions can be initialized by Orthogonal"
)
row = shape[0]
col = 1
for i in shape[1:]:
col *= i
flatten_shape = [max(row, col), min(row, col)]
if framework.in_dygraph_mode():
with no_grad():
place = framework._current_expected_place()
normal_var = _C_ops.gaussian(
flatten_shape, 0.0, 1.0, self._seed, var.dtype, place
)
q, r = _C_ops.qr(normal_var, 'reduced')
r_diag = _C_ops.diag(r, 0, 0)
r_sign = _C_ops.sign(r_diag)
q = _C_ops.multiply(q, r_sign)
if row < col:
q = _C_ops.transpose(q, [1, 0])
q = _C_ops.reshape(q, var.shape)
tmp = _C_ops.scale(q, self._gain, 0.0, True)
tmp._share_underline_tensor_to(var)
return None
elif framework.in_pir_mode():
place = framework._current_expected_place()
normal_var = _C_ops.gaussian(
flatten_shape, 0.0, 1.0, self._seed, var.dtype, place
)
q, r = _C_ops.qr(normal_var, 'reduced')
r_diag = _C_ops.diag(r, 0, 0)
r_sign = _C_ops.sign(r_diag)
q = _C_ops.multiply(q, r_sign)
if row < col:
q = _C_ops.transpose(q, [1, 0])
q = _C_ops.reshape(q, var.shape)
tmp = _C_ops.scale(q, self._gain, 0.0, True)
return tmp
# 'qr' op only support float32/float64 now
check_variable_and_dtype(
var, "Out", ["float32", "float64"], "Orthogonal"
)
normal_var = block.create_var(
name=unique_name.generate('.'.join(['gaussian_random', 'tmp'])),
dtype=var.dtype,
persistable=False,
stop_gradient=True,
)
block.append_op(
type='gaussian_random',
inputs={},
outputs={'Out': normal_var},
attrs={
'mean': 0.0,
'std': 1.0,
'shape': flatten_shape,
'seed': self._seed,
'dtype': var.dtype,
},
stop_gradient=True,
)
q = block.create_var(
name=unique_name.generate('.'.join(['qr', 'q', 'tmp'])),
dtype=normal_var.dtype,
persistable=False,
stop_gradient=True,
)
r = block.create_var(
name=unique_name.generate('.'.join(['qr', 'r', 'tmp'])),
dtype=normal_var.dtype,
persistable=False,
stop_gradient=True,
)
block.append_op(
type='qr',
inputs={'X': [normal_var]},
outputs={
'Q': q,
'R': r,
},
attrs={'mode': 'reduced'},
stop_gradient=True,
)
r_diag = block.create_var(
name=unique_name.generate('.'.join(['diag', 'tmp'])),
dtype=r.dtype,
persistable=False,
stop_gradient=True,
)
block.append_op(
type='diag_v2',
inputs={'X': r},
outputs={'Out': r_diag},
attrs={'offset': 0, 'padding_value': 0},
stop_gradient=True,
)
r_sign = r_diag
block.append_op(
type='sign',
inputs={'X': [r_diag]},
outputs={'Out': r_sign},
stop_gradient=True,
)
block.append_op(
type='elementwise_mul',
inputs={'X': q, 'Y': r_sign},
outputs={'Out': q},
attrs={},
stop_gradient=True,
)
x_shape = block.create_var(
name=unique_name.generate('.'.join(['transpose', 'shape', 'tmp'])),
dtype=q.dtype,
persistable=False,
stop_gradient=True,
)
if row < col:
q_transpose = block.create_var(
name=unique_name.generate('.'.join(['transpose', 'tmp'])),
dtype=q.dtype,
persistable=False,
stop_gradient=True,
)
block.append_op(
type='transpose2',
inputs={'X': q},
outputs={'Out': q_transpose, 'XShape': x_shape},
attrs={'axis': [1, 0]},
stop_gradient=True,
)
q = q_transpose
block.append_op(
type='reshape2',
inputs={'X': q},
outputs={'Out': q, "XShape": x_shape},
attrs={'shape': var.shape},
stop_gradient=True,
)
op = block.append_op(
type='scale',
inputs={'X': q},
outputs={'Out': var},
attrs={'scale': self._gain, 'bias': 0.0},
)
return op
python/paddle/nn/initializer/uniform.py
+239
View File
@@ -0,0 +1,239 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import paddle
from paddle import _C_ops, pir
from ...base import core, framework, unique_name
from ...base.data_feeder import check_variable_and_dtype
from ...base.framework import (
_current_expected_place,
in_dygraph_mode,
in_pir_mode,
)
from .initializer import Initializer
__all__ = []
class UniformInitializer(Initializer):
"""Implements the random uniform distribution initializer
Args:
low (float, optional): Lower boundary of the uniform distribution. Default is :math:`-1.0`.
high (float, optional): Upper boundary of the uniform distribution. Default is :math:`1.0`.
seed (int, optional): Random seed. Default is 0.
diag_num (int, optional): the number of diagonal elements to initialize.
If set to 0, diagonal initialization will be not performed. Default is 0.
diag_step (int, optional): Step size between two diagonal elements,
which is generally the width of the square matrix. Default is 0.
diag_val (float, optional): the value of the diagonal element to be initialized,
default 1.0. It takes effect only if the diag_num is greater than 0. Default is :math:`1.0`.
"""
def __init__(
self,
low: float = -1.0,
high: float = 1.0,
seed: int = 0,
diag_num: int = 0,
diag_step: int = 0,
diag_val: float = 1.0,
) -> None:
assert low is not None
assert high is not None
assert high >= low
assert seed is not None
assert diag_num is not None
assert diag_step is not None
assert diag_val is not None
if diag_num > 0 or diag_step > 0:
assert diag_num > 0 and diag_step > 0
super().__init__()
self._low = low
self._high = high
self._seed = seed
self._diag_num = diag_num
self._diag_step = diag_step
self._diag_val = diag_val
def forward(
self, var: paddle.Tensor, block: pir.Block | None = None
) -> paddle.Tensor | None:
"""Initialize the input tensor with Uniform distribution.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The initialization op
"""
assert not (
isinstance(var, framework.EagerParamBase) and var.is_dist()
), (
"Currently, uniform initializer not support lazy init for dist param."
)
block = self._check_block(block)
assert isinstance(block, (framework.Block, pir.Block))
if not in_dygraph_mode():
check_variable_and_dtype(
var,
"Out",
["uint16", "float16", "float32", "float64"],
"uniform_random",
)
if self._seed == 0:
self._seed = block.program.random_seed
# to be compatible of fp16 initializers
if var.dtype == core.VarDesc.VarType.FP16:
out_dtype = core.VarDesc.VarType.FP32
out_var = block.create_var(
name=unique_name.generate(
".".join(['uniform_random', var.name, 'tmp'])
),
shape=var.shape,
dtype=out_dtype,
type=core.VarDesc.VarType.DENSE_TENSOR,
persistable=False,
)
else:
out_dtype = var.dtype
out_var = var
if in_dygraph_mode():
out_var = _C_ops.uniform(
var.shape,
out_dtype,
self._low,
self._high,
self._seed,
var.place if var.place._type() else _current_expected_place(),
)
if var.dtype == core.VarDesc.VarType.FP16:
var_tmp = _C_ops.cast(out_var, var.dtype)
var_tmp._share_underline_tensor_to(var)
else:
out_var._share_underline_tensor_to(var)
return None
elif in_pir_mode():
if var.dtype == core.DataType.FLOAT16:
out_dtype = core.DataType.FLOAT32
else:
out_dtype = var.dtype
out_var = _C_ops.uniform(
var.shape,
out_dtype,
self._low,
self._high,
self._seed,
_current_expected_place(),
)
if (
var.dtype == core.DataType.FLOAT16
and out_var.dtype != core.DataType.FLOAT16
):
return _C_ops.cast(out_var, var.dtype)
return out_var
else:
op = block.append_op(
type="uniform_random",
inputs={},
outputs={"Out": out_var},
attrs={
"shape": var.shape,
"dtype": out_dtype,
"min": self._low,
"max": self._high,
"seed": self._seed,
"diag_num": self._diag_num,
"diag_step": self._diag_step,
"diag_val": self._diag_val,
},
stop_gradient=True,
)
if var.dtype == core.VarDesc.VarType.FP16:
block.append_op(
type="cast",
inputs={"X": out_var},
outputs={"Out": var},
attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
)
var.op = op
return op
class Uniform(UniformInitializer):
"""The uniform distribution initializer.
Args:
low (float, optional): Lower boundary of the uniform distribution. Default is :math:`-1.0`.
high (float, optional): Upper boundary of the uniform distribution. Default is :math:`1.0`.
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
Returns:
A parameter initialized by uniform distribution.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(1)
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
>>> weight_attr = paddle.framework.ParamAttr(
... name="linear_weight",
... initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5),
... )
>>> bias_attr = paddle.framework.ParamAttr(
... name="linear_bias",
... initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5),
... )
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
>>> print(linear.weight)
Parameter containing:
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[-0.48212373, 0.26492310],
[ 0.17605734, -0.45379421]])
>>> print(linear.bias)
Parameter containing:
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
[-0.11236754, 0.46462214])
>>> res = linear(data)
>>> print(res)
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[[-0.41843393, 0.27575102]],
[[-0.41843393, 0.27575102]],
[[-0.41843393, 0.27575102]]])
"""
def __init__(
self, low: float = -1.0, high: float = 1.0, name: str | None = None
) -> None:
assert low is not None, 'low should not be None'
assert high is not None, 'high should not be None'
assert high >= low, 'high should greater or equal than low'
super().__init__(
low=low, high=high, seed=0, diag_num=0, diag_step=0, diag_val=1.0
)
python/paddle/nn/initializer/xavier.py
+432
View File
@@ -0,0 +1,432 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import math
import paddle
from paddle import _C_ops
from ...base import core, framework, unique_name
from ...base.data_feeder import check_variable_and_dtype
from ...base.framework import (
_current_expected_place,
in_dygraph_mode,
in_pir_mode,
)
from .initializer import Initializer
__all__ = []
class XavierInitializer(Initializer):
r"""
This class implements the Xavier weight initializer from the paper
`Understanding the difficulty of training deep feedforward neural
networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
by Xavier Glorot and Yoshua Bengio.
This initializer is designed to keep the scale of the gradients
approximately same in all the layers. In case of Uniform distribution,
the range is [-x, x], where
.. math::
x = gain \times \sqrt{\\frac{6.0}{fan\_in + fan\_out}}
In case of Normal distribution, the mean is 0 and the standard deviation
is
.. math::
gain \times \sqrt{\\frac{2.0}{fan\_in + fan\_out}}
Args:
uniform (bool, optional): whether to use uniform ,if False use normal distribution. Default is True.
fan_in (float|None, optional): fan_in for Xavier initialization. If None, it is
inferred from the variable. Default is None.
fan_out (float|None, optional): fan_out for Xavier initialization. If None, it is
inferred from the variable. Default is None.
seed (int, optional): Random seed. Default is 0.
gain (float, optional): Scaling Tensor. Default is 1.0.
Note:
It is recommended to set fan_in and fan_out to None for most cases.
"""
def __init__(
self,
uniform: bool = True,
fan_in: float | None = None,
fan_out: float | None = None,
seed: int = 0,
gain: float = 1.0,
) -> None:
assert uniform is not None
assert seed is not None
super().__init__()
self._uniform = uniform
self._fan_in = fan_in
self._fan_out = fan_out
self._seed = seed
self._gain = gain
def forward(
self, var: paddle.Tensor, block: paddle.pir.Block | None = None
) -> paddle.Tensor | None:
"""Initialize the input tensor with Xavier initialization.
Args:
var(Tensor): Tensor that needs to be initialized.
block(Block|None, optional): The block in which initialization ops
should be added. Used in static graph only, default None.
Returns:
The initialization op
"""
block = self._check_block(block)
assert isinstance(block, (framework.Block, paddle.pir.Block))
if not isinstance(var, paddle.pir.core.ParameterMeta):
check_variable_and_dtype(
var,
"Out",
["uint16", "float16", "float32", "float64"],
"xavier_init",
)
f_in, f_out = self._compute_fans(var)
# If fan_in and fan_out are passed, use them
fan_in = f_in if self._fan_in is None else self._fan_in
fan_out = f_out if self._fan_out is None else self._fan_out
if self._seed == 0:
self._seed = block.program.random_seed
out_var_shape = (
var._local_shape
if (isinstance(var, framework.EagerParamBase) and var.is_dist())
else var.shape
)
# to be compatible of fp16 initializers
origin_dtype = var.dtype
if origin_dtype == core.VarDesc.VarType.FP16 or (
origin_dtype == core.VarDesc.VarType.BF16 and not self._uniform
):
out_dtype = core.VarDesc.VarType.FP32
out_var = block.create_var(
name=unique_name.generate(
".".join(['xavier_init', var.name, 'tmp'])
),
shape=out_var_shape,
dtype=out_dtype,
type=core.VarDesc.VarType.DENSE_TENSOR,
persistable=False,
)
elif (
origin_dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
and not self._uniform
):
out_dtype = core.DataType.FLOAT32
out_var = var
else:
out_dtype = origin_dtype
out_var = var
if in_dygraph_mode():
if self._uniform:
if 0 in [fan_in, fan_out]:
limit = 0.0
else:
limit = self._gain * math.sqrt(
6.0 / float(fan_in + fan_out)
)
out_var = _C_ops.uniform(
out_var_shape,
out_dtype,
-limit,
limit,
self._seed,
_current_expected_place(),
)
else:
if 0 in [fan_in, fan_out]:
std = 0.0
else:
std = self._gain * math.sqrt(2.0 / float(fan_in + fan_out))
place = _current_expected_place()
out_var = _C_ops.gaussian(
out_var_shape,
0.0,
std,
self._seed,
out_dtype,
place,
)
if origin_dtype == core.VarDesc.VarType.FP16 or (
origin_dtype
in [
core.VarDesc.VarType.BF16,
core.DataType.FLOAT16,
core.DataType.BFLOAT16,
]
and not self._uniform
):
out_var = _C_ops.cast(out_var, origin_dtype)
if isinstance(var, framework.EagerParamBase) and var.is_dist():
# lazy init for dist tensor
out_var = (
paddle.distributed.auto_parallel.api.dtensor_from_local(
out_var, var.process_mesh, var.placements
)
)
out_var._share_underline_tensor_to(var)
return None
elif in_pir_mode():
if self._uniform:
if 0 in [fan_in, fan_out]:
limit = 0.0
else:
limit = self._gain * math.sqrt(
6.0 / float(fan_in + fan_out)
)
out_var = paddle._pir_ops.uniform(
out_var.shape,
out_dtype,
-limit,
limit,
self._seed,
_current_expected_place(),
)
else:
if 0 in [fan_in, fan_out]:
std = 0.0
else:
std = self._gain * math.sqrt(2.0 / float(fan_in + fan_out))
out_var = _C_ops.gaussian(
out_var.shape,
0.0,
std,
self._seed,
out_dtype,
_current_expected_place(),
)
if (
origin_dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
and not self._uniform
):
return _C_ops.cast(out_var, origin_dtype)
return out_var
else:
if self._uniform:
if 0 in [fan_in, fan_out]:
limit = 0.0
else:
limit = self._gain * math.sqrt(
6.0 / float(fan_in + fan_out)
)
op = block.append_op(
type="uniform_random",
inputs={},
outputs={"Out": out_var},
attrs={
"shape": out_var.shape,
"dtype": out_dtype,
"min": -limit,
"max": limit,
"seed": self._seed,
},
stop_gradient=True,
)
else:
if 0 in [fan_in, fan_out]:
std = 0.0
else:
std = self._gain * math.sqrt(2.0 / float(fan_in + fan_out))
op = block.append_op(
type="gaussian_random",
outputs={"Out": out_var},
attrs={
"shape": out_var.shape,
"dtype": out_var.dtype,
"mean": 0.0,
"std": std,
"seed": self._seed,
},
stop_gradient=True,
)
if origin_dtype == core.VarDesc.VarType.FP16 or (
origin_dtype == core.VarDesc.VarType.BF16 and not self._uniform
):
block.append_op(
type="cast",
inputs={"X": out_var},
outputs={"Out": var},
attrs={
"in_dtype": out_var.dtype,
"out_dtype": origin_dtype,
},
)
var.op = op
return op
class XavierNormal(XavierInitializer):
r"""
This class implements the Xavier weight initializer from the paper
`Understanding the difficulty of training deep feedforward neural
networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
by Xavier Glorot and Yoshua Bengio, using a normal distribution whose mean is :math:`0` and standard deviation is
.. math::
gain \times \sqrt{\frac{2.0}{fan\_in + fan\_out}}.
Args:
fan_in (float|None, optional): fan_in for Xavier initialization, which is
inferred from the Tensor. Default is None.
fan_out (float|None, optional): fan_out for Xavier initialization, which is
inferred from the Tensor. Default is None.
gain (float, optional): Scaling Tensor. Default is 1.0.
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
Returns:
A parameter initialized by Xavier weight, using a normal distribution.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(1)
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
>>> weight_attr = paddle.framework.ParamAttr(
... name="linear_weight",
... initializer=paddle.nn.initializer.XavierNormal(),
... )
>>> bias_attr = paddle.framework.ParamAttr(
... name="linear_bias",
... initializer=paddle.nn.initializer.XavierNormal(),
... )
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
>>> print(linear.weight)
Parameter containing:
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[-0.21607460, 0.08382989],
[ 0.29147008, -0.07049121]])
>>> print(linear.bias)
Parameter containing:
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
[1.06076419, 0.87684733])
>>> res = linear(data)
>>> print(res)
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[[1.13615966, 0.89018601]],
[[1.13615966, 0.89018601]],
[[1.13615966, 0.89018601]]])
"""
def __init__(
self,
fan_in: float | None = None,
fan_out: float | None = None,
gain: float = 1.0,
name: str | None = None,
) -> None:
super().__init__(
uniform=False, fan_in=fan_in, fan_out=fan_out, seed=0, gain=gain
)
class XavierUniform(XavierInitializer):
r"""
This class implements the Xavier weight initializer from the paper
`Understanding the difficulty of training deep feedforward neural
networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
by Xavier Glorot and Yoshua Bengio.
This initializer is designed to keep the scale of the gradients
approximately same in all the layers. In case of Uniform distribution,
the range is :math:`[-x,x]`, where
.. math::
x = gain \times \sqrt{\frac{6.0}{fan\_in + fan\_out}}.
Args:
fan_in (float|None, optional): fan_in for Xavier initialization, which is
inferred from the Tensor. Default is None.
fan_out (float|None, optional): fan_out for Xavier initialization, which is
inferred from the Tensor. Default is None.
gain (float, optional): Scaling Tensor. Default is 1.0.
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
Returns:
A parameter initialized by Xavier weight, using a uniform distribution.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(1)
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
>>> weight_attr = paddle.framework.ParamAttr(
... name="linear_weight",
... initializer=paddle.nn.initializer.XavierUniform(),
... )
>>> bias_attr = paddle.framework.ParamAttr(
... name="linear_bias",
... initializer=paddle.nn.initializer.XavierUniform(),
... )
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
>>> print(linear.weight)
Parameter containing:
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[-1.18095720, 0.64892638],
[ 0.43125069, -1.11156428]])
>>> print(linear.bias)
Parameter containing:
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
[-0.27524316, 1.13808715])
>>> res = linear(data)
>>> print(res)
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
[[[-1.02494967, 0.67544925]],
[[-1.02494967, 0.67544925]],
[[-1.02494967, 0.67544925]]])
"""
def __init__(
self,
fan_in: float | None = None,
fan_out: float | None = None,
gain: float = 1.0,
name: str | None = None,
) -> None:
super().__init__(
uniform=True, fan_in=fan_in, fan_out=fan_out, seed=0, gain=gain
)
python/paddle/nn/layer/__init__.py
+128
View File
@@ -0,0 +1,128 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO: define activation functions of neural network
from . import container, rnn, transformer # noqa: F401
from .activation import ( # noqa: F401
CELU,
LeakyReLU,
LogSoftmax,
PReLU,
ReLU,
ReLU6,
RReLU,
Sigmoid,
Softmax,
Softmax2D,
)
from .common import ( # noqa: F401
AlphaDropout,
Bilinear,
ConstantPad1D,
ConstantPad2D,
ConstantPad3D,
CosineSimilarity,
Dropout,
Dropout2D,
Dropout3D,
Embedding,
FeatureAlphaDropout,
Flatten,
Fold,
Identity,
Linear,
Pad1D,
Pad2D,
Pad3D,
ReflectionPad1D,
ReflectionPad2D,
ReflectionPad3D,
ReplicationPad1D,
ReplicationPad2D,
ReplicationPad3D,
Unflatten,
Upsample,
UpsamplingBilinear2D,
UpsamplingNearest2D,
ZeroPad2D,
)
from .container import LayerDict # noqa: F401
from .conv import ( # noqa: F401
Conv1D,
Conv1DTranspose,
Conv2D,
Conv2DTranspose,
Conv3D,
Conv3DTranspose,
)
from .distance import PairwiseDistance # noqa: F401
from .layers import Layer # noqa: F401
from .loss import ( # noqa: F401
AdaptiveLogSoftmaxWithLoss,
BCELoss,
BCEWithLogitsLoss,
CrossEntropyLoss,
CTCLoss,
GaussianNLLLoss,
HingeEmbeddingLoss,
KLDivLoss,
L1Loss,
MarginRankingLoss,
MSELoss,
MultiLabelMarginLoss,
MultiLabelSoftMarginLoss,
MultiMarginLoss,
NLLLoss,
PoissonNLLLoss,
RNNTLoss,
SmoothL1Loss,
SoftMarginLoss,
TripletMarginLoss,
TripletMarginWithDistanceLoss,
)
from .norm import ( # noqa: F401
BatchNorm1D,
BatchNorm2D,
BatchNorm3D,
GroupNorm,
LayerNorm,
LocalResponseNorm,
SpectralNorm,
SyncBatchNorm,
)
from .pooling import ( # noqa: F401
AdaptiveAvgPool1D,
AdaptiveAvgPool2D,
AdaptiveAvgPool3D,
AdaptiveMaxPool1D,
AdaptiveMaxPool2D,
AdaptiveMaxPool3D,
AvgPool1D,
AvgPool2D,
AvgPool3D,
FractionalMaxPool2D,
FractionalMaxPool3D,
LPPool1D,
LPPool2D,
MaxPool1D,
MaxPool2D,
MaxPool3D,
MaxUnPool1D,
MaxUnPool2D,
MaxUnPool3D,
)
from .vision import ChannelShuffle, PixelShuffle, PixelUnshuffle # noqa: F401
__all__ = []
python/paddle/nn/layer/activation.py
+1816
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/common.py
+2861
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/container.py
+871
View File
@@ -0,0 +1,871 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import typing
from collections import OrderedDict
from collections.abc import Iterable, Iterator, Mapping, Sequence
from itertools import chain
from typing import Any
from typing_extensions import Self
import paddle
from paddle import Tensor
from ...base.dygraph.base import param_guard
from ...base.framework import Parameter
from .layers import Layer
__all__ = []
from paddle.utils.decorator_utils import (
param_one_alias,
)
class LayerDict(Layer):
"""
LayerDict holds sublayers in the ordered dictionary, and sublayers it contains are properly registered.
Held sublayers can be accessed like a regular ordered python dictionary.
Parameters:
sublayers (LayerDict|OrderedDict|list[(key,Layer)...], optional): iterable of key/value pairs, the type of value is 'paddle.nn.Layer' .
Examples:
.. code-block:: pycon
>>> import paddle
>>> import numpy as np
>>> from collections import OrderedDict
>>> sublayers = OrderedDict(
... [
... ('conv1d', paddle.nn.Conv1D(3, 2, 3)),
... ('conv2d', paddle.nn.Conv2D(3, 2, 3)),
... ('conv3d', paddle.nn.Conv3D(4, 6, (3, 3, 3))),
... ]
... )
>>> layers_dict = paddle.nn.LayerDict(sublayers=sublayers)
>>> l = layers_dict['conv1d']
>>> for k in layers_dict:
... l = layers_dict[k]
>>> print(len(layers_dict))
3
>>> del layers_dict['conv2d']
>>> print(len(layers_dict))
2
>>> conv1d = layers_dict.pop('conv1d')
>>> print(len(layers_dict))
1
>>> layers_dict.clear()
>>> print(len(layers_dict))
0
"""
@param_one_alias(["sublayers", "modules"])
def __init__(
self,
sublayers: (
LayerDict
| typing.Mapping[str, Layer]
| Sequence[tuple[str, Layer]]
| None
) = None,
) -> None:
super().__init__()
if sublayers is not None:
self.update(sublayers)
def __getitem__(self, key: str) -> Layer:
return self._sub_layers[key]
def __setitem__(self, key: str, sublayer: Layer) -> Layer:
return self.add_sublayer(key, sublayer)
def __delitem__(self, key: str) -> None:
del self._sub_layers[key]
def __len__(self) -> int:
return len(self._sub_layers)
def __iter__(self) -> Iterator[str]:
return iter(self._sub_layers)
def __contains__(self, key: str) -> bool:
return key in self._sub_layers
def clear(self) -> None:
"""
Clear all the sublayers in the LayerDict.
Parameters:
None.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from collections import OrderedDict
>>> sublayers = OrderedDict([
... ('conv1d', paddle.nn.Conv1D(3, 2, 3)),
... ('conv2d', paddle.nn.Conv2D(3, 2, 3)),
... ('conv3d', paddle.nn.Conv3D(4, 6, (3, 3, 3))),
>>> ])
>>> layer_dict = paddle.nn.LayerDict(sublayers=sublayers)
>>> len(layer_dict)
3
>>> layer_dict.clear()
>>> len(layer_dict)
0
"""
self._sub_layers.clear()
def pop(self, key: str) -> Layer:
"""
Remove the key from the LayerDict and return the layer of the key.
Parameters:
key (str): the key to be removed.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from collections import OrderedDict
>>> sublayers = OrderedDict([
... ('conv1d', paddle.nn.Conv1D(3, 2, 3)),
... ('conv2d', paddle.nn.Conv2D(3, 2, 3)),
... ('conv3d', paddle.nn.Conv3D(4, 6, (3, 3, 3))),
>>> ])
>>> layer_dict = paddle.nn.LayerDict(sublayers=sublayers)
>>> len(layer_dict)
3
>>> layer_dict.pop('conv2d')
>>> len(layer_dict)
2
"""
v = self[key]
del self[key]
return v
def keys(self) -> Iterable[str]:
"""
Return the iterable of the keys in LayerDict.
Parameters:
None.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from collections import OrderedDict
>>> sublayers = OrderedDict([
... ('conv1d', paddle.nn.Conv1D(3, 2, 3)),
... ('conv2d', paddle.nn.Conv2D(3, 2, 3)),
... ('conv3d', paddle.nn.Conv3D(4, 6, (3, 3, 3))),
>>> ])
>>> layer_dict = paddle.nn.LayerDict(sublayers=sublayers)
>>> for k in layer_dict.keys():
... print(k)
conv1d
conv2d
conv3d
"""
return self._sub_layers.keys()
def items(self) -> Iterable[tuple[str, Layer]]:
"""
Return the iterable of the key/value pairs in LayerDict.
Parameters:
None.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from collections import OrderedDict
>>> sublayers = OrderedDict([
... ('conv1d', paddle.nn.Conv1D(3, 2, 3)),
... ('conv2d', paddle.nn.Conv2D(3, 2, 3)),
... ('conv3d', paddle.nn.Conv3D(4, 6, (3, 3, 3))),
>>> ])
>>> layer_dict = paddle.nn.LayerDict(sublayers=sublayers)
>>> for k, v in layer_dict.items():
... print(f"{k}:", v)
conv1d : Conv1D(3, 2, kernel_size=[3], data_format=NCL)
conv2d : Conv2D(3, 2, kernel_size=[3, 3], data_format=NCHW)
conv3d : Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)
"""
return self._sub_layers.items()
def values(self) -> Iterable[Layer]:
"""
Return the iterable of the values in LayerDict.
Parameters:
None.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from collections import OrderedDict
>>> sublayers = OrderedDict([
... ('conv1d', paddle.nn.Conv1D(3, 2, 3)),
... ('conv2d', paddle.nn.Conv2D(3, 2, 3)),
... ('conv3d', paddle.nn.Conv3D(4, 6, (3, 3, 3))),
>>> ])
>>> layer_dict = paddle.nn.LayerDict(sublayers=sublayers)
>>> for v in layer_dict.values():
... print(v)
Conv1D(3, 2, kernel_size=[3], data_format=NCL)
Conv2D(3, 2, kernel_size=[3, 3], data_format=NCHW)
Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)
"""
return self._sub_layers.values()
@param_one_alias(["sublayers", "modules"])
def update(
self,
sublayers: (
LayerDict | typing.Mapping[str, Layer] | Sequence[tuple[str, Layer]]
),
) -> None:
"""
Update the key/values pairs in sublayers to the LayerDict, overwriting the existing keys.
Parameters:
sublayers (LayerDict|OrderedDict|list[(key,Layer)...]): iterable of key/value pairs, the type of value is 'paddle.nn.Layer' .
Examples:
.. code-block:: pycon
>>> import paddle
>>> from collections import OrderedDict
>>> sublayers = OrderedDict([
... ('conv1d', paddle.nn.Conv1D(3, 2, 3)),
... ('conv2d', paddle.nn.Conv2D(3, 2, 3)),
... ('conv3d', paddle.nn.Conv3D(4, 6, (3, 3, 3))),
>>> ])
>>> new_sublayers = OrderedDict([
... ('relu', paddle.nn.ReLU()),
... ('conv2d', paddle.nn.Conv2D(4, 2, 4)),
>>> ])
>>> layer_dict = paddle.nn.LayerDict(sublayers=sublayers)
>>> layer_dict.update(new_sublayers)
>>> for k, v in layer_dict.items():
... print(f"{k}:", v)
conv1d : Conv1D(3, 2, kernel_size=[3], data_format=NCL)
conv2d : Conv2D(4, 2, kernel_size=[4, 4], data_format=NCHW)
conv3d : Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)
relu : ReLU()
"""
assert isinstance(sublayers, Iterable), (
"The type of sublayers is not iterable of key/value pairs, the type of sublayers is "
+ type(sublayers).__name__
)
if isinstance(sublayers, (OrderedDict, LayerDict, Mapping)):
for key, layer in sublayers.items():
self.add_sublayer(key, layer)
else:
# handle this format [(key1, layer1), (key2, layer2)...]
for i, kv in enumerate(sublayers):
if len(kv) != 2:
raise ValueError(
"The length of the "
+ str(i)
+ "'s element in sublayers is "
+ str(len(kv))
+ ", which must be 2."
)
self.add_sublayer(kv[0], kv[1])
class ParameterDict(Layer):
"""
Holds parameters in a dictionary.
ParameterDict can be indexed like a regular Python dictionary, but Parameters it contains are properly registered.
Parameters:
parameters (iterable, optional): a mapping (dictionary) of (string : Any) or an iterable of key-value pairs of type (string, Any)
alias: values
Examples:
.. code-block:: pycon
>>> import paddle
>>> class MyLayer(paddle.nn.Layer):
... def __init__(self, num_stacked_param):
... super().__init__()
... # create ParameterDict with iterable Parameters
... self.params = paddle.nn.ParameterDict(
... {f"t{i}": paddle.create_parameter(shape=[2, 2], dtype='float32') for i in range(num_stacked_param)}
... )
...
... def forward(self, x):
... for i, key in enumerate(self.params):
... x = paddle.matmul(x, self.params[key])
... return x
>>> x = paddle.uniform(shape=[5, 2], dtype='float32')
>>> num_stacked_param = 4
>>> model = MyLayer(num_stacked_param)
>>> print(len(model.params))
4
>>> res = model(x)
>>> print(res.shape)
paddle.Size([5, 2])
>>> replaced_param = paddle.create_parameter(shape=[2, 3], dtype='float32')
>>> model.params['t3'] = replaced_param # replace t3 param
>>> res = model(x)
>>> print(res.shape)
paddle.Size([5, 3])
>>> model.params['t4'] = paddle.create_parameter(shape=[3, 4], dtype='float32') # append param
>>> print(len(model.params))
5
>>> res = model(x)
>>> print(res.shape)
paddle.Size([5, 4])
"""
@param_one_alias(["parameters", "values"])
def __init__(
self,
parameters: (
ParameterDict
| Mapping[str, Tensor]
| Sequence[tuple[str, Tensor]]
| None
) = None,
) -> None:
super().__init__()
if parameters is not None:
self.update(parameters)
def __getitem__(self, key: str) -> Tensor:
with param_guard(self._parameters):
return self._parameters[key]
def __setitem__(self, key: str, param: Tensor) -> None:
assert isinstance(param, Parameter)
setattr(self, key, param)
def __len__(self) -> int:
return len(self._parameters)
def __iter__(self) -> Iterator[str]:
return iter(self._parameters)
def update(
self,
parameters: (
ParameterDict | Mapping[str, Tensor] | Sequence[tuple[str, Tensor]]
),
) -> None:
"""Update a given parameter at the end of the dict.
Parameters:
parameters (Parameter): parameter to update
"""
assert isinstance(parameters, Iterable), (
"The type of parameters is not iterable of key/value pairs, the type of sublayers is "
+ type(parameters).__name__
)
if isinstance(parameters, ParameterDict):
for key, parameter in parameters._parameters.items():
self.add_parameter(key, parameter)
elif isinstance(parameters, (OrderedDict, Mapping)):
for key, parameter in parameters.items():
self.add_parameter(key, parameter)
else:
for i, kv in enumerate(parameters):
if len(kv) != 2:
raise ValueError(
f"The length of the {i}'s element in parameters is {len(kv)}, which must be 2."
)
self.add_parameter(kv[0], kv[1])
def pop(self, key: str) -> Tensor:
"""Remove key from the ParameterDict and return its parameter.
Parameters:
key (str): the key to be removed.
"""
v = self[key]
del self._parameters[key]
return v
def keys(self) -> Iterable[str]:
"""Return an iterable of the keys in the ParameterDict.
Parameters:
None.
"""
return self._parameters.keys()
def values(self) -> Iterable[Tensor]:
"""Return an iterable of the parameters in the ParameterDict.
Parameters:
None.
"""
with param_guard(self._parameters):
return list(self._parameters.values())
class ParameterList(Layer):
"""ParameterList Container.
This container acts like a Python list, but parameters it contains will be properly added.
Parameters:
parameters (iterable, optional): Iterable Parameters to be added.
Alias: ``values``.
Examples:
.. code-block:: pycon
>>> import paddle
>>> class MyLayer(paddle.nn.Layer):
... def __init__(self, num_stacked_param):
... super().__init__()
... # create ParameterList with iterable Parameters
... self.params = paddle.nn.ParameterList(
... [paddle.create_parameter(shape=[2, 2], dtype='float32') for _ in range(num_stacked_param)]
... )
...
... def forward(self, x):
... for i, p in enumerate(self.params):
... x = paddle.matmul(x, p)
... return x
>>> x = paddle.uniform(shape=[5, 2], dtype='float32')
>>> num_stacked_param = 4
>>> model = MyLayer(num_stacked_param)
>>> print(len(model.params))
4
>>> res = model(x)
>>> print(res.shape)
paddle.Size([5, 2])
>>> replaced_param = paddle.create_parameter(shape=[2, 3], dtype='float32')
>>> model.params[num_stacked_param - 1] = replaced_param
>>> res = model(x)
>>> print(res.shape)
paddle.Size([5, 3])
>>> model.params.append(paddle.create_parameter(shape=[3, 4], dtype='float32')) # append param
>>> print(len(model.params))
5
>>> res = model(x)
>>> print(res.shape)
paddle.Size([5, 4])
"""
@param_one_alias(["parameters", "values"])
def __init__(self, parameters: Iterable[Tensor] | None = None) -> None:
super().__init__()
if parameters is not None:
for idx, param in enumerate(parameters):
assert isinstance(param, Parameter)
self.add_parameter(str(idx), param)
def __getitem__(self, idx: int) -> Tensor:
with param_guard(self._parameters):
return self._parameters[str(idx)]
def __setitem__(self, idx: int, param: Tensor) -> None:
if not isinstance(param, (Parameter, Tensor)):
raise TypeError(
f"param should be 'Parameter' or 'Tensor', but received {type(param)}"
)
paddle.assign(param, getattr(self, str(idx)))
def __len__(self) -> int:
return len(self._parameters)
def __iter__(self) -> Iterator[Tensor]:
with param_guard(self._parameters):
return iter(self._parameters.values())
@param_one_alias(["parameter", "value"])
def append(self, parameter: Tensor) -> Self:
"""Appends a given parameter at the end of the list.
Parameters:
parameter (Parameter): parameter to append.
Alias: ``value``.
"""
idx = len(self._parameters)
self.add_parameter(str(idx), parameter)
return self
@param_one_alias(["parameters", "values"])
def extend(self, parameters: Iterable[Tensor]) -> Self:
"""Append values from a Python iterable to the end of the list.
Parameters:
parameters (iterable): iterable of values to append.
Alias: ``values``.
"""
for v in parameters:
self.append(v)
return self
def __iadd__(self, parameters: Iterable[Tensor]) -> Self:
return self.extend(parameters)
class LayerList(Layer):
"""
LayerList holds sublayers, and sublayers it contains are properly registered.
held sublayers can be indexed like a regular python list.
Parameters:
sublayers (iterable of Layer, optional): sublayers to hold
Examples:
.. code-block:: pycon
>>> import paddle
>>> class MyLayer(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
... self.linears = paddle.nn.LayerList(
... [paddle.nn.Linear(10, 10) for i in range(10)],
... )
...
... def forward(self, x):
... # LayerList can act as an iterable, or be indexed using ints
... for i, l in enumerate(self.linears):
... x = self.linears[i // 2](x) + l(x)
... return x
"""
@param_one_alias(["sublayers", "modules"])
def __init__(self, sublayers: Iterable[Layer] | None = None) -> None:
super().__init__()
if sublayers is not None:
for idx, layer in enumerate(sublayers):
self.add_sublayer(str(idx), layer)
def _get_abs_idx(self, idx: int) -> int:
if isinstance(idx, int):
if not (-len(self) <= idx < len(self)):
raise IndexError(
f'index {idx} is out of range, should be an integer in range [{-len(self)}, {len(self)})'
)
if idx < 0:
idx += len(self)
return idx
def _get_abs_string_index(self, idx):
return str(self._get_abs_idx(idx))
def __getitem__(self, idx: int) -> Layer:
if isinstance(idx, slice):
return self.__class__(list(self._sub_layers.values())[idx])
else:
idx = self._get_abs_idx(idx)
return self._sub_layers[str(idx)]
def __setitem__(self, idx: int, sublayer: Layer) -> None:
idx = self._get_abs_idx(idx)
return setattr(self, str(idx), sublayer)
def __delitem__(self, idx: int) -> None:
if isinstance(idx, slice):
for k in range(len(self._sub_layers))[idx]:
delattr(self, str(k))
else:
idx = self._get_abs_idx(idx)
delattr(self, str(idx))
str_indices = [str(i) for i in range(len(self._sub_layers))]
self._sub_layers = OrderedDict(
list(zip(str_indices, self._sub_layers.values()))
)
def __len__(self) -> int:
return len(self._sub_layers)
def __iter__(self) -> Iterator[Layer]:
return iter(self._sub_layers.values())
def __iadd__(self, modules: Iterable[Layer]) -> Self:
return self.extend(modules)
def __add__(self, other: Iterable[Layer]) -> LayerList:
combined = LayerList()
for i, module in enumerate(chain(self, other)):
combined.add_module(str(i), module)
return combined
def __dir__(self) -> list[str]:
keys = super().__dir__()
keys = [key for key in keys if not key.isdigit()]
return keys
@param_one_alias(["sublayer", "module"])
def append(self, sublayer: Layer) -> Self:
"""
Appends a sublayer to the end of the list.
Parameters:
sublayer (Layer): sublayer to append
Examples:
.. code-block:: pycon
>>> import paddle
>>> linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
>>> another = paddle.nn.Linear(10, 10)
>>> linears.append(another)
>>> print(len(linears))
11
"""
self.add_sublayer(str(len(self)), sublayer)
return self
@param_one_alias(["sublayer", "module"])
def insert(self, index: int, sublayer: Layer) -> None:
"""
Insert a sublayer before a given index in the list.
Parameters:
index (int): index to insert.
sublayer (Layer): sublayer to insert
Examples:
.. code-block:: pycon
>>> import paddle
>>> linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
>>> another = paddle.nn.Linear(10, 10)
>>> linears.insert(3, another)
>>> print(linears[3] is another)
True
>>> another = paddle.nn.Linear(10, 10)
>>> linears.insert(-1, another)
>>> print(linears[-2] is another)
True
"""
assert isinstance(index, int) and -len(
self._sub_layers
) <= index <= len(self._sub_layers), (
f"index should be an integer in range [{-len(self)}, {len(self)}]"
)
if index < 0:
index += len(self)
for i in range(len(self._sub_layers), index, -1):
self._sub_layers[str(i)] = self._sub_layers[str(i - 1)]
self._sub_layers[str(index)] = sublayer
@param_one_alias(["sublayers", "modules"])
def extend(self, sublayers: Iterable[Layer]) -> Self:
"""
Appends sublayers to the end of the list.
Parameters:
sublayers (iterable of Layer): iterable of sublayers to append
Returns:
None
Examples:
.. code-block:: pycon
>>> import paddle
>>> linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
>>> another_list = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(5)])
>>> linears.extend(another_list)
>>> print(len(linears))
15
>>> print(another_list[0] is linears[10])
True
"""
offset = len(self)
for i, sublayer in enumerate(sublayers):
idx = str(offset + i)
self.add_sublayer(idx, sublayer)
return self
def pop(self, key: int | slice) -> Layer:
v = self[key]
del self[key]
return v
class Sequential(Layer):
"""Sequential container.
Sub layers will be added to this container in the order of argument in the constructor.
The argument passed to the constructor can be iterable Layers or iterable name Layer pairs.
Parameters:
layers(Layer|list|tuple): Layer or list/tuple of iterable name Layer pair.
Returns:
None.
Examples:
.. code-block:: pycon
>>> import paddle
>>> data = paddle.uniform(shape=[30, 10], dtype='float32')
>>> # create Sequential with iterable Layers
>>> model1 = paddle.nn.Sequential(
... paddle.nn.Linear(10, 1), paddle.nn.Linear(1, 2)
>>> )
>>> model1[0] # access the first layer
>>> res1 = model1(data) # sequential execution
>>> # create Sequential with name Layer pairs
>>> model2 = paddle.nn.Sequential(
... ('l1', paddle.nn.Linear(10, 2)),
... ('l2', paddle.nn.Linear(2, 3))
>>> )
>>> model2['l1'] # access l1 layer
>>> model2.add_sublayer('l3', paddle.nn.Linear(3, 3)) # add sublayer
>>> res2 = model2(data) # sequential execution
>>> # append single layer at the end of sequential
>>> model2 = paddle.nn.Sequential(paddle.nn.Linear(10, 20))
>>> model2.append(paddle.nn.Linear(20, 30))
>>> res2 = model2(data) # [30, 30]
>>> # insert single layer at the given position
>>> model2 = paddle.nn.Sequential(paddle.nn.Linear(20, 30))
>>> model2.insert(0, paddle.nn.Linear(10, 20))
>>> res2 = model2(data) # [30, 30]
>>> # extend sequential with given sequence of layer(s) at the end
>>> model2 = paddle.nn.Sequential()
>>> model2.extend([paddle.nn.Linear(10, 20), paddle.nn.Linear(20, 30)])
>>> res2 = model2(data) # [30, 30]
"""
def __init__(
self,
*layers: Layer
| tuple[str, Layer]
| list[Any]
| OrderedDict[str, Layer],
) -> None:
super().__init__()
if len(layers) == 1 and isinstance(layers[0], OrderedDict):
for name, layer in layers[0].items():
self.add_sublayer(name, layer)
elif len(layers) > 0 and isinstance(layers[0], (list, tuple)):
for name, layer in layers:
self.add_sublayer(name, layer)
else:
for idx, layer in enumerate(layers):
self.add_sublayer(str(idx), layer)
def __getitem__(self, name: str | slice | int) -> Layer:
if isinstance(name, slice):
return self.__class__(*(list(self._sub_layers.values())[name]))
elif isinstance(name, str):
return self._sub_layers[name]
else:
if name >= len(self._sub_layers):
raise IndexError(f'index {name} is out of range')
elif name < 0 and name >= -len(self._sub_layers):
name += len(self._sub_layers)
elif name < -len(self._sub_layers):
raise IndexError(f'index {name} is out of range')
return list(self._sub_layers.values())[name]
def __setitem__(self, name: str, layer: Layer) -> None:
assert isinstance(layer, Layer)
setattr(self, str(name), layer)
def __delitem__(self, name: str) -> None:
name = str(name)
assert name in self._sub_layers
del self._sub_layers[name]
def __len__(self) -> int:
return len(self._sub_layers)
def forward(self, input: Any) -> Any:
for layer in self._sub_layers.values():
input = layer(input)
return input
def append(self, module: Layer) -> Sequential:
self.add_sublayer(str(len(self)), module)
return self
def insert(self, index: int, module: Layer) -> Sequential:
if not isinstance(module, Layer):
raise AssertionError(f'module should be of type: {Layer}')
n = len(self._sub_layers)
if not (-n <= index <= n):
raise IndexError(f'Index out of range: {index}')
if index < 0:
index += n
for i in range(n, index, -1):
self._sub_layers[str(i)] = self._sub_layers[str(i - 1)]
self._sub_layers[str(index)] = module
return self
def extend(self, sequential: Iterable[Layer]) -> Sequential:
for layer in sequential:
self.append(layer)
return self
def __iter__(self) -> Iterator[Layer]:
return iter(self._sub_layers.values())
python/paddle/nn/layer/conv.py
+1582
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/distance.py
+117
View File
@@ -0,0 +1,117 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from paddle.utils.decorator_utils import param_one_alias, param_two_alias
from .. import functional as F
from .layers import Layer
if TYPE_CHECKING:
import paddle
__all__ = []
class PairwiseDistance(Layer):
r"""
It computes the pairwise distance between two vectors. The
distance is calculated by p-order norm:
.. math::
\Vert x \Vert _p = \left( \sum_{i=1}^n \vert x_i \vert ^ p \right) ^ {1/p}.
Parameters:
p (float, optional): The order of norm. Default: :math:`2.0`.
epsilon (float, optional): Add small value to avoid division by zero.
Default: :math:`1e-6`.
keepdim (bool, optional): Whether to reserve the reduced dimension
in the output Tensor. The result tensor is one dimension less than
the result of ``|x-y|`` unless :attr:`keepdim` is True. Default: False.
name (str, optional): For details, please refer to :ref:`api_guide_Name`.
Generally, no setting is required. Default: None.
Shape:
- x: :math:`[N, D]` or :math:`[D]`, where :math:`N` is batch size, :math:`D`
is the dimension of the data. Available data type is float16, float32, float64.
- y: :math:`[N, D]` or :math:`[D]`, y have the same dtype as x.
- output: The same dtype as input tensor.
- If :attr:`keepdim` is True, the output shape is :math:`[N, 1]` or :math:`[1]`,
depending on whether the input has data shaped as :math:`[N, D]`.
- If :attr:`keepdim` is False, the output shape is :math:`[N]` or :math:`[]`,
depending on whether the input has data shaped as :math:`[N, D]`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.to_tensor([[1.0, 3.0], [3.0, 5.0]], dtype=paddle.float64)
>>> y = paddle.to_tensor([[5.0, 6.0], [7.0, 8.0]], dtype=paddle.float64)
>>> dist = paddle.nn.PairwiseDistance()
>>> distance = dist(x, y)
>>> print(distance)
Tensor(shape=[2], dtype=float64, place=Place(cpu), stop_gradient=True,
[4.99999860, 4.99999860])
"""
@param_one_alias(["epsilon", "eps"])
def __init__(
self,
p: float = 2.0,
epsilon: float = 1e-6,
keepdim: bool = False,
name: str | None = None,
):
super().__init__()
self.p = p
self.epsilon = epsilon
self.keepdim = keepdim
self.name = name
@param_two_alias(["x", "x1"], ["y", "x2"])
def forward(self, x: paddle.Tensor, y: paddle.Tensor) -> paddle.Tensor:
return F.pairwise_distance(
x, y, self.p, self.epsilon, self.keepdim, self.name
)
def extra_repr(self) -> str:
main_str = 'p={p}'
if self.epsilon != 1e-6:
main_str += ', epsilon={epsilon}'
if self.keepdim is not False:
main_str += ', keepdim={keepdim}'
if self.name is not None:
main_str += ', name={name}'
return main_str.format(**self.__dict__)
@property
def eps(self) -> float:
return self.epsilon
@eps.setter
def eps(self, value: float) -> None:
self.epsilon = value
@property
def norm(self) -> float:
return self.p
@norm.setter
def norm(self, value: float) -> None:
self.p = value
python/paddle/nn/layer/layers.py
+3969
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/loss.py
+2975
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/norm.py
+2084
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/pooling.py
Executable
+2145
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/rnn.py
+2311
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/transformer.py
+1753
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/layer/vision.py
+253
View File
@@ -0,0 +1,253 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from .. import functional
from .layers import Layer
if TYPE_CHECKING:
from paddle import Tensor
from paddle._typing import DataLayout2D
__all__ = []
class PixelShuffle(Layer):
"""
Rearranges elements in a tensor of shape :math:`[N, C, H, W]`
to a tensor of shape :math:`[N, C/upscale_factor^2, H*upscale_factor, W*upscale_factor]`,
or from shape :math:`[N, H, W, C]` to :math:`[N, H*upscale_factor, W*upscale_factor, C/upscale_factor^2]`.
This is useful for implementing efficient sub-pixel convolution
with a stride of 1/upscale_factor.
Please refer to the paper: `Real-Time Single Image and Video Super-Resolution
Using an Efficient Sub-Pixel Convolutional Neural Network <https://arxiv.org/abs/1609.05158v2>`_ .
by Shi et. al (2016) for more details.
Parameters:
upscale_factor(int): factor to increase spatial resolution.
data_format (str, optional): The data format of the input and output data. An optional string from: `'NCHW'``, ``'NHWC'``. When it is ``'NCHW'``, the data is stored in the order of: [batch_size, input_channels, input_height, input_width]. Default: ``'NCHW'``.
name (str|None, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
Shape:
- x: 4-D tensor with shape of :math:`(N, C, H, W)` or :math:`(N, H, W, C)`.
- out: 4-D tensor with shape of :math:`(N, C/upscale_factor^2, H*upscale_factor, W*upscale_factor)` or :math:`(N, H*upscale_factor, W*upscale_factor, C/upscale_factor^2)`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn as nn
>>> x = paddle.randn(shape=[2, 9, 4, 4])
>>> pixel_shuffle = nn.PixelShuffle(3)
>>> out = pixel_shuffle(x)
>>> print(out.shape)
paddle.Size([2, 1, 12, 12])
"""
def __init__(
self,
upscale_factor: int,
data_format: DataLayout2D = 'NCHW',
name: str | None = None,
) -> None:
super().__init__()
if not isinstance(upscale_factor, int):
raise TypeError("upscale factor must be int type")
if data_format not in ["NCHW", "NHWC"]:
raise ValueError(
"Data format should be 'NCHW' or 'NHWC'."
f"But receive data format: {data_format}"
)
self._upscale_factor = upscale_factor
self._data_format = data_format
self._name = name
def forward(self, x: Tensor) -> Tensor:
return functional.pixel_shuffle(
x, self._upscale_factor, self._data_format, self._name
)
def extra_repr(self) -> str:
main_str = f'upscale_factor={self._upscale_factor}'
if self._data_format != 'NCHW':
main_str += f', data_format={self._data_format}'
if self._name is not None:
main_str += f', name={self._name}'
return main_str
class PixelUnshuffle(Layer):
"""
Rearranges elements in a tensor of shape :math:`[N, C, H, W]`
to a tensor of shape :math:`[N, r^2C, H/r, W/r]`, or from shape
:math:`[N, H, W, C]` to :math:`[N, H/r, W/r, r^2C]`, where :math:`r` is the
downscale factor. This operation is the reversion of PixelShuffle operation.
Please refer to the paper: `Real-Time Single Image and Video Super-Resolution
Using an Efficient Sub-Pixel Convolutional Neural Network <https://arxiv.org/abs/1609.05158v2>`_ .
by Shi et. al (2016) for more details.
Parameters:
downscale_factor (int): Factor to decrease spatial resolution.
data_format (str, optional): The data format of the input and output data. An optional string of ``'NCHW'`` or ``'NHWC'``. When it is ``'NCHW'``, the data is stored in the order of [batch_size, input_channels, input_height, input_width]. Default: ``'NCHW'``.
name (str|None, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`.
Shape:
- **x**: 4-D tensor with shape of :math:`[N, C, H, W]` or :math:`[N, C, H, W]`.
- **out**: 4-D tensor with shape of :math:`[N, r^2C, H/r, W/r]` or :math:`[N, H/r, W/r, r^2C]`, where :math:`r` is :attr:`downscale_factor`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn as nn
>>> x = paddle.randn([2, 1, 12, 12])
>>> pixel_unshuffle = nn.PixelUnshuffle(3)
>>> out = pixel_unshuffle(x)
>>> print(out.shape)
paddle.Size([2, 9, 4, 4])
"""
def __init__(
self,
downscale_factor: int,
data_format: DataLayout2D = 'NCHW',
name: str | None = None,
) -> None:
super().__init__()
if not isinstance(downscale_factor, int):
raise TypeError("Downscale factor must be int type")
if downscale_factor <= 0:
raise ValueError("Downscale factor must be positive")
if data_format not in ["NCHW", "NHWC"]:
raise ValueError(
"Data format should be 'NCHW' or 'NHWC'."
f"But receive data format: {data_format}"
)
self._downscale_factor = downscale_factor
self._data_format = data_format
self._name = name
def forward(self, x: Tensor) -> Tensor:
return functional.pixel_unshuffle(
x, self._downscale_factor, self._data_format, self._name
)
def extra_repr(self) -> str:
main_str = f'downscale_factor={self._downscale_factor}'
if self._data_format != 'NCHW':
main_str += f', data_format={self._data_format}'
if self._name is not None:
main_str += f', name={self._name}'
return main_str
class ChannelShuffle(Layer):
"""
Can divide channels in a tensor of shape [N, C, H, W] or [N, H, W, C] into g groups,
getting a tensor with the shape of [N, g, C/g, H, W] or [N, H, W, g, C/g], and transposes them
as [N, C/g, g, H, W] or [N, H, W, g, C/g], then rearranges them to original tensor shape. This
operation can improve the interaction between channels, using features efficiently. Please
refer to the paper: `ShuffleNet: An Extremely Efficient
Convolutional Neural Network for Mobile Devices <https://arxiv.org/abs/1707.01083>`_ .
by Zhang et. al (2017) for more details.
Parameters:
groups (int): Number of groups to divide channels in.
data_format (str, optional): The data format of the input and output data. An optional string of NCHW or NHWC. The default is NCHW. When it is NCHW, the data is stored in the order of [batch_size, input_channels, input_height, input_width].
name (str|None, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`.
Shape:
- **x**: 4-D tensor with shape of [N, C, H, W] or [N, H, W, C].
- **out**: 4-D tensor with shape and dtype same as x.
Examples:
.. code-block:: pycon
>>> import paddle
>>> import paddle.nn as nn
>>> x = paddle.arange(0, 0.6, 0.1, 'float32')
>>> x = paddle.reshape(x, [1, 6, 1, 1])
>>> print(x)
Tensor(shape=[1, 6, 1, 1], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[0. ]],
[[0.10000000]],
[[0.20000000]],
[[0.30000001]],
[[0.40000001]],
[[0.50000000]]]])
>>> channel_shuffle = nn.ChannelShuffle(3)
>>> y = channel_shuffle(x)
>>> print(y)
Tensor(shape=[1, 6, 1, 1], dtype=float32, place=Place(cpu), stop_gradient=True,
[[[[0. ]],
[[0.20000000]],
[[0.40000001]],
[[0.10000000]],
[[0.30000001]],
[[0.50000000]]]])
"""
def __init__(
self,
groups: int,
data_format: DataLayout2D = 'NCHW',
name: str | None = None,
) -> None:
super().__init__()
if not isinstance(groups, int):
raise TypeError("groups must be int type")
if groups <= 0:
raise ValueError("groups must be positive")
if data_format not in ["NCHW", "NHWC"]:
raise ValueError(
"Data format should be 'NCHW' or 'NHWC'."
f"But receive data format: {data_format}"
)
self._groups = groups
self._data_format = data_format
self._name = name
def forward(self, x: Tensor) -> Tensor:
return functional.channel_shuffle(
x, self._groups, self._data_format, self._name
)
def extra_repr(self) -> str:
main_str = f'groups={self._groups}'
if self._data_format != 'NCHW':
main_str += f', data_format={self._data_format}'
if self._name is not None:
main_str += f', name={self._name}'
return main_str
python/paddle/nn/modules/__init__.py
+19
View File
@@ -0,0 +1,19 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import (
loss, # noqa: F401
utils, # noqa: F401
)
from .module import Module # noqa: F401
python/paddle/nn/modules/container.py
+18
View File
@@ -0,0 +1,18 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle.nn import LayerDict, LayerList
ModuleList = LayerList
ModuleDict = LayerDict
python/paddle/nn/modules/loss.py
+37
View File
@@ -0,0 +1,37 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle.nn.layer.loss import ( # noqa: F401
BCELoss,
BCEWithLogitsLoss,
CosineEmbeddingLoss,
CrossEntropyLoss,
CTCLoss,
GaussianNLLLoss,
HingeEmbeddingLoss,
KLDivLoss,
L1Loss,
MarginRankingLoss,
MSELoss,
MultiLabelMarginLoss,
MultiLabelSoftMarginLoss,
MultiMarginLoss,
NLLLoss,
PoissonNLLLoss,
SmoothL1Loss,
SoftMarginLoss,
TripletMarginLoss,
TripletMarginWithDistanceLoss,
_Loss,
)
python/paddle/nn/modules/module.py
+18
View File
@@ -0,0 +1,18 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle.nn import Layer
Module = Layer
python/paddle/nn/modules/utils.py
+34
View File
@@ -0,0 +1,34 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import collections
from itertools import repeat
def _ntuple(n, name="parse"):
def parse(x):
if isinstance(x, collections.abc.Iterable):
return tuple(x)
return tuple(repeat(x, n))
parse.__name__ = name
return parse
_single = _ntuple(1, "_single")
_pair = _ntuple(2, "_pair")
_triple = _ntuple(3, "_triple")
_quadruple = _ntuple(4, "_quadruple")
python/paddle/nn/parameter.py
+18
View File
@@ -0,0 +1,18 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle.base.framework import EagerParamBase
Parameter = EagerParamBase
python/paddle/nn/quant/__init__.py
+44
View File
@@ -0,0 +1,44 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import qat # noqa: F401
from .functional_layers import ( # noqa: F401
FloatFunctionalLayer,
add,
concat,
divide,
flatten,
matmul,
multiply,
reshape,
subtract,
transpose,
)
from .quant_layers import QuantStub # noqa: F401
from .quantized_linear import ( # noqa: F401
apply_per_channel_scale,
llm_int8_linear,
weight_dequantize,
weight_only_linear,
weight_quantize,
)
from .stub import Stub
__all__ = [
"Stub",
"weight_only_linear",
"llm_int8_linear",
"weight_quantize",
"weight_dequantize",
]
python/paddle/nn/quant/format.py
+494
View File
@@ -0,0 +1,494 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Define some layers used to export quantization model with ONNX style."""
from __future__ import annotations
import abc
import paddle
from paddle import _C_ops, _legacy_C_ops
from paddle.base import unique_name
from paddle.framework import in_dynamic_mode, in_pir_mode
from ..layer.layers import Layer
def fake_fp8_quant(input, scale, axis=-1, type='e4m3'):
# only support channelwise or tensorwise
if axis >= 0:
shape = [1] * len(input.shape)
shape[axis] = scale.numel()
scale = scale.reshape(shape)
inp = input.astype("float32")
if type == 'e4m3':
return paddle.cast(
(inp * 448 / scale).clip(-448, 448), "float8_e4m3fn"
).astype(input.dtype) # clip then cast
elif type == 'e5m2':
return paddle.cast(
(inp * 57344 / scale).clip(-57344, 57344), "float8_e5m2"
).astype(input.dtype) # clip then cast
else:
raise NotImplementedError("only support e4m3 or e5m2 now")
def fake_fp8_dequant(input, scale, axis=-1, type='e4m3'):
# only support channelwise or tensorwise
if axis >= 0:
shape = [1] * len(input.shape)
shape[axis] = scale.numel()
scale = scale.reshape(shape)
if type == 'e4m3':
return (input.astype("float32") / 448 * scale).astype(input.dtype)
elif type == 'e5m2':
return (input.astype("float32") / 57344 * scale).astype(input.dtype)
else:
raise NotImplementedError("only support e4m3 or e5m2 now")
class LinearQuanterDequanter(Layer):
def __init__(self, quanter, dequanter):
super().__init__()
self._quanter = quanter
self._dequanter = dequanter
def forward(self, input):
out = input
if self._quanter is not None:
out = self._quanter(out)
if self._dequanter is not None:
out = self._dequanter(out)
return out
@staticmethod
def from_quanter(quanter):
assert quanter is not None
return LinearQuanterDequanter(
LinearQuanter.from_quanter(quanter),
LinearDequanter.from_quanter(quanter),
)
class LinearQuanter(Layer):
def __init__(
self,
scales,
zero_point=None,
quant_axis=None,
bit_length=8,
group_size=128,
):
super().__init__()
scales = paddle.to_tensor(scales, dtype="float32")
scale_attr = paddle.framework.ParamAttr(
name=paddle.utils.unique_name.generate('quant_dequant.scale'),
initializer=paddle.nn.initializer.Constant(1.0),
trainable=False,
)
self._scales = self.create_parameter(
shape=scales.shape, attr=scale_attr, dtype="float32"
)
self._scales.set_value(scales)
self.in_accum = paddle.to_tensor(0.0, dtype="float32")
self.in_state = paddle.to_tensor(0.0, dtype="float32")
zero_point = zero_point if zero_point is not None else 0.0
zero_point = paddle.to_tensor(zero_point, dtype="float32")
zp_attr = paddle.framework.ParamAttr(
name=paddle.utils.unique_name.generate('quant_dequant.zero_point'),
initializer=paddle.nn.initializer.Constant(0.0),
trainable=False,
)
self._zero_point = self.create_parameter(
shape=zero_point.shape, attr=zp_attr, dtype="float32"
)
self._zero_point.set_value(zero_point)
self._quant_axis = -1 if quant_axis is None else quant_axis
self._bit_length = bit_length
self._group_size = group_size
if isinstance(self._bit_length, tuple):
if (
self._bit_length[0] == 4
and self._bit_length[1] == 3
and len(self._bit_length) == 2
):
self._qmin = -1 * 448
self._qmax = 448
elif (
self._bit_length[0] == 5
and self._bit_length[1] == 2
and len(self._bit_length) == 2
):
self._qmin = -1 * 57344
self._qmax = 57344
else:
raise NotImplementedError(
"Currently, only float8_e4m3 and float8_e5m2 formats are supported. Please set quant_bits to (4,3) or (5,2) for the corresponding format."
)
else:
self._qmax = (1 << (self._bit_length - 1)) - 1
self._qmin = -1 * self._qmax - 1
if isinstance(self._bit_length, tuple):
self._bit_length = self._bit_length[0] + self._bit_length[1] + 1
def forward(self, input):
if in_dynamic_mode():
if self._qmax == 448:
return fake_fp8_quant(
input, self._scales, self._quant_axis, type='e4m3'
)
elif self._qmax == 57344:
return fake_fp8_quant(
input, self._scales, self._quant_axis, type='e5m2'
)
elif len(self._scales.shape) > 1:
if self._zero_point.sum() != 0:
quant_weight = paddle.clip(
paddle.round(input.cast('float32') / self._scales)
+ self._zero_point,
self._qmin,
self._qmax,
)
else:
new_s = paddle.repeat_interleave(
self._scales, self._group_size, 0
)
new_zp = paddle.repeat_interleave(
self._zero_point, self._group_size, 0
)
quant_weight = paddle.clip(
paddle.round(input.cast('float32') / new_s * self._qmax)
+ new_zp,
self._qmin,
self._qmax,
)
return quant_weight.cast(input.dtype)
return _legacy_C_ops.quantize_linear(
input.cast('float32'),
self._scales,
self._zero_point,
"quant_axis",
self._quant_axis,
"bit_length",
self._bit_length,
"qmin",
self._qmin,
"qmax",
self._qmax,
).cast(input.dtype)
if in_pir_mode():
input.stop_gradient = True
quant_out = paddle.pir.core.create_persistable_value(
dtype='float32',
shape=input.shape,
name=unique_name.generate("quant_out"),
initializer=paddle.nn.initializer.Constant(0.0),
stop_gradient=True,
)
# TODO(xiaoluomi): need to add only observer pass for quantize_linear
quant_out, out_state, out_accum, out_scale = _C_ops.quantize_linear(
input,
self._scales,
self._zero_point,
self.in_accum,
self.in_state,
self._quant_axis,
self._bit_length,
self._qmin,
self._qmax,
0,
True,
False,
)
return quant_out
else:
out = self._helper.create_variable_for_type_inference(input.dtype)
self._helper.append_op(
type='quantize_linear',
inputs={
'X': input,
'Scale': self._scales,
'ZeroPoint': self._zero_point,
},
outputs={'Y': out},
attrs={
'quant_axis': self._quant_axis,
'bit_length': self._bit_length,
'qmin': self._qmin,
'qmax': self._qmax,
},
)
return out
@staticmethod
def from_quanter(quanter):
return LinearQuanter(
quanter.scales(),
zero_point=quanter.zero_points(),
quant_axis=quanter.quant_axis(),
bit_length=quanter.bit_length(),
)
class LinearDequanter(Layer):
def __init__(
self,
scales,
zero_point=None,
quant_axis=None,
bit_length=8,
group_size=128,
):
super().__init__()
scales = paddle.to_tensor(scales, dtype="float32")
scale_attr = paddle.framework.ParamAttr(
name=paddle.utils.unique_name.generate('quant_dequant.scale'),
initializer=paddle.nn.initializer.Constant(1.0),
trainable=False,
)
self._scales = self.create_parameter(
shape=scales.shape, attr=scale_attr, dtype="float32"
)
self._scales.set_value(scales)
self.in_accum = paddle.to_tensor(0.0, dtype="float32")
self.in_state = paddle.to_tensor(0.0, dtype="float32")
zero_point = zero_point if zero_point is not None else 0.0
zero_point = paddle.to_tensor(zero_point, dtype="float32")
zp_attr = paddle.framework.ParamAttr(
name=paddle.utils.unique_name.generate('quant_dequant.zero_point'),
initializer=paddle.nn.initializer.Constant(0.0),
trainable=False,
)
self._zero_point = self.create_parameter(
shape=zero_point.shape, attr=zp_attr, dtype="float32"
)
self._zero_point.set_value(zero_point)
self._quant_axis = -1 if quant_axis is None else quant_axis
self._bit_length = bit_length
self._group_size = group_size
if isinstance(self._bit_length, tuple):
if (
self._bit_length[0] == 4
and self._bit_length[1] == 3
and len(self._bit_length) == 2
):
self._qmin = -1 * 448
self._qmax = 448
elif (
self._bit_length[0] == 5
and self._bit_length[1] == 2
and len(self._bit_length) == 2
):
self._qmin = -1 * 57344
self._qmax = 57344
else:
raise NotImplementedError(
"Currently, only float8_e4m3 and float8_e5m2 formats are supported. Please set quant_bits to (4,3) or (5,2) for the corresponding format."
)
else:
self._qmax = (1 << (self._bit_length - 1)) - 1
self._qmin = -1 * self._qmax - 1
if isinstance(self._bit_length, tuple):
self._bit_length = self._bit_length[0] + self._bit_length[1] + 1
def forward(self, input):
if in_dynamic_mode():
if self._qmax == 448:
return fake_fp8_dequant(
input, self._scales, self._quant_axis, type='e4m3'
)
elif self._qmax == 57344:
return fake_fp8_dequant(
input, self._scales, self._quant_axis, type='e5m2'
)
elif len(self._scales.shape) > 1:
if self._zero_point.sum() != 0:
quant_dequant_weight = (
input.cast('float32') - self._zero_point
) * self._scales
else:
new_s = paddle.repeat_interleave(
self._scales, self._group_size, 0
)
new_zp = paddle.repeat_interleave(
self._zero_point, self._group_size, 0
)
quant_dequant_weight = (
(input.cast('float32') - new_zp) / self._qmax * new_s
)
return quant_dequant_weight.cast(input.dtype)
return _legacy_C_ops.dequantize_linear(
input.cast('float32'),
self._scales,
self._zero_point,
"quant_axis",
self._quant_axis,
"bit_length",
self._bit_length,
"qmin",
self._qmin,
"qmax",
self._qmax,
).cast(input.dtype)
if in_pir_mode():
input.stop_gradient = True
dequant_out = paddle.pir.core.create_persistable_value(
dtype='float32',
shape=input.shape,
name=unique_name.generate("quant_out"),
initializer=paddle.nn.initializer.Constant(0.0),
stop_gradient=True,
)
# TODO(xiaoluomi): need to add only observer pass for dequantize_linear
dequant_out, out_state, out_accum, out_scale = (
_C_ops.dequantize_linear(
input,
self._scales,
self._zero_point,
self.in_accum,
self.in_state,
self._quant_axis,
self._bit_length,
self._qmin,
self._qmax,
0,
True,
False,
)
)
return dequant_out
else:
out = self._helper.create_variable_for_type_inference(input.dtype)
self._helper.append_op(
type='dequantize_linear',
inputs={
'X': input,
'Scale': self._scales,
'ZeroPoint': self._zero_point,
},
outputs={'Y': out},
attrs={
'quant_axis': self._quant_axis,
'bit_length': self._bit_length,
'qmin': self._qmin,
'qmax': self._qmax,
},
)
return out
@staticmethod
def from_quanter(quanter):
return LinearDequanter(
quanter.scales(),
zero_point=quanter.zero_points(),
quant_axis=quanter.quant_axis(),
bit_length=quanter.bit_length(),
)
class ConvertibleQuantedLayer(Layer, metaclass=abc.ABCMeta):
r"""Abstract class to help convert quantized layer to inference model.
It defines some functions to convert quantizers and observers to quantize
or dequantize operators that maintain the quantization parameters used
during inference.
Examples:
.. code-block:: pycon
>>> # Given codes in ./customized_quanter.py
>>> class CustomizedQuantedLayer(ConvertibleQuantedLayer):
... def __init__(self):
... super().__init__()
... self.weight_a = paddle.create_parameter(shape=[1], dtype='float32')
... self.weight_b = paddle.create_parameter(shape=[1], dtype='float32')
... self.quanter_for_weight_a = None
... self.activation_weight = None
...
... def forward(self, input):
... qweight_a = self.quanter_for_weight_a(self.weight_a)
... weight_b = self.weight_b
... qinput = self.activation_weight(input)
... # compute with qweight_a, weight_b and qinput.
... return qweight * qinput + weight_b
...
... def weights_to_quanters(self):
... return [('weight_a', 'quanter_for_weight_a')]
...
... def activation_quanters(self):
... return ['activation_weight']
"""
def __init__(self):
super().__init__()
self.converted = False
@abc.abstractmethod
def weights_to_quanters(self) -> list[tuple[str, str]]:
r"""Get the name pairs of weights to be quantized and their corresponding
quantizers. In the convert function of this abstract class, it will call
the ‘weights_to_quanters’ function and do something as follows:
For each pair, the quantizer will be converted to a quantize operator and
a dequantize operator. Then, the weight will be quantized by the quantize
operator. Finally, the quantize operator will be removed and the weights
will be stored in integer data type.
Returns: A list of name pairs. Each pair contains two names. The first is name of weight
to be quantized and the second is name of corresponding quanter.
"""
pass
@abc.abstractmethod
def activation_quanters(self) -> list[str]:
r"""Get the names of quanters used to quantize activations.
All the quanters or observers returned by this function will be converted to quantize
and dequantize operators for deployment.
Returns: A list of quanter names.
"""
pass
def _convert_quanter_to_qdq(self, quanter_name) -> LinearQuanterDequanter:
r"""Convert quanter to an instance of LinearQuanterDequanter."""
if not hasattr(self, quanter_name):
return None
quanter = getattr(self, quanter_name)
if quanter is None:
return None
quanter = LinearQuanterDequanter.from_quanter(quanter)
setattr(self, quanter_name, quanter)
self._sub_layers[quanter_name] = quanter
return quanter
def _quant_weights(self, weight_name, quanter):
r"""Quantize the weight by given quanter."""
weight = getattr(self, weight_name)
qweight = quanter(weight)
weight.set_value(qweight)
def _convert(self, remain_weight=False):
r"""Convert current layer to onnx style for inference."""
assert not self.converted, "The model should be converted only once."
for weight_name, quanter_name in self.weights_to_quanters():
qdq = self._convert_quanter_to_qdq(quanter_name)
if qdq is not None and remain_weight is False:
self._quant_weights(weight_name, qdq._quanter)
qdq._quanter = None
qdq._sub_layers['_quanter'] = None
for quanter_name in self.activation_quanters():
self._convert_quanter_to_qdq(quanter_name)
self.converted = True
python/paddle/nn/quant/functional_layers.py
+95
View File
@@ -0,0 +1,95 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ...tensor import linalg, manipulation, math
from ..layer.layers import Layer
__all__ = []
class FloatFunctionalLayer(Layer):
def __init__(self):
super().__init__()
class add(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, y, name=None):
return math.add(x, y, name=name)
class subtract(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, y, name=None):
return math.subtract(x, y, name=name)
class multiply(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, y, name=None):
return math.multiply(x, y, name=name)
class divide(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, y, name=None):
return math.divide(x, y, name=name)
class reshape(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, shape, name=None):
return manipulation.reshape(x, shape, name=name)
class transpose(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, perm, name=None):
return manipulation.transpose(x, perm, name=name)
class concat(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, axis=0, name=None):
return manipulation.concat(x, axis, name=name)
class flatten(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, start_axis=0, stop_axis=-1, name=None):
return manipulation.flatten(x, start_axis, stop_axis, name=name)
class matmul(FloatFunctionalLayer):
def __init__(self):
super().__init__()
def forward(self, x, y, transpose_x=False, transpose_y=False, name=None):
return linalg.matmul(x, y, transpose_x, transpose_y, name=name)
python/paddle/nn/quant/lsq.py
+370
View File
@@ -0,0 +1,370 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
import paddle
from paddle.autograd import PyLayer
from paddle.framework import ParamAttr
from paddle.nn.initializer import Constant
from paddle.utils import unique_name
from ..layer.layers import Layer
def round(x):
sign = paddle.sign(x)
x = sign * paddle.floor(paddle.abs(x) + 0.5)
return x
class LsqFunc(PyLayer):
@staticmethod
def forward(ctx, weight, alpha, g, Qn, Qp, per_channel=False, quant_axis=0):
ctx.save_for_backward(weight, alpha)
ctx.other = g, Qn, Qp, per_channel, quant_axis
if per_channel:
sizes = weight.shape
weight = weight.reshape((weight.shape[quant_axis], -1))
weight = weight.transpose((1, 0))
alpha = paddle.broadcast_to(alpha, weight.shape)
quant_w = round(paddle.divide(weight, alpha)).clip(Qn, Qp)
quant_w = quant_w * alpha
quant_w = quant_w.transpose((1, 0))
quant_w = quant_w.reshape(sizes)
else:
quant_w = round(paddle.divide(weight, alpha)).clip(Qn, Qp)
quant_w = quant_w * alpha
return quant_w
@staticmethod
def backward(ctx, grad_weight):
weight, alpha = ctx.saved_tensor()
g, Qn, Qp, per_channel, quant_axis = ctx.other
if per_channel:
sizes = weight.shape
weight = weight.reshape((weight.shape[quant_axis], -1))
weight = weight.transpose((1, 0))
alpha = paddle.broadcast_to(alpha, weight.shape)
q_w = paddle.divide(weight, alpha)
q_w = q_w.transpose((1, 0))
q_w = q_w.reshape(sizes)
else:
q_w = paddle.divide(weight, alpha)
lower_flag = paddle.cast((q_w < Qn), 'float32')
upper_flag = paddle.cast((q_w > Qp), 'float32')
middle_flag = 1.0 - lower_flag - upper_flag
if per_channel:
grad_alpha = (
(
lower_flag * Qn
+ upper_flag * Qp
+ middle_flag * round(q_w)
- middle_flag * q_w
)
* grad_weight
* g
)
grad_alpha = grad_alpha.reshape(
(grad_alpha.shape[quant_axis], -1)
).sum(axis=1)
else:
grad_alpha = (
(
(
lower_flag * Qn
+ upper_flag * Qp
+ middle_flag * round(q_w)
- middle_flag * q_w
)
* grad_weight
* g
)
.sum()
.unsqueeze(axis=0)[0]
)
grad_weight = middle_flag * grad_weight
return grad_weight, grad_alpha
class LsqPlusActFunc(PyLayer):
@staticmethod
def forward(ctx, x, alpha, beta, g, Qn, Qp):
ctx.save_for_backward(x, alpha, beta)
ctx.other = g, Qn, Qp
quant_x = round(paddle.divide((x - beta), alpha)).clip(Qn, Qp)
return quant_x * alpha + beta
@staticmethod
def backward(ctx, grad_x):
x, alpha, beta = ctx.saved_tensor()
g, Qn, Qp = ctx.other
q_x = (x - beta) / alpha
lower_flag = paddle.cast((q_x < Qn), 'float32')
upper_flag = paddle.cast((q_x > Qp), 'float32')
middle_flag = 1.0 - lower_flag - upper_flag
grad_alpha = (
(
(
lower_flag * Qn
+ upper_flag * Qp
+ middle_flag * round(q_x)
- middle_flag * q_x
)
* grad_x
* g
)
.sum()
.unsqueeze(axis=0)[0]
)
grad_beta = (
((lower_flag + upper_flag) * grad_x * g).sum().unsqueeze(axis=0)[0]
)
grad_x = middle_flag * grad_x
return grad_x, grad_alpha, grad_beta
class FakeQuantActLSQPlus(Layer):
def __init__(
self,
quant_bits,
all_positive=False,
symmetric=False,
batch_init=20,
dtype='float32',
name=None,
reduce_type=None,
):
super().__init__()
'''
Args:
quant_bits(int): quantization bit number for weights.
all_positive(bool): whether unsigned or signed quantization, where True for unsigned quantization and False for signed quantization.
symmetric(bool): whether symmetric or asymmetric quantization.
batch_init(int): number of batches that collect Gaussian approximation for the weight distribution in each layer.
dtype(str): data type.
name(str): the name of the weight.
reduce_type(str): the reduce type which is needed when parallel training.
'''
self.bits = quant_bits
self.all_positive = all_positive
self.symmetric = symmetric
self.batch_init = batch_init
self.name = name
self.reduce_type = reduce_type
if self.all_positive:
# unsigned activation
self.Qn = 0
self.Qp = 2**self.bits - 1
else:
# signed activation
self.Qn = -(2 ** (self.bits - 1))
self.Qp = 2 ** (self.bits - 1) - 1
scale_prefix = f"{name}.scale" if name else 'quant_dequant.scale'
self._scale_name = unique_name.generate(scale_prefix)
s_attr = ParamAttr(
name=self._scale_name, initializer=Constant(1.0), trainable=True
)
self.s = self.create_parameter(shape=[], attr=s_attr, dtype='float32')
self.s.stop_gradient = False
if not self.symmetric:
beta_prefix = f"{name}.beta" if name else 'quant_dequant.beta'
self._beta_name = unique_name.generate(beta_prefix)
beta_attr = ParamAttr(
name=self._beta_name, initializer=Constant(0.0), trainable=True
)
self.beta = self.create_parameter(
shape=[], attr=beta_attr, dtype='float32'
)
self.beta.stop_gradient = False
self.init_state = 0
def forward(self, activation):
if self.reduce_type == "max":
paddle.distributed.all_reduce(
self.s, op=paddle.distributed.ReduceOp.MAX
)
if not self.symmetric and self.reduce_type == "max":
paddle.distributed.all_reduce(
self.beta, op=paddle.distributed.ReduceOp.MAX
)
if self.init_state == 0:
self.g = paddle.to_tensor(
1.0 / math.sqrt(activation.numel() * self.Qp)
)
min_a = paddle.min(activation.detach())
max_a = paddle.max(activation.detach())
self.s.set_value((max_a - min_a) / (self.Qp - self.Qn))
if not self.symmetric:
self.beta.set_value(min_a - self.s * self.Qn)
self.init_state += 1
elif self.init_state < self.batch_init:
min_a = paddle.min(activation.detach())
max_a = paddle.max(activation.detach())
self.s.set_value(
self.s * 0.9 + 0.1 * (max_a - min_a) / (self.Qp - self.Qn)
)
if not self.symmetric:
self.beta.set_value(
self.s * 0.9 + 0.1 * (min_a - self.s * self.Qn)
)
self.init_state += 1
else:
self.init_state += 1
activation.stop_gradient = False
if not self.symmetric:
q_a = LsqPlusActFunc.apply(
activation, self.s, self.beta, self.g, self.Qn, self.Qp
)
else:
q_a = LsqFunc.apply(
activation, self.s, self.g, self.Qn, self.Qp, per_channel=False
)
return q_a
class FakeQuantWeightLSQPlus(Layer):
def __init__(
self,
quant_bits,
all_positive=False,
per_channel=False,
batch_init=20,
channel_num=None,
quant_linear=False,
dtype='float32',
name=None,
reduce_type=None,
):
super().__init__()
'''
Args:
quant_bits(int): quantization bit number for weights.
all_positive(bool): whether unsigned or signed quantization, where True for unsigned quantization and False for signed quantization.
per_channel(bool): whether layer-wise or channel-wise quantization, where True for layer-wise quantization and False for channel-wise quantization.
batch_init(int): number of batches that collect Gaussian approximation for the weight distribution in each layer.
channel_num(int): the channel number of the weight which is needed when per_channel is True.
quant_linear(bool): whether the weight is from Linear.
dtype(str): data type.
name(str): the name of the weight.
reduce_type(str): the reduce type which is needed when parallel training.
'''
self.bits = quant_bits
self.all_positive = all_positive
self.per_channel = per_channel
self.quant_linear = quant_linear
self.batch_init = batch_init
self.name = name
self.quant_axis = 1 if quant_linear else 0
self.collect_axis = 0 if quant_linear else 1
self.reduce_type = reduce_type
if self.all_positive:
# unsigned weight
self.Qn = 0
self.Qp = 2**self.bits - 1
else:
# signed weight
self.Qn = -(2 ** (self.bits - 1))
self.Qp = 2 ** (self.bits - 1) - 1
self.init_state = 0
scale_prefix = f"{name}.scale" if name else 'quant_dequant.scale'
self._scale_name = unique_name.generate(scale_prefix)
s_attr = ParamAttr(
name=self._scale_name, initializer=Constant(1.0), trainable=True
)
self.s = self.create_parameter(
shape=[channel_num], attr=s_attr, dtype=dtype
)
self.s.stop_gradient = False
def forward(self, weight):
if self.reduce_type == "max":
paddle.distributed.all_reduce(
self.s, op=paddle.distributed.ReduceOp.MAX
)
if self.init_state == 0:
self.g = paddle.to_tensor(1.0 / math.sqrt(weight.numel() * self.Qp))
self.div = 2**self.bits - 1
if self.per_channel:
weight_tmp = weight.detach().reshape((weight.shape[0], -1))
mean = paddle.mean(weight_tmp, axis=self.collect_axis)
std = paddle.std(weight_tmp, axis=self.collect_axis)
s = paddle.max(
paddle.stack(
[paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)]
),
axis=0,
)
self.s.set_value(s / self.div)
else:
mean = paddle.mean(weight.detach())
std = paddle.std(weight.detach())
self.s.set_value(
max(
[paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)]
)
/ self.div
)
self.init_state += 1
elif self.init_state < self.batch_init:
self.div = 2**self.bits - 1
if self.per_channel:
weight_tmp = weight.detach().reshape((weight.shape[0], -1))
mean = paddle.mean(weight_tmp, axis=self.collect_axis)
std = paddle.std(weight_tmp, axis=self.collect_axis)
s = paddle.max(
paddle.stack(
[paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)]
),
axis=0,
)
self.s.set_value(s * 0.9 + 0.1 * s / self.div)
else:
mean = paddle.mean(weight.detach())
std = paddle.std(weight.detach())
self.s.set_value(
self.s * 0.9
+ 0.1
* max(
[paddle.abs(mean - 3 * std), paddle.abs(mean + 3 * std)]
)
/ self.div
)
self.init_state += 1
elif self.init_state == self.batch_init:
self.init_state += 1
weight.stop_gradient = False
w_q = LsqFunc.apply(
weight,
self.s,
self.g,
self.Qn,
self.Qp,
self.per_channel,
self.quant_axis,
)
return w_q
python/paddle/nn/quant/qat/__init__.py
+15
View File
@@ -0,0 +1,15 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .conv import QuantedConv2D # noqa: F401
from .linear import QuantedLinear # noqa: F401
python/paddle/nn/quant/qat/conv.py
+88
View File
@@ -0,0 +1,88 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Layers used for QAT.
"""
from paddle.nn import functional as F
from ...layer.layers import Layer
from ..format import ConvertibleQuantedLayer
class QuantedConv2D(ConvertibleQuantedLayer):
"""
The computational logic of QuantizedConv2D is the same as Conv2D.
The only difference is that its inputs are all fake quantized.
"""
def __init__(self, layer: Layer, q_config):
super().__init__()
# For Conv2D
self._groups = layer._groups
self._stride = layer._stride
self._padding = layer._padding
self._padding_mode = layer._padding_mode
if self._padding_mode != 'zeros':
self._reversed_padding_repeated_twice = (
layer._reversed_padding_repeated_twice
)
self._dilation = layer._dilation
self._data_format = layer._data_format
self.weight = layer.weight
self.bias = layer.bias
self.weight_quanter = None
self.activation_quanter = None
if q_config.weight is not None:
self.weight_quanter = q_config.weight._instance(layer)
if q_config.activation is not None:
self.activation_quanter = q_config.activation._instance(layer)
def forward(self, input):
quant_input = input
quant_weight = self.weight
if self.activation_quanter is not None:
quant_input = self.activation_quanter(input)
if self.weight_quanter is not None:
quant_weight = self.weight_quanter(self.weight)
return self._conv_forward(quant_input, quant_weight)
def _conv_forward(self, inputs, weights):
if self._padding_mode != 'zeros':
inputs = F.pad(
inputs,
self._reversed_padding_repeated_twice,
mode=self._padding_mode,
data_format=self._data_format,
)
self._padding = 0
return F.conv2d(
inputs,
weights,
bias=self.bias,
padding=self._padding,
stride=self._stride,
dilation=self._dilation,
groups=self._groups,
data_format=self._data_format,
)
def weights_to_quanters(self):
return [('weight', 'weight_quanter')]
def activation_quanters(self):
return ['activation_quanter']
python/paddle/nn/quant/qat/linear.py
+60
View File
@@ -0,0 +1,60 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle.nn import functional as F
from ...layer.layers import Layer
from ..format import ConvertibleQuantedLayer
class QuantedLinear(ConvertibleQuantedLayer):
"""
The computational logic of QuantizedLinear is the same as Linear.
The only difference is that its inputs are all fake quantized.
"""
def __init__(self, layer: Layer, q_config):
super().__init__()
# For Linear
self.weight = layer.weight
self.bias = layer.bias
self.name = layer.name
# For FakeQuant
self.weight_quanter = None
self.activation_quanter = None
if q_config.weight is not None:
self.weight_quanter = q_config.weight._instance(layer)
if q_config.activation is not None:
self.activation_quanter = q_config.activation._instance(layer)
def forward(self, input):
quant_input = input
quant_weight = self.weight
if self.activation_quanter is not None:
quant_input = self.activation_quanter(input)
if self.weight_quanter is not None:
quant_weight = self.weight_quanter(self.weight)
return self._linear_forward(quant_input, quant_weight)
def _linear_forward(self, input, weight):
out = F.linear(x=input, weight=weight, bias=self.bias, name=self.name)
return out
def weights_to_quanters(self):
return [('weight', 'weight_quanter')]
def activation_quanters(self):
return ['activation_quanter']
python/paddle/nn/quant/quant_layers.py
+1247
View File
File diff suppressed because it is too large Load Diff
python/paddle/nn/quant/quantized_linear.py
+395
View File
@@ -0,0 +1,395 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING, Literal
import paddle
from paddle import _C_ops
from paddle.base.data_feeder import check_dtype
from paddle.device import (
is_compiled_with_cuda,
)
from paddle.device.cuda import get_device_capability
from paddle.framework import (
LayerHelper,
in_dynamic_or_pir_mode,
)
if TYPE_CHECKING:
from typing import TypeAlias
from paddle import Tensor
from paddle._typing import DTypeLike
_Algo: TypeAlias = Literal[
'weight_only_int8', 'weight_only_int4', 'llm.int8'
]
_GroupSize: TypeAlias = Literal[-1, 64, 128]
def _get_arch_info():
# Get SMVersion from device.
if is_compiled_with_cuda():
cuda_version = paddle.version.cuda()
if (
cuda_version is not None and cuda_version != 'False'
) or paddle.is_compiled_with_rocm():
major, minor = get_device_capability()
arch = int(major * 10 + minor)
return arch
else:
raise ValueError(
"Paddle is not compiled with CUDA, we cannot get SMVersion from device, please try to compile Paddle with CUDA"
)
else:
# Default arch value for type checking.
return 0
def weight_quantize(
x: Tensor,
algo: _Algo = "weight_only_int8",
arch: int | None = None,
group_size: _GroupSize = -1,
) -> tuple[Tensor, Tensor]:
"""
Quantization function for weight_only and llm.int8's weight.
Args:
x (Tensor): The input Tensor to be quantized, the data type is float16 or bfloat16.
algo (str): The algo that is x will be apply, must be one of 'weight_only_int8',
'weight_only_int4', 'llm.int8', 'w4a8' and 'w4afp8, default: 'weight_only_int8'.
arch (int): The compute arch for target device. For example, A100 is 80, v100 is 70, if you do not assign arch, we will get arch from your device, default: None.
group_size (int): The group size for weight quantization. -1 stands for default per-channel mode. Currently only support 64 or 128.
Returns:
out (Tensor): The Tensor which is the quantitative results, the data type is int8, the shape is transposition of x.
scale (Tensor): The scale Tensor which is the scale of pre-channel, the data type is float32.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP('No testing required')
>>> import paddle
>>> from paddle.nn.quant import weight_quantize
>>> paddle.seed(2023)
>>> x = paddle.rand(shape=[64, 32], dtype=paddle.float16)
>>> out, scale = weight_quantize(x, algo='weight_only_int8')
>>> print(out.shape)
paddle.Size([32, 64])
>>> print(scale.shape)
paddle.Size([32])
"""
if arch is None:
arch = _get_arch_info()
if is_compiled_with_cuda():
assert (
arch == 70
or arch == 75
or arch == 80
or arch == 86
or arch == 89
or arch == 90
or arch == 92
or arch == 100
), (
f"Currently weight_quantize only support SM70/75/80/86/89/90/92/100. but got {arch} "
)
assert group_size == -1 or group_size == 64 or group_size == 128, (
f"Currently group_size only support -1/64/128. but got {group_size} "
)
if in_dynamic_or_pir_mode():
return _C_ops.weight_quantize(x, algo, arch, group_size)
else:
type = "weight_quantize"
helper = LayerHelper(type, **locals())
out = helper.create_variable_for_type_inference('int8')
scale = helper.create_variable_for_type_inference('float')
helper.append_op(
type=type,
inputs={"x": x},
outputs={'out': out, "scale": scale},
attrs={"algo": algo, "arch": arch, "group_size": group_size},
)
return (out, scale)
def weight_dequantize(
x: Tensor,
scale: Tensor,
algo: _Algo = "weight_only_int8",
out_dtype: DTypeLike = "float16",
group_size: _GroupSize = -1,
) -> Tensor:
"""
Dequantization function for weight_only and llm.int8's weight.
Args:
x (Tensor): The input Tensor to be dequantized, the data type is int8.
scale (Tensor): The scale Tensor which is the output of weight_quantize, the data type is float32.
algo (str): The algo that is x will be apply, must be one of 'weight_only_int8',
'weight_only_int4' and 'llm.int8', default: 'weight_only_int8'.
out_dtype (str|np.dtype): [Deprecated][Not used] The output Tensor's data type, must be one of 'float16' and 'bfloat16', default: 'float16'.
Returns:
out (Tensor): The Tensor which is the dequantitative results, the data type is float16 or bfloat16, the shape is transposition of x.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP('No testing required')
>>> import paddle
>>> from paddle.nn.quant import weight_quantize, weight_dequantize
>>> paddle.seed(2023)
>>> x = paddle.rand(shape=[64, 32], dtype=paddle.float16)
>>> out, scale = weight_quantize(x, algo='weight_only_int8')
>>> x_dequant = weight_dequantize(out, scale)
"""
assert group_size == -1 or group_size == 64 or group_size == 128, (
f"Currently group_size only support -1/64/128. but got {group_size} "
)
if in_dynamic_or_pir_mode():
return _C_ops.weight_dequantize(x, scale, algo, group_size)
else:
type = "weight_dequantize"
helper = LayerHelper(type, **locals())
out_dtype = scale.dtype
out = helper.create_variable_for_type_inference(out_dtype)
helper.append_op(
type=type,
inputs={"x": x, "scale": scale},
outputs={'out': out},
attrs={
"algo": algo,
"group_size": group_size,
},
)
return out
def weight_only_linear(
x: Tensor,
weight: Tensor,
bias: Tensor | None = None,
weight_scale: Tensor | None = None,
weight_dtype: DTypeLike = "int8",
arch: int | None = None,
group_size: _GroupSize = -1,
) -> Tensor:
"""
Applies matrix multiplication of two tensors and then bias addition if provided.
This method requires CUDA version >= 11.2.
Args:
x (Tensor): The first input Tensor to be multiplied, the data type is float16 or bfloat16.
weight (Tensor): The second input Tensor to be multiplied. Its rank must be 2.
bias (Tensor|None): The input bias Tensor. If it is None, no bias addition would
be performed. Otherwise, The bias is added to the matrix multiplication result.
weight_scale (Tensor|None): The input scale Tensor Provided to weight for dequantization. Its rank must be 1.
weight_dtype(str): The dtype of weight Tensor, must be one of 'int8', 'int4', Defaulted to 'int8'.
arch (int): The compute arch for target device. For example, A100 is 80, v100 is 70, if you do not assign arch, we will get arch from your device, default: None.
group_size (int): The group size for weight quantization. -1 stands for default per-channel mode. Currently only support 64 or 128.
Returns:
Tensor: the output Tensor, the data type is the same as that of x.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP('No testing required')
>>> import paddle
>>> from paddle.nn.quant import weight_only_linear
>>> x = paddle.cast(paddle.randn([1, 2, 64]), dtype='float16')
>>> weight = paddle.cast(paddle.randint(0, 127, [32, 64]), dtype='int8')
>>> scale = paddle.randn([32], dtype='float32')
>>> bias = paddle.cast(paddle.randn([32]), dtype='float16')
>>> if paddle.device.cuda.get_device_capability()[0] >= 8:
... out = weight_only_linear(
... x,
... weight,
... bias=bias,
... weight_scale=scale,
... weight_dtype='int8',
... )
... print(out.shape)
paddle.Size([1, 2, 32])
"""
if arch is None:
arch = _get_arch_info()
if is_compiled_with_cuda():
assert (
arch == 70
or arch == 75
or arch == 80
or arch == 86
or arch == 89
or arch == 90
or arch == 92
or arch == 100
), (
f"Currently weight_quantize only support SM70/75/80/86/89/90/92/100. but got {arch} "
)
assert group_size == -1 or group_size == 64 or group_size == 128, (
f"Currently weight_quantize only support group size of -1, 64 or 128. but got {group_size} "
)
if in_dynamic_or_pir_mode():
out = _C_ops.weight_only_linear(
x, weight, bias, weight_scale, weight_dtype, arch, group_size
)
return out
else:
check_dtype(
weight_dtype, 'weight_dtype', ['int8', 'int4'], 'weight_only_linear'
)
type = "weight_only_linear"
helper = LayerHelper(type, **locals())
dtype = x.dtype
inputs = {
'x': [x],
'weight': [weight],
'weight_scale': [weight_scale],
}
if bias is not None:
inputs["bias"] = [bias]
attrs = {
'weight_dtype': weight_dtype,
'arch': arch,
'group_size': group_size,
}
out = helper.create_variable_for_type_inference(dtype)
helper.append_op(
type=type,
inputs=inputs,
outputs={'out': out},
attrs=attrs,
)
return out
def llm_int8_linear(
x: Tensor,
weight: Tensor,
bias: Tensor | None = None,
weight_scale: Tensor | None = None,
threshold: float = 6.0,
) -> Tensor:
"""
Applies matrix multiplication of two tensors and then bias addition if provided.
This method requires CUDA version >= 11.2.
Args:
x (Tensor): the first input Tensor to be multiplied, the data type is float16 or bfloat16.
weight (Tensor): the second input Tensor to be multiplied. Its rank must be 2.
bias (Tensor|None): the input bias Tensor. If it is None, no bias addition would
be performed. Otherwise, the bias is added to the matrix multiplication result.
weight_scale (Tensor|None): the input scale Tensor Provided to weight for dequantization. Its rank must be 1.
threshold(float): The min value of outlier in activation, outlier's channel will be apply multiply with x.dtype.
Returns:
Tensor: the output Tensor, the data type is the same as that of x.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP('No testing required')
>>> import paddle
>>> from paddle.nn.quant import llm_int8_linear
>>> x = paddle.cast(paddle.randn([1, 2, 64]), dtype='float16')
>>> weight = paddle.cast(paddle.randint(0, 127, [32, 64]), dtype='int8')
>>> scale = paddle.randn([32], dtype='float32')
>>> bias = paddle.cast(paddle.randn([32]), dtype='float16')
>>> if paddle.device.cuda.get_device_capability()[0] >= 8:
... out = llm_int8_linear(x, weight, bias=bias, weight_scale=scale, threshold=6.0)
... print(out.shape)
paddle.Size([1, 2, 32])
"""
if in_dynamic_or_pir_mode():
out = _C_ops.llm_int8_linear(x, weight, bias, weight_scale, threshold)
return out
else:
type = "llm_int8_linear"
helper = LayerHelper(type, **locals())
dtype = x.dtype
inputs = {
'x': [x],
'weight': [weight],
'weight_scale': [weight_scale],
}
if bias:
inputs["bias"] = [bias]
attrs = {'threshold': threshold}
out = helper.create_variable_for_type_inference(dtype)
helper.append_op(
type=type,
inputs=inputs,
outputs={'out': out},
attrs=attrs,
)
return out
def apply_per_channel_scale(x: Tensor, scales: Tensor) -> Tensor:
"""
Apply pre-quant per channel scale on activations
Args:
x (Tensor): Input tensor representing the activations, the data type can be float16 or bfloat16.
scales(Tensor): Per-channel scale factors for pre-quantization. Data type should be compatible with x.
Returns:
out (Tensor): The Tensor which is the pre-quant results, the data type is compatible with x.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP('No testing required')
>>> import paddle
>>> from paddle.nn.quant import apply_per_channel_scale
>>> paddle.seed(2023)
>>> x = paddle.rand(shape=[64, 32], dtype=paddle.float16)
>>> scales = paddle.rand(shape=[32], dtype=paddle.float16)
>>> out = apply_per_channel_scale(x, scales)
"""
if in_dynamic_or_pir_mode():
return _C_ops.apply_per_channel_scale(x, scales)
else:
type = "apply_per_channel_scale"
helper = LayerHelper(type, **locals())
out = helper.create_variable_for_type_inference(x.dtype)
helper.append_op(
type=type,
inputs={"x": [x], "scales": [scales]},
outputs={"out": out},
)
return out
python/paddle/nn/quant/stub.py
+107
View File
@@ -0,0 +1,107 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Define stub used in quantization."""
from __future__ import annotations
from typing import TYPE_CHECKING
from ..layer.layers import Layer
if TYPE_CHECKING:
from paddle import Tensor
from paddle.quantization import QuantConfig
from paddle.quantization.factory import QuanterFactory
class Stub(Layer):
r"""
The stub is used as placeholders that will be replaced by observers before PTQ or QAT.
It is hard to assign a quantization configuration to a functional API called in
the forward of a layer. Instead, we can create a stub and add it to the sublayers of the layer.
And call the stub before the functional API in the forward. The observer held by the
stub will observe or quantize the inputs of the functional API.
Args:
observer(QuanterFactory): The configured information of the observer to be inserted.
It will use a global configuration to create the observers if the 'observer' is none.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.nn.quant import Stub
>>> from paddle.quantization.quanters import FakeQuanterWithAbsMaxObserver
>>> from paddle.nn import Conv2D
>>> from paddle.quantization import QAT, QuantConfig
>>> quanter = FakeQuanterWithAbsMaxObserver(moving_rate=0.9)
>>> class Model(paddle.nn.Layer):
... def __init__(self, num_classes=10):
... super().__init__()
... self.conv = Conv2D(3, 6, 3, stride=1, padding=1)
... self.quant = Stub(quanter)
...
... def forward(self, inputs):
... out = self.conv(inputs)
... out = self.quant(out)
... return paddle.nn.functional.relu(out)
>>> model = Model()
>>> q_config = QuantConfig(activation=quanter, weight=quanter)
>>> qat = QAT(q_config)
>>> quant_model = qat.quantize(model)
>>> print(quant_model)
Model(
(conv): QuantedConv2D(
(weight_quanter): FakeQuanterWithAbsMaxObserverLayer()
(activation_quanter): FakeQuanterWithAbsMaxObserverLayer()
)
(quant): QuanterStub(
(_observer): FakeQuanterWithAbsMaxObserverLayer()
)
)
"""
def __init__(self, observer: QuanterFactory | None = None) -> None:
super().__init__()
self._observer = observer
def forward(self, input: Tensor) -> Tensor:
return input
class QuanterStub(Layer):
r"""
It is an identity layer with an observer observing the input.
Before QAT or PTQ, the stub in the model will be replaced with an instance of QuanterStub.
The user should not use this class directly.
Args:
layer(paddle.nn.Layer): The stub layer with an observer configure factory. If the observer
of the stub layer is none, it will use 'q_config' to create an observer instance.
q_config(QuantConfig): The quantization configuration for the current stub layer.
"""
def __init__(self, layer: Stub, q_config: QuantConfig) -> None:
super().__init__()
self._observer = None
if layer._observer is not None:
self._observer = layer._observer._instance(layer)
elif q_config.activation is not None:
self._observer = q_config.activation._instance(layer)
def forward(self, input: Tensor) -> Tensor:
return self._observer(input) if self._observer is not None else input
python/paddle/nn/utils/__init__.py
+34
View File
@@ -0,0 +1,34 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import rnn # noqa: F401
from .clip_grad_norm_ import clip_grad_norm_
from .clip_grad_value_ import clip_grad_value_
from .spectral_norm_hook import spectral_norm
from .transform_parameters import (
_stride_column, # noqa: F401
parameters_to_vector,
vector_to_parameters,
)
from .weight_norm_hook import remove_weight_norm, weight_norm
__all__ = [
'weight_norm',
'remove_weight_norm',
'spectral_norm',
'parameters_to_vector',
'vector_to_parameters',
'clip_grad_norm_',
'clip_grad_value_',
]
python/paddle/nn/utils/clip_grad_norm_.py
+119
View File
@@ -0,0 +1,119 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
import paddle
if TYPE_CHECKING:
from collections.abc import Iterable
from paddle import Tensor
__all__ = []
@paddle.autograd.no_grad()
def clip_grad_norm_(
parameters: Iterable[Tensor] | Tensor,
max_norm: float,
norm_type: float = 2.0,
error_if_nonfinite: bool = False,
) -> Tensor:
r"""Clips gradient norm of the iterable parameters.
Norms are calculated together on all gradients, just as they are
connected into one vector. The gradient will be modified in place.
This API can only run in dynamic graph mode, not static graph mode.
Args:
parameters (Iterable[paddle.Tensor] or paddle.Tensor): Tensors or a single Tensor
that will be normalized gradients
max_norm (float or int): max norm of the gradients
norm_type (float or int): type of the used p-norm. Can be `inf` for
infinity norm.
error_if_nonfinite (bool): if True, throw an error if the total
norm of the gradients from :attr:`parameters` is `nan`,
`inf`, or `-inf`.
Returns:
Total norm of the parameter gradients (treated as a single vector).
Example:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32')
>>> max_norm = float(5.0)
>>> linear = paddle.nn.Linear(in_features=10, out_features=10)
>>> out = linear(x)
>>> loss = paddle.mean(out)
>>> loss.backward()
>>> paddle.nn.utils.clip_grad_norm_(linear.parameters(), max_norm)
>>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters())
>>> sdg.step()
"""
if not paddle.in_dynamic_mode():
raise RuntimeError('this API can only run in dynamic mode.')
if isinstance(parameters, paddle.Tensor):
parameters = [parameters]
support_norm_type = [float("inf"), 0, 1, 2]
if norm_type not in support_norm_type:
raise ValueError(f'norm_type only support {support_norm_type}')
grads = [p.grad_ for p in parameters if p.grad_ is not None]
max_norm = float(max_norm)
norm_type = float(norm_type)
if len(grads) == 0:
return paddle.to_tensor(0.0)
if norm_type == float("inf"):
norms = [g.detach().abs().max() for g in grads]
total_norm = (
norms[0] if len(norms) == 1 else paddle.max(paddle.stack(norms))
)
else:
total_norm = paddle.linalg.norm(
paddle.stack(
[paddle.linalg.norm(g.detach(), norm_type) for g in grads]
),
norm_type,
)
if error_if_nonfinite and paddle.logical_or(
total_norm.isnan(), total_norm.isinf()
):
raise RuntimeError(
f'The total norm of {norm_type} order of the gradients from '
'`parameters` is non-finite, so it cannot be clipped. In any case, '
'disable this error and scale the gradient by non-finite norm, '
'set `error_if_nonfinite=False`'
)
clip_coef = max_norm / (total_norm + 1e-6)
# Note: when the coef is clamped to 1, it is redundant to multiply the clamped coef, but this
# avoids the `if clip_coef < 1:` condition.
clip_coef_clamped = clip_coef.clip_(max=1.0)
for _, p in enumerate(parameters):
if p.grad_ is not None:
p.grad_.multiply_(y=clip_coef_clamped)
return total_norm
python/paddle/nn/utils/clip_grad_value_.py
+70
View File
@@ -0,0 +1,70 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
import paddle
if TYPE_CHECKING:
from collections.abc import Iterable
from paddle import Tensor
__all__ = []
@paddle.autograd.no_grad()
def clip_grad_value_(
parameters: Iterable[Tensor] | Tensor,
clip_value: float,
) -> None:
r"""
Clips gradient of an iterable of parameters at specified value.
The gradient will be modified in place.
This API can only run in dynamic graph mode, not static graph mode.
Args:
parameters (Iterable[paddle.Tensor]|paddle.Tensor): Tensors or a single Tensor
that will be normalized gradients
clip_value (float|int): maximum allowed value of the gradients.
The gradients are clipped in the range
:math:`\left[\text{-clip\_value}, \text{clip\_value}\right]`
Example:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.uniform([10, 10], min=-10.0, max=10.0, dtype='float32')
>>> clip_value = float(5.0)
>>> linear = paddle.nn.Linear(in_features=10, out_features=10)
>>> out = linear(x)
>>> loss = paddle.mean(out)
>>> loss.backward()
>>> paddle.nn.utils.clip_grad_value_(linear.parameters(), clip_value)
>>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters())
>>> sdg.step()
"""
if not paddle.in_dynamic_mode():
raise RuntimeError('this API can only run in dynamic mode.')
if isinstance(parameters, paddle.Tensor):
parameters = [parameters]
clip_value = float(clip_value)
for _, p in enumerate(parameters):
if p.grad is not None:
p.grad.clip_(min=-clip_value, max=clip_value)
python/paddle/nn/utils/dygraph_utils.py
+33
View File
@@ -0,0 +1,33 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle import _legacy_C_ops
from paddle.framework import dygraph_only
@dygraph_only
def _append_bias_in_dygraph(input, bias=None, axis=1):
"""Append bias operation in dygraph mode.
Args:
input: the input variable.
bias: the bias to be appended
axis: the axis to perform operation
Return the Variable after bias operation
"""
if bias is None:
return input
return _legacy_C_ops.elementwise_add(input, bias, 'axis', axis)
python/paddle/nn/utils/rnn.py
+549
View File
@@ -0,0 +1,549 @@
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from collections.abc import Iterable
from typing import TYPE_CHECKING
import paddle
if TYPE_CHECKING:
from paddle import Tensor
__all__ = [
"PackedSequence",
"invert_permutation",
"pack_padded_sequence",
"pad_packed_sequence",
"pad_sequence",
"unpad_sequence",
"pack_sequence",
"unpack_sequence",
]
def invert_permutation(permutation: Tensor | None) -> Tensor | None:
"""Returns the inverse of ``permutation``.
This is useful for converting between sorted and unsorted indices in
a :class:`~nn.utils.rnn.PackedSequence`.
Args:
permutation (Tensor|None): a 1-D tensor of indices to invert.
Returns:
Tensor|None: the inverse permutation tensor, or None if input is None.
Examples:
>>> import paddle
"""
if permutation is None:
return None
# Use paddle.scatter instead of scatter_ for better static mode support
output = paddle.scatter(
paddle.zeros_like(permutation),
permutation,
paddle.arange(
0,
permutation.numel(),
dtype=permutation.dtype,
device=permutation.place,
),
overwrite=True,
)
return output
class PackedSequence:
"""Holds the data and batch sizes of a packed sequence.
PackedSequence is used to represent a packed sequence, which is typically
produced by ``pack_padded_sequence`` and consumed by ``pad_packed_sequence``.
Args:
data (Tensor): The packed data tensor.
batch_sizes (Tensor): A tensor containing the batch size at each step.
sorted_indices (Tensor|None, optional): The indices used to sort the sequences.
unsorted_indices (Tensor|None, optional): The indices to restore the original order.
Examples:
.. code-block:: pycon
>>> import paddle
"""
def __init__(
self,
data: Tensor,
batch_sizes: Tensor,
sorted_indices: Tensor | None = None,
unsorted_indices: Tensor | None = None,
):
self.data = data
self.batch_sizes = batch_sizes
self.sorted_indices = sorted_indices
self.unsorted_indices = unsorted_indices
@property
def is_pinned(self) -> bool:
return (
self.data.place.is_cuda_pinned_place()
or self.data.place.is_xpu_pinned_place()
)
def to(self, *args, **kwargs) -> PackedSequence:
data = self.data.to(*args, **kwargs)
if data is self.data:
return self
# Only convert indices to same device as data, not dtype
target_device = data.place
sorted_indices = (
self.sorted_indices.to(target_device)
if self.sorted_indices is not None
else None
)
unsorted_indices = (
self.unsorted_indices.to(target_device)
if self.unsorted_indices is not None
else None
)
return PackedSequence(
data, self.batch_sizes, sorted_indices, unsorted_indices
)
def cuda(self) -> PackedSequence:
return self.to(device="gpu")
def cpu(self) -> PackedSequence:
return self.to(device="cpu")
def __repr__(self) -> str:
return (
f"PackedSequence(data={self.data}, batch_sizes={self.batch_sizes}, "
f"sorted_indices={self.sorted_indices}, unsorted_indices={self.unsorted_indices})"
)
def pin_memory(self) -> PackedSequence:
return PackedSequence(
self.data.pin_memory(),
self.batch_sizes,
self.sorted_indices.pin_memory()
if self.sorted_indices is not None
else None,
self.unsorted_indices.pin_memory()
if self.unsorted_indices is not None
else None,
)
@property
def is_cuda(self) -> bool:
return self.data.is_cuda
def double(self) -> PackedSequence:
return self.to(dtype=paddle.float64)
def float(self) -> PackedSequence:
return self.to(dtype=paddle.float32)
def half(self) -> PackedSequence:
return self.to(dtype=paddle.float16)
def long(self) -> PackedSequence:
return self.to(dtype=paddle.int64)
def int(self) -> PackedSequence:
return self.to(dtype=paddle.int32)
def short(self) -> PackedSequence:
return self.to(dtype=paddle.int16)
def char(self) -> PackedSequence:
return self.to(dtype=paddle.int8)
def byte(self) -> PackedSequence:
return self.to(dtype=paddle.uint8)
def pack_padded_sequence(
input: Tensor,
lengths: Tensor | list[int],
batch_first: bool = False,
enforce_sorted: bool = True,
) -> PackedSequence:
r"""Packs a Tensor containing padded sequences of variable length.
This function packs a Tensor containing padded sequences into a PackedSequence
object, which can be used as input to a recurrent neural network.
Args:
input (Tensor): The padded sequence tensor. Shape is ``T x B x *`` if
``batch_first`` is False, or ``B x T x *`` if ``batch_first`` is True,
where ``T`` is the length of the longest sequence, ``B`` is the batch size.
lengths (Tensor|list[int]): The lengths of each sequence in the batch.
batch_first (bool, optional): If True, the input is expected to be in
``B x T x *`` format. Default: False.
enforce_sorted (bool, optional): If True, the input is expected to contain
sequences sorted by length in descending order. Default: True.
Returns:
PackedSequence: A PackedSequence object containing the packed data.
Examples:
.. code-block:: pycon
>>> import paddle
"""
if batch_first:
input = input.transpose([1, 0, *range(2, len(input.shape))])
if isinstance(lengths, paddle.Tensor):
lengths = lengths.tolist()
batch_size = input.shape[1]
if len(lengths) != batch_size:
raise ValueError(
f"Length of lengths ({len(lengths)}) does not match batch size ({batch_size})"
)
sorted_indices = None
unsorted_indices = None
if not enforce_sorted:
sorted_lengths = sorted(
enumerate(lengths), key=lambda x: x[1], reverse=True
)
sorted_indices = paddle.to_tensor(
[i for i, _ in sorted_lengths], place=input.place
)
unsorted_indices = paddle.argsort(sorted_indices)
lengths = [l for _, l in sorted_lengths]
# Use index_select to reorder along batch dimension (axis=1)
input = paddle.index_select(input, sorted_indices, axis=1)
packed_data_list = []
batch_sizes_list = []
# num_steps may be different from actual input shape[0] after sorting
# We need to iterate over the actual sequence length
actual_num_steps = input.shape[0]
for step in range(actual_num_steps):
batch_size_at_step = sum(1 for l in lengths if l > step)
if batch_size_at_step > 0:
packed_data_list.append(input[step, :batch_size_at_step])
batch_sizes_list.append(batch_size_at_step)
packed_data = paddle.concat(packed_data_list, axis=0)
batch_sizes = paddle.to_tensor(batch_sizes_list, dtype="int64")
return PackedSequence(
packed_data, batch_sizes, sorted_indices, unsorted_indices
)
def pad_packed_sequence(
sequence: PackedSequence,
batch_first: bool = False,
padding_value: float = 0.0,
total_length: int | None = None,
) -> tuple[Tensor, Tensor]:
r"""Pads a packed sequence to a Tensor of padded sequences.
This function is the inverse of ``pack_padded_sequence``. It takes a PackedSequence
and returns a padded Tensor and a list of lengths.
Args:
sequence (PackedSequence): The packed sequence to pad.
batch_first (bool, optional): If True, the output will be in ``B x T x *``
format. Default: False.
padding_value (float, optional): The value to use for padding. Default: 0.0.
total_length (int|None, optional): If not None, the output will be padded to
this length. Default: None.
Returns:
tuple[Tensor, Tensor]: A tuple containing:
- The padded sequence tensor.
- A tensor of sequence lengths.
Examples:
.. code-block:: pycon
>>> import paddle
"""
if not isinstance(sequence, PackedSequence):
raise TypeError(f"Expected PackedSequence, got {type(sequence)}")
data = sequence.data
batch_sizes = sequence.batch_sizes.tolist()
unsorted_indices = sequence.unsorted_indices
max_seq_len = len(batch_sizes)
max_batch_size = batch_sizes[0]
if total_length is not None:
if total_length < max_seq_len:
raise ValueError(
f"total_length ({total_length}) must be >= max sequence length ({max_seq_len})"
)
trailing_dims = list(data.shape[1:])
if total_length is not None and total_length > max_seq_len:
output = paddle.full(
[total_length, max_batch_size, *trailing_dims],
padding_value,
dtype=data.dtype,
device=data.place,
)
else:
output = paddle.full(
[max_seq_len, max_batch_size, *trailing_dims],
padding_value,
dtype=data.dtype,
device=data.place,
)
data_offset = 0
for step, batch_size in enumerate(batch_sizes):
output[step, :batch_size] = data[data_offset : data_offset + batch_size]
data_offset += batch_size
# Calculate lengths from batch_sizes
# batch_sizes is in descending order, e.g., [3, 2, 1] means:
# - First time step has 3 sequences
# - Second time step has 2 sequences
# - Third time step has 1 sequence
# This means sequence lengths are [3, 2, 1] in sorted order
lengths_list = []
for i in range(max_batch_size):
# Find the length of the i-th sequence (in sorted order)
# It's the number of time steps where batch_sizes > i
seq_len = sum(1 for bs in batch_sizes if bs > i)
lengths_list.append(seq_len)
lengths = paddle.to_tensor(lengths_list, dtype="int64", place=data.place)
if unsorted_indices is not None:
output = output[:, unsorted_indices]
lengths = lengths[unsorted_indices]
if batch_first:
output = output.transpose([1, 0, *range(2, len(output.shape))])
return output, lengths
def pad_sequence(
sequences: Iterable[Tensor],
batch_first: bool = False,
padding_value: float = 0.0,
padding_side: str = 'right',
) -> Tensor:
r"""Pad a list of variable length Tensors with ``padding_value``.
``pad_sequence`` stacks a list of Tensors along a new dimension, and pads
them to equal length. ``sequences`` can be a list of sequences with size
``L x *``, where ``L`` is the length of the sequence and ``*`` is any
number of dimensions (including 0). If ``batch_first`` is ``False``, the
output is of size ``T x B x *``, and ``B x T x *`` otherwise, where ``B``
is the batch size (the number of elements in ``sequences``), ``T`` is the
length of the longest sequence.
Note:
This function returns a Tensor of size ``T x B x *`` or ``B x T x *``
where ``T`` is the length of the longest sequence. This function
assumes trailing dimensions and type of all the Tensors in sequences
are same.
Args:
sequences (list[Tensor]): list of variable length sequences.
batch_first (bool, optional): if ``True``, the output will be in
``B x T x *`` format, ``T x B x *`` otherwise. Default: ``False``.
padding_value (float, optional): value for padded elements.
Default: ``0.0``.
padding_side (str, optional): the side to pad the sequences on,
either ``'right'`` or ``'left'``. Default: ``'right'``.
Returns:
Tensor: Tensor of size ``T x B x *`` if ``batch_first`` is ``False``,
or ``B x T x *`` otherwise.
Examples:
.. code-block:: pycon
>>> import paddle
>>> a = paddle.ones([25, 300])
>>> b = paddle.ones([22, 300])
>>> c = paddle.ones([15, 300])
>>> padded = paddle.nn.utils.rnn.pad_sequence([a, b, c])
>>> print(padded.shape)
paddle.Size([25, 3, 300])
>>> padded = paddle.nn.utils.rnn.pad_sequence([a, b, c], batch_first=True)
>>> print(padded.shape)
paddle.Size([3, 25, 300])
"""
if not isinstance(sequences, Iterable):
raise TypeError(
f"pad_sequence expects an iterable of Tensors, but got {type(sequences)}"
)
sequences = tuple(sequences)
for seq in sequences:
if not isinstance(seq, paddle.Tensor):
raise TypeError(
f"pad_sequence expects an iterable of Tensors, but got element of type {type(seq)}"
)
if padding_side not in ('right', 'left'):
raise ValueError(
f"padding_side must be 'right' or 'left', but got '{padding_side}'"
)
max_len = max(seq.shape[0] for seq in sequences)
trailing_dims = sequences[0].shape[1:]
dtype = sequences[0].dtype
padded_seqs = []
for seq in sequences:
length = seq.shape[0]
if length == max_len:
padded_seqs.append(seq)
else:
pad_size = [max_len - length, *list(trailing_dims)]
padding = paddle.full(pad_size, padding_value, dtype=dtype)
if padding_side == 'right':
padded_seqs.append(paddle.concat([seq, padding], axis=0))
else:
padded_seqs.append(paddle.concat([padding, seq], axis=0))
out = paddle.stack(padded_seqs, axis=0)
if not batch_first:
# Transpose from B x T x * to T x B x *
perm = [1, 0, *list(range(2, len(out.shape)))]
out = out.transpose(perm)
return out
def unpad_sequence(
padded_sequences: Tensor,
lengths: Tensor,
batch_first: bool = False,
) -> list[Tensor]:
r"""Unpad a padded Tensor into a list of variable length Tensors.
``unpad_sequence`` unstacks a padded Tensor into a list of variable length
Tensors.
Args:
padded_sequences (Tensor): padded sequences.
lengths (Tensor): length of original (unpadded) sequences.
batch_first (bool, optional): whether batch dimension is first or not.
Default: ``False``.
Returns:
list[Tensor]: a list of Tensor objects with original lengths.
Examples:
.. code-block:: pycon
>>> import paddle
>>> a = paddle.ones([25, 300])
>>> b = paddle.ones([22, 300])
>>> c = paddle.ones([15, 300])
>>> sequences = [a, b, c]
>>> padded = paddle.nn.utils.rnn.pad_sequence(sequences)
>>> lengths = paddle.to_tensor([v.shape[0] for v in sequences])
>>> unpadded = paddle.nn.utils.rnn.unpad_sequence(padded, lengths)
>>> paddle.allclose(sequences[0], unpadded[0]).item()
True
>>> paddle.allclose(sequences[1], unpadded[1]).item()
True
>>> paddle.allclose(sequences[2], unpadded[2]).item()
True
"""
if not batch_first:
# Transpose from T x B x * to B x T x *
perm = [1, 0, *list(range(2, len(padded_sequences.shape)))]
padded_sequences = padded_sequences.transpose(perm)
unpadded = []
for seq, length in zip(padded_sequences, lengths):
length_val = length.item()
unpadded.append(seq[:length_val])
return unpadded
def pack_sequence(
sequences: list[Tensor],
enforce_sorted: bool = True,
) -> PackedSequence:
r"""Packs a list of variable length Tensors.
Consecutive call of the next functions: ``pad_sequence``, ``pack_padded_sequence``.
``sequences`` should be a list of Tensors of size ``L x *``, where `L` is
the length of a sequence and `*` is any number of trailing dimensions,
including ``0``.
For unsorted sequences, use `enforce_sorted = False`. If ``enforce_sorted``
is ``True``, the sequences should be sorted in the order of decreasing length.
``enforce_sorted = True`` is only necessary for ONNX export.
Args:
sequences (list[Tensor]): A list of sequences of decreasing length.
enforce_sorted (bool, optional): if ``True``, checks that the input
contains sequences sorted by length in a decreasing order. If
``False``, this condition is not checked. Default: ``True``.
Returns:
PackedSequence: a PackedSequence object.
Examples:
>>> import paddle
"""
lengths = paddle.to_tensor([v.shape[0] for v in sequences])
return pack_padded_sequence(
pad_sequence(sequences), lengths, enforce_sorted=enforce_sorted
)
def unpack_sequence(packed_sequences: PackedSequence) -> list[Tensor]:
r"""Unpack PackedSequence into a list of variable length Tensors.
``packed_sequences`` should be a PackedSequence object.
Args:
packed_sequences (PackedSequence): A PackedSequence object.
Returns:
list[Tensor]: a list of Tensor objects.
Examples:
>>> import paddle
"""
padded_sequences, lengths = pad_packed_sequence(
packed_sequences, batch_first=True
)
unpacked_sequences = unpad_sequence(
padded_sequences, lengths, batch_first=True
)
return unpacked_sequences
python/paddle/nn/utils/spectral_norm_hook.py
+245
View File
@@ -0,0 +1,245 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
import paddle
from .. import functional as F
from ..layer.common import Linear
from ..layer.conv import Conv1DTranspose, Conv2DTranspose, Conv3DTranspose
if TYPE_CHECKING:
from typing_extensions import Never
from paddle import Tensor
from paddle.nn import Layer
__all__ = []
def normal_(x: Tensor, mean: float = 0.0, std: float = 1.0) -> Tensor:
temp_value = paddle.normal(mean, std, shape=x.shape)
paddle.assign(temp_value, x)
return x
class SpectralNorm:
name: str
dim: int
n_power_iterations: int
eps: float
def __init__(
self,
name: str = 'weight',
n_power_iterations: int = 1,
dim: int = 0,
eps: float = 1e-12,
) -> None:
self.name = name
self.dim = dim
if n_power_iterations <= 0:
raise ValueError(
'Expected n_power_iterations to be positive, but '
f'got n_power_iterations={n_power_iterations}'
)
self.n_power_iterations = n_power_iterations
self.eps = eps
def reshape_weight_to_matrix(self, weight: Tensor) -> Tensor:
weight_mat = weight
if self.dim != 0:
# transpose dim to front
weight_mat = weight_mat.transpose(
[self.dim]
+ [d for d in range(weight_mat.dim()) if d != self.dim]
)
height = weight_mat.shape[0]
return weight_mat.reshape([height, -1])
def compute_weight(self, layer: Layer, do_power_iteration: bool) -> Tensor:
weight = getattr(layer, self.name + '_orig')
u = getattr(layer, self.name + '_u')
v = getattr(layer, self.name + '_v')
weight_mat = self.reshape_weight_to_matrix(weight)
if do_power_iteration:
with paddle.no_grad():
for _ in range(self.n_power_iterations):
paddle.assign(
F.normalize(
paddle.matmul(
weight_mat,
u,
transpose_x=True,
transpose_y=False,
),
axis=0,
epsilon=self.eps,
),
v,
)
paddle.assign(
F.normalize(
paddle.matmul(weight_mat, v),
axis=0,
epsilon=self.eps,
),
u,
)
if self.n_power_iterations > 0:
u = u.clone()
v = v.clone()
sigma = paddle.dot(u, paddle.mv(weight_mat, v))
weight = weight / sigma
return weight
def __call__(self, layer: Layer, inputs: Never) -> None:
setattr(
layer,
self.name,
self.compute_weight(layer, do_power_iteration=layer.training),
)
@staticmethod
def apply(
layer: Layer, name: str, n_power_iterations: int, dim: int, eps: float
) -> SpectralNorm:
for k, hook in layer._forward_pre_hooks.items():
if isinstance(hook, SpectralNorm) and hook.name == name:
raise RuntimeError(
"Cannot register two spectral_norm hooks on "
f"the same parameter {name}"
)
fn = SpectralNorm(name, n_power_iterations, dim, eps)
weight = layer._parameters[name]
with paddle.no_grad():
weight_mat = fn.reshape_weight_to_matrix(weight)
h, w = weight_mat.shape
# randomly initialize u and v
u = layer.create_parameter([h])
u = normal_(u, 0.0, 1.0)
v = layer.create_parameter([w])
v = normal_(v, 0.0, 1.0)
u = F.normalize(u, axis=0, epsilon=fn.eps)
v = F.normalize(v, axis=0, epsilon=fn.eps)
# delete fn.name form parameters, otherwise you can not set attribute
del layer._parameters[fn.name]
layer.add_parameter(fn.name + "_orig", weight)
# still need to assign weight back as fn.name because all sorts of
# things may assume that it exists, e.g., when initializing weights.
# However, we can't directly assign as it could be an Parameter and
# gets added as a parameter. Instead, we register weight * 1.0 as a plain
# attribute.
setattr(layer, fn.name, weight * 1.0)
layer.register_buffer(fn.name + "_u", u)
layer.register_buffer(fn.name + "_v", v)
layer.register_forward_pre_hook(fn)
return fn
def spectral_norm(
layer: Layer,
name: str = 'weight',
n_power_iterations: int = 1,
eps: float = 1e-12,
dim: int | None = None,
) -> Layer:
r"""
Applies spectral normalization to a parameter according to the
following Calculation:
Step 1:
Generate vector U in shape of [H], and V in shape of [W].
While H is the :attr:`dim` th dimension of the input weights,
and W is the product result of remaining dimensions.
Step 2:
:attr:`n_power_iterations` should be a positive integer, do following
calculations with U and V for :attr:`power_iters` rounds.
.. math::
\mathbf{v} := \frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2}
\mathbf{u} := \frac{\mathbf{W} \mathbf{v}}{\|\mathbf{W} \mathbf{v}\|_2}
Step 3:
Calculate :math:`\sigma(\mathbf{W})` and normalize weight values.
.. math::
\sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v}
\mathbf{W} = \frac{\mathbf{W}}{\sigma(\mathbf{W})}
Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .
Parameters:
layer(Layer): Layer of paddle, which has weight.
name(str, optional): Name of the weight parameter. Default: 'weight'.
n_power_iterations(int, optional): The number of power iterations to calculate spectral norm. Default: 1.
eps(float, optional): The epsilon for numerical stability in calculating norms. Default: 1e-12.
dim(int|None, optional): The index of dimension which should be permuted to the first before reshaping Input(Weight) to matrix, it should be set as 0 if Input(Weight) is the weight of fc layer, and should be set as 1 if Input(Weight) is the weight of conv layer. Default: None.
Returns:
Layer, the original layer with the spectral norm hook.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.nn import Conv2D
>>> from paddle.nn.utils import spectral_norm
>>> paddle.seed(2023)
>>> conv = Conv2D(3, 1, 3)
>>> sn_conv = spectral_norm(conv)
>>> print(sn_conv)
Conv2D(3, 1, kernel_size=[3, 3], data_format=NCHW)
>>> # Conv2D(3, 1, kernel_size=[3, 3], data_format=NCHW)
>>> print(sn_conv.weight)
Tensor(shape=[1, 3, 3, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
[[[[ 0.01668976, 0.30305523, 0.11405435],
[-0.06765547, -0.50396705, -0.40925547],
[ 0.47344422, 0.03628403, 0.45277366]],
[[-0.15177251, -0.16305730, -0.15723954],
[-0.28081197, -0.09183260, -0.08081978],
[-0.40895155, 0.18298769, -0.29325116]],
[[ 0.21819633, -0.01822380, -0.50351536],
[-0.06262003, 0.17713565, 0.20517939],
[ 0.16659889, -0.14333329, 0.05228264]]]])
"""
if dim is None:
if isinstance(
layer, (Conv1DTranspose, Conv2DTranspose, Conv3DTranspose, Linear)
):
dim = 1
else:
dim = 0
SpectralNorm.apply(layer, name, n_power_iterations, dim, eps)
return layer
python/paddle/nn/utils/transform_parameters.py
+208
View File
@@ -0,0 +1,208 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from functools import reduce
from typing import TYPE_CHECKING
import paddle
from paddle import _C_ops
from paddle.base.framework import (
_create_tensor,
_dygraph_tracer,
dygraph_only,
in_dygraph_mode,
)
if TYPE_CHECKING:
from collections.abc import Iterable
from paddle import Tensor
from paddle._typing import ShapeLike
# input==output, inplace strategy of reshape has no cost almost
def _inplace_reshape_dygraph(x: Tensor, shape: ShapeLike) -> None:
x_shape = _create_tensor(dtype='int64')
if in_dygraph_mode():
with paddle.base.dygraph.no_grad():
tmp_out = _C_ops.reshape(x, shape)
tmp_out._share_underline_tensor_to(x)
else:
_dygraph_tracer().trace_op(
type="reshape2",
inputs={'X': x},
outputs={'Out': x, 'XShape': x_shape},
attrs={'shape': shape},
stop_gradient=True,
)
@dygraph_only
def _stride_column(param: Tensor) -> None:
"""
A tool function. Permute date of parameter as a 'columns' stride. Now, it only support 2-D parameter.
Args:
param(Tensor): The param that will be strided according to 'columns'.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(100)
>>> linear = paddle.nn.Linear(2, 3)
>>> print(linear.weight)
Parameter containing:
Tensor(shape=[2, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
[[ 0.11732829, -0.64161885, -1.06996548],
[ 0.03456247, -0.29862350, -0.52380574]])
>>> paddle.nn.utils._stride_column(linear.weight)
>>> print(linear.weight)
"""
assert len(param.shape) == 2
shape = [param.shape[1], param.shape[0]]
with paddle.base.dygraph.no_grad():
reshape_var = paddle.reshape(param, shape)
transpose_var = paddle.transpose(reshape_var, [1, 0])
transpose_var._share_underline_tensor_to(param)
@dygraph_only
def parameters_to_vector(
parameters: Iterable[Tensor], name: str | None = None
) -> Tensor:
"""
Flatten parameters to a 1-D Tensor.
Args:
parameters(Iterable[Tensor]): Iterable Tensors that are trainable parameters of a Layer.
name(str, optional): The default value is None. Normally there is no need for user to set this
property. For more information, please refer to :ref:`api_guide_Name`.
Returns:
A 1-D Tensor, which represents the parameters of a Layer.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(2023)
>>> linear = paddle.nn.Linear(10, 15)
>>> t = paddle.nn.utils.parameters_to_vector(linear.parameters())
>>> print(t.shape)
paddle.Size([165])
"""
dtype = parameters[0].dtype
origin_shapes = []
for param in parameters:
origin_shapes.append(param.shape)
_inplace_reshape_dygraph(param, [-1])
out = _create_tensor(dtype=dtype)
if in_dygraph_mode():
with paddle.base.dygraph.no_grad():
tmp = _C_ops.concat(parameters, 0)
tmp._share_underline_tensor_to(out)
else:
_dygraph_tracer().trace_op(
type='concat',
inputs={'X': parameters},
outputs={'Out': [out]},
attrs={'axis': 0},
stop_gradient=True,
)
for i, param in enumerate(parameters):
_inplace_reshape_dygraph(param, origin_shapes[i])
out.stop_gradient = False
return out
@dygraph_only
def vector_to_parameters(
vec: Tensor, parameters: Iterable[Tensor], name: str | None = None
) -> None:
"""
Transform a 1-D Tensor to the input ``parameters`` .
Args:
vec (Tensor): A 1-D Tensor, which will be sliced and copied to the input ``parameters`` .
parameters (Iterable[Tensor]): Iterable Tensors that are trainable parameters of a Layer.
name(str, optional): The default value is None. Normally there is no need for user to set this
property. For more information, please refer to :ref:`api_guide_Name`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Constant(3.0))
>>> linear1 = paddle.nn.Linear(10, 15, weight_attr)
>>> vec = paddle.nn.utils.parameters_to_vector(linear1.parameters())
>>> linear2 = paddle.nn.Linear(10, 15)
>>> # copy weight of linear1 to linear2
>>> paddle.nn.utils.vector_to_parameters(vec, linear2.parameters())
>>> print((linear1.weight == linear2.weight).all())
Tensor(shape=[], dtype=bool, place=Place(cpu), stop_gradient=True,
True)
"""
assert len(vec.shape) == 1
origin_shapes = []
sections = []
total_elements = 0
for param in parameters:
shape = param.shape
origin_shapes.append(shape)
numel = reduce(lambda x, y: x * y, shape, 1)
total_elements += numel
sections.append(numel)
if len(sections) == 1:
sections.append(0)
if in_dygraph_mode():
with paddle.base.dygraph.no_grad():
res = []
if total_elements == vec.shape[0]:
res = _C_ops.split(vec, sections, 0)
elif total_elements < vec.shape[0]:
pointer = 0
for section in sections:
res.append(vec[pointer : pointer + section])
pointer += section
else:
raise ValueError(
"The total_elements of vec should be equal to or larger than the number of elements in parameters."
)
for i in range(0, len(parameters)):
res[i]._share_underline_tensor_to(parameters[i])
else:
_dygraph_tracer().trace_op(
type='split',
inputs={'X': [vec]},
outputs={'Out': parameters},
attrs={'axis': 0, 'sections': sections},
stop_gradient=True,
)
for i, param in enumerate(parameters):
_inplace_reshape_dygraph(param, origin_shapes[i])
python/paddle/nn/utils/weight_norm_hook.py
+263
View File
@@ -0,0 +1,263 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
import paddle
from paddle import _C_ops
from paddle.utils.decorator_utils import param_one_alias
from ...base.data_feeder import check_variable_and_dtype
from ...base.layer_helper import LayerHelper
from ...framework import in_dynamic_or_pir_mode
if TYPE_CHECKING:
from typing_extensions import Never
from paddle import Tensor
from paddle.nn import Layer
__all__ = []
def l2_norm(
x: Tensor, axis: int, epsilon: float = 1e-12, name: str | None = None
) -> Tensor:
if len(x.shape) == 1:
axis = 0
if in_dynamic_or_pir_mode():
out, norm = _C_ops.norm(x, 1 if axis is None else axis, epsilon, False)
return paddle.squeeze(norm, axis=[axis])
check_variable_and_dtype(x, "X", ("float32", "float64"), "norm")
helper = LayerHelper("l2_normalize", **locals())
out = helper.create_variable_for_type_inference(dtype=x.dtype)
norm = helper.create_variable_for_type_inference(dtype=x.dtype)
helper.append_op(
type="norm",
inputs={"X": x},
outputs={"Out": out, "Norm": norm},
attrs={
"axis": 1 if axis is None else axis,
"epsilon": epsilon,
},
)
return paddle.squeeze(norm, axis=[axis])
def norm_except_dim(p: Tensor, dim: int) -> Tensor:
shape = p.shape
ndims = len(shape)
if dim == -1:
return paddle.sqrt(paddle.sum(paddle.square(p)) + 1e-12)
elif dim == 0:
p_matrix = paddle.reshape(p, (shape[0], -1))
return l2_norm(p_matrix, axis=1)
elif dim == ndims - 1:
p_matrix = paddle.reshape(p, (-1, shape[-1]))
return l2_norm(p_matrix, axis=0)
else:
perm = list(range(ndims))
perm[0] = dim
perm[dim] = 0
p_transposed = paddle.transpose(p, perm)
return norm_except_dim(p_transposed, 0)
def _weight_norm(v: Tensor, g: Tensor, dim: int) -> Tensor:
shape = v.shape
ndims = len(shape)
if dim == -1:
v_normalized = v / (paddle.sqrt(paddle.sum(paddle.square(v))) + 1e-12)
elif dim == 0:
p_matrix = paddle.reshape(v, (shape[0], -1))
v_normalized = paddle.nn.functional.normalize(p_matrix, axis=1)
v_normalized = paddle.reshape(v_normalized, shape)
elif dim == ndims - 1:
p_matrix = paddle.reshape(v, (-1, shape[-1]))
v_normalized = paddle.nn.functional.normalize(p_matrix, axis=0)
v_normalized = paddle.reshape(v_normalized, shape)
else:
perm = list(range(ndims))
perm[0] = dim
perm[dim] = 0
p_transposed = paddle.transpose(v, perm)
transposed_shape = p_transposed.shape
p_matrix = paddle.reshape(p_transposed, (p_transposed.shape[0], -1))
v_normalized = paddle.nn.functional.normalize(p_matrix, axis=1)
v_normalized = paddle.reshape(v_normalized, transposed_shape)
v_normalized = paddle.transpose(v_normalized, perm)
weight = paddle.tensor.math._multiply_with_axis(
v_normalized, g, axis=dim if dim is not None else -1
)
return weight
class WeightNorm:
name: str
dim: int
def __init__(self, name: str, dim: int) -> None:
if dim is None:
dim = -1
self.name = name
self.dim = dim
def compute_weight(self, layer: Layer) -> Tensor:
g = getattr(layer, self.name + '_g')
v = getattr(layer, self.name + '_v')
return _weight_norm(v, g, self.dim)
@staticmethod
def apply(layer: Layer, name: str, dim: int) -> WeightNorm:
for k, hook in layer._forward_pre_hooks.items():
if isinstance(hook, WeightNorm) and hook.name == name:
raise RuntimeError(
"Cannot register two weight_norm hooks on "
f"the same parameter {name}"
)
if dim is None:
dim = -1
# support dim is negative number, (dim = -1) == (dim = None)
weight_dim = len(layer._parameters[name].shape)
assert dim < weight_dim and dim >= -1 * weight_dim, (
"dim must set between [-R, R), R means the dimension of weight."
)
if dim != -1:
dim = (dim + weight_dim) % weight_dim
fn = WeightNorm(name, dim)
w = getattr(layer, name)
del layer._parameters[name]
g_var = norm_except_dim(w, dim)
v = layer.create_parameter(w.shape, dtype=w.dtype)
layer.add_parameter(name + "_v", v)
g = layer.create_parameter(g_var.shape, dtype=g_var.dtype)
layer.add_parameter(name + '_g', g)
with paddle.no_grad():
paddle.assign(w, v)
paddle.assign(g_var, g)
setattr(layer, name, fn.compute_weight(layer))
layer.register_forward_pre_hook(fn)
return fn
def remove(self, layer: Layer) -> None:
w_var = self.compute_weight(layer)
delattr(layer, self.name)
del layer._parameters[self.name + '_g']
del layer._parameters[self.name + '_v']
w = layer.create_parameter(w_var.shape, dtype=w_var.dtype)
layer.add_parameter(self.name, w)
with paddle.no_grad():
paddle.assign(w_var, w)
def __call__(self, layer: Layer, inputs: Never) -> None:
setattr(layer, self.name, self.compute_weight(layer))
@param_one_alias(["layer", "module"])
def weight_norm(layer: Layer, name: str = 'weight', dim: int = 0) -> Layer:
r"""
Applies weight normalization to a parameter according to the
following formula:
.. math::
\mathbf{w} = g \dfrac{v}{\|v\|}
Weight normalization is a reparameterization of the weight vectors in a neural network that
decouples the magnitude of those weight vectors from their direction. Weight normalization
replaces the parameter specified by ``name`` (eg: 'weight') with two parameters: one parameter
specifying the magnitude (eg: 'weight_g') and one parameter specifying the direction
(eg: 'weight_v'). Weight normalization has been implemented as discussed in this paper:
`Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks
<https://arxiv.org/pdf/1602.07868.pdf>`_.
Parameters:
layer(Layer): Layer of paddle, which has weight.
Alias: ``module``.
name(str, optional): Name of the weight parameter. Default: 'weight'.
dim(int, optional): Dimension over which to compute the norm. Dim is a non-negative number
which is less than the rank of weight Tensor. For Example, dim can be chosen from 0,
1, 2, 3 for convolution whose weight shape is [cout, cin, kh, kw] and rank is 4.
If dim is set to None, meaning that all elements will be normalized. Default: 0.
Returns:
Origin layer with weight norm hook.
Examples:
.. code-block:: pycon
>>> from paddle.nn import Conv2D
>>> from paddle.nn.utils import weight_norm
>>> conv = Conv2D(3, 5, 3)
>>> wn = weight_norm(conv)
>>> print(conv.weight_g.shape)
paddle.Size([5])
>>> print(conv.weight_v.shape)
paddle.Size([5, 3, 3, 3])
"""
WeightNorm.apply(layer, name, dim)
return layer
def remove_weight_norm(layer: Layer, name: str = 'weight') -> Layer:
"""
remove weight normalization from layer.
Parameters:
layer(Layer): Layer of paddle, which has weight.
name(str, optional): Name of the weight parameter. Default: 'weight'.
Returns:
Layer, the origin layer without weight norm
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.nn import Conv2D
>>> from paddle.nn.utils import weight_norm, remove_weight_norm
>>> paddle.seed(2023)
>>> conv = Conv2D(3, 5, 3)
>>> wn = weight_norm(conv)
>>> print(conv.weight_g)
Parameter containing:
Tensor(shape=[5], dtype=float32, place=Place(cpu), stop_gradient=False,
[1.35883713, 1.32126212, 1.56303072, 1.20874095, 1.22893476])
>>> remove_weight_norm(conv)
>>> # The following is the effect after removing the weight norm:
>>> # print(conv.weight_g)
>>> # AttributeError: 'Conv2D' object has no attribute 'weight_g'
"""
for k, hook in layer._forward_pre_hooks.items():
if isinstance(hook, WeightNorm) and hook.name == name:
hook.remove(layer)
del layer._forward_pre_hooks[k]
return layer
raise ValueError(f"weight_norm of '{name}' not found in {layer}")