chore: import upstream snapshot with attribution

python/paddle/nn/utils/__init__.py

@@ -0,0 +1,34 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import rnn  # noqa: F401
+from .clip_grad_norm_ import clip_grad_norm_
+from .clip_grad_value_ import clip_grad_value_
+from .spectral_norm_hook import spectral_norm
+from .transform_parameters import (
+    _stride_column,  # noqa: F401
+    parameters_to_vector,
+    vector_to_parameters,
+)
+from .weight_norm_hook import remove_weight_norm, weight_norm
+
+__all__ = [
+    'weight_norm',
+    'remove_weight_norm',
+    'spectral_norm',
+    'parameters_to_vector',
+    'vector_to_parameters',
+    'clip_grad_norm_',
+    'clip_grad_value_',
+]

python/paddle/nn/utils/clip_grad_norm_.py

@@ -0,0 +1,119 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import paddle
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from paddle import Tensor
+
+__all__ = []
+
+
+@paddle.autograd.no_grad()
+def clip_grad_norm_(
+    parameters: Iterable[Tensor] | Tensor,
+    max_norm: float,
+    norm_type: float = 2.0,
+    error_if_nonfinite: bool = False,
+) -> Tensor:
+    r"""Clips gradient norm of the iterable parameters.
+
+    Norms are calculated together on all gradients, just as they are
+    connected into one vector. The gradient will be modified in place.
+
+    This API can only run in dynamic graph mode, not static graph mode.
+
+    Args:
+        parameters (Iterable[paddle.Tensor] or paddle.Tensor): Tensors or a single Tensor
+            that will be normalized gradients
+        max_norm (float or int): max norm of the gradients
+        norm_type (float or int): type of the used p-norm. Can be `inf` for
+            infinity norm.
+        error_if_nonfinite (bool): if True, throw an error if the total
+            norm of the gradients from :attr:`parameters` is `nan`,
+            `inf`, or `-inf`.
+
+    Returns:
+        Total norm of the parameter gradients (treated as a single vector).
+
+    Example:
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32')
+            >>> max_norm = float(5.0)
+            >>> linear = paddle.nn.Linear(in_features=10, out_features=10)
+            >>> out = linear(x)
+            >>> loss = paddle.mean(out)
+            >>> loss.backward()
+
+            >>> paddle.nn.utils.clip_grad_norm_(linear.parameters(), max_norm)
+
+            >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters())
+            >>> sdg.step()
+    """
+    if not paddle.in_dynamic_mode():
+        raise RuntimeError('this API can only run in dynamic mode.')
+
+    if isinstance(parameters, paddle.Tensor):
+        parameters = [parameters]
+
+    support_norm_type = [float("inf"), 0, 1, 2]
+    if norm_type not in support_norm_type:
+        raise ValueError(f'norm_type only support {support_norm_type}')
+
+    grads = [p.grad_ for p in parameters if p.grad_ is not None]
+    max_norm = float(max_norm)
+    norm_type = float(norm_type)
+    if len(grads) == 0:
+        return paddle.to_tensor(0.0)
+    if norm_type == float("inf"):
+        norms = [g.detach().abs().max() for g in grads]
+        total_norm = (
+            norms[0] if len(norms) == 1 else paddle.max(paddle.stack(norms))
+        )
+    else:
+        total_norm = paddle.linalg.norm(
+            paddle.stack(
+                [paddle.linalg.norm(g.detach(), norm_type) for g in grads]
+            ),
+            norm_type,
+        )
+
+    if error_if_nonfinite and paddle.logical_or(
+        total_norm.isnan(), total_norm.isinf()
+    ):
+        raise RuntimeError(
+            f'The total norm of {norm_type} order of the gradients from '
+            '`parameters` is non-finite, so it cannot be clipped. In any case, '
+            'disable this error and scale the gradient by non-finite norm, '
+            'set `error_if_nonfinite=False`'
+        )
+    clip_coef = max_norm / (total_norm + 1e-6)
+    # Note: when the coef is clamped to 1, it is redundant to multiply the clamped coef, but this
+    # avoids the `if clip_coef < 1:` condition.
+    clip_coef_clamped = clip_coef.clip_(max=1.0)
+
+    for _, p in enumerate(parameters):
+        if p.grad_ is not None:
+            p.grad_.multiply_(y=clip_coef_clamped)
+
+    return total_norm

python/paddle/nn/utils/clip_grad_value_.py

@@ -0,0 +1,70 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import paddle
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from paddle import Tensor
+
+__all__ = []
+
+
+@paddle.autograd.no_grad()
+def clip_grad_value_(
+    parameters: Iterable[Tensor] | Tensor,
+    clip_value: float,
+) -> None:
+    r"""
+    Clips gradient of an iterable of parameters at specified value.
+    The gradient will be modified in place.
+    This API can only run in dynamic graph mode, not static graph mode.
+
+    Args:
+        parameters (Iterable[paddle.Tensor]|paddle.Tensor): Tensors or a single Tensor
+            that will be normalized gradients
+        clip_value (float|int): maximum allowed value of the gradients.
+            The gradients are clipped in the range
+            :math:`\left[\text{-clip\_value}, \text{clip\_value}\right]`
+
+    Example:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> x = paddle.uniform([10, 10], min=-10.0, max=10.0, dtype='float32')
+            >>> clip_value = float(5.0)
+            >>> linear = paddle.nn.Linear(in_features=10, out_features=10)
+            >>> out = linear(x)
+            >>> loss = paddle.mean(out)
+            >>> loss.backward()
+            >>> paddle.nn.utils.clip_grad_value_(linear.parameters(), clip_value)
+            >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters())
+            >>> sdg.step()
+    """
+    if not paddle.in_dynamic_mode():
+        raise RuntimeError('this API can only run in dynamic mode.')
+
+    if isinstance(parameters, paddle.Tensor):
+        parameters = [parameters]
+
+    clip_value = float(clip_value)
+
+    for _, p in enumerate(parameters):
+        if p.grad is not None:
+            p.grad.clip_(min=-clip_value, max=clip_value)

python/paddle/nn/utils/dygraph_utils.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle import _legacy_C_ops
+from paddle.framework import dygraph_only
+
+
+@dygraph_only
+def _append_bias_in_dygraph(input, bias=None, axis=1):
+    """Append bias operation in dygraph mode.
+
+        Args:
+            input: the input variable.
+            bias:  the bias to be appended
+            axis:  the axis to perform operation
+
+    Return the Variable after bias operation
+    """
+    if bias is None:
+        return input
+
+    return _legacy_C_ops.elementwise_add(input, bias, 'axis', axis)

python/paddle/nn/utils/rnn.py

@@ -0,0 +1,549 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
+
+import paddle
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+
+__all__ = [
+    "PackedSequence",
+    "invert_permutation",
+    "pack_padded_sequence",
+    "pad_packed_sequence",
+    "pad_sequence",
+    "unpad_sequence",
+    "pack_sequence",
+    "unpack_sequence",
+]
+
+
+def invert_permutation(permutation: Tensor | None) -> Tensor | None:
+    """Returns the inverse of ``permutation``.
+
+    This is useful for converting between sorted and unsorted indices in
+    a :class:`~nn.utils.rnn.PackedSequence`.
+
+    Args:
+        permutation (Tensor|None): a 1-D tensor of indices to invert.
+
+    Returns:
+        Tensor|None: the inverse permutation tensor, or None if input is None.
+
+    Examples:
+        >>> import paddle
+    """
+    if permutation is None:
+        return None
+    # Use paddle.scatter instead of scatter_ for better static mode support
+    output = paddle.scatter(
+        paddle.zeros_like(permutation),
+        permutation,
+        paddle.arange(
+            0,
+            permutation.numel(),
+            dtype=permutation.dtype,
+            device=permutation.place,
+        ),
+        overwrite=True,
+    )
+    return output
+
+
+class PackedSequence:
+    """Holds the data and batch sizes of a packed sequence.
+
+    PackedSequence is used to represent a packed sequence, which is typically
+    produced by ``pack_padded_sequence`` and consumed by ``pad_packed_sequence``.
+
+    Args:
+        data (Tensor): The packed data tensor.
+        batch_sizes (Tensor): A tensor containing the batch size at each step.
+        sorted_indices (Tensor|None, optional): The indices used to sort the sequences.
+        unsorted_indices (Tensor|None, optional): The indices to restore the original order.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+
+    """
+
+    def __init__(
+        self,
+        data: Tensor,
+        batch_sizes: Tensor,
+        sorted_indices: Tensor | None = None,
+        unsorted_indices: Tensor | None = None,
+    ):
+        self.data = data
+        self.batch_sizes = batch_sizes
+        self.sorted_indices = sorted_indices
+        self.unsorted_indices = unsorted_indices
+
+    @property
+    def is_pinned(self) -> bool:
+        return (
+            self.data.place.is_cuda_pinned_place()
+            or self.data.place.is_xpu_pinned_place()
+        )
+
+    def to(self, *args, **kwargs) -> PackedSequence:
+        data = self.data.to(*args, **kwargs)
+        if data is self.data:
+            return self
+        # Only convert indices to same device as data, not dtype
+        target_device = data.place
+        sorted_indices = (
+            self.sorted_indices.to(target_device)
+            if self.sorted_indices is not None
+            else None
+        )
+        unsorted_indices = (
+            self.unsorted_indices.to(target_device)
+            if self.unsorted_indices is not None
+            else None
+        )
+        return PackedSequence(
+            data, self.batch_sizes, sorted_indices, unsorted_indices
+        )
+
+    def cuda(self) -> PackedSequence:
+        return self.to(device="gpu")
+
+    def cpu(self) -> PackedSequence:
+        return self.to(device="cpu")
+
+    def __repr__(self) -> str:
+        return (
+            f"PackedSequence(data={self.data}, batch_sizes={self.batch_sizes}, "
+            f"sorted_indices={self.sorted_indices}, unsorted_indices={self.unsorted_indices})"
+        )
+
+    def pin_memory(self) -> PackedSequence:
+        return PackedSequence(
+            self.data.pin_memory(),
+            self.batch_sizes,
+            self.sorted_indices.pin_memory()
+            if self.sorted_indices is not None
+            else None,
+            self.unsorted_indices.pin_memory()
+            if self.unsorted_indices is not None
+            else None,
+        )
+
+    @property
+    def is_cuda(self) -> bool:
+        return self.data.is_cuda
+
+    def double(self) -> PackedSequence:
+        return self.to(dtype=paddle.float64)
+
+    def float(self) -> PackedSequence:
+        return self.to(dtype=paddle.float32)
+
+    def half(self) -> PackedSequence:
+        return self.to(dtype=paddle.float16)
+
+    def long(self) -> PackedSequence:
+        return self.to(dtype=paddle.int64)
+
+    def int(self) -> PackedSequence:
+        return self.to(dtype=paddle.int32)
+
+    def short(self) -> PackedSequence:
+        return self.to(dtype=paddle.int16)
+
+    def char(self) -> PackedSequence:
+        return self.to(dtype=paddle.int8)
+
+    def byte(self) -> PackedSequence:
+        return self.to(dtype=paddle.uint8)
+
+
+def pack_padded_sequence(
+    input: Tensor,
+    lengths: Tensor | list[int],
+    batch_first: bool = False,
+    enforce_sorted: bool = True,
+) -> PackedSequence:
+    r"""Packs a Tensor containing padded sequences of variable length.
+
+    This function packs a Tensor containing padded sequences into a PackedSequence
+    object, which can be used as input to a recurrent neural network.
+
+    Args:
+        input (Tensor): The padded sequence tensor. Shape is ``T x B x *`` if
+            ``batch_first`` is False, or ``B x T x *`` if ``batch_first`` is True,
+            where ``T`` is the length of the longest sequence, ``B`` is the batch size.
+        lengths (Tensor|list[int]): The lengths of each sequence in the batch.
+        batch_first (bool, optional): If True, the input is expected to be in
+            ``B x T x *`` format. Default: False.
+        enforce_sorted (bool, optional): If True, the input is expected to contain
+            sequences sorted by length in descending order. Default: True.
+
+    Returns:
+        PackedSequence: A PackedSequence object containing the packed data.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+
+    """
+    if batch_first:
+        input = input.transpose([1, 0, *range(2, len(input.shape))])
+
+    if isinstance(lengths, paddle.Tensor):
+        lengths = lengths.tolist()
+
+    batch_size = input.shape[1]
+
+    if len(lengths) != batch_size:
+        raise ValueError(
+            f"Length of lengths ({len(lengths)}) does not match batch size ({batch_size})"
+        )
+
+    sorted_indices = None
+    unsorted_indices = None
+
+    if not enforce_sorted:
+        sorted_lengths = sorted(
+            enumerate(lengths), key=lambda x: x[1], reverse=True
+        )
+        sorted_indices = paddle.to_tensor(
+            [i for i, _ in sorted_lengths], place=input.place
+        )
+        unsorted_indices = paddle.argsort(sorted_indices)
+        lengths = [l for _, l in sorted_lengths]
+        # Use index_select to reorder along batch dimension (axis=1)
+        input = paddle.index_select(input, sorted_indices, axis=1)
+
+    packed_data_list = []
+    batch_sizes_list = []
+
+    # num_steps may be different from actual input shape[0] after sorting
+    # We need to iterate over the actual sequence length
+    actual_num_steps = input.shape[0]
+    for step in range(actual_num_steps):
+        batch_size_at_step = sum(1 for l in lengths if l > step)
+        if batch_size_at_step > 0:
+            packed_data_list.append(input[step, :batch_size_at_step])
+            batch_sizes_list.append(batch_size_at_step)
+
+    packed_data = paddle.concat(packed_data_list, axis=0)
+    batch_sizes = paddle.to_tensor(batch_sizes_list, dtype="int64")
+
+    return PackedSequence(
+        packed_data, batch_sizes, sorted_indices, unsorted_indices
+    )
+
+
+def pad_packed_sequence(
+    sequence: PackedSequence,
+    batch_first: bool = False,
+    padding_value: float = 0.0,
+    total_length: int | None = None,
+) -> tuple[Tensor, Tensor]:
+    r"""Pads a packed sequence to a Tensor of padded sequences.
+
+    This function is the inverse of ``pack_padded_sequence``. It takes a PackedSequence
+    and returns a padded Tensor and a list of lengths.
+
+    Args:
+        sequence (PackedSequence): The packed sequence to pad.
+        batch_first (bool, optional): If True, the output will be in ``B x T x *``
+            format. Default: False.
+        padding_value (float, optional): The value to use for padding. Default: 0.0.
+        total_length (int|None, optional): If not None, the output will be padded to
+            this length. Default: None.
+
+    Returns:
+        tuple[Tensor, Tensor]: A tuple containing:
+            - The padded sequence tensor.
+            - A tensor of sequence lengths.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+
+    """
+    if not isinstance(sequence, PackedSequence):
+        raise TypeError(f"Expected PackedSequence, got {type(sequence)}")
+
+    data = sequence.data
+    batch_sizes = sequence.batch_sizes.tolist()
+    unsorted_indices = sequence.unsorted_indices
+
+    max_seq_len = len(batch_sizes)
+    max_batch_size = batch_sizes[0]
+
+    if total_length is not None:
+        if total_length < max_seq_len:
+            raise ValueError(
+                f"total_length ({total_length}) must be >= max sequence length ({max_seq_len})"
+            )
+
+    trailing_dims = list(data.shape[1:])
+
+    if total_length is not None and total_length > max_seq_len:
+        output = paddle.full(
+            [total_length, max_batch_size, *trailing_dims],
+            padding_value,
+            dtype=data.dtype,
+            device=data.place,
+        )
+    else:
+        output = paddle.full(
+            [max_seq_len, max_batch_size, *trailing_dims],
+            padding_value,
+            dtype=data.dtype,
+            device=data.place,
+        )
+
+    data_offset = 0
+    for step, batch_size in enumerate(batch_sizes):
+        output[step, :batch_size] = data[data_offset : data_offset + batch_size]
+        data_offset += batch_size
+
+    # Calculate lengths from batch_sizes
+    # batch_sizes is in descending order, e.g., [3, 2, 1] means:
+    # - First time step has 3 sequences
+    # - Second time step has 2 sequences
+    # - Third time step has 1 sequence
+    # This means sequence lengths are [3, 2, 1] in sorted order
+    lengths_list = []
+    for i in range(max_batch_size):
+        # Find the length of the i-th sequence (in sorted order)
+        # It's the number of time steps where batch_sizes > i
+        seq_len = sum(1 for bs in batch_sizes if bs > i)
+        lengths_list.append(seq_len)
+    lengths = paddle.to_tensor(lengths_list, dtype="int64", place=data.place)
+
+    if unsorted_indices is not None:
+        output = output[:, unsorted_indices]
+        lengths = lengths[unsorted_indices]
+
+    if batch_first:
+        output = output.transpose([1, 0, *range(2, len(output.shape))])
+
+    return output, lengths
+
+
+def pad_sequence(
+    sequences: Iterable[Tensor],
+    batch_first: bool = False,
+    padding_value: float = 0.0,
+    padding_side: str = 'right',
+) -> Tensor:
+    r"""Pad a list of variable length Tensors with ``padding_value``.
+
+    ``pad_sequence`` stacks a list of Tensors along a new dimension, and pads
+    them to equal length. ``sequences`` can be a list of sequences with size
+    ``L x *``, where ``L`` is the length of the sequence and ``*`` is any
+    number of dimensions (including 0). If ``batch_first`` is ``False``, the
+    output is of size ``T x B x *``, and ``B x T x *`` otherwise, where ``B``
+    is the batch size (the number of elements in ``sequences``), ``T`` is the
+    length of the longest sequence.
+
+    Note:
+        This function returns a Tensor of size ``T x B x *`` or ``B x T x *``
+        where ``T`` is the length of the longest sequence. This function
+        assumes trailing dimensions and type of all the Tensors in sequences
+        are same.
+
+    Args:
+        sequences (list[Tensor]): list of variable length sequences.
+        batch_first (bool, optional): if ``True``, the output will be in
+            ``B x T x *`` format, ``T x B x *`` otherwise. Default: ``False``.
+        padding_value (float, optional): value for padded elements.
+            Default: ``0.0``.
+        padding_side (str, optional): the side to pad the sequences on,
+            either ``'right'`` or ``'left'``. Default: ``'right'``.
+
+    Returns:
+        Tensor: Tensor of size ``T x B x *`` if ``batch_first`` is ``False``,
+        or ``B x T x *`` otherwise.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> a = paddle.ones([25, 300])
+            >>> b = paddle.ones([22, 300])
+            >>> c = paddle.ones([15, 300])
+            >>> padded = paddle.nn.utils.rnn.pad_sequence([a, b, c])
+            >>> print(padded.shape)
+            paddle.Size([25, 3, 300])
+            >>> padded = paddle.nn.utils.rnn.pad_sequence([a, b, c], batch_first=True)
+            >>> print(padded.shape)
+            paddle.Size([3, 25, 300])
+
+    """
+    if not isinstance(sequences, Iterable):
+        raise TypeError(
+            f"pad_sequence expects an iterable of Tensors, but got {type(sequences)}"
+        )
+    sequences = tuple(sequences)
+    for seq in sequences:
+        if not isinstance(seq, paddle.Tensor):
+            raise TypeError(
+                f"pad_sequence expects an iterable of Tensors, but got element of type {type(seq)}"
+            )
+    if padding_side not in ('right', 'left'):
+        raise ValueError(
+            f"padding_side must be 'right' or 'left', but got '{padding_side}'"
+        )
+
+    max_len = max(seq.shape[0] for seq in sequences)
+    trailing_dims = sequences[0].shape[1:]
+    dtype = sequences[0].dtype
+
+    padded_seqs = []
+    for seq in sequences:
+        length = seq.shape[0]
+        if length == max_len:
+            padded_seqs.append(seq)
+        else:
+            pad_size = [max_len - length, *list(trailing_dims)]
+            padding = paddle.full(pad_size, padding_value, dtype=dtype)
+            if padding_side == 'right':
+                padded_seqs.append(paddle.concat([seq, padding], axis=0))
+            else:
+                padded_seqs.append(paddle.concat([padding, seq], axis=0))
+
+    out = paddle.stack(padded_seqs, axis=0)
+
+    if not batch_first:
+        # Transpose from B x T x * to T x B x *
+        perm = [1, 0, *list(range(2, len(out.shape)))]
+        out = out.transpose(perm)
+
+    return out
+
+
+def unpad_sequence(
+    padded_sequences: Tensor,
+    lengths: Tensor,
+    batch_first: bool = False,
+) -> list[Tensor]:
+    r"""Unpad a padded Tensor into a list of variable length Tensors.
+
+    ``unpad_sequence`` unstacks a padded Tensor into a list of variable length
+    Tensors.
+
+    Args:
+        padded_sequences (Tensor): padded sequences.
+        lengths (Tensor): length of original (unpadded) sequences.
+        batch_first (bool, optional): whether batch dimension is first or not.
+            Default: ``False``.
+
+    Returns:
+        list[Tensor]: a list of Tensor objects with original lengths.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> a = paddle.ones([25, 300])
+            >>> b = paddle.ones([22, 300])
+            >>> c = paddle.ones([15, 300])
+            >>> sequences = [a, b, c]
+            >>> padded = paddle.nn.utils.rnn.pad_sequence(sequences)
+            >>> lengths = paddle.to_tensor([v.shape[0] for v in sequences])
+            >>> unpadded = paddle.nn.utils.rnn.unpad_sequence(padded, lengths)
+            >>> paddle.allclose(sequences[0], unpadded[0]).item()
+            True
+            >>> paddle.allclose(sequences[1], unpadded[1]).item()
+            True
+            >>> paddle.allclose(sequences[2], unpadded[2]).item()
+            True
+
+    """
+    if not batch_first:
+        # Transpose from T x B x * to B x T x *
+        perm = [1, 0, *list(range(2, len(padded_sequences.shape)))]
+        padded_sequences = padded_sequences.transpose(perm)
+
+    unpadded = []
+    for seq, length in zip(padded_sequences, lengths):
+        length_val = length.item()
+        unpadded.append(seq[:length_val])
+
+    return unpadded
+
+
+def pack_sequence(
+    sequences: list[Tensor],
+    enforce_sorted: bool = True,
+) -> PackedSequence:
+    r"""Packs a list of variable length Tensors.
+
+    Consecutive call of the next functions: ``pad_sequence``, ``pack_padded_sequence``.
+
+    ``sequences`` should be a list of Tensors of size ``L x *``, where `L` is
+    the length of a sequence and `*` is any number of trailing dimensions,
+    including ``0``.
+
+    For unsorted sequences, use `enforce_sorted = False`. If ``enforce_sorted``
+    is ``True``, the sequences should be sorted in the order of decreasing length.
+    ``enforce_sorted = True`` is only necessary for ONNX export.
+
+    Args:
+        sequences (list[Tensor]): A list of sequences of decreasing length.
+        enforce_sorted (bool, optional): if ``True``, checks that the input
+            contains sequences sorted by length in a decreasing order. If
+            ``False``, this condition is not checked. Default: ``True``.
+
+    Returns:
+        PackedSequence: a PackedSequence object.
+
+    Examples:
+        >>> import paddle
+
+    """
+    lengths = paddle.to_tensor([v.shape[0] for v in sequences])
+    return pack_padded_sequence(
+        pad_sequence(sequences), lengths, enforce_sorted=enforce_sorted
+    )
+
+
+def unpack_sequence(packed_sequences: PackedSequence) -> list[Tensor]:
+    r"""Unpack PackedSequence into a list of variable length Tensors.
+
+    ``packed_sequences`` should be a PackedSequence object.
+
+    Args:
+        packed_sequences (PackedSequence): A PackedSequence object.
+
+    Returns:
+        list[Tensor]: a list of Tensor objects.
+
+    Examples:
+        >>> import paddle
+
+    """
+    padded_sequences, lengths = pad_packed_sequence(
+        packed_sequences, batch_first=True
+    )
+    unpacked_sequences = unpad_sequence(
+        padded_sequences, lengths, batch_first=True
+    )
+    return unpacked_sequences

python/paddle/nn/utils/spectral_norm_hook.py

@@ -0,0 +1,245 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import paddle
+
+from .. import functional as F
+from ..layer.common import Linear
+from ..layer.conv import Conv1DTranspose, Conv2DTranspose, Conv3DTranspose
+
+if TYPE_CHECKING:
+    from typing_extensions import Never
+
+    from paddle import Tensor
+    from paddle.nn import Layer
+
+__all__ = []
+
+
+def normal_(x: Tensor, mean: float = 0.0, std: float = 1.0) -> Tensor:
+    temp_value = paddle.normal(mean, std, shape=x.shape)
+    paddle.assign(temp_value, x)
+    return x
+
+
+class SpectralNorm:
+    name: str
+    dim: int
+    n_power_iterations: int
+    eps: float
+
+    def __init__(
+        self,
+        name: str = 'weight',
+        n_power_iterations: int = 1,
+        dim: int = 0,
+        eps: float = 1e-12,
+    ) -> None:
+        self.name = name
+        self.dim = dim
+        if n_power_iterations <= 0:
+            raise ValueError(
+                'Expected n_power_iterations to be positive, but '
+                f'got n_power_iterations={n_power_iterations}'
+            )
+        self.n_power_iterations = n_power_iterations
+        self.eps = eps
+
+    def reshape_weight_to_matrix(self, weight: Tensor) -> Tensor:
+        weight_mat = weight
+        if self.dim != 0:
+            # transpose dim to front
+            weight_mat = weight_mat.transpose(
+                [self.dim]
+                + [d for d in range(weight_mat.dim()) if d != self.dim]
+            )
+
+        height = weight_mat.shape[0]
+
+        return weight_mat.reshape([height, -1])
+
+    def compute_weight(self, layer: Layer, do_power_iteration: bool) -> Tensor:
+        weight = getattr(layer, self.name + '_orig')
+        u = getattr(layer, self.name + '_u')
+        v = getattr(layer, self.name + '_v')
+        weight_mat = self.reshape_weight_to_matrix(weight)
+
+        if do_power_iteration:
+            with paddle.no_grad():
+                for _ in range(self.n_power_iterations):
+                    paddle.assign(
+                        F.normalize(
+                            paddle.matmul(
+                                weight_mat,
+                                u,
+                                transpose_x=True,
+                                transpose_y=False,
+                            ),
+                            axis=0,
+                            epsilon=self.eps,
+                        ),
+                        v,
+                    )
+
+                    paddle.assign(
+                        F.normalize(
+                            paddle.matmul(weight_mat, v),
+                            axis=0,
+                            epsilon=self.eps,
+                        ),
+                        u,
+                    )
+                if self.n_power_iterations > 0:
+                    u = u.clone()
+                    v = v.clone()
+
+        sigma = paddle.dot(u, paddle.mv(weight_mat, v))
+        weight = weight / sigma
+        return weight
+
+    def __call__(self, layer: Layer, inputs: Never) -> None:
+        setattr(
+            layer,
+            self.name,
+            self.compute_weight(layer, do_power_iteration=layer.training),
+        )
+
+    @staticmethod
+    def apply(
+        layer: Layer, name: str, n_power_iterations: int, dim: int, eps: float
+    ) -> SpectralNorm:
+        for k, hook in layer._forward_pre_hooks.items():
+            if isinstance(hook, SpectralNorm) and hook.name == name:
+                raise RuntimeError(
+                    "Cannot register two spectral_norm hooks on "
+                    f"the same parameter {name}"
+                )
+
+        fn = SpectralNorm(name, n_power_iterations, dim, eps)
+        weight = layer._parameters[name]
+
+        with paddle.no_grad():
+            weight_mat = fn.reshape_weight_to_matrix(weight)
+            h, w = weight_mat.shape
+
+            # randomly initialize u and v
+            u = layer.create_parameter([h])
+            u = normal_(u, 0.0, 1.0)
+            v = layer.create_parameter([w])
+            v = normal_(v, 0.0, 1.0)
+            u = F.normalize(u, axis=0, epsilon=fn.eps)
+            v = F.normalize(v, axis=0, epsilon=fn.eps)
+
+        # delete fn.name form parameters, otherwise you can not set attribute
+        del layer._parameters[fn.name]
+        layer.add_parameter(fn.name + "_orig", weight)
+        # still need to assign weight back as fn.name because all sorts of
+        # things may assume that it exists, e.g., when initializing weights.
+        # However, we can't directly assign as it could be an Parameter and
+        # gets added as a parameter. Instead, we register weight * 1.0 as a plain
+        # attribute.
+        setattr(layer, fn.name, weight * 1.0)
+        layer.register_buffer(fn.name + "_u", u)
+        layer.register_buffer(fn.name + "_v", v)
+        layer.register_forward_pre_hook(fn)
+        return fn
+
+
+def spectral_norm(
+    layer: Layer,
+    name: str = 'weight',
+    n_power_iterations: int = 1,
+    eps: float = 1e-12,
+    dim: int | None = None,
+) -> Layer:
+    r"""
+    Applies spectral normalization to a parameter according to the
+    following Calculation:
+
+    Step 1:
+    Generate vector U in shape of [H], and V in shape of [W].
+    While H is the :attr:`dim` th dimension of the input weights,
+    and W is the product result of remaining dimensions.
+
+    Step 2:
+    :attr:`n_power_iterations` should be a positive integer, do following
+    calculations with U and V for :attr:`power_iters` rounds.
+
+    .. math::
+
+        \mathbf{v} := \frac{\mathbf{W}^{T} \mathbf{u}}{\|\mathbf{W}^{T} \mathbf{u}\|_2}
+
+        \mathbf{u} := \frac{\mathbf{W} \mathbf{v}}{\|\mathbf{W} \mathbf{v}\|_2}
+
+    Step 3:
+    Calculate :math:`\sigma(\mathbf{W})` and normalize weight values.
+
+    .. math::
+
+        \sigma(\mathbf{W}) = \mathbf{u}^{T} \mathbf{W} \mathbf{v}
+
+        \mathbf{W} = \frac{\mathbf{W}}{\sigma(\mathbf{W})}
+
+
+    Refer to `Spectral Normalization <https://arxiv.org/abs/1802.05957>`_ .
+
+    Parameters:
+        layer(Layer): Layer of paddle, which has weight.
+        name(str, optional): Name of the weight parameter. Default: 'weight'.
+        n_power_iterations(int, optional): The number of power iterations to calculate spectral norm. Default: 1.
+        eps(float, optional): The epsilon for numerical stability in calculating norms. Default: 1e-12.
+        dim(int|None, optional): The index of dimension which should be permuted to the first before reshaping Input(Weight) to matrix, it should be set as 0 if Input(Weight) is the weight of fc layer, and should be set as 1 if Input(Weight) is the weight of conv layer. Default: None.
+
+    Returns:
+        Layer, the original layer with the spectral norm hook.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> from paddle.nn import Conv2D
+            >>> from paddle.nn.utils import spectral_norm
+            >>> paddle.seed(2023)
+            >>> conv = Conv2D(3, 1, 3)
+            >>> sn_conv = spectral_norm(conv)
+            >>> print(sn_conv)
+            Conv2D(3, 1, kernel_size=[3, 3], data_format=NCHW)
+            >>> # Conv2D(3, 1, kernel_size=[3, 3], data_format=NCHW)
+            >>> print(sn_conv.weight)
+            Tensor(shape=[1, 3, 3, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
+            [[[[ 0.01668976,  0.30305523,  0.11405435],
+               [-0.06765547, -0.50396705, -0.40925547],
+               [ 0.47344422,  0.03628403,  0.45277366]],
+              [[-0.15177251, -0.16305730, -0.15723954],
+               [-0.28081197, -0.09183260, -0.08081978],
+               [-0.40895155,  0.18298769, -0.29325116]],
+              [[ 0.21819633, -0.01822380, -0.50351536],
+               [-0.06262003,  0.17713565,  0.20517939],
+               [ 0.16659889, -0.14333329,  0.05228264]]]])
+
+    """
+
+    if dim is None:
+        if isinstance(
+            layer, (Conv1DTranspose, Conv2DTranspose, Conv3DTranspose, Linear)
+        ):
+            dim = 1
+        else:
+            dim = 0
+    SpectralNorm.apply(layer, name, n_power_iterations, dim, eps)
+    return layer

python/paddle/nn/utils/transform_parameters.py

@@ -0,0 +1,208 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from functools import reduce
+from typing import TYPE_CHECKING
+
+import paddle
+from paddle import _C_ops
+from paddle.base.framework import (
+    _create_tensor,
+    _dygraph_tracer,
+    dygraph_only,
+    in_dygraph_mode,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from paddle import Tensor
+    from paddle._typing import ShapeLike
+
+
+# input==output, inplace strategy of reshape has no cost almost
+def _inplace_reshape_dygraph(x: Tensor, shape: ShapeLike) -> None:
+    x_shape = _create_tensor(dtype='int64')
+    if in_dygraph_mode():
+        with paddle.base.dygraph.no_grad():
+            tmp_out = _C_ops.reshape(x, shape)
+            tmp_out._share_underline_tensor_to(x)
+    else:
+        _dygraph_tracer().trace_op(
+            type="reshape2",
+            inputs={'X': x},
+            outputs={'Out': x, 'XShape': x_shape},
+            attrs={'shape': shape},
+            stop_gradient=True,
+        )
+
+
+@dygraph_only
+def _stride_column(param: Tensor) -> None:
+    """
+    A tool function. Permute date of parameter as a 'columns' stride. Now, it only support 2-D parameter.
+
+    Args:
+        param(Tensor): The param that will be strided according to 'columns'.
+
+    Examples:
+       .. code-block:: pycon
+
+            >>> import paddle
+            >>> paddle.seed(100)
+
+            >>> linear = paddle.nn.Linear(2, 3)
+            >>> print(linear.weight)
+            Parameter containing:
+            Tensor(shape=[2, 3], dtype=float32, place=Place(cpu), stop_gradient=False,
+                   [[ 0.11732829, -0.64161885, -1.06996548],
+                    [ 0.03456247, -0.29862350, -0.52380574]])
+
+            >>> paddle.nn.utils._stride_column(linear.weight)
+            >>> print(linear.weight)
+
+    """
+    assert len(param.shape) == 2
+    shape = [param.shape[1], param.shape[0]]
+    with paddle.base.dygraph.no_grad():
+        reshape_var = paddle.reshape(param, shape)
+        transpose_var = paddle.transpose(reshape_var, [1, 0])
+        transpose_var._share_underline_tensor_to(param)
+
+
+@dygraph_only
+def parameters_to_vector(
+    parameters: Iterable[Tensor], name: str | None = None
+) -> Tensor:
+    """
+    Flatten parameters to a 1-D Tensor.
+
+    Args:
+        parameters(Iterable[Tensor]): Iterable Tensors that are trainable parameters of a Layer.
+        name(str, optional): The default value is None. Normally there is no need for user to set this
+            property. For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        A 1-D Tensor, which represents the parameters of a Layer.
+
+
+    Examples:
+       .. code-block:: pycon
+
+            >>> import paddle
+            >>> paddle.seed(2023)
+            >>> linear = paddle.nn.Linear(10, 15)
+
+            >>> t = paddle.nn.utils.parameters_to_vector(linear.parameters())
+            >>> print(t.shape)
+            paddle.Size([165])
+
+    """
+    dtype = parameters[0].dtype
+    origin_shapes = []
+    for param in parameters:
+        origin_shapes.append(param.shape)
+        _inplace_reshape_dygraph(param, [-1])
+
+    out = _create_tensor(dtype=dtype)
+    if in_dygraph_mode():
+        with paddle.base.dygraph.no_grad():
+            tmp = _C_ops.concat(parameters, 0)
+            tmp._share_underline_tensor_to(out)
+    else:
+        _dygraph_tracer().trace_op(
+            type='concat',
+            inputs={'X': parameters},
+            outputs={'Out': [out]},
+            attrs={'axis': 0},
+            stop_gradient=True,
+        )
+    for i, param in enumerate(parameters):
+        _inplace_reshape_dygraph(param, origin_shapes[i])
+    out.stop_gradient = False
+    return out
+
+
+@dygraph_only
+def vector_to_parameters(
+    vec: Tensor, parameters: Iterable[Tensor], name: str | None = None
+) -> None:
+    """
+    Transform a 1-D Tensor to the input ``parameters`` .
+
+    Args:
+        vec (Tensor): A 1-D Tensor, which will be sliced and copied to the input ``parameters`` .
+        parameters (Iterable[Tensor]): Iterable Tensors that are trainable parameters of a Layer.
+        name(str, optional): The default value is None. Normally there is no need for user to set this
+            property. For more information, please refer to :ref:`api_guide_Name`.
+
+    Examples:
+       .. code-block:: pycon
+
+            >>> import paddle
+            >>> weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Constant(3.0))
+            >>> linear1 = paddle.nn.Linear(10, 15, weight_attr)
+
+            >>> vec = paddle.nn.utils.parameters_to_vector(linear1.parameters())
+
+            >>> linear2 = paddle.nn.Linear(10, 15)
+            >>> # copy weight of linear1 to linear2
+            >>> paddle.nn.utils.vector_to_parameters(vec, linear2.parameters())
+            >>> print((linear1.weight == linear2.weight).all())
+            Tensor(shape=[], dtype=bool, place=Place(cpu), stop_gradient=True,
+            True)
+    """
+    assert len(vec.shape) == 1
+    origin_shapes = []
+    sections = []
+    total_elements = 0
+    for param in parameters:
+        shape = param.shape
+        origin_shapes.append(shape)
+        numel = reduce(lambda x, y: x * y, shape, 1)
+        total_elements += numel
+        sections.append(numel)
+
+    if len(sections) == 1:
+        sections.append(0)
+
+    if in_dygraph_mode():
+        with paddle.base.dygraph.no_grad():
+            res = []
+            if total_elements == vec.shape[0]:
+                res = _C_ops.split(vec, sections, 0)
+            elif total_elements < vec.shape[0]:
+                pointer = 0
+                for section in sections:
+                    res.append(vec[pointer : pointer + section])
+                    pointer += section
+            else:
+                raise ValueError(
+                    "The total_elements of vec should be equal to or larger than the number of elements in parameters."
+                )
+            for i in range(0, len(parameters)):
+                res[i]._share_underline_tensor_to(parameters[i])
+    else:
+        _dygraph_tracer().trace_op(
+            type='split',
+            inputs={'X': [vec]},
+            outputs={'Out': parameters},
+            attrs={'axis': 0, 'sections': sections},
+            stop_gradient=True,
+        )
+
+    for i, param in enumerate(parameters):
+        _inplace_reshape_dygraph(param, origin_shapes[i])

python/paddle/nn/utils/weight_norm_hook.py

@@ -0,0 +1,263 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import paddle
+from paddle import _C_ops
+from paddle.utils.decorator_utils import param_one_alias
+
+from ...base.data_feeder import check_variable_and_dtype
+from ...base.layer_helper import LayerHelper
+from ...framework import in_dynamic_or_pir_mode
+
+if TYPE_CHECKING:
+    from typing_extensions import Never
+
+    from paddle import Tensor
+    from paddle.nn import Layer
+
+__all__ = []
+
+
+def l2_norm(
+    x: Tensor, axis: int, epsilon: float = 1e-12, name: str | None = None
+) -> Tensor:
+    if len(x.shape) == 1:
+        axis = 0
+
+    if in_dynamic_or_pir_mode():
+        out, norm = _C_ops.norm(x, 1 if axis is None else axis, epsilon, False)
+        return paddle.squeeze(norm, axis=[axis])
+
+    check_variable_and_dtype(x, "X", ("float32", "float64"), "norm")
+
+    helper = LayerHelper("l2_normalize", **locals())
+    out = helper.create_variable_for_type_inference(dtype=x.dtype)
+    norm = helper.create_variable_for_type_inference(dtype=x.dtype)
+    helper.append_op(
+        type="norm",
+        inputs={"X": x},
+        outputs={"Out": out, "Norm": norm},
+        attrs={
+            "axis": 1 if axis is None else axis,
+            "epsilon": epsilon,
+        },
+    )
+    return paddle.squeeze(norm, axis=[axis])
+
+
+def norm_except_dim(p: Tensor, dim: int) -> Tensor:
+    shape = p.shape
+    ndims = len(shape)
+    if dim == -1:
+        return paddle.sqrt(paddle.sum(paddle.square(p)) + 1e-12)
+    elif dim == 0:
+        p_matrix = paddle.reshape(p, (shape[0], -1))
+        return l2_norm(p_matrix, axis=1)
+    elif dim == ndims - 1:
+        p_matrix = paddle.reshape(p, (-1, shape[-1]))
+        return l2_norm(p_matrix, axis=0)
+    else:
+        perm = list(range(ndims))
+        perm[0] = dim
+        perm[dim] = 0
+        p_transposed = paddle.transpose(p, perm)
+        return norm_except_dim(p_transposed, 0)
+
+
+def _weight_norm(v: Tensor, g: Tensor, dim: int) -> Tensor:
+    shape = v.shape
+    ndims = len(shape)
+
+    if dim == -1:
+        v_normalized = v / (paddle.sqrt(paddle.sum(paddle.square(v))) + 1e-12)
+    elif dim == 0:
+        p_matrix = paddle.reshape(v, (shape[0], -1))
+        v_normalized = paddle.nn.functional.normalize(p_matrix, axis=1)
+        v_normalized = paddle.reshape(v_normalized, shape)
+    elif dim == ndims - 1:
+        p_matrix = paddle.reshape(v, (-1, shape[-1]))
+        v_normalized = paddle.nn.functional.normalize(p_matrix, axis=0)
+        v_normalized = paddle.reshape(v_normalized, shape)
+    else:
+        perm = list(range(ndims))
+        perm[0] = dim
+        perm[dim] = 0
+        p_transposed = paddle.transpose(v, perm)
+        transposed_shape = p_transposed.shape
+        p_matrix = paddle.reshape(p_transposed, (p_transposed.shape[0], -1))
+        v_normalized = paddle.nn.functional.normalize(p_matrix, axis=1)
+        v_normalized = paddle.reshape(v_normalized, transposed_shape)
+        v_normalized = paddle.transpose(v_normalized, perm)
+    weight = paddle.tensor.math._multiply_with_axis(
+        v_normalized, g, axis=dim if dim is not None else -1
+    )
+    return weight
+
+
+class WeightNorm:
+    name: str
+    dim: int
+
+    def __init__(self, name: str, dim: int) -> None:
+        if dim is None:
+            dim = -1
+        self.name = name
+        self.dim = dim
+
+    def compute_weight(self, layer: Layer) -> Tensor:
+        g = getattr(layer, self.name + '_g')
+        v = getattr(layer, self.name + '_v')
+        return _weight_norm(v, g, self.dim)
+
+    @staticmethod
+    def apply(layer: Layer, name: str, dim: int) -> WeightNorm:
+        for k, hook in layer._forward_pre_hooks.items():
+            if isinstance(hook, WeightNorm) and hook.name == name:
+                raise RuntimeError(
+                    "Cannot register two weight_norm hooks on "
+                    f"the same parameter {name}"
+                )
+
+        if dim is None:
+            dim = -1
+
+        # support dim is negative number, (dim = -1) == (dim = None)
+        weight_dim = len(layer._parameters[name].shape)
+        assert dim < weight_dim and dim >= -1 * weight_dim, (
+            "dim must set between [-R, R), R means the dimension of weight."
+        )
+        if dim != -1:
+            dim = (dim + weight_dim) % weight_dim
+
+        fn = WeightNorm(name, dim)
+
+        w = getattr(layer, name)
+        del layer._parameters[name]
+
+        g_var = norm_except_dim(w, dim)
+        v = layer.create_parameter(w.shape, dtype=w.dtype)
+        layer.add_parameter(name + "_v", v)
+        g = layer.create_parameter(g_var.shape, dtype=g_var.dtype)
+        layer.add_parameter(name + '_g', g)
+        with paddle.no_grad():
+            paddle.assign(w, v)
+            paddle.assign(g_var, g)
+        setattr(layer, name, fn.compute_weight(layer))
+
+        layer.register_forward_pre_hook(fn)
+        return fn
+
+    def remove(self, layer: Layer) -> None:
+        w_var = self.compute_weight(layer)
+        delattr(layer, self.name)
+        del layer._parameters[self.name + '_g']
+        del layer._parameters[self.name + '_v']
+        w = layer.create_parameter(w_var.shape, dtype=w_var.dtype)
+        layer.add_parameter(self.name, w)
+        with paddle.no_grad():
+            paddle.assign(w_var, w)
+
+    def __call__(self, layer: Layer, inputs: Never) -> None:
+        setattr(layer, self.name, self.compute_weight(layer))
+
+
+@param_one_alias(["layer", "module"])
+def weight_norm(layer: Layer, name: str = 'weight', dim: int = 0) -> Layer:
+    r"""
+    Applies weight normalization to a parameter according to the
+    following formula:
+
+    .. math::
+
+        \mathbf{w} = g \dfrac{v}{\|v\|}
+
+    Weight normalization is a reparameterization of the weight vectors in a neural network that
+    decouples the magnitude of those weight vectors from their direction. Weight normalization
+    replaces the parameter specified by ``name`` (eg: 'weight') with two parameters: one parameter
+    specifying the magnitude (eg: 'weight_g') and one parameter specifying the direction
+    (eg: 'weight_v'). Weight normalization has been implemented as discussed in this paper:
+
+    `Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks
+    <https://arxiv.org/pdf/1602.07868.pdf>`_.
+
+    Parameters:
+        layer(Layer): Layer of paddle, which has weight.
+            Alias: ``module``.
+        name(str, optional): Name of the weight parameter. Default: 'weight'.
+        dim(int, optional): Dimension over which to compute the norm. Dim is a non-negative number
+              which is less than the rank of weight Tensor. For Example, dim can be chosen from 0,
+              1, 2, 3 for convolution whose weight shape is [cout, cin, kh, kw] and rank is 4.
+              If dim is set to None, meaning that all elements will be normalized. Default: 0.
+
+    Returns:
+        Origin layer with weight norm hook.
+
+    Examples:
+        .. code-block:: pycon
+
+          >>> from paddle.nn import Conv2D
+          >>> from paddle.nn.utils import weight_norm
+
+          >>> conv = Conv2D(3, 5, 3)
+          >>> wn = weight_norm(conv)
+          >>> print(conv.weight_g.shape)
+          paddle.Size([5])
+          >>> print(conv.weight_v.shape)
+          paddle.Size([5, 3, 3, 3])
+    """
+    WeightNorm.apply(layer, name, dim)
+    return layer
+
+
+def remove_weight_norm(layer: Layer, name: str = 'weight') -> Layer:
+    """
+    remove weight normalization from layer.
+
+    Parameters:
+        layer(Layer): Layer of paddle, which has weight.
+        name(str, optional): Name of the weight parameter. Default: 'weight'.
+
+    Returns:
+        Layer, the origin layer without weight norm
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> from paddle.nn import Conv2D
+            >>> from paddle.nn.utils import weight_norm, remove_weight_norm
+            >>> paddle.seed(2023)
+
+            >>> conv = Conv2D(3, 5, 3)
+            >>> wn = weight_norm(conv)
+            >>> print(conv.weight_g)
+            Parameter containing:
+            Tensor(shape=[5], dtype=float32, place=Place(cpu), stop_gradient=False,
+                   [1.35883713, 1.32126212, 1.56303072, 1.20874095, 1.22893476])
+            >>> remove_weight_norm(conv)
+            >>> # The following is the effect after removing the weight norm:
+            >>> # print(conv.weight_g)
+            >>> # AttributeError: 'Conv2D' object has no attribute 'weight_g'
+    """
+    for k, hook in layer._forward_pre_hooks.items():
+        if isinstance(hook, WeightNorm) and hook.name == name:
+            hook.remove(layer)
+            del layer._forward_pre_hooks[k]
+            return layer
+
+    raise ValueError(f"weight_norm of '{name}' not found in {layer}")