chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,363 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops
|
||||
from paddle.utils.decorator_utils import param_one_alias
|
||||
|
||||
from ...base.data_feeder import check_variable_and_dtype
|
||||
from ...base.layer_helper import LayerHelper
|
||||
from ...common_ops_import import Variable
|
||||
from ...framework import in_dynamic_or_pir_mode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from paddle import Tensor
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
@param_one_alias(["x", "input"])
|
||||
def one_hot(
|
||||
x: Tensor,
|
||||
num_classes: int = -1,
|
||||
name: str | None = None,
|
||||
) -> Tensor:
|
||||
"""
|
||||
|
||||
The operator converts each id in the input `x` to an one-hot vector with a
|
||||
`num_classes` length. The value in the vector dimension corresponding to the id
|
||||
is 1, and the value in the remaining dimension is 0.
|
||||
|
||||
The shape of output Tensor is generated by appending `num_classes` dimension
|
||||
behind the last dimension of the `x` shape.
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
Example 1:
|
||||
|
||||
input:
|
||||
x.shape = [4]
|
||||
x.data = [1, 1, 3, 0]
|
||||
num_classes = 4
|
||||
|
||||
output:
|
||||
Out.shape = [4, 4]
|
||||
Out.data = [[0., 1., 0., 0.],
|
||||
[0., 1., 0., 0.],
|
||||
[0., 0., 0., 1.],
|
||||
[1., 0., 0., 0.]]
|
||||
|
||||
Example 2:
|
||||
|
||||
input:
|
||||
x.shape = [4]
|
||||
x.data = [1, 1, 5, 0]
|
||||
num_classes = 4
|
||||
|
||||
output: Throw an exception for Illegal value
|
||||
The second dimension in X is 5, which is greater than num_classes,
|
||||
so it throws an exception.
|
||||
|
||||
|
||||
.. note::
|
||||
Alias Support: The parameter name ``input`` can be used as an alias for ``x``.
|
||||
For example, ``one_hot(input=tensor_x, ...)`` is equivalent to ``one_hot(x=tensor_x, ...)``.
|
||||
|
||||
|
||||
Args:
|
||||
x (Tensor): Tensor with shape :math:`[N_1, N_2, ..., N_k]` ,
|
||||
which contains at least one dimension. The data type is int32 or int64.
|
||||
Alias: ``input``.
|
||||
num_classes (int): An integer defining the `num_classes` of the one hot dimension. If input `x`
|
||||
is word id, `num_classes` is generally the dictionary size. Default value: -1.
|
||||
name (str|None, optional): For detailed information, please refer
|
||||
to :ref:`api_guide_Name`. Usually name is no need to set and
|
||||
None by default.
|
||||
|
||||
Returns:
|
||||
Tensor, The one-hot representations of `x`. A Tensor with type float32.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> # Correspond to the first example above, where label.shape is 4 and one_hot_label.shape is [4, 4].
|
||||
>>> label = paddle.to_tensor([1, 1, 3, 0], dtype='int64')
|
||||
>>> print(label.shape)
|
||||
paddle.Size([4])
|
||||
>>> one_hot_label = paddle.nn.functional.one_hot(label, num_classes=4)
|
||||
>>> print(one_hot_label.shape)
|
||||
paddle.Size([4, 4])
|
||||
>>> print(one_hot_label)
|
||||
Tensor(shape=[4, 4], dtype=float32, place=Place(cpu), stop_gradient=True,
|
||||
[[0., 1., 0., 0.],
|
||||
[0., 1., 0., 0.],
|
||||
[0., 0., 0., 1.],
|
||||
[1., 0., 0., 0.]])
|
||||
|
||||
"""
|
||||
if not isinstance(num_classes, paddle.pir.Value) and num_classes == -1:
|
||||
num_classes = x.max() + 1
|
||||
if in_dynamic_or_pir_mode():
|
||||
return _C_ops.one_hot(x, num_classes)
|
||||
else:
|
||||
check_variable_and_dtype(x, 'input', ['int32', 'int64'], 'one_hot_v2')
|
||||
helper = LayerHelper("one_hot_v2", **locals())
|
||||
|
||||
one_hot_out = helper.create_variable_for_type_inference(dtype='float32')
|
||||
if not isinstance(num_classes, Variable):
|
||||
# user attribute
|
||||
inputs = {'X': x}
|
||||
attrs = {'depth': num_classes, 'allow_out_of_range': False}
|
||||
else:
|
||||
num_classes.stop_gradient = True
|
||||
inputs = {'X': x, 'depth_tensor': num_classes}
|
||||
attrs = {'allow_out_of_range': False}
|
||||
helper.append_op(
|
||||
type="one_hot_v2",
|
||||
inputs=inputs,
|
||||
attrs=attrs,
|
||||
outputs={'Out': one_hot_out},
|
||||
stop_gradient=True,
|
||||
)
|
||||
return one_hot_out
|
||||
|
||||
|
||||
def embedding_renorm_(
|
||||
x: Tensor, weight: Tensor, max_norm: float, norm_type: float = 2.0
|
||||
) -> Tensor:
|
||||
r"""
|
||||
Renorm the weight of embedding with respect to the provided :attr:`max_norm` and :attr:`norm_type` .
|
||||
|
||||
Note:
|
||||
In the dynamic graph mode, the input weight will be updated in-place, and the return value will be the changed weight.
|
||||
|
||||
Args:
|
||||
x (Tensor): A Tensor with type int32/int64, which contains the id information. The value of the input id should
|
||||
satisfy :math:`0<= id < weight.shape[0]` .
|
||||
weight (Tensor): The weight. A Tensor with shape of lookup table parameter. It should have two elements which
|
||||
indicates the size of the dictionary of embeddings and the size of each embedding vector respectively.
|
||||
max_norm (float): The maximum norm for each embedding vector.
|
||||
norm_type (float, optional): The p of the p-norm to compute for the max_norm option. Default: 2.0.
|
||||
|
||||
Returns:
|
||||
Tensor, The updated weight. The data type is the same as :attr:`weight`.
|
||||
"""
|
||||
with paddle.set_grad_enabled(False):
|
||||
unique_x = paddle.unique(x)
|
||||
selected_rows = paddle.index_select(weight, unique_x)
|
||||
norm = paddle.norm(selected_rows, p=norm_type, axis=1, keepdim=True)
|
||||
mask = norm > max_norm
|
||||
scale = max_norm / (norm + 1e-7)
|
||||
scale = paddle.where(mask, scale, paddle.ones_like(scale))
|
||||
scale = paddle.expand_as(scale, selected_rows)
|
||||
updated_rows = selected_rows * scale
|
||||
paddle.scatter_(weight, unique_x, updated_rows, overwrite=True)
|
||||
return weight
|
||||
|
||||
|
||||
@param_one_alias(["x", "input"])
|
||||
def embedding(
|
||||
x: Tensor,
|
||||
weight: Tensor,
|
||||
padding_idx: int | None = None,
|
||||
max_norm: float | None = None,
|
||||
norm_type: float = 2.0,
|
||||
sparse: bool = False,
|
||||
scale_grad_by_freq: bool = False,
|
||||
name: str | None = None,
|
||||
) -> Tensor:
|
||||
r"""
|
||||
Used to lookup embeddings vector of ids provided by :attr:`x` .
|
||||
|
||||
The shape of output Tensor is generated by appending the last dimension of the input Tensor shape
|
||||
with embedding size.
|
||||
|
||||
Note:
|
||||
The id in :attr:`x` must satisfy :math:`0 <= id < weight.shape[0]` ,
|
||||
otherwise the program will throw an exception and exit.
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
x is a Tensor.
|
||||
padding_idx = -1
|
||||
x.data = [[1, 3], [2, 4], [4, 127]]
|
||||
x.shape = [3, 2]
|
||||
weight.shape = [128, 16]
|
||||
output is a Tensor:
|
||||
out.shape = [3, 2, 16]
|
||||
out.data = [[[0.129435295, 0.244512452, ..., 0.436322452],
|
||||
[0.345421456, 0.524563927, ..., 0.144534654]],
|
||||
[[0.345249859, 0.124939536, ..., 0.194353745],
|
||||
[0.945345345, 0.435394634, ..., 0.435345365]],
|
||||
[[0.945345345, 0.435394634, ..., 0.435345365],
|
||||
[0.0, 0.0, ..., 0.0 ]]] # padding data
|
||||
|
||||
The input padding_idx is less than 0, it is automatically converted to padding_idx = -1 + 128 = 127
|
||||
It will pad all-zero data when id is 127.
|
||||
|
||||
.. note::
|
||||
Alias Support: The parameter name ``input`` can be used as an alias for ``x``.
|
||||
For example, ``embedding(input=tensor_x, ...)`` is equivalent to ``embedding(x=tensor_x, ...)``.
|
||||
|
||||
Args:
|
||||
x (Tensor): A Tensor with type int32/int64, which contains the id information. The value of the input id should
|
||||
satisfy :math:`0 <= id < weight.shape[0]` .
|
||||
Alias: ``input``.
|
||||
weight (Tensor): The weight. A Tensor with shape of lookup table parameter. It should have two elements which
|
||||
indicates the size of the dictionary of embeddings and the size of each embedding vector respectively.
|
||||
sparse (bool, optional): The flag indicating whether to use sparse update. This parameter only
|
||||
affects the performance of the backwards gradient update. It is recommended to set
|
||||
True because sparse update is faster. But some optimizers does not support sparse update,
|
||||
such as :ref:`api_paddle_optimizer_adadelta_Adadelta` , :ref:`api_paddle_optimizer_adamax_Adamax` , :ref:`api_paddle_optimizer_lamb_Lamb`.
|
||||
In these cases, sparse must be False. Default: False.
|
||||
padding_idx (int|None, optional): padding_idx needs to be in the interval [-weight.shape[0], weight.shape[0]).
|
||||
If :math:`padding\_idx < 0`, the :math:`padding\_idx` will automatically be converted
|
||||
to :math:`weight.shape[0] + padding\_idx` . It will output all-zero padding data whenever lookup
|
||||
encounters :math:`padding\_idx` in id. And the padding data will not be updated while training.
|
||||
If set None, it makes no effect to output. Default: None.
|
||||
max_norm (float, optional): If provided, will renormalize the embedding vectors to have a norm larger than
|
||||
:attr:`max\_norm` . It will inplace update the input embedding weight in dynamic graph mode. Default: None.
|
||||
norm_type (float, optional): The p of the p-norm to compute for the max_norm option. Default: 2.0.
|
||||
scale_grad_by_freq (bool, optional): Indicating whether to scale the gradients by the inverse frequency of the
|
||||
word ids in input `x`. Default: False.
|
||||
name (str|None, optional): For detailed information, please refer
|
||||
to :ref:`api_guide_Name`. Usually name is no need to set and
|
||||
None by default.
|
||||
|
||||
Returns:
|
||||
Tensor, Embedding Tensor mapped by x. The data type is the same as :attr:`weight`.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> import paddle.nn as nn
|
||||
|
||||
>>> x0 = paddle.arange(3, 6).reshape((3, 1)).astype(paddle.int64)
|
||||
>>> w0 = paddle.full(shape=(10, 3), fill_value=2).astype(paddle.float32)
|
||||
|
||||
>>> x = paddle.to_tensor(x0, stop_gradient=False)
|
||||
>>> print(x.numpy())
|
||||
[[3]
|
||||
[4]
|
||||
[5]]
|
||||
>>> print(x.shape)
|
||||
paddle.Size([3, 1])
|
||||
|
||||
>>> w = paddle.to_tensor(w0, stop_gradient=False)
|
||||
>>> print(w.numpy())
|
||||
[[2. 2. 2.]
|
||||
[2. 2. 2.]
|
||||
[2. 2. 2.]
|
||||
[2. 2. 2.]
|
||||
[2. 2. 2.]
|
||||
[2. 2. 2.]
|
||||
[2. 2. 2.]
|
||||
[2. 2. 2.]
|
||||
[2. 2. 2.]
|
||||
[2. 2. 2.]]
|
||||
>>> print(w.shape)
|
||||
paddle.Size([10, 3])
|
||||
|
||||
>>> emb = nn.functional.embedding(x=x, weight=w, sparse=True, name="embedding")
|
||||
>>> print(emb.numpy())
|
||||
[[[2. 2. 2.]]
|
||||
[[2. 2. 2.]]
|
||||
[[2. 2. 2.]]]
|
||||
>>> print(emb.shape)
|
||||
paddle.Size([3, 1, 3])
|
||||
|
||||
"""
|
||||
padding_idx = (
|
||||
-1
|
||||
if padding_idx is None
|
||||
else (
|
||||
padding_idx if padding_idx >= 0 else (weight.shape[0] + padding_idx)
|
||||
)
|
||||
)
|
||||
|
||||
if weight.shape[0] != 0 and (
|
||||
padding_idx >= weight.shape[0] or padding_idx < -weight.shape[0]
|
||||
):
|
||||
raise ValueError(
|
||||
f"padding_idx must be within [-{weight.shape[0]}, {weight.shape[0]})"
|
||||
)
|
||||
|
||||
if max_norm and weight.size != 0:
|
||||
weight = embedding_renorm_(
|
||||
x, weight, max_norm=max_norm, norm_type=norm_type
|
||||
)
|
||||
|
||||
if scale_grad_by_freq:
|
||||
if sparse:
|
||||
raise AttributeError(
|
||||
"scale_grad_by_freq = True is not supported with sparse update."
|
||||
)
|
||||
if in_dynamic_or_pir_mode():
|
||||
return _C_ops.embedding_with_scaled_gradient(x, weight, padding_idx)
|
||||
else:
|
||||
helper = LayerHelper('embedding_with_scaled_gradient', **locals())
|
||||
dtype = helper.input_dtype(input_param_name='weight')
|
||||
|
||||
check_variable_and_dtype(
|
||||
x,
|
||||
'input',
|
||||
['uint8', 'int8', 'int16', 'int32', 'int64'],
|
||||
'embedding_with_scaled_gradient',
|
||||
)
|
||||
tmp = helper.create_variable_for_type_inference(dtype)
|
||||
|
||||
helper.append_op(
|
||||
type='embedding_with_scaled_gradient',
|
||||
inputs={'x': x, 'weight': weight},
|
||||
outputs={'out': tmp},
|
||||
attrs={'padding_idx': padding_idx},
|
||||
)
|
||||
return tmp
|
||||
else:
|
||||
if in_dynamic_or_pir_mode():
|
||||
return _C_ops.embedding(x, weight, padding_idx, sparse)
|
||||
else:
|
||||
helper = LayerHelper('embedding', **locals())
|
||||
dtype = helper.input_dtype(input_param_name='weight')
|
||||
|
||||
check_variable_and_dtype(
|
||||
x,
|
||||
'input',
|
||||
['uint8', 'int8', 'int16', 'int32', 'int64'],
|
||||
'embedding',
|
||||
)
|
||||
|
||||
is_distributed = False
|
||||
remote_prefetch = sparse and (not is_distributed)
|
||||
|
||||
tmp = helper.create_variable_for_type_inference(dtype)
|
||||
|
||||
helper.append_op(
|
||||
type='lookup_table_v2',
|
||||
inputs={'Ids': x, 'W': weight},
|
||||
outputs={'Out': tmp},
|
||||
attrs={
|
||||
'is_sparse': sparse,
|
||||
'is_distributed': is_distributed,
|
||||
'remote_prefetch': remote_prefetch,
|
||||
'padding_idx': padding_idx,
|
||||
},
|
||||
)
|
||||
return tmp
|
||||
Reference in New Issue
Block a user