chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,266 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import numpy as np
|
||||
|
||||
from paddle.base.framework import static_only
|
||||
|
||||
# TODO: define loss functions of neural network
|
||||
from paddle.base.layer_helper import LayerHelper
|
||||
from paddle.base.param_attr import ParamAttr
|
||||
from paddle.nn.initializer import Assign
|
||||
|
||||
from ...base.data_feeder import check_variable_and_dtype
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
# FIXME(wuyi): let docstring_checker.py understand @autodoc.
|
||||
# For now, the comments in c++ use types like Tensor, but in python side
|
||||
# the type is often "Variable", and arguments may vary.
|
||||
@static_only
|
||||
def nce(
|
||||
input,
|
||||
label,
|
||||
num_total_classes,
|
||||
sample_weight=None,
|
||||
param_attr=None,
|
||||
bias_attr=None,
|
||||
num_neg_samples=None,
|
||||
name=None,
|
||||
sampler="uniform",
|
||||
custom_dist=None,
|
||||
seed=0,
|
||||
is_sparse=False,
|
||||
):
|
||||
"""
|
||||
:api_attr: Static Graph
|
||||
|
||||
Compute and return the noise-contrastive estimation training loss. See `Noise-contrastive estimation: A new estimation principle
|
||||
for unnormalized statistical models <http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf>`_.
|
||||
By default this operator uses a uniform distribution for sampling.
|
||||
|
||||
Args:
|
||||
input (Tensor): Input tensor, 2-D tensor with shape [batch_size, dim],
|
||||
and data type is float32 or float64.
|
||||
label (Tensor): Input label, 2-D tensor with shape [batch_size, num_true_class],
|
||||
and data type is int64.
|
||||
num_total_classes (int): Total number of classes in all samples.
|
||||
sample_weight (Tensor|None): A Tensor of shape [batch_size, 1]
|
||||
storing a weight for each sample. The default weight for each
|
||||
sample is 1.0.
|
||||
param_attr (ParamAttr|None): To specify the weight parameter attribute.
|
||||
Default: None, which means the default weight parameter property is
|
||||
used. See usage for details in :ref:`api_paddle_ParamAttr` .
|
||||
bias_attr (ParamAttr|None): To specify the bias parameter attribute.
|
||||
Default: None, which means the default bias parameter property is
|
||||
used. See usage for details in :ref:`api_paddle_ParamAttr` .
|
||||
num_neg_samples (int): The number of negative classes. The default value is 10.
|
||||
name(str|None): For detailed information, please refer to
|
||||
:ref:`api_guide_Name` . Usually name is no need to set and None by default.
|
||||
sampler (str, optional): The sampler used to sample class from negative classes.
|
||||
It can be 'uniform', 'log_uniform' or 'custom_dist'.
|
||||
default: 'uniform'.
|
||||
custom_dist (nd.array|None): A numpy ndarray with size=num_total_classes.
|
||||
It is used when sampler is set to 'custom_dist'.
|
||||
custom_dist[i] is the probability of i-th class to be sampled.
|
||||
default: None.
|
||||
seed (int, optional): The seed used in sampler. Default 0, means no random seed.
|
||||
is_sparse(bool, optional): The flag indicating whether to use sparse update,
|
||||
the weight@GRAD and bias@GRAD will be changed to SelectedRows. Default False.
|
||||
|
||||
Returns:
|
||||
Tensor: The output nce loss.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> # doctest: +SKIP("paddle.static.nn.nce doesn't support PIR mode")
|
||||
>>> import paddle
|
||||
>>> import numpy as np
|
||||
|
||||
>>> paddle.enable_static()
|
||||
|
||||
>>> window_size = 5
|
||||
>>> words = []
|
||||
>>> for i in range(window_size):
|
||||
... words.append(paddle.static.data(name='word_{0}'.format(i), shape=[-1, 1], dtype='int64'))
|
||||
|
||||
>>> dict_size = 10000
|
||||
>>> label_word = int(window_size / 2) + 1
|
||||
|
||||
>>> embs = []
|
||||
>>> for i in range(window_size):
|
||||
... if i == label_word:
|
||||
... continue
|
||||
...
|
||||
... emb = paddle.static.nn.embedding(input=words[i], size=[dict_size, 32], param_attr='embed', is_sparse=True)
|
||||
... embs.append(emb)
|
||||
|
||||
>>> embs = paddle.concat(x=embs, axis=1) # concat from 4 * [(-1, 1, 32)] to (-1, 4, 32)
|
||||
>>> embs = paddle.reshape(x=embs, shape=(-1, 4 * 32)) # reshape to (batch_size = -1, dim = 4*32)
|
||||
>>> loss = paddle.static.nn.nce(
|
||||
... input=embs, label=words[label_word], num_total_classes=dict_size, param_attr='nce.w_0', bias_attr='nce.b_0'
|
||||
... )
|
||||
|
||||
# or use custom distribution
|
||||
>>> dist = np.array([0.05, 0.5, 0.1, 0.3, 0.05])
|
||||
>>> loss = paddle.static.nn.nce(
|
||||
... input=embs,
|
||||
... label=words[label_word],
|
||||
... num_total_classes=5,
|
||||
... param_attr='nce.w_1',
|
||||
... bias_attr='nce.b_1',
|
||||
... num_neg_samples=3,
|
||||
... sampler="custom_dist",
|
||||
... custom_dist=dist,
|
||||
... )
|
||||
"""
|
||||
helper = LayerHelper('nce', **locals())
|
||||
check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'nce')
|
||||
check_variable_and_dtype(label, 'label', ['int64'], 'nce')
|
||||
|
||||
if input.ndim != 2:
|
||||
raise ValueError(
|
||||
f'The rank of `input` must be 2, but received {input.ndim}.'
|
||||
)
|
||||
|
||||
dim = input.shape[1]
|
||||
num_true_class = label.shape[1]
|
||||
w = helper.create_parameter(
|
||||
attr=helper.param_attr,
|
||||
shape=[num_total_classes, dim],
|
||||
is_bias=False,
|
||||
dtype=input.dtype,
|
||||
)
|
||||
inputs = {}
|
||||
if helper.bias_attr:
|
||||
b = helper.create_parameter(
|
||||
attr=helper.bias_attr,
|
||||
shape=[num_total_classes, 1],
|
||||
is_bias=True,
|
||||
dtype=input.dtype,
|
||||
)
|
||||
inputs['Bias'] = b
|
||||
cost = helper.create_variable_for_type_inference(dtype=input.dtype)
|
||||
sample_logits = helper.create_variable_for_type_inference(dtype=input.dtype)
|
||||
sample_labels = helper.create_variable_for_type_inference(dtype=label.dtype)
|
||||
|
||||
inputs['Input'] = input
|
||||
inputs['Label'] = label
|
||||
inputs['Weight'] = w
|
||||
inputs['SampleWeight'] = sample_weight if sample_weight is not None else []
|
||||
|
||||
if sampler == "uniform":
|
||||
sampler = 0
|
||||
elif sampler == "log_uniform":
|
||||
sampler = 1
|
||||
elif sampler == "custom_dist":
|
||||
assert custom_dist is not None
|
||||
|
||||
custom_dist_len = num_total_classes
|
||||
alias_probs_ = [0] * custom_dist_len
|
||||
alias_ = [0] * custom_dist_len
|
||||
bigs = []
|
||||
littles = []
|
||||
for i in range(custom_dist_len):
|
||||
normal_prob = custom_dist[i] * custom_dist_len
|
||||
if normal_prob - 1.0 > 0:
|
||||
bigs.append((i, normal_prob))
|
||||
elif 1.0 - normal_prob > 0:
|
||||
littles.append((i, normal_prob))
|
||||
else:
|
||||
alias_probs_[i] = normal_prob
|
||||
alias_[i] = -1
|
||||
|
||||
while len(bigs) and len(littles):
|
||||
big = bigs.pop(0)
|
||||
little = littles.pop(0)
|
||||
|
||||
big_idx = big[0]
|
||||
big_prob = big[1]
|
||||
|
||||
alias_probs_[little[0]] = little[1]
|
||||
alias_[little[0]] = big_idx
|
||||
big_left = big[1] + little[1] - 1
|
||||
if big_left - 1.0 > 0:
|
||||
bigs.append((big_idx, big_left))
|
||||
elif 1.0 - big_left > 0:
|
||||
littles.append((big_idx, big_left))
|
||||
else:
|
||||
alias_probs_[big_idx] = big_left
|
||||
alias_[big_idx] = -1
|
||||
|
||||
if len(bigs):
|
||||
big = bigs.pop(0)
|
||||
alias_probs_[big[0]] = 1.0
|
||||
alias_[big[0]] = -1
|
||||
if len(littles):
|
||||
little = littles.pop(0)
|
||||
alias_probs_[little[0]] = 1.0
|
||||
alias_[little[0]] = -1
|
||||
|
||||
def _init_by_numpy_array(numpy_array):
|
||||
ret = helper.create_parameter(
|
||||
attr=ParamAttr(),
|
||||
shape=numpy_array.shape,
|
||||
dtype=numpy_array.dtype,
|
||||
default_initializer=Assign(numpy_array),
|
||||
)
|
||||
ret.stop_gradient = True
|
||||
return ret
|
||||
|
||||
inputs['CustomDistProbs'] = _init_by_numpy_array(
|
||||
np.array(custom_dist).astype('float32')
|
||||
)
|
||||
inputs['CustomDistAlias'] = _init_by_numpy_array(
|
||||
np.array(alias_).astype('int32')
|
||||
)
|
||||
inputs['CustomDistAliasProbs'] = _init_by_numpy_array(
|
||||
np.array(alias_probs_).astype('float32')
|
||||
)
|
||||
sampler = 2
|
||||
else:
|
||||
raise Exception("Unsupported sampler type.")
|
||||
|
||||
if num_neg_samples is None:
|
||||
num_neg_samples = 10
|
||||
else:
|
||||
num_neg_samples = int(num_neg_samples)
|
||||
|
||||
remote_prefetch = is_sparse
|
||||
print(
|
||||
"With sparse mode, if your models has only small parameter prefetch may cause speed down"
|
||||
)
|
||||
|
||||
attrs = {
|
||||
'num_total_classes': int(num_total_classes),
|
||||
'num_neg_samples': num_neg_samples,
|
||||
'seed': seed,
|
||||
'sampler': sampler,
|
||||
'is_sparse': is_sparse,
|
||||
'remote_prefetch': remote_prefetch,
|
||||
}
|
||||
|
||||
helper.append_op(
|
||||
type='nce',
|
||||
inputs=inputs,
|
||||
outputs={
|
||||
'Cost': cost,
|
||||
'SampleLogits': sample_logits,
|
||||
'SampleLabels': sample_labels,
|
||||
},
|
||||
attrs=attrs,
|
||||
)
|
||||
return cost / (num_neg_samples + 1)
|
||||
Reference in New Issue
Block a user