wehub/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/paddle/static/nn/__init__.py
Executable
+79
View File
@@ -0,0 +1,79 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ...tensor.creation import create_parameter # noqa: F401
from .common import (
batch_norm,
bilinear_tensor_product,
continuous_value_model, # noqa: F401
conv2d,
conv2d_transpose,
conv3d,
conv3d_transpose,
deform_conv2d,
embedding,
fc,
group_norm,
instance_norm,
layer_norm,
prelu,
py_func,
row_conv,
sparse_embedding,
spectral_norm,
)
from .control_flow import case, cond, switch_case, while_loop
from .loss import nce
from .sequence_lod import (
sequence_conv,
sequence_expand,
sequence_first_step,
sequence_last_step,
sequence_pool,
sequence_softmax,
)
from .static_pylayer import static_pylayer
__all__ = [
'fc',
'batch_norm',
'bilinear_tensor_product',
'embedding',
'case',
'cond',
'static_pylayer',
'conv2d',
'conv2d_transpose',
'conv3d',
'conv3d_transpose',
'deform_conv2d',
'group_norm',
'instance_norm',
'layer_norm',
'nce',
'prelu',
'py_func',
'row_conv',
'spectral_norm',
'switch_case',
'while_loop',
'sparse_embedding',
'sequence_conv',
'sequence_softmax',
'sequence_pool',
'sequence_first_step',
'sequence_last_step',
'sequence_expand',
'prelu',
]
python/paddle/static/nn/common.py
+4113
View File
File diff suppressed because it is too large Load Diff
python/paddle/static/nn/control_flow.py
+2410
View File
File diff suppressed because it is too large Load Diff
python/paddle/static/nn/loss.py
+266
View File
@@ -0,0 +1,266 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
from paddle.base.framework import static_only
# TODO: define loss functions of neural network
from paddle.base.layer_helper import LayerHelper
from paddle.base.param_attr import ParamAttr
from paddle.nn.initializer import Assign
from ...base.data_feeder import check_variable_and_dtype
__all__ = []
# FIXME(wuyi): let docstring_checker.py understand @autodoc.
# For now, the comments in c++ use types like Tensor, but in python side
# the type is often "Variable", and arguments may vary.
@static_only
def nce(
input,
label,
num_total_classes,
sample_weight=None,
param_attr=None,
bias_attr=None,
num_neg_samples=None,
name=None,
sampler="uniform",
custom_dist=None,
seed=0,
is_sparse=False,
):
"""
:api_attr: Static Graph
Compute and return the noise-contrastive estimation training loss. See `Noise-contrastive estimation: A new estimation principle
for unnormalized statistical models <http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf>`_.
By default this operator uses a uniform distribution for sampling.
Args:
input (Tensor): Input tensor, 2-D tensor with shape [batch_size, dim],
and data type is float32 or float64.
label (Tensor): Input label, 2-D tensor with shape [batch_size, num_true_class],
and data type is int64.
num_total_classes (int): Total number of classes in all samples.
sample_weight (Tensor|None): A Tensor of shape [batch_size, 1]
storing a weight for each sample. The default weight for each
sample is 1.0.
param_attr (ParamAttr|None): To specify the weight parameter attribute.
Default: None, which means the default weight parameter property is
used. See usage for details in :ref:`api_paddle_ParamAttr` .
bias_attr (ParamAttr|None): To specify the bias parameter attribute.
Default: None, which means the default bias parameter property is
used. See usage for details in :ref:`api_paddle_ParamAttr` .
num_neg_samples (int): The number of negative classes. The default value is 10.
name(str|None): For detailed information, please refer to
:ref:`api_guide_Name` . Usually name is no need to set and None by default.
sampler (str, optional): The sampler used to sample class from negative classes.
It can be 'uniform', 'log_uniform' or 'custom_dist'.
default: 'uniform'.
custom_dist (nd.array|None): A numpy ndarray with size=num_total_classes.
It is used when sampler is set to 'custom_dist'.
custom_dist[i] is the probability of i-th class to be sampled.
default: None.
seed (int, optional): The seed used in sampler. Default 0, means no random seed.
is_sparse(bool, optional): The flag indicating whether to use sparse update,
the weight@GRAD and bias@GRAD will be changed to SelectedRows. Default False.
Returns:
Tensor: The output nce loss.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP("paddle.static.nn.nce doesn't support PIR mode")
>>> import paddle
>>> import numpy as np
>>> paddle.enable_static()
>>> window_size = 5
>>> words = []
>>> for i in range(window_size):
... words.append(paddle.static.data(name='word_{0}'.format(i), shape=[-1, 1], dtype='int64'))
>>> dict_size = 10000
>>> label_word = int(window_size / 2) + 1
>>> embs = []
>>> for i in range(window_size):
... if i == label_word:
... continue
...
... emb = paddle.static.nn.embedding(input=words[i], size=[dict_size, 32], param_attr='embed', is_sparse=True)
... embs.append(emb)
>>> embs = paddle.concat(x=embs, axis=1) # concat from 4 * [(-1, 1, 32)] to (-1, 4, 32)
>>> embs = paddle.reshape(x=embs, shape=(-1, 4 * 32)) # reshape to (batch_size = -1, dim = 4*32)
>>> loss = paddle.static.nn.nce(
... input=embs, label=words[label_word], num_total_classes=dict_size, param_attr='nce.w_0', bias_attr='nce.b_0'
... )
# or use custom distribution
>>> dist = np.array([0.05, 0.5, 0.1, 0.3, 0.05])
>>> loss = paddle.static.nn.nce(
... input=embs,
... label=words[label_word],
... num_total_classes=5,
... param_attr='nce.w_1',
... bias_attr='nce.b_1',
... num_neg_samples=3,
... sampler="custom_dist",
... custom_dist=dist,
... )
"""
helper = LayerHelper('nce', **locals())
check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'nce')
check_variable_and_dtype(label, 'label', ['int64'], 'nce')
if input.ndim != 2:
raise ValueError(
f'The rank of `input` must be 2, but received {input.ndim}.'
)
dim = input.shape[1]
num_true_class = label.shape[1]
w = helper.create_parameter(
attr=helper.param_attr,
shape=[num_total_classes, dim],
is_bias=False,
dtype=input.dtype,
)
inputs = {}
if helper.bias_attr:
b = helper.create_parameter(
attr=helper.bias_attr,
shape=[num_total_classes, 1],
is_bias=True,
dtype=input.dtype,
)
inputs['Bias'] = b
cost = helper.create_variable_for_type_inference(dtype=input.dtype)
sample_logits = helper.create_variable_for_type_inference(dtype=input.dtype)
sample_labels = helper.create_variable_for_type_inference(dtype=label.dtype)
inputs['Input'] = input
inputs['Label'] = label
inputs['Weight'] = w
inputs['SampleWeight'] = sample_weight if sample_weight is not None else []
if sampler == "uniform":
sampler = 0
elif sampler == "log_uniform":
sampler = 1
elif sampler == "custom_dist":
assert custom_dist is not None
custom_dist_len = num_total_classes
alias_probs_ = [0] * custom_dist_len
alias_ = [0] * custom_dist_len
bigs = []
littles = []
for i in range(custom_dist_len):
normal_prob = custom_dist[i] * custom_dist_len
if normal_prob - 1.0 > 0:
bigs.append((i, normal_prob))
elif 1.0 - normal_prob > 0:
littles.append((i, normal_prob))
else:
alias_probs_[i] = normal_prob
alias_[i] = -1
while len(bigs) and len(littles):
big = bigs.pop(0)
little = littles.pop(0)
big_idx = big[0]
big_prob = big[1]
alias_probs_[little[0]] = little[1]
alias_[little[0]] = big_idx
big_left = big[1] + little[1] - 1
if big_left - 1.0 > 0:
bigs.append((big_idx, big_left))
elif 1.0 - big_left > 0:
littles.append((big_idx, big_left))
else:
alias_probs_[big_idx] = big_left
alias_[big_idx] = -1
if len(bigs):
big = bigs.pop(0)
alias_probs_[big[0]] = 1.0
alias_[big[0]] = -1
if len(littles):
little = littles.pop(0)
alias_probs_[little[0]] = 1.0
alias_[little[0]] = -1
def _init_by_numpy_array(numpy_array):
ret = helper.create_parameter(
attr=ParamAttr(),
shape=numpy_array.shape,
dtype=numpy_array.dtype,
default_initializer=Assign(numpy_array),
)
ret.stop_gradient = True
return ret
inputs['CustomDistProbs'] = _init_by_numpy_array(
np.array(custom_dist).astype('float32')
)
inputs['CustomDistAlias'] = _init_by_numpy_array(
np.array(alias_).astype('int32')
)
inputs['CustomDistAliasProbs'] = _init_by_numpy_array(
np.array(alias_probs_).astype('float32')
)
sampler = 2
else:
raise Exception("Unsupported sampler type.")
if num_neg_samples is None:
num_neg_samples = 10
else:
num_neg_samples = int(num_neg_samples)
remote_prefetch = is_sparse
print(
"With sparse mode, if your models has only small parameter prefetch may cause speed down"
)
attrs = {
'num_total_classes': int(num_total_classes),
'num_neg_samples': num_neg_samples,
'seed': seed,
'sampler': sampler,
'is_sparse': is_sparse,
'remote_prefetch': remote_prefetch,
}
helper.append_op(
type='nce',
inputs=inputs,
outputs={
'Cost': cost,
'SampleLogits': sample_logits,
'SampleLabels': sample_labels,
},
attrs=attrs,
)
return cost / (num_neg_samples + 1)
python/paddle/static/nn/metric.py
+627
View File
@@ -0,0 +1,627 @@
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
All layers just related to metric.
"""
import numpy as np
import paddle
from paddle import _C_ops, _legacy_C_ops
from paddle.base.data_feeder import check_variable_and_dtype
from paddle.base.framework import (
Variable,
_create_tensor,
in_dygraph_mode,
in_pir_mode,
)
from paddle.base.layer_helper import LayerHelper
from paddle.nn.initializer import ConstantInitializer
__all__ = []
def accuracy(input, label, k=1, correct=None, total=None):
"""
accuracy layer.
Refer to the https://en.wikipedia.org/wiki/Precision_and_recall
This function computes the accuracy using the input and label.
If the correct label occurs in top k predictions, then correct will increment by one.
Note:
the dtype of accuracy is determined by input. the input and label dtype can be different.
Args:
input(Tensor): The input of accuracy layer, which is the predictions of network. A Tensor with type float32,float64.
The shape is ``[sample_number, class_dim]`` .
label(Tensor): The label of dataset. Tensor with type int32,int64. The shape is ``[sample_number, 1]`` .
k(int, optional): The top k predictions for each class will be checked. Data type is int64 or int32. Default is 1.
correct(Tensor, optional): The correct predictions count. A Tensor with type int64 or int32. Default is None.
total(Tensor, optional): The total entries count. A tensor with type int64 or int32. Default is None.
Returns:
Tensor, The correct rate. A Tensor with type float32.
Examples:
.. code-block:: pycon
>>> import numpy as np
>>> import paddle
>>> import paddle.static as static
>>> import paddle.nn.functional as F
>>> paddle.seed(2023)
>>> paddle.enable_static()
>>> data = static.data(name="input", shape=[-1, 32, 32], dtype="float32")
>>> label = static.data(name="label", shape=[-1, 1], dtype="int64")
>>> fc_out = static.nn.fc(x=data, size=10)
>>> predict = F.softmax(x=fc_out)
>>> result = static.accuracy(input=predict, label=label, k=5)
>>> place = paddle.CPUPlace()
>>> exe = static.Executor(place)
>>> exe.run(static.default_startup_program())
>>> np.random.seed(1107)
>>> x = np.random.rand(3, 32, 32).astype("float32")
>>> y = np.array([[1], [0], [1]])
>>> output = exe.run(
... feed={"input": x, "label": y},
... fetch_list=[result],
... )
>>> print(output)
[array(0.33333334, dtype=float32)]
"""
if in_dygraph_mode():
if correct is None:
correct = _create_tensor(dtype="int32")
if total is None:
total = _create_tensor(dtype="int32")
_k = np.array(k).item(0) if isinstance(k, Variable) else k
topk_out, topk_indices = _legacy_C_ops.top_k_v2(
input, 'k', _k, 'sorted', False
)
_acc, _, _ = _legacy_C_ops.accuracy(
topk_out, topk_indices, label, correct, total
)
return _acc
elif in_pir_mode():
topk_out, topk_indices = paddle.topk(input, k=k, sorted=False)
_acc, _, _ = _C_ops.accuracy(topk_out, topk_indices, label)
return _acc
helper = LayerHelper("accuracy", **locals())
check_variable_and_dtype(
input, 'input', ['float16', 'uint16', 'float32', 'float64'], 'accuracy'
)
topk_out = helper.create_variable_for_type_inference(dtype=input.dtype)
topk_indices = helper.create_variable_for_type_inference(dtype="int64")
inputs = {"X": [input]}
if isinstance(k, Variable):
inputs['K'] = [k]
else:
attrs = {'k': k}
attrs['sorted'] = False
helper.append_op(
type="top_k_v2",
inputs=inputs,
attrs=attrs,
outputs={"Out": [topk_out], "Indices": [topk_indices]},
)
acc_out = helper.create_variable_for_type_inference(dtype="float32")
if correct is None:
correct = helper.create_variable_for_type_inference(dtype="int32")
if total is None:
total = helper.create_variable_for_type_inference(dtype="int32")
helper.append_op(
type="accuracy",
inputs={"Out": [topk_out], "Indices": [topk_indices], "Label": [label]},
outputs={
"Accuracy": [acc_out],
"Correct": [correct],
"Total": [total],
},
)
return acc_out
def auc(
input,
label,
curve='ROC',
num_thresholds=2**12 - 1,
topk=1,
slide_steps=1,
ins_tag_weight=None,
):
"""
**Area Under the Curve (AUC) Layer**
This implementation computes the AUC according to forward output and label.
It is used very widely in binary classification evaluation.
Note: If input label contains values other than 0 and 1, it will be cast
to `bool`. Find the relevant definitions `here <https://en.wikipedia.org\
/wiki/Receiver_operating_characteristic#Area_under_the_curve>`_.
There are two types of possible curves:
1. ROC: Receiver operating characteristic;
2. PR: Precision Recall
Args:
input(Tensor): A floating-point 2D Tensor, values are in the range
[0, 1]. Each row is sorted in descending order. This
input should be the output of topk. Typically, this
Tensor indicates the probability of each label.
A Tensor with type float32,float64.
label(Tensor): A 2D int Tensor indicating the label of the training
data. The height is batch size and width is always 1.
A Tensor with type int32,int64.
curve(str, optional): Curve type, can be 'ROC' or 'PR'. Default 'ROC'.
num_thresholds(int, optional): The number of thresholds to use when discretizing
the roc curve. Default 4095.
topk(int, optional): only topk number of prediction output will be used for auc.
slide_steps(int, optional): when calc batch auc, we can not only use step currently but the previous steps can be used. slide_steps=1 means use the current step, slide_steps=3 means use current step and the previous second steps, slide_steps=0 use all of the steps.
ins_tag_weight(Tensor, optional): A 2D int Tensor indicating the data's tag weight, 1 means real data, 0 means fake data. Default None, and it will be assigned to a tensor of value 1.
A Tensor with type float32,float64.
Returns:
Tensor: A tuple representing the current AUC. Data type is Tensor, supporting float32, float64.
The return tuple is auc_out, batch_auc_out, [batch_stat_pos, batch_stat_neg, stat_pos, stat_neg ]
auc_out: the result of the accuracy rate
batch_auc_out: the result of the batch accuracy
batch_stat_pos: the statistic value for label=1 at the time of batch calculation
batch_stat_neg: the statistic value for label=0 at the time of batch calculation
stat_pos: the statistic for label=1 at the time of calculation
stat_neg: the statistic for label=0 at the time of calculation
Examples:
.. code-block:: pycon
:name: example-1
>>> # doctest: +SKIP("This has diff in xdoctest env")
>>> import paddle
>>> import numpy as np
>>> paddle.enable_static()
>>> paddle.seed(2023)
>>> data = paddle.static.data(name="input", shape=[-1, 32,32], dtype="float32")
>>> label = paddle.static.data(name="label", shape=[-1], dtype="int64")
>>> fc_out = paddle.static.nn.fc(x=data, size=2)
>>> predict = paddle.nn.functional.softmax(x=fc_out)
>>> result=paddle.static.auc(input=predict, label=label)
>>> place = paddle.CPUPlace()
>>> exe = paddle.static.Executor(place)
>>> exe.run(paddle.static.default_startup_program())
>>> np.random.seed(1107)
>>> x = np.random.rand(3,32,32).astype("float32")
>>> y = np.array([1,0,1])
>>> output= exe.run(feed={"input": x,"label": y},
... fetch_list=[result[0]])
>>> print(output)
[array(1.)]
.. code-block:: pycon
:name: example-2
# you can learn the usage of ins_tag_weight by the following code.
>>> # doctest: +SKIP("This has diff in xdoctest env")
>>> import paddle
>>> import numpy as np
>>> paddle.enable_static()
>>> paddle.seed(2023)
>>> data = paddle.static.data(name="input", shape=[-1, 32,32], dtype="float32")
>>> label = paddle.static.data(name="label", shape=[-1], dtype="int64")
>>> ins_tag_weight = paddle.static.data(name='ins_tag_weight', shape=[-1,16], dtype='float64')
>>> fc_out = paddle.static.nn.fc(x=data, size=2)
>>> predict = paddle.nn.functional.softmax(x=fc_out)
>>> result=paddle.static.auc(input=predict, label=label, ins_tag_weight=ins_tag_weight)
>>> place = paddle.CPUPlace()
>>> exe = paddle.static.Executor(place)
>>> exe.run(paddle.static.default_startup_program())
>>> np.random.seed(1107)
>>> x = np.random.rand(3,32,32).astype("float32")
>>> y = np.array([1,0,1])
>>> z = np.array([1,0,1]).astype("float64")
>>> output= exe.run(feed={"input": x,"label": y, "ins_tag_weight":z},
... fetch_list=[result[0]])
>>> print(output)
[array(1.)]
"""
if in_pir_mode():
if ins_tag_weight is None:
ins_tag_weight = paddle.full(
shape=[1, 1], dtype="float32", fill_value=1.0
)
check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'auc')
check_variable_and_dtype(label, 'label', ['int32', 'int64'], 'auc')
check_variable_and_dtype(
ins_tag_weight, 'ins_tag_weight', ['float32', 'float64'], 'auc'
)
stat_pos = paddle.zeros(shape=[1, num_thresholds + 1], dtype="int64")
stat_neg = paddle.zeros(shape=[1, num_thresholds + 1], dtype="int64")
auc_out, batch_stat_pos, batch_stat_neg = _C_ops.auc(
input,
label,
stat_pos,
stat_neg,
ins_tag_weight,
curve,
num_thresholds,
0,
)
return (
auc_out,
batch_stat_pos,
batch_stat_neg,
)
helper = LayerHelper("auc", **locals())
if ins_tag_weight is None:
ins_tag_weight = paddle.tensor.fill_constant(
shape=[1, 1], dtype="float32", value=1.0
)
check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'auc')
check_variable_and_dtype(label, 'label', ['int32', 'int64'], 'auc')
check_variable_and_dtype(
ins_tag_weight, 'ins_tag_weight', ['float32', 'float64'], 'auc'
)
auc_out = helper.create_variable_for_type_inference(dtype="float64")
batch_auc_out = helper.create_variable_for_type_inference(dtype="float64")
# make tp, tn, fp, fn persistable, so that can accumulate all batches.
# for batch auc
# we create slide_step+1 buckets, the first slide_steps buckets store
# historical batch-level values, and the last bucket stores the sum values of
# previous slide_step buckets.
# The index of bucket that the newest batch will use is determined by batch_id mod slide_steps,
# and batch_id is store in the last position of following variable
batch_stat_pos = helper.create_global_variable(
persistable=True,
dtype='int64',
shape=[(1 + slide_steps) * (num_thresholds + 1) + 1],
)
batch_stat_neg = helper.create_global_variable(
persistable=True,
dtype='int64',
shape=[(1 + slide_steps) * (num_thresholds + 1) + 1],
)
# for global auc
# Needn't maintain the batch id
stat_pos = helper.create_global_variable(
persistable=True, dtype='int64', shape=[1, num_thresholds + 1]
)
stat_neg = helper.create_global_variable(
persistable=True, dtype='int64', shape=[1, num_thresholds + 1]
)
for var in [batch_stat_pos, batch_stat_neg, stat_pos, stat_neg]:
helper.set_variable_initializer(
var,
ConstantInitializer(value=0.0, force_cpu=False),
)
# "InsTagWeight": [ins_tag_weight]
# Batch AUC
helper.append_op(
type="auc",
inputs={
"Predict": [input],
"Label": [label],
"StatPos": [batch_stat_pos],
"StatNeg": [batch_stat_neg],
},
attrs={
"curve": curve,
"num_thresholds": num_thresholds,
"slide_steps": slide_steps,
},
outputs={
"AUC": [batch_auc_out],
"StatPosOut": [batch_stat_pos],
"StatNegOut": [batch_stat_neg],
},
)
# Global AUC
helper.append_op(
type="auc",
inputs={
"Predict": [input],
"Label": [label],
"StatPos": [stat_pos],
"StatNeg": [stat_neg],
},
attrs={
"curve": curve,
"num_thresholds": num_thresholds,
"slide_steps": 0,
},
outputs={
"AUC": [auc_out],
"StatPosOut": [stat_pos],
"StatNegOut": [stat_neg],
},
)
return (
auc_out,
batch_auc_out,
[batch_stat_pos, batch_stat_neg, stat_pos, stat_neg],
)
def ctr_metric_bundle(input, label, ins_tag_weight=None):
"""
ctr related metric layer
This function help compute the ctr related metrics: RMSE, MAE, predicted_ctr, q_value.
To compute the final values of these metrics, we should do following computations using
total instance number:
MAE = local_abserr / instance number
RMSE = sqrt(local_sqrerr / instance number)
predicted_ctr = local_prob / instance number
q = local_q / instance number
Note that if you are doing distribute job, you should all reduce these metrics and instance
number first
Args:
input(Tensor): A floating-point 2D Tensor, values are in the range
[0, 1]. Each row is sorted in descending order. This
input should be the output of topk. Typically, this
Tensor indicates the probability of each label.
label(Tensor): A 2D int Tensor indicating the label of the training
data. The height is batch size and width is always 1.
ins_tag_weight(Tensor): A 2D int Tensor indicating the ins_tag_weight of the training
data. 1 means real data, 0 means fake data.
A DenseTensor or Tensor with type float32,float64.
Returns:
local_sqrerr(Tensor): Local sum of squared error
local_abserr(Tensor): Local sum of abs error
local_prob(Tensor): Local sum of predicted ctr
local_q(Tensor): Local sum of q value
local_pos_num (Tensor): Local number of positive examples
local_ins_num (Tensor): Local number of instances
Examples:
.. code-block:: pycon
:name: example-1
>>> # doctest: +SKIP("This has diff in xdoctest env")
>>> import paddle
>>> paddle.enable_static()
>>> data = paddle.static.data(name="data", shape=[-1, 32], dtype="float32")
>>> label = paddle.static.data(name="label", shape=[-1, 1], dtype="int32")
>>> predict = paddle.nn.functional.sigmoid(paddle.static.nn.fc(x=data, size=1))
>>> auc_out = paddle.static.ctr_metric_bundle(input=predict, label=label)
.. code-block:: pycon
:name: example-2
>>> # doctest: +SKIP("This has diff in xdoctest env")
>>> import paddle
>>> paddle.enable_static()
>>> data = paddle.static.data(name="data", shape=[-1, 32], dtype="float32")
>>> label = paddle.static.data(name="label", shape=[-1, 1], dtype="int32")
>>> predict = paddle.nn.functional.sigmoid(paddle.static.nn.fc(x=data, size=1))
>>> ins_tag_weight = paddle.static.data(name='ins_tag_weight', shape=[-1, 1], dtype='int64')
>>> auc_out = paddle.static.ctr_metric_bundle(input=predict, label=label, ins_tag_weight=ins_tag_weight)
"""
if ins_tag_weight is None:
ins_tag_weight = paddle.tensor.fill_constant(
shape=[1, 1], dtype="float32", value=1.0
)
assert input.shape == label.shape
helper = LayerHelper("ctr_metric_bundle", **locals())
local_abserr = helper.create_global_variable(
persistable=True, dtype='float32', shape=[1]
)
local_sqrerr = helper.create_global_variable(
persistable=True, dtype='float32', shape=[1]
)
local_prob = helper.create_global_variable(
persistable=True, dtype='float32', shape=[1]
)
local_q = helper.create_global_variable(
persistable=True, dtype='float32', shape=[1]
)
local_pos_num = helper.create_global_variable(
persistable=True, dtype='float32', shape=[1]
)
local_ins_num = helper.create_global_variable(
persistable=True, dtype='float32', shape=[1]
)
tmp_res_elesub = helper.create_global_variable(
persistable=False, dtype='float32', shape=[-1]
)
tmp_res_sigmoid = helper.create_global_variable(
persistable=False, dtype='float32', shape=[-1]
)
tmp_ones = helper.create_global_variable(
persistable=False, dtype='float32', shape=[-1]
)
batch_prob = helper.create_global_variable(
persistable=False, dtype='float32', shape=[1]
)
batch_abserr = helper.create_global_variable(
persistable=False, dtype='float32', shape=[1]
)
batch_sqrerr = helper.create_global_variable(
persistable=False, dtype='float32', shape=[1]
)
batch_q = helper.create_global_variable(
persistable=False, dtype='float32', shape=[1]
)
batch_pos_num = helper.create_global_variable(
persistable=False, dtype='float32', shape=[1]
)
batch_ins_num = helper.create_global_variable(
persistable=False, dtype='float32', shape=[1]
)
for var in [
local_abserr,
batch_abserr,
local_sqrerr,
batch_sqrerr,
local_prob,
batch_prob,
local_q,
batch_q,
batch_pos_num,
batch_ins_num,
local_pos_num,
local_ins_num,
]:
helper.set_variable_initializer(
var,
paddle.nn.initializer.ConstantInitializer(
value=0.0, force_cpu=True
),
)
helper.append_op(
type="elementwise_sub",
inputs={"X": [input], "Y": [label]},
outputs={"Out": [tmp_res_elesub]},
)
helper.append_op(
type="squared_l2_norm",
inputs={"X": [tmp_res_elesub]},
outputs={"Out": [batch_sqrerr]},
)
helper.append_op(
type="elementwise_add",
inputs={"X": [batch_sqrerr], "Y": [local_sqrerr]},
outputs={"Out": [local_sqrerr]},
)
helper.append_op(
type="l1_norm",
inputs={"X": [tmp_res_elesub]},
outputs={"Out": [batch_abserr]},
)
helper.append_op(
type="elementwise_add",
inputs={"X": [batch_abserr], "Y": [local_abserr]},
outputs={"Out": [local_abserr]},
)
helper.append_op(
type="reduce_sum", inputs={"X": [input]}, outputs={"Out": [batch_prob]}
)
helper.append_op(
type="elementwise_add",
inputs={"X": [batch_prob], "Y": [local_prob]},
outputs={"Out": [local_prob]},
)
helper.append_op(
type="sigmoid",
inputs={"X": [input]},
outputs={"Out": [tmp_res_sigmoid]},
)
helper.append_op(
type="reduce_sum",
inputs={"X": [tmp_res_sigmoid]},
outputs={"Out": [batch_q]},
)
helper.append_op(
type="reduce_sum",
inputs={"X": [label]},
outputs={"Out": [batch_pos_num]},
)
helper.append_op(
type="elementwise_add",
inputs={"X": [batch_pos_num], "Y": [local_pos_num]},
outputs={"Out": [local_pos_num]},
)
helper.append_op(
type='fill_constant_batch_size_like',
inputs={"Input": label},
outputs={'Out': [tmp_ones]},
attrs={
'shape': [-1, 1],
'dtype': tmp_ones.dtype,
'value': 1.0,
},
)
helper.append_op(
type="reduce_sum",
inputs={"X": [tmp_ones]},
outputs={"Out": [batch_ins_num]},
)
# if data is fake, return 0
inputs_slice = {'Input': ins_tag_weight}
attrs = {'axes': [0]}
attrs['starts'] = [0]
attrs['ends'] = [1]
helper.append_op(
type="slice",
inputs=inputs_slice,
attrs=attrs,
outputs={"Out": ins_tag_weight},
)
axis = helper.kwargs.get('axis', 0)
helper.append_op(
type="elementwise_mul",
inputs={"X": [batch_ins_num], "Y": [ins_tag_weight]},
outputs={"Out": [batch_ins_num]},
attrs={'axis': axis},
)
helper.append_op(
type="elementwise_add",
inputs={"X": [batch_ins_num], "Y": [local_ins_num]},
outputs={"Out": [local_ins_num]},
)
helper.append_op(
type="elementwise_mul",
inputs={"X": [batch_q], "Y": [ins_tag_weight]},
outputs={"Out": [batch_q]},
attrs={'axis': axis},
)
helper.append_op(
type="elementwise_add",
inputs={"X": [batch_q], "Y": [local_q]},
outputs={"Out": [local_q]},
)
return (
local_sqrerr,
local_abserr,
local_prob,
local_q,
local_pos_num,
local_ins_num,
)
python/paddle/static/nn/sequence_lod.py
+755
View File
@@ -0,0 +1,755 @@
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle
from paddle.base.data_feeder import check_variable_and_dtype
from paddle.base.framework import in_dygraph_mode, in_pir_mode
from paddle.base.layer_helper import LayerHelper
from paddle.utils import deprecated
__all__ = []
@deprecated(
since="3.0.0",
level=1,
reason="This API will be deprecated in the future, because it's just for old statics mode.",
)
def sequence_conv(
input,
num_filters,
filter_size=3,
filter_stride=1,
padding=True,
padding_start=None,
bias_attr=None,
param_attr=None,
act=None,
name=None,
):
r"""
Note:
Only receives Tensor as input. If your input is Tensor, please use conv2d Op.(base.layers.** :ref:`api_paddle_nn_functional_conv2d` ).
This operator receives input sequences with variable length and other convolutional
configuration parameters(num_filters, filter_size) to apply the convolution operation.
It fills all-zero padding data on both sides of the sequence by default to ensure that
the output is the same length as the input. You can customize the padding behavior by
configuring the parameter :attr:`padding\_start` .
**Warning:** the parameter :attr:`padding` take no effect and will be deprecated in the future.
.. code-block:: text
Here we will illustrate the details of the padding operation:
For a mini-batch of 2 variable lengths sentences, containing 3, and 1 time-steps:
Assumed input (X) is a [4, N] float Tensor, and for the sake of simplicity, we assume N=2.
input.data = [[1, 1],
[2, 2],
[3, 3],
[4, 4]]
This is to say that input (X) has 4 words and the dimension of each word
representation is 2.
* Case1:
If padding_start is -1 and filter_size is 3.
The length of padding data is calculated as follows:
up_pad_len = max(0, -padding_start) = 1
down_pad_len = max(0, filter_size + padding_start - 1) = 1
The output of the input sequence after padding is:
data_after_padding = [[0, 0, 1, 1, 2, 2],
[1, 1, 2, 2, 3, 3],
[2, 2, 3, 3, 0, 0],
[0, 0, 4, 4, 0, 0]]
It will be multiplied by the filter weight to get the final output.
Assume num_filters = 3
output.data = [[ 0.3234, -0.2334, 0.7433],
[ 0.5646, 0.9464, -0.1223],
[-0.1343, 0.5653, 0.4555],
[ 0.9954, -0.1234, -0.1234]]
output.shape = [4, 3] # 3 = num_filters
output.lod = [[0, 3, 4]] # Remain the same
Args:
input (Tensor): Tensor with shape :math:`(M, K)`, where M is the total time-step of mini-batch
and K is hidden_size of input. Only lod_level of 1 is supported. The data type should be float32 or
float64.
num_filters (int): the number of filters.
filter_size (int): the height of filter. Specified filter width is not supported, the width is
hidden_size by default. Default: 3.
filter_stride (int, optional): stride of the filter. Currently only supports :attr:`stride` = 1.
padding (bool, optional): the parameter :attr:`padding` take no effect and will be discarded in the
future. Currently, it will always pad input to make sure the length of the output is
the same as input whether :attr:`padding` is set true or false. Because the length of
input sequence may be shorter than :attr:`filter\_size`, which will cause the convolution
result to not be computed correctly. These padding data will not be trainable or updated
while training. Default: True.
padding_start (int): It is used to indicate the start index for padding the input
sequence, which can be negative. The negative number means to pad
:attr:`|padding_start|` time-steps of all-zero data at the beginning of each instance.
The positive number means to skip :attr:`padding_start` time-steps of each instance,
and it will pad :math:`filter\_size + padding\_start - 1` time-steps of all-zero data
at the end of the sequence to ensure that the output is the same length as the input.
If set None, the same length :math:`\\frac{filter\_size}{2}` of data will be filled
on both sides of the sequence. If set 0, the length of :math:`filter\_size - 1` data
is padded at the end of each input sequence. Default: None.
bias_attr (ParamAttr): To specify the bias parameter property. Default: None, which means the
default bias parameter property is used. See usage for details in :ref:`api_paddle_ParamAttr` .
param_attr (ParamAttr): To specify the weight parameter property. Default: None, which means the
default weight parameter property is used. See usage for details in :ref:`api_paddle_ParamAttr` .
act (str): Activation to be applied to the output of this layer, such as tanh, softmax,
sigmoid, relu. For more information, please refer to :ref:`api_guide_activations_en` . Default: None.
name (str, optional): The default value is None. Normally there is no need for user to set this property.
For more information, please refer to :ref:`api_guide_Name` .
Returns:
Tensor: Tensor with the same length as input. The data type is float32 or float64, which is same as input.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP("env set will not work in ci check because import paddle in global_exec")
>>> # set env var before import paddle to disable pir mode, following example code use os module.
>>> import os
>>> os.environ['FLAGS_enable_pir_api'] = '0'
>>> import paddle
>>> paddle.enable_static()
>>> x = paddle.static.data(name='x', shape=[-1, 10], dtype='float32', lod_level=1)
>>> x_conved = paddle.static.nn.sequence_conv(input=x, num_filters=2, filter_size=3, padding_start=-1)
"""
assert not in_dygraph_mode(), (
"sequence layer is not supported in dygraph mode yet."
)
assert not in_pir_mode(), (
"sequence layer is not supported in pir mode, please set the environment variable FLAGS_enable_pir_api=0 to switch old static mode."
)
check_variable_and_dtype(
input, 'input', ['float32', 'float64'], 'sequence_conv'
)
helper = LayerHelper('sequence_conv', **locals())
dtype = helper.input_dtype()
filter_shape = [filter_size * input.shape[1], num_filters]
filter_param = helper.create_parameter(
attr=helper.param_attr, shape=filter_shape, dtype=dtype
)
pre_bias = helper.create_variable_for_type_inference(dtype)
if padding_start is None:
padding_start = -int(filter_size // 2)
helper.append_op(
type='sequence_conv',
inputs={
'X': [input],
'Filter': [filter_param],
},
outputs={"Out": pre_bias},
attrs={
'contextStride': filter_stride,
'contextStart': padding_start,
'contextLength': filter_size,
},
)
pre_act = helper.append_bias_op(pre_bias)
return helper.append_activation(pre_act)
@deprecated(
since="3.0.0",
level=1,
reason="This API will be deprecated in the future, because it's just for old statics mode.",
)
def sequence_softmax(input, use_cudnn=False, name=None):
r"""
Note:
The input type of the OP must be Tensor. For Tensor, use:** :ref:`api_paddle_nn_functional_softmax`
A LoD-tensor can be regarded as several sequences, and this op apply softmax algo on each sequence.
The shape of input Tensor can be :math:`[N, 1]` or :math:`[N]`, where :math:`N`
is the sum of the length of all sequences. Recommended usage: :math:`[N]`.
For i-th sequence in a mini-batch:
.. math::
Out(X[lod[i]:lod[i+1]], :) = \\frac{\exp(X[lod[i]:lod[i+1], :])}{\sum(\exp(X[lod[i]:lod[i+1], :]))}
For example, for a LoD-Tensor with 6 sequences ([3, 2, 4, 1, 2, 3] - sequence length list in order),
the lod in the runtime is [[0, 3, 5, 9, 10, 12, 15]],
then softmax will be computed among :math:`X[0:3,:],X[3:5,:],X[5:9,:],X[9:10,:],X[10:12,:],X[12:15,:]`,
and :math:`N` turns out to be 15.
.. code-block:: text
*Case 1:
Given:
input.data = [0.7, 1, 0.6,
1.5, 1.1,
1.2, 0.2, 0.6, 1.9,
3.1,
2.5, 0.8,
0.1, 2.4, 1.3]
input.lod = [[0, 3, 5, 9, 10, 12, 15]]
then:
output.data = [0.30724832, 0.41474187, 0.2780098,
0.59868765, 0.40131235,
0.2544242, 0.09359743, 0.13963096, 0.5123474,
1.,
0.84553474, 0.15446526,
0.06995796, 0.69777346, 0.23226859]
output.lod = [[0, 3, 5, 9, 10, 12, 15]]
Args:
input (Tensor):A Tensor with shape of :math:`[N, 1]` or :math:`[N]`, Recommended usage: :math:`[N]`.
Supported data types: float32, float64.
use_cudnn (bool, optional): Use cudnn kernel or not. Effective only when the cudnn version of the paddle
library is installed and GPU is used for training or reasoning. Default: False.
name (str, optional): The default value is None. Normally there is no need for user to set this property.
For more information, please refer to :ref:`api_guide_Name`
Returns:
Tensor: A LoD-Tensor which has the same shape and data type with input.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP("env set will not work in ci check because import paddle in global_exec")
>>> # set env var before import paddle to disable pir mode, following example code use os module.
>>> import os
>>> os.environ['FLAGS_enable_pir_api'] = '0'
>>> import paddle
>>> paddle.enable_static()
>>> x = paddle.static.data(name='x', shape=[7, 1], dtype='float32', lod_level=1)
>>> x_sequence_softmax_1 = paddle.static.nn.sequence_softmax(input=x)
>>> y = paddle.static.data(name='y', shape=[7], dtype='float32', lod_level=1)
>>> x_sequence_softmax_2 = paddle.static.nn.sequence_softmax(input=y)
"""
assert not in_dygraph_mode(), (
"sequence layer is not supported in dygraph mode yet."
)
assert not in_pir_mode(), (
"sequence layer is not supported in pir mode, please set the environment variable FLAGS_enable_pir_api=0 to switch old static mode."
)
helper = LayerHelper('sequence_softmax', **locals())
check_variable_and_dtype(
input, 'input', ['float32', 'float64'], 'sequence_softmax'
)
dtype = helper.input_dtype()
softmax_out = helper.create_variable_for_type_inference(dtype)
helper.append_op(
type="sequence_softmax",
inputs={"X": input},
outputs={"Out": softmax_out},
attrs={"use_cudnn": use_cudnn},
)
return softmax_out
@deprecated(
since="3.0.0",
level=1,
reason="This API will be deprecated in the future, because it's just for old statics mode.",
)
def sequence_pool(input, pool_type, is_test=False, pad_value=0.0):
r"""
Note:
Only receives Tensor as input. If your input is Tensor, please use pool2d Op.(static.nn.** :ref:`api_paddle_nn_functional_avg_pool2d` or :ref:`api_paddle_nn_functional_max_pool2d` ).
This operator only supports Tensor as input. It will apply specified pooling
operation on the input Tensor. It pools features of all time-steps of each
sequence at the last lod_level using :attr:`pool_type` mentioned in the parameters,
such as sum, average, sqrt, etc.
It supports six pool_type:
- average: :math:`Out[i] = \\frac{\sum_i X_i}{N}`
- sum: :math:`Out[i] = \sum_jX_{ij}`
- sqrt: :math:`Out[i] = \\frac{\sum_jX_{ij}}{\sqrt{len(X_i)}}`
- max: :math:`Out[i] = max(X_i)`
- last: :math:`Out[i] = X_{N_i}`
- first: :math:`Out[i]` = X_0
where :math:`N_i` is the length of i-th input sequence.
.. code-block:: text
Case 1:
input is a 1-level Tensor and pad_value = 0.0:
input.lod = [[0, 2, 5, 7, 7]]
input.data = [[1.], [3.], [2.], [4.], [6.], [5.], [1.]]
input.shape = [7, 1]
output is Tensor:
out.shape = [4, 1]
with condition out.shape[0] == len(x.lod[-1]) == 4
for different pool_type:
average: out.data = [[2.], [4.], [3.], [0.0]], where 2.=(1. + 3.)/2, 4.=(2. + 4. + 6.)/3, 3.=(5. + 1.)/2
sum : out.data = [[4.], [12.], [6.], [0.0]], where 4.=1. + 3., 12.=2. + 4. + 6., 6.=5. + 1.
sqrt : out.data = [[2.82], [6.93], [4.24], [0.0]], where 2.82=(1. + 3.)/sqrt(2), 6.93=(2. + 4. + 6.)/sqrt(3), 4.24=(5. + 1.)/sqrt(2)
max : out.data = [[3.], [6.], [5.], [0.0]], where 3.=max(1., 3.), 6.=max(2., 4., 6.), 5.=max(5., 1.)
last : out.data = [[3.], [6.], [1.], [0.0]], where 3.=last(1., 3.), 6.=last(2., 4., 6.), 1.=last(5., 1.)
first : out.data = [[1.], [2.], [5.], [0.0]], where 1.=first(1., 3.), 2.=first(2., 4., 6.), 5.=first(5., 1.)
and all above [0.0] at last of out.data is padding data.
Case 2:
input is a 2-level Tensor containing 3 sequences with length info [2, 0, 3],
where 0 means empty sequence.
The first sequence contains 2 subsequence with length info [1, 2];
The last sequence contains 3 subsequence with length info [1, 0, 3].
input.lod = [[0, 2, 2, 5], [0, 1, 3, 4, 4, 7]]
input.data = [[1.], [3.], [2.], [4.], [6.], [5.], [1.]]
input.shape = [7, 1]
If pool_typ = sum, it will apply pooling on last lod_level [0, 1, 3, 4, 4, 7]. pad_value = 0.0
output is Tensor:
out.shape= [5, 1]
out.lod = [[0, 2, 2, 5]]
where out.shape[0] == len(x.lod[-1]) == 5
sum: out.data = [[1.], [5.], [4.], [0.0], [12.]]
where 1.=1., 5.=3. + 2., 4.=4., 0.0=pad_value, 12.=6. + 5. + 1.
Args:
input (variable): Tensor with lod_level no more than 2. The data type should be float32 or float64.
pool_type (str): The pooling type that supports average, sum, sqrt, max, last or first.
is_test (bool): Only works when :attr:`pool_type` is max. If set False, a temporary Tensor maxIndex is
created to record the index information corresponding to the maximum value, which is used for backward
gradient calculation in the training phase. Default: False.
pad_value (float): Used to pad the pooling result for empty input sequence. Default: 0.0
Returns:
Tensor: Tensor after pooling with data type float32 or float64.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP("env set will not work in ci check because import paddle in global_exec")
>>> # set env var before import paddle to disable pir mode, following example code use os module.
>>> import os
>>> os.environ['FLAGS_enable_pir_api'] = '0'
>>> import paddle
>>> paddle.enable_static()
>>> x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1)
>>> avg_x = paddle.static.nn.sequence_pool(input=x, pool_type='average')
>>> sum_x = paddle.static.nn.sequence_pool(input=x, pool_type='sum')
>>> sqrt_x = paddle.static.nn.sequence_pool(input=x, pool_type='sqrt')
>>> max_x = paddle.static.nn.sequence_pool(input=x, pool_type='max')
>>> last_x = paddle.static.nn.sequence_pool(input=x, pool_type='last')
>>> first_x = paddle.static.nn.sequence_pool(input=x, pool_type='first')
"""
assert not in_dygraph_mode(), (
"sequence layer is not supported in dygraph mode yet."
)
assert not in_pir_mode(), (
"sequence layer is not supported in pir mode, please set the environment variable FLAGS_enable_pir_api=0 to switch old static mode."
)
check_variable_and_dtype(
input, 'input', ['float32', 'float64'], 'sequence_pool'
)
helper = LayerHelper('sequence_pool', **locals())
dtype = helper.input_dtype()
pool_out = helper.create_variable_for_type_inference(dtype)
max_index = helper.create_variable_for_type_inference(dtype)
helper.append_op(
type="sequence_pool",
inputs={"X": input},
outputs={"Out": pool_out, "MaxIndex": max_index},
attrs={
"pooltype": pool_type.upper(),
"is_test": is_test,
"pad_value": pad_value,
},
)
# when pool_type is max, variable max_index is initialized,
# so we stop the gradient explicitly here
if pool_type == 'max':
max_index.stop_gradient = True
return pool_out
@deprecated(
since="3.0.0",
level=1,
reason="This API will be deprecated in the future, because it's just for old statics mode.",
)
def sequence_first_step(input):
"""
Only supports Tensor as input. Given the input Tensor, it will
select first time-step feature of each sequence as output.
.. code-block:: text
Case 1:
input is 1-level Tensor:
input.lod = [[0, 2, 5, 7]]
input.data = [[1.], [3.], [2.], [4.], [6.], [5.], [1.]]
input.shape = [7, 1]
output is a Tensor:
out.shape = [3, 1]
out.shape[0] == len(x.lod[-1]) == 3
out.data = [[1.], [2.], [5.]], where 1.=first(1., 3.), 2.=first(2., 4., 6.), 5.=first(5., 1.)
Case 2:
input is a 2-level Tensor containing 3 sequences with length info [2, 0, 3],
where 0 means empty sequence.
The first sequence contains 2 subsequence with length info [1, 2];
The last sequence contains 3 subsequence with length info [1, 0, 3].
input.lod = [[0, 2, 2, 5], [0, 1, 3, 4, 4, 7]]
input.data = [[1.], [3.], [2.], [4.], [6.], [5.], [1.]]
input.shape = [7, 1]
It will apply pooling on last lod_level [0, 1, 3, 4, 4, 7]. pad_value = 0.0
output is a Tensor:
out.shape= [5, 1]
out.lod = [[0, 2, 2, 5]]
out.shape[0] == len(x.lod[-1]) == 5
out.data = [[1.], [3.], [4.], [0.0], [6.]]
where 1.=first(1.), 3.=first(3., 2.), 4.=first(4.), 0.0 = pad_value, 6.=first(6., 5., 1.)
Args:
input(Tensor): Tensor with lod_level no more than 2. The data type should be float32 or float64.
Returns:
Tensor: Tensor consist of the sequence's first step vector. The data type is float32 or float64.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP("env set will not work in ci check because import paddle in global_exec")
>>> # set env var before import paddle to disable pir mode, following example code use os module.
>>> import os
>>> os.environ['FLAGS_enable_pir_api'] = '0'
>>> import paddle
>>> paddle.enable_static()
>>> x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1)
>>> x_first_step = paddle.static.nn.sequence_first_step(input=x)
"""
check_variable_and_dtype(
input, 'input', ['float32', 'float64'], 'sequence_first_step'
)
return sequence_pool(input=input, pool_type="first")
@deprecated(
since="3.0.0",
level=1,
reason="This API will be deprecated in the future, because it's just for old statics mode.",
)
def sequence_last_step(input):
"""
Only supports Tensor as input. Given the input Tensor, it will
select last time-step feature of each sequence as output.
.. code-block:: text
Case 1:
input is 1-level Tensor:
input.lod = [[0, 2, 5, 7]]
input.data = [[1.], [3.], [2.], [4.], [6.], [5.], [1.]]
input.shape = [7, 1]
output is a Tensor:
out.shape = [3, 1]
out.shape[0] == len(x.lod[-1]) == 3
out.data = [[3.], [6.], [1.]], where 3.=last(1., 3.), 6.=last(2., 4., 6.), 1.=last(5., 1.)
Case 2:
input is a 2-level Tensor containing 3 sequences with length info [2, 0, 3],
where 0 means empty sequence.
The first sequence contains 2 subsequence with length info [1, 2];
The last sequence contains 3 subsequence with length info [1, 0, 3].
input.lod = [[0, 2, 2, 5], [0, 1, 3, 4, 4, 7]]
input.data = [[1.], [3.], [2.], [4.], [6.], [5.], [1.]]
input.shape = [7, 1]
It will apply pooling on last lod_level [0, 1, 3, 4, 4, 7]. pad_value = 0.0
output is a Tensor:
out.shape= [5, 1]
out.lod = [[0, 2, 2, 5]]
out.shape[0] == len(x.lod[-1]) == 5
out.data = [[1.], [2.], [4.], [0.0], [1.]]
where 1.=last(1.), 2.=last(3., 2.), 4.=last(4.), 0.0 = pad_value, 1=last(6., 5., 1.)
Args:
input(Tensor): Tensor with lod_level no more than 2. The data type should be float32.
Returns:
Tensor: Tensor consist of the sequence's last step vector. The data type is float32.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP("env set will not work in ci check because import paddle in global_exec")
>>> # set env var before import paddle to disable pir mode, following example code use os module.
>>> import os
>>> os.environ['FLAGS_enable_pir_api'] = '0'
>>> import paddle
>>> paddle.enable_static()
>>> x = paddle.static.data(name='x', shape=[None, 10], dtype='float32', lod_level=1)
>>> x_last_step = paddle.static.nn.sequence_last_step(input=x)
"""
check_variable_and_dtype(
input, 'input', ['float32', 'float64'], 'sequence_last_step'
)
return sequence_pool(input=input, pool_type="last")
@deprecated(
since="3.0.0",
level=1,
reason="This API will be deprecated in the future, because it's just for old statics mode.",
)
def sequence_expand(x, y, ref_level=-1, name=None):
r"""
Sequence Expand Layer. This layer will expand the input variable ``x`` \
according to specified level ``ref_level`` lod of ``y``. Please note that \
the lod level of ``x`` is at most 1. If the lod level of ``x`` is 1, than \
the size of lod of ``x`` must be equal to the length of ``ref_level`` lod \
of ``y``. If the lod level of ``x`` is 0, then the first dim of ``x`` should \
be equal to the size of ``ref_level`` of ``y``. The rank of **x** is at least 2. \
When rank of ``x`` is greater than 2, then it would be viewed as a 2-D tensor.
Note:
Please note that the input ``x`` should be Tensor or Tensor, \
and input ``y`` must be Tensor.
**Following examples will explain how sequence_expand works:**
.. code-block:: text
Case 1
Consider 2 sequences [a][b] and [c][d], now we want to expand them to [a][b], [a][b], [c][d] and [c][d].
Sequence [a][b] expand twice and [c][d] expands twice, so the lod which according to is [2, 2].
Input x is a 1-level Tensor:
x.lod = [[2, 2]] #lod based on length may be easier to understand
x.data = [[a], [b], [c], [d]]
x.dims = [4, 1]
input y is a Tensor:
y.lod = [[2, 2], #the 0th level lod, according to this level
[3, 3, 1, 1]] #the 1st level lod, it has nothing to do with this level
ref_level: 0
then output is a 1-level Tensor out:
out.lod = [[2, 2, 2, 2]] #lod based on offset
out.data = [[a], [b], [a], [b], [c], [d], [c], [d]]
out.dims = [8, 1]
Case 2
Consider 3 sequences [a], [b], [c], now we want to expand them to [a][a], [c][c][c].
It's obvious that the lod info of expanded sequences is [2, 0, 3].
x is a Tensor:
x.data = [[a], [b], [c]]
x.dims = [3, 1]
y is a Tensor:
y.lod = [[2, 0, 3]]
ref_level: -1
then output is a 1-level Tensor:
out.data = [[a], [a], [c], [c], [c]]
out.dims = [5, 1]
Args:
x (Tensor): The input variable which is a Tensor or Tensor, with the \
dims ``[M, K]``. The lod level is at most 1. The data type should be \
float32, float64, int32 or int64.
y (Tensor): The input variable which is a Tensor, the lod level is \
at least 1.
ref_level (int): Lod level of ``y`` to be referred by ``x``. If set to -1, \
refer the last level of lod.
name(str, optional): For detailed information, please refer \
to :ref:`api_guide_Name`. Usually name is no need to set and \
None by default.
Returns:
Tensor, The expanded variable which is a Tensor, with dims ``[N, K]``. \
``N`` depends on the lod info of ``x`` and ``y``. \
The data type is same as input.
Examples:
.. code-block:: pycon
>>> # doctest: +SKIP("env set will not work in ci check because import paddle in global_exec")
>>> # set env var before import paddle to disable pir mode, following example code use os module.
>>> import os
>>> os.environ['FLAGS_enable_pir_api'] = '0'
>>> import paddle
>>> from paddle import base
>>> paddle.enable_static()
>>> import numpy as np
>>> x = paddle.static.data(name='x', shape=[4, 1], dtype='float32')
>>> y = paddle.static.data(name='y', shape=[8, 1],
... dtype='float32', lod_level=1)
>>> out = paddle.static.nn.sequence_expand(x=x, y=y, ref_level=0)
>>> exe = paddle.static.Executor(base.CPUPlace())
>>> place = paddle.CPUPlace()
>>> np_data = np.array([[1], [2], [3], [4]]).astype('float32')
>>> x_lod_tensor = base.create_lod_tensor(np_data, [[2, 2]], place)
>>> print(x_lod_tensor)
- lod: {{0, 2, 4}}
- place: Place(cpu)
- shape: [4, 1]
- layout: NCHW
- dtype: float32
- data: [1 2 3 4]
>>> np_data = np.array([[1], [2], [3], [4], [5], [6], [7], [8]]).astype('float32')
>>> y_lod_tensor = base.create_lod_tensor(np_data, [[2, 2], [3,3,1,1]], place)
>>> print(y_lod_tensor)
- lod: {{0, 2, 4}{0, 3, 6, 7, 8}}
- place: Place(cpu)
- shape: [8, 1]
- layout: NCHW
- dtype: float32
- data: [1 2 3 4 5 6 7 8]
>>> out_main = exe.run(base.default_main_program(),
... feed={'x': x_lod_tensor, 'y': y_lod_tensor},
... fetch_list=[out], return_numpy=False)
>>> print(out_main[0])
- lod: {{0, 2, 4, 6, 8}}
- place: Place(cpu)
- shape: [8, 1]
- layout: NCHW
- dtype: float32
- data: [1 2 1 2 3 4 3 4]
"""
assert not in_dygraph_mode(), (
"sequence layer is not supported in dygraph mode yet."
)
assert not in_pir_mode(), (
"sequence layer is not supported in pir mode, please set the environment variable FLAGS_enable_pir_api=0 to switch old static mode."
)
check_variable_and_dtype(
x, 'x', ['float32', 'float64', 'int32', 'int64'], 'sequence_expand'
)
helper = LayerHelper('sequence_expand', **locals())
dtype = helper.input_dtype(input_param_name='x')
tmp = helper.create_variable_for_type_inference(dtype)
helper.append_op(
type='sequence_expand',
inputs={'X': x, 'Y': y},
outputs={'Out': tmp},
attrs={'ref_level': ref_level},
)
return tmp
@deprecated(
update_to="paddle.nn.functional.sequence_mask",
level=1,
)
def sequence_mask(x, maxlen=None, dtype='int64', name=None):
r"""
**SequenceMask Layer**
This layer outputs a mask according to the input :code:`x` and
:code:`maxlen` with data type of :code:`dtype`.
Supposing :code:`x` is a Tensor with shape [d_1, d_2, ..., d_n], the
:code:`y` is a mask with shape [d_1, d_2, ..., d_n, maxlen], where:
.. math::
y(i_1, i_2,..., i_n, j) = (j < x(i_1, i_2,..., i_n))
.. code-block:: text
Case:
Consider input:
x = [3, 1, 1, 0] max_len = 4
then we get out:
mask = [[1, 1, 1, 0],
[1, 0, 0, 0],
[1, 0, 0, 0],
[0, 0, 0, 0]]
Args:
x (Tensor): Input tensor of sequence_mask layer, \
whose elements are integers less than :code:`maxlen`. \
Tensor or Tensor with shape [d_1, d_2, ..., d_n].
maxlen (int, optional): Maximum length of the sequence. If :code:`maxlen` \
is None, it would be replace with :math:`max(x)`.
dtype (np.dtype|paddle.dtype|str, optional): Data type of the output, \
``int64`` by default.
name(str, optional): For detailed information, please refer \
to :ref:`api_guide_Name`. Usually name is no need to set and \
None by default.
Returns:
Tensor, The output sequence mask. Tensor with shape [d_1, d_2, ..., d_n, maxlen]
and data type of :code:`dtype`. The data type should be bool, float32, float64, int8,
int32 or int64.
Examples:
.. code-block:: pycon
>>> import paddle
>>> lengths = paddle.to_tensor([10, 9, 8])
>>> mask = paddle.nn.functional.sequence_mask(lengths)
>>> print(mask.numpy())
[[1 1 1 1 1 1 1 1 1 1]
[1 1 1 1 1 1 1 1 1 0]
[1 1 1 1 1 1 1 1 0 0]]
"""
return paddle.nn.functional.sequence_mask(x, maxlen, dtype, name)
python/paddle/static/nn/static_pylayer.py
+604
View File
@@ -0,0 +1,604 @@
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle
from paddle.base import core
from paddle.base.backward import _append_grad_suffix_
from paddle.base.framework import Variable, in_pir_mode
from paddle.base.libpaddle.pir import build_pylayer_op, cf_yield
from paddle.common_ops_import import LayerHelper, check_type, in_dygraph_mode
from paddle.utils import flatten, map_structure
# NOTE(MarioLulab): Borrowed from `python/paddle/static/nn/control_flow.py`
from .control_flow import BlockGuard, copy_var_to_parent_block
class StaticPyLayerBlockGuard(BlockGuard):
def __init__(self, block_manager):
check_type(
block_manager,
"block",
StaticPyLayerBlock,
"StaticPyLayerBlockGuard",
)
super().__init__(block_manager.helper.main_program)
self.block_manager = block_manager
def __enter__(self):
super().__enter__()
return self.block_manager
def __exit__(self, exc_type, exc_val, exc_tb):
self.block_manager.complete()
return super().__exit__(exc_type, exc_val, exc_tb)
class StaticPyLayerBlock:
def __init__(self, inputs, name=None, pylayer_context=None):
# used to specify the Variable type `Input` to `pylayer` op
self.fwd_inputs = [
each_input
for each_input in inputs
if isinstance(each_input, Variable)
] # filter non-Variable inputs
# used to specify the `Out` to `pylayer` op
self.fwd_outputs = []
self.context = pylayer_context
self.helper = LayerHelper("static_pylayer_block", name=name)
self.fwd_op_id = None
self._forward_block_id = None
self._backward_block_id = None
self.var_old_to_new = {}
def block(self, is_backward_block=False):
self.is_backward_block = is_backward_block
return StaticPyLayerBlockGuard(self)
@property
def forward_block_index(self):
return self._forward_block_id
@property
def backward_block_index(self):
return self._backward_block_id
@property
def fwd_op_index(self):
return self.fwd_op_id
def complete_forward_block(self):
inside_block = self.helper.main_program.current_block()
parent_block = self.helper.main_program.block(inside_block.parent_idx)
self._forward_block_id = inside_block.idx
step_scope = parent_block.create_var(
type=core.VarDesc.VarType.STEP_SCOPES
)
pylayer_op = parent_block.append_op(
type='pylayer',
inputs={
'Input': self.fwd_inputs,
},
outputs={"Out": self.fwd_outputs, "Scope": [step_scope]},
attrs={
'blocks': [inside_block],
},
)
self.fwd_op_id = pylayer_op.idx
self.helper.main_program._sync_with_cpp()
def complete_backward_block(self):
inside_block = self.helper.main_program.current_block()
parent_block = self.helper.main_program.block(inside_block.parent_idx)
self._backward_block_id = inside_block.idx
# Set OpRole to `backward`. The operators marked as `backward` are expected to be pruned in PruneBackward.
for op in inside_block.ops:
op_role_attr_name = (
core.op_proto_and_checker_maker.kOpRoleAttrName()
)
backward = core.op_proto_and_checker_maker.OpRole.Backward
op.desc._set_attr(op_role_attr_name, backward)
inside_block._set_forward_block_idx(self.forward_block_index)
# NOTE(MarioLulab): The reason of renaming the var name in the inside block is that
# we need to associating `inside_grads` and `outside_grads` at
# runtime `RunImpl` in pylayer op
_rename_var_recursively_(inside_block, self.var_old_to_new)
# update `blocks` attr by appending backward_block
forward_block_desc = parent_block.program.block(
self.forward_block_index
).desc
backward_block_desc = inside_block.desc
parent_block.ops[self.fwd_op_index].desc.set_blocks_attr(
"blocks", [forward_block_desc, backward_block_desc]
)
# remove temporary vars created by `StaticPyLayerContext.saved_tensor`
if self.context:
for var in self.context.saved_vars:
if not inside_block.has_var(var.name):
raise ValueError(
f"{var.name} was saved in forward block but could not be found in backward block. Maybe {var.name} was renamed somewhere."
)
inside_block._remove_var(var.name)
self.helper.main_program._sync_with_cpp()
def complete(self):
if not self.is_backward_block:
return self.complete_forward_block()
else:
return self.complete_backward_block()
def _get_ctx_from_func_(func):
if func is None:
return None
fn_bind_args = getattr(func, "args", None)
if fn_bind_args is None:
return None
from paddle.jit.dy2static.py_layer import StaticPyLayerContext
fn_ctx = None
if len(fn_bind_args) > 0 and isinstance(
fn_bind_args[0], StaticPyLayerContext
):
fn_ctx = fn_bind_args[0]
return fn_ctx
def _rename_var_recursively_(cur_block, var_old_to_new):
"""
Rename the var both the Variable instances and all ops' input and output arg names
in `cur_block` based on dict `var_old_to_new`.
Dict `var_old_to_new` should be the following format:
{
old_name_0 : new_name_0,
old_name_1 : new_name_1,
...
old_name_n : new_name_n,
}
"""
for old_var_name, new_var_name in var_old_to_new.items():
# NOTE(MarioLulab): The reason why not using `Block._rename_var`` is that `Block._rename_var` will raise ValueError, when `old_var_name` does not correspond to a Variable instance in Block.
if cur_block.has_var(old_var_name):
# `Block.desc._rename_var` can rename var in block and then rename var name in all ops
cur_block.desc._rename_var(
old_var_name.encode(), new_var_name.encode()
)
else:
# When cur_block does not have the var, `Block.desc._rename_var` can't rename var name in ops.
# In this case, we should traverse all ops and perform renaming manually.
for op in cur_block.ops:
op._rename_input(old_var_name, new_var_name)
op._rename_output(old_var_name, new_var_name)
# NOTE(MarioLulab): block attr type with the name of "blocks" or "sub_block" indicates
# the block might be executed. We should rename the var name in these blocks recursively
block_attr_names = ["blocks", "sub_block"]
for op in cur_block.ops:
for attr_name in op.all_attrs():
if attr_name not in block_attr_names:
continue
if op.attr_type(attr_name) == core.AttrType.BLOCK:
sub_block_id = op._block_attr_id(attr_name)
sub_block = cur_block.program.block(sub_block_id)
_rename_var_recursively_(sub_block, var_old_to_new)
elif op.attr_type(attr_name) == core.AttrType.BLOCKS:
sub_blocks_ids = op._blocks_attr_ids(attr_name)
for sub_block_id in sub_blocks_ids:
sub_block = cur_block.program.block(sub_block_id)
_rename_var_recursively_(sub_block, var_old_to_new)
def copy_var_from_parent_block(parent_block_var, layer_helper):
if not isinstance(parent_block_var, Variable):
return parent_block_var
prog = layer_helper.main_program
current_block = prog.current_block()
if (
parent_block_var.type == core.VarDesc.VarType.DENSE_TENSOR_ARRAY
and current_block._find_var_recursive(parent_block_var.name)
):
current_block_var = parent_block_var
else:
current_block_var = current_block.create_var(
dtype=parent_block_var.dtype,
shape=parent_block_var.shape,
type=parent_block_var.type,
)
paddle.assign(parent_block_var, current_block_var)
return current_block_var
class PyLayerBackwardFunction:
_register_backward_funcs = []
def __init__(self, backward_function, hook_check_func):
if backward_function is None or not callable(backward_function):
raise TypeError('func must be a Python function')
self._func = backward_function
# Note: Used to verify the number of `Value` inputs to ``forward_fn`` the same as the
# number of `Value` outputs to ``backward_fn``, and the number of `Value` outputs to ``forward_fn``
# the same as the number of `Value` inputs to ``backward_fn``.
self._hook_check_func = hook_check_func
'''
Why record self here?
For increasing reference count of self.
It seems that to release Python object
whose reference count is 1 would cause
segmentation fault error in C++ side.
May be lack of Python GC in C++ side?
'''
PyLayerBackwardFunction._register_backward_funcs.append(self)
def __call__(self, *output_grads):
assert self._hook_check_func
input_grads = self._func(*output_grads)
if not isinstance(input_grads, (list, tuple)):
input_grads = (input_grads,)
self._hook_check_func(output_grads, input_grads)
input_grads = [
input_grad
for input_grad in flatten(input_grads)
if isinstance(input_grad, (paddle.pir.Value, type(None)))
]
return input_grads
def static_pylayer(forward_fn, inputs, backward_fn=None, name=None):
"""
This API returns ``forward_fn(inputs)``, and two sub-block are created based on
the logic of ``forward_fn`` and ``backward_fn``, with the operator ``pylayer``
holding information about the two blocks.
``forward_fn`` and ``backward_fn`` should return a nest structure of Variables.
A nest structure of Variables in PaddlePaddle is Variable(s), or tuple of Variables, or
list of Variables.
Note:
1. If ``backward_fn`` is not None, user needs to keep the number of `Variable` inputs to ``forward_fn`` the same as the
number of `Variable` outputs to ``backward_fn``, and the number of `Variable` outputs to ``forward_fn``
the same as the number of `Variable` inputs to ``backward_fn``.
2. If ``backward_fn`` is None, ``stop_gradient`` attr of all Variable in ``inputs`` is expected to be True.
Otherwise it might get unexpected results in backward propagation.
3. This API can only be used under static graph mode.
Args:
forward_fn (callable): A callable to be performed in forward propagation
inputs (list[Variable]): The list of input Variable to the ``forward_fn``
backward_fn (callable, optional): A callable to be performed in backward propagation. Default: None, which means no need to do backward propagation.
name (str, optional): The default value is ``None`` . Normally users
don't have to set this parameter. For more information, please
refer to :ref:`api_guide_Name` .
Returns:
Variable|list(Variable)|tuple(Variable): returns the output of ``forward_fn(inputs)``
Examples:
.. code-block:: pycon
>>> import paddle
>>> import numpy as np
>>> paddle.enable_static()
>>> def forward_fn(x):
... return paddle.exp(x)
>>> def backward_fn(dy):
... return 2 * paddle.exp(dy)
>>> main_program = paddle.static.Program()
>>> start_program = paddle.static.Program()
>>> place = paddle.CPUPlace()
>>> exe = paddle.static.Executor(place)
>>> with paddle.static.program_guard(main_program, start_program):
... data = paddle.static.data(name="X", shape=[None, 5], dtype="float32")
... data.stop_gradient = False
... ret = paddle.static.nn.static_pylayer(forward_fn, [data], backward_fn)
... data_grad = paddle.static.gradients([ret], data)[0]
>>> exe.run(start_program)
>>> x = np.array([[1.0, 2.0, 3.0, 4.0, 5.0]], dtype=np.float32)
>>> x, x_grad, y = exe.run(
... main_program,
... feed={"X": x},
... fetch_list=[data, data_grad, ret],
... )
>>> print(x)
[[1. 2. 3. 4. 5.]]
>>> print(x_grad)
[[5.4365635 5.4365635 5.4365635 5.4365635 5.4365635]]
>>> print(y)
[[ 2.7182817 7.389056 20.085537 54.59815 148.41316 ]]
"""
assert in_dygraph_mode() is False, (
"please use PyLayer instead of static_pylayer in dygraph mode"
)
assert isinstance(inputs, list)
if backward_fn is None:
for input_var in inputs:
if input_var.stop_gradient is False:
raise ValueError(
f"``stop_gradient`` attr of all inputs to ``forward_fn`` are expected to be True, when ``backward_fn == None``, but {input_var.name}.stop_gradient got {input_var.stop_gradient}"
)
# judge if in dy2st or not, by checking binding args of `forward_fn` and `backward_fn`
fwd_fn_ctx = _get_ctx_from_func_(forward_fn)
bwd_fn_ctx = _get_ctx_from_func_(backward_fn)
static_pylayer_context = (
fwd_fn_ctx if fwd_fn_ctx and (fwd_fn_ctx == bwd_fn_ctx) else None
)
if in_pir_mode():
fwd_inputs = [
inp for inp in flatten(inputs) if isinstance(inp, paddle.pir.Value)
]
pylayer_op = build_pylayer_op(fwd_inputs)
outputs = None
if forward_fn is not None:
if not callable(forward_fn):
raise ValueError("`forward_fn` should be callable")
with pylayer_op.forward_block():
outputs = forward_fn(*inputs)
if outputs is None:
return None
fwd_outputs = [
out
for out in flatten(outputs)
if isinstance(out, paddle.pir.Value)
]
with pylayer_op.forward_block():
if fwd_outputs is not None:
cf_yield(flatten(fwd_outputs))
pylayer_op.update_output()
if backward_fn is not None:
if not callable(backward_fn):
raise ValueError("`backward_fn` should be callable")
def hook_inputs_outputs_check_function(output_grads, input_grads):
# 1. Verify the number of `Value` inputs to ``forward_fn`` the same as the
# number of `Value` outputs to ``backward_fn``
forward_inputs = [
x
for x in flatten(inputs)
if isinstance(x, paddle.pir.Value)
]
input_grads = [
x
for x in flatten(input_grads)
if isinstance(x, (paddle.pir.Value, type(None)))
]
if len(input_grads) != len(forward_inputs):
raise ValueError(
f"The number of input grads should be equal to the number of inputs, but got {len(input_grads)} and {len(forward_inputs)}."
)
for inp_grad, fwd_input in zip(input_grads, forward_inputs):
# NOTE: inp_grad will be None if fwd_input.stop_gradients=True
if inp_grad is None:
continue
assert inp_grad.dtype == fwd_input.dtype, (
f"dtype of inp_grad({inp_grad.dtype}) and fwd_input({fwd_input.dtype}) should be the same"
)
assert inp_grad.shape == fwd_input.shape, (
f"shape of inp_grad({inp_grad.shape}) and fwd_input({fwd_input.shape}) should be the same"
)
if fwd_input.is_dist():
# NOTE: placements may be not the same, so do not check it.
assert inp_grad.is_dist(), (
"fwd_input and inp_grad should both be distributed"
)
assert (
fwd_input.dist_attr().process_mesh
== inp_grad.dist_attr().process_mesh
), (
f"process_mesh of fwd_input({fwd_input.dist_attr().process_mesh}) and inp_grad({inp_grad.dist_attr().process_mesh}) should be the same"
)
else:
assert inp_grad.type() == fwd_input.type(), (
f"type of inp_grad({inp_grad.type()}) and fwd_input({fwd_input.type()}) should be the same"
)
# 2. Verify the number of `Value` outputs to ``forward_fn``
# the same as the number of `Value` inputs to ``backward_fn``
forward_outputs = [
x
for x in flatten(fwd_outputs)
if isinstance(x, paddle.pir.Value)
]
if len(output_grads) != len(forward_outputs):
raise ValueError(
f"The number of output grads should be equal to the number of outputs, but got {len(output_grads)} and {len(fwd_outputs)}."
)
for out_grad, fwd_output in zip(output_grads, forward_outputs):
if out_grad is None:
continue
assert out_grad.dtype == fwd_output.dtype, (
f"dtype of out_grad({out_grad.dtype}) and fwd_output({fwd_output.dtype}) should be the same"
)
assert out_grad.shape == fwd_output.shape, (
f"shape of out_grad({out_grad.shape}) and fwd_output({fwd_output.shape}) should be the same"
)
if fwd_output.is_dist():
# NOTE: placements may be not the same, so do not check it.
assert out_grad.is_dist(), (
"fwd_output and out_grad should both be distributed"
)
assert (
fwd_output.dist_attr().process_mesh
== out_grad.dist_attr().process_mesh
), (
f"process_mesh of fwd_output({fwd_output.dist_attr().process_mesh}) and out_grad({out_grad.dist_attr().process_mesh}) should be the same"
)
else:
assert out_grad.type() == fwd_output.type(), (
f"type of out_grad({out_grad.type}) and fwd_output({fwd_output.type}) should be the same"
)
bwd_fn = PyLayerBackwardFunction(
backward_fn, hook_check_func=hook_inputs_outputs_check_function
)
pylayer_op.register_backward_function(bwd_fn)
# NOTE: Replace pir.Value of `outputs` with pylayer_op.result, because value of `outputs` which is inside pylayer block can't be reference outside the block.
op_result_idx = 0
outputs = flatten(outputs)
for i in range(len(outputs)):
if isinstance(outputs[i], paddle.pir.Value):
outputs[i] = pylayer_op.results()[op_result_idx]
op_result_idx += 1
return outputs[0] if len(outputs) == 1 else outputs
check_type(name, "name", (str, type(None)), "base.layers.static_pylayer")
helper = LayerHelper('static_pylayer', **locals())
copy_to_parent_func = lambda var: copy_var_to_parent_block(var, helper)
assert forward_fn is not None and callable(forward_fn)
pylayer_block_manager = StaticPyLayerBlock(
inputs, pylayer_context=static_pylayer_context
)
with pylayer_block_manager.block(is_backward_block=False) as mgr:
origin_output = forward_fn(*inputs)
if origin_output is not None:
output = map_structure(copy_to_parent_func, origin_output)
mgr.fwd_outputs = [
x for x in flatten(output) if isinstance(x, Variable)
]
else:
mgr.fwd_outputs = []
current_block = helper.main_program.current_block()
current_block._sync_with_cpp()
if backward_fn is not None:
assert callable(backward_fn)
if origin_output is None:
output = []
# **Create the backward input** from the output of the op to build the
# backward block, and then delete it.
grad_var_ins = []
for fwd_var in pylayer_block_manager.fwd_outputs:
fwd_var_name = fwd_var.name
bwd_var_name = _append_grad_suffix_(fwd_var_name)
if not current_block.desc.has_var_recursive(fwd_var_name.encode()):
raise ValueError(
f"Grad var {bwd_var_name} , we can't find its related forward var {fwd_var_name}"
)
var = current_block.create_var(
dtype=fwd_var.dtype,
shape=fwd_var.shape,
type=fwd_var.type,
name=bwd_var_name,
)
grad_var_ins.append(var)
copy_from_parent_func = lambda var: copy_var_from_parent_block(
var, helper
)
assert isinstance(grad_var_ins, list)
with pylayer_block_manager.block(is_backward_block=True) as mgr:
# Step1. Copy var from parent block
inside_block_inputs = map_structure(
copy_from_parent_func, grad_var_ins
)
# Step2. Do backward propagation
grad_origin_output = backward_fn(*inside_block_inputs)
if grad_origin_output is not None:
# Step3. Check the number of inputs to ``forward_fn`` the
# same as the number of outputs to ``backward_fn``
flat_grad_origin = flatten(grad_origin_output)
# NOTE(MarioLulab): ``current_block`` was defined outside
forward_input_names = current_block.ops[
pylayer_block_manager.fwd_op_index
].desc.input_arg_names()
assert len(forward_input_names) == len(flat_grad_origin), (
f"needs to keep the number of inputs to ``forward_fn`` the same as the number of outputs to ``backward_fn``, \
but got {len(forward_input_names)} and {len(flat_grad_origin)}"
)
# Step4. Rename var name with suffix of "@GRAD"
for bwd_output, fwd_input_name in zip(
flat_grad_origin, forward_input_names
):
# NOTE(MarioLulab): Because `flat_grad_origin` are the Variables inside the backward block, which one by one corresponds
# to the gradients of the inputs to the forward function, we need to establish a link between `flat_grad_origin`,
# and the Variable outside the backward block which represent the gradient of the input ot the forward function.
# The approach we have taken is renaming `flat_grad_origin` by forward input name with suffix of "@GRAD", and aligning
# the order of `Out@GRAD` in `pylayer_grad` op with `flat_grad_origin`. And in the runtime `RunImpl` in `pylayer_grad` op,
# we will find inside_grad with the name of forward input name with suffix of "@GRAD" in the scope, and assign `inside_grads`
# to `outside_grads`.
#
# Example:
# after run the code below to create forward and backward block:
#
# out = forward_fn(x, y) # create forward block
# x_grad, y_grad = backward_fn(out_grad) # create backward block
#
# x.name is "X", y.name is "Y", and out.name is "tmp_0", but x_grad.name is "_generate_0", y_grad.name is "_generate_1".
# we rename x_grad by "X@GRAD", and y_grad by "Y@GRAD" inside backward block.
# One thing to keep in mind is that we assume there were no Variable naming "X@GRAD" inside backward block before performing rename operation.
# TODO(MarioLulab): We will validate the assumption above is whether a strong hypothesis or not.
# attach old var name into new
if isinstance(bwd_output, Variable):
bwd_out_new = _append_grad_suffix_(
fwd_input_name
) # "X" => "X@GRAD"
mgr.var_old_to_new[bwd_output.name] = (
bwd_out_new # e.g. "tmp_0.mean_0": "X@GRAD"
)
# **Delete the backward input**
for bwd_var in grad_var_ins:
current_block._remove_var(bwd_var.name)
if origin_output is None:
return None
return output