chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,68 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
from ...base.initializer import set_global_initializer
|
||||
from .assign import (
|
||||
Assign,
|
||||
NumpyArrayInitializer, # noqa: F401
|
||||
)
|
||||
from .bilinear import Bilinear
|
||||
from .constant import (
|
||||
Constant,
|
||||
ConstantInitializer, # noqa: F401
|
||||
)
|
||||
from .dirac import Dirac
|
||||
from .initializer import (
|
||||
Initializer, # noqa: F401
|
||||
calculate_gain,
|
||||
)
|
||||
from .kaiming import (
|
||||
KaimingNormal,
|
||||
KaimingUniform,
|
||||
MSRAInitializer, # noqa: F401
|
||||
)
|
||||
from .normal import (
|
||||
Normal,
|
||||
NormalInitializer, # noqa: F401
|
||||
TruncatedNormal,
|
||||
TruncatedNormalInitializer, # noqa: F401
|
||||
)
|
||||
from .orthogonal import Orthogonal
|
||||
from .uniform import (
|
||||
Uniform,
|
||||
UniformInitializer, # noqa: F401
|
||||
)
|
||||
from .xavier import (
|
||||
XavierInitializer, # noqa: F401
|
||||
XavierNormal,
|
||||
XavierUniform,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'Bilinear',
|
||||
'Constant',
|
||||
'KaimingUniform',
|
||||
'KaimingNormal',
|
||||
'XavierNormal',
|
||||
'XavierUniform',
|
||||
'Assign',
|
||||
'Normal',
|
||||
'TruncatedNormal',
|
||||
'Uniform',
|
||||
'Orthogonal',
|
||||
'Dirac',
|
||||
'set_global_initializer',
|
||||
'calculate_gain',
|
||||
]
|
||||
@@ -0,0 +1,297 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops
|
||||
|
||||
from ...base import core, framework, unique_name
|
||||
from ...base.data_feeder import check_type
|
||||
from ...base.framework import (
|
||||
_current_expected_place,
|
||||
in_dygraph_mode,
|
||||
in_pir_mode,
|
||||
)
|
||||
from .initializer import Initializer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
import numpy.typing as npt
|
||||
|
||||
from paddle._typing import NestedSequence
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class NumpyArrayInitializer(Initializer):
|
||||
"""Init an parameter with an numpy array
|
||||
This api initialize the tensor by numpy array.
|
||||
|
||||
Args:
|
||||
value (numpy): numpy array to initialize the tensor
|
||||
|
||||
Returns:
|
||||
A Tensor initialized by numpy.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, value: npt.NDArray[Any]) -> None:
|
||||
import numpy
|
||||
|
||||
assert isinstance(value, numpy.ndarray)
|
||||
super().__init__()
|
||||
self._value = value
|
||||
|
||||
def forward(
|
||||
self, var: paddle.Tensor, block: paddle.pir.Block | None = None
|
||||
) -> paddle.Tensor | None:
|
||||
"""Initialize the input tensor with Numpy array.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The initialization op
|
||||
"""
|
||||
assert not (
|
||||
isinstance(var, framework.EagerParamBase) and var.is_dist()
|
||||
), "Currently, assign initializer not support lazy init for dist param."
|
||||
block = self._check_block(block)
|
||||
|
||||
assert isinstance(
|
||||
var, (framework.Variable, paddle.pir.core.ParameterMeta)
|
||||
)
|
||||
assert isinstance(block, (framework.Block, paddle.pir.Block))
|
||||
|
||||
# to be compatible of fp16 initializers
|
||||
origin_dtype = var.dtype
|
||||
if origin_dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
]:
|
||||
out_dtype = core.VarDesc.VarType.FP32
|
||||
np_value = self._value.astype("float32")
|
||||
out_var = block.create_var(
|
||||
name=unique_name.generate(
|
||||
".".join(['numpy_array_init', var.name, 'tmp'])
|
||||
),
|
||||
shape=var.shape,
|
||||
dtype=out_dtype,
|
||||
type=core.VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
)
|
||||
elif origin_dtype in [core.DataType.FLOAT16, core.DataType.BFLOAT16]:
|
||||
out_var = var
|
||||
out_dtype = core.DataType.FLOAT32
|
||||
np_value = self._value.astype("float32")
|
||||
else:
|
||||
out_var = var
|
||||
out_dtype = origin_dtype
|
||||
np_value = self._value
|
||||
|
||||
if out_dtype in (core.VarDesc.VarType.FP32, core.DataType.FLOAT32):
|
||||
value_name = "values"
|
||||
values = [float(v) for v in np_value.flat]
|
||||
elif out_dtype in (core.VarDesc.VarType.FP64, core.DataType.FLOAT64):
|
||||
value_name = "values"
|
||||
values = [float(v) for v in np_value.flat]
|
||||
elif out_dtype in (core.VarDesc.VarType.INT32, core.DataType.INT32):
|
||||
value_name = "values"
|
||||
values = [int(v) for v in np_value.flat]
|
||||
elif out_dtype in (
|
||||
core.VarDesc.VarType.INT8,
|
||||
core.VarDesc.VarType.UINT8,
|
||||
core.DataType.INT8,
|
||||
core.DataType.UINT8,
|
||||
):
|
||||
value_name = "int8_values"
|
||||
values = [int(v) for v in np_value.flat]
|
||||
else:
|
||||
raise ValueError(f"Unsupported dtype {self._value.dtype}")
|
||||
if self._value.size > 1024 * 1024 * 1024:
|
||||
raise ValueError(
|
||||
"The size of input is too big. Please consider "
|
||||
"saving it to file and 'load_op' to load it"
|
||||
)
|
||||
|
||||
if in_dygraph_mode():
|
||||
_C_ops.assign_value_(
|
||||
out_var,
|
||||
list(self._value.shape),
|
||||
out_dtype,
|
||||
values,
|
||||
_current_expected_place(),
|
||||
)
|
||||
if origin_dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
core.DataType.FLOAT16,
|
||||
core.DataType.BFLOAT16,
|
||||
]:
|
||||
var_tmp = _C_ops.cast(out_var, origin_dtype)
|
||||
var_tmp._share_underline_tensor_to(var)
|
||||
else:
|
||||
out_var._share_underline_tensor_to(var)
|
||||
return None
|
||||
elif in_pir_mode():
|
||||
out_var = _C_ops.assign_value(
|
||||
list(self._value.shape),
|
||||
out_dtype,
|
||||
values,
|
||||
_current_expected_place(),
|
||||
)
|
||||
if origin_dtype in [core.DataType.FLOAT16, core.DataType.BFLOAT16]:
|
||||
out_var = _C_ops.cast(out_var, origin_dtype)
|
||||
return out_var
|
||||
else:
|
||||
op = block.append_op(
|
||||
type='assign_value',
|
||||
outputs={'Out': out_var},
|
||||
attrs={
|
||||
'dtype': out_dtype,
|
||||
'shape': list(self._value.shape),
|
||||
value_name: values,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if origin_dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
]:
|
||||
block.append_op(
|
||||
type="cast",
|
||||
inputs={"X": out_var},
|
||||
outputs={"Out": var},
|
||||
attrs={
|
||||
"in_dtype": out_var.dtype,
|
||||
"out_dtype": origin_dtype,
|
||||
},
|
||||
)
|
||||
|
||||
var.op = op
|
||||
return op
|
||||
|
||||
|
||||
class Assign(NumpyArrayInitializer):
|
||||
"""Init an parameter with a numpy array, list, or tensor.
|
||||
|
||||
Args:
|
||||
value (Tensor|numpy.ndarray|list|tuple): numpy array, list, tuple, or tensor to initialize the parameter.
|
||||
name(str|None, optional): Normally there is no need for user to set this
|
||||
property. For more information, please refer to :ref:`api_guide_Name`. Default is None.
|
||||
|
||||
Returns:
|
||||
A parameter initialized by the input numpy array, list, or tensor.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> import numpy as np
|
||||
|
||||
>>> # numpy array
|
||||
>>> data_1 = paddle.ones(shape=[1, 2], dtype='float32')
|
||||
>>> weight_attr_1 = paddle.ParamAttr(
|
||||
... name="linear_weight_1",
|
||||
... initializer=paddle.nn.initializer.Assign(np.array([[2, 2], [2, 2]])),
|
||||
... )
|
||||
>>> bias_attr_1 = paddle.ParamAttr(
|
||||
... name="linear_bias_1",
|
||||
... initializer=paddle.nn.initializer.Assign(np.array([2, 2])),
|
||||
... )
|
||||
>>> linear_1 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_1, bias_attr=bias_attr_1)
|
||||
>>> print(linear_1.weight.numpy())
|
||||
[[2. 2.]
|
||||
[2. 2.]]
|
||||
>>> print(linear_1.bias.numpy())
|
||||
[2. 2.]
|
||||
|
||||
>>> res_1 = linear_1(data_1)
|
||||
>>> print(res_1.numpy())
|
||||
[[6. 6.]]
|
||||
|
||||
>>> # python list
|
||||
>>> data_2 = paddle.ones(shape=[1, 2], dtype='float32')
|
||||
>>> weight_attr_2 = paddle.ParamAttr(
|
||||
... name="linear_weight_2",
|
||||
... initializer=paddle.nn.initializer.Assign([[2, 2], [2, 2]]),
|
||||
... )
|
||||
>>> bias_attr_2 = paddle.ParamAttr(
|
||||
... name="linear_bias_2",
|
||||
... initializer=paddle.nn.initializer.Assign([2, 2]),
|
||||
... )
|
||||
>>> linear_2 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_2, bias_attr=bias_attr_2)
|
||||
>>> print(linear_2.weight.numpy())
|
||||
[[2. 2.]
|
||||
[2. 2.]]
|
||||
>>> print(linear_2.bias.numpy())
|
||||
[2. 2.]
|
||||
|
||||
>>> res_2 = linear_2(data_2)
|
||||
>>> print(res_2.numpy())
|
||||
[[6. 6.]]
|
||||
|
||||
>>> # tensor
|
||||
>>> data_3 = paddle.ones(shape=[1, 2], dtype='float32')
|
||||
>>> weight_attr_3 = paddle.ParamAttr(
|
||||
... name="linear_weight_3",
|
||||
... initializer=paddle.nn.initializer.Assign(paddle.full([2, 2], 2)),
|
||||
... )
|
||||
>>> bias_attr_3 = paddle.ParamAttr(
|
||||
... name="linear_bias_3",
|
||||
... initializer=paddle.nn.initializer.Assign(paddle.full([2], 2)),
|
||||
... )
|
||||
>>> linear_3 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_3, bias_attr=bias_attr_3)
|
||||
>>> print(linear_3.weight.numpy())
|
||||
[[2. 2.]
|
||||
[2. 2.]]
|
||||
>>> print(linear_3.bias.numpy())
|
||||
[2. 2.]
|
||||
|
||||
>>> res_3 = linear_3(data_3)
|
||||
>>> print(res_3.numpy())
|
||||
[[6. 6.]]
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
value: npt.NDArray[Any]
|
||||
| Sequence[NestedSequence[int | float | bool | complex]]
|
||||
| paddle.Tensor,
|
||||
name: str | None = None,
|
||||
) -> None:
|
||||
import numpy
|
||||
|
||||
check_type(
|
||||
value,
|
||||
'value',
|
||||
(numpy.ndarray, list, tuple, paddle.static.Variable),
|
||||
'Assign',
|
||||
)
|
||||
|
||||
if isinstance(value, (list, tuple)):
|
||||
value = numpy.array(value)
|
||||
|
||||
# TODO: value is already is a tensor, accounting efficiency maybe it does not need to convert tensor to numpy data and then initialized.
|
||||
if isinstance(value, paddle.static.Variable):
|
||||
value = value.numpy(False)
|
||||
|
||||
super().__init__(value)
|
||||
@@ -0,0 +1,225 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops, pir
|
||||
|
||||
from ...base import core, framework, unique_name
|
||||
from ...base.framework import (
|
||||
_current_expected_place,
|
||||
in_dygraph_mode,
|
||||
in_pir_mode,
|
||||
)
|
||||
from .initializer import Initializer
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class Bilinear(Initializer):
|
||||
"""
|
||||
This initializer can be used in transposed convolution operator to
|
||||
act as upsampling. Users can upsample a feature map with shape of
|
||||
(B, C, H, W) by any integer factor.
|
||||
|
||||
Returns:
|
||||
Bilinear initializer instance objects.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import math
|
||||
|
||||
>>> import paddle
|
||||
>>> import paddle.nn as nn
|
||||
>>> from paddle.regularizer import L2Decay
|
||||
|
||||
>>> factor = 2
|
||||
>>> C = 2
|
||||
>>> B = 8
|
||||
>>> H = W = 32
|
||||
>>> w_attr = paddle.ParamAttr(
|
||||
... learning_rate=0.0,
|
||||
... regularizer=L2Decay(0.0),
|
||||
... initializer=nn.initializer.Bilinear(),
|
||||
... )
|
||||
>>> data = paddle.rand([B, 3, H, W], dtype='float32')
|
||||
>>> conv_up = nn.Conv2DTranspose(
|
||||
... 3,
|
||||
... out_channels=C,
|
||||
... kernel_size=2 * factor - factor % 2,
|
||||
... padding=int(math.ceil((factor - 1) / 2.0)),
|
||||
... stride=factor,
|
||||
... weight_attr=w_attr,
|
||||
... bias_attr=False,
|
||||
... )
|
||||
>>> x = conv_up(data)
|
||||
|
||||
Where, `out_channels=C` and `groups=C` means this is channel-wise transposed
|
||||
convolution. The filter shape will be (C, 1, K, K) where K is `kernel_size`,
|
||||
This initializer will set a (K, K) interpolation kernel for every channel
|
||||
of the filter identically. The resulting shape of the output feature map
|
||||
will be (B, C, factor * H, factor * W). Note that the learning rate and the
|
||||
weight decay are set to 0 in order to keep coefficient values of bilinear
|
||||
interpolation unchanged during training.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Constructor for BilinearInitializer."""
|
||||
super().__init__()
|
||||
|
||||
def forward(
|
||||
self, var: paddle.Tensor, block: pir.Block | None = None
|
||||
) -> paddle.Tensor | None:
|
||||
"""Initialize the input tensor with Bilinear initialization.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The initialization op
|
||||
"""
|
||||
assert not (
|
||||
isinstance(var, framework.EagerParamBase) and var.is_dist()
|
||||
), (
|
||||
"Currently, Bilinear initializer not support lazy init for dist param."
|
||||
)
|
||||
block = self._check_block(block)
|
||||
|
||||
if not isinstance(var, (framework.Variable, pir.core.ParameterMeta)):
|
||||
raise ValueError(
|
||||
"var must be framework.Variable or pir.core.ParameterMeta."
|
||||
)
|
||||
|
||||
if not isinstance(block, (framework.Block, pir.Block)):
|
||||
raise ValueError("block must be framework.Block or pir.Block.")
|
||||
|
||||
shape = var.shape
|
||||
if len(shape) != 4:
|
||||
raise ValueError("the length of shape must be 4.")
|
||||
if shape[2] != shape[3]:
|
||||
raise ValueError("shape[2] must be equal to shape[3].")
|
||||
|
||||
weight = np.zeros(np.prod(var.shape), dtype='float32')
|
||||
size = shape[3]
|
||||
# factor
|
||||
f = np.ceil(size / 2.0)
|
||||
# center
|
||||
c = (2 * f - 1 - f % 2) / (2.0 * f)
|
||||
for i in range(np.prod(shape)):
|
||||
x = i % size
|
||||
y = (i / size) % size
|
||||
weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))
|
||||
weight = np.reshape(weight, shape)
|
||||
|
||||
# to be compatible of fp16 initializers
|
||||
if var.dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
core.VarDesc.VarType.FP64,
|
||||
]:
|
||||
out_dtype = core.VarDesc.VarType.FP32
|
||||
out_var = block.create_var(
|
||||
name=unique_name.generate(
|
||||
".".join(['bilinear_init', var.name, 'tmp'])
|
||||
),
|
||||
shape=var.shape,
|
||||
dtype=out_dtype,
|
||||
type=core.VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
)
|
||||
elif var.dtype in [
|
||||
core.DataType.FLOAT16,
|
||||
core.DataType.BFLOAT16,
|
||||
core.DataType.FLOAT64,
|
||||
]:
|
||||
out_dtype = core.DataType.FLOAT32
|
||||
out_var = var
|
||||
else:
|
||||
out_dtype = var.dtype
|
||||
out_var = var
|
||||
|
||||
if out_dtype in (core.VarDesc.VarType.FP32, core.DataType.FLOAT32):
|
||||
value_name = "values"
|
||||
values = [float(v) for v in weight.flat]
|
||||
else:
|
||||
raise TypeError(f"Unsupported dtype {var.dtype}")
|
||||
|
||||
if np.prod(shape) > 1024 * 1024:
|
||||
raise ValueError("The size of input is too big. ")
|
||||
|
||||
if in_dygraph_mode():
|
||||
_C_ops.assign_value_(
|
||||
out_var,
|
||||
list(shape),
|
||||
out_dtype,
|
||||
values,
|
||||
_current_expected_place(),
|
||||
)
|
||||
if var.dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
core.VarDesc.VarType.FP64,
|
||||
]:
|
||||
var_tmp = _C_ops.cast(out_var, var.dtype)
|
||||
var_tmp._share_underline_tensor_to(var)
|
||||
else:
|
||||
out_var._share_underline_tensor_to(var)
|
||||
return None
|
||||
elif in_pir_mode():
|
||||
out_var = _C_ops.assign_value(
|
||||
list(shape),
|
||||
out_dtype,
|
||||
values,
|
||||
_current_expected_place(),
|
||||
)
|
||||
if var.dtype in [
|
||||
core.DataType.FLOAT16,
|
||||
core.DataType.BFLOAT16,
|
||||
core.DataType.FLOAT64,
|
||||
]:
|
||||
out_var = _C_ops.cast(out_var, var.dtype)
|
||||
return out_var
|
||||
else:
|
||||
op = block.append_op(
|
||||
type='assign_value',
|
||||
outputs={'Out': [out_var]},
|
||||
attrs={
|
||||
'dtype': out_dtype,
|
||||
'shape': list(shape),
|
||||
value_name: values,
|
||||
},
|
||||
)
|
||||
|
||||
if var.dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
core.VarDesc.VarType.FP64,
|
||||
]:
|
||||
block.append_op(
|
||||
type="cast",
|
||||
inputs={"X": out_var},
|
||||
outputs={"Out": var},
|
||||
attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
|
||||
)
|
||||
|
||||
var.op = op
|
||||
return op
|
||||
@@ -0,0 +1,144 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops
|
||||
|
||||
from ...base import core, framework
|
||||
from ...base.framework import (
|
||||
_current_expected_place,
|
||||
in_dygraph_mode,
|
||||
in_dynamic_or_pir_mode,
|
||||
)
|
||||
|
||||
# TODO: define the initializers of Constant in neural network
|
||||
from .initializer import Initializer
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class ConstantInitializer(Initializer):
|
||||
"""Implements the constant initializer
|
||||
|
||||
Args:
|
||||
value (float32, optional): constant value to initialize the variable. Default: 0.0.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, value: float = 0.0, force_cpu: bool = False) -> None:
|
||||
assert value is not None
|
||||
super().__init__()
|
||||
self._value = value
|
||||
self._force_cpu = force_cpu
|
||||
|
||||
def forward(
|
||||
self,
|
||||
var: paddle.Tensor,
|
||||
block: paddle.pir.Block | None = None,
|
||||
) -> paddle.Tensor | None:
|
||||
"""Initialize the input tensor with constant.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The initialization op
|
||||
"""
|
||||
|
||||
block = self._check_block(block)
|
||||
|
||||
assert isinstance(
|
||||
var,
|
||||
(
|
||||
framework.Variable,
|
||||
framework.EagerParamBase,
|
||||
paddle.pir.Value,
|
||||
paddle.pir.core.ParameterMeta,
|
||||
),
|
||||
)
|
||||
assert isinstance(block, (framework.Block, paddle.pir.Block))
|
||||
|
||||
if in_dynamic_or_pir_mode():
|
||||
place = _current_expected_place()
|
||||
if self._force_cpu:
|
||||
place = core.CPUPlace()
|
||||
if in_dygraph_mode():
|
||||
if isinstance(var, framework.EagerParamBase) and var.is_dist():
|
||||
out_var = _C_ops.full(
|
||||
var._local_shape, float(self._value), var.dtype, place
|
||||
)
|
||||
out_var = (
|
||||
paddle.distributed.auto_parallel.api.dtensor_from_local(
|
||||
out_var, var.process_mesh, var.placements
|
||||
)
|
||||
)
|
||||
out_var._share_underline_tensor_to(var)
|
||||
else:
|
||||
_C_ops.full_(
|
||||
var, var.shape, float(self._value), var.dtype, place
|
||||
)
|
||||
return None
|
||||
else:
|
||||
return _C_ops.full(
|
||||
var.shape, float(self._value), var.dtype, place
|
||||
)
|
||||
else:
|
||||
op = block.append_op(
|
||||
type="fill_constant",
|
||||
outputs={"Out": var},
|
||||
attrs={
|
||||
"shape": var.shape,
|
||||
"dtype": int(var.dtype),
|
||||
"value": float(self._value),
|
||||
'str_value': str(float(self._value)),
|
||||
'force_cpu': self._force_cpu,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
var.op = op
|
||||
return op
|
||||
|
||||
|
||||
class Constant(ConstantInitializer):
|
||||
"""Implement the constant initializer.
|
||||
|
||||
Args:
|
||||
value (float32|float64, optional): constant value to initialize the parameter. Default: 0.0.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> import paddle.nn as nn
|
||||
|
||||
>>> paddle.seed(2023)
|
||||
>>> data = paddle.rand([30, 10, 2], dtype='float32')
|
||||
>>> linear = nn.Linear(2, 4, weight_attr=nn.initializer.Constant(value=2.0))
|
||||
>>> res = linear(data)
|
||||
>>> print(linear.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2, 4], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[2., 2., 2., 2.],
|
||||
[2., 2., 2., 2.]])
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, value: float = 0.0) -> None:
|
||||
if value is None:
|
||||
raise ValueError("value must not be none.")
|
||||
super().__init__(value=value, force_cpu=False)
|
||||
@@ -0,0 +1,366 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops, in_dynamic_mode, pir
|
||||
from paddle.utils import unique_name
|
||||
|
||||
from ... import base
|
||||
from ...base import core, framework
|
||||
from ...base.core import VarDesc
|
||||
from ...base.data_feeder import check_variable_and_dtype
|
||||
from ...base.framework import _current_expected_place
|
||||
from .initializer import Initializer
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class Dirac(Initializer):
|
||||
r"""Initialize the 3D/4D/5D Tensor with Dirac delta function.
|
||||
|
||||
It can reserve the feature of convolution layer input, which means that
|
||||
as many channels are reserved as possible.
|
||||
|
||||
In this initialize method, elements in the middle of convolution kernels will
|
||||
be set to 1 . The formula can be described as follow.
|
||||
|
||||
.. math::
|
||||
|
||||
X[d, d, shape[2]//2, shape[3]//2, ...]=1, \ d=0,1...N
|
||||
|
||||
where, ``N`` is the minimum value of ``in_channels`` and ``out_channels``
|
||||
|
||||
Args:
|
||||
groups(int|None, optional): 0-dimension of the Tensor will be divided by groups,
|
||||
each group has the same value. Default: 1.
|
||||
name(str|None, optional): The default value is None. Normally there is no need for user to set this
|
||||
property. For more information, please refer to :ref:`api_guide_Name`.
|
||||
|
||||
Returns:
|
||||
Dirac initializer instance objects.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
|
||||
>>> # 1. For kernel_size is uneven number:
|
||||
>>> attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Dirac())
|
||||
>>> conv = paddle.nn.Conv1D(3, 2, 3, weight_attr=attr)
|
||||
>>> print(conv.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2, 3, 3], dtype=float32, place=CPUPlace, stop_gradient=False,
|
||||
[[[0., 1., 0.],
|
||||
[0., 0., 0.],
|
||||
[0., 0., 0.]],
|
||||
[[0., 0., 0.],
|
||||
[0., 1., 0.],
|
||||
[0., 0., 0.]]])
|
||||
>>> input = paddle.rand([8, 3, 10])
|
||||
>>> output = conv(input)
|
||||
>>> output == input[:, 0:2, 1:9]
|
||||
>>> print(output.shape)
|
||||
paddle.Size([8, 2, 8])
|
||||
>>> # It means output is almost the same with input, 2 channels are reserved
|
||||
|
||||
>>> # 2. For kernel_size is even number:
|
||||
>>> attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Dirac())
|
||||
>>> conv = paddle.nn.Conv1D(3, 2, 4, weight_attr=attr)
|
||||
>>> print(conv.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2, 3, 4], dtype=float32, place=CPUPlace, stop_gradient=False,
|
||||
[[[0., 0., 1., 0.],
|
||||
[0., 0., 0., 0.],
|
||||
[0., 0., 0., 0.]],
|
||||
[[0., 0., 0., 0.],
|
||||
[0., 0., 1., 0.],
|
||||
[0., 0., 0., 0.]]])
|
||||
"""
|
||||
|
||||
def __init__(self, groups: int = 1, name: str | None = None) -> None:
|
||||
assert groups > 0 and isinstance(groups, int), (
|
||||
" 'groups' must be a positive integer. "
|
||||
)
|
||||
super().__init__()
|
||||
self._groups = groups
|
||||
|
||||
def __call__(
|
||||
self, var: paddle.Tensor, block: pir.Block | None = None
|
||||
) -> paddle.Tensor:
|
||||
"""Initialize the input tensor with dirac initializer.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The most critical OP(scatter) in this initializer, which contains 7~8 ops in total.
|
||||
"""
|
||||
assert not (
|
||||
isinstance(var, framework.EagerParamBase) and var.is_dist()
|
||||
), "Currently, dirac initializer not support lazy init for dist param."
|
||||
block = self._check_block(block)
|
||||
assert isinstance(
|
||||
var, (framework.Variable, paddle.pir.Value, pir.core.ParameterMeta)
|
||||
)
|
||||
assert isinstance(block, (framework.Block, pir.Block))
|
||||
check_variable_and_dtype(
|
||||
var, "Out", ['float16', 'bfloat16', 'float32', 'float64'], 'Dirac'
|
||||
)
|
||||
|
||||
assert len(var.shape) in [
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
], "Only Tensor with 3/4/5 dimensions can be initialized by Dirac"
|
||||
assert (var.shape[0] % self._groups) == 0, (
|
||||
"Tensor 0-dimension must be divisible by groups"
|
||||
)
|
||||
|
||||
if framework.in_pir_mode():
|
||||
if var.dtype != core.DataType.FLOAT32:
|
||||
out_dtype = core.DataType.FLOAT32
|
||||
out_var = var
|
||||
else:
|
||||
out_dtype = var.dtype
|
||||
out_var = var
|
||||
else:
|
||||
if var.dtype != VarDesc.VarType.FP32:
|
||||
out_dtype = VarDesc.VarType.FP32
|
||||
out_var = block.create_var(
|
||||
name=unique_name.generate(
|
||||
".".join(['dirac', var.name, 'tmp'])
|
||||
),
|
||||
shape=var.shape,
|
||||
dtype=out_dtype,
|
||||
type=VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
)
|
||||
else:
|
||||
out_dtype = var.dtype
|
||||
out_var = var
|
||||
|
||||
op = None
|
||||
if framework.in_dygraph_mode():
|
||||
with base.dygraph.no_grad():
|
||||
place = _current_expected_place()
|
||||
_C_ops.full_(
|
||||
out_var, out_var.shape, str(float(0)), out_dtype, place
|
||||
)
|
||||
elif framework.in_pir_mode():
|
||||
place = _current_expected_place()
|
||||
out_var = _C_ops.full(out_var.shape, float(0), out_dtype, place)
|
||||
else:
|
||||
block.append_op(
|
||||
type='fill_constant',
|
||||
inputs={},
|
||||
outputs={'Out': out_var},
|
||||
attrs={
|
||||
'value': float(0),
|
||||
'dtype': out_var.dtype,
|
||||
'shape': out_var.shape,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
origin_shape = var.shape
|
||||
num_per_group = origin_shape[0] // self._groups
|
||||
min_shape = min(num_per_group, origin_shape[1])
|
||||
|
||||
idx_list = []
|
||||
value_list = []
|
||||
strides = []
|
||||
prod = 1
|
||||
for dim in reversed(origin_shape):
|
||||
strides.insert(0, prod)
|
||||
prod *= dim
|
||||
for i in range(self._groups):
|
||||
for j in range(min_shape):
|
||||
value_list.append(1.0)
|
||||
offset = 0
|
||||
for k, stride in enumerate(strides):
|
||||
if k == 0:
|
||||
offset += (j + i * num_per_group) * stride
|
||||
elif k == 1:
|
||||
offset += j * stride
|
||||
else:
|
||||
offset += origin_shape[k] // 2 * stride
|
||||
idx_list.append(offset)
|
||||
if framework.in_dygraph_mode():
|
||||
with base.dygraph.no_grad():
|
||||
tmp_out = _C_ops.reshape(out_var, [-1])
|
||||
tmp_out._share_underline_tensor_to(out_var)
|
||||
elif framework.in_pir_mode():
|
||||
out_var = _C_ops.reshape(out_var, [-1])
|
||||
else:
|
||||
x_shape = block.create_var(
|
||||
name=unique_name.generate(".".join([out_var.name, "XShape"])),
|
||||
dtype=out_dtype,
|
||||
shape=out_var.shape,
|
||||
type=VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
block.append_op(
|
||||
type="reshape2",
|
||||
inputs={"X": out_var},
|
||||
attrs={'shape': [-1]},
|
||||
outputs={"Out": out_var, "XShape": x_shape},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if framework.in_pir_mode():
|
||||
index_tensor = paddle.zeros(
|
||||
[len(idx_list)], dtype=core.DataType.INT64
|
||||
)
|
||||
index_tensor.stop_gradient = True
|
||||
else:
|
||||
index_tensor = block.create_var(
|
||||
name=unique_name.generate('scatter_index'),
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if framework.in_dygraph_mode():
|
||||
with base.dygraph.no_grad():
|
||||
tmp_tensor = framework._create_tensor()
|
||||
_C_ops.assign_value_(
|
||||
tmp_tensor,
|
||||
[len(idx_list)],
|
||||
VarDesc.VarType.INT64,
|
||||
idx_list,
|
||||
_current_expected_place(),
|
||||
)
|
||||
tmp_tensor._share_underline_tensor_to(index_tensor)
|
||||
elif framework.in_pir_mode():
|
||||
_C_ops.assign_value_(
|
||||
index_tensor,
|
||||
[len(idx_list)],
|
||||
core.DataType.INT64,
|
||||
idx_list,
|
||||
_current_expected_place(),
|
||||
)
|
||||
else:
|
||||
block.append_op(
|
||||
type='assign_value',
|
||||
outputs={'Out': index_tensor},
|
||||
attrs={
|
||||
'dtype': VarDesc.VarType.INT64,
|
||||
'shape': [len(idx_list)],
|
||||
'values': idx_list,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
if framework.in_pir_mode():
|
||||
value_tensor = paddle.zeros(
|
||||
[len(value_list)], dtype=core.DataType.FLOAT32
|
||||
)
|
||||
value_tensor.stop_gradient = True
|
||||
else:
|
||||
value_tensor = block.create_var(
|
||||
name=unique_name.generate('scatter_value'),
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if framework.in_dygraph_mode():
|
||||
with base.dygraph.no_grad():
|
||||
tmp_tensor = framework._create_tensor()
|
||||
_C_ops.assign_value_(
|
||||
tmp_tensor,
|
||||
[len(value_list)],
|
||||
VarDesc.VarType.FP32,
|
||||
value_list,
|
||||
_current_expected_place(),
|
||||
)
|
||||
|
||||
tmp_tensor._share_underline_tensor_to(value_tensor)
|
||||
elif framework.in_pir_mode():
|
||||
_C_ops.assign_value_(
|
||||
value_tensor,
|
||||
[len(value_list)],
|
||||
core.DataType.FLOAT32,
|
||||
value_list,
|
||||
_current_expected_place(),
|
||||
)
|
||||
else:
|
||||
block.append_op(
|
||||
type='assign_value',
|
||||
outputs={'Out': value_tensor},
|
||||
attrs={
|
||||
'dtype': VarDesc.VarType.FP32,
|
||||
'shape': [len(value_list)],
|
||||
'values': value_list,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if framework.in_dygraph_mode():
|
||||
with base.dygraph.no_grad():
|
||||
tmp_out = _C_ops.scatter(
|
||||
out_var, index_tensor, value_tensor, True
|
||||
)
|
||||
tmp_out._share_underline_tensor_to(out_var)
|
||||
tmp_reshape_out = _C_ops.reshape(out_var, origin_shape)
|
||||
tmp_reshape_out._share_underline_tensor_to(out_var)
|
||||
if var.dtype != VarDesc.VarType.FP32:
|
||||
tmp_cast_out = _C_ops.cast(out_var, var.dtype)
|
||||
tmp_cast_out._share_underline_tensor_to(var)
|
||||
elif framework.in_pir_mode():
|
||||
out_var = _C_ops.scatter(out_var, index_tensor, value_tensor, True)
|
||||
out_var = _C_ops.reshape(out_var, origin_shape)
|
||||
if var.dtype != core.DataType.FLOAT32:
|
||||
return _C_ops.cast(out_var, var.dtype)
|
||||
return out_var
|
||||
else:
|
||||
op = block.append_op(
|
||||
type="scatter",
|
||||
inputs={
|
||||
"X": out_var,
|
||||
"Ids": index_tensor,
|
||||
"Updates": value_tensor,
|
||||
},
|
||||
attrs={'overwrite': True},
|
||||
outputs={"Out": out_var},
|
||||
stop_gradient=True,
|
||||
)
|
||||
x_shape = block.create_var(
|
||||
name=unique_name.generate(".".join([out_var.name, "XShape"])),
|
||||
dtype=out_dtype,
|
||||
shape=out_var.shape,
|
||||
type=VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
block.append_op(
|
||||
type="reshape2",
|
||||
inputs={"X": out_var},
|
||||
attrs={'shape': origin_shape},
|
||||
outputs={"Out": out_var, "XShape": x_shape},
|
||||
stop_gradient=True,
|
||||
)
|
||||
if var.dtype != VarDesc.VarType.FP32:
|
||||
block.append_op(
|
||||
type="cast",
|
||||
inputs={"X": out_var},
|
||||
outputs={"Out": var},
|
||||
attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
|
||||
stop_gradient=True,
|
||||
)
|
||||
if not in_dynamic_mode():
|
||||
var.op = op
|
||||
return op
|
||||
@@ -0,0 +1,213 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import math
|
||||
from typing import TYPE_CHECKING, Literal, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ...base.framework import (
|
||||
EagerParamBase,
|
||||
default_main_program,
|
||||
in_dygraph_mode,
|
||||
use_pir_api,
|
||||
)
|
||||
from .lazy_init import lazy_init_helper
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import paddle
|
||||
|
||||
_NonLinearity: TypeAlias = Literal[ # noqa: PYI047
|
||||
"sigmoid",
|
||||
"linear",
|
||||
"conv1d",
|
||||
"conv2d",
|
||||
"conv3d",
|
||||
"conv1d_transpose",
|
||||
"conv_transpose1d",
|
||||
"conv2d_transpose",
|
||||
"conv_transpose2d",
|
||||
"conv3d_transpose",
|
||||
"conv_transpose3d",
|
||||
"tanh",
|
||||
"relu",
|
||||
"leaky_relu",
|
||||
"selu",
|
||||
]
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class Initializer:
|
||||
"""Base class for parameter initializers
|
||||
|
||||
Defines the common interface of parameter initializers.
|
||||
They add operations to the init program that are used
|
||||
to initialize parameter. Users should not use this class
|
||||
directly, but need to use one of its implementations.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def __call__(
|
||||
self, param: paddle.Tensor, block: paddle.pir.Block | None = None
|
||||
):
|
||||
if not lazy_init_helper().state:
|
||||
return self.forward(param, block)
|
||||
|
||||
return self._lazy_init(param, block)
|
||||
|
||||
def forward(
|
||||
self, param: paddle.Tensor, block: paddle.pir.Block | None = None
|
||||
) -> paddle.Tensor | None:
|
||||
"""Add corresponding initialization operations to the network."""
|
||||
raise NotImplementedError
|
||||
|
||||
def _lazy_init(
|
||||
self, param: paddle.Tensor, block: paddle.pir.Block | None = None
|
||||
):
|
||||
"""
|
||||
Apply lazy initialization
|
||||
"""
|
||||
assert in_dygraph_mode()
|
||||
|
||||
def init_op_creator(
|
||||
forward, param: paddle.Tensor, block: paddle.pir.Block | None
|
||||
):
|
||||
if use_pir_api():
|
||||
new_var = param
|
||||
else:
|
||||
new_var = param._to_static_var(True, block=block)
|
||||
# Record initializer operator
|
||||
with lazy_init_helper():
|
||||
forward(new_var, block)
|
||||
|
||||
# Add hook function for initializing param in dygraph mode
|
||||
param.set_init_func(functools.partial(self.forward))
|
||||
param._init_op_creator = functools.partial(
|
||||
init_op_creator, self.forward
|
||||
)
|
||||
|
||||
return param
|
||||
|
||||
def _check_block(self, block: paddle.pir.Block | None) -> paddle.pir.Block:
|
||||
if block is None:
|
||||
block = default_main_program().global_block()
|
||||
|
||||
return block
|
||||
|
||||
def _compute_fans(self, var: paddle.Tensor) -> tuple[int, int]:
|
||||
"""Compute the fan_in and the fan_out for layers
|
||||
|
||||
This method computes the fan_in and the fan_out
|
||||
for neural network layers, if not specified. It is
|
||||
not possible to perfectly estimate fan_in and fan_out.
|
||||
This method will estimate it correctly for matrix multiply and
|
||||
convolutions.
|
||||
|
||||
Args:
|
||||
var: variable for which fan_in and fan_out have to be computed.
|
||||
|
||||
Returns:
|
||||
tuple of two integers (fan_in, fan_out).
|
||||
"""
|
||||
shape = (
|
||||
var._local_shape
|
||||
if (isinstance(var, EagerParamBase) and var.is_dist())
|
||||
else var.shape
|
||||
)
|
||||
if not shape or len(shape) == 0:
|
||||
fan_in = fan_out = 1
|
||||
elif len(shape) == 1:
|
||||
fan_in = fan_out = shape[0]
|
||||
elif len(shape) == 2:
|
||||
# This is the case for simple matrix multiply
|
||||
fan_in = shape[0]
|
||||
fan_out = shape[1]
|
||||
else:
|
||||
# Assume this to be a convolutional kernel
|
||||
# In PaddlePaddle, the shape of the kernel is like:
|
||||
# [num_filters, num_filter_channels, ...] where the remaining
|
||||
# dimensions are the filter_size
|
||||
receptive_field_size = np.prod(shape[2:])
|
||||
fan_in = shape[1] * receptive_field_size
|
||||
fan_out = shape[0] * receptive_field_size
|
||||
|
||||
return (fan_in, fan_out)
|
||||
|
||||
|
||||
def calculate_gain(
|
||||
nonlinearity: str, param: bool | float | None = None
|
||||
) -> float:
|
||||
"""
|
||||
Get the recommended ``gain`` value of some nonlinearity function. ``gain`` value can be used in some
|
||||
``paddle.nn.initializer`` api to adjust the initialization value.
|
||||
|
||||
Args:
|
||||
nonlinearity(str): name of nonlinearity activation function. If it is a linear function, such as:
|
||||
`linear/conv1d/conv2d/conv3d/conv1d_transpose/conv2d_transpose/conv3d_transpose` , 1.0 will be returned.
|
||||
param(bool|int|float|None, optional): optional parameter for some nonlinearity function. Now, it only applies to
|
||||
'leaky_relu'. Default: None, it will be calculated as 0.01 in the formula.
|
||||
|
||||
Returns:
|
||||
A float value, which is the recommended gain for this nonlinearity function.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
|
||||
>>> gain = paddle.nn.initializer.calculate_gain('tanh')
|
||||
>>> print(gain)
|
||||
1.6666666666666667
|
||||
>>> # 5.0 / 3
|
||||
>>> gain = paddle.nn.initializer.calculate_gain('leaky_relu', param=1.0)
|
||||
>>> print(gain)
|
||||
1.0
|
||||
>>> # math.sqrt(2.0 / (1+param^2))
|
||||
>>> initializer = paddle.nn.initializer.Orthogonal(gain)
|
||||
|
||||
"""
|
||||
if param is None:
|
||||
param = 0.01
|
||||
else:
|
||||
assert isinstance(param, (bool, int, float))
|
||||
param = float(param)
|
||||
recommended_gain = {
|
||||
'sigmoid': 1,
|
||||
'linear': 1,
|
||||
'conv1d': 1,
|
||||
'conv2d': 1,
|
||||
'conv3d': 1,
|
||||
'conv1d_transpose': 1,
|
||||
'conv_transpose1d': 1,
|
||||
'conv2d_transpose': 1,
|
||||
'conv_transpose2d': 1,
|
||||
'conv3d_transpose': 1,
|
||||
'conv_transpose3d': 1,
|
||||
'tanh': 5.0 / 3,
|
||||
'relu': math.sqrt(2.0),
|
||||
'leaky_relu': math.sqrt(2.0 / (1 + param**2)),
|
||||
'selu': 3.0 / 4,
|
||||
}
|
||||
if nonlinearity in recommended_gain.keys():
|
||||
return recommended_gain[nonlinearity]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"nonlinearity function {nonlinearity} is not supported now."
|
||||
)
|
||||
@@ -0,0 +1,394 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# TODO: define the initializers of Kaiming functions in neural network
|
||||
import math
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops
|
||||
|
||||
from ...base import core, framework, unique_name
|
||||
from ...base.framework import (
|
||||
_current_expected_place,
|
||||
in_dygraph_mode,
|
||||
in_pir_mode,
|
||||
)
|
||||
from .initializer import Initializer, calculate_gain
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .initializer import _NonLinearity
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class MSRAInitializer(Initializer):
|
||||
r"""Implements the MSRA initializer a.k.a. Kaiming Initializer
|
||||
|
||||
This class implements the weight initialization from the paper
|
||||
`Delving Deep into Rectifiers: Surpassing Human-Level Performance on
|
||||
ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
|
||||
by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
|
||||
robust initialization method that particularly considers the rectifier
|
||||
nonlinearities. In case of Uniform distribution, the range is [-x, x], where
|
||||
|
||||
.. math::
|
||||
|
||||
x = gain \times \sqrt{\frac{3}{fan\_in}}
|
||||
|
||||
In case of Normal distribution, the mean is 0 and the standard deviation
|
||||
is
|
||||
|
||||
.. math::
|
||||
|
||||
\frac{gain}{\sqrt{{fan\_in}}}
|
||||
|
||||
Args:
|
||||
uniform (bool, optional): whether to use uniform or normal distribution. Default is True.
|
||||
fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be inferred automatically. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None.
|
||||
seed (int32, optional): random seed. Default is 0.
|
||||
negative_slope (float, optional): negative_slope (only used with leaky_relu). Default is 0.0.
|
||||
nonlinearity(str, optional): the non-linear function. Default is relu.
|
||||
mode(str, optional): the mode of initialization, can be 'fan_in' or 'fan_out'. When set to 'fan_in', the fan_in parameter is used for initialization. When set to 'fan_out', the out_features of trainable Tensor will be used. Default is 'fan_in'.
|
||||
|
||||
Note:
|
||||
It is recommended to set fan_in to None for most cases.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
uniform: bool = True,
|
||||
fan_in: float | None = None,
|
||||
seed: int = 0,
|
||||
negative_slope: float = 0,
|
||||
nonlinearity: _NonLinearity = 'relu',
|
||||
mode: str = 'fan_in',
|
||||
) -> None:
|
||||
"""Constructor for MSRAInitializer"""
|
||||
assert uniform is not None
|
||||
assert seed is not None
|
||||
super().__init__()
|
||||
self._uniform = uniform
|
||||
self._fan_in = fan_in
|
||||
self._seed = seed
|
||||
self._negative_slope = negative_slope
|
||||
self._nonlinearity = nonlinearity
|
||||
self._mode = mode
|
||||
if self._mode not in ['fan_in', 'fan_out']:
|
||||
raise ValueError(
|
||||
"The mode of KaimingNormal/KaimingUniform should be 'fan_in' or 'fan_out', "
|
||||
f"but received {self._mode}."
|
||||
)
|
||||
if self._mode == 'fan_out' and self._fan_in is not None:
|
||||
raise ValueError(
|
||||
"The mode of KaimingNormal/KaimingUniform is 'fan_out', "
|
||||
"but fan_in is set. Please set fan_in to None."
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, var: paddle.Tensor, block: paddle.pir.Block | None = None
|
||||
) -> paddle.Tensor | None:
|
||||
"""Initialize the input tensor with MSRA initialization.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The initialization op.
|
||||
"""
|
||||
assert not (
|
||||
isinstance(var, framework.EagerParamBase) and var.is_dist()
|
||||
), (
|
||||
"Currently, kaiming initializer not support lazy init for dist param."
|
||||
)
|
||||
block = self._check_block(block)
|
||||
assert isinstance(
|
||||
var,
|
||||
(
|
||||
framework.Variable,
|
||||
paddle.pir.Value,
|
||||
paddle.pir.core.ParameterMeta,
|
||||
),
|
||||
)
|
||||
assert isinstance(block, (framework.Block, paddle.pir.Block))
|
||||
f_in, f_out = self._compute_fans(var)
|
||||
|
||||
# If fan_in is passed, use it
|
||||
if self._mode == 'fan_in':
|
||||
fan_in = f_in if self._fan_in is None else self._fan_in
|
||||
if self._mode == 'fan_out':
|
||||
fan_in = f_out
|
||||
|
||||
if self._seed == 0:
|
||||
self._seed = block.program.random_seed
|
||||
|
||||
# to be compatible of fp16 initializers
|
||||
origin_dtype = var.dtype
|
||||
if origin_dtype == core.VarDesc.VarType.FP16 or (
|
||||
origin_dtype == core.VarDesc.VarType.BF16 and not self._uniform
|
||||
):
|
||||
out_dtype = core.VarDesc.VarType.FP32
|
||||
out_var = block.create_var(
|
||||
name=unique_name.generate(
|
||||
".".join(['masra_init', var.name, 'tmp'])
|
||||
),
|
||||
shape=var.shape,
|
||||
dtype=out_dtype,
|
||||
type=core.VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
)
|
||||
elif (
|
||||
origin_dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
|
||||
and not self._uniform
|
||||
):
|
||||
out_dtype = core.DataType.FLOAT32
|
||||
out_var = var
|
||||
else:
|
||||
out_dtype = origin_dtype
|
||||
out_var = var
|
||||
|
||||
if in_dygraph_mode():
|
||||
if self._uniform:
|
||||
gain = calculate_gain(self._nonlinearity, self._negative_slope)
|
||||
std = gain / math.sqrt(float(fan_in))
|
||||
limit = math.sqrt(3.0) * std
|
||||
out_var = _C_ops.uniform(
|
||||
var.shape,
|
||||
out_dtype,
|
||||
-limit,
|
||||
limit,
|
||||
self._seed,
|
||||
var.place
|
||||
if var.place._type()
|
||||
else _current_expected_place(),
|
||||
)
|
||||
else:
|
||||
gain = calculate_gain(self._nonlinearity, self._negative_slope)
|
||||
std = gain / math.sqrt(float(fan_in))
|
||||
# var.place._type() means undefined, happens when initializer is specified in ParamAttr
|
||||
place = (
|
||||
var.place
|
||||
if var.place._type()
|
||||
else _current_expected_place()
|
||||
)
|
||||
out_var = _C_ops.gaussian(
|
||||
out_var.shape, 0.0, std, self._seed, out_dtype, place
|
||||
)
|
||||
|
||||
if origin_dtype == core.VarDesc.VarType.FP16 or (
|
||||
origin_dtype
|
||||
in [
|
||||
core.VarDesc.VarType.BF16,
|
||||
core.DataType.FLOAT16,
|
||||
core.DataType.BFLOAT16,
|
||||
]
|
||||
and not self._uniform
|
||||
):
|
||||
var_tmp = _C_ops.cast(out_var, origin_dtype)
|
||||
var_tmp._share_underline_tensor_to(var)
|
||||
else:
|
||||
out_var._share_underline_tensor_to(var)
|
||||
return None
|
||||
elif in_pir_mode():
|
||||
if self._uniform:
|
||||
gain = calculate_gain(self._nonlinearity, self._negative_slope)
|
||||
std = gain / math.sqrt(float(fan_in))
|
||||
limit = math.sqrt(3.0) * std
|
||||
out_var = _C_ops.uniform(
|
||||
var.shape,
|
||||
out_dtype,
|
||||
-limit,
|
||||
limit,
|
||||
self._seed,
|
||||
_current_expected_place(),
|
||||
)
|
||||
else:
|
||||
gain = calculate_gain(self._nonlinearity, self._negative_slope)
|
||||
std = gain / math.sqrt(float(fan_in))
|
||||
place = _current_expected_place()
|
||||
out_var = _C_ops.gaussian(
|
||||
out_var.shape, 0.0, std, self._seed, out_dtype, place
|
||||
)
|
||||
|
||||
if (
|
||||
origin_dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
|
||||
and not self._uniform
|
||||
):
|
||||
return _C_ops.cast(out_var, origin_dtype)
|
||||
|
||||
return out_var
|
||||
else:
|
||||
if self._uniform:
|
||||
gain = calculate_gain(self._nonlinearity, self._negative_slope)
|
||||
std = gain / math.sqrt(float(fan_in))
|
||||
limit = math.sqrt(3.0) * std
|
||||
op = block.append_op(
|
||||
type="uniform_random",
|
||||
inputs={},
|
||||
outputs={"Out": out_var},
|
||||
attrs={
|
||||
"shape": out_var.shape,
|
||||
"dtype": int(out_dtype),
|
||||
"min": -limit,
|
||||
"max": limit,
|
||||
"seed": self._seed,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
else:
|
||||
gain = calculate_gain(self._nonlinearity, self._negative_slope)
|
||||
std = gain / math.sqrt(float(fan_in))
|
||||
op = block.append_op(
|
||||
type="gaussian_random",
|
||||
outputs={"Out": out_var},
|
||||
attrs={
|
||||
"shape": out_var.shape,
|
||||
"dtype": int(out_dtype),
|
||||
"mean": 0.0,
|
||||
"std": std,
|
||||
"seed": self._seed,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if origin_dtype == core.VarDesc.VarType.FP16 or (
|
||||
origin_dtype == core.VarDesc.VarType.BF16 and not self._uniform
|
||||
):
|
||||
block.append_op(
|
||||
type="cast",
|
||||
inputs={"X": out_var},
|
||||
outputs={"Out": var},
|
||||
attrs={
|
||||
"in_dtype": out_var.dtype,
|
||||
"out_dtype": origin_dtype,
|
||||
},
|
||||
)
|
||||
|
||||
var.op = op
|
||||
return op
|
||||
|
||||
|
||||
class KaimingNormal(MSRAInitializer):
|
||||
r"""Implements the Kaiming Normal initializer
|
||||
|
||||
This class implements the weight initialization from the paper
|
||||
`Delving Deep into Rectifiers: Surpassing Human-Level Performance on
|
||||
ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
|
||||
by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
|
||||
robust initialization method that particularly considers the rectifier
|
||||
nonlinearities.
|
||||
|
||||
In case of Normal distribution, the mean is 0 and the standard deviation
|
||||
is
|
||||
|
||||
.. math::
|
||||
|
||||
\frac{gain}{\sqrt{{fan\_in}}}
|
||||
|
||||
Args:
|
||||
fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be inferred automatically. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None.
|
||||
negative_slope (float, optional): negative_slope (only used with leaky_relu). Default is 0.0.
|
||||
nonlinearity(str, optional): the non-linear function. Default is relu.
|
||||
mode(str, optional): the mode of initialization, can be 'fan_in' or 'fan_out'. When set to 'fan_in', the fan_in parameter is used for initialization. When set to 'fan_out', the out_features of trainable Tensor will be used. Default is 'fan_in'.
|
||||
|
||||
Note:
|
||||
It is recommended to set fan_in to None for most cases.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> import paddle.nn as nn
|
||||
|
||||
>>> linear = nn.Linear(2, 4, weight_attr=nn.initializer.KaimingNormal())
|
||||
>>> data = paddle.rand([30, 10, 2], dtype='float32')
|
||||
>>> res = linear(data)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fan_in: float | None = None,
|
||||
negative_slope: float = 0.0,
|
||||
nonlinearity: str = 'relu',
|
||||
mode: str = 'fan_in',
|
||||
) -> None:
|
||||
super().__init__(
|
||||
uniform=False,
|
||||
fan_in=fan_in,
|
||||
seed=0,
|
||||
negative_slope=negative_slope,
|
||||
nonlinearity=nonlinearity,
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
|
||||
class KaimingUniform(MSRAInitializer):
|
||||
r"""Implements the Kaiming Uniform initializer
|
||||
|
||||
This class implements the weight initialization from the paper
|
||||
`Delving Deep into Rectifiers: Surpassing Human-Level Performance on
|
||||
ImageNet Classification <https://arxiv.org/abs/1502.01852>`_
|
||||
by Kaiming He, Xiangyu Zhang, Shaoqing Ren and Jian Sun. This is a
|
||||
robust initialization method that particularly considers the rectifier
|
||||
nonlinearities.
|
||||
|
||||
In case of Uniform distribution, the range is [-x, x], where
|
||||
|
||||
.. math::
|
||||
|
||||
x = gain \times \sqrt{\frac{3}{fan\_in}}
|
||||
|
||||
Args:
|
||||
fan_in (float32|None, optional): fan_in (in_features) of trainable Tensor, If None, it will be inferred automatically. If you don't want to use in_features of the Tensor, you can set the value of 'fan_in' smartly by yourself. Default is None.
|
||||
negative_slope (float, optional): negative_slope (only used with leaky_relu). Default is 0.0.
|
||||
nonlinearity(str, optional): the non-linear function. Default is relu.
|
||||
mode(str, optional): the mode of initialization, can be 'fan_in' or 'fan_out'. When set to 'fan_in', the fan_in parameter is used for initialization. When set to 'fan_out', the out_features of trainable Tensor will be used. Default is 'fan_in'.
|
||||
|
||||
Note:
|
||||
It is recommended to set fan_in to None for most cases.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> import paddle.nn as nn
|
||||
|
||||
>>> linear = nn.Linear(2, 4, weight_attr=nn.initializer.KaimingUniform())
|
||||
>>> data = paddle.rand([30, 10, 2], dtype='float32')
|
||||
>>> res = linear(data)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fan_in: float | None = None,
|
||||
negative_slope: float = 0.0,
|
||||
nonlinearity: str = 'relu',
|
||||
mode: str = 'fan_in',
|
||||
) -> None:
|
||||
super().__init__(
|
||||
uniform=True,
|
||||
fan_in=fan_in,
|
||||
seed=0,
|
||||
negative_slope=negative_slope,
|
||||
nonlinearity=nonlinearity,
|
||||
mode=mode,
|
||||
)
|
||||
@@ -0,0 +1,144 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ...base import framework
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from types import TracebackType
|
||||
|
||||
|
||||
__all__ = ["LazyGuard"]
|
||||
|
||||
|
||||
class LazyInitHelper:
|
||||
"""
|
||||
A Helper Context to trigger switching mode between dygraph and static graph mode,
|
||||
and holds the startup program resource.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._state = False
|
||||
self._tracer = None
|
||||
self._in_guard = False
|
||||
|
||||
def enable(self):
|
||||
"""
|
||||
Switch into lazy mode.
|
||||
|
||||
NOTE(dev): This is a very low level API and not exposed for user.
|
||||
"""
|
||||
if self._state:
|
||||
return
|
||||
assert framework.in_dygraph_mode(), (
|
||||
"LazyInit.enable() is only available in dygraph mode."
|
||||
)
|
||||
self._state = True
|
||||
|
||||
def disable(self):
|
||||
"""
|
||||
Exit from lazy mode.
|
||||
|
||||
NOTE(dev): This is a very low level API and not exposed for user.
|
||||
"""
|
||||
if not self._state:
|
||||
return
|
||||
self._state = False
|
||||
|
||||
def __enter__(self):
|
||||
"""
|
||||
Switch into lazy mode and set _dygraph_tracer_ with None to convert
|
||||
dygraph mode into static graph mode.
|
||||
"""
|
||||
self.enable()
|
||||
if self._in_guard:
|
||||
return
|
||||
self._tracer = framework.global_var._dygraph_tracer_
|
||||
framework.global_var._dygraph_tracer_ = None
|
||||
self._in_guard = True
|
||||
|
||||
def __exit__(self, *args, **kwargs):
|
||||
"""
|
||||
Exit from lazy mode and recover _dygraph_tracer_.
|
||||
"""
|
||||
self.disable()
|
||||
if not self._in_guard:
|
||||
return
|
||||
assert self._tracer is not None
|
||||
framework.global_var._dygraph_tracer_ = self._tracer
|
||||
self._tracer = None
|
||||
self._in_guard = False
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return self._state
|
||||
|
||||
|
||||
_lazy_init_helper = LazyInitHelper()
|
||||
|
||||
|
||||
def lazy_init_helper():
|
||||
global _lazy_init_helper
|
||||
return _lazy_init_helper
|
||||
|
||||
|
||||
class LazyGuard:
|
||||
"""
|
||||
LazyGuard is a wrapper interface for nn.Layer, it forwards the construct
|
||||
process of user defined Layer. Meanwhile, it provides necessary API to
|
||||
trigger EagerParamBase Lazy Initialization and get startup Program.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> from paddle import LazyGuard
|
||||
>>> from paddle.nn import Linear
|
||||
|
||||
>>> with LazyGuard():
|
||||
... # w and b are initialized lazily and have no memory.
|
||||
... net = Linear(10, 10)
|
||||
>>> for param in net.parameters():
|
||||
... # Initialize param and allocate memory explicitly.
|
||||
... param.initialize()
|
||||
"""
|
||||
|
||||
def __enter__(self) -> None:
|
||||
"""
|
||||
Construct instance from class_obj by Lazy Initializing parameters.
|
||||
|
||||
Examples:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> from paddle import LazyGuard
|
||||
>>> from paddle.nn import Linear
|
||||
|
||||
>>> with LazyGuard():
|
||||
... fc = LazyInit(Linear)(10, 10)
|
||||
>>> for param in fc.parameters():
|
||||
... param.initialize()
|
||||
"""
|
||||
lazy_init_helper().enable()
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_val: BaseException | None,
|
||||
exc_tb: TracebackType | None,
|
||||
) -> None:
|
||||
lazy_init_helper().disable()
|
||||
@@ -0,0 +1,408 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops, pir
|
||||
|
||||
from ...base import core, framework, unique_name
|
||||
from ...base.data_feeder import check_variable_and_dtype
|
||||
from ...base.framework import (
|
||||
_current_expected_place,
|
||||
in_dygraph_mode,
|
||||
in_pir_mode,
|
||||
)
|
||||
from .initializer import Initializer
|
||||
from .lazy_init import lazy_init_helper
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class NormalInitializer(Initializer):
|
||||
"""Implements the Random Normal(Gaussian) distribution initializer
|
||||
|
||||
Args:
|
||||
loc (float|complex, optional): mean of the normal distribution. Default is 0.0.
|
||||
scale (float, optional): standard deviation of the normal distribution. Default is 1.0.
|
||||
seed (int, optional): random seed. Default is 0.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, loc: float = 0.0, scale: float = 1.0, seed: int = 0
|
||||
) -> None:
|
||||
assert loc is not None
|
||||
assert scale is not None
|
||||
assert seed is not None
|
||||
super().__init__()
|
||||
self._mean = loc
|
||||
self._std_dev = scale
|
||||
self._seed = seed
|
||||
if isinstance(self._mean, complex):
|
||||
if self._mean.real != self._mean.imag:
|
||||
raise ValueError(
|
||||
"if mean is a complex number, its real part should equal imag part, "
|
||||
f"but got real part: {self._mean.real} != imag part: {self._mean.imag}"
|
||||
)
|
||||
self._mean = self._mean.real
|
||||
|
||||
def forward(
|
||||
self, var: paddle.Tensor, block: pir.Block | None = None
|
||||
) -> paddle.Tensor | None:
|
||||
"""Initialize the input tensor with Normal distribution.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The initialization op.
|
||||
"""
|
||||
assert not (
|
||||
isinstance(var, framework.EagerParamBase) and var.is_dist()
|
||||
), "Currently, normal initializer not support lazy init for dist param."
|
||||
block = self._check_block(block)
|
||||
|
||||
assert isinstance(block, (framework.Block, pir.Block))
|
||||
|
||||
check_variable_and_dtype(
|
||||
var,
|
||||
"Out",
|
||||
[
|
||||
"uint16",
|
||||
"float16",
|
||||
"float32",
|
||||
"float64",
|
||||
"complex64",
|
||||
"complex128",
|
||||
],
|
||||
"gaussian_random",
|
||||
)
|
||||
|
||||
if self._seed == 0:
|
||||
self._seed = block.program.random_seed
|
||||
|
||||
if in_dygraph_mode():
|
||||
place = _current_expected_place()
|
||||
out_var = _C_ops.gaussian(
|
||||
var.shape,
|
||||
self._mean,
|
||||
self._std_dev,
|
||||
self._seed,
|
||||
var.dtype,
|
||||
place,
|
||||
)
|
||||
out_var._share_underline_tensor_to(var)
|
||||
return None
|
||||
elif in_pir_mode():
|
||||
place = _current_expected_place()
|
||||
out_var = _C_ops.gaussian(
|
||||
var.shape,
|
||||
self._mean,
|
||||
self._std_dev,
|
||||
self._seed,
|
||||
var.dtype,
|
||||
place,
|
||||
)
|
||||
return out_var
|
||||
else:
|
||||
op = block.append_op(
|
||||
type="gaussian_random",
|
||||
outputs={"Out": var},
|
||||
attrs={
|
||||
"shape": var.shape,
|
||||
"dtype": var.dtype,
|
||||
"mean": self._mean,
|
||||
"std": self._std_dev,
|
||||
"seed": self._seed,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
var.op = op
|
||||
return op
|
||||
|
||||
|
||||
class Normal(NormalInitializer):
|
||||
"""The Random Normal (Gaussian) distribution initializer.
|
||||
|
||||
Args:
|
||||
mean (float|complex, optional): mean of the normal distribution. Default is 0.0.
|
||||
std (float, optional): standard deviation of the normal distribution. Default is 1.0.
|
||||
name(str|None, optional): The default value is None. Normally there is no need for user to set this
|
||||
property. For more information, please refer to :ref:`api_guide_Name`. Default: None.
|
||||
|
||||
Returns:
|
||||
A parameter initialized by Random Normal (Gaussian) distribution.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
|
||||
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
|
||||
>>> weight_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_weight",
|
||||
... initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0),
|
||||
... )
|
||||
>>> bias_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_bias",
|
||||
... initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0),
|
||||
... )
|
||||
>>> # doctest: +SKIP('name has been used')
|
||||
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
|
||||
>>> print(linear.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[ 2.1973135 -2.2697184],
|
||||
[-1.9104223 -1.0541488]])
|
||||
>>> print(linear.bias)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[ 0.7885926 -0.74719954])
|
||||
>>> res = linear(data)
|
||||
>>> print(res)
|
||||
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[[ 1.0754838 -4.071067 ]],
|
||||
[[ 1.0754838 -4.071067 ]],
|
||||
[[ 1.0754838 -4.071067 ]]])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, mean: float = 0.0, std: float = 1.0, name: str | None = None
|
||||
) -> None:
|
||||
assert mean is not None, 'mean should not be None'
|
||||
assert std is not None, 'std should not be None'
|
||||
super().__init__(loc=mean, scale=std, seed=0)
|
||||
|
||||
|
||||
class TruncatedNormalInitializer(Initializer):
|
||||
"""Implements the Random TruncatedNormal(Gaussian) distribution initializer
|
||||
|
||||
Note:
|
||||
It is better to set `a <= mean <= b`.
|
||||
If `mean < a - 2*std` or `mean > b + 2*std`, the distribution of values may be incorrect.
|
||||
|
||||
Args:
|
||||
loc (float, optional): Mean of the normal distribution. Default is :math:`0.0`.
|
||||
scale (float, optional): Standard deviation of the normal distribution. Default is :math:`1.0`.
|
||||
seed (int, optional): random seed. Default is 0.
|
||||
a (float, optional): The minimum cutoff value. Default is -2.0.
|
||||
b (float, optional): The maximum cutoff value. Default is 2.0.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
loc: float = 0.0,
|
||||
scale: float = 1.0,
|
||||
seed: int = 0,
|
||||
a: float = -2.0,
|
||||
b: float = 2.0,
|
||||
) -> None:
|
||||
assert loc is not None
|
||||
assert scale is not None
|
||||
assert seed is not None
|
||||
assert a is not None
|
||||
assert b is not None
|
||||
super().__init__()
|
||||
self._mean = loc
|
||||
self._std_dev = scale
|
||||
self._seed = seed
|
||||
self._a = a
|
||||
self._b = b
|
||||
|
||||
def forward(
|
||||
self, var: paddle.Tensor, block: pir.Block | None = None
|
||||
) -> paddle.Tensor | None:
|
||||
"""Initialize the input tensor with TruncatedNormal distribution.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The initialization op
|
||||
"""
|
||||
block = self._check_block(block)
|
||||
if lazy_init_helper().state:
|
||||
expected = (
|
||||
framework.Variable,
|
||||
paddle.pir.core.ParameterMeta,
|
||||
core.eager.Tensor,
|
||||
)
|
||||
else:
|
||||
expected = (
|
||||
framework.Variable,
|
||||
paddle.pir.Value,
|
||||
paddle.pir.core.ParameterMeta,
|
||||
)
|
||||
|
||||
assert isinstance(var, expected)
|
||||
assert isinstance(block, (framework.Block, pir.Block))
|
||||
|
||||
if self._seed == 0:
|
||||
self._seed = block.program.random_seed
|
||||
|
||||
# to be compatible of fp16 initializers
|
||||
if var.dtype in [core.VarDesc.VarType.FP16, core.VarDesc.VarType.BF16]:
|
||||
out_dtype = core.VarDesc.VarType.FP32
|
||||
out_var = block.create_var(
|
||||
name=unique_name.generate(
|
||||
".".join(['truncated_gaussian_random', var.name, 'tmp'])
|
||||
),
|
||||
shape=var.shape,
|
||||
dtype=out_dtype,
|
||||
type=core.VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
)
|
||||
else:
|
||||
out_dtype = var.dtype
|
||||
out_var = var
|
||||
|
||||
if in_dygraph_mode():
|
||||
out_var = _C_ops.truncated_gaussian_random(
|
||||
var.shape,
|
||||
self._mean,
|
||||
self._std_dev,
|
||||
self._seed,
|
||||
self._a,
|
||||
self._b,
|
||||
out_dtype,
|
||||
_current_expected_place(),
|
||||
)
|
||||
if var.dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
]:
|
||||
var_tmp = _C_ops.cast(out_var, var.dtype)
|
||||
var_tmp._share_underline_tensor_to(var)
|
||||
else:
|
||||
out_var._share_underline_tensor_to(var)
|
||||
return None
|
||||
|
||||
elif in_pir_mode():
|
||||
out_var = _C_ops.truncated_gaussian_random(
|
||||
var.shape,
|
||||
self._mean,
|
||||
self._std_dev,
|
||||
self._seed,
|
||||
self._a,
|
||||
self._b,
|
||||
out_dtype,
|
||||
_current_expected_place(),
|
||||
)
|
||||
if var.dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
]:
|
||||
var_tmp = _C_ops.cast(out_var, var.dtype)
|
||||
var_tmp._share_underline_tensor_to(var)
|
||||
return out_var
|
||||
|
||||
else:
|
||||
op = block.append_op(
|
||||
type="truncated_gaussian_random",
|
||||
outputs={"Out": out_var},
|
||||
attrs={
|
||||
"shape": var.shape,
|
||||
"dtype": out_dtype,
|
||||
"mean": self._mean,
|
||||
"std": self._std_dev,
|
||||
"seed": self._seed,
|
||||
"a": self._a,
|
||||
"b": self._b,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if var.dtype in [
|
||||
core.VarDesc.VarType.FP16,
|
||||
core.VarDesc.VarType.BF16,
|
||||
]:
|
||||
block.append_op(
|
||||
type="cast",
|
||||
inputs={"X": out_var},
|
||||
outputs={"Out": var},
|
||||
attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
|
||||
)
|
||||
var.op = op
|
||||
return op
|
||||
|
||||
|
||||
class TruncatedNormal(TruncatedNormalInitializer):
|
||||
"""The truncated normal distribution (Gaussian distribution) initializer.
|
||||
|
||||
Note:
|
||||
It is better to set `a <= mean <= b`.
|
||||
If `mean < a - 2*std` or `mean > b + 2*std`, the distribution of values may be incorrect.
|
||||
|
||||
Args:
|
||||
mean (float, optional): Mean of the normal distribution. Default is :math:`0.0`.
|
||||
std (float, optional): Standard deviation of the normal distribution. Default is :math:`1.0`.
|
||||
a (float, optional): The minimum cutoff value. Default is -2.0.
|
||||
b (float, optional): The maximum cutoff value. Default is 2.0.
|
||||
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
|
||||
|
||||
Returns:
|
||||
A parameter initialized by truncated normal distribution (Gaussian distribution).
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
|
||||
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
|
||||
>>> weight_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_weight",
|
||||
... initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, std=2.0),
|
||||
... )
|
||||
>>> bias_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_bias",
|
||||
... initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, std=2.0),
|
||||
... )
|
||||
>>> # doctest: +SKIP('name has been used')
|
||||
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
|
||||
>>> print(linear.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[-1.0981836 1.4140984],
|
||||
[ 3.1390522 -2.8266568]])
|
||||
>>> print(linear.bias)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[ -2.1546738 -1.6570673])
|
||||
>>> res = linear(data)
|
||||
>>> print(res)
|
||||
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[[-0.11380529 -3.0696259 ]],
|
||||
[[-0.11380529 -3.0696259 ]],
|
||||
[[-0.11380529 -3.0696259 ]]])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mean: float = 0.0,
|
||||
std: float = 1.0,
|
||||
a: float = -2.0,
|
||||
b: float = 2.0,
|
||||
name: str | None = None,
|
||||
) -> None:
|
||||
assert mean is not None, 'mean should not be None'
|
||||
assert std is not None, 'std should not be None'
|
||||
assert a is not None, 'a should not be None'
|
||||
assert b is not None, 'b should not be None'
|
||||
super().__init__(loc=mean, scale=std, seed=0, a=a, b=b)
|
||||
@@ -0,0 +1,271 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops, pir
|
||||
from paddle.utils import unique_name
|
||||
|
||||
from ...base import framework
|
||||
from ...base.data_feeder import check_variable_and_dtype
|
||||
from ...base.dygraph import no_grad
|
||||
from .initializer import Initializer
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class Orthogonal(Initializer):
|
||||
"""The orthogonal initializer. The initialized tensor is (semi) orthogonal.
|
||||
|
||||
It's only applied to Tensor whose dimension is greater than or equal to 2.
|
||||
|
||||
For the Tensor whose dimension is greater than 2, the 0 dimension is seen as ``rows`` ,
|
||||
and the >=1 dimension are flattened as ``cols`` .
|
||||
|
||||
Which can be describe as:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
rows = shape[0]
|
||||
cols = shape[1]·shape[2]···shape[N]
|
||||
|
||||
if rows < cols:
|
||||
The rows are orthogonal vectors
|
||||
elif rows > cols:
|
||||
The columns are orthogonal vectors
|
||||
else rows = cols:
|
||||
Both rows and columns are orthogonal vectors
|
||||
|
||||
Args:
|
||||
gain(float, optional): The multiplication coefficient for initialized tensor. Default: 1.0.
|
||||
name(str|None, optional): The default value is None. Normally there is no need for user to set this
|
||||
property. For more information, please refer to :ref:`api_guide_Name`.
|
||||
|
||||
Returns:
|
||||
A parameter initialized by orthogonal initialized.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
|
||||
>>> weight_attr = paddle.ParamAttr(initializer=paddle.nn.initializer.Orthogonal())
|
||||
>>> linear = paddle.nn.Linear(10, 15, weight_attr=weight_attr)
|
||||
>>> # linear.weight: X * X' = I
|
||||
>>> linear = paddle.nn.Linear(15, 10, weight_attr=weight_attr)
|
||||
>>> # linear.weight: X' * X = I
|
||||
"""
|
||||
|
||||
def __init__(self, gain: float = 1.0, name: str | None = None) -> None:
|
||||
assert gain is not None, 'gain should not be None'
|
||||
super().__init__()
|
||||
self._gain = gain
|
||||
|
||||
def __call__(self, var: paddle.Tensor, block: pir.Block | None = None):
|
||||
"""Initialize the input tensor with orthogonal initializer.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The last initialization op, it contain 8 ops in orthogonal initializer.
|
||||
"""
|
||||
assert not (
|
||||
isinstance(var, framework.EagerParamBase) and var.is_dist()
|
||||
), (
|
||||
"Currently, orthogonal initializer not support lazy init for dist param."
|
||||
)
|
||||
block = self._check_block(block)
|
||||
assert isinstance(
|
||||
var, (framework.Variable, paddle.pir.Value, pir.core.ParameterMeta)
|
||||
)
|
||||
assert isinstance(block, (framework.Block, pir.Block))
|
||||
self._seed = block.program.random_seed
|
||||
|
||||
shape = var.shape
|
||||
assert len(shape) >= 2, (
|
||||
"Only Tensor with 2 or more dimensions can be initialized by Orthogonal"
|
||||
)
|
||||
|
||||
row = shape[0]
|
||||
col = 1
|
||||
for i in shape[1:]:
|
||||
col *= i
|
||||
|
||||
flatten_shape = [max(row, col), min(row, col)]
|
||||
|
||||
if framework.in_dygraph_mode():
|
||||
with no_grad():
|
||||
place = framework._current_expected_place()
|
||||
normal_var = _C_ops.gaussian(
|
||||
flatten_shape, 0.0, 1.0, self._seed, var.dtype, place
|
||||
)
|
||||
q, r = _C_ops.qr(normal_var, 'reduced')
|
||||
|
||||
r_diag = _C_ops.diag(r, 0, 0)
|
||||
|
||||
r_sign = _C_ops.sign(r_diag)
|
||||
|
||||
q = _C_ops.multiply(q, r_sign)
|
||||
|
||||
if row < col:
|
||||
q = _C_ops.transpose(q, [1, 0])
|
||||
|
||||
q = _C_ops.reshape(q, var.shape)
|
||||
|
||||
tmp = _C_ops.scale(q, self._gain, 0.0, True)
|
||||
|
||||
tmp._share_underline_tensor_to(var)
|
||||
|
||||
return None
|
||||
elif framework.in_pir_mode():
|
||||
place = framework._current_expected_place()
|
||||
normal_var = _C_ops.gaussian(
|
||||
flatten_shape, 0.0, 1.0, self._seed, var.dtype, place
|
||||
)
|
||||
q, r = _C_ops.qr(normal_var, 'reduced')
|
||||
|
||||
r_diag = _C_ops.diag(r, 0, 0)
|
||||
|
||||
r_sign = _C_ops.sign(r_diag)
|
||||
|
||||
q = _C_ops.multiply(q, r_sign)
|
||||
|
||||
if row < col:
|
||||
q = _C_ops.transpose(q, [1, 0])
|
||||
|
||||
q = _C_ops.reshape(q, var.shape)
|
||||
|
||||
tmp = _C_ops.scale(q, self._gain, 0.0, True)
|
||||
|
||||
return tmp
|
||||
|
||||
# 'qr' op only support float32/float64 now
|
||||
check_variable_and_dtype(
|
||||
var, "Out", ["float32", "float64"], "Orthogonal"
|
||||
)
|
||||
|
||||
normal_var = block.create_var(
|
||||
name=unique_name.generate('.'.join(['gaussian_random', 'tmp'])),
|
||||
dtype=var.dtype,
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
block.append_op(
|
||||
type='gaussian_random',
|
||||
inputs={},
|
||||
outputs={'Out': normal_var},
|
||||
attrs={
|
||||
'mean': 0.0,
|
||||
'std': 1.0,
|
||||
'shape': flatten_shape,
|
||||
'seed': self._seed,
|
||||
'dtype': var.dtype,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
q = block.create_var(
|
||||
name=unique_name.generate('.'.join(['qr', 'q', 'tmp'])),
|
||||
dtype=normal_var.dtype,
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
r = block.create_var(
|
||||
name=unique_name.generate('.'.join(['qr', 'r', 'tmp'])),
|
||||
dtype=normal_var.dtype,
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
block.append_op(
|
||||
type='qr',
|
||||
inputs={'X': [normal_var]},
|
||||
outputs={
|
||||
'Q': q,
|
||||
'R': r,
|
||||
},
|
||||
attrs={'mode': 'reduced'},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
r_diag = block.create_var(
|
||||
name=unique_name.generate('.'.join(['diag', 'tmp'])),
|
||||
dtype=r.dtype,
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
block.append_op(
|
||||
type='diag_v2',
|
||||
inputs={'X': r},
|
||||
outputs={'Out': r_diag},
|
||||
attrs={'offset': 0, 'padding_value': 0},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
r_sign = r_diag
|
||||
block.append_op(
|
||||
type='sign',
|
||||
inputs={'X': [r_diag]},
|
||||
outputs={'Out': r_sign},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
block.append_op(
|
||||
type='elementwise_mul',
|
||||
inputs={'X': q, 'Y': r_sign},
|
||||
outputs={'Out': q},
|
||||
attrs={},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
x_shape = block.create_var(
|
||||
name=unique_name.generate('.'.join(['transpose', 'shape', 'tmp'])),
|
||||
dtype=q.dtype,
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
if row < col:
|
||||
q_transpose = block.create_var(
|
||||
name=unique_name.generate('.'.join(['transpose', 'tmp'])),
|
||||
dtype=q.dtype,
|
||||
persistable=False,
|
||||
stop_gradient=True,
|
||||
)
|
||||
block.append_op(
|
||||
type='transpose2',
|
||||
inputs={'X': q},
|
||||
outputs={'Out': q_transpose, 'XShape': x_shape},
|
||||
attrs={'axis': [1, 0]},
|
||||
stop_gradient=True,
|
||||
)
|
||||
q = q_transpose
|
||||
|
||||
block.append_op(
|
||||
type='reshape2',
|
||||
inputs={'X': q},
|
||||
outputs={'Out': q, "XShape": x_shape},
|
||||
attrs={'shape': var.shape},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
op = block.append_op(
|
||||
type='scale',
|
||||
inputs={'X': q},
|
||||
outputs={'Out': var},
|
||||
attrs={'scale': self._gain, 'bias': 0.0},
|
||||
)
|
||||
|
||||
return op
|
||||
@@ -0,0 +1,239 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops, pir
|
||||
|
||||
from ...base import core, framework, unique_name
|
||||
from ...base.data_feeder import check_variable_and_dtype
|
||||
from ...base.framework import (
|
||||
_current_expected_place,
|
||||
in_dygraph_mode,
|
||||
in_pir_mode,
|
||||
)
|
||||
from .initializer import Initializer
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class UniformInitializer(Initializer):
|
||||
"""Implements the random uniform distribution initializer
|
||||
|
||||
Args:
|
||||
low (float, optional): Lower boundary of the uniform distribution. Default is :math:`-1.0`.
|
||||
high (float, optional): Upper boundary of the uniform distribution. Default is :math:`1.0`.
|
||||
seed (int, optional): Random seed. Default is 0.
|
||||
diag_num (int, optional): the number of diagonal elements to initialize.
|
||||
If set to 0, diagonal initialization will be not performed. Default is 0.
|
||||
diag_step (int, optional): Step size between two diagonal elements,
|
||||
which is generally the width of the square matrix. Default is 0.
|
||||
diag_val (float, optional): the value of the diagonal element to be initialized,
|
||||
default 1.0. It takes effect only if the diag_num is greater than 0. Default is :math:`1.0`.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
low: float = -1.0,
|
||||
high: float = 1.0,
|
||||
seed: int = 0,
|
||||
diag_num: int = 0,
|
||||
diag_step: int = 0,
|
||||
diag_val: float = 1.0,
|
||||
) -> None:
|
||||
assert low is not None
|
||||
assert high is not None
|
||||
assert high >= low
|
||||
assert seed is not None
|
||||
assert diag_num is not None
|
||||
assert diag_step is not None
|
||||
assert diag_val is not None
|
||||
if diag_num > 0 or diag_step > 0:
|
||||
assert diag_num > 0 and diag_step > 0
|
||||
super().__init__()
|
||||
self._low = low
|
||||
self._high = high
|
||||
self._seed = seed
|
||||
self._diag_num = diag_num
|
||||
self._diag_step = diag_step
|
||||
self._diag_val = diag_val
|
||||
|
||||
def forward(
|
||||
self, var: paddle.Tensor, block: pir.Block | None = None
|
||||
) -> paddle.Tensor | None:
|
||||
"""Initialize the input tensor with Uniform distribution.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The initialization op
|
||||
"""
|
||||
assert not (
|
||||
isinstance(var, framework.EagerParamBase) and var.is_dist()
|
||||
), (
|
||||
"Currently, uniform initializer not support lazy init for dist param."
|
||||
)
|
||||
block = self._check_block(block)
|
||||
|
||||
assert isinstance(block, (framework.Block, pir.Block))
|
||||
if not in_dygraph_mode():
|
||||
check_variable_and_dtype(
|
||||
var,
|
||||
"Out",
|
||||
["uint16", "float16", "float32", "float64"],
|
||||
"uniform_random",
|
||||
)
|
||||
|
||||
if self._seed == 0:
|
||||
self._seed = block.program.random_seed
|
||||
|
||||
# to be compatible of fp16 initializers
|
||||
if var.dtype == core.VarDesc.VarType.FP16:
|
||||
out_dtype = core.VarDesc.VarType.FP32
|
||||
out_var = block.create_var(
|
||||
name=unique_name.generate(
|
||||
".".join(['uniform_random', var.name, 'tmp'])
|
||||
),
|
||||
shape=var.shape,
|
||||
dtype=out_dtype,
|
||||
type=core.VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
)
|
||||
else:
|
||||
out_dtype = var.dtype
|
||||
out_var = var
|
||||
|
||||
if in_dygraph_mode():
|
||||
out_var = _C_ops.uniform(
|
||||
var.shape,
|
||||
out_dtype,
|
||||
self._low,
|
||||
self._high,
|
||||
self._seed,
|
||||
var.place if var.place._type() else _current_expected_place(),
|
||||
)
|
||||
if var.dtype == core.VarDesc.VarType.FP16:
|
||||
var_tmp = _C_ops.cast(out_var, var.dtype)
|
||||
var_tmp._share_underline_tensor_to(var)
|
||||
else:
|
||||
out_var._share_underline_tensor_to(var)
|
||||
return None
|
||||
elif in_pir_mode():
|
||||
if var.dtype == core.DataType.FLOAT16:
|
||||
out_dtype = core.DataType.FLOAT32
|
||||
else:
|
||||
out_dtype = var.dtype
|
||||
out_var = _C_ops.uniform(
|
||||
var.shape,
|
||||
out_dtype,
|
||||
self._low,
|
||||
self._high,
|
||||
self._seed,
|
||||
_current_expected_place(),
|
||||
)
|
||||
if (
|
||||
var.dtype == core.DataType.FLOAT16
|
||||
and out_var.dtype != core.DataType.FLOAT16
|
||||
):
|
||||
return _C_ops.cast(out_var, var.dtype)
|
||||
return out_var
|
||||
else:
|
||||
op = block.append_op(
|
||||
type="uniform_random",
|
||||
inputs={},
|
||||
outputs={"Out": out_var},
|
||||
attrs={
|
||||
"shape": var.shape,
|
||||
"dtype": out_dtype,
|
||||
"min": self._low,
|
||||
"max": self._high,
|
||||
"seed": self._seed,
|
||||
"diag_num": self._diag_num,
|
||||
"diag_step": self._diag_step,
|
||||
"diag_val": self._diag_val,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if var.dtype == core.VarDesc.VarType.FP16:
|
||||
block.append_op(
|
||||
type="cast",
|
||||
inputs={"X": out_var},
|
||||
outputs={"Out": var},
|
||||
attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype},
|
||||
)
|
||||
|
||||
var.op = op
|
||||
return op
|
||||
|
||||
|
||||
class Uniform(UniformInitializer):
|
||||
"""The uniform distribution initializer.
|
||||
|
||||
Args:
|
||||
low (float, optional): Lower boundary of the uniform distribution. Default is :math:`-1.0`.
|
||||
high (float, optional): Upper boundary of the uniform distribution. Default is :math:`1.0`.
|
||||
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
|
||||
|
||||
Returns:
|
||||
A parameter initialized by uniform distribution.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> paddle.seed(1)
|
||||
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
|
||||
>>> weight_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_weight",
|
||||
... initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5),
|
||||
... )
|
||||
>>> bias_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_bias",
|
||||
... initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5),
|
||||
... )
|
||||
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
|
||||
>>> print(linear.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[-0.48212373, 0.26492310],
|
||||
[ 0.17605734, -0.45379421]])
|
||||
|
||||
>>> print(linear.bias)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[-0.11236754, 0.46462214])
|
||||
|
||||
>>> res = linear(data)
|
||||
>>> print(res)
|
||||
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[[-0.41843393, 0.27575102]],
|
||||
[[-0.41843393, 0.27575102]],
|
||||
[[-0.41843393, 0.27575102]]])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, low: float = -1.0, high: float = 1.0, name: str | None = None
|
||||
) -> None:
|
||||
assert low is not None, 'low should not be None'
|
||||
assert high is not None, 'high should not be None'
|
||||
assert high >= low, 'high should greater or equal than low'
|
||||
super().__init__(
|
||||
low=low, high=high, seed=0, diag_num=0, diag_step=0, diag_val=1.0
|
||||
)
|
||||
@@ -0,0 +1,432 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import paddle
|
||||
from paddle import _C_ops
|
||||
|
||||
from ...base import core, framework, unique_name
|
||||
from ...base.data_feeder import check_variable_and_dtype
|
||||
from ...base.framework import (
|
||||
_current_expected_place,
|
||||
in_dygraph_mode,
|
||||
in_pir_mode,
|
||||
)
|
||||
from .initializer import Initializer
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
class XavierInitializer(Initializer):
|
||||
r"""
|
||||
This class implements the Xavier weight initializer from the paper
|
||||
`Understanding the difficulty of training deep feedforward neural
|
||||
networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
|
||||
by Xavier Glorot and Yoshua Bengio.
|
||||
|
||||
This initializer is designed to keep the scale of the gradients
|
||||
approximately same in all the layers. In case of Uniform distribution,
|
||||
the range is [-x, x], where
|
||||
|
||||
.. math::
|
||||
|
||||
x = gain \times \sqrt{\\frac{6.0}{fan\_in + fan\_out}}
|
||||
|
||||
In case of Normal distribution, the mean is 0 and the standard deviation
|
||||
is
|
||||
|
||||
.. math::
|
||||
|
||||
gain \times \sqrt{\\frac{2.0}{fan\_in + fan\_out}}
|
||||
|
||||
|
||||
Args:
|
||||
uniform (bool, optional): whether to use uniform ,if False use normal distribution. Default is True.
|
||||
fan_in (float|None, optional): fan_in for Xavier initialization. If None, it is
|
||||
inferred from the variable. Default is None.
|
||||
fan_out (float|None, optional): fan_out for Xavier initialization. If None, it is
|
||||
inferred from the variable. Default is None.
|
||||
seed (int, optional): Random seed. Default is 0.
|
||||
gain (float, optional): Scaling Tensor. Default is 1.0.
|
||||
|
||||
Note:
|
||||
It is recommended to set fan_in and fan_out to None for most cases.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
uniform: bool = True,
|
||||
fan_in: float | None = None,
|
||||
fan_out: float | None = None,
|
||||
seed: int = 0,
|
||||
gain: float = 1.0,
|
||||
) -> None:
|
||||
assert uniform is not None
|
||||
assert seed is not None
|
||||
super().__init__()
|
||||
self._uniform = uniform
|
||||
self._fan_in = fan_in
|
||||
self._fan_out = fan_out
|
||||
self._seed = seed
|
||||
self._gain = gain
|
||||
|
||||
def forward(
|
||||
self, var: paddle.Tensor, block: paddle.pir.Block | None = None
|
||||
) -> paddle.Tensor | None:
|
||||
"""Initialize the input tensor with Xavier initialization.
|
||||
|
||||
Args:
|
||||
var(Tensor): Tensor that needs to be initialized.
|
||||
block(Block|None, optional): The block in which initialization ops
|
||||
should be added. Used in static graph only, default None.
|
||||
|
||||
Returns:
|
||||
The initialization op
|
||||
"""
|
||||
|
||||
block = self._check_block(block)
|
||||
assert isinstance(block, (framework.Block, paddle.pir.Block))
|
||||
if not isinstance(var, paddle.pir.core.ParameterMeta):
|
||||
check_variable_and_dtype(
|
||||
var,
|
||||
"Out",
|
||||
["uint16", "float16", "float32", "float64"],
|
||||
"xavier_init",
|
||||
)
|
||||
|
||||
f_in, f_out = self._compute_fans(var)
|
||||
|
||||
# If fan_in and fan_out are passed, use them
|
||||
fan_in = f_in if self._fan_in is None else self._fan_in
|
||||
fan_out = f_out if self._fan_out is None else self._fan_out
|
||||
|
||||
if self._seed == 0:
|
||||
self._seed = block.program.random_seed
|
||||
|
||||
out_var_shape = (
|
||||
var._local_shape
|
||||
if (isinstance(var, framework.EagerParamBase) and var.is_dist())
|
||||
else var.shape
|
||||
)
|
||||
# to be compatible of fp16 initializers
|
||||
origin_dtype = var.dtype
|
||||
if origin_dtype == core.VarDesc.VarType.FP16 or (
|
||||
origin_dtype == core.VarDesc.VarType.BF16 and not self._uniform
|
||||
):
|
||||
out_dtype = core.VarDesc.VarType.FP32
|
||||
out_var = block.create_var(
|
||||
name=unique_name.generate(
|
||||
".".join(['xavier_init', var.name, 'tmp'])
|
||||
),
|
||||
shape=out_var_shape,
|
||||
dtype=out_dtype,
|
||||
type=core.VarDesc.VarType.DENSE_TENSOR,
|
||||
persistable=False,
|
||||
)
|
||||
elif (
|
||||
origin_dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
|
||||
and not self._uniform
|
||||
):
|
||||
out_dtype = core.DataType.FLOAT32
|
||||
out_var = var
|
||||
else:
|
||||
out_dtype = origin_dtype
|
||||
out_var = var
|
||||
|
||||
if in_dygraph_mode():
|
||||
if self._uniform:
|
||||
if 0 in [fan_in, fan_out]:
|
||||
limit = 0.0
|
||||
else:
|
||||
limit = self._gain * math.sqrt(
|
||||
6.0 / float(fan_in + fan_out)
|
||||
)
|
||||
out_var = _C_ops.uniform(
|
||||
out_var_shape,
|
||||
out_dtype,
|
||||
-limit,
|
||||
limit,
|
||||
self._seed,
|
||||
_current_expected_place(),
|
||||
)
|
||||
else:
|
||||
if 0 in [fan_in, fan_out]:
|
||||
std = 0.0
|
||||
else:
|
||||
std = self._gain * math.sqrt(2.0 / float(fan_in + fan_out))
|
||||
|
||||
place = _current_expected_place()
|
||||
out_var = _C_ops.gaussian(
|
||||
out_var_shape,
|
||||
0.0,
|
||||
std,
|
||||
self._seed,
|
||||
out_dtype,
|
||||
place,
|
||||
)
|
||||
|
||||
if origin_dtype == core.VarDesc.VarType.FP16 or (
|
||||
origin_dtype
|
||||
in [
|
||||
core.VarDesc.VarType.BF16,
|
||||
core.DataType.FLOAT16,
|
||||
core.DataType.BFLOAT16,
|
||||
]
|
||||
and not self._uniform
|
||||
):
|
||||
out_var = _C_ops.cast(out_var, origin_dtype)
|
||||
if isinstance(var, framework.EagerParamBase) and var.is_dist():
|
||||
# lazy init for dist tensor
|
||||
out_var = (
|
||||
paddle.distributed.auto_parallel.api.dtensor_from_local(
|
||||
out_var, var.process_mesh, var.placements
|
||||
)
|
||||
)
|
||||
out_var._share_underline_tensor_to(var)
|
||||
return None
|
||||
elif in_pir_mode():
|
||||
if self._uniform:
|
||||
if 0 in [fan_in, fan_out]:
|
||||
limit = 0.0
|
||||
else:
|
||||
limit = self._gain * math.sqrt(
|
||||
6.0 / float(fan_in + fan_out)
|
||||
)
|
||||
out_var = paddle._pir_ops.uniform(
|
||||
out_var.shape,
|
||||
out_dtype,
|
||||
-limit,
|
||||
limit,
|
||||
self._seed,
|
||||
_current_expected_place(),
|
||||
)
|
||||
else:
|
||||
if 0 in [fan_in, fan_out]:
|
||||
std = 0.0
|
||||
else:
|
||||
std = self._gain * math.sqrt(2.0 / float(fan_in + fan_out))
|
||||
out_var = _C_ops.gaussian(
|
||||
out_var.shape,
|
||||
0.0,
|
||||
std,
|
||||
self._seed,
|
||||
out_dtype,
|
||||
_current_expected_place(),
|
||||
)
|
||||
|
||||
if (
|
||||
origin_dtype in (core.DataType.FLOAT16, core.DataType.BFLOAT16)
|
||||
and not self._uniform
|
||||
):
|
||||
return _C_ops.cast(out_var, origin_dtype)
|
||||
|
||||
return out_var
|
||||
else:
|
||||
if self._uniform:
|
||||
if 0 in [fan_in, fan_out]:
|
||||
limit = 0.0
|
||||
else:
|
||||
limit = self._gain * math.sqrt(
|
||||
6.0 / float(fan_in + fan_out)
|
||||
)
|
||||
op = block.append_op(
|
||||
type="uniform_random",
|
||||
inputs={},
|
||||
outputs={"Out": out_var},
|
||||
attrs={
|
||||
"shape": out_var.shape,
|
||||
"dtype": out_dtype,
|
||||
"min": -limit,
|
||||
"max": limit,
|
||||
"seed": self._seed,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
else:
|
||||
if 0 in [fan_in, fan_out]:
|
||||
std = 0.0
|
||||
else:
|
||||
std = self._gain * math.sqrt(2.0 / float(fan_in + fan_out))
|
||||
op = block.append_op(
|
||||
type="gaussian_random",
|
||||
outputs={"Out": out_var},
|
||||
attrs={
|
||||
"shape": out_var.shape,
|
||||
"dtype": out_var.dtype,
|
||||
"mean": 0.0,
|
||||
"std": std,
|
||||
"seed": self._seed,
|
||||
},
|
||||
stop_gradient=True,
|
||||
)
|
||||
|
||||
if origin_dtype == core.VarDesc.VarType.FP16 or (
|
||||
origin_dtype == core.VarDesc.VarType.BF16 and not self._uniform
|
||||
):
|
||||
block.append_op(
|
||||
type="cast",
|
||||
inputs={"X": out_var},
|
||||
outputs={"Out": var},
|
||||
attrs={
|
||||
"in_dtype": out_var.dtype,
|
||||
"out_dtype": origin_dtype,
|
||||
},
|
||||
)
|
||||
|
||||
var.op = op
|
||||
return op
|
||||
|
||||
|
||||
class XavierNormal(XavierInitializer):
|
||||
r"""
|
||||
This class implements the Xavier weight initializer from the paper
|
||||
`Understanding the difficulty of training deep feedforward neural
|
||||
networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
|
||||
by Xavier Glorot and Yoshua Bengio, using a normal distribution whose mean is :math:`0` and standard deviation is
|
||||
|
||||
.. math::
|
||||
|
||||
gain \times \sqrt{\frac{2.0}{fan\_in + fan\_out}}.
|
||||
|
||||
|
||||
Args:
|
||||
fan_in (float|None, optional): fan_in for Xavier initialization, which is
|
||||
inferred from the Tensor. Default is None.
|
||||
fan_out (float|None, optional): fan_out for Xavier initialization, which is
|
||||
inferred from the Tensor. Default is None.
|
||||
gain (float, optional): Scaling Tensor. Default is 1.0.
|
||||
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
|
||||
|
||||
Returns:
|
||||
A parameter initialized by Xavier weight, using a normal distribution.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> paddle.seed(1)
|
||||
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
|
||||
>>> weight_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_weight",
|
||||
... initializer=paddle.nn.initializer.XavierNormal(),
|
||||
... )
|
||||
>>> bias_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_bias",
|
||||
... initializer=paddle.nn.initializer.XavierNormal(),
|
||||
... )
|
||||
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
|
||||
>>> print(linear.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[-0.21607460, 0.08382989],
|
||||
[ 0.29147008, -0.07049121]])
|
||||
|
||||
>>> print(linear.bias)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[1.06076419, 0.87684733])
|
||||
|
||||
>>> res = linear(data)
|
||||
>>> print(res)
|
||||
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[[1.13615966, 0.89018601]],
|
||||
[[1.13615966, 0.89018601]],
|
||||
[[1.13615966, 0.89018601]]])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fan_in: float | None = None,
|
||||
fan_out: float | None = None,
|
||||
gain: float = 1.0,
|
||||
name: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
uniform=False, fan_in=fan_in, fan_out=fan_out, seed=0, gain=gain
|
||||
)
|
||||
|
||||
|
||||
class XavierUniform(XavierInitializer):
|
||||
r"""
|
||||
This class implements the Xavier weight initializer from the paper
|
||||
`Understanding the difficulty of training deep feedforward neural
|
||||
networks <http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf>`_
|
||||
by Xavier Glorot and Yoshua Bengio.
|
||||
|
||||
This initializer is designed to keep the scale of the gradients
|
||||
approximately same in all the layers. In case of Uniform distribution,
|
||||
the range is :math:`[-x,x]`, where
|
||||
|
||||
.. math::
|
||||
|
||||
x = gain \times \sqrt{\frac{6.0}{fan\_in + fan\_out}}.
|
||||
|
||||
Args:
|
||||
fan_in (float|None, optional): fan_in for Xavier initialization, which is
|
||||
inferred from the Tensor. Default is None.
|
||||
fan_out (float|None, optional): fan_out for Xavier initialization, which is
|
||||
inferred from the Tensor. Default is None.
|
||||
gain (float, optional): Scaling Tensor. Default is 1.0.
|
||||
name (str|None, optional): For details, please refer to :ref:`api_guide_Name`. Generally, no setting is required. Default: None.
|
||||
|
||||
Returns:
|
||||
A parameter initialized by Xavier weight, using a uniform distribution.
|
||||
|
||||
Examples:
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> import paddle
|
||||
>>> paddle.seed(1)
|
||||
>>> data = paddle.ones(shape=[3, 1, 2], dtype='float32')
|
||||
>>> weight_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_weight",
|
||||
... initializer=paddle.nn.initializer.XavierUniform(),
|
||||
... )
|
||||
>>> bias_attr = paddle.framework.ParamAttr(
|
||||
... name="linear_bias",
|
||||
... initializer=paddle.nn.initializer.XavierUniform(),
|
||||
... )
|
||||
>>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr)
|
||||
>>> print(linear.weight)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[-1.18095720, 0.64892638],
|
||||
[ 0.43125069, -1.11156428]])
|
||||
>>> print(linear.bias)
|
||||
Parameter containing:
|
||||
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[-0.27524316, 1.13808715])
|
||||
|
||||
>>> res = linear(data)
|
||||
>>> print(res)
|
||||
Tensor(shape=[3, 1, 2], dtype=float32, place=Place(cpu), stop_gradient=False,
|
||||
[[[-1.02494967, 0.67544925]],
|
||||
[[-1.02494967, 0.67544925]],
|
||||
[[-1.02494967, 0.67544925]]])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fan_in: float | None = None,
|
||||
fan_out: float | None = None,
|
||||
gain: float = 1.0,
|
||||
name: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
uniform=True, fan_in=fan_in, fan_out=fan_out, seed=0, gain=gain
|
||||
)
|
||||
Reference in New Issue
Block a user