wehub-resource-sync
626 lines
25 KiB
Python
626 lines
25 KiB
Python
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
import paddle
|
|
from paddle import _C_ops
|
|
from paddle.base import framework
|
|
from paddle.base.dygraph import base as imperative_base
|
|
from paddle.base.layer_helper import LayerHelper
|
|
from paddle.base.wrapped_decorator import signature_safe_contextmanager
|
|
from paddle.framework import (
|
|
in_dynamic_mode,
|
|
in_dynamic_or_pir_mode,
|
|
in_pir_mode,
|
|
)
|
|
from paddle.optimizer import Optimizer
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Generator, Sequence
|
|
|
|
from paddle import Tensor
|
|
from paddle.optimizer.optimizer import _ParameterConfig
|
|
from paddle.static import Executor, Program
|
|
|
|
|
|
__all__ = []
|
|
|
|
|
|
class ModelAverage(Optimizer):
|
|
r"""
|
|
The ModelAverage optimizer accumulates specific continuous historical
|
|
parameters during training. The accumulated historical range can be controlled
|
|
by the passed ``average_window_rate`` argument. The averaged ``Parameter`` are
|
|
used in the prediction, which usually can improve the accuracy of the prediction.
|
|
|
|
Accumulate the average of the ``Parameter`` in the sliding window, the result will be saved
|
|
in a temporary variable, can be applied to the current model's ``Parameter`` by calling
|
|
the ``apply()`` method, and the current model ``Parameter`` can be restored by calling
|
|
the ``restore()`` method.
|
|
|
|
The window size for calculating the average is determined by ``average_window_rate``,
|
|
``min_average_window``, ``max_average_window`` and the current ``Parameter`` update times (num_updates).
|
|
|
|
When the cumulative times (num_accumulates) is greater than the specific window
|
|
threshold (average_window), the accumulated ``Parameter`` temporary variable is set to 0.0.
|
|
The following example will help to understand the role of these arguments:
|
|
|
|
::
|
|
|
|
if num_accumulates >= min_average_window and num_accumulates >= min(max_average_window, num_updates * average_window_rate):
|
|
num_accumulates = 0
|
|
|
|
In the above conditional judgment statement, ``num_accumulates`` indicates the current
|
|
accumulated number, which can be abstractly understood as the length of the cumulative window.
|
|
The length of the window must be at least the length set by the ``min_average_window`` argument,
|
|
and cannot exceed the length specified by the ``max_average_window`` argument or
|
|
``num_updates * average_window_rate``, where ``num_updates`` indicates the current ``Parameter``
|
|
update times, ``average_window_rate`` is a coefficient that calculates the length of the window.
|
|
|
|
Args:
|
|
average_window_rate (float): The calculate ratio of the window length relative to ``Parameter`` update times.
|
|
parameters (list, optional): List of ``Tensor`` names to update to minimize ``loss``. \
|
|
This parameter is required in dygraph mode. \
|
|
The default value is None in static graph mode, at this time all parameters will be updated.
|
|
min_average_window (int, optional): the minimum size of average window length. The default value is 10000.
|
|
max_average_window (int, optional): The maximum size of average window length. The default value is 10000.
|
|
name (str, optional): Normally there is no need for user to set this property.
|
|
For more information, please refer to :ref:`api_guide_Name`.
|
|
The default value is None.
|
|
|
|
Examples:
|
|
|
|
.. code-block:: pycon
|
|
|
|
>>> # doctest: +SKIP("Cannot get source code by to_static in REPL")
|
|
>>> import numpy as np
|
|
>>> import paddle
|
|
>>> import paddle.nn as nn
|
|
>>> import paddle.optimizer as opt
|
|
|
|
>>> BATCH_SIZE = 16
|
|
>>> BATCH_NUM = 4
|
|
>>> EPOCH_NUM = 4
|
|
|
|
>>> IMAGE_SIZE = 784
|
|
>>> CLASS_NUM = 10
|
|
|
|
>>> # define a random dataset
|
|
>>> class RandomDataset(paddle.io.Dataset): # type: ignore[type-arg]
|
|
... def __init__(self, num_samples):
|
|
... self.num_samples = num_samples
|
|
... def __getitem__(self, idx):
|
|
... image = np.random.random([IMAGE_SIZE]).astype('float32')
|
|
... label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
|
|
... return image, label
|
|
... def __len__(self):
|
|
... return self.num_samples
|
|
...
|
|
>>> class LinearNet(nn.Layer):
|
|
... def __init__(self):
|
|
... super().__init__()
|
|
... self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
|
|
... self.bias = self._linear.bias
|
|
...
|
|
... @paddle.jit.to_static
|
|
... def forward(self, x):
|
|
... return self._linear(x)
|
|
...
|
|
>>> def train(layer, loader, loss_fn, opt, model_average):
|
|
... for epoch_id in range(EPOCH_NUM):
|
|
... for batch_id, (image, label) in enumerate(loader()):
|
|
... out = layer(image)
|
|
... loss = loss_fn(out, label)
|
|
... loss.backward()
|
|
... opt.step()
|
|
... model_average.step()
|
|
... opt.clear_grad()
|
|
... model_average.clear_grad()
|
|
... print("Train Epoch {} batch {}: loss = {}, bias = {}".format(
|
|
... epoch_id, batch_id, np.mean(loss.numpy()), layer.bias.numpy()))
|
|
...
|
|
>>> def evaluate(layer, loader, loss_fn):
|
|
... for batch_id, (image, label) in enumerate(loader()):
|
|
... out = layer(image)
|
|
... loss = loss_fn(out, label)
|
|
... loss.backward()
|
|
... print("Evaluate batch {}: loss = {}, bias = {}".format(
|
|
... batch_id, np.mean(loss.numpy()), layer.bias.numpy()))
|
|
...
|
|
>>> # create network
|
|
>>> layer = LinearNet()
|
|
>>> loss_fn = nn.CrossEntropyLoss()
|
|
>>> optimizer = opt.Momentum(learning_rate=0.2, momentum=0.1, parameters=layer.parameters())
|
|
>>> model_average = paddle.incubate.ModelAverage(
|
|
... 0.15,
|
|
... parameters=layer.parameters(),
|
|
... min_average_window=2,
|
|
... max_average_window=10
|
|
... )
|
|
...
|
|
>>> # create data loader
|
|
>>> dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
|
|
>>> loader = paddle.io.DataLoader(dataset,
|
|
... batch_size=BATCH_SIZE,
|
|
... shuffle=True,
|
|
... drop_last=True,
|
|
... num_workers=2)
|
|
...
|
|
>>> # create data loader
|
|
>>> eval_loader = paddle.io.DataLoader(dataset,
|
|
... batch_size=BATCH_SIZE,
|
|
... shuffle=True,
|
|
... drop_last=True,
|
|
... num_workers=1
|
|
... )
|
|
...
|
|
>>> # train
|
|
>>> train(layer, loader, loss_fn, optimizer, model_average)
|
|
|
|
>>> print("\nEvaluate With ModelAverage")
|
|
>>> with model_average.apply(need_restore=False):
|
|
... evaluate(layer, eval_loader, loss_fn)
|
|
|
|
>>> print("\nEvaluate With Restored Parameters")
|
|
>>> model_average.restore()
|
|
>>> evaluate(layer, eval_loader, loss_fn)
|
|
|
|
"""
|
|
|
|
helper: LayerHelper
|
|
average_window: float
|
|
min_average_window: int
|
|
max_average_window: int
|
|
type: str
|
|
apply_program: Program
|
|
restore_program: Program
|
|
|
|
def __init__(
|
|
self,
|
|
average_window_rate: float,
|
|
parameters: Sequence[Tensor] | Sequence[_ParameterConfig] | None = None,
|
|
min_average_window: int = 10000,
|
|
max_average_window: int = 10000,
|
|
name: str | None = None,
|
|
) -> None:
|
|
super().__init__(
|
|
learning_rate=0.0,
|
|
parameters=parameters,
|
|
weight_decay=None,
|
|
grad_clip=None,
|
|
name=name,
|
|
)
|
|
|
|
self.helper = LayerHelper(self.__class__.__name__)
|
|
self.average_window = average_window_rate
|
|
self.min_average_window = min_average_window
|
|
self.max_average_window = max_average_window
|
|
self.type = "average_accumulates"
|
|
|
|
if not in_dynamic_mode():
|
|
global_block = paddle.static.default_main_program().global_block()
|
|
all_parameters = (
|
|
parameters if parameters else global_block.all_parameters()
|
|
)
|
|
|
|
self._create_accumulators(global_block, all_parameters)
|
|
for param in all_parameters:
|
|
self._append_optimize_op(global_block, [param, None])
|
|
self.apply_program = paddle.static.Program()
|
|
block = self.apply_program.global_block()
|
|
with paddle.static.program_guard(main_program=self.apply_program):
|
|
for param in all_parameters:
|
|
self._add_average_apply_op(block, param)
|
|
self.restore_program = paddle.static.Program()
|
|
block = self.restore_program.global_block()
|
|
with paddle.static.program_guard(main_program=self.restore_program):
|
|
for param in all_parameters:
|
|
self._add_average_restore_op(block, param)
|
|
|
|
def _create_accumulators(self, block, parameters):
|
|
assert isinstance(block, (framework.Block, paddle.pir.Block))
|
|
|
|
for param in parameters:
|
|
self._add_accumulator('sum_1', param)
|
|
self._add_accumulator('sum_2', param)
|
|
self._add_accumulator('sum_3', param)
|
|
self._add_accumulator('restore', param)
|
|
self._add_accumulator(
|
|
'num_accumulates', param, dtype='int64', shape=[1]
|
|
)
|
|
self._add_accumulator(
|
|
'old_num_accumulates', param, dtype='int64', shape=[1]
|
|
)
|
|
self._add_accumulator(
|
|
'num_updates', param, dtype='int64', shape=[1]
|
|
)
|
|
|
|
def _append_optimize_op(self, block, param_and_grad):
|
|
assert isinstance(block, (framework.Block, paddle.pir.Block))
|
|
|
|
sum_1 = self._get_accumulator('sum_1', param_and_grad[0])
|
|
sum_2 = self._get_accumulator('sum_2', param_and_grad[0])
|
|
sum_3 = self._get_accumulator('sum_3', param_and_grad[0])
|
|
num_accumulates = self._get_accumulator(
|
|
'num_accumulates', param_and_grad[0]
|
|
)
|
|
old_num_accumulates = self._get_accumulator(
|
|
'old_num_accumulates', param_and_grad[0]
|
|
)
|
|
num_updates = self._get_accumulator('num_updates', param_and_grad[0])
|
|
|
|
if in_dynamic_or_pir_mode():
|
|
_, _, _, _, _, _ = _C_ops.average_accumulates_(
|
|
param_and_grad[0],
|
|
sum_1,
|
|
sum_2,
|
|
sum_3,
|
|
num_accumulates,
|
|
old_num_accumulates,
|
|
num_updates,
|
|
self.average_window,
|
|
self.max_average_window,
|
|
self.min_average_window,
|
|
)
|
|
return None
|
|
|
|
block = framework.default_main_program().global_block()
|
|
attrs = {
|
|
"average_window": self.average_window,
|
|
"min_average_window": self.min_average_window,
|
|
"max_average_window": self.max_average_window,
|
|
}
|
|
|
|
inputs = {
|
|
"param": param_and_grad[0],
|
|
"in_sum_1": sum_1,
|
|
"in_sum_2": sum_2,
|
|
"in_sum_3": sum_3,
|
|
"in_num_accumulates": num_accumulates,
|
|
"in_old_num_accumulates": old_num_accumulates,
|
|
"in_num_updates": num_updates,
|
|
}
|
|
|
|
outputs = {
|
|
"out_sum_1": sum_1,
|
|
"out_sum_2": sum_2,
|
|
"out_sum_3": sum_3,
|
|
"out_num_accumulates": num_accumulates,
|
|
"out_old_num_accumulates": old_num_accumulates,
|
|
"out_num_updates": num_updates,
|
|
}
|
|
|
|
average_accumulates_op = block.append_op(
|
|
type=self.type,
|
|
inputs=inputs,
|
|
outputs=outputs,
|
|
attrs=attrs,
|
|
stop_gradient=True,
|
|
)
|
|
|
|
return average_accumulates_op
|
|
|
|
@imperative_base.no_grad
|
|
def minimize(
|
|
self,
|
|
loss: Tensor,
|
|
startup_program: Program | None = None,
|
|
parameters: list[Tensor] | None = None,
|
|
no_grad_set: set[Tensor] | set[str] | None = None,
|
|
) -> None:
|
|
"""
|
|
Add operations to minimize ``loss`` by updating ``parameters``.
|
|
|
|
Args:
|
|
loss (Tensor): A ``Tensor`` containing the value to minimize.
|
|
startup_program (Program, optional): :ref:`api_paddle_static_Program` for
|
|
initializing parameters in ``parameters``. The default value
|
|
is None, at this time :ref:`api_paddle_static_default_startup_program` will be used.
|
|
parameters (list, optional): List of ``Tensor`` or ``Tensor.name`` to update
|
|
to minimize ``loss``. The default value is None, at this time all parameters
|
|
will be updated.
|
|
no_grad_set (set, optional): Set of ``Tensor`` or ``Tensor.name`` that don't need
|
|
to be updated. The default value is None.
|
|
|
|
Returns:
|
|
tuple: tuple (optimize_ops, params_grads), A list of operators appended
|
|
by minimize and a list of (param, grad) tensor pairs, param is
|
|
``Parameter``, grad is the gradient value corresponding to the parameter.
|
|
In static graph mode, the returned tuple can be passed to ``fetch_list`` in ``Executor.run()`` to
|
|
indicate program pruning. If so, the program will be pruned by ``feed`` and
|
|
``fetch_list`` before run, see details in ``Executor``.
|
|
|
|
Examples:
|
|
|
|
.. code-block:: pycon
|
|
|
|
>>> import paddle
|
|
>>> inp = paddle.rand([1, 10], dtype="float32")
|
|
>>> linear = paddle.nn.Linear(10, 1)
|
|
>>> out = linear(inp)
|
|
>>> loss = paddle.mean(out)
|
|
>>> loss.backward()
|
|
|
|
>>> sgd = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters())
|
|
>>> sgd.minimize(loss)
|
|
|
|
>>> modelaverage = paddle.incubate.ModelAverage(
|
|
... 0.15,
|
|
... parameters=linear.parameters(),
|
|
... min_average_window=2,
|
|
... max_average_window=4,
|
|
... )
|
|
>>> modelaverage.minimize(loss)
|
|
>>> sgd.clear_grad()
|
|
>>> modelaverage.clear_grad()
|
|
|
|
"""
|
|
if in_dynamic_mode():
|
|
self.step()
|
|
|
|
@framework.dygraph_only
|
|
@imperative_base.no_grad
|
|
def step(self) -> None:
|
|
"""
|
|
Execute the optimizer and update parameters once.
|
|
|
|
Returns:
|
|
None
|
|
|
|
Examples:
|
|
|
|
.. code-block:: pycon
|
|
|
|
>>> import paddle
|
|
>>> inp = paddle.rand([1, 10], dtype="float32")
|
|
>>> linear = paddle.nn.Linear(10, 1)
|
|
>>> out = linear(inp)
|
|
>>> loss = paddle.mean(out)
|
|
>>> sgd = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters())
|
|
>>> modelaverage = paddle.incubate.ModelAverage(
|
|
... 0.15,
|
|
... parameters=linear.parameters(),
|
|
... min_average_window=2,
|
|
... max_average_window=4,
|
|
... )
|
|
>>> loss.backward()
|
|
>>> sgd.step()
|
|
>>> modelaverage.step()
|
|
>>> sgd.clear_grad()
|
|
>>> modelaverage.clear_grad()
|
|
"""
|
|
|
|
params_grads = []
|
|
for param in self._parameter_list:
|
|
if not param.trainable:
|
|
continue
|
|
if param._grad_ivar() is not None:
|
|
grad_var = param._grad_ivar()
|
|
params_grads.append((param, grad_var))
|
|
|
|
block = framework.default_main_program().global_block()
|
|
self._create_accumulators(block, self._parameter_list)
|
|
for param_and_grad in params_grads:
|
|
self._append_optimize_op(block, param_and_grad)
|
|
|
|
@signature_safe_contextmanager
|
|
@imperative_base.no_grad
|
|
def apply(
|
|
self, executor: Executor | None = None, need_restore: bool = True
|
|
) -> Generator[None, None, None]:
|
|
"""
|
|
Apply the average of the cumulative ``Parameter`` to the parameters of the current model.
|
|
|
|
Args:
|
|
executor(Executor): The network executor in static-graph mode. The default value is None in dygraph mode.
|
|
need_restore(bool): Restore flag variable, if set to True, the network will restore
|
|
the parameters of the network to the default value, if set to False,
|
|
it will not be restored. The default value is True.
|
|
|
|
Examples:
|
|
|
|
.. code-block:: pycon
|
|
|
|
>>> import paddle
|
|
>>> inp = paddle.rand([1, 10], dtype="float32")
|
|
>>> linear = paddle.nn.Linear(10, 1)
|
|
>>> out = linear(inp)
|
|
>>> loss = paddle.mean(out)
|
|
>>> loss.backward()
|
|
|
|
>>> sgd = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters())
|
|
|
|
>>> modelaverage = paddle.incubate.ModelAverage(
|
|
... 0.15,
|
|
... parameters=linear.parameters(),
|
|
... min_average_window=2,
|
|
... max_average_window=4,
|
|
... )
|
|
>>> sgd.step()
|
|
>>> modelaverage.step()
|
|
|
|
>>> with modelaverage.apply():
|
|
... for param in linear.parameters():
|
|
... print(param)
|
|
|
|
>>> for param in linear.parameters():
|
|
... print(param)
|
|
"""
|
|
if in_dynamic_mode():
|
|
for param in self._parameter_list:
|
|
num_accumulates = self._get_accumulator(
|
|
'num_accumulates', param
|
|
)
|
|
old_num_accumulates = self._get_accumulator(
|
|
'old_num_accumulates', param
|
|
)
|
|
sum_1 = self._get_accumulator('sum_1', param)
|
|
sum_2 = self._get_accumulator('sum_2', param)
|
|
sum_3 = self._get_accumulator('sum_3', param)
|
|
param_restore = self._get_accumulator('restore', param)
|
|
|
|
paddle.assign(param, param_restore)
|
|
total_param = sum_1 + sum_2 + sum_3
|
|
total_accumulates = num_accumulates + old_num_accumulates
|
|
total_param = paddle.cast(total_param, dtype='float32')
|
|
total_accumulates = paddle.cast(
|
|
total_accumulates, dtype='float32'
|
|
)
|
|
average_param = total_param / total_accumulates
|
|
paddle.assign(average_param, param)
|
|
try:
|
|
yield
|
|
finally:
|
|
if need_restore:
|
|
self.restore()
|
|
return
|
|
if executor is None:
|
|
raise RuntimeError(
|
|
"Executor should not be None in static graph mode."
|
|
)
|
|
executor.run(self.apply_program)
|
|
try:
|
|
yield
|
|
finally:
|
|
if need_restore:
|
|
self.restore(executor)
|
|
|
|
@imperative_base.no_grad
|
|
def restore(self, executor: Executor | None = None) -> None:
|
|
"""
|
|
Restore ``Parameter`` values of current model.
|
|
|
|
Args:
|
|
executor(Executor): The network executor in static-graph mode. The default value is None in dygraph mode
|
|
|
|
Examples:
|
|
|
|
.. code-block:: pycon
|
|
|
|
>>> import paddle
|
|
>>> inp = paddle.rand([1, 10], dtype="float32")
|
|
>>> linear = paddle.nn.Linear(10, 1)
|
|
>>> out = linear(inp)
|
|
>>> loss = paddle.mean(out)
|
|
>>> loss.backward()
|
|
|
|
>>> sgd = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters())
|
|
|
|
>>> modelaverage = paddle.incubate.ModelAverage(
|
|
... 0.15,
|
|
... parameters=linear.parameters(),
|
|
... min_average_window=2,
|
|
... max_average_window=4,
|
|
... )
|
|
>>> sgd.step()
|
|
>>> modelaverage.step()
|
|
|
|
>>> with modelaverage.apply(need_restore=False):
|
|
... for param in linear.parameters():
|
|
... print(param)
|
|
|
|
>>> for param in linear.parameters():
|
|
... print(param)
|
|
|
|
>>> modelaverage.restore()
|
|
|
|
>>> for param in linear.parameters():
|
|
... print(param)
|
|
"""
|
|
if in_dynamic_mode():
|
|
for param in self._parameter_list:
|
|
param_restore = self._get_accumulator('restore', param)
|
|
paddle.assign(param_restore, param)
|
|
return
|
|
if executor is None:
|
|
raise RuntimeError(
|
|
"Executor should not be None in static graph mode."
|
|
)
|
|
executor.run(self.restore_program)
|
|
|
|
def _add_average_apply_op(self, block, param):
|
|
if in_pir_mode():
|
|
target_program = paddle.static.default_main_program()
|
|
param = paddle.pir.core._get_parameter(target_program, param)
|
|
restore_value = self._get_accumulator('restore', param)
|
|
grad = paddle.pir.core._get_persistable_value(
|
|
target_program, restore_value
|
|
)
|
|
sum_1 = self._get_accumulator('sum_1', param)
|
|
sum_1 = paddle.pir.core._get_persistable_value(
|
|
target_program, sum_1
|
|
)
|
|
sum_2 = self._get_accumulator('sum_2', param)
|
|
sum_2 = paddle.pir.core._get_persistable_value(
|
|
target_program, sum_2
|
|
)
|
|
sum_3 = self._get_accumulator('sum_3', param)
|
|
sum_3 = paddle.pir.core._get_persistable_value(
|
|
target_program, sum_3
|
|
)
|
|
num_accumulates = self._get_accumulator('num_accumulates', param)
|
|
num_accumulates = paddle.pir.core._get_persistable_value(
|
|
target_program, num_accumulates
|
|
)
|
|
old_num_accumulates = self._get_accumulator(
|
|
'old_num_accumulates', param
|
|
)
|
|
old_num_accumulates = paddle.pir.core._get_persistable_value(
|
|
target_program, old_num_accumulates
|
|
)
|
|
else:
|
|
param = block._clone_variable(param)
|
|
grad = block._clone_variable(
|
|
self._get_accumulator('restore', param)
|
|
)
|
|
sum_1 = block._clone_variable(self._get_accumulator('sum_1', param))
|
|
sum_2 = block._clone_variable(self._get_accumulator('sum_2', param))
|
|
sum_3 = block._clone_variable(self._get_accumulator('sum_3', param))
|
|
num_accumulates = block._clone_variable(
|
|
self._get_accumulator('num_accumulates', param)
|
|
)
|
|
old_num_accumulates = block._clone_variable(
|
|
self._get_accumulator('old_num_accumulates', param)
|
|
)
|
|
# backup param value to grad
|
|
paddle.assign(param, output=grad)
|
|
# param = (sum_1 + sum_2 + sum_3) / (num_accumulates + old_num_accumulates)
|
|
tmp = paddle.add_n([num_accumulates, old_num_accumulates])
|
|
sum = paddle.add_n([sum_1, sum_2, sum_3])
|
|
tmp = paddle.cast(
|
|
x=tmp, dtype='float32' if self._dtype is None else self._dtype
|
|
)
|
|
sum = paddle.cast(
|
|
x=sum, dtype='float32' if self._dtype is None else self._dtype
|
|
)
|
|
divide_out = paddle.divide(x=sum, y=tmp)
|
|
paddle.assign(divide_out, output=param)
|
|
|
|
def _add_average_restore_op(self, block, param):
|
|
if in_pir_mode():
|
|
target_program = paddle.static.default_main_program()
|
|
param = paddle.pir.core._get_parameter(target_program, param)
|
|
restore_value = self._get_accumulator('restore', param)
|
|
grad = paddle.pir.core._get_persistable_value(
|
|
target_program, restore_value
|
|
)
|
|
else:
|
|
param = block._clone_variable(param)
|
|
grad = block._clone_variable(
|
|
self._get_accumulator('restore', param)
|
|
)
|
|
paddle.assign(grad, output=param)
|