wehub/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/paddle/base/dygraph/__init__.py
+30
View File
@@ -0,0 +1,30 @@
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import ( # noqa: F401
base,
tracer,
)
from .base import ( # noqa: F401
disable_dygraph,
enable_dygraph,
enabled,
grad,
guard,
no_grad,
no_grad_,
)
from .tracer import Tracer # noqa: F401
__all__ = []
python/paddle/base/dygraph/base.py
+942
View File
@@ -0,0 +1,942 @@
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import functools
import inspect
import sys
import warnings
from typing import (
TYPE_CHECKING,
Any,
TypeVar,
overload,
)
from typing_extensions import ParamSpec
import paddle
from paddle.base import core, framework
from paddle.base.framework import global_var
from paddle.base.multiprocess_utils import CleanupFuncRegistrar
from paddle.utils.decorator_utils import param_one_alias
from paddle.utils.download import check_and_create_dir
from ..framework import _get_paddle_place
from ..wrapped_decorator import (
copy_signature,
signature_safe_contextmanager,
wrap_decorator,
)
from .tracer import Tracer
if TYPE_CHECKING:
from collections import OrderedDict
from collections.abc import Callable, Generator, Sequence
from contextlib import AbstractContextManager
from types import TracebackType
from typing_extensions import Self
from paddle import Tensor
from paddle._typing import PlaceLike
__all__ = []
_InputT = ParamSpec("_InputT")
_RetT = TypeVar("_RetT")
NON_PERSISTABLE_VAR_NAME_SUFFIX = "__non_persistable"
def in_to_static_mode() -> bool:
"""
Return a bool value that indicates whether running code under `@to_static`
"""
return global_var._in_to_static_mode_
def in_sot_simulation_mode() -> bool:
"""
Returns whether the code is running under the SOT simulation context.
NOTE: Always returns False because if this function is called directly from native Python,
it is not within the SOT simulation process. In that case, returning False is correct.
If the code is running within the SOT simulation process, the function will be represented
by UserDefinedFunctionVariable, which is specially handled in its `call_function` method
to return True when this function is called.
This design avoids introducing `global_var` into the guard logic.
"""
return False
# TODO(Aurelius84): Need to remove this alias after clean usage in PaddleX
in_declarative_mode = in_to_static_mode
def to_static_unsupported_argument_warning(
func_name, input_names, inputs, support_values
):
"""
Warning if inputs do not elementwisely equals to support_values.
It's a utility function for dy2static when dygraph interface have
more inputs than static interface such as paddle.grad.
"""
for name, inp, sup in zip(input_names, inputs, support_values):
if inp != sup:
warnings.warn(
f"{func_name} has unsupported parameter in jit: "
+ f"{name}, jit will discard it"
)
def _switch_to_static_graph_(
func: Callable[_InputT, _RetT],
) -> Callable[_InputT, _RetT]:
def __impl__(*args: _InputT.args, **kwargs: _InputT.kwargs) -> _RetT:
with framework._dygraph_guard(None):
return func(*args, **kwargs)
return __impl__
switch_to_static_graph = wrap_decorator(_switch_to_static_graph_)
@signature_safe_contextmanager
def to_static_mode_guard(
is_to_static: bool = True,
) -> Generator[None, None, None]:
global global_var
original_val = global_var._in_to_static_mode_
global_var._in_to_static_mode_ = is_to_static
try:
yield
finally:
global_var._in_to_static_mode_ = original_val
@signature_safe_contextmanager
def param_guard(
parameters: OrderedDict[str, Tensor],
) -> Generator[None, None, None]:
# Note: parameters is a reference of self._parameters or self._buffers
if in_to_static_mode() and not paddle.in_dynamic_mode() and parameters:
try:
origin_parameters = parameters.copy()
for name, var_base in parameters.items():
if isinstance(var_base, list):
new_var = [_convert_into_variable(var) for var in var_base]
else:
new_var = _convert_into_variable(var_base)
parameters[name] = new_var
yield
finally:
parameters.update(origin_parameters)
else:
yield
def _convert_into_variable(tensor):
"""
Convert Tensor into Variable.
"""
if paddle.framework.use_pir_api():
return paddle.pir.core._convert_into_value(tensor)
if isinstance(tensor, paddle.Tensor):
# Check whether has been created before.
new_var = tensor.block._find_var_recursive(tensor.name)
if new_var is not None:
assert isinstance(new_var, framework.Variable)
# Convert EagerParamBase into Parameter with same attributes in dy2stat.
elif isinstance(tensor, framework.EagerParamBase):
new_var = tensor._to_static_var(to_parameter=True)
else:
# Note(Aurelius84): Convert Tensor in self._buffers into Variable with
# same attributes and set persistable=True to allow saving this var.
# Because users can create a Tensor in `__init__` like a
# `mask` Tensor or `hidden_0` in RNN layers, which is equivalent to a Parameter
# and necessary for inferring. It will be pruned if it's not necessary for inferring.
# But if its shape is empty while created from `create_variable()`, we consider this buffer
# non-persistable. See case of `dropout_state` in lstm api.
is_persistable = True
# NOTE(SigureMo): Why do not use `tensor.name.endswith(NON_PERSISTABLE_VAR_NAME_SUFFIX)`?
# Because the tensor maybe copied, the name of the tensor will be appended with a new suffix.
# Such as `lstm_0.dropout_state__non_persistable_deepcopy_204`
if NON_PERSISTABLE_VAR_NAME_SUFFIX in tensor.name:
is_persistable = False
new_var = tensor._to_static_var(
to_parameter=False, persistable=is_persistable
)
# add param into parameter recorder to collect all the params used in this program.
if new_var.persistable is True:
from paddle.jit.dy2static.program_translator import (
ProgramTranslator,
)
ProgramTranslator.get_instance()._params_recorder.add(
tensor.block.program, tensor
)
return new_var
else:
return tensor
def enabled() -> bool:
"""
This function checks whether the program runs in dynamic graph mode or not.
You can enable dynamic graph mode with :ref:`api_paddle_disable_static` api,
or disable dynamic graph mode with :ref:`api_paddle_enable_static` .
**Note**:
``base.dygraph.enabled`` is the alias of ``base.in_dygraph_mode``, and
``base.in_dygraph_mode`` is recommended to use for now.
Returns:
bool: Whether the program is running in dynamic graph mode.
Examples:
.. code-block:: pycon
>>> import paddle.base as base
>>> base.enable_dygraph() # Now we are in dygragh mode
>>> print(base.dygraph.enabled())
True
>>> base.disable_dygraph()
>>> print(base.dygraph.enabled())
False
"""
# TODO(jiabin): Make this check as in_dygraph_mode when we support default eager mode.
return framework.in_dygraph_mode()
def enable_dygraph(place: PlaceLike | None = None) -> None:
"""
.. note::
Dynamic graph mode is turn ON by default since paddle 2.0.0
This API turn OFF static graph mode. You can turn ON static graph mode by `enable_static <./disable_dygraph_en.html>`_ .
Parameters:
place(paddle.CPUPlace|paddle.CUDAPlace|str, optional): Place to run dynamic graph. Default: None. Which means that the running place will be
determined according to the way of paddle compilation. If ``place`` is string, It can be ``cpu``, and ``gpu:x``, where ``x`` is the
index of the GPUs.
return:
None
Examples:
.. code-block:: pycon
>>> import paddle
>>> print(paddle.in_dynamic_mode())
True
>>> paddle.enable_static()
>>> print(paddle.in_dynamic_mode())
False
>>> paddle.disable_static()
>>> print(paddle.in_dynamic_mode())
True
"""
global global_var
if global_var._functional_dygraph_context_manager is None:
global_var._functional_dygraph_context_manager = guard(
place=_get_paddle_place(place)
)
global_var._functional_dygraph_context_manager.__enter__()
# call disable_dygraph when Python exit
CleanupFuncRegistrar.register(disable_dygraph)
def disable_dygraph() -> None:
"""
.. note::
Dynamic graph mode is turn ON by default since paddle 2.0.0
This API turn ON static graph mode. You can turn ON static graph mode by `disable_static <./enable_dygraph_en.html>`_ .
return:
None
Examples:
.. code-block:: pycon
>>> import paddle
>>> print(paddle.in_dynamic_mode())
True
>>> paddle.enable_static()
>>> print(paddle.in_dynamic_mode())
False
>>> paddle.disable_static()
>>> print(paddle.in_dynamic_mode())
True
"""
global global_var
if global_var._functional_dygraph_context_manager is not None:
global_var._functional_dygraph_context_manager.__exit__(*sys.exc_info())
global_var._functional_dygraph_context_manager = None
@signature_safe_contextmanager
def _switch_tracer_mode_guard_(
is_train: bool = True,
) -> Generator[None, None, None]:
has_grad = core._has_grad()
core._set_has_grad(is_train)
try:
yield
finally:
core._set_has_grad(has_grad)
@overload
def no_grad(func: None = ...) -> AbstractContextManager: ...
@overload
def no_grad(func: Callable[_InputT, _RetT]) -> Callable[_InputT, _RetT]: ...
@param_one_alias(["func", "orig_func"])
def no_grad(func=None):
"""
:api_attr: imperative
Create a context which disables dygraph gradient calculation.
In this mode, the result of every computation will have `stop_gradient=True`.
Also functions as a decorator. (Make sure to instantiate without parenthesis.)
.. note::
Alias Support: The parameter name ``orig_func`` can be used as an alias for ``func``.
Examples:
.. code-block:: pycon
>>> import numpy as np
>>> import paddle.base as base
>>> # use as generator
>>> data = np.array([[2, 3], [4, 5]]).astype('float32')
>>> with base.dygraph.guard():
... l0 = paddle.nn.Linear(2, 2) # l0.weight.gradient() is None
... l1 = paddle.nn.Linear(2, 2)
... with base.dygraph.no_grad():
... # l1.weight.stop_gradient is False
... tmp = l1.weight * 2 # tmp.stop_gradient is True
... x = paddle.to_tensor(data)
... y = l0(x) + tmp
... o = l1(y)
... o.backward()
... print(tmp.grad is None)
... print(l0.weight.grad is None)
True
False
>>> @base.dygraph.no_grad
>>> def test_layer():
... with base.dygraph.guard():
... inp = np.ones([3, 1024], dtype='float32')
... t = paddle.to_tensor(inp)
... linear1 = paddle.nn.Linear(1024, 4, bias_attr=False)
... linear2 = paddle.nn.Linear(4, 4)
... ret = linear1(t)
... dy_ret = linear2(ret)
>>> test_layer()
"""
if func is None:
return _switch_tracer_mode_guard_(is_train=False)
else:
@functools.wraps(func)
def __impl__(
*args: _InputT.args,
**kwargs: _InputT.kwargs,
) -> _RetT:
with _switch_tracer_mode_guard_(is_train=False):
return func(*args, **kwargs)
copy_signature(func, __impl__)
return __impl__
class _DecoratorContextManager:
"""Allow a context manager to be used as a decorator"""
DECORATED_BY_MARKER_ATTR = "__decorated_by__"
def __call__(
self, func: Callable[_InputT, _RetT]
) -> Callable[_InputT, _RetT]:
@functools.wraps(func)
def _decorate_function(*args, **kwargs):
with self:
return func(*args, **kwargs)
@functools.wraps(func)
def _decorate_generator(*args, **kwargs):
gen = func(*args, **kwargs)
with self:
yield from gen
if inspect.isgeneratorfunction(func):
decorated_fn = _decorate_generator
else:
decorated_fn = _decorate_function
copy_signature(func, decorated_fn)
setattr(
decorated_fn,
_DecoratorContextManager.DECORATED_BY_MARKER_ATTR,
self,
)
return decorated_fn
def __enter__(self) -> Any:
raise NotImplementedError
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> bool:
raise NotImplementedError
def clone(self) -> Self:
# override this method if your children class takes __init__ parameters
return self.__class__()
def is_grad_enabled() -> bool:
"""
Returns whether current gradient calculation mode is enabled.
Returns:
bool: True if current gradient calculation mode is enabled, otherwise false.
Examples:
.. code-block:: pycon
>>> import paddle
>>> # Gradient calculation mode is enabled by default.
>>> paddle.is_grad_enabled()
True
>>> with paddle.set_grad_enabled(False):
... paddle.is_grad_enabled()
False
>>> paddle.enable_static()
>>> paddle.is_grad_enabled()
True
"""
return core._has_grad()
def _set_grad_enabled(mode: bool) -> None:
core._set_has_grad(mode)
class set_grad_enabled(_DecoratorContextManager):
"""
Create a context which enables or disables dygraph gradient calculation.
Args:
mode(bool): whether to enable (`True`), or disable (`False`) grad.
Returns:
None.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.to_tensor([1.0], stop_gradient=False)
>>> is_train = False
>>> with paddle.set_grad_enabled(is_train):
... y = x * 2
>>> print(y.stop_gradient)
True
>>> paddle.set_grad_enabled(True)
>>> y = x * 2
>>> print(y.stop_gradient)
False
>>> paddle.set_grad_enabled(False)
>>> y = x * 2
>>> print(y.stop_gradient)
True
"""
def __init__(self, mode) -> None:
self.prev = is_grad_enabled()
self.mode = mode
_set_grad_enabled(mode)
def __call__(
self, func: Callable[_InputT, _RetT]
) -> Callable[_InputT, _RetT]:
_set_grad_enabled(self.prev)
return super().__call__(func)
def __enter__(self) -> None:
_set_grad_enabled(self.mode)
def __exit__(self, *args: object) -> None:
_set_grad_enabled(self.prev)
def clone(self) -> Self:
return self.__class__(self.mode)
class no_grad_(_DecoratorContextManager):
"""
:api_attr: imperative
Create a context which disables dygraph gradient calculation.
In this mode, the result of every computation will have `stop_gradient` set
to `True`.
Also functions as a decorator. (Make sure to use an instance.)
Examples:
.. code-block:: pycon
>>> import numpy as np
>>> import paddle
>>> # use as generator
>>> data = np.array([[2, 3], [4, 5]]).astype('float32')
>>> l0 = paddle.nn.Linear(2, 2) # l0.weight.gradient() is None
>>> l1 = paddle.nn.Linear(2, 2)
>>> with paddle.no_grad():
... # l1.weight.stop_gradient is False
... tmp = l1.weight * 2 # tmp.stop_gradient is True
>>> x = paddle.to_tensor(data)
>>> y = l0(x) + tmp
>>> o = l1(y)
>>> o.backward()
>>> print(tmp.grad is None)
True
>>> print(l0.weight.grad is None)
False
>>> # use as decorator
>>> @paddle.no_grad()
>>> def test_layer():
... inp = np.ones([3, 1024], dtype='float32')
... t = paddle.to_tensor(inp)
... linear1 = paddle.nn.Linear(1024, 4, bias_attr=False)
... linear2 = paddle.nn.Linear(4, 4)
... ret = linear1(t)
... dy_ret = linear2(ret)
>>> test_layer()
"""
def __enter__(self) -> None:
self.prev = is_grad_enabled()
_set_grad_enabled(False)
def __exit__(self, *args: object) -> None:
_set_grad_enabled(self.prev)
class enable_grad(_DecoratorContextManager):
"""
:api_attr: imperative
Create a context which enable dygraph gradient calculation,
if it has been disabled by `no_grad` or `set_grad_enabled`.
In this mode, the result of every computation will have `stop_gradient` set
to `False`.
Also functions as a decorator. (Make sure to use an instance.)
Examples:
.. code-block:: pycon
>>> import paddle
>>> # use as generator
>>> x = paddle.to_tensor([1.0], stop_gradient=False)
>>> with paddle.no_grad():
... with paddle.enable_grad():
... y = x * 2
>>> assert y.stop_gradient == False
>>> y.backward()
>>> assert x.grad is not None
>>> # use as decorator
>>> @paddle.enable_grad()
>>> def double(x):
... return x * 2
>>> with paddle.no_grad():
... z = double(x)
>>> assert z.stop_gradient == False
"""
def __enter__(self) -> None:
self.prev = is_grad_enabled()
_set_grad_enabled(True)
def __exit__(self, *args: object) -> None:
_set_grad_enabled(self.prev)
class inference_mode(_DecoratorContextManager):
"""
Context-manager/decorator that enables or disables inference mode.
In this mode, the result of every computation will have `stop_gradient` set
to `True`. When ``mode=False``, gradient calculation is enabled.
Also functions as a decorator.
"""
def __init__(self, mode=True) -> None:
self.mode = mode
def __new__(cls, mode=True):
if isinstance(mode, bool):
return super().__new__(cls)
return cls()(mode)
def __enter__(self) -> None:
self._inference_mode_context = set_grad_enabled(not self.mode)
self._inference_mode_context.__enter__()
def __exit__(self, *args: object) -> None:
self._inference_mode_context.__exit__(*args)
def clone(self) -> Self:
return self.__class__(self.mode)
@signature_safe_contextmanager
def guard(place: PlaceLike | None = None) -> Generator[None, None, None]:
"""
:api_attr: imperative
This context will create a dygraph context for dygraph to run, using python ``with`` statement.
Parameters:
place(base.CPUPlace| base.CUDAPlace|str, optional): Place to execute dygraph.
If None, the running place will be determined according to the way of paddle compilation.
If ``place`` is string, It can be ``cpu``, ``gpu:x`` and ``xpu:x``, where ``x`` is the
index of the GPUs or XPUs. Default: None
return:
None
Examples:
.. code-block:: pycon
>>> import numpy as np
>>> import paddle.base as base
>>> with base.dygraph.guard():
... inp = np.ones([3, 1024], dtype='float32')
... t = paddle.to_tensor(inp)
... linear1 = paddle.nn.Linear(1024, 4, bias_attr=False)
... linear2 = paddle.nn.Linear(4, 4)
... ret = linear1(t)
... dy_ret = linear2(ret)
"""
train = framework.Program()
startup = framework.Program()
tracer = Tracer()
if place is not None:
expected_place = _get_paddle_place(place)
else:
expected_place = framework._current_expected_place_()
with (
framework.program_guard(train, startup),
framework.unique_name.guard(),
framework._dygraph_guard(tracer),
framework._dygraph_place_guard(expected_place),
):
yield
@framework.non_static_only
def grad(
outputs: Tensor | Sequence[Tensor],
inputs: Tensor | Sequence[Tensor],
grad_outputs: Tensor | Sequence[Tensor | None] | None = None,
retain_graph: bool | None = None,
create_graph: bool = False,
only_inputs: bool = True,
allow_unused: bool = False,
no_grad_vars: Tensor | Sequence[Tensor] | set[Tensor] | None = None,
*,
dump_backward_graph_path: str | None = None,
) -> list[Tensor]:
'''
.. note::
**This API is ONLY available in imperative mode.**
This API computes the sum of gradients of `outputs` with respect to each `inputs` .
Parameters:
outputs (Tensor|list[Tensor]|tuple[Tensor]): the output Tensor or
Tensor list/tuple of the graph to compute gradients.
inputs (Tensor|list[Tensor]|tuple[Tensor]): the input Tensor or
Tensor list/tuple of the graph to compute gradients. The returned
values of this API are the gradients of `inputs` .
grad_outputs (Tensor|list[Tensor|None]|tuple[Tensor|None], optional):
initial gradient values of `outputs` . If `grad_outputs` is None,
the initial gradient values of `outputs` would be Tensors filled with 1;
if `grad_outputs` is not None, it must have the same length as `outputs` ,
and in this case, the initial gradient value of the i-th `outputs` would
be: (1) a Tensor filled with 1 when the i-th element of `grad_outputs`
is None; (2) the i-th element of `grad_outputs` when the i-th element of
`grad_outputs` is a Tensor. Default None.
retain_graph (bool|None, optional): whether to retain the forward graph which
is used to calculate the gradient. When it is True, the graph would
be retained, in which way users can calculate backward twice for the
same graph. When it is False, the graph would be freed. Default None,
which means it is equal to `create_graph` .
create_graph (bool, optional): whether to create the gradient graphs of
the computing process. When it is True, higher order derivatives are
supported to compute; when it is False, the gradient graphs of the
computing process would be discarded. Default False.
only_inputs (bool, optional): whether to only compute the gradients of
`inputs` . If it is False, the gradients of all remaining leaf
Tensors in the graph would be also computed and accumulated.
If it is True, only the gradients of `inputs` would be computed.
Default True. only_inputs=False is under development, and it is
not supported yet.
allow_unused (bool, optional): whether to raise error or return None if some
Tensors of `inputs` are unreachable in the graph. If some Tensors of
`inputs` are unreachable in the graph (i.e., their gradients are None),
error would be raised if allow_unused=False, or None would be returned as
their gradients if allow_unused=True. Default False.
no_grad_vars (Tensor|list[Tensor]|tuple[Tensor]|set[Tensor], optional):
the Tensors whose gradients are not needed to compute. Default None.
dump_backward_graph_path (str, optional): specifies the directory path for storing the debug file.
If this parameter is specified, the backward-related graph (in dot format)
and the debugging call stack information will be generated in this directory.
Returns:
list: a list of Tensors, whose length is the same as the Tensor number
inside `inputs`, and the i-th returned Tensor is the sum of gradients of
`outputs` with respect to the i-th `inputs`.
Examples:
.. code-block:: pycon
:name: code-example-1
>>> import paddle
>>> def test_dygraph_grad(create_graph):
... x = paddle.ones(shape=[1], dtype='float32')
... x.stop_gradient = False
... y = x * x
...
... # Since y = x * x, dx = 2 * x
... dx = paddle.grad(
... outputs=[y],
... inputs=[x],
... create_graph=create_graph,
... retain_graph=True,
... )[0]
...
... z = y + dx
...
... # If create_graph = False, the gradient of dx
... # would not be backpropagated. Therefore,
... # z = x * x + dx, and x.gradient() = 2 * x = 2.0
...
... # If create_graph = True, the gradient of dx
... # would be backpropagated. Therefore,
... # z = x * x + dx = x * x + 2 * x, and
... # x.gradient() = 2 * x + 2 = 4.0
...
... z.backward()
... return x.grad
>>> print(test_dygraph_grad(create_graph=False))
Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=False,
[2.])
>>> print(test_dygraph_grad(create_graph=True))
Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=False,
[4.])
.. code-block:: pycon
:name: code-example-2
>>> import paddle
>>> def test_dygraph_grad(grad_outputs=None):
... x = paddle.to_tensor(2.0)
... x.stop_gradient = False
...
... y1 = x * x
... y2 = x * 3
...
... # If grad_outputs=None, dy1 = [1], dy2 = [1].
... # If grad_outputs=[g1, g2], then:
... # - dy1 = [1] if g1 is None else g1
... # - dy2 = [1] if g2 is None else g2
...
... # Since y1 = x * x, dx = 2 * x * dy1.
... # Since y2 = x * 3, dx = 3 * dy2.
... # Therefore, the final result would be:
... # dx = 2 * x * dy1 + 3 * dy2 = 4 * dy1 + 3 * dy2.
...
... dx = paddle.grad(
... outputs=[y1, y2],
... inputs=[x],
... grad_outputs=grad_outputs,
... )[0]
...
... return dx.numpy()
>>> grad_value = paddle.to_tensor(4.0)
>>> # dy1 = [1], dy2 = [1]
>>> print(test_dygraph_grad(None))
7.0
>>> # dy1 = [1], dy2 = [4]
>>> print(test_dygraph_grad([None, grad_value]))
16.0
>>> # dy1 = [4], dy2 = [1]
>>> print(test_dygraph_grad([grad_value, None]))
19.0
>>> # dy1 = [3], dy2 = [4]
>>> grad_y1 = paddle.to_tensor(3.0)
>>> print(test_dygraph_grad([grad_y1, grad_value]))
24.0
'''
if in_to_static_mode():
# In dy2static context, we call static interface `gradients`
# to calculate grads.
from paddle.static import gradients
to_static_unsupported_argument_warning(
"paddle.grad",
["retain_graph", "create_graph", "only_inputs", "allow_unused"],
[retain_graph, create_graph, only_inputs, allow_unused],
[None, False, True, False],
)
return gradients(outputs, inputs, grad_outputs, no_grad_vars)
def check_in_out(in_out_list, name):
assert in_out_list is not None, f"{name} should not be None"
if isinstance(in_out_list, (list, tuple)):
assert len(in_out_list) > 0, f"{name} cannot be empty"
for each_var in in_out_list:
assert isinstance(each_var, core.eager.Tensor), (
f"Elements of {name} must be Tensor"
)
return in_out_list
else:
assert isinstance(in_out_list, core.eager.Tensor), (
f"{name} must be Tensor or list of Tensor"
)
return [in_out_list]
outputs = check_in_out(outputs, 'outputs')
inputs = check_in_out(inputs, 'inputs')
if grad_outputs is not None:
if not isinstance(grad_outputs, (list, tuple)):
grad_outputs = [grad_outputs]
for each_var in grad_outputs:
if each_var is not None:
assert isinstance(each_var, core.eager.Tensor), (
"grad_outputs must be None, a Variable or a list containing None or Variables"
)
else:
grad_outputs = []
if len(grad_outputs) > 0:
assert len(grad_outputs) == len(outputs), (
"The length of grad_outputs must be equal to outputs"
)
if no_grad_vars is None:
no_grad_vars = []
elif isinstance(no_grad_vars, core.eager.Tensor):
no_grad_vars = [no_grad_vars]
elif isinstance(no_grad_vars, (list, tuple, set)):
no_grad_vars = list(no_grad_vars)
for var in no_grad_vars:
assert isinstance(var, core.eager.Tensor), (
"no_grad_vars can only contains Tensor"
)
else:
raise AssertionError(
"no_grad_vars must be None, Tensor or list/tuple/set of Tensors"
)
assert isinstance(create_graph, bool), "create_graph must be True or False"
if retain_graph is None:
retain_graph = create_graph
assert isinstance(retain_graph, bool), (
"retain_graph must be None, True or False"
)
assert isinstance(allow_unused, bool), "allow_unused must be True or False"
assert isinstance(only_inputs, bool), "only_inputs must be True or False"
assert only_inputs, "only_inputs=False is not supported yet"
check_and_create_dir(dump_backward_graph_path)
return core.eager.run_partial_grad(
outputs,
inputs,
grad_outputs,
retain_graph,
create_graph,
only_inputs,
allow_unused,
no_grad_vars,
dump_backward_graph_path,
)
python/paddle/base/dygraph/math_op_patch.py
+852
View File
@@ -0,0 +1,852 @@
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
import numpy as np
import paddle
from paddle import _C_ops
from paddle.utils.decorator_utils import (
size_args_decorator_patch,
)
from .. import core
from ..framework import convert_nptype_to_datatype_or_vartype
if TYPE_CHECKING:
from typing import Any
from numpy.typing import NDArray
from paddle import Tensor
from paddle._typing import (
DTypeLike,
NestedNumericSequence,
PlaceLike,
ShapeLike,
TensorLike,
)
_supported_int_dtype_ = [
core.VarDesc.VarType.UINT8,
core.VarDesc.VarType.INT8,
core.VarDesc.VarType.INT16,
core.VarDesc.VarType.INT32,
core.VarDesc.VarType.INT64,
core.VarDesc.VarType.BOOL,
]
# NOTE(chenweihang): We currently do not fully support the type promotion
# between tensors. Parting support here is because the interoperation of
# real and complex numbers in paddle quantum is very frequent, such as the
# binary operation between `float` and `complex64`, so we must support the
# correct type promotion on the APIs paddle quantum used.
# Now only check in dygraph (paddle quantum based dygraph)
# Full type promotion support will need to be fully verified later.
_supported_promote_complex_types_ = [
'__add__',
'__radd__',
'__sub__',
'__rsub__',
'__mul__',
'__rmul__',
'__div__',
'__truediv__',
'__rdiv__',
'__rtruediv__',
'__matmul__',
]
_complex_dtypes = [
core.VarDesc.VarType.COMPLEX64,
core.VarDesc.VarType.COMPLEX128,
]
_already_patch_eager_tensor = False
_supported_dtype_conversions = {
# float
'float16': 'float16',
'half': 'float16',
'bfloat16': 'bfloat16',
'float32': 'float32',
'float': 'float32',
'float64': 'float64',
'double': 'float64',
# int
'int8': 'int8',
'char': 'int8',
# We handle uint8 conversion separately
# 'uint8': 'uint8',
# 'byte': 'uint8',
'int16': 'int16',
'short': 'int16',
'int32': 'int32',
'int': 'int32',
'int64': 'int64',
'long': 'int64',
# other
'bool': 'bool',
'complex64': 'complex64',
'complex128': 'complex128',
'cfloat': 'complex64',
'cdouble': 'complex128',
}
def _rebuild_tensor(
data: NDArray[Any],
dtype: DTypeLike,
device: PlaceLike,
requires_grad,
) -> Tensor:
return paddle.tensor(
data,
dtype,
device,
requires_grad,
)
class TensorSize(int):
as_shape: list[int]
def __new__(cls, shape):
instance = super().__new__(cls, int(np.prod(shape)))
instance.as_shape = shape
return instance
def __call__(self, dim=None):
shape = paddle.Size(self.as_shape)
if dim is None:
return shape
return shape[dim]
def monkey_patch_math_tensor():
"""
Similar to monkey_patch_variable.
The difference is, in dygraph mode, use auto-generated op functions for better performance.
"""
global paddle
def astype(self: Tensor, dtype: DTypeLike) -> Tensor:
"""
Cast a Tensor to a specified data type if it differs from the current dtype;
otherwise, return the original Tensor.
Args:
dtype: The target data type.
Returns:
Tensor: a new Tensor with target dtype
Examples:
.. code-block:: pycon
>>> import paddle
>>> import numpy as np
>>> original_tensor = paddle.ones([2, 2])
>>> print("original tensor's dtype is: {}".format(original_tensor.dtype))
original tensor's dtype is: paddle.float32
>>> new_tensor = original_tensor.astype('float32')
>>> print("new tensor's dtype is: {}".format(new_tensor.dtype))
new tensor's dtype is: paddle.float32
"""
if not isinstance(dtype, (core.VarDesc.VarType, core.DataType)):
dtype = convert_nptype_to_datatype_or_vartype(dtype)
if self.dtype == dtype:
return self
return _C_ops.cast(self, dtype)
def byte(self: Tensor) -> Tensor:
# since paddle don't support float to uint8, so we need to convert it to int8 first
if self.is_floating_point():
tensor = astype(self, 'int8')
return astype(tensor, 'uint8')
elif self.is_complex():
real = astype(self.real(), 'int8')
logging.warning(
"Casting complex values to real discards the imaginary part"
)
return astype(real, 'uint8')
else:
return astype(self, 'uint8')
def _create_dtype_conversion_methods():
"""
Batch create all data type conversion methods
"""
methods = []
for method_name, target_dtype in _supported_dtype_conversions.items():
def make_conversion_method(dtype):
def conversion_method(self: Tensor) -> Tensor:
return astype(self, dtype)
return conversion_method
method_impl = make_conversion_method(target_dtype)
method_impl.__name__ = method_name
method_impl.__doc__ = f"""
Cast a Tensor to {target_dtype} data type if it differs from the current dtype;
otherwise, return the original Tensor.
Returns:
Tensor: a new Tensor with {target_dtype} dtype
"""
methods.append((method_name, method_impl))
return methods
def type_as(self: Tensor, other: Tensor) -> Tensor:
return self.astype(other.dtype)
def _scalar_elementwise_op_(
var: Tensor, scale: float, bias: float
) -> Tensor:
return _C_ops.scale(var, float(scale), bias, True)
def _neg_(var: Tensor) -> Tensor:
return _scalar_elementwise_op_(var, -1.0, 0.0)
def _abs_(var: Tensor) -> Tensor:
return var.abs()
def _complex_(var: Tensor) -> complex:
numel = np.prod(var.shape, dtype="int64")
assert numel == 1, (
"only one element variable can be converted to complex."
)
assert var._is_initialized(), "variable's tensor is not initialized"
if not var.is_complex():
var = var.astype('complex64')
return complex(var.item())
def _float_(var: Tensor) -> float:
numel = np.prod(var.shape, dtype="int64")
assert numel == 1, (
"only one element variable can be converted to float."
)
assert var._is_initialized(), "variable's tensor is not initialized"
if (
var.dtype == core.VarDesc.VarType.BF16
or var.dtype == core.DataType.BFLOAT16
):
var = var.astype('float32')
return float(var.item())
def _int_(var: Tensor) -> int:
numel = np.prod(var.shape, dtype="int64")
assert numel == 1, "only one element variable can be converted to int."
assert var._is_initialized(), "variable's tensor is not initialized"
if (
var.dtype == core.VarDesc.VarType.BF16
or var.dtype == core.DataType.BFLOAT16
):
var = var.astype('float32')
return int(var.item())
def _len_(var: Tensor) -> int:
assert var.ndim > 0, "len() of a 0-D tensor is wrong"
if var.type == core.VarDesc.VarType.VOCAB:
return len(var.value().get_map_tensor())
elif var.type == core.VarDesc.VarType.STRINGS:
return len(var.value().get_string_tensor())
else:
return var.shape[0]
def _index_(var: Tensor) -> int:
numel = np.prod(var.shape, dtype="int64")
assert numel == 1, (
"only one element variable can be converted to python index."
)
assert var._is_initialized(), "variable's tensor is not initialized"
if (
var.dtype == core.VarDesc.VarType.BF16
or var.dtype == core.DataType.BFLOAT16
):
var = var.astype('float32')
return int(var.item())
@property
def _ndim(var: Tensor) -> int:
return len(var.shape)
def ndimension(var: Tensor) -> int:
return len(var.shape)
def dim(var: Tensor) -> int:
return len(var.shape)
@property
def _size_(var: Tensor) -> int:
return TensorSize(var.shape)
def nelement(var: Tensor) -> int:
"""
Returns the number of elements for current Tensor. Alias for attribute ``size``.
Returns:
int: the number of elements for current Tensor
"""
return int(np.prod(var.shape))
@property
def _T_(var: Tensor) -> Tensor:
if len(var.shape) == 1:
return var
perm = list(reversed(range(len(var.shape))))
out = _C_ops.transpose(var, perm)
return out
@property
def _mT_(var: Tensor) -> Tensor:
"""
Return the last two dimensions of a Tensor transposed.
Args:
var (Tensor): The input Tensor, which must have at least 2 dimensions.
Returns:
Tensor: A new Tensor with its last two dimensions swapped.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.randn([2, 3, 4])
>>> x_transposed = x.mT
>>> x_transposed.shape
paddle.Size([2, 4, 3])
"""
if len(var.shape) < 2:
raise ValueError(
f"Tensor.ndim({var.ndim}) is required to be greater than or equal to 2."
)
perm = list(range(len(var.shape)))
perm[-1], perm[-2] = perm[-2], perm[-1]
out = _C_ops.transpose(var, perm)
return out
@property
def _mH_(var: Tensor) -> Tensor:
"""
Return the conjugate transpose of the last two dimensions of a Tensor.
Accessing this property is equivalent to calling x.mT.conj().
Args:
var (Tensor): The input Tensor, which must be at least 2-D or 0-D.
Returns:
Tensor: A new Tensor with its last two dimensions swapped and
the elements conjugated. If the input is 0-D, returns the
Tensor itself.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.to_tensor([[1.0 + 1.0j, 2.0 + 2.0j], [3.0 + 3.0j, 4.0 + 4.0j]])
>>> x_mH = x.mH
>>> print(x_mH)
Tensor(shape=[2, 2], dtype=complex64, place=Place(cpu), stop_gradient=True,
[[(1-1j), (3-3j)],
[(2-2j), (4-4j)]])
>>> x_0d = paddle.to_tensor(1.0 + 1.0j)
>>> x_0d_mH = x_0d.mH
>>> print(x_0d_mH)
Tensor(shape=[], dtype=complex64, place=Place(cpu), stop_gradient=True,
(1+1j))
"""
if len(var.shape) == 0:
return _C_ops.conj(var)
if len(var.shape) < 2:
raise ValueError(
f"Tensor.ndim({var.ndim}) is required to be greater than or equal to 2 "
f"or 0-D."
)
perm = list(range(len(var.shape)))
perm[-1], perm[-2] = perm[-2], perm[-1]
out = _C_ops.transpose(var, perm)
out = _C_ops.conj(out)
return out
@property
def _H_(var: Tensor) -> Tensor:
"""
Return the conjugate transpose of a Tensor.
The conjugate transpose of a 2-D Tensor is equivalent to transposing the
Tensor and then taking the conjugate of each element (i.e., x.T.conj()).
For 0-D Tensor, returns the conjugated Tensor.
Args:
var (Tensor): The input Tensor, which must be 0-D or 2-D.
Returns:
Tensor: A new Tensor with its dimensions transposed and elements conjugated.
If the input is 0-D, returns the conjugated Tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.to_tensor([[1.0 + 1.0j, 2.0 + 2.0j], [3.0 + 3.0j, 4.0 + 4.0j]])
>>> x_H = x.H
>>> print(x_H)
Tensor(shape=[2, 2], dtype=complex64, place=Place(cpu), stop_gradient=True,
[[(1-1j), (3-3j)],
[(2-2j), (4-4j)]])
>>> x_0d = paddle.to_tensor(1.0 + 1.0j)
>>> x_0d_H = x_0d.H
>>> print(x_0d_H)
Tensor(shape=[], dtype=complex64, place=Place(cpu), stop_gradient=True,
(1+1j))
"""
if len(var.shape) == 0:
return _C_ops.conj(var)
if len(var.shape) != 2:
raise ValueError(
f"Only 0-D or 2-D tensors support .H (conjugate transpose), "
f"but got tensor with {len(var.shape)} dimension(s)."
)
out = _C_ops.transpose(var, [1, 0])
out = _C_ops.conj(out)
return out
def _new_full_(
var: Tensor,
size: ShapeLike,
fill_value: bool | float | paddle.Tensor,
*,
dtype: DTypeLike | None = None,
device: PlaceLike | None = None,
requires_grad: bool = False,
pin_memory: bool = False,
) -> Tensor:
"""
Create a new Tensor of specified shape and fill it with a given value.
Args:
var (Tensor): A reference Tensor for default dtype and device.
size (ShapeLike): Shape of the new Tensor.
fill_value (bool | float | Tensor): Value to fill the Tensor with.
dtype (DTypeLike, optional): Desired data type of the new Tensor. Defaults to `var.dtype`.
device (PlaceLike, optional): Device on which to place the new Tensor. Defaults to `var.place`.
requires_grad (bool, optional): Whether to track gradients. Default: False.
pin_memory (bool, optional): Whether to pin memory. Default: False.
Returns:
Tensor: A new Tensor filled with `fill_value`.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.ones([2, 2])
>>> y = x.new_full([3, 3], 5.0)
>>> y.numpy()
array([[5., 5., 5.],
[5., 5., 5.],
[5., 5., 5.]], dtype=float32)
"""
if dtype is None:
dtype = var.dtype
if device is None:
device = var.place
return paddle.full(
size,
fill_value,
dtype=dtype,
device=device,
requires_grad=requires_grad,
pin_memory=pin_memory,
)
def _new_tensor_(
var: Tensor,
data: TensorLike | NestedNumericSequence,
dtype: DTypeLike | None = None,
device: PlaceLike | None = None,
requires_grad: bool = False,
) -> Tensor:
"""
Creates a new tensor from ``data`` with the same device and dtype as this tensor.
Args:
var (Tensor): A reference Tensor for default dtype and device.
data: Data for the new tensor. Can be a list, numpy array, or Tensor.
dtype (DTypeLike|None, optional): Desired data type. If None, uses
the dtype of this tensor. Default: None.
device (PlaceLike|None, optional): Desired device. If None, uses
the place of this tensor. Default: None.
requires_grad (bool, optional): If True, gradient computation will
be enabled for the new tensor. Default: False.
Returns:
Tensor: A new tensor on the specified device.
"""
if dtype is None:
dtype = var.dtype
if device is None:
device = var.place
return paddle.to_tensor(
data, dtype=dtype, place=device, stop_gradient=not requires_grad
)
@size_args_decorator_patch
def _new_empty_(
var: Tensor,
size: ShapeLike,
*,
dtype: DTypeLike | None = None,
device: PlaceLike | None = None,
requires_grad: bool = False,
pin_memory: bool = False,
) -> Tensor:
"""
Create a new uninitialized Tensor of the specified shape.
Args:
var (Tensor): A reference Tensor for default dtype and device.
size (ShapeLike): Shape of the new Tensor.
dtype (DTypeLike, optional): Desired data type of the new Tensor. Defaults to `var.dtype`.
device (PlaceLike, optional): Device on which to place the new Tensor. Defaults to `var.place`.
requires_grad (bool, optional): Whether to track gradients. Default: False.
pin_memory (bool, optional): Whether to pin memory. Default: False.
Returns:
Tensor: A new uninitialized Tensor with the specified shape.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.ones([2, 2])
>>> y = x.new_empty(3, 3) # type: ignore
>>> y.shape
paddle.Size([3, 3])
"""
if dtype is None:
dtype = var.dtype
if device is None:
device = var.place
return paddle.empty(
size,
dtype,
device=device,
requires_grad=requires_grad,
pin_memory=pin_memory,
)
@size_args_decorator_patch
def _new_ones_(
var: Tensor,
size: ShapeLike,
*,
dtype: DTypeLike | None = None,
device: PlaceLike | None = None,
requires_grad: bool = False,
pin_memory: bool = False,
) -> Tensor:
"""
Create a new Tensor of the specified shape filled with ones.
Args:
var (Tensor): A reference Tensor for default dtype and device.
size (ShapeLike): Shape of the new Tensor.
dtype (DTypeLike, optional): Desired data type of the new Tensor. Defaults to `var.dtype`.
device (PlaceLike, optional): Device on which to place the new Tensor. Defaults to `var.place`.
requires_grad (bool, optional): Whether to track gradients. Default: False.
pin_memory (bool, optional): Whether to pin memory. Default: False.
Returns:
Tensor: A new Tensor filled with ones.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.zeros([2, 2])
>>> y = x.new_ones(3, 3) # type: ignore
>>> y.numpy()
array([[1., 1., 1.],
[1., 1., 1.],
[1., 1., 1.]], dtype=float32)
"""
if dtype is None:
dtype = var.dtype
if device is None:
device = var.place
return paddle.full(
size,
1,
dtype,
device=device,
requires_grad=requires_grad,
pin_memory=pin_memory,
)
@size_args_decorator_patch
def _new_zeros_(
var: Tensor,
size: ShapeLike,
*,
dtype: DTypeLike | None = None,
device: PlaceLike | None = None,
requires_grad: bool = False,
pin_memory: bool = False,
) -> Tensor:
"""
Create a new Tensor of the specified shape filled with zeros.
Args:
var (Tensor): A reference Tensor for default dtype and device.
size (ShapeLike): Shape of the new Tensor.
dtype (DTypeLike, optional): Desired data type of the new Tensor. Defaults to `var.dtype`.
device (PlaceLike, optional): Device on which to place the new Tensor. Defaults to `var.place`.
requires_grad (bool, optional): Whether to track gradients. Default: False.
pin_memory (bool, optional): Whether to pin memory. Default: False.
Returns:
Tensor: A new Tensor filled with zeros.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.ones([2, 2])
>>> y = x.new_zeros(3, 3) # type: ignore[misc, arg-type]
>>> y.numpy()
array([[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.]], dtype=float32)
"""
if dtype is None:
dtype = var.dtype
if device is None:
device = var.place
return paddle.full(
size,
0,
dtype,
device=device,
requires_grad=requires_grad,
pin_memory=pin_memory,
)
@property
def requires_grad(self: Tensor) -> bool:
"""
Whether this Tensor requires gradient computation.
This is a convenience property that returns the opposite of stop_gradient.
Setting requires_grad=True is equivalent to setting stop_gradient=False.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.randn([2, 3])
>>> print(x.requires_grad) # False by default
>>>
>>> x.requires_grad = False
>>> print(x.stop_gradient) # True
"""
return not self.stop_gradient
@requires_grad.setter
def requires_grad(self: Tensor, value: bool) -> None:
"""
Set whether this Tensor requires gradient computation.
Args:
value (bool): True to enable gradient computation, False to disable.
"""
if not isinstance(value, bool):
raise TypeError(
f"requires_grad must be bool, but got {type(value)}"
)
self.stop_gradient = not value
def requires_grad_(self, requires_grad: bool = True) -> Tensor:
"""
Set whether this Tensor requires gradient computation.
Args:
requires_grad (bool): True to enable gradient computation, False to disable.
"""
if not isinstance(requires_grad, bool):
raise TypeError(
f"requires_grad must be bool, but got {type(requires_grad)}"
)
self.stop_gradient = not requires_grad
return self
@property
def itemsize(self: Tensor) -> int:
"""
Returns the number of bytes allocated on the machine for a single element of the Tensor.
Examples:
.. code-block:: pycon
>>> import paddle
>>> x = paddle.randn((2, 3), dtype=paddle.float64)
>>> x.itemsize
8
"""
return self.element_size()
@property
def nbytes(self: Tensor) -> int:
"""
Returns the number of bytes allocated for elements of the dense Tensor. Defined to be ``size`` * ``element_size()``
"""
if self.is_sparse():
raise RuntimeError(
"nbytes is not defined for sparse tensors. "
"Add nbytes of indices and values for sparse storage size, "
"or multiply numel by element_size for the equivalent dense tensor."
)
return self.size * self.element_size()
def _reduce_ex_(self: Tensor, proto):
data_numpy = self.numpy()
place = str(self.place)[6:-1] # Place(gpu:1) -> gpu:1
dtype = str(self.dtype)[7:] # paddle.int32 -> int32
requires_grad = self.requires_grad
return _rebuild_tensor, (
data_numpy,
dtype,
place,
requires_grad,
)
eager_methods = [
('__neg__', _neg_),
('__abs__', _abs_),
('__complex__', _complex_),
('__float__', _float_),
('__int__', _int_),
('__len__', _len_),
('__index__', _index_),
('astype', astype),
('byte', byte),
('uint8', byte),
('type_as', type_as),
('dim', dim),
('ndimension', ndimension),
('ndim', _ndim),
('size', _size_),
('nelement', nelement),
('T', _T_),
('mT', _mT_),
('mH', _mH_),
('H', _H_),
('new_full', _new_full_),
('new_tensor', _new_tensor_),
('new_empty', _new_empty_),
('new_ones', _new_ones_),
('new_zeros', _new_zeros_),
("requires_grad", requires_grad),
("requires_grad_", requires_grad_),
# for logical compare
('__array_ufunc__', None),
('itemsize', itemsize),
('nbytes', nbytes),
('__reduce_ex__', _reduce_ex_),
]
dtype_conversion_methods = _create_dtype_conversion_methods()
eager_methods.extend(dtype_conversion_methods)
eager_cpp_level_patch = [
"__add__",
"__radd__",
'__sub__',
'__rsub__',
'__mul__',
'__rmul__',
'__div__',
'__truediv__',
'__rdiv__',
'__rtruediv__',
'__mod__',
'__rmod__',
'__matmul__',
'__rmatmul__',
'__gt__',
'__ge__',
'__lt__',
'__le__',
'__floordiv__',
'__rfloordiv__',
'__pow__',
'__rpow__',
'__eq__',
'__ne__',
]
global _already_patch_eager_tensor
local_already_patch = _already_patch_eager_tensor
_already_patch_eager_tensor = True
local_tensor = core.eager.Tensor
if not local_already_patch:
for method_name in eager_cpp_level_patch:
method_impl = getattr(local_tensor, method_name, None)
if method_impl:
setattr(local_tensor, method_name, method_impl)
for method in eager_methods:
method_name = method[0]
method_impl = method[1]
setattr(local_tensor, method_name, method_impl)
else:
import paddle.tensor
# Tensor method from module paddle.tensor
for method_name in paddle.tensor.tensor_method_func:
if hasattr(local_tensor, method_name):
continue
method_impl = getattr(paddle.tensor, method_name, None)
if method_impl:
setattr(local_tensor, method_name, method_impl)
for magic_method, origin_method in paddle.tensor.magic_method_func:
impl = getattr(paddle.tensor, origin_method, None)
if impl:
setattr(local_tensor, magic_method, impl)
python/paddle/base/dygraph/tensor_patch_methods.py
+1822
View File
File diff suppressed because it is too large Load Diff
python/paddle/base/dygraph/tracer.py
+329
View File
@@ -0,0 +1,329 @@
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle import _C_ops, _legacy_C_ops
from paddle.base import core, framework
name_mapping = {
"graph_send_recv": {
"final_op_name": "graph_send_recv",
"x": "X",
"src_index": "Src_index",
"dst_index": "Dst_index",
"out": "Out",
"dst_count": "Dst_count",
},
"matmul_v2": {
"final_op_name": "matmul",
"transpose_x": "trans_x",
"transpose_y": "trans_y",
"x": "X",
"y": "Y",
"out": "Out",
},
# "elementwise_add": {
# "final_op_name": "add",
# "x": "X",
# "y": "Y",
# },
"trunc": {
"final_op_name": "trunc",
"x": "X",
"out": "Out",
},
# "pool2d": {
# "final_op_name": "pool2d",
# "x": "X",
# "kernel_size": "ksize",
# "out": "Out",
# },
"abs": {
"final_op_name": "abs",
"x": "X",
"out": "Out",
},
"digamma": {
"final_op_name": "digamma",
"x": "X",
"out": "Out",
},
"diagonal": {
"final_op_name": "diagonal",
"x": "Input",
"offset": "offset",
"axis1": "axis1",
"axis2": "axis2",
"out": "Out",
},
"roi_align": {
"final_op_name": "roi_align",
"x": "X",
"boxes": "ROIs",
"boxes_num": "RoisNum",
"pooled_height": "pooled_height",
"pooled_width": "pooled_width",
"spatial_scale": "spatial_scale",
"sampling_ratio": "sampling_ratio",
"aligned": "aligned",
},
# "one_hot": {
# "final_op_name": "one_hot",
# "x": "X",
# "num_class": "depth",
# "out": "Out",
# }
}
core_ops_args_info = _legacy_C_ops.get_core_ops_args_info()
core_ops_args_type_info = _legacy_C_ops.get_core_ops_args_type_info()
core_ops_returns_info = _legacy_C_ops.get_core_ops_returns_info()
class Tracer(core.Tracer):
"""
:api_attr: imperative
Tracer is used to execute and record the operators executed, to construct the
computation graph in dygraph model. Tracer has two mode, :code:`train_mode`
and :code:`eval_mode`. In :code:`train_mode`, Tracer would add backward network
automatically and perform AutoGrad by method :code:`loss.backward()`.
In :code:`eval_mode`, Tracer would not add backward network.
This is a low level API, users don't need to use it directly.
"""
def __init__(self):
super().__init__()
self._train_mode = True
def eager_legacy_trace_op(
self,
op_type,
inputs,
outputs,
attrs,
stop_gradient=False,
inplace_map=None,
):
function_ptr = _legacy_C_ops.__dict__[op_type]
op_args = core_ops_args_info[op_type]
op_args_type = core_ops_args_type_info[op_type]
op_returns = core_ops_returns_info[op_type]
arg_list = []
for i in range(len(op_args)):
# initialized with None
arg_to_append = None
arg_name = op_args[i]
arg_type = op_args_type[i]
if arg_name in inputs.keys():
arg_to_append = inputs[arg_name]
elif arg_name in outputs.keys():
arg_to_append = outputs[arg_name]
else:
if "Num" in arg_name[-3:]:
# Remove "Num" suffix to get out_name
out_name = arg_name[:-3]
assert out_name in outputs.keys()
num_outs = len(outputs[out_name])
arg_to_append = num_outs
# NOTE(dev): For MasterParam/MasterParamOut in optimizer op
elif "Var" in arg_name[-3:]:
out_name = arg_name[:-3]
print(out_name)
if out_name in outputs.keys():
arg_to_append = outputs[out_name]
elif out_name in inputs.keys():
arg_to_append = inputs[out_name]
if arg_to_append is None:
arg_list.append(arg_to_append)
elif arg_type == "tensor":
if isinstance(arg_to_append, list):
arg_list.append(arg_to_append[0])
else:
arg_list.append(arg_to_append)
elif arg_type == "list":
assert isinstance(arg_to_append, list)
arg_list.append(arg_to_append)
else:
assert arg_type == "int"
assert isinstance(arg_to_append, int)
arg_list.append(arg_to_append)
attrs_list = []
for k, v in attrs.items():
attrs_list.append(k)
attrs_list.append(v)
returns = function_ptr(*arg_list, *attrs_list)
if op_type == 'load_combine':
assert len(outputs.keys()) == 1
key = next(iter(outputs.keys()))
for j in range(len(returns)):
returns[j]._share_underline_tensor_to(outputs[key][j])
return
if isinstance(returns, tuple):
for i in range(len(op_returns)):
retname = op_returns[i]
if retname in outputs.keys():
# Replaced outputs by function returns
if isinstance(returns[i], list):
for j in range(len(returns[i])):
outputs[retname][j].reconstruct_from_(
returns[i][j], False
)
else:
if isinstance(outputs[retname], list):
outputs[retname][0].reconstruct_from_(
returns[i], False
)
else:
outputs[retname].reconstruct_from_(
returns[i], False
)
elif isinstance(returns, list):
assert len(outputs.keys()) == 1
key = next(iter(outputs.keys()))
for j in range(len(returns)):
outputs[key][j].reconstruct_from_(returns[j], False)
else:
assert len(outputs.keys()) == 1
key = next(iter(outputs.keys()))
if isinstance(outputs[key], list):
outputs[key][0].reconstruct_from_(returns, False)
else:
outputs[key].reconstruct_from_(returns, False)
def eager_trace_op(
self,
op_type,
inputs,
outputs,
attrs,
stop_gradient=False,
inplace_map=None,
):
assert op_type in name_mapping.keys()
op_type = name_mapping[op_type]["final_op_name"]
function_ptr = _C_ops.__dict__[op_type]
core_ops_args_info = _C_ops.get_core_ops_args_info()
core_ops_args_type_info = _C_ops.get_core_ops_args_type_info()
core_ops_returns_info = _C_ops.get_core_ops_returns_info()
op_args = core_ops_args_info[op_type]
op_args_type = core_ops_args_type_info[op_type]
op_returns = core_ops_returns_info[op_type]
arg_list = []
for i in range(len(op_args)):
eager_arg_name = op_args[i]
arg_type = op_args_type[i]
assert eager_arg_name in name_mapping[op_type].keys()
arg_name = name_mapping[op_type][eager_arg_name]
if arg_name in inputs.keys():
arg_to_append = inputs[arg_name]
elif arg_name in outputs.keys():
arg_to_append = outputs[arg_name]
elif arg_name in attrs.keys() and arg_type == "":
arg_to_append = attrs[arg_name]
else:
# dispensable
arg_to_append = None
if arg_type == "":
# attribute
arg_list.append(arg_to_append)
elif arg_type == "tensor":
if isinstance(arg_to_append, list):
arg_list.append(arg_to_append[0])
else:
arg_list.append(arg_to_append)
elif arg_type == "list":
assert isinstance(arg_to_append, list)
arg_list.append(arg_to_append)
else:
assert arg_to_append is None
arg_list.append(arg_to_append)
returns = function_ptr(*arg_list)
if isinstance(returns, tuple):
for i in range(len(op_returns)):
eager_retname = op_returns[i]
assert eager_retname in name_mapping[op_type].keys()
retname = name_mapping[op_type][eager_retname]
if retname in outputs.keys():
# Replaced outputs by function returns
if isinstance(returns[i], list):
for j in range(len(returns[i])):
outputs[retname][j].reconstruct_from_(
returns[i][j], False
)
else:
outputs[retname][0].reconstruct_from_(returns[i], False)
elif isinstance(returns, list):
assert len(outputs.keys()) == 1
key = next(iter(outputs.keys()))
for j in range(len(returns)):
outputs[key][j].reconstruct_from_(returns[j], False)
else:
assert len(outputs.keys()) == 1
key = next(iter(outputs.keys()))
if isinstance(outputs[key], list):
outputs[key][0].reconstruct_from_(returns, False)
else:
outputs[key].reconstruct_from_(returns, False)
def trace_op(
self,
type,
inputs,
outputs,
attrs,
stop_gradient=False,
inplace_map=None,
):
if framework.in_dygraph_mode():
# inputs : {"sum": [tensor], ...}
# outputs : {"sum": [tensor], ...}
if type in name_mapping.keys():
type = name_mapping[type]["final_op_name"]
assert type in _legacy_C_ops.__dict__
self.eager_trace_op(
type, inputs, outputs, attrs, stop_gradient, inplace_map
)
else:
self.eager_legacy_trace_op(
type, inputs, outputs, attrs, stop_gradient, inplace_map
)
else:
raise ValueError("trace_op only work in dygraph mode")
def train_mode(self):
self._train_mode = True
def eval_mode(self):
self._train_mode = False