chore: import upstream snapshot with attribution

python/paddle/incubate/operators/__init__.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .graph_khop_sampler import graph_khop_sampler  # noqa: F401
+from .graph_reindex import graph_reindex  # noqa: F401
+from .graph_sample_neighbors import graph_sample_neighbors  # noqa: F401
+from .graph_send_recv import graph_send_recv  # noqa: F401
+from .resnet_unit import ResNetUnit  # noqa: F401
+from .softmax_mask_fuse import softmax_mask_fuse  # noqa: F401
+from .softmax_mask_fuse_upper_triangle import (  # noqa: F401
+    softmax_mask_fuse_upper_triangle,
+)

python/paddle/incubate/operators/graph_khop_sampler.py

@@ -0,0 +1,216 @@
+#   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal, overload
+
+from paddle import _C_ops
+from paddle.base.data_feeder import check_variable_and_dtype
+from paddle.base.layer_helper import LayerHelper
+from paddle.framework import in_dynamic_or_pir_mode
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+
+
+@overload
+def graph_khop_sampler(
+    row: Tensor,
+    colptr: Tensor,
+    input_nodes: Tensor,
+    sample_sizes: list[int] | tuple[int, ...],
+    sorted_eids: Tensor | None = ...,
+    return_eids: Literal[True] = ...,
+    name: str | None = None,
+) -> tuple[Tensor, Tensor, Tensor, Tensor]: ...
+
+
+@overload
+def graph_khop_sampler(
+    row: Tensor,
+    colptr: Tensor,
+    input_nodes: Tensor,
+    sample_sizes: list[int] | tuple[int, ...],
+    sorted_eids: Tensor | None = ...,
+    return_eids: Literal[False] = ...,
+    name: str | None = None,
+) -> tuple[Tensor, Tensor, Tensor]: ...
+
+
+@overload
+def graph_khop_sampler(
+    row: Tensor,
+    colptr: Tensor,
+    input_nodes: Tensor,
+    sample_sizes: list[int] | tuple[int, ...],
+    sorted_eids: Tensor | None = ...,
+    return_eids: bool = ...,
+    name: str | None = None,
+) -> tuple[Tensor, Tensor, Tensor, Tensor] | tuple[Tensor, Tensor, Tensor]: ...
+
+
+def graph_khop_sampler(
+    row,
+    colptr,
+    input_nodes,
+    sample_sizes,
+    sorted_eids=None,
+    return_eids=False,
+    name=None,
+):
+    """
+
+    Graph Khop Sampler API.
+
+    This API is mainly used in Graph Learning domain, and the main purpose is to
+    provide high performance graph khop sampling method with subgraph reindex step.
+    For example, we get the CSC(Compressed Sparse Column) format of the input graph
+    edges as `row` and `colptr`, so as to convert graph data into a suitable format
+    for sampling. And the `input_nodes` means the nodes we need to sample neighbors,
+    and `sample_sizes` means the number of neighbors and number of layers we want
+    to sample.
+
+    Args:
+        row (Tensor): One of the components of the CSC format of the input graph, and
+                      the shape should be [num_edges, 1] or [num_edges]. The available
+                      data type is int32, int64.
+        colptr (Tensor): One of the components of the CSC format of the input graph,
+                         and the shape should be [num_nodes + 1, 1] or [num_nodes].
+                         The data type should be the same with `row`.
+        input_nodes (Tensor): The input nodes we need to sample neighbors for, and the
+                              data type should be the same with `row`.
+        sample_sizes (list|tuple): The number of neighbors and number of layers we want
+                                   to sample. The data type should be int, and the shape
+                                   should only have one dimension.
+        sorted_eids (Tensor|None, optional): The sorted edge ids, should not be None when `return_eids`
+                              is True. The shape should be [num_edges, 1], and the data
+                              type should be the same with `row`. Default is None.
+        return_eids (bool, optional): Whether to return the id of the sample edges. Default is False.
+        name (str, optional): Name for the operation (optional, default is None).
+                              For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        - edge_src (Tensor), The src index of the output edges, also means the first column of
+          the edges. The shape is [num_sample_edges, 1] currently.
+        - edge_dst (Tensor), The dst index of the output edges, also means the second column
+          of the edges. The shape is [num_sample_edges, 1] currently.
+        - sample_index (Tensor), The original id of the input nodes and sampled neighbor nodes.
+        - reindex_nodes (Tensor), The reindex id of the input nodes.
+        - edge_eids (Tensor), Return the id of the sample edges if `return_eids` is True.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> # doctest: +SKIP('Number of sample edges mismatch, the sample kernel has error.')
+            >>> import paddle
+
+            >>> row = [3, 7, 0, 9, 1, 4, 2, 9, 3, 9, 1, 9, 7]
+            >>> colptr = [0, 2, 4, 5, 6, 7, 9, 11, 11, 13, 13]
+            >>> nodes = [0, 8, 1, 2]
+            >>> sample_sizes = [2, 2]
+            >>> row = paddle.to_tensor(row, dtype="int64")
+            >>> colptr = paddle.to_tensor(colptr, dtype="int64")
+            >>> nodes = paddle.to_tensor(nodes, dtype="int64")
+
+            >>> edge_src, edge_dst, sample_index, reindex_nodes = paddle.incubate.graph_khop_sampler(  # type: ignore[operator]
+            ...     row, colptr, nodes, sample_sizes, False
+            ... )
+
+    """
+
+    if in_dynamic_or_pir_mode():
+        if return_eids:
+            if sorted_eids is None:
+                raise ValueError(
+                    "`sorted_eid` should not be None if return_eids is True."
+                )
+            (
+                edge_src,
+                edge_dst,
+                sample_index,
+                reindex_nodes,
+                edge_eids,
+            ) = _C_ops.graph_khop_sampler(
+                row,
+                colptr,
+                input_nodes,
+                sorted_eids,
+                sample_sizes,
+                True,
+            )
+            return edge_src, edge_dst, sample_index, reindex_nodes, edge_eids
+        else:
+            (
+                edge_src,
+                edge_dst,
+                sample_index,
+                reindex_nodes,
+                _,
+            ) = _C_ops.graph_khop_sampler(
+                row,
+                colptr,
+                input_nodes,
+                None,
+                sample_sizes,
+                False,
+            )
+            return edge_src, edge_dst, sample_index, reindex_nodes
+
+    check_variable_and_dtype(
+        row, "Row", ("int32", "int64"), "graph_khop_sampler"
+    )
+
+    if return_eids:
+        if sorted_eids is None:
+            raise ValueError(
+                "`sorted_eid` should not be None if return_eids is True."
+            )
+        check_variable_and_dtype(
+            sorted_eids, "Eids", ("int32", "int64"), "graph_khop_sampler"
+        )
+
+    check_variable_and_dtype(
+        colptr, "Col_Ptr", ("int32", "int64"), "graph_khop_sampler"
+    )
+    check_variable_and_dtype(
+        input_nodes, "X", ("int32", "int64"), "graph_khop_sampler"
+    )
+
+    helper = LayerHelper("graph_khop_sampler", **locals())
+    edge_src = helper.create_variable_for_type_inference(dtype=row.dtype)
+    edge_dst = helper.create_variable_for_type_inference(dtype=row.dtype)
+    sample_index = helper.create_variable_for_type_inference(dtype=row.dtype)
+    reindex_nodes = helper.create_variable_for_type_inference(dtype=row.dtype)
+    edge_eids = helper.create_variable_for_type_inference(dtype=row.dtype)
+    helper.append_op(
+        type="graph_khop_sampler",
+        inputs={
+            "Row": row,
+            "Eids": sorted_eids,
+            "Col_Ptr": colptr,
+            "X": input_nodes,
+        },
+        outputs={
+            "Out_Src": edge_src,
+            "Out_Dst": edge_dst,
+            "Sample_Index": sample_index,
+            "Reindex_X": reindex_nodes,
+            "Out_Eids": edge_eids,
+        },
+        attrs={"sample_sizes": sample_sizes, "return_eids": return_eids},
+    )
+    if return_eids:
+        return edge_src, edge_dst, sample_index, reindex_nodes, edge_eids
+    else:
+        return edge_src, edge_dst, sample_index, reindex_nodes

python/paddle/incubate/operators/graph_reindex.py

@@ -0,0 +1,187 @@
+#   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from paddle import _C_ops
+from paddle.base.data_feeder import check_variable_and_dtype
+from paddle.base.layer_helper import LayerHelper
+from paddle.framework import in_dynamic_or_pir_mode
+from paddle.utils import deprecated
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+
+
+@deprecated(
+    since="2.4.0",
+    update_to="paddle.geometric.reindex_graph",
+    level=1,
+    reason="paddle.incubate.graph_reindex will be removed in future",
+)
+def graph_reindex(
+    x: Tensor,
+    neighbors: Tensor,
+    count: Tensor,
+    value_buffer: Tensor | None = None,
+    index_buffer: Tensor | None = None,
+    flag_buffer_hashtable: bool = False,
+    name: str | None = None,
+) -> tuple[Tensor, Tensor, Tensor]:
+    """
+
+    Graph Reindex API.
+
+    This API is mainly used in Graph Learning domain, which should be used
+    in conjunction with `graph_sample_neighbors` API. And the main purpose
+    is to reindex the ids information of the input nodes, and return the
+    corresponding graph edges after reindex.
+
+    Notes:
+        The number in x should be unique, otherwise it would cause potential errors.
+        Besides, we also support multi-edge-types neighbors reindexing. If we have different
+        edge_type neighbors for x, we should concatenate all the neighbors and count of x.
+        We will reindex all the nodes from 0.
+
+    Take input nodes x = [0, 1, 2] as an example.
+    If we have neighbors = [8, 9, 0, 4, 7, 6, 7], and count = [2, 3, 2],
+    then we know that the neighbors of 0 is [8, 9], the neighbors of 1
+    is [0, 4, 7], and the neighbors of 2 is [6, 7].
+
+    Args:
+        x (Tensor): The input nodes which we sample neighbors for. The available
+                    data type is int32, int64.
+        neighbors (Tensor): The neighbors of the input nodes `x`. The data type
+                            should be the same with `x`.
+        count (Tensor): The neighbor count of the input nodes `x`. And the
+                        data type should be int32.
+        value_buffer (Tensor, optional): Value buffer for hashtable. The data type should
+                                    be int32, and should be filled with -1. Default is None.
+        index_buffer (Tensor, optional): Index buffer for hashtable. The data type should
+                                    be int32, and should be filled with -1. Default is None.
+        flag_buffer_hashtable (bool, optional): Whether to use buffer for hashtable to speed up.
+                                      Default is False. Only useful for gpu version currently.
+        name (str, optional): Name for the operation (optional, default is None).
+                              For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        - reindex_src (Tensor), The source node index of graph edges after reindex.
+        - reindex_dst (Tensor), The destination node index of graph edges after reindex.
+        - out_nodes (Tensor), The index of unique input nodes and neighbors before reindex,
+          where we put the input nodes `x` in the front, and put neighbor
+          nodes in the back.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> x = [0, 1, 2]
+            >>> neighbors_e1 = [8, 9, 0, 4, 7, 6, 7]
+            >>> count_e1 = [2, 3, 2]
+            >>> x = paddle.to_tensor(x, dtype="int64")
+            >>> neighbors_e1 = paddle.to_tensor(neighbors_e1, dtype="int64")
+            >>> count_e1 = paddle.to_tensor(count_e1, dtype="int32")
+
+            >>> reindex_src, reindex_dst, out_nodes = paddle.incubate.graph_reindex(  # type: ignore[operator]
+            ...     x,
+            ...     neighbors_e1,
+            ...     count_e1,
+            ... )
+            >>> print(reindex_src)
+            Tensor(shape=[7], dtype=int64, place=Place(cpu), stop_gradient=True,
+            [3, 4, 0, 5, 6, 7, 6])
+            >>> print(reindex_dst)
+            Tensor(shape=[7], dtype=int64, place=Place(cpu), stop_gradient=True,
+            [0, 0, 1, 1, 1, 2, 2])
+            >>> print(out_nodes)
+            Tensor(shape=[8], dtype=int64, place=Place(cpu), stop_gradient=True,
+            [0, 1, 2, 8, 9, 4, 7, 6])
+
+            >>> neighbors_e2 = [0, 2, 3, 5, 1]
+            >>> count_e2 = [1, 3, 1]
+            >>> neighbors_e2 = paddle.to_tensor(neighbors_e2, dtype="int64")
+            >>> count_e2 = paddle.to_tensor(count_e2, dtype="int32")
+
+            >>> neighbors = paddle.concat([neighbors_e1, neighbors_e2])
+            >>> count = paddle.concat([count_e1, count_e2])
+            >>> reindex_src, reindex_dst, out_nodes = paddle.incubate.graph_reindex(  # type: ignore[operator]
+            ...     x,
+            ...     neighbors,
+            ...     count,
+            ... )
+            >>> print(reindex_src)
+            Tensor(shape=[12], dtype=int64, place=Place(cpu), stop_gradient=True,
+            [3, 4, 0, 5, 6, 7, 6, 0, 2, 8, 9, 1])
+            >>> print(reindex_dst)
+            Tensor(shape=[12], dtype=int64, place=Place(cpu), stop_gradient=True,
+            [0, 0, 1, 1, 1, 2, 2, 0, 1, 1, 1, 2])
+            >>> print(out_nodes)
+            Tensor(shape=[10], dtype=int64, place=Place(cpu), stop_gradient=True,
+            [0, 1, 2, 8, 9, 4, 7, 6, 3, 5])
+
+    """
+    if flag_buffer_hashtable:
+        if value_buffer is None or index_buffer is None:
+            raise ValueError(
+                "`value_buffer` and `index_buffer` should not"
+                "be None if `flag_buffer_hashtable` is True."
+            )
+
+    if in_dynamic_or_pir_mode():
+        reindex_src, reindex_dst, out_nodes = _C_ops.reindex_graph(
+            x,
+            neighbors,
+            count,
+            value_buffer,
+            index_buffer,
+        )
+        return reindex_src, reindex_dst, out_nodes
+
+    check_variable_and_dtype(x, "X", ("int32", "int64"), "graph_reindex")
+    check_variable_and_dtype(
+        neighbors, "Neighbors", ("int32", "int64"), "graph_reindex"
+    )
+    check_variable_and_dtype(count, "Count", ("int32"), "graph_reindex")
+
+    if flag_buffer_hashtable:
+        check_variable_and_dtype(
+            value_buffer, "HashTable_Value", ("int32"), "graph_reindex"
+        )
+        check_variable_and_dtype(
+            index_buffer, "HashTable_Index", ("int32"), "graph_reindex"
+        )
+
+    helper = LayerHelper("graph_reindex", **locals())
+    reindex_src = helper.create_variable_for_type_inference(dtype=x.dtype)
+    reindex_dst = helper.create_variable_for_type_inference(dtype=x.dtype)
+    out_nodes = helper.create_variable_for_type_inference(dtype=x.dtype)
+    helper.append_op(
+        type="graph_reindex",
+        inputs={
+            "X": x,
+            "Neighbors": neighbors,
+            "Count": count,
+            "HashTable_Value": value_buffer if flag_buffer_hashtable else None,
+            "HashTable_Index": index_buffer if flag_buffer_hashtable else None,
+        },
+        outputs={
+            "Reindex_Src": reindex_src,
+            "Reindex_Dst": reindex_dst,
+            "Out_Nodes": out_nodes,
+        },
+    )
+    return reindex_src, reindex_dst, out_nodes

python/paddle/incubate/operators/graph_sample_neighbors.py

@@ -0,0 +1,229 @@
+#   Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal, overload
+
+from paddle import _C_ops
+from paddle.base.data_feeder import check_variable_and_dtype
+from paddle.base.layer_helper import LayerHelper
+from paddle.framework import in_dynamic_or_pir_mode
+from paddle.utils import deprecated
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+
+
+@overload
+def graph_sample_neighbors(
+    row: Tensor,
+    colptr: Tensor,
+    input_nodes: Tensor,
+    eids: Tensor | None = ...,
+    perm_buffer: Tensor | None = ...,
+    sample_size: int = ...,
+    return_eids: Literal[True] = ...,
+    flag_perm_buffer: bool = ...,
+    name: str | None = ...,
+) -> tuple[Tensor, Tensor, Tensor]: ...
+
+
+@overload
+def graph_sample_neighbors(
+    row: Tensor,
+    colptr: Tensor,
+    input_nodes: Tensor,
+    eids: Tensor | None = ...,
+    perm_buffer: Tensor | None = ...,
+    sample_size: int = ...,
+    return_eids: Literal[False] = ...,
+    flag_perm_buffer: bool = ...,
+    name: str | None = ...,
+) -> tuple[Tensor, Tensor]: ...
+
+
+@overload
+def graph_sample_neighbors(
+    row: Tensor,
+    colptr: Tensor,
+    input_nodes: Tensor,
+    eids: Tensor | None = ...,
+    perm_buffer: Tensor | None = ...,
+    sample_size: int = ...,
+    return_eids: bool = ...,
+    flag_perm_buffer: bool = ...,
+    name: str | None = ...,
+) -> tuple[Tensor, Tensor] | tuple[Tensor, Tensor, Tensor]: ...
+
+
+@deprecated(
+    since="2.4.0",
+    update_to="paddle.geometric.sample_neighbors",
+    level=1,
+    reason="paddle.incubate.graph_sample_neighbors will be removed in future",
+)
+def graph_sample_neighbors(
+    row,
+    colptr,
+    input_nodes,
+    eids=None,
+    perm_buffer=None,
+    sample_size=-1,
+    return_eids=False,
+    flag_perm_buffer=False,
+    name=None,
+):
+    """
+
+    Graph Sample Neighbors API.
+
+    This API is mainly used in Graph Learning domain, and the main purpose is to
+    provide high performance of graph sampling method. For example, we get the
+    CSC(Compressed Sparse Column) format of the input graph edges as `row` and
+    `colptr`, so as to convert graph data into a suitable format for sampling.
+    `input_nodes` means the nodes we need to sample neighbors, and `sample_sizes`
+    means the number of neighbors and number of layers we want to sample.
+
+    Besides, we support fisher-yates sampling in GPU version.
+
+    Args:
+        row (Tensor): One of the components of the CSC format of the input graph, and
+                      the shape should be [num_edges, 1] or [num_edges]. The available
+                      data type is int32, int64.
+        colptr (Tensor): One of the components of the CSC format of the input graph,
+                         and the shape should be [num_nodes + 1, 1] or [num_nodes + 1].
+                         The data type should be the same with `row`.
+        input_nodes (Tensor): The input nodes we need to sample neighbors for, and the
+                              data type should be the same with `row`.
+        eids (Tensor): The eid information of the input graph. If return_eids is True,
+                            then `eids` should not be None. The data type should be the
+                            same with `row`. Default is None.
+        perm_buffer (Tensor): Permutation buffer for fisher-yates sampling. If `flag_perm_buffer`
+                              is True, then `perm_buffer` should not be None. The data type should
+                              be the same with `row`. Default is None.
+        sample_size (int): The number of neighbors we need to sample. Default value is
+                           -1, which means returning all the neighbors of the input nodes.
+        return_eids (bool): Whether to return eid information of sample edges. Default is False.
+        flag_perm_buffer (bool): Using the permutation for fisher-yates sampling in GPU. Default
+                                 value is false, means not using it.
+        name (str, optional): Name for the operation (optional, default is None).
+                              For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        - out_neighbors (Tensor): The sample neighbors of the input nodes.
+        - out_count (Tensor): The number of sampling neighbors of each input node, and the shape should be the same with `input_nodes`.
+        - out_eids (Tensor): If `return_eids` is True, we will return the eid information of the sample edges.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> # edges: (3, 0), (7, 0), (0, 1), (9, 1), (1, 2), (4, 3), (2, 4),
+            >>> #        (9, 5), (3, 5), (9, 6), (1, 6), (9, 8), (7, 8)
+            >>> row = [3, 7, 0, 9, 1, 4, 2, 9, 3, 9, 1, 9, 7]
+            >>> colptr = [0, 2, 4, 5, 6, 7, 9, 11, 11, 13, 13]
+            >>> nodes = [0, 8, 1, 2]
+            >>> sample_size = 2
+            >>> row = paddle.to_tensor(row, dtype="int64")
+            >>> colptr = paddle.to_tensor(colptr, dtype="int64")
+            >>> nodes = paddle.to_tensor(nodes, dtype="int64")
+            >>> out_neighbors, out_count = paddle.incubate.graph_sample_neighbors(
+            ...     row,
+            ...     colptr,
+            ...     nodes,
+            ...     sample_size=sample_size,
+            ... )  # type: ignore[operator]
+
+    """
+
+    if return_eids:
+        if eids is None:
+            raise ValueError(
+                "`eids` should not be None if `return_eids` is True."
+            )
+
+    if flag_perm_buffer:
+        if perm_buffer is None:
+            raise ValueError(
+                "`perm_buffer` should not be None if `flag_perm_buffer` is True."
+            )
+
+    if in_dynamic_or_pir_mode():
+        (
+            out_neighbors,
+            out_count,
+            out_eids,
+        ) = _C_ops.graph_sample_neighbors(
+            row,
+            colptr,
+            input_nodes,
+            eids,
+            perm_buffer,
+            sample_size,
+            return_eids,
+            flag_perm_buffer,
+        )
+        if return_eids:
+            return out_neighbors, out_count, out_eids
+        return out_neighbors, out_count
+
+    check_variable_and_dtype(
+        row, "Row", ("int32", "int64"), "graph_sample_neighbors"
+    )
+    check_variable_and_dtype(
+        colptr, "Col_Ptr", ("int32", "int64"), "graph_sample_neighbors"
+    )
+    check_variable_and_dtype(
+        input_nodes, "X", ("int32", "int64"), "graph_sample_neighbors"
+    )
+    if return_eids:
+        check_variable_and_dtype(
+            eids, "Eids", ("int32", "int64"), "graph_sample_neighbors"
+        )
+    if flag_perm_buffer:
+        check_variable_and_dtype(
+            perm_buffer,
+            "Perm_Buffer",
+            ("int32", "int64"),
+            "graph_sample_neighbors",
+        )
+
+    helper = LayerHelper("graph_sample_neighbors", **locals())
+    out_neighbors = helper.create_variable_for_type_inference(dtype=row.dtype)
+    out_count = helper.create_variable_for_type_inference(dtype=row.dtype)
+    out_eids = helper.create_variable_for_type_inference(dtype=row.dtype)
+    helper.append_op(
+        type="graph_sample_neighbors",
+        inputs={
+            "Row": row,
+            "Col_Ptr": colptr,
+            "X": input_nodes,
+            "Eids": eids if return_eids else None,
+            "Perm_Buffer": perm_buffer if flag_perm_buffer else None,
+        },
+        outputs={
+            "Out": out_neighbors,
+            "Out_Count": out_count,
+            "Out_Eids": out_eids,
+        },
+        attrs={
+            "sample_size": sample_size,
+            "return_eids": return_eids,
+            "flag_perm_buffer": flag_perm_buffer,
+        },
+    )
+    if return_eids:
+        return out_neighbors, out_count, out_eids
+    return out_neighbors, out_count

python/paddle/incubate/operators/graph_send_recv.py

@@ -0,0 +1,201 @@
+#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal
+
+import numpy as np
+
+from paddle import _C_ops
+from paddle.base.data_feeder import (
+    check_dtype,
+    check_type,
+    check_variable_and_dtype,
+)
+from paddle.base.framework import Variable
+from paddle.base.layer_helper import LayerHelper
+from paddle.framework import in_dynamic_or_pir_mode
+from paddle.geometric.message_passing.utils import (
+    convert_out_size_to_list,
+    get_out_size_tensor_inputs,
+)
+from paddle.utils import deprecated
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+
+
+@deprecated(
+    since="2.4.0",
+    update_to="paddle.geometric.send_u_recv",
+    level=1,
+    reason="graph_send_recv in paddle.incubate will be removed in future",
+)
+def graph_send_recv(
+    x: Tensor,
+    src_index: Tensor,
+    dst_index: Tensor,
+    pool_type: Literal["sum", "mean", "max", "min"] = "sum",
+    out_size: int | Tensor | None = None,
+    name: str | None = None,
+) -> Tensor:
+    """
+
+    Graph Learning Send_Recv combine operator.
+
+    This operator is mainly used in Graph Learning domain, and the main purpose is to reduce intermediate memory
+    consumption in the process of message passing. Take `x` as the input tensor, we first use `src_index`
+    to gather the corresponding data, and then use `dst_index` to update the corresponding position of output tensor
+    in different pooling types, like sum, mean, max, or min. Besides, we can set `out_size` to get necessary output shape.
+
+    .. code-block:: text
+
+           Given:
+
+           X = [[0, 2, 3],
+                [1, 4, 5],
+                [2, 6, 7]]
+
+           src_index = [0, 1, 2, 0]
+
+           dst_index = [1, 2, 1, 0]
+
+           pool_type = "sum"
+
+           out_size = None
+
+           Then:
+
+           Out = [[0, 2, 3],
+                  [2, 8, 10],
+                  [1, 4, 5]]
+
+    Args:
+        x (Tensor): The input tensor, and the available data type is float32, float64, int32, int64.
+        src_index (Tensor): An 1-D tensor, and the available data type is int32, int64.
+        dst_index (Tensor): An 1-D tensor, and should have the same shape as `src_index`.
+                            The available data type is int32, int64.
+        pool_type (str): The pooling types of graph_send_recv, including `sum`, `mean`, `max`, `min`.
+                         Default value is `sum`.
+        out_size (int|Tensor|None): We can set `out_size` to get necessary output shape. If not set or
+                                    out_size is smaller or equal to 0, then this input will not be used.
+                                    Otherwise, `out_size` should be equal with or larger than
+                                    max(dst_index) + 1.
+        name (str, optional): Name for the operation (optional, default is None).
+                              For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        out (Tensor): The output tensor, should have the same shape and same dtype as input tensor `x`.
+                      If `out_size` is set correctly, then it should have the same shape as `x` except
+                      the 0th dimension.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> x = paddle.to_tensor([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype="float32")
+            >>> indexes = paddle.to_tensor([[0, 1], [1, 2], [2, 1], [0, 0]], dtype="int32")
+            >>> src_index = indexes[:, 0]
+            >>> dst_index = indexes[:, 1]
+            >>> out = paddle.incubate.graph_send_recv(x, src_index, dst_index, pool_type="sum")  # type: ignore[operator]
+            >>> print(out)
+            Tensor(shape=[3, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[0. , 2. , 3. ],
+             [2. , 8. , 10.],
+             [1. , 4. , 5. ]])
+
+            >>> x = paddle.to_tensor([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype="float32")
+            >>> indexes = paddle.to_tensor([[0, 1], [2, 1], [0, 0]], dtype="int32")
+            >>> src_index = indexes[:, 0]
+            >>> dst_index = indexes[:, 1]
+            >>> out_size = paddle.max(dst_index) + 1
+            >>> out = paddle.incubate.graph_send_recv(x, src_index, dst_index, pool_type="sum", out_size=out_size)  # type: ignore[operator]
+            >>> print(out)
+            Tensor(shape=[2, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[0. , 2. , 3. ],
+             [2. , 8. , 10.]])
+
+            >>> x = paddle.to_tensor([[0, 2, 3], [1, 4, 5], [2, 6, 7]], dtype="float32")
+            >>> indexes = paddle.to_tensor([[0, 1], [2, 1], [0, 0]], dtype="int32")
+            >>> src_index = indexes[:, 0]
+            >>> dst_index = indexes[:, 1]
+            >>> out = paddle.incubate.graph_send_recv(x, src_index, dst_index, pool_type="sum")  # type: ignore[operator]
+            >>> print(out)
+            Tensor(shape=[3, 3], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[0. , 2. , 3. ],
+             [2. , 8. , 10.],
+             [0. , 0. , 0. ]])
+    """
+
+    if pool_type not in ["sum", "mean", "max", "min"]:
+        raise ValueError(
+            f"pool_type should be `sum`, `mean`, `max` or `min`, but received {pool_type}"
+        )
+
+    # TODO(daisiming): Should we add judgement for out_size: max(dst_index) + 1.
+    if in_dynamic_or_pir_mode():
+        out_size = convert_out_size_to_list(out_size, 'graph_send_recv')
+        return _C_ops.send_u_recv(
+            x, src_index, dst_index, pool_type.upper(), out_size
+        )
+    else:
+        check_variable_and_dtype(
+            x, "X", ("float32", "float64", "int32", "int64"), "graph_send_recv"
+        )
+        check_variable_and_dtype(
+            src_index, "Src_index", ("int32", "int64"), "graph_send_recv"
+        )
+        check_variable_and_dtype(
+            dst_index, "Dst_index", ("int32", "int64"), "graph_send_recv"
+        )
+        if out_size:
+            check_type(
+                out_size,
+                'out_size',
+                (int, np.int32, np.int64, Variable),
+                'graph_send_recv',
+            )
+        if isinstance(out_size, Variable):
+            check_dtype(
+                out_size.dtype,
+                'out_size',
+                ['int32', 'int64'],
+                'graph_send_recv',
+            )
+
+        helper = LayerHelper("graph_send_recv", **locals())
+        out = helper.create_variable_for_type_inference(dtype=x.dtype)
+        dst_count = helper.create_variable_for_type_inference(
+            dtype="int32", stop_gradient=True
+        )
+
+        inputs = {"X": x, "Src_index": src_index, "Dst_index": dst_index}
+        attrs = {"reduce_op": pool_type.upper()}
+        get_out_size_tensor_inputs(
+            inputs=inputs,
+            attrs=attrs,
+            out_size=out_size,
+            op_type='graph_send_recv',
+        )
+
+        helper.append_op(
+            type="graph_send_recv",
+            inputs=inputs,
+            outputs={"Out": out, "Dst_count": dst_count},
+            attrs=attrs,
+        )
+    return out

python/paddle/incubate/operators/resnet_unit.py

@@ -0,0 +1,368 @@
+#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+import paddle
+from paddle import base
+from paddle.base.layer_helper import LayerHelper
+from paddle.base.param_attr import ParamAttr
+from paddle.nn import (
+    Layer,
+    initializer as I,
+)
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+    from paddle._typing import DataLayout2D, ParamAttrLike
+
+
+def resnet_unit(
+    x: Tensor,
+    filter_x: Tensor,
+    scale_x: Tensor,
+    bias_x: Tensor,
+    mean_x: Tensor,
+    var_x: Tensor,
+    z: Tensor | None,
+    filter_z: Tensor | None,
+    scale_z: Tensor | None,
+    bias_z: Tensor | None,
+    mean_z: Tensor | None,
+    var_z: Tensor | None,
+    stride: int,
+    stride_z: int,
+    padding: int,
+    dilation: int,
+    groups: int,
+    momentum: float,
+    eps: float,
+    data_format: DataLayout2D,
+    fuse_add: bool,
+    has_shortcut: bool,
+    use_global_stats: bool,
+    is_test: bool,
+    act: str,
+) -> Tensor:
+    helper = LayerHelper('resnet_unit', **locals())
+    bn_param_dtype = base.core.VarDesc.VarType.FP32
+    bit_mask_dtype = base.core.VarDesc.VarType.INT32
+    out = helper.create_variable_for_type_inference(x.dtype)
+    bit_mask = helper.create_variable_for_type_inference(
+        dtype=bit_mask_dtype, stop_gradient=True
+    )
+    # intermediate_out for x
+    conv_x = helper.create_variable_for_type_inference(
+        dtype=x.dtype, stop_gradient=True
+    )
+    saved_mean_x = helper.create_variable_for_type_inference(
+        dtype=bn_param_dtype, stop_gradient=True
+    )
+    saved_invstd_x = helper.create_variable_for_type_inference(
+        dtype=bn_param_dtype, stop_gradient=True
+    )
+    running_mean_x = mean_x
+    running_var_x = var_x
+    # intermediate_out for z
+    conv_z = helper.create_variable_for_type_inference(
+        dtype=x.dtype, stop_gradient=True
+    )
+    saved_mean_z = helper.create_variable_for_type_inference(
+        dtype=bn_param_dtype, stop_gradient=True
+    )
+    saved_invstd_z = helper.create_variable_for_type_inference(
+        dtype=bn_param_dtype, stop_gradient=True
+    )
+    running_mean_z = (
+        helper.create_variable_for_type_inference(
+            dtype=bn_param_dtype, stop_gradient=True
+        )
+        if mean_z is None
+        else mean_z
+    )
+    running_var_z = (
+        helper.create_variable_for_type_inference(
+            dtype=bn_param_dtype, stop_gradient=True
+        )
+        if var_z is None
+        else var_z
+    )
+
+    inputs = {
+        'X': x,
+        'FilterX': filter_x,
+        'ScaleX': scale_x,
+        'BiasX': bias_x,
+        'MeanX': mean_x,
+        'VarX': var_x,
+        'Z': z,
+        'FilterZ': filter_z,
+        'ScaleZ': scale_z,
+        'BiasZ': bias_z,
+        'MeanZ': mean_z,
+        'VarZ': var_z,
+    }
+
+    attrs = {
+        'stride': stride,
+        'stride_z': stride_z,
+        'padding': padding,
+        'dilation': dilation,
+        'group': groups,
+        'momentum': momentum,
+        'epsilon': eps,
+        'data_format': data_format,
+        'fuse_add': fuse_add,
+        'has_shortcut': has_shortcut,
+        'use_global_stats': use_global_stats,
+        'is_test': is_test,
+        'act_type': act,
+    }
+
+    outputs = {
+        'Y': out,
+        'BitMask': bit_mask,
+        'ConvX': conv_x,
+        'SavedMeanX': saved_mean_x,
+        'SavedInvstdX': saved_invstd_x,
+        'RunningMeanX': running_mean_x,
+        'RunningVarX': running_var_x,
+        'ConvZ': conv_z,
+        'SavedMeanZ': saved_mean_z,
+        'SavedInvstdZ': saved_invstd_z,
+        'RunningMeanZ': running_mean_z,
+        'RunningVarZ': running_var_z,
+    }
+
+    helper.append_op(
+        type='resnet_unit', inputs=inputs, outputs=outputs, attrs=attrs
+    )
+
+    return out
+
+
+class ResNetUnit(Layer):
+    r"""
+    ******Temporary version******.
+    ResNetUnit is designed for optimize the performance by using cudnnv8 API.
+    """
+
+    def __init__(
+        self,
+        num_channels_x: int,
+        num_filters: int,
+        filter_size: int,
+        stride: int = 1,
+        momentum: float = 0.9,
+        eps: float = 1e-5,
+        data_format: DataLayout2D = 'NHWC',
+        act: str = 'relu',
+        fuse_add: bool = False,
+        has_shortcut: bool = False,
+        use_global_stats: bool = False,
+        is_test: bool = False,
+        filter_x_attr: ParamAttrLike | None = None,
+        scale_x_attr: ParamAttrLike | None = None,
+        bias_x_attr: ParamAttrLike | None = None,
+        moving_mean_x_name: str | None = None,
+        moving_var_x_name: str | None = None,
+        num_channels_z: int = 1,
+        stride_z: int = 1,
+        filter_z_attr: ParamAttrLike | None = None,
+        scale_z_attr: ParamAttrLike | None = None,
+        bias_z_attr: ParamAttrLike | None = None,
+        moving_mean_z_name: str | None = None,
+        moving_var_z_name: str | None = None,
+    ) -> None:
+        super().__init__()
+        self._stride = stride
+        self._stride_z = stride_z
+        self._dilation = 1
+        self._kernel_size = paddle.utils.convert_to_list(
+            filter_size, 2, 'kernel_size'
+        )
+        self._padding = (filter_size - 1) // 2
+        self._groups = 1
+        self._momentum = momentum
+        self._eps = eps
+        self._data_format = data_format
+        self._act = act
+        self._fuse_add = fuse_add
+        self._has_shortcut = has_shortcut
+        self._use_global_stats = use_global_stats
+        self._is_test = is_test
+
+        # check format
+        valid_format = {'NHWC', 'NCHW'}
+        if data_format not in valid_format:
+            raise ValueError(
+                f"conv_format must be one of {valid_format}, but got conv_format='{data_format}'"
+            )
+
+        def _get_default_param_initializer(channels):
+            filter_elem_num = np.prod(self._kernel_size) * channels
+            std = (2.0 / filter_elem_num) ** 0.5
+            return I.Normal(0.0, std)
+
+        is_nchw = data_format == 'NCHW'
+        # initial filter
+        bn_param_dtype = base.core.VarDesc.VarType.FP32
+        if not is_nchw:
+            bn_param_shape = [1, 1, 1, num_filters]
+            filter_x_shape = [
+                num_filters,
+                filter_size,
+                filter_size,
+                num_channels_x,
+            ]
+            filter_z_shape = [
+                num_filters,
+                filter_size,
+                filter_size,
+                num_channels_z,
+            ]
+        else:
+            bn_param_shape = [1, num_filters, 1, 1]
+            filter_x_shape = [
+                num_filters,
+                num_channels_x,
+                filter_size,
+                filter_size,
+            ]
+            filter_z_shape = [
+                num_filters,
+                num_channels_z,
+                filter_size,
+                filter_size,
+            ]
+
+        self.filter_x = self.create_parameter(
+            shape=filter_x_shape,
+            attr=filter_x_attr,
+            default_initializer=_get_default_param_initializer(num_channels_x),
+        )
+        self.scale_x = self.create_parameter(
+            shape=bn_param_shape,
+            attr=scale_x_attr,
+            dtype=bn_param_dtype,
+            default_initializer=I.Constant(1.0),
+        )
+        self.bias_x = self.create_parameter(
+            shape=bn_param_shape,
+            attr=bias_x_attr,
+            dtype=bn_param_dtype,
+            is_bias=True,
+        )
+        self.mean_x = self.create_parameter(
+            attr=ParamAttr(
+                name=moving_mean_x_name,
+                initializer=I.Constant(0.0),
+                trainable=False,
+            ),
+            shape=bn_param_shape,
+            dtype=bn_param_dtype,
+        )
+        self.mean_x.stop_gradient = True
+        self.var_x = self.create_parameter(
+            attr=ParamAttr(
+                name=moving_var_x_name,
+                initializer=I.Constant(1.0),
+                trainable=False,
+            ),
+            shape=bn_param_shape,
+            dtype=bn_param_dtype,
+        )
+        self.var_x.stop_gradient = True
+        if has_shortcut:
+            self.filter_z = self.create_parameter(
+                shape=filter_z_shape,
+                attr=filter_z_attr,
+                default_initializer=_get_default_param_initializer(
+                    num_channels_z
+                ),
+            )
+            self.scale_z = self.create_parameter(
+                shape=bn_param_shape,
+                attr=scale_z_attr,
+                dtype=bn_param_dtype,
+                default_initializer=I.Constant(1.0),
+            )
+            self.bias_z = self.create_parameter(
+                shape=bn_param_shape,
+                attr=bias_z_attr,
+                dtype=bn_param_dtype,
+                is_bias=True,
+            )
+            self.mean_z = self.create_parameter(
+                attr=ParamAttr(
+                    name=moving_mean_z_name,
+                    initializer=I.Constant(0.0),
+                    trainable=False,
+                ),
+                shape=bn_param_shape,
+                dtype=bn_param_dtype,
+            )
+            self.mean_z.stop_gradient = True
+            self.var_z = self.create_parameter(
+                attr=ParamAttr(
+                    name=moving_var_z_name,
+                    initializer=I.Constant(1.0),
+                    trainable=False,
+                ),
+                shape=bn_param_shape,
+                dtype=bn_param_dtype,
+            )
+            self.var_z.stop_gradient = True
+        else:
+            self.filter_z = None
+            self.scale_z = None
+            self.bias_z = None
+            self.mean_z = None
+            self.var_z = None
+
+    def forward(self, x: Tensor, z: Tensor | None = None) -> Tensor:
+        if self._fuse_add and z is None:
+            raise ValueError("z can not be None")
+
+        out = resnet_unit(
+            x,
+            self.filter_x,
+            self.scale_x,
+            self.bias_x,
+            self.mean_x,
+            self.var_x,
+            z,
+            self.filter_z,
+            self.scale_z,
+            self.bias_z,
+            self.mean_z,
+            self.var_z,
+            self._stride,
+            self._stride_z,
+            self._padding,
+            self._dilation,
+            self._groups,
+            self._momentum,
+            self._eps,
+            self._data_format,
+            self._fuse_add,
+            self._has_shortcut,
+            self._use_global_stats,
+            self._is_test,
+            self._act,
+        )
+        return out

python/paddle/incubate/operators/softmax_mask_fuse.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import paddle
+from paddle import _C_ops
+from paddle.base.layer_helper import LayerHelper
+from paddle.framework import in_dynamic_or_pir_mode
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+
+
+def softmax_mask_fuse(
+    x: Tensor, mask: Tensor, name: str | None = None
+) -> Tensor:
+    """
+    Do a masked softmax on x.
+
+    This is designed for speeding up Transformer structure.
+    Used for reducing operation such as: tmp = x + mask, out = softmax(tmp).
+    The equation is:
+
+    .. math::
+        out = softmax(x + mask)
+
+    Note:
+        This API only supports GPU.
+
+    Args:
+        x (4-D Tensor): The input tensor, should be in 4D shape, it's data type should be float16, float32.
+                        The fourth dimension of x must be larger or equal to 32 and less then 8192.
+        mask (4-D Tensor): The input tensor, should be in 4D shape, it's data type should be float16, float32.
+                           The second dimension of mask must be 1, and other dimensions must be same with x.
+        name (str, optional): Name for the operation (optional, default is None).
+                              For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        4-D Tensor. A location into which the result is stored. It's dimension is 4D. Has same shape with x.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> # doctest: +REQUIRES(env:GPU)
+            >>> import paddle
+            >>> import paddle.incubate as incubate
+
+            >>> x = paddle.rand([2, 8, 8, 32])
+            >>> mask = paddle.rand([2, 1, 8, 32])
+
+            >>> rst = incubate.softmax_mask_fuse(x, mask)  # type: ignore[operator]
+            >>> rst.shape
+            paddle.Size([2, 8, 8, 32])
+    """
+    if in_dynamic_or_pir_mode():
+        if isinstance(mask, (paddle.Tensor)) and mask.size == 0:
+            return x + mask
+        out = _C_ops.fused_softmax_mask(x, mask)
+        return out
+    helper = LayerHelper('fused_softmax_mask', **locals())
+    out = helper.create_variable_for_type_inference(dtype=x.dtype)
+    helper.append_op(
+        type='fused_softmax_mask',
+        inputs={'X': [x], 'Mask': [mask]},
+        outputs={'Out': [out]},
+    )
+    return out

python/paddle/incubate/operators/softmax_mask_fuse_upper_triangle.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from paddle import _C_ops
+from paddle.base.layer_helper import LayerHelper
+from paddle.framework import in_dynamic_or_pir_mode
+from paddle.utils.deprecated import deprecated
+
+if TYPE_CHECKING:
+    from paddle import Tensor
+
+
+@deprecated(
+    since="3.4.0",
+    level=1,
+    update_to="paddle.nn.functional.scaled_dot_product_attention",
+)
+def softmax_mask_fuse_upper_triangle(x: Tensor) -> Tensor:
+    """
+    Do a masked softmax on x, which will always mask upper triangle part of x.
+
+    This is designed for speeding up GPT kind Transformer structure.
+    Used for reducing operation such as: tmp = x + mask, out = softmax(tmp), where the mask is
+    always be an upper triangle matrix.
+    The equation is:
+
+    .. math::
+        out = softmax(LowerTriangular(x))
+
+    Note:
+        This API only supports GPU.
+
+    Args:
+        x (4-D Tensor): The input tensor, should be in 4D shape, it's data type should be float16, float32
+                        The fourth dimension of x must be larger or equal to 32 and less then 8192.
+                        The third dimension of x must be same with the fourth dimension of x.
+
+    Returns:
+        4-D Tensor. A location into which the result is stored. It's dimension is 4D. Has same dimension with x.
+
+    Examples:
+        .. code-block:: pycon
+
+            >>> # doctest: +REQUIRES(env:GPU)
+            >>> import paddle
+            >>> import paddle.incubate as incubate
+
+            >>> paddle.seed(1)
+            >>> paddle.set_device("gpu")
+            >>> x = paddle.rand((1, 1, 32, 32))
+
+            >>> rst = incubate.softmax_mask_fuse_upper_triangle(x)  # type: ignore[operator]
+            >>> print(rst)
+            Tensor(shape=[1, 1, 32, 32], dtype=float32, place=Place(gpu:0), stop_gradient=True,
+            [[[[1.        , 0.        , 0.        , ..., 0.        ,
+                0.        , 0.        ],
+               [0.49575609, 0.50424391, 0.        , ..., 0.        ,
+                0.        , 0.        ],
+               [0.26035303, 0.25114325, 0.48850375, ..., 0.        ,
+                0.        , 0.        ],
+                ...,
+               [0.04379999, 0.04194880, 0.05150032, ..., 0.02721255,
+                0.        , 0.        ],
+               [0.02348574, 0.01959674, 0.02609110, ..., 0.04046615,
+                0.02248267, 0.        ],
+               [0.02280738, 0.03144657, 0.02892209, ..., 0.03885521,
+                0.03342311, 0.02842640]]]])
+    """
+    if in_dynamic_or_pir_mode():
+        out = _C_ops.fused_softmax_mask_upper_triangle(x)
+        return out
+
+    helper = LayerHelper('fused_softmax_mask_upper_triangle', **locals())
+
+    out = helper.create_variable_for_type_inference(dtype=x.dtype)
+
+    helper.append_op(
+        type='fused_softmax_mask_upper_triangle',
+        inputs={'X': [x]},
+        outputs={'Out': [out]},
+    )
+    return out