chore: import upstream snapshot with attribution

test/auto_parallel/custom_op/CMakeLists.txt

@@ -0,0 +1,16 @@
+set(LOCAL_ALL_ARCH ON)
+set(LOCAL_ALL_PLAT ON)
+if(WITH_DISTRIBUTE
+   AND WITH_GPU
+   AND (LINUX))
+  py_test_modules(
+    test_semi_auto_parallel_custom_op
+    MODULES
+    test_semi_auto_parallel_custom_op
+    ENVS
+    "http_proxy=;https_proxy=;PYTHONPATH=../..:${PADDLE_BINARY_DIR}/python;PADDLE_SOURCE_DIR=${PROJECT_SOURCE_DIR};WITH_ONEDNN=${WITH_ONEDNN};ONEDNN_INSTALL_DIR=${ONEDNN_INSTALL_DIR};WITH_ONEDNN=${WITH_ONEDNN};WITH_GPU=${WITH_GPU};WITH_ROCM=${WITH_ROCM};externalError_INCLUDE_DIR=${externalError_INCLUDE_DIR};PYBIND_INCLUDE_DIR=${PYBIND_INCLUDE_DIR}"
+  )
+  set_tests_properties(test_semi_auto_parallel_custom_op
+                       PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE" TIMEOUT 120)
+
+endif()

test/auto_parallel/custom_op/custom_relu_op.cc

@@ -0,0 +1,138 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <vector>
+
+#include "paddle/extension.h"
+#include "paddle/phi/api/ext/spmd_infer.h"
+#include "paddle/phi/infermeta/spmd_rules/rules.h"
+
+#define CHECK_CPU_INPUT(x) \
+  PADDLE_ENFORCE_EQ(       \
+      x.is_cpu(), true, common::errors::Fatal(#x " must be a CPU Tensor."))
+
+template <typename data_t>
+void relu_cpu_forward_kernel(const data_t* x_data,
+                             data_t* out_data,
+                             int64_t x_numel) {
+  PADDLE_ENFORCE_NE(
+      x_data, nullptr, common::errors::Fatal("x_data is nullptr."));
+  PADDLE_ENFORCE_NE(
+      out_data, nullptr, common::errors::Fatal("out_data is nullptr."));
+  for (int64_t i = 0; i < x_numel; ++i) {
+    out_data[i] = std::max(static_cast<data_t>(0.), x_data[i]);
+  }
+}
+
+template <typename data_t>
+void relu_cpu_backward_kernel(const data_t* grad_out_data,
+                              const data_t* out_data,
+                              data_t* grad_x_data,
+                              int64_t out_numel) {
+  for (int64_t i = 0; i < out_numel; ++i) {
+    grad_x_data[i] =
+        grad_out_data[i] * (out_data[i] > static_cast<data_t>(0) ? 1. : 0.);
+  }
+}
+
+std::vector<paddle::Tensor> relu_cpu_forward(const paddle::Tensor& x) {
+  auto out = paddle::empty_like(x);
+
+  PD_DISPATCH_FLOATING_TYPES(
+      x.type(), "relu_cpu_forward", ([&] {
+        relu_cpu_forward_kernel<data_t>(
+            x.data<data_t>(), out.data<data_t>(), x.numel());
+      }));
+
+  return {out};
+}
+
+std::vector<paddle::Tensor> relu_cpu_backward(const paddle::Tensor& x,
+                                              const paddle::Tensor& out,
+                                              const paddle::Tensor& grad_out) {
+  auto grad_x = paddle::empty_like(x);
+
+  PD_DISPATCH_FLOATING_TYPES(out.type(), "relu_cpu_backward", ([&] {
+                               relu_cpu_backward_kernel<data_t>(
+                                   grad_out.data<data_t>(),
+                                   out.data<data_t>(),
+                                   grad_x.data<data_t>(),
+                                   out.size());
+                             }));
+
+  return {grad_x};
+}
+
+std::vector<paddle::Tensor> relu_cuda_forward(const paddle::Tensor& x);
+std::vector<paddle::Tensor> relu_cuda_backward(const paddle::Tensor& x,
+                                               const paddle::Tensor& out,
+                                               const paddle::Tensor& grad_out);
+
+std::vector<paddle::Tensor> ReluForward(const paddle::Tensor& x) {
+  if (x.is_cpu()) {
+    return relu_cpu_forward(x);
+  } else if (x.is_gpu()) {
+    return relu_cuda_forward(x);
+  } else {
+    PD_THROW("Not implemented.");
+  }
+}
+
+std::vector<paddle::Tensor> ReluBackward(const paddle::Tensor& x,
+                                         const paddle::Tensor& out,
+                                         const paddle::Tensor& grad_out) {
+  if (x.is_cpu()) {
+    return relu_cpu_backward(x, out, grad_out);
+  } else if (x.is_gpu()) {
+    return relu_cuda_backward(x, out, grad_out);
+  } else {
+    PD_THROW("Not implemented.");
+  }
+}
+
+phi::distributed::SpmdInfo ReluGradInferSpmd(
+    const phi::distributed::DistMetaTensor& x,
+    const phi::distributed::DistMetaTensor& out,
+    const phi::distributed::DistMetaTensor& out_grad) {
+  return phi::distributed::ElementwiseUnaryGradInferSpmd(x, out, out_grad);
+}
+
+PD_BUILD_OP(custom_relu)
+    .Inputs({"X"})
+    .Outputs({"Out"})
+    .SetKernelFn(PD_KERNEL(ReluForward))
+    .SetInferSpmdFn(
+        PD_INFER_SPMD_RULE(phi::distributed::ElementwiseUnaryInferSpmd));
+
+PD_BUILD_GRAD_OP(custom_relu)
+    .Inputs({"X", "Out", paddle::Grad("Out")})
+    .Outputs({paddle::Grad("X")})
+    .SetKernelFn(PD_KERNEL(ReluBackward))
+    .SetInferSpmdFn(PD_INFER_SPMD_RULE(ReluGradInferSpmd));
+
+PD_BUILD_OP(custom_relu_no_spmd)
+    .Inputs({"X"})
+    .Outputs({"Out"})
+    .SetKernelFn(PD_KERNEL(ReluForward));
+
+PD_BUILD_GRAD_OP(custom_relu_no_spmd)
+    .Inputs({"X", "Out", paddle::Grad("Out")})
+    .Outputs({paddle::Grad("X")})
+    .SetKernelFn(PD_KERNEL(ReluBackward));
+
+PD_REGISTER_SPMD_RULE(
+    custom_relu,
+    PD_INFER_SPMD(phi::distributed::ElementwiseUnaryInferSpmd),
+    PD_INFER_SPMD(phi::distributed::ElementwiseUnaryInferSpmdReverse));

test/auto_parallel/custom_op/custom_relu_op.cu

@@ -0,0 +1,93 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/extension.h"
+
+#define CHECK_GPU_INPUT(x)                                         \
+  PADDLE_ENFORCE_EQ(                                               \
+      x.is_gpu(),                                                  \
+      true,                                                        \
+      common::errors::InvalidArgument("Input tensor `x` must be a" \
+                                      "GPU Tensor."));
+
+template <typename data_t>
+__global__ void relu_cuda_forward_kernel(const data_t* x,
+                                         data_t* y,
+                                         int64_t num) {
+  int64_t gid = blockIdx.x * blockDim.x + threadIdx.x;
+  for (int64_t i = gid; i < num; i += blockDim.x * gridDim.x) {
+    y[i] = x[i] > static_cast<data_t>(0.) ? x[i] : static_cast<data_t>(0.);
+  }
+}
+
+template <typename data_t>
+__global__ void relu_cuda_backward_kernel(const data_t* dy,
+                                          const data_t* y,
+                                          data_t* dx,
+                                          int64_t num) {
+  int64_t gid = blockIdx.x * blockDim.x + threadIdx.x;
+  for (int64_t i = gid; i < num; i += blockDim.x * gridDim.x) {
+    dx[i] = dy[i] * (y[i] > static_cast<data_t>(0.) ? static_cast<data_t>(1.)
+                                                    : static_cast<data_t>(0.));
+  }
+}
+
+std::vector<paddle::Tensor> relu_cuda_forward(const paddle::Tensor& x) {
+  CHECK_GPU_INPUT(x);
+  auto out = paddle::empty_like(x);
+
+  PADDLE_ENFORCE_EQ(x.is_gpu(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "Input tensor `x` must be a GPU Tensor."));
+
+  int64_t numel = x.numel();
+  int64_t block = 512;
+  int64_t grid = (numel + block - 1) / block;
+  PD_DISPATCH_FLOATING_AND_HALF_TYPES(
+      x.type(), "relu_cuda_forward_kernel", ([&] {
+        relu_cuda_forward_kernel<data_t><<<grid, block, 0, x.stream()>>>(
+            x.data<data_t>(), out.data<data_t>(), numel);
+      }));
+
+  return {out};
+}
+
+std::vector<paddle::Tensor> relu_cuda_backward(const paddle::Tensor& x,
+                                               const paddle::Tensor& out,
+                                               const paddle::Tensor& grad_out) {
+  CHECK_GPU_INPUT(x);
+  CHECK_GPU_INPUT(out);
+  CHECK_GPU_INPUT(grad_out);
+  auto grad_x = paddle::empty_like(x);
+
+  PADDLE_ENFORCE_EQ(x.is_gpu(),
+                    true,
+                    common::errors::InvalidArgument(
+                        "Input tensor `x` must be a GPU Tensor."));
+
+  int64_t numel = out.numel();
+  int64_t block = 512;
+  int64_t grid = (numel + block - 1) / block;
+  PD_DISPATCH_FLOATING_AND_HALF_TYPES(
+      out.type(), "relu_cuda_backward_kernel", ([&] {
+        relu_cuda_backward_kernel<data_t><<<grid, block, 0, x.stream()>>>(
+            grad_out.data<data_t>(),
+            out.data<data_t>(),
+            grad_x.mutable_data<data_t>(x.place()),
+            numel);
+      }));
+
+  return {grad_x};
+}

test/auto_parallel/custom_op/custom_relu_setup.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from utils import extra_compile_args, paddle_includes
+
+from paddle.utils.cpp_extension import CUDAExtension, setup
+
+# Mac-CI don't support GPU
+Extension = CUDAExtension
+sources = ['custom_relu_op.cc', 'custom_relu_op.cu']
+
+setup(
+    name='custom_relu',
+    ext_modules=Extension(
+        sources=sources,
+        include_dirs=paddle_includes,
+        extra_compile_args=extra_compile_args,
+        verbose=True,
+    ),
+)

test/auto_parallel/custom_op/semi_auto_parallel_for_custom_op.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import unittest
+
+__dir__ = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
+
+from semi_auto_parallel_util import SemiAutoParallelTestBase
+
+import paddle
+import paddle.distributed as dist
+from paddle.distributed.auto_parallel.static.mix_to_dist_pass import (
+    apply_mix2dist_pass,
+)
+from paddle.framework import core
+
+import custom_relu  # pylint: disable=unused-import # isort:skip
+
+assert core.contains_spmd_rule("custom_relu")
+
+
+class TestCustomOpSemiAutoParallel(SemiAutoParallelTestBase):
+    def __init__(self):
+        super().__init__()
+        self._backend = os.getenv("backend")
+        self._seed = eval(os.getenv("seed"))
+
+    def check_placements(self, output, expected_placements):
+        assert output.placements == expected_placements, (
+            f"{output.placements}  vs {expected_placements}"
+        )
+
+    def test_custom_relu(self):
+        shapes = [16, 4, 4]
+        specs = ['x', None, None]
+        inputs, outputs = self.runfunc_and_check(
+            inputs_shape=shapes,
+            inputs_specs=specs,
+            op_func=custom_relu.custom_relu,
+            with_backward=True,
+        )
+        self.check_placements(outputs, [dist.Shard(0)])
+
+    def test_custom_relu_no_spmd(self):
+        shapes = [16, 4, 4]
+        specs = ['x', None, None]
+        inputs, outputs = self.runfunc_and_check(
+            inputs_shape=shapes,
+            inputs_specs=specs,
+            op_func=custom_relu.custom_relu_no_spmd,
+            with_backward=True,
+        )
+        self.check_placements(outputs, [dist.Replicate()])
+
+    def test_custom_relu_no_shard(self):
+        shapes = [16, 4, 4]
+        specs = [None, None, None]
+        inputs, outputs = self.runfunc_and_check(
+            inputs_shape=shapes,
+            inputs_specs=specs,
+            op_func=custom_relu.custom_relu,
+            with_backward=True,
+        )
+        self.check_placements(outputs, [dist.Replicate()])
+
+    def run_test_case(self):
+        if self._backend == "cpu":
+            paddle.set_device("cpu")
+        elif self._backend == "gpu":
+            paddle.set_device("gpu:" + str(dist.get_rank()))
+        else:
+            raise ValueError("Only support cpu or gpu backend.")
+        self.test_custom_relu_no_shard()
+        self.test_custom_relu()
+        self.test_custom_relu_no_spmd()
+
+
+class TestBuildFakeProgramWithCustomOp(unittest.TestCase):
+    def test_build_with_custom_relu(self):
+        shapes = [16, 4, 4]
+        paddle.enable_static()
+        with paddle.pir_utils.IrGuard():
+            main_program = paddle.base.Program()
+            with paddle.base.program_guard(main_program):
+                mesh = dist.ProcessMesh([0, 1], dim_names=['mp'])
+                input = paddle.static.data(
+                    name='input',
+                    shape=shapes,
+                )
+                dist_input = dist.shard_tensor(input, mesh, [dist.Shard(0)])
+                dist_out = custom_relu.custom_relu(dist_input)
+        apply_mix2dist_pass(main_program)
+
+        self.assertTrue(dist_out.is_dist_dense_tensor_type())
+        self.assertEqual(dist_out._local_shape, [16 // 2, 4, 4])
+        self.assertEqual(dist_out.dist_attr().dims_mapping, [0, -1, -1])
+        self.assertEqual(dist_out.dist_attr().process_mesh, mesh)
+        op_dist_attr = dist_out.get_defining_op().dist_attr
+        self.assertEqual(op_dist_attr.process_mesh, mesh)
+        self.assertEqual(
+            op_dist_attr.result(0).as_tensor_dist_attr().dims_mapping,
+            [0, -1, -1],
+        )
+        self.assertEqual(
+            op_dist_attr.operand(0).as_tensor_dist_attr().dims_mapping,
+            [0, -1, -1],
+        )
+
+
+if __name__ == '__main__':
+    TestCustomOpSemiAutoParallel().run_test_case()
+    TestBuildFakeProgramWithCustomOp().test_build_with_custom_relu()

test/auto_parallel/custom_op/test_semi_auto_parallel_custom_op.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import site
+import sys
+import unittest
+
+import collective.test_communication_api_base as test_base
+
+from paddle.utils.cpp_extension.extension_utils import run_cmd
+
+
+class TestCustomOp(test_base.CommunicationTestDistBase):
+    def setUp(self):
+        super().setUp(num_of_devices=2, timeout=200, nnode=1)
+        self._default_envs = {"dtype": "float32", "seed": "2023"}
+        self._changeable_envs = {"backend": ["cpu", "gpu"]}
+        cur_dir = os.path.dirname(os.path.abspath(__file__))
+        # compile, install the custom op egg into site-packages under background
+        if os.name == 'nt':
+            cmd = f'cd /d {cur_dir} && python custom_relu_setup.py install'
+        else:
+            site_dir = site.getsitepackages()[0]
+            cmd = f'cd {cur_dir} && {sys.executable} custom_relu_setup.py install --install-lib={site_dir}'
+        run_cmd(cmd)
+
+    # test dynamic auto parallel run
+    def test_dynamic_auto_parallel(self):
+        envs_list = test_base.gen_product_envs_list(
+            self._default_envs, self._changeable_envs
+        )
+        for envs in envs_list:
+            self.run_test_case(
+                "semi_auto_parallel_for_custom_op.py",
+                user_defined_envs=envs,
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/auto_parallel/custom_op/utils.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from pathlib import Path
+from site import getsitepackages
+
+from paddle.utils.cpp_extension.extension_utils import (
+    _get_all_paddle_includes_from_include_root,
+)
+
+# Test for extra compile args
+extra_cc_args = ['-w', '-g']
+extra_nvcc_args = ['-O3']
+extra_compile_args = {'cc': extra_cc_args, 'nvcc': extra_nvcc_args}
+
+
+def get_paddle_includes():
+    env_dict = os.environ
+    paddle_includes = []
+    paddle_includes.append(f"{env_dict.get('PADDLE_SOURCE_DIR')}")
+
+    # onednn
+    if env_dict.get("WITH_ONEDNN") == 'ON':
+        paddle_includes.append(f"{env_dict.get('ONEDNN_INSTALL_DIR')}/include")
+    if env_dict.get("WITH_GPU") == 'ON' or env_dict.get("WITH_ROCM") == 'ON':
+        paddle_includes.append(f"{env_dict.get('externalError_INCLUDE_DIR')}")
+    paddle_includes.append(f"{env_dict.get('PYBIND_INCLUDE_DIR')}")
+
+    for site_packages_path in getsitepackages():
+        paddle_include_dir = Path(site_packages_path) / "paddle/include"
+        paddle_includes.extend(
+            _get_all_paddle_includes_from_include_root(str(paddle_include_dir))
+        )
+
+    return paddle_includes
+
+
+paddle_includes = get_paddle_includes()