chore: import upstream snapshot with attribution

test/cpp_extension/CMakeLists.txt

@@ -0,0 +1,13 @@
+if(WITH_TESTING)
+  if(WITH_GPU)
+    py_test(test_cpp_extension_setup SRCS test_cpp_extension_setup.py)
+    py_test(test_cpp_extension_jit SRCS test_cpp_extension_jit.py)
+    py_test(test_cpp_extension_ninja SRCS test_cpp_extension_ninja.py)
+
+    set_tests_properties(test_cpp_extension_setup PROPERTIES TIMEOUT 120)
+    set_tests_properties(test_cpp_extension_jit PROPERTIES TIMEOUT 120)
+    set_tests_properties(test_cpp_extension_ninja PROPERTIES TIMEOUT 120)
+  endif()
+  py_test(test_mixed_extension_setup SRCS test_mixed_extension_setup.py)
+  set_tests_properties(test_mixed_extension_setup PROPERTIES TIMEOUT 120)
+endif()

test/cpp_extension/cpp_extension_setup.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from pathlib import Path
+from site import getsitepackages
+
+from utils import extra_compile_args
+
+import paddle
+from paddle.utils.cpp_extension import CppExtension, CUDAExtension, setup
+from paddle.utils.cpp_extension.extension_utils import (
+    _get_all_paddle_includes_from_include_root,
+)
+
+paddle_includes = []
+for site_packages_path in getsitepackages():
+    paddle_include_dir = Path(site_packages_path) / "paddle/include"
+    paddle_includes.extend(
+        _get_all_paddle_includes_from_include_root(paddle_include_dir)
+    )
+
+# Add current dir, search custom_power.h
+paddle_includes.append(os.path.dirname(os.path.abspath(__file__)))
+
+sources = ["custom_extension.cc", "custom_sub.cc"]
+Extension = CppExtension
+if paddle.is_compiled_with_cuda():
+    sources.append("custom_relu_forward.cu")
+    Extension = CUDAExtension
+
+setup(
+    name='custom_cpp_extension',
+    ext_modules=Extension(
+        sources=sources,
+        include_dirs=paddle_includes,
+        extra_compile_args=extra_compile_args,
+        verbose=True,
+    ),
+)

test/cpp_extension/custom_extension.cc

@@ -0,0 +1,78 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <vector>
+
+#include "custom_power.h"  // NOLINT
+#include "paddle/extension.h"
+
+paddle::Tensor custom_sub(paddle::Tensor x, paddle::Tensor y);
+
+paddle::Tensor relu_cuda_forward(const paddle::Tensor& x);
+
+paddle::Tensor custom_add(const paddle::Tensor& x, const paddle::Tensor& y) {
+  return x.exp() + y.exp();
+}
+
+paddle::Tensor custom_optional_add(const paddle::Tensor& x,
+                                   const paddle::optional<paddle::Tensor>& y) {
+  if (y) {
+    return x.exp() + *y;
+  }
+  return x.exp();
+}
+
+std::vector<paddle::Tensor> custom_tensor(
+    const std::vector<paddle::Tensor>& inputs) {
+  std::vector<paddle::Tensor> out;
+  out.reserve(inputs.size());
+  for (const auto& input : inputs) {
+    out.push_back(input + 1.0);
+  }
+  return out;
+}
+
+paddle::Tensor nullable_tensor(bool return_none = false) {
+  paddle::Tensor t;
+  if (!return_none) {
+    t = paddle::ones({2, 2});
+  }
+  return t;
+}
+
+paddle::optional<paddle::Tensor> optional_tensor(bool return_option = false) {
+  paddle::optional<paddle::Tensor> t;
+  if (!return_option) {
+    t = paddle::ones({2, 2});
+  }
+  return t;
+}
+
+PYBIND11_MODULE(PADDLE_EXTENSION_NAME, m) {
+  m.def("custom_add", &custom_add, "exp(x) + exp(y)");
+  m.def("custom_optional_add", &custom_optional_add, "exp(x) + optional(y)");
+  m.def("custom_sub", &custom_sub, "exp(x) - exp(y)");
+  m.def("custom_tensor", &custom_tensor, "x + 1");
+  m.def("nullable_tensor", &nullable_tensor, "returned Tensor might be None");
+  m.def(
+      "optional_tensor", &optional_tensor, "returned Tensor might be optional");
+  m.def("relu_cuda_forward", &relu_cuda_forward, "relu(x)");
+
+  py::class_<Power>(m, "Power")
+      .def(py::init<int, int>())
+      .def(py::init<paddle::Tensor>())
+      .def("forward", &Power::forward)
+      .def("get", &Power::get);
+}

test/cpp_extension/custom_power.h

@@ -0,0 +1,28 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "paddle/extension.h"
+
+struct Power {
+  Power(int A, int B) {
+    tensor_ = paddle::ones({A, B}, phi::DataType::FLOAT32, phi::CPUPlace());
+  }
+  explicit Power(paddle::Tensor x) { tensor_ = x; }
+  paddle::Tensor forward() { return paddle::experimental::pow(tensor_, 2); }
+  paddle::Tensor get() const { return tensor_; }
+
+ private:
+  paddle::Tensor tensor_;
+};

test/cpp_extension/custom_relu_forward.cu

@@ -0,0 +1,51 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/common/enforce.h"
+#include "paddle/extension.h"
+
+#define CHECK_GPU_INPUT(x) \
+  PADDLE_ENFORCE_EQ(       \
+      x.is_gpu(), true, common::errors::Fatal(#x " must be a GPU Tensor."))
+
+template <typename data_t>
+__global__ void relu_cuda_forward_kernel(const data_t* x,
+                                         data_t* y,
+                                         int64_t num) {
+  int64_t gid = blockIdx.x * blockDim.x + threadIdx.x;
+  for (int64_t i = gid; i < num; i += blockDim.x * gridDim.x) {
+    y[i] = x[i] > static_cast<data_t>(0.) ? x[i] : static_cast<data_t>(0.);
+  }
+}
+
+paddle::Tensor relu_cuda_forward(const paddle::Tensor& x) {
+  CHECK_GPU_INPUT(x);
+  auto out = paddle::empty_like(x);
+
+  PADDLE_ENFORCE_EQ(
+      x.place() == paddle::DefaultGPUPlace(),
+      true,
+      common::errors::InvalidArgument("Input tensor `x` should be on GPU"));
+
+  int64_t numel = x.numel();
+  int64_t block = 512;
+  int64_t grid = (numel + block - 1) / block;
+  PD_DISPATCH_FLOATING_AND_HALF_TYPES(
+      x.type(), "relu_cuda_forward_kernel", ([&] {
+        relu_cuda_forward_kernel<data_t><<<grid, block, 0, x.stream()>>>(
+            x.data<data_t>(), out.data<data_t>(), numel);
+      }));
+
+  return out;
+}

test/cpp_extension/custom_sub.cc

@@ -0,0 +1,19 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/extension.h"
+
+paddle::Tensor custom_sub(paddle::Tensor x, paddle::Tensor y) {
+  return paddle::subtract(paddle::exp(x), paddle::exp(y));
+}

test/cpp_extension/mix_relu_and_extension.cc

@@ -0,0 +1,167 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <vector>
+
+#include "custom_power.h"  // NOLINT
+#include "paddle/extension.h"
+
+#define CHECK_CPU_INPUT(x) \
+  PADDLE_ENFORCE_EQ(       \
+      x.is_cpu(), true, common::errors::Fatal(#x " must be a CPU Tensor."))
+
+template <typename data_t>
+void relu_cpu_forward_kernel(const data_t* x_data,
+                             data_t* out_data,
+                             int64_t x_numel) {
+  PADDLE_ENFORCE_NE(
+      x_data, nullptr, common::errors::Fatal("x_data is nullptr."));
+  PADDLE_ENFORCE_NE(
+      out_data, nullptr, common::errors::Fatal("out_data is nullptr."));
+  for (int64_t i = 0; i < x_numel; ++i) {
+    out_data[i] = std::max(static_cast<data_t>(0.), x_data[i]);
+  }
+}
+
+template <typename data_t>
+void relu_cpu_backward_kernel(const data_t* grad_out_data,
+                              const data_t* out_data,
+                              data_t* grad_x_data,
+                              int64_t out_numel) {
+  for (int64_t i = 0; i < out_numel; ++i) {
+    grad_x_data[i] =
+        grad_out_data[i] * (out_data[i] > static_cast<data_t>(0) ? 1. : 0.);
+  }
+}
+
+template <typename data_t>
+void relu_cpu_double_backward_kernel(const data_t* out_data,
+                                     const data_t* ddx_data,
+                                     data_t* ddout_data,
+                                     int64_t ddout_numel) {
+  for (int64_t i = 0; i < ddout_numel; ++i) {
+    ddout_data[i] =
+        ddx_data[i] * (out_data[i] > static_cast<data_t>(0) ? 1. : 0.);
+  }
+}
+
+std::vector<paddle::Tensor> relu_cpu_forward(const paddle::Tensor& x) {
+  CHECK_CPU_INPUT(x);
+  auto out = paddle::empty_like(x);
+
+  PD_DISPATCH_FLOATING_TYPES(
+      x.type(), "relu_cpu_forward", ([&] {
+        relu_cpu_forward_kernel<data_t>(
+            x.data<data_t>(), out.data<data_t>(), x.numel());
+      }));
+
+  return {out};
+}
+
+std::vector<paddle::Tensor> relu_cpu_backward(const paddle::Tensor& x,
+                                              const paddle::Tensor& out,
+                                              const paddle::Tensor& grad_out) {
+  auto grad_x = paddle::empty_like(x);
+
+  PD_DISPATCH_FLOATING_TYPES(out.type(), "relu_cpu_backward", ([&] {
+                               relu_cpu_backward_kernel<data_t>(
+                                   grad_out.data<data_t>(),
+                                   out.data<data_t>(),
+                                   grad_x.data<data_t>(),
+                                   out.size());
+                             }));
+
+  return {grad_x};
+}
+
+std::vector<paddle::Tensor> relu_cpu_double_backward(
+    const paddle::Tensor& out, const paddle::Tensor& ddx) {
+  CHECK_CPU_INPUT(out);
+  CHECK_CPU_INPUT(ddx);
+  auto ddout = paddle::empty(out.shape(), out.dtype(), out.place());
+
+  PD_DISPATCH_FLOATING_TYPES(out.type(), "relu_cpu_double_backward", ([&] {
+                               relu_cpu_double_backward_kernel<data_t>(
+                                   out.data<data_t>(),
+                                   ddx.data<data_t>(),
+                                   ddout.mutable_data<data_t>(out.place()),
+                                   ddout.size());
+                             }));
+  return {ddout};
+}
+
+std::vector<paddle::Tensor> ReluForward(const paddle::Tensor& x) {
+  if (x.is_cpu()) {
+    return relu_cpu_forward(x);
+  } else {
+    PD_THROW("Not implemented.");
+  }
+}
+
+std::vector<paddle::Tensor> ReluBackward(const paddle::Tensor& x,
+                                         const paddle::Tensor& out,
+                                         const paddle::Tensor& grad_out) {
+  if (x.is_cpu()) {
+    return relu_cpu_backward(x, out, grad_out);
+  } else {
+    PD_THROW("Not implemented.");
+  }
+}
+
+std::vector<paddle::Tensor> ReluDoubleBackward(const paddle::Tensor& out,
+                                               const paddle::Tensor& ddx) {
+  if (out.is_cpu()) {
+    return relu_cpu_double_backward(out, ddx);
+  } else {
+    PD_THROW("Not implemented.");
+  }
+}
+
+std::vector<std::vector<int64_t>> ReluDoubleBackwardInferShape(
+    const std::vector<int64_t>& out_shape,
+    const std::vector<int64_t>& ddx_shape) {
+  return {out_shape};
+}
+
+PD_BUILD_OP(custom_relu)
+    .Inputs({"X"})
+    .Outputs({"Out"})
+    .SetKernelFn(PD_KERNEL(ReluForward));
+
+PD_BUILD_GRAD_OP(custom_relu)
+    .Inputs({"X", "Out", paddle::Grad("Out")})
+    .Outputs({paddle::Grad("X")})
+    .SetKernelFn(PD_KERNEL(ReluBackward));
+
+PD_BUILD_DOUBLE_GRAD_OP(custom_relu)
+    .Inputs({"Out", paddle::Grad(paddle::Grad("X"))})
+    .Outputs({paddle::Grad(paddle::Grad("Out"))})
+    .SetKernelFn(PD_KERNEL(ReluDoubleBackward))
+    .SetInferShapeFn(PD_INFER_SHAPE(ReluDoubleBackwardInferShape));
+
+// Extension with tensor operator overloading
+paddle::Tensor custom_sub2(paddle::Tensor x, paddle::Tensor y) {
+  return paddle::exp(x) - paddle::exp(y);
+}
+
+// Extension with tensor operator overloading
+paddle::Tensor custom_add2(const paddle::Tensor& x, const paddle::Tensor& y) {
+  return paddle::exp(x) + paddle::exp(y);
+}
+
+PYBIND11_MODULE(mix_relu_extension, m) {
+  m.def("custom_add2", &custom_add2, "exp(x) + exp(y)");
+  m.def("custom_sub2", &custom_sub2, "exp(x) - exp(y)");
+}

test/cpp_extension/mix_relu_and_extension_setup.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+from utils import paddle_includes
+
+from paddle.utils.cpp_extension import CppExtension, setup
+
+setup(
+    name='mix_relu_extension',
+    ext_modules=CppExtension(
+        sources=["mix_relu_and_extension.cc"],
+        include_dirs=[
+            *paddle_includes,
+            Path(__file__).parent.resolve(),
+        ],
+        extra_compile_args={'cc': ['-w', '-g']},
+        verbose=True,
+    ),
+)

test/cpp_extension/test_cpp_extension_jit.py

@@ -0,0 +1,171 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import unittest
+from pathlib import Path
+from site import getsitepackages
+
+import numpy as np
+from utils import check_output
+
+import paddle
+from paddle.utils.cpp_extension import load
+from paddle.utils.cpp_extension.extension_utils import (
+    _get_all_paddle_includes_from_include_root,
+)
+
+if os.name == 'nt' or sys.platform.startswith('darwin'):
+    # only support Linux now
+    sys.exit()
+
+# Compile and load cpp extension Just-In-Time.
+sources = ["custom_extension.cc", "custom_sub.cc"]
+if paddle.is_compiled_with_cuda():
+    sources.append("custom_relu_forward.cu")
+
+paddle_includes = []
+for site_packages_path in getsitepackages():
+    paddle_include_dir = Path(site_packages_path) / "paddle/include"
+    paddle_includes.extend(
+        _get_all_paddle_includes_from_include_root(str(paddle_include_dir))
+    )
+
+# include "custom_power.h"
+paddle_includes.append(os.path.dirname(os.path.abspath(__file__)))
+
+custom_cpp_extension = load(
+    name='custom_cpp_extension',
+    sources=sources,
+    extra_include_paths=paddle_includes,  # add for Coverage CI
+    extra_cxx_cflags=['-w', '-g'],
+    verbose=True,
+)
+
+
+class TestCppExtensionJITInstall(unittest.TestCase):
+    """
+    Tests setup install cpp extensions.
+    """
+
+    def setUp(self):
+        # config seed
+        SEED = 2021
+        paddle.seed(SEED)
+        paddle.framework.random._manual_program_seed(SEED)
+
+        self.dtypes = ['float32', 'float64']
+
+    def tearDown(self):
+        pass
+
+    def test_cpp_extension(self):
+        self._test_extension_function()
+        self._test_extension_class()
+        self._test_vector_tensor()
+        self._test_nullable_tensor()
+        self._test_optional_tensor()
+        if paddle.is_compiled_with_cuda():
+            self._test_cuda_relu()
+
+    def _test_extension_function(self):
+        for dtype in self.dtypes:
+            np_x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            x = paddle.to_tensor(np_x, dtype=dtype)
+            np_y = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            y = paddle.to_tensor(np_y, dtype=dtype)
+
+            out = custom_cpp_extension.custom_add(x, y)
+            target_out = np.exp(np_x) + np.exp(np_y)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+            out = custom_cpp_extension.custom_optional_add(x, None)
+            target_out = np.exp(np_x)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+            # Test we can call a method not defined in the main C++ file.
+            out = custom_cpp_extension.custom_sub(x, y)
+            target_out = np.exp(np_x) - np.exp(np_y)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+    def _test_extension_class(self):
+        for dtype in self.dtypes:
+            # Test we can use CppExtension class with C++ methods.
+            power = custom_cpp_extension.Power(3, 3)
+            self.assertEqual(power.get().sum(), 9)
+            self.assertEqual(power.forward().sum(), 9)
+
+            np_x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            x = paddle.to_tensor(np_x, dtype=dtype)
+
+            power = custom_cpp_extension.Power(x)
+            np.testing.assert_allclose(
+                power.get().sum().numpy(), np.sum(np_x), atol=1e-5
+            )
+            np.testing.assert_allclose(
+                power.forward().sum().numpy(),
+                np.sum(np.power(np_x, 2)),
+                atol=1e-5,
+            )
+
+    def _test_vector_tensor(self):
+        for dtype in self.dtypes:
+            np_inputs = [
+                np.random.uniform(-1, 1, [4, 8]).astype(dtype) for _ in range(3)
+            ]
+            inputs = [paddle.to_tensor(np_x, dtype=dtype) for np_x in np_inputs]
+
+            out = custom_cpp_extension.custom_tensor(inputs)
+            target_out = [x + 1.0 for x in inputs]
+            for i in range(3):
+                np.testing.assert_allclose(
+                    out[i].numpy(), target_out[i].numpy(), atol=1e-5
+                )
+
+    def _test_nullable_tensor(self):
+        x = custom_cpp_extension.nullable_tensor(True)
+        assert x is None, "Return None when input parameter return_none = True"
+        x = custom_cpp_extension.nullable_tensor(False).numpy()
+        x_np = np.ones(shape=[2, 2])
+        np.testing.assert_array_equal(
+            x,
+            x_np,
+            err_msg=f'extension out: {x},\n numpy out: {x_np}',
+        )
+
+    def _test_optional_tensor(self):
+        x = custom_cpp_extension.optional_tensor(True)
+        assert x is None, (
+            "Return None when input parameter return_option = True"
+        )
+        x = custom_cpp_extension.optional_tensor(False).numpy()
+        x_np = np.ones(shape=[2, 2])
+        np.testing.assert_array_equal(
+            x,
+            x_np,
+            err_msg=f'extension out: {x},\n numpy out: {x_np}',
+        )
+
+    def _test_cuda_relu(self):
+        paddle.set_device('gpu')
+        x = np.random.uniform(-1, 1, [4, 8]).astype('float32')
+        x = paddle.to_tensor(x, dtype='float32')
+        out = custom_cpp_extension.relu_cuda_forward(x)
+        pd_out = paddle.nn.functional.relu(x)
+        check_output(out, pd_out, "out")
+
+
+if __name__ == '__main__':
+    unittest.main()

test/cpp_extension/test_cpp_extension_setup.py

@@ -0,0 +1,170 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import site
+import sys
+import unittest
+
+import numpy as np
+from utils import check_output
+
+import paddle
+from paddle.utils.cpp_extension.extension_utils import run_cmd
+
+
+class TestCppExtensionSetupInstall(unittest.TestCase):
+    """
+    Tests setup install cpp extensions.
+    """
+
+    def setUp(self):
+        cur_dir = os.path.dirname(os.path.abspath(__file__))
+        # install general extension
+        # compile, install the custom op egg into site-packages under background
+        site_dir = site.getsitepackages()[0]
+        cmd = f'cd {cur_dir} && {sys.executable} cpp_extension_setup.py install'
+        if os.name != 'nt':
+            cmd += f' --install-lib={site_dir}'
+        run_cmd(cmd)
+
+        custom_install_path = [
+            x for x in os.listdir(site_dir) if 'custom_cpp_extension' in x
+        ]
+
+        assert len(custom_install_path) == 2, (
+            f"Matched egg number is {len(custom_install_path)}."
+        )
+
+        sys.path.append(os.path.join(site_dir, custom_install_path[0]))
+        #################################
+
+        # config seed
+        SEED = 2021
+        paddle.seed(SEED)
+        paddle.framework.random._manual_program_seed(SEED)
+
+        self.dtypes = ['float32', 'float64']
+
+    def tearDown(self):
+        pass
+
+    def test_cpp_extension(self):
+        self._test_extension_function_plain()
+        self._test_vector_tensor()
+        self._test_extension_class()
+        self._test_nullable_tensor()
+        self._test_optional_tensor()
+
+    def _test_extension_function_plain(self):
+        import custom_cpp_extension
+
+        for dtype in self.dtypes:
+            np_x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            x = paddle.to_tensor(np_x, dtype=dtype)
+            np_y = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            y = paddle.to_tensor(np_y, dtype=dtype)
+            # Test custom_cpp_extension
+            out = custom_cpp_extension.custom_add(x, y)
+            target_out = np.exp(np_x) + np.exp(np_y)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+            # Test we can call a method not defined in the main C++ file.
+            out = custom_cpp_extension.custom_sub(x, y)
+            target_out = np.exp(np_x) - np.exp(np_y)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+    def _test_extension_class(self):
+        import custom_cpp_extension
+
+        for dtype in self.dtypes:
+            # Test custom_cpp_extension
+            # Test we can use CppExtension class with C++ methods.
+            power = custom_cpp_extension.Power(3, 3)
+            self.assertEqual(power.get().sum(), 9)
+            self.assertEqual(power.forward().sum(), 9)
+
+            np_x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            x = paddle.to_tensor(np_x, dtype=dtype)
+
+            power = custom_cpp_extension.Power(x)
+            np.testing.assert_allclose(
+                power.get().sum().numpy(), np.sum(np_x), atol=1e-5
+            )
+            np.testing.assert_allclose(
+                power.forward().sum().numpy(),
+                np.sum(np.power(np_x, 2)),
+                atol=1e-5,
+            )
+
+    def _test_vector_tensor(self):
+        import custom_cpp_extension
+
+        for dtype in self.dtypes:
+            np_inputs = [
+                np.random.uniform(-1, 1, [4, 8]).astype(dtype) for _ in range(3)
+            ]
+            inputs = [paddle.to_tensor(np_x, dtype=dtype) for np_x in np_inputs]
+
+            out = custom_cpp_extension.custom_tensor(inputs)
+            target_out = [x + 1 for x in inputs]
+            for i in range(3):
+                np.testing.assert_allclose(
+                    out[i].numpy(), target_out[i].numpy(), atol=1e-5
+                )
+
+    def _test_nullable_tensor(self):
+        import custom_cpp_extension
+
+        x = custom_cpp_extension.nullable_tensor(True)
+        assert x is None, "Return None when input parameter return_none = True"
+        x = custom_cpp_extension.nullable_tensor(False).numpy()
+        x_np = np.ones(shape=[2, 2])
+        np.testing.assert_array_equal(
+            x,
+            x_np,
+            err_msg=f'extension out: {x},\n numpy out: {x_np}',
+        )
+
+    def _test_optional_tensor(self):
+        import custom_cpp_extension
+
+        x = custom_cpp_extension.optional_tensor(True)
+        assert x is None, (
+            "Return None when input parameter return_option = True"
+        )
+        x = custom_cpp_extension.optional_tensor(False).numpy()
+        x_np = np.ones(shape=[2, 2])
+        np.testing.assert_array_equal(
+            x,
+            x_np,
+            err_msg=f'extension out: {x},\n numpy out: {x_np}',
+        )
+
+    def _test_cuda_relu(self):
+        import custom_cpp_extension
+
+        paddle.set_device('gpu')
+        x = np.random.uniform(-1, 1, [4, 8]).astype('float32')
+        x = paddle.to_tensor(x, dtype='float32')
+        out = custom_cpp_extension.relu_cuda_forward(x)
+        pd_out = paddle.nn.functional.relu(x)
+        check_output(out, pd_out, "out")
+
+
+if __name__ == '__main__':
+    if os.name == 'nt' or sys.platform.startswith('darwin'):
+        # only support Linux now
+        sys.exit()
+    unittest.main()

test/cpp_extension/test_mixed_extension_setup.py

@@ -0,0 +1,228 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import site
+import sys
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle import static
+from paddle.utils.cpp_extension.extension_utils import run_cmd
+
+
+def custom_relu_static(
+    func, device, dtype, np_x, use_func=True, test_infer=False
+):
+    paddle.enable_static()
+    paddle.set_device(device)
+
+    with (
+        static.scope_guard(static.Scope()),
+        static.program_guard(static.Program()),
+    ):
+        x = static.data(name='X', shape=[None, 8], dtype=dtype)
+        x.stop_gradient = False
+        out = func(x) if use_func else paddle.nn.functional.relu(x)
+        static.append_backward(out)
+
+        exe = static.Executor()
+        exe.run(static.default_startup_program())
+        # in static graph mode, x data has been covered by out
+        out_v = exe.run(
+            static.default_main_program(),
+            feed={'X': np_x},
+            fetch_list=[out],
+        )
+
+    paddle.disable_static()
+    return out_v
+
+
+def custom_relu_dynamic(func, device, dtype, np_x, use_func=True):
+    paddle.set_device(device)
+
+    t = paddle.to_tensor(np_x, dtype=dtype)
+    t.stop_gradient = False
+
+    out = func(t) if use_func else paddle.nn.functional.relu(t)
+    out.stop_gradient = False
+
+    out.backward()
+
+    if t.grad is None:
+        return out.numpy(), t.grad
+    else:
+        return out.numpy(), t.grad.numpy()
+
+
+def custom_relu_double_grad_dynamic(func, device, dtype, np_x, use_func=True):
+    paddle.set_device(device)
+
+    t = paddle.to_tensor(np_x, dtype=dtype, stop_gradient=False)
+    t.retain_grads()
+
+    out = func(t) if use_func else paddle.nn.functional.relu(t)
+    out.retain_grads()
+    dx = paddle.grad(
+        outputs=out,
+        inputs=t,
+        grad_outputs=paddle.ones_like(t),
+        create_graph=True,
+        retain_graph=True,
+    )
+
+    ddout = paddle.grad(
+        outputs=dx[0],
+        inputs=out.grad,
+        grad_outputs=paddle.ones_like(t),
+        create_graph=False,
+    )
+
+    assert ddout[0].numpy() is not None
+    return dx[0].numpy(), ddout[0].numpy()
+
+
+class TestCppExtensionSetupInstall(unittest.TestCase):
+    """
+    Tests setup install cpp extensions.
+    """
+
+    def setUp(self):
+        cur_dir = os.path.dirname(os.path.abspath(__file__))
+        # install mixed custom_op and extension
+        # compile, install the custom op egg into site-packages under background
+        site_dir = site.getsitepackages()[0]
+        cmd = f'cd {cur_dir} && {sys.executable} mix_relu_and_extension_setup.py install'
+        if os.name != 'nt':
+            cmd += f' --install-lib={site_dir}'
+        run_cmd(cmd)
+
+        custom_install_path = [
+            x for x in os.listdir(site_dir) if 'mix_relu_extension' in x
+        ]
+
+        assert len(custom_install_path) == 2, (
+            f"Matched egg number is {len(custom_install_path)}."
+        )
+
+        sys.path.append(os.path.join(site_dir, custom_install_path[0]))
+        #################################
+
+        # config seed
+        SEED = 2021
+        paddle.seed(SEED)
+        paddle.framework.random._manual_program_seed(SEED)
+
+        self.dtypes = ['float32', 'float64']
+
+    def tearDown(self):
+        pass
+
+    def test_cpp_extension(self):
+        # Extension
+        self._test_extension_function_mixed()
+        # Custom op
+        self._test_static()
+        self._test_dynamic()
+        self._test_double_grad_dynamic()
+
+    def _test_extension_function_mixed(self):
+        import mix_relu_extension
+
+        for dtype in self.dtypes:
+            np_x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            x = paddle.to_tensor(np_x, dtype=dtype)
+            np_y = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            y = paddle.to_tensor(np_y, dtype=dtype)
+
+            # Test mix_relu_extension
+            out = mix_relu_extension.custom_add2(x, y)
+            target_out = np.exp(np_x) + np.exp(np_y)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+            # Test we can call a method not defined in the main C++ file.
+            out = mix_relu_extension.custom_sub2(x, y)
+            target_out = np.exp(np_x) - np.exp(np_y)
+            np.testing.assert_allclose(out.numpy(), target_out, atol=1e-5)
+
+    def _test_static(self):
+        import mix_relu_extension
+
+        for dtype in self.dtypes:
+            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            out = custom_relu_static(
+                mix_relu_extension.custom_relu, "CPU", dtype, x
+            )
+            pd_out = custom_relu_static(
+                mix_relu_extension.custom_relu, "CPU", dtype, x, False
+            )
+            np.testing.assert_array_equal(
+                out,
+                pd_out,
+                err_msg=f'custom op out: {out},\n paddle api out: {pd_out}',
+            )
+
+    def _test_dynamic(self):
+        import mix_relu_extension
+
+        for dtype in self.dtypes:
+            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            out, x_grad = custom_relu_dynamic(
+                mix_relu_extension.custom_relu, "CPU", dtype, x
+            )
+            pd_out, pd_x_grad = custom_relu_dynamic(
+                mix_relu_extension.custom_relu, "CPU", dtype, x, False
+            )
+            np.testing.assert_array_equal(
+                out,
+                pd_out,
+                err_msg=f'custom op out: {out},\n paddle api out: {pd_out}',
+            )
+            np.testing.assert_array_equal(
+                x_grad,
+                pd_x_grad,
+                err_msg=f'custom op x grad: {x_grad},\n paddle api x grad: {pd_x_grad}',
+            )
+
+    def _test_double_grad_dynamic(self):
+        import mix_relu_extension
+
+        for dtype in self.dtypes:
+            x = np.random.uniform(-1, 1, [4, 8]).astype(dtype)
+            out, dx_grad = custom_relu_double_grad_dynamic(
+                mix_relu_extension.custom_relu, "CPU", dtype, x
+            )
+            pd_out, pd_dx_grad = custom_relu_double_grad_dynamic(
+                mix_relu_extension.custom_relu, "CPU", dtype, x, False
+            )
+            np.testing.assert_array_equal(
+                out,
+                pd_out,
+                err_msg=f'custom op out: {out},\n paddle api out: {pd_out}',
+            )
+            np.testing.assert_array_equal(
+                dx_grad,
+                pd_dx_grad,
+                err_msg=f'custom op dx grad: {dx_grad},\n paddle api dx grad: {pd_dx_grad}',
+            )
+
+
+if __name__ == '__main__':
+    if os.name == 'nt' or sys.platform.startswith('darwin'):
+        # only support Linux now
+        sys.exit()
+    unittest.main()

test/cpp_extension/utils.py

@@ -0,0 +1,77 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from pathlib import Path
+from site import getsitepackages
+
+import numpy as np
+
+from paddle.utils.cpp_extension.extension_utils import (
+    IS_WINDOWS,
+    _get_all_paddle_includes_from_include_root,
+)
+
+IS_MAC = sys.platform.startswith('darwin')
+
+# Note(Aurelius84): We use `add_test` in Cmake to config how to run unittest in CI.
+# `PYTHONPATH` will be set as `build/python/paddle` that will make no way to find
+# paddle include directory. Because the following path is generated after installing
+# PaddlePaddle whl. So here we specific `include_dirs` to avoid errors in CI.
+paddle_includes = []
+for site_packages_path in getsitepackages():
+    paddle_include_dir = Path(site_packages_path) / "paddle/include"
+    paddle_includes.extend(
+        _get_all_paddle_includes_from_include_root(str(paddle_include_dir))
+    )
+
+
+# Test for extra compile args
+extra_cc_args = ['-w', '-g'] if not IS_WINDOWS else ['/w']
+extra_nvcc_args = ['-O3']
+extra_compile_args = {'cc': extra_cc_args, 'nvcc': extra_nvcc_args}
+
+
+def check_output(out, pd_out, name):
+    if out is None and pd_out is None:
+        return
+    assert out is not None, "out value of " + name + " is None"
+    assert pd_out is not None, "pd_out value of " + name + " is None"
+    if isinstance(out, list) and isinstance(pd_out, list):
+        for idx in range(len(out)):
+            np.testing.assert_array_equal(
+                out[idx],
+                pd_out[idx],
+                err_msg=f'custom op {name}: {out[idx]},\n paddle api {name}: {pd_out[idx]}',
+            )
+    else:
+        np.testing.assert_array_equal(
+            out,
+            pd_out,
+            err_msg=f'custom op {name}: {out},\n paddle api {name}: {pd_out}',
+        )
+
+
+def check_output_allclose(out, pd_out, name, rtol=5e-5, atol=1e-2):
+    if out is None and pd_out is None:
+        return
+    assert out is not None, "out value of " + name + " is None"
+    assert pd_out is not None, "pd_out value of " + name + " is None"
+    np.testing.assert_allclose(
+        out,
+        pd_out,
+        rtol,
+        atol,
+        err_msg=f'custom op {name}: {out},\n paddle api {name}: {pd_out}',
+    )