paddlepaddle--paddle/paddle/phi/kernels/xpu/gather_nd_kernel.cc

// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/phi/kernels/gather_nd_kernel.h"

#include "paddle/phi/backends/xpu/enforce_xpu.h"
#include "paddle/phi/core/kernel_registry.h"
#include "paddle/phi/kernels/tile_kernel.h"

namespace phi {

template <typename T, typename Context>
void GatherNdKernel(const Context &dev_ctx,
                    const DenseTensor &x,
                    const DenseTensor &index,
                    DenseTensor *out) {
  using XPUType = typename XPUTypeTrait<T>::Type;
  dev_ctx.template Alloc<T>(out);

  if (x.numel() == 0 || out->numel() == 0) return;
  // The result dims is
  //   Index.shape[:-1] + X.shape[Index.shape[-1]:]
  // If the last dimension of index is 0, set it to 1 and tile x.
  auto index_dims = index.dims();
  std::vector<int64_t> out_dims;
  if (index_dims[index_dims.size() - 1] == 0) {
    for (int i = 0; i < index_dims.size() - 1; ++i) {
      out_dims.emplace_back(index_dims[i]);
    }
    for (int i = 0; i < x.dims().size(); ++i) {
      out_dims.emplace_back(1);
    }
    phi::TileKernel<T, Context>(dev_ctx, x, phi::IntArray(out_dims), out);
    return;
  }
  if (index.dims()[0] == 0 && index.numel() == 0) return;

  if (index.numel() == 0) {
    auto index_dims = index.dims();
    auto index_dims_size = index_dims.size();
    // final dim
    int64_t end_size = index_dims[index_dims_size - 1];
    PADDLE_ENFORCE_EQ(
        end_size,
        0,
        common::errors::InvalidArgument("end_size[%d] should be 0", end_size));
    // remain dim
    auto remain_ddim = slice_ddim(index_dims, 0, index_dims_size - 1);
    int64_t remain_numel = common::product(remain_ddim);

    int64_t x_numel = x.numel();
    int64_t y_numel = out->numel();
    PADDLE_ENFORCE_EQ(
        x_numel * remain_numel,
        y_numel,
        common::errors::InvalidArgument(
            "x_numel[%d] * remain_numel[%d] should match y_numel[%d]",
            x_numel,
            remain_numel,
            y_numel));

    // int broadcast(Context* dev_ctx, const T* x, T* y, const
    // std::vector<int64_t>& xshape, const std::vector<int64_t>& yshape)
    int r = xpu::broadcast(dev_ctx.x_context(),
                           reinterpret_cast<const XPUType *>(x.data<T>()),
                           reinterpret_cast<XPUType *>(out->data<T>()),
                           {1, x_numel},
                           {remain_numel, x_numel});
    PADDLE_ENFORCE_XDNN_SUCCESS(r, "broadcast");
    return;
  }

  const auto &index_type = index.dtype();
  bool index_type_match =
      index_type == DataType::INT32 || index_type == DataType::INT64;
  PADDLE_ENFORCE_EQ(index_type_match,
                    true,
                    common::errors::InvalidArgument(
                        "Index holds the wrong type, it holds [%s],"
                        "but desires to be [%s] or [%s]",
                        index_type,
                        DataType::INT32,
                        DataType::INT64));

  auto x_shape = vectorize<int64_t>(x.dims());
  auto index_shape = vectorize<int64_t>(index.dims());
  if (index_shape.size() == 1) {
    index_shape.insert(index_shape.begin(), 1);
  }
  xpu::VectorParam<int64_t> x_vec = {
      x_shape.data(), static_cast<int64_t>(x_shape.size()), nullptr};

  int ret = 0;
#ifndef PADDLE_WITH_XPU_PLUGIN
  if (index_type == DataType::INT32) {
    ret = xpu::gather_nd<XPUType, int>(
        dev_ctx.x_context(),
        reinterpret_cast<const XPUType *>(x.data<T>()),
        index.data<int>(),
        reinterpret_cast<XPUType *>(out->data<T>()),
        x_vec,
        index_shape);
  } else {
    ret = xpu::gather_nd<XPUType, int64_t>(
        dev_ctx.x_context(),
        reinterpret_cast<const XPUType *>(x.data<T>()),
        index.data<int64_t>(),
        reinterpret_cast<XPUType *>(out->data<T>()),
        x_vec,
        index_shape);
  }
  PADDLE_ENFORCE_XDNN_SUCCESS(ret, "gather_nd");
#else
  if (index_type == DataType::INT32) {
    ret = xpu::plugin::fast_gather_nd<XPUType, int>(
        dev_ctx.x_context(),
        reinterpret_cast<const XPUType *>(x.data<T>()),
        index.data<int>(),
        reinterpret_cast<XPUType *>(out->data<T>()),
        x_vec,
        index_shape);
  } else {
    ret = xpu::plugin::fast_gather_nd<XPUType, int64_t>(
        dev_ctx.x_context(),
        reinterpret_cast<const XPUType *>(x.data<T>()),
        index.data<int64_t>(),
        reinterpret_cast<XPUType *>(out->data<T>()),
        x_vec,
        index_shape);
  }
  PADDLE_ENFORCE_XDNN_SUCCESS(ret, "fast_gather_nd");
#endif
}

}  // namespace phi

PD_REGISTER_KERNEL(gather_nd,
                   XPU,
                   ALL_LAYOUT,
                   phi::GatherNdKernel,
                   float,
                   int64_t,
                   int,
                   phi::float16,
                   phi::bfloat16) {}