chore: import upstream snapshot with attribution

paddle/phi/kernels/strings/gpu/copy_utils.h

@@ -0,0 +1,198 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/phi/backends/gpu/gpu_helper.h"
+#include "paddle/phi/backends/gpu/gpu_info.h"
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/string_tensor.h"
+
+namespace phi {
+namespace strings {
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+__global__ void SerializeStringsData(const phi::dtype::pstring* src_str,
+                                     uint8_t* strings_data,
+                                     int32_t* strings_offset,
+                                     int64_t numel,
+                                     int32_t start_offset) {
+  if (threadIdx.x == 0 && blockIdx.x == 0) {
+    strings_offset[0] = start_offset;
+    for (int64_t i = 1; i <= numel; ++i) {
+      strings_offset[i] = strings_offset[i - 1] + src_str[i - 1].length() + 1;
+    }
+  }
+  __syncthreads();
+  CUDA_KERNEL_LOOP(i, numel) {
+    memcpy(strings_data + strings_offset[i],
+           src_str[i].data(),
+           src_str[i].length() + 1);
+  }
+}
+
+__global__ void SumStringsLen(const phi::dtype::pstring* src_ptr,
+                              int64_t numel,
+                              int* num) {
+  extern __shared__ int counter[];
+  int thread_counter = 0;
+  CUDA_KERNEL_LOOP(i, numel) { thread_counter += src_ptr[i].length() + 1; }
+  counter[threadIdx.x] = thread_counter;
+  __syncthreads();
+  if (threadIdx.x == 0) {
+    int block_counter = 0;
+    for (int i = 0; i < blockDim.x; ++i) {
+      block_counter += counter[i];
+    }
+    atomicAdd(num, block_counter);
+  }
+}
+
+template <typename Context>
+int GetAllStringsSize(const Context& dev_ctx,
+                      const phi::dtype::pstring* src_ptr,
+                      size_t numel) {
+  auto nums_meta = phi::DenseTensorMeta(DataType::INT32, {1}, DataLayout::NCHW);
+  DenseTensor nums_tensor = Empty(dev_ctx, std::move(nums_meta));
+
+  int* nums_ptr = dev_ctx.template Alloc<int>(&nums_tensor);
+  phi::backends::gpu::GpuMemsetAsync(
+      nums_ptr, 0, sizeof(int), dev_ctx.stream());
+
+  dim3 block_size = dim3(PREDEFINED_BLOCK_SIZE, 1);
+  dim3 grid_size =
+      dim3((numel + PREDEFINED_BLOCK_SIZE - 1) / PREDEFINED_BLOCK_SIZE, 1);
+  SumStringsLen<<<grid_size,
+                  block_size,
+                  PREDEFINED_BLOCK_SIZE * sizeof(int),
+                  dev_ctx.stream()>>>(src_ptr, numel, nums_ptr);
+  int num = -1;
+#ifdef PADDLE_WITH_HIP
+  phi::backends::gpu::GpuMemcpyAsync(
+      &num, nums_ptr, sizeof(int), hipMemcpyDeviceToHost, dev_ctx.stream());
+#else
+  phi::backends::gpu::GpuMemcpyAsync(
+      &num, nums_ptr, sizeof(int), cudaMemcpyDeviceToHost, dev_ctx.stream());
+#endif
+  return num;
+}
+
+__global__ void DeserializeCUDAKernel(const char* strings_data,
+                                      const int* strings_offset,
+                                      phi::dtype::pstring* dst_str,
+                                      int numel) {
+  CUDA_KERNEL_LOOP(i, numel) {
+    // -1 not include '\0'
+    auto len = strings_offset[i + 1] - strings_offset[i] - 1;
+    dst_str[i] = phi::dtype::pstring(strings_data + strings_offset[i], len);
+  }
+}
+#endif
+
+template <typename Context>
+void SerializeOnCPU(const Context& dev_ctx,
+                    const StringTensor& src,
+                    DenseTensor* dst) {
+  int64_t numel = src.numel();
+  int64_t num = sizeof(int) * (numel + 1);
+  auto* src_str = src.data();
+  for (int64_t i = 0; i < numel; ++i) {
+    num += src_str[i].length() + 1;
+  }
+  dst->Resize({num});
+  uint8_t* strings_data = dev_ctx.template HostAlloc<uint8_t>(dst);
+  auto* strings_offset = reinterpret_cast<int*>(strings_data);
+  int start_offset = sizeof(int) * (numel + 1);
+  for (int64_t i = 0; i <= numel; ++i) {
+    if (i == 0) {
+      strings_offset[i] = start_offset;
+    } else {
+      strings_offset[i] = strings_offset[i - 1] + src_str[i - 1].length() + 1;
+    }
+  }
+  for (int64_t i = 0; i < numel; ++i) {
+    memcpy(strings_data + strings_offset[i],
+           src_str[i].data(),
+           src_str[i].length() + 1);
+  }
+}
+
+template <typename Context>
+void DeserializeOnCPU(const Context& dev_ctx,
+                      const DenseTensor& src,
+                      StringTensor* dst) {
+  auto* strings_data = reinterpret_cast<const char*>(src.data<uint8_t>());
+  auto* strings_offset = reinterpret_cast<const int*>(strings_data);
+  int numel = strings_offset[0] / sizeof(int) - 1;
+  dst->Resize({numel});
+  dtype::pstring* dst_str = dev_ctx.template HostAlloc<dtype::pstring>(dst);
+  for (int i = 0; i < numel; ++i) {
+    // -1 not include '\0'
+    auto len = strings_offset[i + 1] - strings_offset[i] - 1;
+    dst_str[i] = phi::dtype::pstring(strings_data + strings_offset[i], len);
+  }
+}
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+void SerializeOnGPU(const phi::GPUContext& dev_ctx,
+                    const StringTensor& src,
+                    DenseTensor* dst) {
+  int64_t numel = src.numel();
+  auto* src_str = src.data();
+  // 1.get the number of bytes of all strings in string tensor
+  auto strings_size = GetAllStringsSize(dev_ctx, src_str, numel);
+  strings_size += sizeof(int32_t) * (numel + 1);
+
+  dst->Resize({strings_size});
+  uint8_t* strings_data = dev_ctx.template Alloc<uint8_t>(dst);
+  auto* strings_offset = reinterpret_cast<int*>(strings_data);
+
+  int32_t start_offset = sizeof(int32_t) * (numel + 1);
+  // 2. serialize strings data to dense tensor
+  dim3 block_size = dim3(PREDEFINED_BLOCK_SIZE, 1);
+  dim3 grid_size =
+      dim3((numel + PREDEFINED_BLOCK_SIZE - 1) / PREDEFINED_BLOCK_SIZE, 1);
+
+  SerializeStringsData<<<grid_size, block_size, 0, dev_ctx.stream()>>>(
+      src_str, strings_data, strings_offset, numel, start_offset);
+}
+
+void DeserializeOnGPU(const phi::GPUContext& dev_ctx,
+                      const DenseTensor& src,
+                      StringTensor* dst) {
+  auto* strings_data = reinterpret_cast<const char*>(src.data<uint8_t>());
+  auto* strings_offset = reinterpret_cast<const int*>(strings_data);
+  int numel = 0;
+#ifdef PADDLE_WITH_HIP
+  phi::backends::gpu::GpuMemcpySync(
+      &numel, strings_data, sizeof(numel), hipMemcpyDeviceToHost);
+#else
+  phi::backends::gpu::GpuMemcpySync(
+      &numel, strings_data, sizeof(numel), cudaMemcpyDeviceToHost);
+#endif
+  numel = numel / sizeof(int) - 1;
+  dst->Resize({numel});
+  dtype::pstring* dst_str = dev_ctx.template Alloc<dtype::pstring>(dst);
+
+  dim3 block_size = dim3(PREDEFINED_BLOCK_SIZE, 1);
+  dim3 grid_size =
+      dim3((numel + PREDEFINED_BLOCK_SIZE - 1) / PREDEFINED_BLOCK_SIZE, 1);
+  DeserializeCUDAKernel<<<grid_size, block_size, 0, dev_ctx.stream()>>>(
+      strings_data, strings_offset, dst_str, numel);
+}
+#endif
+
+}  // namespace strings
+}  // namespace phi

paddle/phi/kernels/strings/gpu/strings_copy_kernel.cu

@@ -0,0 +1,127 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/strings/strings_copy_kernel.h"
+
+#include "glog/logging.h"
+
+#include "paddle/phi/backends/all_context.h"
+#include "paddle/phi/backends/gpu/gpu_helper.h"
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/core/tensor_utils.h"
+#include "paddle/phi/kernels/empty_kernel.h"
+#include "paddle/phi/kernels/strings/gpu/copy_utils.h"
+
+namespace phi {
+namespace strings {
+
+__global__ void CopyFromStringTensor(pstring* dst,
+                                     const pstring* src,
+                                     int64_t num) {
+  CUDA_KERNEL_LOOP(i, num) { dst[i] = src[i]; }
+}
+
+template <typename Context>
+void Copy(const Context& dev_ctx,
+          const StringTensor& src,
+          bool blocking,
+          StringTensor* dst) {
+  auto* src_ptr = src.data();
+  const auto& src_place = src.place();
+  auto dst_place = dst->place();
+
+  if (src_place == dst_place && src_place.GetType() == AllocationType::CPU) {
+    PADDLE_THROW(common::errors::InvalidArgument(
+        "The src and dst string tensor are all "
+        "CPU string tensor, you should call copy "
+        "function in CPU mode."));
+  }
+  VLOG(3) << "StringTensorCopy " << src.dims() << " from " << src.place()
+          << " to " << dst_place;
+
+  dst->Resize(src.dims());
+  auto* dst_ptr = dev_ctx.template Alloc<dtype::pstring>(dst);
+
+  if (src_ptr == dst_ptr && src_place == dst_place) {
+    VLOG(3) << "Skip copy the same string data async from " << src_place
+            << " to " << dst_place;
+    return;
+  }
+
+  VLOG(4) << "src:" << src_ptr << ", dst:" << dst_ptr;
+
+  if (src_place.GetType() == AllocationType::GPU &&
+      dst_place.GetType() == AllocationType::CPU) {
+    // Situation 1: gpu_place->cpu_place
+    DenseTensor gpu_serialized = Empty<uint8_t, GPUContext>(dev_ctx, {1});
+    phi::strings::SerializeOnGPU(dev_ctx, src, &gpu_serialized);
+
+    DenseTensor cpu_serialized;
+    cpu_serialized.Resize(gpu_serialized.dims());
+    dev_ctx.template HostAlloc<uint8_t>(&cpu_serialized);
+
+    phi::Copy(dev_ctx, gpu_serialized, dst_place, false, &cpu_serialized);
+
+    phi::strings::DeserializeOnCPU(dev_ctx, cpu_serialized, dst);
+
+  } else if (src_place.GetType() == AllocationType::CPU &&
+             dst_place.GetType() == AllocationType::GPU) {
+    // Situation 2: cpu_place->gpu_place
+    DenseTensor cpu_serialized;
+    cpu_serialized.Resize({1});
+    dev_ctx.template HostAlloc<uint8_t>(&cpu_serialized);
+
+    phi::strings::SerializeOnCPU(dev_ctx, src, &cpu_serialized);
+
+    DenseTensor gpu_serialized = EmptyLike<uint8_t>(dev_ctx, cpu_serialized);
+    phi::Copy(
+        dev_ctx, cpu_serialized, dev_ctx.GetPlace(), false, &gpu_serialized);
+
+    phi::strings::DeserializeOnGPU(dev_ctx, gpu_serialized, dst);
+  } else if (src_place.GetType() == AllocationType::GPU &&
+             dst_place.GetType() == AllocationType::GPU) {
+    // Situation 3: gpu_place->gpu_place
+    auto src_gpu_place = src_place;
+    auto dst_gpu_place = dst_place;
+    auto ctx_place = dev_ctx.GetPlace();
+    PADDLE_ENFORCE_EQ(
+        ctx_place.GetType(),
+        AllocationType::GPU,
+        common::errors::PreconditionNotMet(
+            "Context place error, excepted GPUPlace, but actually %s.",
+            ctx_place));
+    int64_t numel = src.numel();
+    dim3 block_size = dim3(PREDEFINED_BLOCK_SIZE, 1);
+    dim3 grid_size =
+        dim3((numel + PREDEFINED_BLOCK_SIZE - 1) / PREDEFINED_BLOCK_SIZE, 1);
+    // Copy
+    CopyFromStringTensor<<<grid_size, block_size, 0, dev_ctx.stream()>>>(
+        dst_ptr, src_ptr, numel);
+  }
+}
+#ifdef _WIN32
+template PADDLE_API void Copy<GPUContext>(const GPUContext&,
+                                          const StringTensor&,
+                                          bool,
+                                          StringTensor*);
+#endif
+}  // namespace strings
+}  // namespace phi
+
+PD_REGISTER_KERNEL_FOR_ALL_DTYPE(strings_copy,
+                                 GPU,
+                                 ALL_LAYOUT,
+                                 phi::strings::Copy<phi::GPUContext>) {}

paddle/phi/kernels/strings/gpu/strings_lower_upper_kernel.cu

@@ -0,0 +1,193 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/kernels/strings/strings_lower_upper_kernel.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/backends/gpu/gpu_launch_config.h"
+#include "paddle/phi/common/pstring.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/strings/unicode.h"
+
+namespace phi {
+namespace strings {
+
+template <typename CharConverter>
+__global__ void StringCaseConvertCUDAKernel(pstring* out,
+                                            const pstring* in,
+                                            size_t num) {
+  CUDA_KERNEL_LOOP(i, num) {
+    out[i] = pstring(in[i]);
+    thrust::transform(thrust::device,
+                      in[i].begin(),
+                      in[i].end(),
+                      out[i].mdata(),
+                      CharConverter());
+  }
+}
+
+template <typename CharConverter>
+struct AsciiCaseConverter<phi::GPUContext, CharConverter> {
+  void operator()(const phi::GPUContext& dev_ctx,
+                  const pstring* in,
+                  pstring* out,
+                  size_t num) const {
+#ifdef PADDLE_WITH_HIP
+    dim3 block_size = dim3(256, 1);
+#else
+    dim3 block_size = dim3(PREDEFINED_BLOCK_SIZE, 1);
+#endif
+    dim3 grid_size =
+        dim3((num + PREDEFINED_BLOCK_SIZE - 1) / PREDEFINED_BLOCK_SIZE, 1);
+    StringCaseConvertCUDAKernel<CharConverter>
+        <<<grid_size, block_size, 0, dev_ctx.stream()>>>(out, in, num);
+  }
+};
+
+template <template <typename DeviceContextT> typename CharConverter>
+struct UTF8CaseConverter<phi::GPUContext, CharConverter> {
+  void operator()(const phi::GPUContext& dev_ctx,
+                  const pstring* in,
+                  pstring* out,
+                  size_t num) const {
+    auto unicode_flag_map = GetGPUUniflagMap();
+    auto cases_map = GetGPUCharCasesMap();
+    thrust::device_vector<uint32_t> unicode_offsets(num + 1, 0);
+    uint32_t* unicode_offsets_ptr =
+        thrust::raw_pointer_cast(unicode_offsets.data());
+
+    thrust::for_each_n(thrust::device,
+                       thrust::make_counting_iterator<unsigned int>(0),
+                       num,
+                       [unicode_offsets_ptr, in] __device__(uint32_t idx) {
+                         unicode_offsets_ptr[idx + 1] =
+                             GetUnicodeStrLen(in[idx].data(), in[idx].size());
+                       });
+    uint32_t total_lengths = thrust::reduce(
+        thrust::device, unicode_offsets_ptr, unicode_offsets_ptr + num + 1, 0);
+    if (total_lengths == 0) {
+      return;
+    }
+
+    thrust::device_vector<uint32_t> unicode_output(total_lengths, 0);
+    uint32_t* unicode_output_ptr =
+        thrust::raw_pointer_cast(unicode_output.data());
+
+    CharConverter<GPUContext> converter(unicode_flag_map, cases_map);
+    thrust::for_each_n(
+        thrust::device,
+        thrust::make_counting_iterator<unsigned int>(0),
+        num,
+        [in,
+         out,
+         unicode_output_ptr,
+         unicode_offsets_ptr,
+         converter] __device__(uint32_t idx) {
+          uint32_t unicode_len =
+              unicode_offsets_ptr[idx + 1] - unicode_offsets_ptr[idx];
+          GetUnicodeStr(in[idx].data(),
+                        unicode_output_ptr + unicode_offsets_ptr[idx],
+                        unicode_len);
+          uint32_t* curr_unicode_output_ptr =
+              unicode_output_ptr + unicode_offsets_ptr[idx];
+          for (uint32_t i = 0; i < unicode_len; ++i) {
+            curr_unicode_output_ptr[i] = converter(curr_unicode_output_ptr[i]);
+          }
+          thrust::transform(thrust::device,
+                            unicode_output_ptr + unicode_offsets_ptr[idx],
+                            unicode_output_ptr + unicode_offsets_ptr[idx + 1],
+                            unicode_output_ptr + unicode_offsets_ptr[idx],
+                            converter);
+        });
+
+    thrust::device_vector<uint32_t> utf8_offsets(num + 1, 0);
+    uint32_t* utf8_offsets_ptr = thrust::raw_pointer_cast(utf8_offsets.data());
+
+    thrust::for_each_n(
+        thrust::device,
+        thrust::make_counting_iterator<unsigned int>(0),
+        num,
+        [utf8_offsets_ptr, unicode_output_ptr, unicode_offsets_ptr] __device__(
+            uint32_t idx) {
+          uint32_t unicode_len =
+              unicode_offsets_ptr[idx + 1] - unicode_offsets_ptr[idx];
+          utf8_offsets_ptr[idx + 1] = GetUTF8StrLen(
+              unicode_output_ptr + unicode_offsets_ptr[idx], unicode_len);
+        });
+    uint32_t total_utf8_lengths = thrust::reduce(
+        thrust::device, utf8_offsets_ptr, utf8_offsets_ptr + num + 1, 0);
+
+    thrust::device_vector<char> utf8_output(total_utf8_lengths, 0);
+    char* utf8_output_ptr = thrust::raw_pointer_cast(utf8_output.data());
+    thrust::for_each_n(thrust::device,
+                       thrust::make_counting_iterator<unsigned int>(0),
+                       num,
+                       [utf8_output_ptr,
+                        utf8_offsets_ptr,
+                        unicode_output_ptr,
+                        unicode_offsets_ptr,
+                        out] __device__(uint32_t idx) {
+                         uint32_t unicode_len = unicode_offsets_ptr[idx + 1] -
+                                                unicode_offsets_ptr[idx];
+                         const uint32_t* input_ptr =
+                             unicode_output_ptr + unicode_offsets_ptr[idx];
+                         char* result_ptr =
+                             utf8_output_ptr + utf8_offsets_ptr[idx];
+                         GetUTF8Str(input_ptr, result_ptr, unicode_len);
+                         out[idx] = result_ptr;
+                       });
+  }
+};
+
+template <typename ContextT>
+void StringLowerKernel(const ContextT& dev_ctx,
+                       const StringTensor& x,
+                       bool use_utf8_encoding,
+                       StringTensor* out) {
+  StringCaseConvertKernel<AsciiCaseConverter<ContextT, AsciiToLower>,
+                          UTF8CaseConverter<ContextT, UTF8ToLower>,
+                          ContextT>()(dev_ctx, x, use_utf8_encoding, out);
+}
+
+template <typename ContextT>
+void StringUpperKernel(const ContextT& dev_ctx,
+                       const StringTensor& x,
+                       bool use_utf8_encoding,
+                       StringTensor* out) {
+  StringCaseConvertKernel<AsciiCaseConverter<ContextT, AsciiToUpper>,
+                          UTF8CaseConverter<ContextT, UTF8ToUpper>,
+                          ContextT>()(dev_ctx, x, use_utf8_encoding, out);
+}
+#ifdef _WIN32
+template PADDLE_API void StringLowerKernel<GPUContext>(const GPUContext&,
+                                                       const StringTensor& x,
+                                                       bool,
+                                                       StringTensor*);
+
+template PADDLE_API void StringUpperKernel<GPUContext>(const GPUContext&,
+                                                       const StringTensor& x,
+                                                       bool,
+                                                       StringTensor*);
+#endif
+}  // namespace strings
+}  // namespace phi
+
+PD_REGISTER_KERNEL_FOR_ALL_DTYPE(
+    strings_lower,
+    GPU,
+    ALL_LAYOUT,
+    phi::strings::StringLowerKernel<phi::GPUContext>) {}
+
+PD_REGISTER_KERNEL_FOR_ALL_DTYPE(
+    strings_upper,
+    GPU,
+    ALL_LAYOUT,
+    phi::strings::StringUpperKernel<phi::GPUContext>) {}