chore: import upstream snapshot with attribution

paddle/phi/api/profiler/CMakeLists.txt

@@ -0,0 +1,29 @@
+proto_library(phi_profiler_proto SRCS profiler.proto)
+
+if(WITH_PYTHON AND EXISTS ${PADDLE_BINARY_DIR})
+  set(FLUID_PATH ${PADDLE_BINARY_DIR}/python/paddle/base)
+  py_proto_compile(profiler_py_proto SRCS profiler.proto)
+  file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/__init__.py)
+  if(NOT WIN32)
+    add_custom_command(
+      TARGET profiler_py_proto
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLUID_PATH}/proto/profiler
+      COMMAND cp *.py ${FLUID_PATH}/proto/profiler
+      COMMENT
+        "Copy generated python proto into directory paddle/fluid/proto/profiler."
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  else()
+    string(REPLACE "/" "\\" proto_dstpath "${FLUID_PATH}/proto/profiler/")
+    add_custom_command(
+      TARGET profiler_py_proto
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E make_directory ${FLUID_PATH}/proto/profiler
+      COMMAND copy /Y *.py ${proto_dstpath}
+      COMMENT
+        "Copy generated python proto into directory paddle/fluid/proto/profiler."
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  endif()
+endif()
+
+collect_srcs(api_srcs SRCS device_tracer.cc event.cc profiler.cc)

paddle/phi/api/profiler/common_event.h

@@ -0,0 +1,135 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <cstring>
+#include <functional>
+#include <string>
+
+#include "paddle/common/ddim.h"
+#include "paddle/phi/api/profiler/event.h"  // import EventRole, TODO(TIEXING): remove later
+#include "paddle/phi/api/profiler/trace_event.h"
+#include "paddle/phi/core/attribute.h"
+
+namespace phi {
+
+struct CommonEvent {
+ public:
+  CommonEvent(const char *name,
+              uint64_t start_ns,
+              uint64_t end_ns,
+              EventRole role,
+              TracerEventType type)
+      : name(name),
+        start_ns(start_ns),
+        end_ns(end_ns),
+        role(role),
+        type(type) {}
+
+  CommonEvent(std::function<void *(size_t)> arena_allocator,
+              const std::string &name_str,
+              uint64_t start_ns,
+              uint64_t end_ns,
+              EventRole role,
+              TracerEventType type,
+              const std::string &attr_str)
+      : start_ns(start_ns), end_ns(end_ns), role(role), type(type) {
+    auto buf = static_cast<char *>(arena_allocator(name_str.length() + 1));
+    strncpy(buf, name_str.c_str(), name_str.length() + 1);
+    name = buf;
+    buf = static_cast<char *>(arena_allocator(attr_str.length() + 1));
+    strncpy(buf, attr_str.c_str(), attr_str.length() + 1);
+    attr = buf;
+  }
+
+  CommonEvent(std::function<void *(size_t)> arena_allocator,
+              const std::string &name_str,
+              uint64_t start_ns,
+              uint64_t end_ns,
+              EventRole role,
+              TracerEventType type)
+      : start_ns(start_ns), end_ns(end_ns), role(role), type(type) {
+    auto buf = static_cast<char *>(arena_allocator(name_str.length() + 1));
+    strncpy(buf, name_str.c_str(), name_str.length() + 1);
+    name = buf;
+  }
+
+  const char *name = nullptr;  // not owned, designed for performance
+  uint64_t start_ns = 0;
+  uint64_t end_ns = 0;
+  EventRole role = EventRole::kOrdinary;
+  TracerEventType type = TracerEventType::NumTypes;
+  const char *attr = nullptr;  // not owned, designed for performance
+};
+
+struct CommonMemEvent {
+ public:
+  CommonMemEvent(uint64_t timestamp_ns,
+                 uint64_t addr,
+                 TracerMemEventType type,
+                 int64_t increase_bytes,
+                 const Place &place,
+                 uint64_t current_allocated,
+                 uint64_t current_reserved,
+                 uint64_t peak_allocated,
+                 uint64_t peak_reserved)
+      : timestamp_ns(timestamp_ns),
+        addr(addr),
+        type(type),
+        increase_bytes(increase_bytes),
+        place(place),
+        current_allocated(current_allocated),
+        current_reserved(current_reserved),
+        peak_allocated(peak_allocated),
+        peak_reserved(peak_reserved) {}
+  uint64_t timestamp_ns;
+  uint64_t addr;
+  TracerMemEventType type;
+  int64_t increase_bytes;
+  Place place;
+  uint64_t current_allocated;
+  uint64_t current_reserved;
+  uint64_t peak_allocated;
+  uint64_t peak_reserved;
+};
+
+struct OperatorSupplementOriginEvent {
+ public:
+  OperatorSupplementOriginEvent(
+      std::function<void *(size_t)> arena_allocator,
+      uint64_t timestamp_ns,
+      const std::string &type_name,
+      const std::vector<std::pair<const char *, std::vector<DDim>>> &shapes,
+      const AttributeMap &attributes,
+      uint64_t op_id)
+      : timestamp_ns(timestamp_ns), attributes(attributes), op_id(op_id) {
+    auto buf = static_cast<char *>(arena_allocator(type_name.length() + 1));
+    strncpy(buf, type_name.c_str(), type_name.length() + 1);
+    op_type = buf;
+    for (auto it = shapes.begin(); it != shapes.end(); it++) {
+      input_shapes[std::string((*it).first)] = (*it).second;
+    }
+  }
+  uint64_t timestamp_ns;
+  const char *op_type = nullptr;  // not owned, designed for performance
+  // input shapes
+  std::map<std::string, std::vector<DDim>> input_shapes;
+  // op attributes
+  AttributeMap attributes;
+  // op id
+  uint64_t op_id;
+};
+
+}  // namespace phi

paddle/phi/api/profiler/device_tracer.cc

@@ -0,0 +1,913 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/api/profiler/device_tracer.h"
+
+#include <deque>
+#include <forward_list>
+#include <fstream>
+#include <mutex>  // NOLINT
+#include <string>
+#include <thread>  // NOLINT
+
+#include "glog/logging.h"
+#include "paddle/common/flags.h"
+#include "paddle/phi/core/enforce.h"
+
+PHI_DECLARE_bool(enable_host_event_recorder_hook);
+
+namespace phi {
+
+// Used only by DeviceTracer
+uint64_t GetThreadIdFromSystemThreadId(uint32_t id);
+
+namespace {
+// Tracking the nested block stacks of each thread.
+#ifdef PADDLE_WITH_SW
+// sw not supported thread_local
+std::deque<int> block_id_stack;
+std::deque<Event *> annotation_stack;
+#else
+// Tracking the nested event stacks.
+thread_local std::deque<int> block_id_stack;
+// Tracking the nested event stacks.
+thread_local std::deque<Event *> annotation_stack;
+#endif
+// stack to store event such as pe and so on
+static std::deque<Event *> main_thread_annotation_stack{};
+static std::deque<std::string> main_thread_annotation_stack_name{};
+
+std::map<uint32_t, uint64_t> system_thread_id_map;
+std::mutex system_thread_id_map_mutex;
+
+std::once_flag tracer_once_flag;
+DeviceTracer *tracer = nullptr;
+
+void PrintCuptiHint() {
+  static bool showed = false;
+  if (showed) return;
+  showed = true;
+  LOG(WARNING) << "Invalid timestamp occurred. Please try increasing the "
+                  "FLAGS_multiple_of_cupti_buffer_size.";
+}
+
+}  // namespace
+#ifdef PADDLE_WITH_CUPTI
+
+namespace {
+// The experimental best performance is
+// the same size with CUPTI device buffer size(8M)
+uint64_t kBufSize = 1024 * 1024 * 8;
+uint64_t kAlignSize = 8;
+std::unordered_map<CUpti_CallbackId, std::string> runtime_cbid_str,
+    driver_cbid_str;
+
+#define ALIGN_BUFFER(buffer, align)                                 \
+  (((uintptr_t)(buffer) & ((align)-1))                              \
+       ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) \
+       : (buffer))
+
+#define CUPTI_CALL(call)                                           \
+  do {                                                             \
+    CUptiResult _status = call;                                    \
+    if (_status != CUPTI_SUCCESS) {                                \
+      const char *errstr;                                          \
+      dynload::cuptiGetResultString(_status, &errstr);             \
+      fprintf(stderr,                                              \
+              "%s:%d: error: function %s failed with error %s.\n", \
+              __FILE__,                                            \
+              __LINE__,                                            \
+              #call,                                               \
+              errstr);                                             \
+      exit(-1);                                                    \
+    }                                                              \
+  } while (0)
+
+std::string MemcpyKind(CUpti_ActivityMemcpyKind kind) {
+  switch (kind) {
+    case CUPTI_ACTIVITY_MEMCPY_KIND_HTOD:
+      return "MEMCPY_HtoD";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_DTOH:
+      return "MEMCPY_DtoH";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_HTOA:
+      return "MEMCPY_HtoA";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_ATOH:
+      return "MEMCPY_AtoH";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_ATOA:
+      return "MEMCPY_AtoA";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_ATOD:
+      return "MEMCPY_AtoD";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_DTOA:
+      return "MEMCPY_DtoA";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_DTOD:
+      return "MEMCPY_DtoD";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_HTOH:
+      return "MEMCPY_HtoH";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_PTOP:
+      return "MEMCPY_PtoP";
+    case CUPTI_ACTIVITY_MEMCPY_KIND_FORCE_INT:
+      return "MEMCPY_FORCE_INT";
+    default:
+      break;
+  }
+  return "MEMCPY";
+}
+
+std::string DriverKind(CUpti_CallbackId cbid) {
+  auto iter = driver_cbid_str.find(cbid);
+  if (iter == driver_cbid_str.end())
+    return "Driver API " + std::to_string(cbid);
+  return iter->second;
+}
+
+std::string RuntimeKind(CUpti_CallbackId cbid) {
+  auto iter = runtime_cbid_str.find(cbid);
+  if (iter == runtime_cbid_str.end())
+    return "Runtime API " + std::to_string(cbid);
+  return iter->second;
+}
+
+void EnableActivity() {
+  // Device activity record is created when CUDA initializes, so we
+  // want to enable it before cuInit() or any CUDA runtime call.
+  CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY));
+  CUPTI_CALL(
+      dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
+  // CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_KERNEL));
+  CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DRIVER));
+  CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME));
+  // We don't track these activities for now.
+  CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMSET));
+  // CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_OVERHEAD));
+  // CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DEVICE));
+  // CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONTEXT));
+  // CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_DRIVER));
+  // CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME));
+  // CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_NAME));
+  // CUPTI_CALL(dynload::cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MARKER));
+}
+
+void DisableActivity() {
+  CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY));
+  CUPTI_CALL(
+      dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
+  // CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_DEVICE));
+  // Disable all other activity record kinds.
+  // CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONTEXT));
+  CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_DRIVER));
+  CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME));
+  CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMSET));
+  // CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_NAME));
+  // CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MARKER));
+  // CUPTI_CALL(dynload::cuptiActivityDisable(CUPTI_ACTIVITY_KIND_OVERHEAD));
+}
+
+void CUPTIAPI bufferRequested(uint8_t **buffer,
+                              size_t *size,
+                              size_t *maxNumRecords) {
+  uint8_t *buf = reinterpret_cast<uint8_t *>(malloc(kBufSize + kAlignSize));
+  *size = kBufSize;
+  *buffer = ALIGN_BUFFER(buf, kAlignSize);
+  *maxNumRecords = 0;
+}
+
+void CUPTIAPI bufferCompleted(CUcontext ctx,
+                              uint32_t streamId,
+                              uint8_t *buffer,
+                              size_t size,
+                              size_t validSize) {
+  static std::thread::id cupti_thread_id(0);
+  if (cupti_thread_id == std::thread::id(0))
+    cupti_thread_id = std::this_thread::get_id();
+  PADDLE_ENFORCE_EQ(
+      std::this_thread::get_id(),
+      cupti_thread_id,
+      errors::PermissionDenied(
+          "Only one thread is allowed to call bufferCompleted()."));
+  CUptiResult status;
+  CUpti_Activity *record = nullptr;
+  if (validSize > 0) {
+    do {
+      status = dynload::cuptiActivityGetNextRecord(buffer, validSize, &record);
+      if (status == CUPTI_SUCCESS) {
+        switch (record->kind) {
+          case CUPTI_ACTIVITY_KIND_KERNEL:
+          case CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL: {
+#if CUDA_VERSION >= 9000 || defined(PADDLE_WITH_XPU)
+            auto *kernel =
+                reinterpret_cast<const CUpti_ActivityKernel4 *>(record);
+#else
+            auto *kernel =
+                reinterpret_cast<const CUpti_ActivityKernel3 *>(record);
+#endif
+            tracer->AddKernelRecords(kernel->name,
+                                     kernel->start,
+                                     kernel->end,
+                                     kernel->deviceId,
+                                     kernel->streamId,
+                                     kernel->correlationId);
+            break;
+          }
+          case CUPTI_ACTIVITY_KIND_MEMCPY: {
+            auto *memcpy =
+                reinterpret_cast<const CUpti_ActivityMemcpy *>(record);
+            tracer->AddMemRecords(
+                MemcpyKind(
+                    static_cast<CUpti_ActivityMemcpyKind>(memcpy->copyKind)),
+                memcpy->start,
+                memcpy->end,
+                memcpy->deviceId,
+                memcpy->streamId,
+                memcpy->correlationId,
+                memcpy->bytes);
+            break;
+          }
+          case CUPTI_ACTIVITY_KIND_MEMCPY2: {
+            auto *memcpy =
+                reinterpret_cast<const CUpti_ActivityMemcpy2 *>(record);
+            tracer->AddMemRecords(
+                MemcpyKind(
+                    static_cast<CUpti_ActivityMemcpyKind>(memcpy->copyKind)),
+                memcpy->start,
+                memcpy->end,
+                memcpy->deviceId,
+                memcpy->streamId,
+                memcpy->correlationId,
+                memcpy->bytes);
+            break;
+          }
+          case CUPTI_ACTIVITY_KIND_MEMSET: {
+            auto *memset =
+                reinterpret_cast<const CUpti_ActivityMemset *>(record);
+            tracer->AddKernelRecords("MEMSET",
+                                     memset->start,
+                                     memset->end,
+                                     memset->deviceId,
+                                     memset->streamId,
+                                     memset->correlationId);
+            break;
+          }
+          case CUPTI_ACTIVITY_KIND_DRIVER: {
+            auto *api = reinterpret_cast<const CUpti_ActivityAPI *>(record);
+            if (api->start != 0 && api->end != 0) {
+              // -1 device id represents ActiveKind api call
+              tracer->AddActiveKindRecords(
+                  DriverKind(api->cbid),
+                  api->start,
+                  api->end,
+                  -1,
+                  GetThreadIdFromSystemThreadId(api->threadId),
+                  api->correlationId);
+            }
+            break;
+          }
+          case CUPTI_ACTIVITY_KIND_RUNTIME: {
+            auto *api = reinterpret_cast<const CUpti_ActivityAPI *>(record);
+            if (api->start != 0 && api->end != 0) {
+              // -1 device id represents ActiveKind api call
+              tracer->AddActiveKindRecords(
+                  RuntimeKind(api->cbid),
+                  api->start,
+                  api->end,
+                  -1,
+                  GetThreadIdFromSystemThreadId(api->threadId),
+                  api->correlationId);
+            }
+            break;
+          }
+          default: {
+            break;
+          }
+        }
+      } else if (status == CUPTI_ERROR_MAX_LIMIT_REACHED) {
+        // Seems not an error in this case.
+        break;
+      } else {
+        CUPTI_CALL(status);
+      }
+    } while (true);
+
+    size_t dropped;
+    CUPTI_CALL(
+        dynload::cuptiActivityGetNumDroppedRecords(ctx, streamId, &dropped));
+    if (dropped != 0) {
+      fprintf(stderr, "Dropped %u activity records\n", (unsigned int)dropped);
+      PrintCuptiHint();
+    }
+  }
+  free(buffer);
+}
+
+void initCuptiCbidStr();
+
+}  // namespace
+
+#endif  // PADDLE_WITH_CUPTI
+
+class DeviceTracerImpl : public DeviceTracer {
+ public:
+  DeviceTracerImpl() : enabled_(false), start_ns_(0), end_ns_(0) {
+#ifdef PADDLE_WITH_CUPTI
+    initCuptiCbidStr();
+#endif
+  }
+
+  void AddAnnotation(uint32_t id, Event *event) override {
+#ifdef PADDLE_WITH_SW
+    std::forward_list<std::pair<uint32_t, Event *>> *local_correlations_pairs =
+        nullptr;
+#else
+    thread_local std::forward_list<std::pair<uint32_t, Event *>>
+        *local_correlations_pairs = nullptr;
+#endif
+    if (local_correlations_pairs == nullptr) {
+      std::lock_guard<std::mutex> l(trace_mu_);
+      correlations_pairs.emplace_front();
+      local_correlations_pairs = &correlations_pairs.front();
+    }
+    local_correlations_pairs->push_front(std::make_pair(id, event));
+  }
+
+  void AddAnnotations(
+      const std::map<uint64_t, ThreadEvents> &thr_events) override {
+    for (auto &tmp : active_kind_records_) {
+      for (const ActiveKindRecord &r : tmp) {
+        auto iter = thr_events.find(r.thread_id);
+        if (iter == thr_events.end()) {
+          VLOG(10) << __func__ << " " << r.name
+                   << " Missing tid: " << r.thread_id;
+          continue;
+        }
+        const ThreadEvents &evts = iter->second;
+        auto evt_iter = evts.upper_bound(r.end_ns);
+        if (evt_iter == evts.end()) {
+          VLOG(10) << __func__ << " Missing Record " << r.name
+                   << " tid: " << r.thread_id << " end_ns: " << r.end_ns;
+          continue;
+        }
+        if (evt_iter != evts.begin()) {
+          auto prev_iter = std::prev(evt_iter);
+          if (prev_iter->first >= r.end_ns) {
+            evt_iter = prev_iter;
+          } else {
+            VLOG(10) << __func__ << " prev end_ns " << prev_iter->first
+                     << " end_ns: " << r.end_ns;
+          }
+        }
+        Event *evt = evt_iter->second.first;
+        uint64_t start_ns = evt_iter->second.second;
+        if (start_ns > r.start_ns) {
+          VLOG(10) << __func__ << " Mismatch Record " << r.name
+                   << " tid: " << r.thread_id << " start_ns: " << r.start_ns
+                   << " end_ns: " << r.end_ns << ", event " << evt->name()
+                   << " start_ns: " << start_ns;
+          continue;
+        }
+        VLOG(10) << __func__ << " tid: " << r.thread_id << " Add correlation "
+                 << r.correlation_id << "<->" << evt->name();
+        AddAnnotation(r.correlation_id, evt);
+      }
+    }
+  }
+
+  void AddCPURecords(const std::string &anno,
+                     uint64_t start_ns,
+                     uint64_t end_ns,
+                     int64_t device_id,
+                     uint64_t thread_id) override {
+    if (anno.empty()) {
+      VLOG(1) << "Empty timeline annotation.";
+      return;
+    }
+#ifdef PADDLE_WITH_SW
+    std::forward_list<CPURecord> *local_cpu_records_ = nullptr;
+#else
+    thread_local std::forward_list<CPURecord> *local_cpu_records_ = nullptr;
+#endif
+    if (local_cpu_records_ == nullptr) {
+      std::lock_guard<std::mutex> l(trace_mu_);
+      cpu_records_.emplace_front();
+      local_cpu_records_ = &cpu_records_.front();
+    }
+    local_cpu_records_->push_front(
+        CPURecord{anno, start_ns, end_ns, device_id, thread_id});
+  }
+
+  void AddMemRecords(const std::string &name,
+                     uint64_t start_ns,
+                     uint64_t end_ns,
+                     int64_t device_id,
+                     int64_t stream_id,
+                     uint32_t correlation_id,
+                     uint64_t bytes) override {
+    // 0 means timestamp information could not be collected for the kernel.
+    if (start_ns == 0 || end_ns == 0 || start_ns == end_ns) {
+      VLOG(3) << name << " cannot be traced";
+      PrintCuptiHint();
+      return;
+    }
+    // NOTE(liangdun): lock is not needed, only one thread call this function.
+    mem_records_.push_front(MemRecord{
+        name, start_ns, end_ns, device_id, stream_id, correlation_id, bytes});
+  }
+
+  void AddMemInfoRecord(uint64_t start_ns,
+                        uint64_t end_ns,
+                        size_t bytes,
+                        const Place &place,
+                        const std::string &alloc_in,
+                        const std::string &free_in,
+                        uint64_t thread_id) override {
+    if (0 == start_ns || 0 == end_ns) {
+      VLOG(3) << alloc_in << ", " << free_in << " Cannot be traced.";
+      return;
+    }
+#ifdef PADDLE_WITH_SW
+    std::forward_list<MemInfoRecord> *local_mem_info_record = nullptr;
+#else
+    thread_local std::forward_list<MemInfoRecord> *local_mem_info_record =
+        nullptr;
+#endif
+    if (local_mem_info_record == nullptr) {
+      std::lock_guard<std::mutex> l(trace_mu_);
+      mem_info_record_.emplace_front();
+      local_mem_info_record = &mem_info_record_.front();
+    }
+    local_mem_info_record->emplace_front(MemInfoRecord{
+        start_ns, end_ns, bytes, place, thread_id, alloc_in, free_in});
+  }
+
+  void AddActiveKindRecords(const std::string &anno,
+                            uint64_t start_ns,
+                            uint64_t end_ns,
+                            int64_t device_id,
+                            uint64_t thread_id,
+                            uint32_t correlation_id) override {
+    if (anno.empty()) {
+      VLOG(1) << "Empty timeline annotation.";
+      return;
+    }
+#ifdef PADDLE_WITH_SW
+    std::forward_list<ActiveKindRecord> *local_active_kind_records = nullptr;
+#else
+    thread_local std::forward_list<ActiveKindRecord>
+        *local_active_kind_records = nullptr;
+#endif
+    if (local_active_kind_records == nullptr) {
+      std::lock_guard<std::mutex> l(trace_mu_);
+      active_kind_records_.emplace_front();
+      local_active_kind_records = &active_kind_records_.front();
+    }
+    //  lock is not needed, only one thread call this function.
+    local_active_kind_records->push_front(ActiveKindRecord{
+        anno, start_ns, end_ns, device_id, thread_id, correlation_id});
+  }
+
+  void AddKernelRecords(std::string name,
+                        uint64_t start,
+                        uint64_t end,
+                        int64_t device_id,
+                        int64_t stream_id,
+                        uint32_t correlation_id) override {
+    // 0 means timestamp information could not be collected for the kernel.
+    if (start == 0 || end == 0 || start == end) {
+      VLOG(3) << correlation_id << " cannot be traced";
+      PrintCuptiHint();
+      return;
+    }
+    // NOTE(liangdun): lock is not needed, only one thread call this function.
+    kernel_records_.push_front(
+        KernelRecord{name, start, end, device_id, stream_id, correlation_id});
+  }
+
+  bool IsEnabled() override {
+    std::lock_guard<std::mutex> l(trace_mu_);
+    return enabled_;
+  }
+
+  void Enable() override {
+    std::lock_guard<std::mutex> l(trace_mu_);
+    if (enabled_) {
+      return;
+    }
+
+#ifdef PADDLE_WITH_CUPTI
+    EnableActivity();
+
+    // Register callbacks for buffer requests and completed by CUPTI.
+    CUPTI_CALL(dynload::cuptiActivityRegisterCallbacks(bufferRequested,
+                                                       bufferCompleted));
+
+    CUptiResult ret;
+    ret = dynload::cuptiSubscribe(
+        &subscriber_, static_cast<CUpti_CallbackFunc>(ApiCallback), this);
+    if (ret == CUPTI_ERROR_MAX_LIMIT_REACHED) {
+      fprintf(stderr, "CUPTI subscriber limit reached.\n");
+    } else if (ret != CUPTI_SUCCESS) {
+      fprintf(stderr, "Failed to create CUPTI subscriber.\n");
+    }
+    const std::vector<int> runtime_cbids {
+      CUPTI_RUNTIME_TRACE_CBID_cudaMemcpy_v3020,
+          CUPTI_RUNTIME_TRACE_CBID_cudaSetupArgument_v3020,
+          CUPTI_RUNTIME_TRACE_CBID_cudaMemcpyAsync_v3020,
+          CUPTI_RUNTIME_TRACE_CBID_cudaMemset_v3020,
+          CUPTI_RUNTIME_TRACE_CBID_cudaMemsetAsync_v3020,
+          CUPTI_RUNTIME_TRACE_CBID_cudaLaunch_v3020,
+          CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000
+#if CUDA_VERSION >= 9000 || defined(PADDLE_WITH_XPU)
+          ,
+          CUPTI_RUNTIME_TRACE_CBID_cudaLaunchCooperativeKernel_v9000,
+          CUPTI_RUNTIME_TRACE_CBID_cudaLaunchCooperativeKernelMultiDevice_v9000
+#endif
+    };
+    const std::vector<int> driver_cbids{CUPTI_DRIVER_TRACE_CBID_cuLaunch,
+                                        CUPTI_DRIVER_TRACE_CBID_cuLaunchGrid,
+                                        CUPTI_DRIVER_TRACE_CBID_cuLaunchKernel};
+    for (auto cbid : runtime_cbids)
+      CUPTI_CALL(dynload::cuptiEnableCallback(
+          1, subscriber_, CUPTI_CB_DOMAIN_RUNTIME_API, cbid));
+    for (auto cbid : driver_cbids)
+      CUPTI_CALL(dynload::cuptiEnableCallback(
+          1, subscriber_, CUPTI_CB_DOMAIN_DRIVER_API, cbid));
+    CUPTI_CALL(dynload::cuptiGetTimestamp(&start_ns_));
+#endif  // PADDLE_WITH_CUPTI
+    enabled_ = true;
+  }
+
+  void Reset() override {
+#ifdef PADDLE_WITH_CUPTI
+    CUPTI_CALL(
+        dynload::cuptiActivityFlushAll(CUPTI_ACTIVITY_FLAG_FLUSH_FORCED));
+#endif
+    std::lock_guard<std::mutex> l(trace_mu_);
+    kernel_records_.clear();
+    mem_records_.clear();
+    correlations_.clear();
+    for (auto &tmp : correlations_pairs) tmp.clear();
+    for (auto &tmp : cpu_records_) tmp.clear();
+    for (auto &tmp : mem_info_record_) tmp.clear();
+    for (auto &tmp : active_kind_records_) tmp.clear();
+  }
+
+  void GenEventKernelCudaElapsedTime() override {
+#ifdef PADDLE_WITH_CUPTI
+    if (correlations_.empty())
+      for (auto &tmp : correlations_pairs)
+        for (auto &pair : tmp) correlations_[pair.first] = pair.second;
+    for (const KernelRecord &r : kernel_records_) {
+      auto c = correlations_.find(r.correlation_id);
+      if (c != correlations_.end() && c->second != nullptr) {
+        Event *e = c->second;
+        Event *parent = e->parent();
+        while (parent) {
+          parent->AddCudaElapsedTime(r.start_ns, r.end_ns);  // NOLINT
+          parent = parent->parent();
+        }
+        e->AddCudaElapsedTime(r.start_ns, r.end_ns);  // NOLINT
+      }
+    }
+    for (const auto &r : mem_records_) {
+      auto c = correlations_.find(r.correlation_id);
+      if (c != correlations_.end() && c->second != nullptr) {
+        Event *e = c->second;
+        Event *parent = e->parent();
+        while (parent) {
+          parent->AddCudaElapsedTime(r.start_ns, r.end_ns);  // NOLINT
+          parent = parent->parent();
+        }
+        e->AddCudaElapsedTime(r.start_ns, r.end_ns);  // NOLINT
+      }
+    }
+#endif
+  }
+
+  proto::Profile GenProfile(const std::string &profile_path) override {
+    proto::Profile profile_pb = this->GetProfile();
+    std::ofstream profile_f;
+    profile_f.open(profile_path,
+                   std::ios::out | std::ios::trunc | std::ios::binary);
+    profile_pb.SerializeToOstream(&profile_f);
+    profile_f.close();
+    return profile_pb;
+  }
+
+  proto::Profile GetProfile() override {
+    int miss = 0, find = 0;
+    std::lock_guard<std::mutex> l(trace_mu_);
+    proto::Profile profile_pb;
+    profile_pb.set_start_ns(start_ns_);
+    profile_pb.set_end_ns(end_ns_);
+    if (correlations_.empty()) {
+      for (auto &tmp : correlations_pairs) {
+        for (auto &pair : tmp) correlations_[pair.first] = pair.second;
+      }
+    }
+
+    for (const KernelRecord &r : kernel_records_) {
+      auto *event = profile_pb.add_events();
+      event->set_type(proto::Event::GPUKernel);
+      auto c = correlations_.find(r.correlation_id);
+      if (c != correlations_.end() && c->second != nullptr) {
+        event->set_name(c->second->name());
+        event->set_detail_info(c->second->attr());
+        find++;
+      } else {
+        VLOG(10) << __func__ << " Missing Kernel Event: " + r.name;
+        miss++;
+        event->set_name(r.name);
+      }
+      event->set_start_ns(r.start_ns);
+      event->set_end_ns(r.end_ns);
+      event->set_sub_device_id(r.stream_id);
+      event->set_device_id(r.device_id);
+    }
+    VLOG(1) << __func__ << " KernelRecord event miss: " << miss
+            << " find: " << find;
+
+    for (auto &tmp : cpu_records_) {
+      for (const CPURecord &r : tmp) {
+        auto *event = profile_pb.add_events();
+        event->set_type(proto::Event::CPU);
+        event->set_name(r.name);
+        event->set_start_ns(r.start_ns);
+        event->set_end_ns(r.end_ns);
+        event->set_sub_device_id(r.thread_id);
+        event->set_device_id(r.device_id);
+      }
+    }
+
+    for (auto &tmp : active_kind_records_) {
+      for (const ActiveKindRecord &r : tmp) {
+        auto *event = profile_pb.add_events();
+        event->set_type(proto::Event::CPU);
+        auto c = correlations_.find(r.correlation_id);
+        if (c != correlations_.end() && c->second != nullptr) {
+          event->set_name(c->second->name());
+          event->set_detail_info(r.name);
+        } else {
+          event->set_name(r.name);
+        }
+        event->set_start_ns(r.start_ns);
+        event->set_end_ns(r.end_ns);
+        event->set_sub_device_id(r.thread_id);
+        event->set_device_id(r.device_id);
+      }
+    }
+    miss = find = 0;
+    for (const MemRecord &r : mem_records_) {
+      auto *event = profile_pb.add_events();
+      event->set_type(proto::Event::GPUKernel);
+      auto c = correlations_.find(r.correlation_id);
+      if (c != correlations_.end() && c->second != nullptr) {
+        event->set_name(c->second->name());
+        event->set_detail_info(r.name);
+        find++;
+      } else {
+        miss++;
+        event->set_name(r.name);
+      }
+      event->set_start_ns(r.start_ns);
+      event->set_end_ns(r.end_ns);
+      event->set_sub_device_id(r.stream_id);
+      event->set_device_id(r.device_id);
+      event->mutable_memcopy()->set_bytes(r.bytes);
+    }
+    VLOG(1) << __func__ << " MemRecord event miss: " << miss
+            << " find: " << find;
+
+    for (auto &tmp : mem_info_record_) {
+      for (const auto &r : tmp) {
+        auto *event = profile_pb.add_mem_events();
+        event->set_device_id(0);
+        if (r.place.GetType() == phi::AllocationType::CPU) {
+          event->set_place(proto::MemEvent::CPUPlace);
+        } else if (r.place.GetType() == phi::AllocationType::GPU) {
+          event->set_place(proto::MemEvent::CUDAPlace);
+          event->set_device_id(r.place.GetDeviceId());
+        } else if (r.place.GetType() == phi::AllocationType::GPUPINNED) {
+          event->set_place(proto::MemEvent::CUDAPinnedPlace);
+        } else {
+          PADDLE_THROW(
+              errors::Unimplemented("The current place is not supported."));
+        }
+        event->set_alloc_in(r.alloc_in);
+        event->set_free_in(r.free_in);
+        event->set_start_ns(r.start_ns);
+        event->set_end_ns(r.end_ns);
+        event->set_bytes(r.bytes);
+        event->set_thread_id(r.thread_id);
+      }
+    }
+    return profile_pb;
+  }
+
+  void Disable() override {
+#ifdef PADDLE_WITH_CUPTI
+    // flush might cause additional calls to DeviceTracker.
+    CUPTI_CALL(
+        dynload::cuptiActivityFlushAll(CUPTI_ACTIVITY_FLAG_FLUSH_FORCED));
+#endif  // PADDLE_WITH_CUPTI
+    std::lock_guard<std::mutex> l(trace_mu_);
+#ifdef PADDLE_WITH_CUPTI
+    DisableActivity();
+    CUPTI_CALL(dynload::cuptiUnsubscribe(subscriber_));
+    CUPTI_CALL(dynload::cuptiGetTimestamp(&end_ns_));
+#endif  // PADDLE_WITH_CUPTI
+    enabled_ = false;
+  }
+
+ private:
+#ifdef PADDLE_WITH_CUPTI
+  static void CUPTIAPI ApiCallback(void *userdata,
+                                   CUpti_CallbackDomain domain,
+                                   CUpti_CallbackId cbid,
+                                   const void *cbdata) {
+    if (LIKELY(FLAGS_enable_host_event_recorder_hook)) {
+      return;
+    }
+    auto *cbInfo = reinterpret_cast<const CUpti_CallbackData *>(cbdata);
+    DeviceTracerImpl *tracer = reinterpret_cast<DeviceTracerImpl *>(userdata);
+    if (cbInfo->callbackSite == CUPTI_API_ENTER) {
+      Event *event = CurAnnotation();
+      tracer->AddAnnotation(cbInfo->correlationId, event);
+    }
+  }
+  CUpti_SubscriberHandle subscriber_;
+#endif  // PADDLE_WITH_CUPTI
+  std::mutex trace_mu_;
+  bool enabled_;
+  uint64_t start_ns_;
+  uint64_t end_ns_;
+  std::forward_list<KernelRecord> kernel_records_;
+  std::forward_list<MemRecord> mem_records_;
+  std::forward_list<std::forward_list<CPURecord>> cpu_records_;
+  std::forward_list<std::forward_list<MemInfoRecord>> mem_info_record_;
+  std::forward_list<std::forward_list<ActiveKindRecord>> active_kind_records_;
+  std::forward_list<std::forward_list<std::pair<uint32_t, Event *>>>
+      correlations_pairs;
+  std::unordered_map<uint32_t, Event *> correlations_;
+};
+
+void CreateTracer(DeviceTracer **t) { *t = new DeviceTracerImpl(); }
+
+DeviceTracer *GetDeviceTracer() {
+  std::call_once(tracer_once_flag, CreateTracer, &tracer);
+  return tracer;
+}
+
+// In order to record PE time, we add main_thread_annotation_stack
+// for all event between PE run, we treat it as PE's child Event,
+// so when event is not in same thread of PE event, we need add
+// father event(PE::run event) for this event
+void SetCurAnnotation(Event *event) {
+  if (!annotation_stack.empty()) {
+    event->set_parent(annotation_stack.back());
+    event->set_name(annotation_stack.back()->name() + "/" + event->name());
+  }
+  if (annotation_stack.empty() && !main_thread_annotation_stack.empty() &&
+      main_thread_annotation_stack.back()->thread_id() != event->thread_id()) {
+    event->set_parent(main_thread_annotation_stack.back());
+    event->set_name(main_thread_annotation_stack.back()->name() + "/" +
+                    event->name());
+  }
+  annotation_stack.push_back(event);
+
+  if (event->role() == EventRole::kSpecial) {
+    std::string name = event->name();
+    if (!main_thread_annotation_stack_name.empty()) {
+      name = main_thread_annotation_stack_name.back() + "/" + event->name();
+    }
+    main_thread_annotation_stack_name.push_back(name);
+    main_thread_annotation_stack.push_back(event);
+  }
+}
+
+void ClearCurAnnotation() {
+  if (!main_thread_annotation_stack.empty()) {
+    std::string name = annotation_stack.back()->name();
+    std::string main_name = main_thread_annotation_stack.back()->name();
+    int main_name_len = static_cast<int>(main_name.length());
+    int name_len = static_cast<int>(name.length());
+    int prefix_len = main_name_len - name_len;
+
+    if ((prefix_len > 0 && main_name.at(prefix_len - 1) == '/' &&
+         name == main_name.substr(prefix_len, name_len)) ||
+        (name == main_name)) {
+      main_thread_annotation_stack_name.pop_back();
+      main_thread_annotation_stack.pop_back();
+    }
+  }
+  annotation_stack.pop_back();
+}
+
+Event *CurAnnotation() {
+  if (annotation_stack.empty()) return nullptr;
+  return annotation_stack.back();
+}
+
+std::string CurAnnotationName() {
+  if (annotation_stack.empty()) return "Unknown";
+  return annotation_stack.back()->name();
+}
+
+void SetCurBlock(int block_id) { block_id_stack.push_back(block_id); }
+
+void ClearCurBlock() { block_id_stack.pop_back(); }
+
+int BlockDepth() { return static_cast<int>(block_id_stack.size()); }
+
+uint32_t GetCurSystemThreadId() {
+  std::stringstream ss;
+  ss << std::this_thread::get_id();
+  uint32_t id = static_cast<uint32_t>(std::stoull(ss.str()));
+  return id;
+}
+
+void RecordCurThreadId(uint64_t id) {
+  std::lock_guard<std::mutex> lock(system_thread_id_map_mutex);
+  auto gid = GetCurSystemThreadId();
+  system_thread_id_map[gid] = id;
+}
+
+uint64_t GetThreadIdFromSystemThreadId(uint32_t id) {
+  auto it = system_thread_id_map.find(id);
+  if (it != system_thread_id_map.end()) return it->second;
+  // return origin id if no event is recorded in this thread.
+  return static_cast<int32_t>(id);
+}
+
+#ifdef PADDLE_WITH_CUPTI
+namespace {
+
+void initCuptiCbidStr() {
+  static bool called = false;
+  if (called) return;
+  called = true;
+#define REGISTER_RUNTIME_CBID_STR(cbid) \
+  runtime_cbid_str[CUPTI_RUNTIME_TRACE_CBID_##cbid] = #cbid
+
+  REGISTER_RUNTIME_CBID_STR(cudaBindTexture_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaConfigureCall_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaDeviceGetAttribute_v5000);
+  REGISTER_RUNTIME_CBID_STR(cudaDeviceGetStreamPriorityRange_v5050);
+  REGISTER_RUNTIME_CBID_STR(cudaDeviceSynchronize_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaDriverGetVersion_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaEventCreateWithFlags_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaEventDestroy_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaEventDestroy_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaEventQuery_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaEventRecord_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaFreeHost_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaFree_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaFuncGetAttributes_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaGetDeviceCount_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaGetDeviceProperties_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaGetDevice_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaGetErrorString_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaGetLastError_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaHostAlloc_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaHostGetDevicePointer_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaLaunchKernel_v7000);
+  REGISTER_RUNTIME_CBID_STR(cudaMallocHost_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaMalloc_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaMemcpyAsync_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaMemcpy_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaMemsetAsync_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaMemset_v3020);
+  REGISTER_RUNTIME_CBID_STR(
+      cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags_v7000);
+  REGISTER_RUNTIME_CBID_STR(cudaPeekAtLastError_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaRuntimeGetVersion_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaSetDevice_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaStreamCreate_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaStreamCreateWithFlags_v5000);
+  REGISTER_RUNTIME_CBID_STR(cudaStreamCreateWithPriority_v5050);
+  REGISTER_RUNTIME_CBID_STR(cudaStreamDestroy_v5050);
+  REGISTER_RUNTIME_CBID_STR(cudaStreamSynchronize_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaStreamWaitEvent_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaUnbindTexture_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaSetupArgument_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaLaunch_v3020);
+  REGISTER_RUNTIME_CBID_STR(cudaDeviceGetPCIBusId_v4010);
+#if CUDA_VERSION >= 9000 || defined(PADDLE_WITH_XPU)
+  REGISTER_RUNTIME_CBID_STR(cudaLaunchCooperativeKernel_v9000);
+  REGISTER_RUNTIME_CBID_STR(cudaLaunchCooperativeKernelMultiDevice_v9000);
+#endif
+
+#undef REGISTER_RUNTIME_CBID_STR
+}
+}  // namespace
+#endif  // PADDLE_WITH_CUPTI
+
+}  // namespace phi

paddle/phi/api/profiler/device_tracer.h

@@ -0,0 +1,166 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#pragma once
+
+#include <chrono>  // NOLINT
+#include <string>
+
+#include "paddle/phi/api/profiler/event.h"
+#include "paddle/phi/api/profiler/profiler.pb.h"
+#include "paddle/phi/backends/dynload/cupti.h"
+#include "paddle/phi/common/place.h"
+#include "paddle/phi/core/os_info.h"
+
+namespace phi {
+
+///////////////////////
+// WARN: Under Development. Don't depend on it yet.
+//////////////////////
+class Event;
+
+// DeviceTracer performs the following tasks:
+// 1. Register cuda callbacks for various events: kernel, memcpy, etc.
+// 2. Collect cuda statistics: start/end ts, memory, etc.
+// 3. Generate a protobuf for further analysis.
+class DeviceTracer {
+ public:
+  struct KernelRecord {
+    std::string name;
+    uint64_t start_ns;
+    uint64_t end_ns;
+    int64_t device_id;
+    int64_t stream_id;
+    uint32_t correlation_id;
+  };
+
+  struct CPURecord {
+    std::string name;
+    uint64_t start_ns;
+    uint64_t end_ns;
+    int64_t device_id;
+    uint64_t thread_id;
+  };
+
+  struct MemRecord {
+    std::string name;
+    uint64_t start_ns;
+    uint64_t end_ns;
+    int64_t device_id;
+    int64_t stream_id;
+    uint32_t correlation_id;
+    uint64_t bytes;
+  };
+
+  struct MemInfoRecord {
+    uint64_t start_ns;
+    uint64_t end_ns;
+    size_t bytes;
+    Place place;
+    uint64_t thread_id;
+    std::string alloc_in;
+    std::string free_in;
+  };
+
+  struct ActiveKindRecord {
+    std::string name;
+    uint64_t start_ns;
+    uint64_t end_ns;
+    int64_t device_id;
+    uint64_t thread_id;
+    uint32_t correlation_id;
+  };
+
+  virtual ~DeviceTracer() {}
+  // Needs to be called once before use.
+  virtual void Enable() = 0;
+  // Needs to be called once after use.
+  virtual void Disable() = 0;
+  // Needs to be called once before reuse.
+  virtual void Reset() = 0;
+
+  // Add a pair to correlate internal cuda id with high level
+  // annotation event(with string). So cuda statistics can be represented by
+  // human-readable annotations.
+  virtual void AddAnnotation(uint32_t id, Event* event) = 0;
+
+  virtual void AddAnnotations(
+      const std::map<uint64_t, ThreadEvents>& thr_events) = 0;
+
+  virtual void AddMemRecords(const std::string& name,
+                             uint64_t start_ns,
+                             uint64_t end_ns,
+                             int64_t device_id,
+                             int64_t stream_id,
+                             uint32_t correlation_id,
+                             uint64_t bytes) = 0;
+
+  virtual void AddCPURecords(const std::string& anno,
+                             uint64_t start_ns,
+                             uint64_t end_ns,
+                             int64_t device_id,
+                             uint64_t thread_id) = 0;
+  virtual void AddActiveKindRecords(const std::string& anno,
+                                    uint64_t start_ns,
+                                    uint64_t end_ns,
+                                    int64_t device_id,
+                                    uint64_t thread_id,
+                                    uint32_t correlation_id) = 0;
+
+  virtual void AddMemInfoRecord(uint64_t start_ns,
+                                uint64_t end_ns,
+                                size_t bytes,
+                                const Place& place,
+                                const std::string& alloc_in,
+                                const std::string& free_in,
+                                uint64_t thread_id) = 0;
+
+  // Add a cuda kernel stats. `correlation_id` will be mapped to annotation
+  // added before for human readability.
+  virtual void AddKernelRecords(std::string name,
+                                uint64_t start,
+                                uint64_t end,
+                                int64_t device_id,
+                                int64_t stream_id,
+                                uint32_t correlation_id) = 0;
+
+  // Get a proto after done
+  virtual proto::Profile GetProfile() = 0;
+
+  // Generate a proto after done (Disabled).
+  virtual proto::Profile GenProfile(const std::string& profile_path) = 0;
+
+  // generate kernel elapsed time into Event
+  virtual void GenEventKernelCudaElapsedTime() = 0;
+
+  virtual bool IsEnabled() = 0;
+};
+
+// Get a DeviceTracer.
+DeviceTracer* GetDeviceTracer();
+
+// Set a name for the cuda kernel operation being launched by the thread.
+void SetCurAnnotation(Event* event);
+// Clear the name after the operation is done.
+void ClearCurAnnotation();
+// Current name of the operation being run in the thread.
+std::string CurAnnotationName();
+Event* CurAnnotation();
+
+void SetCurBlock(int block_id);
+void ClearCurBlock();
+int BlockDepth();
+
+// Set current thread id, so we can map the system thread id to thread id.
+void RecordCurThreadId(uint64_t id);
+}  // namespace phi

paddle/phi/api/profiler/event.cc

@@ -0,0 +1,81 @@
+// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/api/profiler/event.h"
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+#include "glog/logging.h"
+#endif
+
+namespace phi {
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+
+CudaEvent::CudaEvent() {
+#ifdef PADDLE_WITH_HIP
+  hipEventCreateWithFlags(&event_, flags_);
+#else
+  cudaEventCreateWithFlags(&event_, flags_);
+#endif
+  VLOG(4) << "CudaEvent " << event_;
+}
+
+CudaEvent::CudaEvent(unsigned int flags) : flags_(flags) {
+#ifdef PADDLE_WITH_HIP
+  hipEventCreateWithFlags(&event_, flags_);
+#else
+  cudaEventCreateWithFlags(&event_, flags_);
+#endif
+  VLOG(4) << "CudaEvent " << event_;
+}
+
+bool CudaEvent::Query() {
+#ifdef PADDLE_WITH_HIP
+  gpuError_t err = hipEventQuery(event_);
+  if (err == hipSuccess) {
+    return true;
+  }
+  if (err == hipErrorNotReady) {
+    return false;
+  }
+#else
+  gpuError_t err = cudaEventQuery(event_);
+  if (err == cudaSuccess) {
+    return true;
+  }
+  if (err == cudaErrorNotReady) {
+    return false;
+  }
+#endif
+  PADDLE_ENFORCE_GPU_SUCCESS(err);
+  return false;
+}
+
+float CudaEvent::ElapsedTime(CudaEvent *end_event) {
+  float milliseconds = 0;
+#ifdef PADDLE_WITH_HIP
+  hipEventSynchronize(end_event->GetRawCudaEvent());
+  PADDLE_ENFORCE_GPU_SUCCESS(
+      hipEventElapsedTime(&milliseconds, event_, end_event->GetRawCudaEvent()));
+#else
+  cudaEventSynchronize(end_event->GetRawCudaEvent());
+  PADDLE_ENFORCE_GPU_SUCCESS(cudaEventElapsedTime(
+      &milliseconds, event_, end_event->GetRawCudaEvent()));
+#endif
+  return milliseconds;
+}
+
+#endif
+
+}  // namespace phi

paddle/phi/api/profiler/event.h

@@ -0,0 +1,190 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <functional>
+#include <map>
+#include <string>
+#include <utility>
+
+#include "paddle/phi/common/place.h"
+
+#ifdef PADDLE_WITH_CUDA
+#include <cuda_runtime.h>
+#endif
+#ifdef PADDLE_WITH_HIP
+#include <hip/hip_runtime.h>
+#endif
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+#include "paddle/phi/core/cuda_stream.h"
+#elif defined(PADDLE_WITH_XPU)
+#include "paddle/phi/core/xpu_cuda_stream.h"
+#endif
+
+namespace phi {
+
+enum class EventType { kMark, kPushRange, kPopRange };
+
+enum class EventRole {
+  kOrdinary,  // only record op time with op type key
+  kInnerOp,   // record op detail time with op type key
+  kUniqueOp,  // record op detail time with op unique name key
+  kSpecial,   // record event such as PE which is outer of thread local
+};
+
+class PADDLE_API Event {
+ public:
+  // The DeviceContext is used to get the cuda stream.
+  // If CPU profiling mode, can pass nullptr.
+  Event(EventType type,
+        std::string name,
+        uint32_t thread_id,
+        EventRole role = EventRole::kOrdinary,
+        std::string attr = "none");
+
+  const EventType &type() const;
+  Event *parent() const { return parent_; }
+  void set_parent(Event *parent) { parent_ = parent; }
+  std::string name() const { return name_; }
+  EventRole role() const { return role_; }
+  uint64_t thread_id() const { return thread_id_; }
+  void set_name(std::string name) { name_ = name; }
+  void set_role(EventRole role) { role_ = role; }
+  std::string attr() const { return attr_; }
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) || \
+    defined(PADDLE_WITH_XPU)
+#ifndef PADDLE_WITH_CUPTI
+  gpuEvent_t event() const { return event_; }
+  int device() const { return device_; }
+#endif
+#endif
+
+  double CpuElapsedMs(const Event &e) const;
+  double CudaElapsedMs(const Event &e) const;
+
+ private:
+  EventType type_;
+  std::string name_{};
+  Event *parent_{nullptr};
+  uint64_t thread_id_;
+  EventRole role_{};
+  uint64_t cpu_ns_;
+  bool visited_status_{false};
+  std::string attr_;
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) || \
+    defined(PADDLE_WITH_XPU)
+#ifdef PADDLE_WITH_CUPTI
+  int64_t gpu_ns_ = 0;
+
+ public:
+  void AddCudaElapsedTime(int64_t start_ns, int64_t end_ns) {
+    gpu_ns_ += end_ns - start_ns;
+  }
+
+ private:
+#else
+  gpuEvent_t event_ = nullptr;
+  int device_ = -1;
+#endif
+#endif
+};
+
+using EventWithStartNs = std::pair<Event *, uint64_t>;
+using ThreadEvents = std::map<uint64_t, EventWithStartNs>;
+
+class MemEvent {
+ public:
+  MemEvent(EventType type,
+           uint64_t start_ns,
+           uint64_t end_ns,
+           size_t bytes,
+           Place place,
+           int64_t thread_id,
+           const std::string &annotation)
+      : type_(type),
+        start_ns_(start_ns),
+        end_ns_(end_ns),
+        bytes_(bytes),
+        place_(place),
+        thread_id_(thread_id),
+        annotation_(annotation) {}
+
+  const EventType &type() const { return type_; }
+  uint64_t start_ns() const { return start_ns_; }
+  uint64_t end_ns() const { return end_ns_; }
+  size_t bytes() const { return bytes_; }
+  Place place() const { return place_; }
+  uint64_t thread_id() const { return thread_id_; }
+  const std::string &annotation() const { return annotation_; }
+
+ private:
+  EventType type_;
+  uint64_t start_ns_ = 0;
+  uint64_t end_ns_ = 0;
+  size_t bytes_;
+  Place place_;
+  uint64_t thread_id_;
+  std::string annotation_;
+};
+
+class CudaEvent {
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+
+ public:
+  PADDLE_API CudaEvent();
+
+  PADDLE_API explicit CudaEvent(unsigned int flags);
+
+  ~CudaEvent() {
+#ifdef PADDLE_WITH_HIP
+    hipEventDestroy(event_);
+#else
+    cudaEventDestroy(event_);
+#endif
+  }
+
+  void Record(gpuStream_t stream) {
+#ifdef PADDLE_WITH_HIP
+    PADDLE_ENFORCE_GPU_SUCCESS(hipEventRecord(event_, stream));
+#else
+    PADDLE_ENFORCE_GPU_SUCCESS(cudaEventRecord(event_, stream));
+#endif
+  }
+
+  PADDLE_API bool Query();
+
+  PADDLE_API float ElapsedTime(CudaEvent *end_event);
+
+  void Synchronize() {
+#ifdef PADDLE_WITH_HIP
+    PADDLE_ENFORCE_GPU_SUCCESS(hipEventSynchronize(event_));
+#else
+    PADDLE_ENFORCE_GPU_SUCCESS(cudaEventSynchronize(event_));
+#endif
+  }
+  gpuEvent_t GetRawCudaEvent() { return event_; }
+
+ private:
+#ifdef PADDLE_WITH_HIP
+  unsigned int flags_ = hipEventDefault;
+#else
+  unsigned int flags_ = cudaEventDefault;
+#endif
+  gpuEvent_t event_;
+#endif
+};
+
+}  // namespace phi

paddle/phi/api/profiler/event_tracing.h

@@ -0,0 +1,97 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <string>
+
+#include "paddle/phi/api/profiler/event.h"
+#include "paddle/phi/api/profiler/trace_event.h"
+#include "paddle/utils/test_macros.h"
+
+namespace phi {
+
+// Default tracing level.
+// It is Recommended to set the level explicitly.
+static constexpr uint32_t kDefaultTraceLevel = 4;
+
+// Host event tracing. A trace starts when an object of this class is created
+// and stops when the object is destroyed.
+// Chrome Trace Viewer Format: Duration Event/Complete Event
+class PADDLE_API RecordEvent {
+ public:
+  static bool IsEnabled();
+  /**
+   * @param name If your string argument has a longer lifetime (e.g.: string
+   * literal, static variables, etc) than the event, use 'const char* name'.
+   * Do your best to avoid using 'std::string' as the argument type. It will
+   * cause deep-copy to harm performance.
+   * @param type Classification which is used to instruct the profiling
+   * data statistics.
+   * @param level Used to filter events, works like glog VLOG(level).
+   * RecordEvent will works if HostTraceLevel >= level.
+   */
+  explicit RecordEvent(
+      const std::string& name,
+      const TracerEventType type = TracerEventType::UserDefined,
+      uint32_t level = kDefaultTraceLevel,
+      const EventRole role = EventRole::kOrdinary);
+
+  /**
+   * @param name It is the caller's responsibility to manage the underlying
+   * storage. RecordEvent stores the pointer.
+   * @param type Classification which is used to instruct the profiling
+   * data statistics.
+   * @param level Used to filter events, works like glog VLOG(level).
+   * RecordEvent will works if HostTraceLevel >= level.
+   */
+  explicit RecordEvent(
+      const char* name,
+      const TracerEventType type = TracerEventType::UserDefined,
+      uint32_t level = kDefaultTraceLevel,
+      const EventRole role = EventRole::kOrdinary);
+
+  RecordEvent(const std::string& name,
+              const std::string& attr,
+              const TracerEventType type = TracerEventType::UserDefined,
+              uint32_t level = kDefaultTraceLevel,
+              const EventRole role = EventRole::kOrdinary);
+
+  // Stop event tracing explicitly before the object goes out of scope.
+  // Sometimes it's inconvenient to use RAII
+  void End();
+
+  ~RecordEvent() { End(); }
+
+ private:
+  void OriginalConstruct(const std::string& name,
+                         const EventRole role,
+                         const std::string& attr);
+
+  bool is_enabled_{false};
+  bool is_pushed_{false};
+  // Event name
+  std::string* name_{nullptr};
+  const char* shallow_copy_name_{nullptr};
+  uint64_t start_ns_;
+  // Need to distinguish name by op type, block_id, program_id and perhaps
+  // different kernel invocations within an op.
+  // std::string full_name_;
+  EventRole role_{EventRole::kOrdinary};
+  TracerEventType type_{TracerEventType::UserDefined};
+  std::string* attr_{nullptr};
+  bool finished_{false};
+};
+
+}  // namespace phi

paddle/phi/api/profiler/host_event_recorder.h

@@ -0,0 +1,292 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "paddle/common/macros.h"
+#include "paddle/phi/common/thread_data_registry.h"
+#include "paddle/phi/core/os_info.h"
+
+namespace phi {
+
+template <typename HeadType, typename... RestTypes>
+struct ContainsStdString
+    : std::conditional_t<
+          std::is_same<
+              std::string,
+              std::remove_cv_t<std::remove_reference_t<HeadType>>>::value,
+          std::true_type,
+          ContainsStdString<RestTypes...>> {};
+
+template <typename TailType>
+struct ContainsStdString<TailType>
+    : std::is_same<std::string,
+                   std::remove_cv_t<std::remove_reference_t<TailType>>> {};
+
+template <typename EventType>
+class EventContainer {
+ public:
+  EventContainer() {
+    event_blocks_ = cur_event_block_ = new EventBlock;
+    str_blocks_ = cur_str_block_ = new StringBlock;
+  }
+  ~EventContainer() {
+    Reduce();
+    delete event_blocks_;
+    for (auto cur = str_blocks_; cur != nullptr;) {
+      auto next = cur->next;
+      delete cur;
+      cur = next;
+    }
+  }
+  DISABLE_COPY_AND_ASSIGN(EventContainer);
+
+ public:
+  // Record an event
+  template <typename... Args>
+  void Record(Args &&...args) {
+    DoRecord(ContainsStdString<Args...>(), std::forward<Args>(args)...);
+  }
+
+  // Get all events and clear the container
+  std::vector<EventType> Reduce();
+
+  // Return a buffer to store the string attribute of Event.
+  // HostEventRecorder locates in the static data section.
+  // So it's safe to use arena to avoid fragmented allocations.
+  char *GetStrBufFromArena(size_t size) { return GetStringStorage(size); }
+
+ private:
+  struct EventBlock {
+    union InitDeferredEvent {
+      InitDeferredEvent() {}
+      ~InitDeferredEvent() {}
+
+      EventType event;
+    };
+
+    static constexpr size_t kBlockSize = 1 << 24;  // 16 MB
+    static constexpr size_t kAvailSize =
+        kBlockSize - sizeof(size_t) - sizeof(nullptr);
+    static constexpr size_t kNumEvents = kAvailSize / sizeof(InitDeferredEvent);
+    static constexpr size_t kPadSize =
+        kAvailSize - kNumEvents * sizeof(InitDeferredEvent);
+    static constexpr size_t kMinimumEventsPerBlock = 1024;
+    static_assert(
+        kNumEvents >= kMinimumEventsPerBlock,
+        "EventType is too large for kBlockSize, make kBlockSize larger");
+
+    size_t offset = 0;
+    EventBlock *next = nullptr;
+    InitDeferredEvent events[kNumEvents];
+    char padding[kPadSize];
+  };
+  static_assert(sizeof(EventBlock) == EventBlock::kBlockSize,
+                "sizeof EventBlock must equal to kBlockSize");
+
+  struct StringBlock {
+    static constexpr size_t kBlockSize = 1 << 22;  // 4 MB
+    static constexpr size_t kAvailSize =
+        kBlockSize - sizeof(size_t) - sizeof(nullptr);
+
+    size_t offset = 0;
+    StringBlock *next = nullptr;
+    char storage[kAvailSize];
+  };
+  static_assert(sizeof(StringBlock) == StringBlock::kBlockSize,
+                "sizeof StringBlock must equal to kBlockSize");
+
+  // Record an event with string arguments
+  template <typename... Args>
+  void DoRecord(std::true_type, Args &&...args) {
+    auto *storage = GetEventStorage();
+    std::function<void *(size_t)> allocator = [this](size_t size) {
+      return GetStrBufFromArena(size);
+    };
+    new (storage) EventType(allocator, std::forward<Args>(args)...);
+  }
+
+  // Record an event without any string argument
+  template <typename... Args>
+  void DoRecord(std::false_type, Args &&...args) {
+    auto *storage = GetEventStorage();
+    new (storage) EventType(std::forward<Args>(args)...);
+  }
+
+  EventType *GetEventStorage();
+
+  char *GetStringStorage(size_t sz);
+
+  EventBlock *event_blocks_ = nullptr;
+  EventBlock *cur_event_block_ = nullptr;
+  StringBlock *str_blocks_ = nullptr;
+  StringBlock *cur_str_block_ = nullptr;
+};
+
+template <typename EventType>
+std::vector<EventType> EventContainer<EventType>::Reduce() {
+  std::vector<EventType> all_events;
+  size_t event_cnt = 0;
+  for (auto cur = event_blocks_; cur != nullptr; cur = cur->next) {
+    event_cnt += cur->offset;
+  }
+  all_events.reserve(event_cnt);
+  for (auto cur = event_blocks_; cur != nullptr;) {
+    for (size_t i = 0; i < cur->offset; ++i) {
+      all_events.emplace_back(cur->events[i].event);
+    }
+    auto next = cur->next;
+    delete cur;
+    cur = next;
+  }
+  event_blocks_ = cur_event_block_ = new EventBlock;
+  return all_events;
+}
+
+template <typename EventType>
+EventType *EventContainer<EventType>::GetEventStorage() {
+  if (UNLIKELY(cur_event_block_->offset >=
+               EventBlock::kNumEvents)) {  // another block
+    cur_event_block_->next = new EventBlock;
+    cur_event_block_ = cur_event_block_->next;
+  }
+  auto &obj = cur_event_block_->events[cur_event_block_->offset].event;
+  ++cur_event_block_->offset;
+  return &obj;
+}
+
+template <typename EventType>
+char *EventContainer<EventType>::GetStringStorage(size_t sz) {
+  if (UNLIKELY(cur_str_block_->offset + sz >
+               StringBlock::kAvailSize)) {  // another block
+    cur_str_block_->next = new StringBlock;
+    cur_str_block_ = cur_str_block_->next;
+  }
+  char *storage = cur_str_block_->storage + cur_str_block_->offset;
+  cur_str_block_->offset += sz;
+  return storage;
+}
+
+template <typename EventType>
+struct ThreadEventSection {
+  std::string thread_name;
+  uint64_t thread_id;
+  std::vector<EventType> events;
+};
+
+template <typename EventType>
+class ThreadEventRecorder {
+ public:
+  ThreadEventRecorder() {
+    thread_id_ = GetCurrentThreadSysId();
+    thread_name_ = GetCurrentThreadName();
+  }
+
+  DISABLE_COPY_AND_ASSIGN(ThreadEventRecorder);
+
+ public:
+  // Forward call to EventContainer::Record
+  template <typename... Args>
+  void RecordEvent(Args &&...args) {
+    base_evt_cntr_.Record(std::forward<Args>(args)...);
+  }
+
+  ThreadEventSection<EventType> GatherEvents() {
+    ThreadEventSection<EventType> thr_sec;
+    thr_sec.thread_name = thread_name_;
+    thr_sec.thread_id = thread_id_;
+    thr_sec.events = std::move(base_evt_cntr_.Reduce());
+    return thr_sec;
+  }
+
+ private:
+  uint64_t thread_id_;
+  std::string thread_name_;
+  EventContainer<EventType> base_evt_cntr_;
+};
+
+template <typename EventType>
+struct HostEventSection {
+  std::string process_name;
+  uint64_t process_id;
+  std::vector<ThreadEventSection<EventType>> thr_sections;
+};
+
+template <typename EventType>
+class HostEventRecorder {
+ public:
+  // singleton
+  static HostEventRecorder &GetInstance() {
+    static HostEventRecorder instance;
+    return instance;
+  }
+
+  // thread-safe
+  // If your string argument has a longer lifetime than the Event,
+  // use 'const char*'. e.g.: string literal, op name, etc.
+  // Do your best to avoid using 'std::string' as the argument type.
+  // It will cause deep-copy to harm performance.
+  template <typename... Args>
+  void RecordEvent(Args &&...args) {
+    // Get thread local ThreadEventRecorder
+    // If not exists, we create a new one.
+    // Both HostEventRecorder and thread-local variable in
+    // ThreadEventRecorderRegistry keep the shared pointer. We add this to
+    // prevent ThreadEventRecorder being destroyed by thread-local variable in
+    // ThreadEventRecorderRegistry and lose data.
+    if (GetThreadLocalRecorder()->get() == nullptr) {
+      std::shared_ptr<ThreadEventRecorder<EventType>>
+          thread_event_recorder_ptr =
+              std::make_shared<ThreadEventRecorder<EventType>>();
+      *(GetThreadLocalRecorder()) = thread_event_recorder_ptr;
+      thr_recorders_.push_back(thread_event_recorder_ptr);
+    }
+    (*GetThreadLocalRecorder())->RecordEvent(std::forward<Args>(args)...);
+  }
+
+  // thread-unsafe, make sure make sure there is no running tracing.
+  // Poor performance, call it at the ending
+  HostEventSection<EventType> GatherEvents() {
+    HostEventSection<EventType> host_sec;
+    host_sec.process_id = GetProcessId();
+    host_sec.thr_sections.reserve(thr_recorders_.size());
+    for (auto &v : thr_recorders_) {
+      host_sec.thr_sections.emplace_back(std::move(v->GatherEvents()));
+    }
+    return host_sec;
+  }
+
+ private:
+  using ThreadEventRecorderRegistry =
+      phi::ThreadDataRegistry<std::shared_ptr<ThreadEventRecorder<EventType>>>;
+
+  HostEventRecorder() = default;
+  DISABLE_COPY_AND_ASSIGN(HostEventRecorder);
+
+  std::shared_ptr<ThreadEventRecorder<EventType>> *GetThreadLocalRecorder() {
+    return ThreadEventRecorderRegistry::GetInstance()
+        .GetMutableCurrentThreadData();
+  }
+  // Hold all thread-local ThreadEventRecorders
+  // ThreadEventRecorderRegistry and HostEventRecorder both take care of this
+  // shared pointer. We add this to prevent ThreadEventRecorder being destroyed
+  // by thread-local variable in ThreadEventRecorderRegistry and lose data.
+  std::vector<std::shared_ptr<ThreadEventRecorder<EventType>>> thr_recorders_;
+};
+
+}  // namespace phi

paddle/phi/api/profiler/host_tracer.h

@@ -0,0 +1,43 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+namespace phi {
+
+class HostTraceLevel {
+ public:
+  static constexpr int64_t kDisabled = -1;
+
+  static HostTraceLevel& GetInstance() {
+    static HostTraceLevel instance;
+    return instance;
+  }
+
+  bool NeedTrace(uint32_t level) {
+    return trace_level_ >= static_cast<int64_t>(level);
+  }
+
+  void SetLevel(int64_t trace_level) { trace_level_ = trace_level; }
+
+ private:
+  // Verbose trace level, works like VLOG(level)
+  int trace_level_ = kDisabled;
+};
+
+struct HostTracerOptions {
+  uint32_t trace_level = 0;
+};
+
+}  // namespace phi

paddle/phi/api/profiler/profiler.cc

@@ -0,0 +1,295 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/phi/api/profiler/profiler.h"
+
+#include <mutex>  // NOLINT
+#include <random>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+#include "glog/logging.h"
+
+#include "paddle/phi/api/profiler/common_event.h"
+#include "paddle/phi/api/profiler/device_tracer.h"
+#include "paddle/phi/api/profiler/host_event_recorder.h"
+#include "paddle/phi/api/profiler/host_tracer.h"
+#include "paddle/phi/api/profiler/profiler_helper.h"
+#include "paddle/phi/core/enforce.h"
+#include "paddle/phi/core/os_info.h"
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_XPU)
+#include "paddle/phi/backends/dynload/nvtx.h"
+#endif
+
+PHI_DEFINE_bool(enable_host_event_recorder_hook,
+                false,
+                "enable HostEventRecorder, hook Profiler");
+
+PHI_DEFINE_bool(enable_record_op_info,
+                false,
+                "enable operator supplement info recorder");
+
+namespace phi {
+
+ProfilerState ProfilerHelper::g_state = ProfilerState::kDisabled;
+bool ProfilerHelper::g_enable_nvprof_hook = false;
+thread_local uint64_t ProfilerHelper::g_thread_id;
+uint32_t ProfilerHelper::g_next_thread_id = 0;
+std::mutex ProfilerHelper::g_all_event_lists_mutex;
+std::list<std::shared_ptr<EventList<Event>>> ProfilerHelper::g_all_event_lists;
+thread_local std::shared_ptr<EventList<Event>> ProfilerHelper::g_event_list;
+std::list<std::shared_ptr<EventList<MemEvent>>>
+    ProfilerHelper::g_all_mem_event_lists;
+thread_local std::shared_ptr<EventList<MemEvent>>
+    ProfilerHelper::g_mem_event_list;
+std::mutex ProfilerHelper::g_all_mem_event_lists_mutex;
+
+Event::Event(EventType type,
+             std::string name,
+             uint32_t thread_id,
+             EventRole role,
+             std::string attr)
+    : type_(type),
+      name_(name),
+      thread_id_(thread_id),
+      role_(role),
+      attr_(attr) {
+  cpu_ns_ = GetTimeInNsec();
+}
+
+const EventType &Event::type() const { return type_; }
+
+double Event::CpuElapsedMs(const Event &e) const {
+  return (static_cast<double>(e.cpu_ns_ - cpu_ns_)) / (1000000.0);
+}
+
+double Event::CudaElapsedMs(const Event &e) const {
+#ifdef PADDLE_WITH_CUPTI
+  return static_cast<double>(gpu_ns_) / 1000000.0;
+#else
+  LOG_FIRST_N(WARNING, 1) << "CUDA CUPTI is not enabled";
+  return 0;
+#endif
+}
+
+Event *PushEvent(const std::string &name,
+                 const EventRole role,
+                 std::string attr) {
+  return GetEventList().Record(
+      EventType::kPushRange, name, ProfilerHelper::g_thread_id, role, attr);
+}
+
+void PopEvent(const std::string &name, const EventRole role, std::string attr) {
+  GetEventList().Record(
+      EventType::kPopRange, name, ProfilerHelper::g_thread_id, role, attr);
+}
+
+RecordEvent::RecordEvent(const char *name,
+                         const TracerEventType type,
+                         uint32_t level,
+                         const EventRole role) {
+#ifndef _WIN32
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_XPU)
+  if (ProfilerHelper::g_enable_nvprof_hook) {
+    dynload::nvtxRangePushA(name);
+    is_pushed_ = true;
+  }
+#endif
+#endif
+  if (UNLIKELY(HostTraceLevel::GetInstance().NeedTrace(level) == false)) {
+    return;
+  }
+  if (FLAGS_enable_host_event_recorder_hook == false) {
+    if (ProfilerHelper::g_state !=
+        ProfilerState::kDisabled) {  // avoid temp string
+      if (type == TracerEventType::Operator ||
+          type == TracerEventType::OperatorInner ||
+          type == TracerEventType::UserDefined) {
+        OriginalConstruct(name, role, "none");
+      }
+    }
+    return;
+  }
+
+  is_enabled_ = true;
+  shallow_copy_name_ = name;
+  role_ = role;
+  type_ = type;
+  start_ns_ = PosixInNsec();
+}
+
+RecordEvent::RecordEvent(const std::string &name,
+                         const TracerEventType type,
+                         uint32_t level,
+                         const EventRole role) {
+#ifndef _WIN32
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_XPU)
+  if (ProfilerHelper::g_enable_nvprof_hook) {
+    dynload::nvtxRangePushA(name.c_str());
+    is_pushed_ = true;
+  }
+#endif
+#endif
+  if (UNLIKELY(HostTraceLevel::GetInstance().NeedTrace(level) == false)) {
+    return;
+  }
+
+  if (FLAGS_enable_host_event_recorder_hook == false) {
+    if (type == TracerEventType::Operator ||
+        type == TracerEventType::OperatorInner ||
+        type == TracerEventType::UserDefined) {
+      OriginalConstruct(name, role, "none");
+    }
+    return;
+  }
+
+  is_enabled_ = true;
+  name_ = new std::string(name);
+  role_ = role;
+  type_ = type;
+  start_ns_ = PosixInNsec();
+}
+
+RecordEvent::RecordEvent(const std::string &name,
+                         const std::string &attr,
+                         const TracerEventType type,
+                         uint32_t level,
+                         const EventRole role) {
+#ifndef _WIN32
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_XPU)
+  if (ProfilerHelper::g_enable_nvprof_hook) {
+    dynload::nvtxRangePushA(name.c_str());
+    is_pushed_ = true;
+  }
+#endif
+#endif
+
+  if (UNLIKELY(HostTraceLevel::GetInstance().NeedTrace(level) == false)) {
+    return;
+  }
+
+  if (FLAGS_enable_host_event_recorder_hook == false) {
+    if (type == TracerEventType::Operator ||
+        type == TracerEventType::OperatorInner ||
+        type == TracerEventType::UserDefined) {
+      OriginalConstruct(name, role, attr);
+    }
+    return;
+  }
+
+  is_enabled_ = true;
+  type_ = type;
+  name_ = new std::string(name);
+  start_ns_ = PosixInNsec();
+  attr_ = new std::string(attr);
+}
+
+void RecordEvent::OriginalConstruct(const std::string &name,
+                                    const EventRole role,
+                                    const std::string &attr) {
+  if (ProfilerHelper::g_state == ProfilerState::kDisabled || name.empty())
+    return;
+
+  // do some initialization
+  name_ = new std::string(name);
+  start_ns_ = PosixInNsec();
+  role_ = role;
+  attr_ = new std::string(attr);
+  is_enabled_ = true;
+  // lock is not needed, the code below is thread-safe
+  // Maybe need the same push/pop behavior.
+  Event *e = PushEvent(name, role, attr);
+  SetCurAnnotation(e);
+  *name_ = e->name();
+}
+
+void RecordEvent::End() {
+#ifndef _WIN32
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_XPU)
+  if (ProfilerHelper::g_enable_nvprof_hook && is_pushed_) {
+    dynload::nvtxRangePop();
+    is_pushed_ = false;
+  }
+#endif
+#endif
+  if (LIKELY(FLAGS_enable_host_event_recorder_hook && is_enabled_)) {
+    uint64_t end_ns = PosixInNsec();
+    if (LIKELY(shallow_copy_name_ != nullptr)) {
+      HostEventRecorder<CommonEvent>::GetInstance().RecordEvent(
+          shallow_copy_name_, start_ns_, end_ns, role_, type_);
+    } else if (name_ != nullptr) {
+      if (attr_ == nullptr) {
+        HostEventRecorder<CommonEvent>::GetInstance().RecordEvent(
+            *name_, start_ns_, end_ns, role_, type_);
+      } else {
+        HostEventRecorder<CommonEvent>::GetInstance().RecordEvent(
+            *name_, start_ns_, end_ns, role_, type_, *attr_);
+        delete attr_;
+      }
+      delete name_;
+    }
+    // use this flag to avoid double End();
+    is_enabled_ = false;
+    return;
+  }
+
+  if (ProfilerHelper::g_state == ProfilerState::kDisabled || !is_enabled_)
+    return;
+  // lock is not needed, the code below is thread-safe
+  DeviceTracer *tracer = GetDeviceTracer();
+  if (tracer) {
+    uint64_t end_ns = PosixInNsec();
+    tracer->AddCPURecords(CurAnnotationName(),
+                          start_ns_,
+                          end_ns,
+                          BlockDepth(),
+                          ProfilerHelper::g_thread_id);
+  }
+  ClearCurAnnotation();
+  PopEvent(*name_, role_);
+  delete name_;
+  delete attr_;
+  // use this flag to avoid double End();
+  is_enabled_ = false;
+}
+
+bool RecordEvent::IsEnabled() {
+  return FLAGS_enable_host_event_recorder_hook ||
+         ProfilerHelper::g_enable_nvprof_hook ||
+         ProfilerHelper::g_state != ProfilerState::kDisabled;
+}
+
+RecordOpInfoSupplement::RecordOpInfoSupplement(
+    const std::string &type,
+    const std::vector<std::pair<const char *, std::vector<DDim>>> &input_shapes,
+    const AttributeMap &attrs) {
+  if (FLAGS_enable_host_event_recorder_hook == false) {
+    return;
+  }
+  if (IsEnabled() == false) {
+    return;
+  }
+  uint64_t op_id = 0;
+  HostEventRecorder<OperatorSupplementOriginEvent>::GetInstance().RecordEvent(
+      PosixInNsec(), type, input_shapes, attrs, op_id);
+}
+
+bool RecordOpInfoSupplement::IsEnabled() { return FLAGS_enable_record_op_info; }
+
+void EnableOpInfoRecorder() { FLAGS_enable_record_op_info = true; }
+
+void DisableOpInfoRecorder() { FLAGS_enable_record_op_info = false; }
+
+}  // namespace phi

paddle/phi/api/profiler/profiler.h

@@ -0,0 +1,95 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include <forward_list>
+#include <list>
+#include <map>
+#include <memory>
+#include <mutex>  // NOLINT
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "paddle/common/flags.h"
+#include "paddle/phi/api/profiler/event_tracing.h"
+#include "paddle/phi/api/profiler/supplement_tracing.h"
+
+COMMON_DECLARE_bool(enable_host_event_recorder_hook);
+
+namespace phi {
+
+enum class ProfilerState {
+  kDisabled,  // disabled state
+  kCPU,       // CPU profiling state
+  kCUDA,      // GPU profiling state
+  kAll,       // Profile both CPU and GPU. (Currently experimental).
+};
+
+// it is the flag to control to print the profiling result
+enum class TracerOption {
+  kDefault,      // print the different op type profiling result
+  kOpDetail,     // print the detail profiling result of different op type
+  kAllOpDetail,  // print the detail profiling result of different op name
+};
+
+template <typename T>
+struct EventList {
+  constexpr static size_t kMB = 1024 * 1024;
+  constexpr static size_t kEventBlockSize = 16 * kMB;
+  constexpr static size_t kEventSize = sizeof(T);
+  constexpr static size_t kEventAlign = alignof(T);
+  constexpr static size_t kNumBlock =
+      kEventBlockSize /
+      ((kEventSize + kEventAlign - 1) / kEventAlign * kEventAlign);
+
+  template <typename... Args>
+  T* Record(Args&&... args) {
+    if (event_blocks.empty() || event_blocks.front().size() == kNumBlock) {
+      event_blocks.emplace_front();
+      event_blocks.front().reserve(kNumBlock);
+    }
+    event_blocks.front().emplace_back(std::forward<Args>(args)...);
+    return &event_blocks.front().back();
+  }
+
+  std::vector<T> Reduce() {
+    std::vector<T> result;
+    for (auto& block : event_blocks) {
+      result.insert(result.begin(),
+                    std::make_move_iterator(block.begin()),
+                    std::make_move_iterator(block.end()));
+    }
+    event_blocks.clear();
+    return result;
+  }
+
+  void Clear() { event_blocks.clear(); }
+
+  std::forward_list<std::vector<T>> event_blocks;
+};
+
+PADDLE_API Event* PushEvent(const std::string& name,
+                            const EventRole role,
+                            const std::string attr = "none");
+PADDLE_API void PopEvent(const std::string& name,
+                         const EventRole role,
+                         const std::string attr = "none");
+
+PADDLE_API void EnableOpInfoRecorder();
+PADDLE_API void DisableOpInfoRecorder();
+
+}  // namespace phi

paddle/phi/api/profiler/profiler.proto

@@ -0,0 +1,61 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+syntax = "proto2";
+package phi.proto;
+
+message MemCopy { optional uint64 bytes = 1; }
+
+message Event {
+  enum EventType {
+    CPU = 0;
+    GPUKernel = 1;
+    NPUKernel = 2;
+  }
+  optional EventType type = 8;
+  optional string name = 1;
+  optional uint64 start_ns = 2;
+  optional uint64 end_ns = 3;
+  // When positive, it represents gpu id. When -1, it represents CPU.
+  optional int64 device_id = 5;
+  optional uint64 sub_device_id = 6;
+
+  optional MemCopy memcopy = 7;
+  optional string detail_info = 9;
+}
+
+message MemEvent {
+  enum Place {
+    CUDAPlace = 0;
+    CPUPlace = 1;
+    CUDAPinnedPlace = 2;
+    XPUPlace = 3;
+    NPUPlace = 4;
+  }
+  optional uint64 start_ns = 1;
+  optional uint64 end_ns = 2;
+  optional uint64 bytes = 3;
+  optional Place place = 4;
+  optional uint64 thread_id = 5;
+  optional uint32 device_id = 6;
+  optional string alloc_in = 7;
+  optional string free_in = 8;
+}
+
+message Profile {
+  repeated Event events = 1;
+  optional uint64 start_ns = 2;
+  optional uint64 end_ns = 3;
+  repeated MemEvent mem_events = 4;
+}

paddle/phi/api/profiler/profiler_helper.h

@@ -0,0 +1,82 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <algorithm>
+#include <iomanip>
+#include <limits>
+#include <list>
+#include <map>
+#include <memory>
+#include <mutex>  // NOLINT
+#include <random>
+#include <set>
+#include <stack>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "paddle/phi/api/profiler/device_tracer.h"
+#include "paddle/phi/api/profiler/profiler.h"
+
+namespace phi {
+
+class ProfilerHelper {
+ public:
+  // The profiler state, the initial value is ProfilerState::kDisabled
+  static ProfilerState g_state;
+  // To hook RecordEvent's events, use it to nvtx timeline
+  static bool g_enable_nvprof_hook;
+  // The thread local event list only can be accessed by the specific thread
+  // The thread index of each thread
+  static thread_local uint64_t g_thread_id;
+  // The g_next_thread_id is a global counter for threads, by the g_thread_id
+  // and g_next_thread_id, we can know how many threads have created EventList.
+  static uint32_t g_next_thread_id;
+  // The global mutex
+  static std::mutex g_all_event_lists_mutex;
+  // The total event lists of all threads
+  static std::list<std::shared_ptr<EventList<Event>>> g_all_event_lists;
+  // The thread local event list only can be accessed by the specific thread
+  static thread_local std::shared_ptr<EventList<Event>> g_event_list;
+
+  static std::list<std::shared_ptr<EventList<MemEvent>>> g_all_mem_event_lists;
+  static thread_local std::shared_ptr<EventList<MemEvent>> g_mem_event_list;
+  static std::mutex g_all_mem_event_lists_mutex;
+};
+
+inline uint64_t GetTimeInNsec() {
+  using clock = std::conditional<std::chrono::high_resolution_clock::is_steady,
+                                 std::chrono::high_resolution_clock,
+                                 std::chrono::steady_clock>::type;
+  return std::chrono::duration_cast<std::chrono::nanoseconds>(
+             clock::now().time_since_epoch())
+      .count();
+}
+
+inline EventList<Event> &GetEventList() {
+  if (!ProfilerHelper::g_event_list) {
+    std::lock_guard<std::mutex> guard(ProfilerHelper::g_all_event_lists_mutex);
+    ProfilerHelper::g_event_list = std::make_shared<EventList<Event>>();
+    ProfilerHelper::g_thread_id = ProfilerHelper::g_next_thread_id++;
+    ProfilerHelper::g_all_event_lists.emplace_front(
+        ProfilerHelper::g_event_list);
+    RecordCurThreadId(ProfilerHelper::g_thread_id);
+  }
+  return *ProfilerHelper::g_event_list;
+}
+
+}  // namespace phi

paddle/phi/api/profiler/supplement_tracing.h

@@ -0,0 +1,39 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <utility>
+
+#include "paddle/phi/core/attribute.h"
+#include "paddle/phi/core/ddim.h"
+
+namespace phi {
+
+class RecordOpInfoSupplement {
+ public:
+  PADDLE_API static bool IsEnabled();
+
+  RecordOpInfoSupplement() = default;
+
+  explicit RecordOpInfoSupplement(
+      const std::string& type,
+      const std::vector<std::pair<const char*, std::vector<DDim>>>&
+          input_shapes,
+      const AttributeMap& attrs);
+};
+
+}  // namespace phi

paddle/phi/api/profiler/trace_event.h

@@ -0,0 +1,347 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace phi {
+
+#define FOR_EACH_TRACER_EVENT_TYPES(_)                                      \
+  /* Used to mark operator record */                                        \
+  _(Operator)                                                               \
+  /* Used to mark dataloader record */                                      \
+  _(Dataloader)                                                             \
+  /* Used to mark profile step record */                                    \
+  _(ProfileStep)                                                            \
+  /* Used to mark cuda runtime record returned by cupti */                  \
+  _(CudaRuntime)                                                            \
+  /* Used to mark kernel computation record returned by cupti */            \
+  _(Kernel)                                                                 \
+  /* Used to mark memcpy record returned by cupti */                        \
+  _(Memcpy)                                                                 \
+  /* Used to mark memset record returned by cupti */                        \
+  _(Memset)                                                                 \
+  /* Used to mark record defined by user */                                 \
+  _(UserDefined)                                                            \
+  /* Used to mark operator detail, (such as infer shape, compute) */        \
+  _(OperatorInner)                                                          \
+  /* Used to mark model training or testing perspective, forward process */ \
+  _(Forward)                                                                \
+  /* Used to mark model training perspective, backward process */           \
+  _(Backward)                                                               \
+  /* Used to mark model training perspective, optimization process */       \
+  _(Optimization)                                                           \
+  /* Used to mark distributed training perspective */                       \
+  _(Communication)                                                          \
+  /* Used to mark python api */                                             \
+  _(PythonOp)                                                               \
+  /* Used to mark python level user-defined */                              \
+  _(PythonUserDefined)                                                      \
+  /* Used to mark kernel call in dynamic graph mode */                      \
+  _(DygraphKernelLaunch)                                                    \
+  /* Used to mark kernel call in static graph mode */                       \
+  _(StaticKernelLaunch)
+
+enum class TracerEventType {
+#define DEFINE_ENUM_ITEM(name) name,
+  FOR_EACH_TRACER_EVENT_TYPES(DEFINE_ENUM_ITEM)
+#undef DEFINE_ENUM_ITEM
+      NumTypes
+};
+
+#define FOR_EACH_TRACER_MEM_EVENT_TYPES(_)                                    \
+  /* Used to mark memory allocation which is managed by paddle */             \
+  _(Allocate)                                                                 \
+  /* Used to mark memory free which is managed by paddle */                   \
+  _(Free)                                                                     \
+  /* Used to mark reserved memory allocation which is applied from device. */ \
+  _(ReservedAllocate)                                                         \
+  /* Used to mark reserved memory free which is released to device. */        \
+  _(ReservedFree)
+
+enum class TracerMemEventType {
+#define DEFINE_ENUM_ITEM(name) name,
+  FOR_EACH_TRACER_MEM_EVENT_TYPES(DEFINE_ENUM_ITEM)
+#undef DEFINE_ENUM_ITEM
+      NumTypes
+};
+
+struct KernelEventInfo {
+  // The X-dimension block size for the kernel.
+  uint32_t block_x;
+  // The Y-dimension block size for the kernel.
+  uint32_t block_y;
+  // The Z-dimension grid size for the kernel.
+  uint32_t block_z;
+  // X-dimension of a grid.
+  uint32_t grid_x;
+  // Y-dimension of a grid.
+  uint32_t grid_y;
+  // Z-dimension of a grid.
+  uint32_t grid_z;
+  // The dynamic shared memory reserved for the kernel, in bytes.
+  uint32_t dynamic_shared_memory;
+  // The static shared memory allocated for the kernel, in bytes.
+  uint32_t static_shared_memory;
+  // The number of registers required for each thread executing the kernel.
+  uint32_t registers_per_thread;
+  // The amount of local memory reserved for each thread, in bytes.
+  uint32_t local_memory_per_thread;
+  // The total amount of local memory reserved for the kernel, in bytes.
+  uint32_t local_memory_total;
+  // The timestamp when the kernel is queued up in the command buffer, in ns.
+  // This timestamp is not collected by default. Use API
+  // cuptiActivityEnableLatencyTimestamps() to enable collection.
+  uint64_t queued;
+  // The timestamp when the command buffer containing the kernel launch is
+  // submitted to the GPU, in ns.
+  // This timestamp is not collected by default. Use API
+  // cuptiActivityEnableLatencyTimestamps() to enable collection.
+  uint64_t submitted;
+  // The completed timestamp for the kernel execution, in ns.
+  uint64_t completed;
+
+  float blocks_per_sm;
+  float warps_per_sm;
+  // theoretical achieved occupancy
+  float occupancy;
+};
+
+static constexpr size_t kMemKindMaxLen = 50;
+
+struct MemcpyEventInfo {
+  // The number of bytes transferred by the memory copy.
+  uint64_t num_bytes;
+  // The kind of the memory copy.
+  // Each kind represents the source and destination targets of a memory copy.
+  // Targets are host, device, and array. Refer to CUpti_ActivityMemcpyKind
+  char copy_kind[kMemKindMaxLen];
+  // The source memory kind read by the memory copy.
+  // Each kind represents the type of the memory accessed by a memory
+  // operation/copy. Refer to CUpti_ActivityMemoryKind
+  char src_kind[kMemKindMaxLen];
+  // The destination memory kind read by the memory copy.
+  char dst_kind[kMemKindMaxLen];
+};
+
+struct MemsetEventInfo {
+  // The number of bytes being set by the memory set.
+  uint64_t num_bytes;
+  // The memory kind of the memory set. Refer to CUpti_ActivityMemoryKind
+  char memory_kind[kMemKindMaxLen];
+  // the value being assigned to memory by the memory set.
+  uint32_t value;
+};
+
+struct HostTraceEvent {
+  HostTraceEvent() = default;
+  HostTraceEvent(const std::string& name,
+                 TracerEventType type,
+                 uint64_t start_ns,
+                 uint64_t end_ns,
+                 uint64_t process_id,
+                 uint64_t thread_id)
+      : name(name),
+        type(type),
+        start_ns(start_ns),
+        end_ns(end_ns),
+        process_id(process_id),
+        thread_id(thread_id) {}
+  // record name
+  std::string name;
+  // record type, one of TracerEventType
+  TracerEventType type;
+  // start timestamp of the record
+  uint64_t start_ns;
+  // end timestamp of the record
+  uint64_t end_ns;
+  // process id of the record
+  uint64_t process_id;
+  // thread id of the record
+  uint64_t thread_id;
+};
+
+struct RuntimeTraceEvent {
+  RuntimeTraceEvent() = default;
+  RuntimeTraceEvent(const std::string& name,
+                    uint64_t start_ns,
+                    uint64_t end_ns,
+                    uint64_t process_id,
+                    uint64_t thread_id,
+                    uint32_t correlation_id,
+                    uint32_t callback_id)
+      : name(name),
+        start_ns(start_ns),
+        end_ns(end_ns),
+        process_id(process_id),
+        thread_id(thread_id),
+        correlation_id(correlation_id),
+        callback_id(callback_id) {}
+
+  // record name
+  std::string name;
+  // record type, one of TracerEventType
+  TracerEventType type{TracerEventType::CudaRuntime};
+  // start timestamp of the record
+  uint64_t start_ns;
+  // end timestamp of the record
+  uint64_t end_ns;
+  // process id of the record
+  uint64_t process_id;
+  // thread id of the record
+  uint64_t thread_id;
+  // correlation id, used for correlating async activities happened on device
+  uint32_t correlation_id;
+  // callback id, used to identify which cuda runtime api is called
+  uint32_t callback_id;
+};
+
+struct DeviceTraceEvent {
+  DeviceTraceEvent() = default;
+  DeviceTraceEvent(const std::string& name,
+                   TracerEventType type,
+                   uint64_t start_ns,
+                   uint64_t end_ns,
+                   uint64_t device_id,
+                   uint64_t context_id,
+                   uint64_t stream_id,
+                   uint32_t correlation_id,
+                   const KernelEventInfo& kernel_info)
+      : name(name),
+        type(type),
+        start_ns(start_ns),
+        end_ns(end_ns),
+        device_id(device_id),
+        context_id(context_id),
+        stream_id(stream_id),
+        correlation_id(correlation_id),
+        kernel_info(kernel_info) {}
+  DeviceTraceEvent(const std::string& name,
+                   TracerEventType type,
+                   uint64_t start_ns,
+                   uint64_t end_ns,
+                   uint64_t device_id,
+                   uint64_t context_id,
+                   uint64_t stream_id,
+                   uint32_t correlation_id,
+                   const MemcpyEventInfo& memcpy_info)
+      : name(name),
+        type(type),
+        start_ns(start_ns),
+        end_ns(end_ns),
+        device_id(device_id),
+        context_id(context_id),
+        stream_id(stream_id),
+        correlation_id(correlation_id),
+        memcpy_info(memcpy_info) {}
+  DeviceTraceEvent(const std::string& name,
+                   TracerEventType type,
+                   uint64_t start_ns,
+                   uint64_t end_ns,
+                   uint64_t device_id,
+                   uint64_t context_id,
+                   uint64_t stream_id,
+                   uint32_t correlation_id,
+                   const MemsetEventInfo& memset_info)
+      : name(name),
+        type(type),
+        start_ns(start_ns),
+        end_ns(end_ns),
+        device_id(device_id),
+        context_id(context_id),
+        stream_id(stream_id),
+        correlation_id(correlation_id),
+        memset_info(memset_info) {}
+  // record name
+  std::string name;
+  // record type, one of TracerEventType
+  TracerEventType type;
+  // start timestamp of the record
+  uint64_t start_ns;
+  // end timestamp of the record
+  uint64_t end_ns;
+  // device id
+  uint64_t device_id;
+  // context id
+  uint64_t context_id;
+  // stream id
+  uint64_t stream_id;
+  // correlation id, used for correlating async activities happened on device
+  uint32_t correlation_id;
+  // union, specific device record type has different detail information
+  union {
+    // used for TracerEventType::Kernel
+    KernelEventInfo kernel_info;
+    // used for TracerEventType::Memcpy
+    MemcpyEventInfo memcpy_info;
+    // used for TracerEventType::Memset
+    MemsetEventInfo memset_info;
+  };
+};
+
+struct MemTraceEvent {
+  MemTraceEvent() = default;
+  MemTraceEvent(uint64_t timestamp_ns,
+                uint64_t addr,
+                TracerMemEventType type,
+                uint64_t process_id,
+                uint64_t thread_id,
+                int64_t increase_bytes,
+                const std::string& place,
+                uint64_t current_allocated,
+                uint64_t current_reserved,
+                uint64_t peak_allocated,
+                uint64_t peak_reserved)
+      : timestamp_ns(timestamp_ns),
+        addr(addr),
+        type(type),
+        process_id(process_id),
+        thread_id(thread_id),
+        increase_bytes(increase_bytes),
+        place(place),
+        current_allocated(current_allocated),
+        current_reserved(current_reserved),
+        peak_allocated(peak_allocated),
+        peak_reserved(peak_reserved) {}
+
+  // timestamp of the record
+  uint64_t timestamp_ns;
+  // memory addr of allocation or free
+  uint64_t addr;
+  // memory manipulation type
+  TracerMemEventType type;
+  // process id of the record
+  uint64_t process_id;
+  // thread id of the record
+  uint64_t thread_id;
+  // increase bytes after this manipulation, allocation for sign +, free for
+  // sign -
+  int64_t increase_bytes;
+  // place
+  std::string place;
+  // current total allocated memory
+  uint64_t current_allocated;
+  // current total reserved memory
+  uint64_t current_reserved;
+  // current peak allocated memory
+  uint64_t peak_allocated;
+  // current peak reserved memory
+  uint64_t peak_reserved;
+};
+
+}  // namespace phi

paddle/phi/api/profiler/trace_event_collector.h

@@ -0,0 +1,75 @@
+/* Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+
+licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <list>
+#include <string>
+#include <unordered_map>
+
+#include "paddle/phi/api/profiler/trace_event.h"
+
+namespace phi {
+
+class TraceEventCollector {
+ public:
+  void AddHostEvent(HostTraceEvent&& event) { host_events_.push_back(event); }
+
+  void AddRuntimeEvent(RuntimeTraceEvent&& event) {
+    runtime_events_.push_back(event);
+  }
+
+  void AddDeviceEvent(DeviceTraceEvent&& event) {
+    device_events_.push_back(event);
+  }
+
+  void AddThreadName(uint64_t tid, const std::string& name) {
+    thread_names_[tid] = name;
+  }
+
+  void AddMemEvent(MemTraceEvent&& event) { mem_events_.push_back(event); }
+
+  const std::list<HostTraceEvent>& HostEvents() const { return host_events_; }
+
+  const std::list<RuntimeTraceEvent>& RuntimeEvents() const {
+    return runtime_events_;
+  }
+
+  const std::list<DeviceTraceEvent>& DeviceEvents() const {
+    return device_events_;
+  }
+
+  const std::list<MemTraceEvent>& MemEvents() const { return mem_events_; }
+
+  const std::unordered_map<uint64_t, std::string>& ThreadNames() const {
+    return thread_names_;
+  }
+
+  void ClearAll() {
+    thread_names_.clear();
+    host_events_.clear();
+    runtime_events_.clear();
+    device_events_.clear();
+    mem_events_.clear();
+  }
+
+ protected:
+  std::unordered_map<uint64_t, std::string> thread_names_;
+  std::list<HostTraceEvent> host_events_;
+  std::list<RuntimeTraceEvent> runtime_events_;
+  std::list<DeviceTraceEvent> device_events_;
+  std::list<MemTraceEvent> mem_events_;
+};
+
+}  // namespace phi

paddle/phi/api/profiler/tracer_base.h

@@ -0,0 +1,40 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/phi/api/profiler/trace_event_collector.h"
+
+namespace phi {
+
+class TracerBase {
+ public:
+  // The state machine for a Tracer.
+  enum class TracerState { UNINITED, READY, STARTED, STOPPED };
+
+  virtual void PrepareTracing() { state_ = TracerState::READY; }
+
+  virtual void StartTracing() = 0;
+
+  virtual void StopTracing() = 0;
+
+  virtual void CollectTraceData(TraceEventCollector* collector) = 0;
+
+  virtual ~TracerBase() {}
+
+ protected:
+  TracerState state_ = TracerState::UNINITED;
+};
+
+}  // namespace phi