chore: import upstream snapshot with attribution

libnd4j/include/memory/impl/AllocationEntry.cpp

@@ -0,0 +1,39 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.05.19.
+//
+#include <memory/AllocationEntry.h>
+
+namespace sd {
+namespace memory {
+AllocationEntry::AllocationEntry(MemoryType type, LongType ptr, LongType numBytes, std::string &stack) {
+  _pointer = ptr;
+  _numBytes = numBytes;
+  _stack = stack;
+  _memoryType = type;
+}
+
+std::string AllocationEntry::stackTrace() { return _stack; }
+
+LongType AllocationEntry::numBytes() { return _numBytes; }
+
+MemoryType AllocationEntry::memoryType() { return _memoryType; }
+}  // namespace memory
+}  // namespace sd

libnd4j/include/memory/impl/ExternalWorkspace.cpp

@@ -0,0 +1,42 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+#include <memory/ExternalWorkspace.h>
+
+namespace sd {
+namespace memory {
+ExternalWorkspace::ExternalWorkspace(Pointer ptrH, LongType sizeH, Pointer ptrD, LongType sizeD) {
+  _ptrH = ptrH;
+  _sizeH = sizeH;
+
+  _ptrD = ptrD;
+  _sizeD = sizeD;
+};
+
+void* ExternalWorkspace::pointerHost() { return _ptrH; }
+
+void* ExternalWorkspace::pointerDevice() { return _ptrD; }
+
+LongType ExternalWorkspace::sizeHost() { return _sizeH; }
+
+LongType ExternalWorkspace::sizeDevice() { return _sizeD; }
+}  // namespace memory
+}  // namespace sd

libnd4j/include/memory/impl/MemoryCounter.cpp

@@ -0,0 +1,129 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include "../MemoryCounter.h"
+
+#include <execution/AffinityManager.h>
+#include <helpers/logger.h>
+#include <system/Environment.h>
+
+namespace sd {
+namespace memory {
+
+MemoryCounter::MemoryCounter() {
+  auto numDevices = AffinityManager::numberOfDevices();
+
+  // setting default 0s
+  for (int e = 0; e < numDevices; e++) {
+    _deviceLimits[e] = 0;
+    _deviceCounters[e] = 0;
+  }
+
+  // setting initial values for limits
+  _groupLimits[HOST] = Environment::getInstance().maxPrimaryMemory();
+  _groupLimits[DEVICE] = Environment::getInstance().maxSpecialMemory();
+
+  // setting initial counter values
+  _groupCounters[HOST] = 0;
+  _groupCounters[DEVICE] = 0;
+}
+
+MemoryCounter& MemoryCounter::getInstance() {
+  static MemoryCounter instance;
+  return instance;
+}
+
+void MemoryCounter::countIn(int deviceId, LongType numBytes) {
+  std::lock_guard<std::mutex> lock(_locker);
+  _deviceCounters[deviceId] += numBytes;
+}
+
+void MemoryCounter::countIn(MemoryType group, LongType numBytes) {
+  std::lock_guard<std::mutex> lock(_locker);
+  _groupCounters[group] += numBytes;
+}
+
+void MemoryCounter::countOut(int deviceId, LongType numBytes) {
+  std::lock_guard<std::mutex> lock(_locker);
+  _deviceCounters[deviceId] -= numBytes;
+}
+
+void MemoryCounter::countOut(MemoryType group, LongType numBytes) {
+  std::lock_guard<std::mutex> lock(_locker);
+  _groupCounters[group] -= numBytes;
+}
+
+bool MemoryCounter::validate(LongType numBytes) {
+  auto deviceId = AffinityManager::currentDeviceId();
+  return validateDevice(deviceId, numBytes);
+}
+
+bool MemoryCounter::validateDevice(int deviceId, LongType numBytes) {
+  std::lock_guard<std::mutex> lock(_locker);
+  auto dLimit = _deviceLimits[deviceId];
+  if (dLimit <= 0) return true;
+
+  auto dAlloc = _deviceCounters[deviceId];
+
+  return numBytes + dAlloc <= dLimit;
+}
+
+bool MemoryCounter::validateGroup(MemoryType group, LongType numBytes) {
+  std::lock_guard<std::mutex> lock(_locker);
+  auto gLimit = _groupLimits[group];
+  if (gLimit <= 0) return true;
+
+  auto gAlloc = _groupCounters[group];
+
+  return numBytes + gAlloc <= gLimit;
+}
+
+LongType MemoryCounter::allocatedDevice(int deviceId) {
+  std::lock_guard<std::mutex> lock(_locker);
+  return _deviceCounters[deviceId];
+}
+
+LongType MemoryCounter::allocatedGroup(MemoryType group) {
+  std::lock_guard<std::mutex> lock(_locker);
+  return _groupCounters[group];
+}
+
+void MemoryCounter::setDeviceLimit(int deviceId, LongType numBytes) {
+  std::lock_guard<std::mutex> lock(_locker);
+  _deviceLimits[deviceId] = numBytes;
+}
+
+void MemoryCounter::setGroupLimit(MemoryType group, LongType numBytes) {
+  std::lock_guard<std::mutex> lock(_locker);
+  _groupLimits[group] = numBytes;
+}
+
+LongType MemoryCounter::deviceLimit(int deviceId) {
+  std::lock_guard<std::mutex> lock(_locker);
+  return _deviceLimits[deviceId];
+}
+
+LongType MemoryCounter::groupLimit(MemoryType group) {
+  std::lock_guard<std::mutex> lock(_locker);
+  return _groupLimits[group];
+}
+}  // namespace memory
+}  // namespace sd

libnd4j/include/memory/impl/MemoryRegistrator.cpp

@@ -0,0 +1,74 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.10.2017.
+//
+#include <memory/MemoryRegistrator.h>
+
+namespace sd {
+namespace memory {
+
+MemoryRegistrator::MemoryRegistrator() { _workspace = nullptr; };
+
+MemoryRegistrator& MemoryRegistrator::getInstance() {
+  static MemoryRegistrator instance;
+  return instance;
+}
+
+bool MemoryRegistrator::hasWorkspaceAttached() { return _workspace != nullptr; }
+
+Workspace* MemoryRegistrator::getWorkspace() { return _workspace; }
+
+void MemoryRegistrator::attachWorkspace(Workspace* workspace) { _workspace = workspace; }
+
+void MemoryRegistrator::forgetWorkspace() { _workspace = nullptr; }
+
+void MemoryRegistrator::setGraphMemoryFootprint(LongType hash, LongType bytes) {
+  _lock.lock();
+
+  _footprint[hash] = bytes;
+
+  _lock.unlock();
+}
+
+void MemoryRegistrator::setGraphMemoryFootprintIfGreater(LongType hash, LongType bytes) {
+  _lock.lock();
+
+  if (_footprint.count(hash) == 0)
+    _footprint[hash] = bytes;
+  else {
+    LongType cv = _footprint[hash];
+    if (bytes > cv) _footprint[hash] = bytes;
+  }
+
+  _lock.unlock();
+}
+
+LongType MemoryRegistrator::getGraphMemoryFootprint(LongType hash) {
+  _lock.lock();
+
+  LongType result = 0L;
+  if (_footprint.count(hash) > 0) result = _footprint[hash];
+
+  _lock.unlock();
+
+  return result;
+}
+}  // namespace memory
+}  // namespace sd

libnd4j/include/memory/impl/MemoryReport.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 11.10.2017.
+//
+#include "memory/MemoryReport.h"
+
+bool sd::memory::MemoryReport::operator<(const MemoryReport &other) const {
+  return this->_rss < other._rss;
+}
+
+bool sd::memory::MemoryReport::operator>(const MemoryReport &other) const {
+  return this->_rss > other._rss;
+}
+
+bool sd::memory::MemoryReport::operator==(const MemoryReport &other) const {
+  return this->_rss == other._rss;
+}
+
+bool sd::memory::MemoryReport::operator!=(const MemoryReport &other) const {
+  return this->_rss != other._rss;
+}
+
+bool sd::memory::MemoryReport::operator<=(const MemoryReport &other) const {
+  return this->_rss <= other._rss;
+}
+
+bool sd::memory::MemoryReport::operator>=(const MemoryReport &other) const {
+  return this->_rss >= other._rss;
+}
+
+sd::LongType sd::memory::MemoryReport::getVM() const { return _vm; }
+
+void sd::memory::MemoryReport::setVM(LongType _vm) { MemoryReport::_vm = _vm; }
+
+sd::LongType sd::memory::MemoryReport::getRSS() const { return _rss; }
+
+void sd::memory::MemoryReport::setRSS(LongType _rss) { MemoryReport::_rss = _rss; }

libnd4j/include/memory/impl/MemoryTracker.cpp

@@ -0,0 +1,169 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.05.19.
+//
+#include <helpers/logger.h>
+#include <memory/MemoryTracker.h>
+#include <stdlib.h>
+
+#include <stdexcept>
+
+#if defined(__GNUC__) && !defined(SD_WINDOWS_BUILD) && !defined(__MINGW64__) && !defined(SD_ANDROID_BUILD) && \
+    !defined(SD_IOS_BUILD) && !defined(SD_APPLE_BUILD)
+
+#include <cxxabi.h>
+#include <execinfo.h>
+#include <unistd.h>
+
+#endif
+
+namespace sd {
+namespace memory {
+
+MemoryTracker::MemoryTracker() {
+  //
+}
+
+MemoryTracker &MemoryTracker::getInstance() {
+  static MemoryTracker instance;
+  return instance;
+}
+
+#if defined(__GNUC__) && !defined(__CYGWIN__) && !defined(SD_WINDOWS) && !defined(__MINGW64__) && \
+    !defined(SD_ANDROID_BUILD) && !defined(SD_IOS_BUILD) && !defined(SD_APPLE_BUILD)
+std::string demangle(char *message) {
+  char *mangled_name = 0, *offset_begin = 0, *offset_end = 0;
+
+  // find parantheses and +address offset surrounding mangled name
+  for (char *p = message; *p; ++p) {
+    if (*p == '(') {
+      mangled_name = p;
+    } else if (*p == '+') {
+      offset_begin = p;
+    } else if (*p == ')') {
+      offset_end = p;
+      break;
+    }
+  }
+
+  // if the line could be processed, attempt to demangle the symbol
+  if (mangled_name && offset_begin && offset_end && mangled_name < offset_begin) {
+    *mangled_name++ = '\0';
+    *offset_begin++ = '\0';
+    *offset_end++ = '\0';
+
+    int status;
+    char *real_name = abi::__cxa_demangle(mangled_name, 0, 0, &status);
+
+    // if demangling is successful, output the demangled function name
+    if (status == 0) {
+      std::string result(real_name);
+      free(real_name);
+      return result;
+    } else {
+      // otherwise, output the mangled function name
+      std::string result(message);
+      free(real_name);
+      return result;
+    }
+  }
+
+  // safe return
+  return std::string("");
+}
+
+#endif
+
+void MemoryTracker::countIn(MemoryType type, Pointer ptr, LongType numBytes) {
+#if defined(__GNUC__) && !defined(__MINGW64__) && !defined(__CYGWIN__) && !defined(SD_ANDROID_BUILD) && \
+    !defined(SD_WINDOWS) && !defined(SD_IOS_BUILD) && !defined(SD_APPLE_BUILD)
+  if (Environment::getInstance().isDetectingLeaks()) {
+    auto lptr = reinterpret_cast<LongType>(ptr);
+
+    _locker.lock();
+
+    void *array[50];
+    size_t size;
+    char **messages;
+    size = backtrace(array, 50);
+
+    std::string stack("");
+    messages = backtrace_symbols(array, size);
+    for (size_t i = 1; i < size && messages != NULL; ++i) {
+      stack += demangle(messages[i]) + "\n";
+    }
+
+    free(messages);
+
+    if (stack.find("ConstantTad") != std::string::npos || stack.find("ConstantShape") != std::string::npos) {
+      _locker.unlock();
+      return;
+    }
+
+    std::pair<LongType, AllocationEntry> pair(lptr, AllocationEntry(type, lptr, numBytes, stack));
+    _allocations.insert(pair);
+
+    _locker.unlock();
+  }
+#endif
+}
+
+void MemoryTracker::countOut(Pointer ptr) {
+#if defined(__GNUC__) && !defined(__MINGW64__) && !defined(SD_ANDROID_BUILD) && !defined(SD_IOS_BUILD) && \
+    !defined(SD_APPLE_BUILD)
+  if (Environment::getInstance().isDetectingLeaks()) {
+    auto lptr = reinterpret_cast<LongType>(ptr);
+
+    _locker.lock();
+    if (_released.count(lptr) > 0) {
+       THROW_EXCEPTION("Double free!");
+    }
+
+    if (_allocations.count(lptr) > 0) {
+       auto entry = _allocations[lptr];
+       std::pair<sd::LongType, AllocationEntry> pair(lptr, entry);
+      _released.insert(pair);
+      _allocations.erase(lptr);
+    }
+
+    _locker.unlock();
+  }
+#endif
+}
+
+void MemoryTracker::summarize() {
+  if (!_allocations.empty()) {
+    sd_printf("\n%i leaked allocations\n", (int)_allocations.size());
+
+    for (auto &v : _allocations) {
+      sd_printf("Leak of %i [%s] bytes\n%s\n\n", (int)v.second.numBytes(),
+                v.second.memoryType() == MemoryType::HOST ? "HOST" : "DEVICE", v.second.stackTrace().c_str());
+    }
+
+    THROW_EXCEPTION("Non-released allocations found");
+  }
+}
+
+void MemoryTracker::reset() {
+  _allocations.clear();
+  _released.clear();
+}
+}  // namespace memory
+}  // namespace sd

libnd4j/include/memory/impl/MemoryUtils.cpp

@@ -0,0 +1,87 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 11.10.2017.
+//
+#include "memory/MemoryUtils.h"
+
+#include <helpers/logger.h>
+
+#if defined(__APPLE__)
+#include <mach/mach.h>
+#include <sys/resource.h>
+#elif defined(_WIN32) || defined(__WIN32__) || defined(WIN32)
+
+#else
+// linux
+#include <fcntl.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include <cstring>
+#endif
+
+bool sd::memory::MemoryUtils::retrieveMemoryStatistics(MemoryReport &report) {
+#if defined(__APPLE__)
+  sd_debug("APPLE route\n", "");
+  /*
+      struct task_basic_info t_info;
+      mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
+
+      if (KERN_SUCCESS != task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count))
+          return false;
+
+      report.setVM(t_info.resident_size);
+      report.setRSS(t_info.resident_size);
+
+
+      sd_debug("RSS: %lld; VM: %lld;\n", report.getRSS(), report.getVM());
+  */
+  struct rusage _usage;
+
+  auto res = getrusage(RUSAGE_SELF, &_usage);
+
+  report.setRSS(_usage.ru_maxrss);
+
+  sd_debug("Usage: %lld; %lld; %lld; %lld;\n", _usage.ru_ixrss, _usage.ru_idrss, _usage.ru_isrss, _usage.ru_maxrss);
+
+  return true;
+#elif defined(_WIN32) || defined(__WIN32__) || defined(WIN32)
+  sd_debug("WIN32 route\n", "");
+
+#else
+  sd_debug("LINUX route\n", "");
+  int fd = open("/proc/self/statm", O_RDONLY, 0);
+  if (fd >= 0) {
+    char line[256];
+    char* s;
+    int n;
+    lseek(fd, 0, SEEK_SET);
+    if ((n = read(fd, line, sizeof(line))) > 0 && (s = (char*)memchr(line, ' ', n)) != NULL) {
+      report.setRSS((LongType)(atoll(s + 1) * getpagesize()));
+    }
+    close(fd);
+  }
+
+
+  return true;
+#endif
+
+  return false;
+}