chore: import upstream snapshot with attribution

libnd4j/include/graph/impl/ArgumentsList.cpp

@@ -0,0 +1,40 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.01.18.
+//
+#include <graph/ArgumentsList.h>
+
+namespace sd {
+namespace graph {
+ArgumentsList::ArgumentsList(std::initializer_list<Pair> arguments) { _arguments = arguments; }
+
+ArgumentsList::ArgumentsList(std::initializer_list<int> arguments) {
+  std::vector<int> args(arguments);
+  for (size_t e = 0; e < args.size(); e++) {
+    Pair pair(args[e]);
+    _arguments.emplace_back(pair);
+  }
+}
+
+int ArgumentsList::size() { return (int)_arguments.size(); }
+
+Pair& ArgumentsList::at(int index) { return _arguments.at(index); }
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/Context.cpp

@@ -0,0 +1,878 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <array/InteropDataBuffer.h>
+#include <graph/Context.h>
+#include <helpers/ShapeUtils.h>
+
+#if defined(SD_GCC_FUNCTRACE)
+#include <graph/OpContextLifecycleTracker.h>
+#endif
+
+namespace sd {
+namespace graph {
+Context::Context(ContextPrototype *prototype, VariableSpace *variableSpace) {
+  _variableSpace = variableSpace;
+  _dataType = prototype->dataType();
+
+  if (prototype != nullptr) {
+    for (const auto &v : *(prototype->inputs())) {
+      this->_inputs.push_back(v);
+    }
+
+    for (const auto &v : *(prototype->getTArguments())) {
+      this->_tArgs.push_back(v);
+    }
+
+    for (const auto &v : *(prototype->getIArguments())) {
+      this->_iArgs.push_back(v);
+    }
+
+    for (const auto &v : *(prototype->getBArguments())) {
+      this->_bArgs.push_back(v);
+    }
+
+    for (const auto &v : *(prototype->getAxis())) {
+      this->_axis.push_back(v);
+    }
+
+    for(auto v : *(prototype->getDArguments())) {
+      this->_dataTypes.push_back(v);
+    }
+
+    this->_opNum = prototype->opNum();
+    this->_isInplace = prototype->isInplace();
+    this->_nodeId = prototype->nodeId();
+    this->_useONEDNN = prototype->isUseONEDNN();
+  }
+
+  if (variableSpace != nullptr && variableSpace->launchContext()->getWorkspace() != nullptr)
+    this->_workspace = variableSpace->launchContext()->getWorkspace();
+
+#if defined(SD_GCC_FUNCTRACE)
+  // Track OpContext allocation
+  OpContextLifecycleTracker::getInstance().recordAllocation(
+      this, _nodeId,
+      _fastpath_in.size(), _fastpath_out.size(),
+      _intermediateResults.size(), _handles.size(),
+      _workspace != nullptr, isFastPath());
+#endif
+}
+DataType Context::dataType(int index) {
+  if(numD() < 1) {
+    if(width() > 0) {
+      return this->array(index)->dataType();
+    } else {
+      std::string errorMessage;
+      errorMessage += std::string("Context::dataType: Unable to determine data type. Both d args and inputs are empty.");
+      errorMessage += std::string(" Index: ");
+      errorMessage += std::to_string(index);
+      errorMessage += std::string(" Width: ");
+      errorMessage += std::to_string(width());
+      THROW_EXCEPTION(errorMessage.c_str());
+    };
+  }
+
+
+  return getDArguments()->at(index);
+}
+
+DataType Context::dataType() { return dataType(0); }
+
+void Context::setDataType(int index, DataType type) {
+  if (this->_dataTypes.size() > (size_t)index) _dataTypes[index] = type;
+  _dataType = type;
+}
+
+Context::Context(int nodeId, VariableSpace *variableSpace) {
+  this->_nodeId = nodeId;
+  this->_variableSpace = variableSpace;
+  this->_isInplace = false;
+  this->_workspace = nullptr;
+
+  this->_executionTime.first = 0;
+  this->_executionTime.second = 0;
+
+  if (variableSpace != nullptr && variableSpace->launchContext()->getWorkspace() != nullptr)
+    this->_workspace = variableSpace->launchContext()->getWorkspace();
+
+#if defined(SD_GCC_FUNCTRACE)
+  // Track OpContext allocation
+  OpContextLifecycleTracker::getInstance().recordAllocation(
+      this, _nodeId,
+      _fastpath_in.size(), _fastpath_out.size(),
+      _intermediateResults.size(), _handles.size(),
+      _workspace != nullptr, isFastPath());
+#endif
+}
+
+Context::Context(int nodeId, VariableSpace *variableSpace, bool isInplace) : Context(nodeId, variableSpace) {
+  this->_isInplace = isInplace;
+}
+
+Context::~Context() {
+#if defined(SD_GCC_FUNCTRACE)
+  // Track OpContext deallocation before cleanup
+  OpContextLifecycleTracker::getInstance().recordDeallocation(this);
+#endif
+
+  this->_iArgs.clear();
+  this->_tArgs.clear();
+  this->_inputs.clear();
+
+  // IMPORTANT: Do NOT delete arrays in _fastpath_in and _fastpath_out!
+  // These are BORROWED pointers - the Context does not own them.
+  // The caller (e.g., DeclarableOp::execute) owns these arrays and is
+  // responsible for their lifecycle. Deleting them here causes use-after-free
+  // bugs when operations call sub-operations (e.g., layer_norm calling standardize).
+  // Only _handles contains arrays explicitly marked as owned by this Context.
+  this->_fastpath_in.clear();
+  this->_fastpath_out.clear();
+
+  // Clean up intermediate results - these ARE owned by the Context
+  for (auto v : _intermediateResults) {
+    if (v != nullptr) delete v;
+  }
+  _intermediateResults.clear();
+
+  // Clean up handles - these are arrays explicitly marked as removable/owned
+  for (auto v : _handles) delete v;
+
+  if (_context != nullptr) delete _context;
+}
+
+void Context::setTargetEngine(samediff::Engine engine) { _engine = engine; }
+
+bool Context::hasWorkspaceProvided() { return this->_workspace != nullptr; }
+
+void Context::attachWorkspace(sd::memory::Workspace *workspace) { this->_workspace = workspace; }
+
+void Context::setVariableSpace(VariableSpace *variableSpace) { this->_variableSpace = variableSpace; }
+
+void Context::forgetWorkspace() { _workspace = nullptr; }
+
+std::vector<NDArray *> &Context::fastpath_in() { return _fastpath_in; }
+
+std::vector<NDArray *> &Context::fastpath_out() { return _fastpath_out; }
+
+bool Context::isFastPath() {
+  auto ie = _fastpath_in.empty();
+  auto io = _fastpath_out.empty();
+  // two options here.
+  // either both IN/OUT are filled
+  auto b1 = (!ie && !io) || (!ie && _isInplace);
+
+  // or at least something is filled, and FastPath is NOT forbidden
+  auto b2 = (!ie || !io) && !_forbidFastPath;
+  return b1 || b2;
+}
+
+void Context::forbidFastPath(bool reallyForbid) { _forbidFastPath = reallyForbid; }
+
+VariableSpace *Context::getVariableSpace() { return _variableSpace; }
+
+memory::Workspace *Context::getWorkspace() { return _workspace; }
+
+memory::Workspace *Context::workspace() { return _workspace; }
+
+random::RandomBuffer *Context::getRNG() { return _rng; }
+
+void Context::setRNG(random::RandomBuffer *rng) { _rng = rng; }
+
+
+Stash *Context::getStash() {
+  if (_variableSpace == nullptr) {
+    THROW_EXCEPTION("Context::getStash: VariableSpace is null. Context was not properly initialized.");
+  }
+  return _variableSpace->getStash();
+}
+
+void Context::trackList(NDArrayList *list) {
+  if (_variableSpace == nullptr) {
+    THROW_EXCEPTION("Context::trackList: VariableSpace is null. Context was not properly initialized.");
+  }
+  _variableSpace->trackList(list);
+}
+
+int Context::getBranch() {
+  if (_variableSpace == nullptr) {
+    THROW_EXCEPTION("Context::getBranch: VariableSpace is null. Context was not properly initialized.");
+  }
+  if (_variableSpace->flowPath() == nullptr) {
+    return 0;  // Default branch when no flow path is set
+  }
+  return _variableSpace->flowPath()->branch(this->nodeId());
+}
+
+void Context::setBranch(int branch) {
+  //_branch = branch;
+  if (_variableSpace != nullptr && _variableSpace->flowPath() != nullptr) {
+    _variableSpace->flowPath()->markBranch(this->nodeId(), branch);
+  }
+}
+
+LongType Context::getOuterTime() { return this->_executionTime.first; }
+
+LongType Context::getInnerTime() { return this->_executionTime.second; }
+
+void Context::setOuterTime(LongType time) { this->_executionTime.first = time; }
+
+void Context::setInnerTime(LongType time) { this->_executionTime.second = time; }
+
+Variable *Context::getVariable(int idx) {
+  if (static_cast<size_t>(idx) >= this->_inputs.size()) {
+    std::string errorMessage;
+    errorMessage += "Node ";
+    errorMessage += std::to_string(this->_nodeId);
+    errorMessage += "; Variable [";
+    errorMessage += std::to_string(idx);
+    errorMessage += " requested, but only ";
+    errorMessage += std::to_string(this->_inputs.size());
+    errorMessage += " available";
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+
+  auto p = this->_inputs[idx];
+
+  auto v = variable(p);
+  // preconditioned with v->variableType()==VariableType::NDARRAY as for other cases getNDArray() can throw exception
+  if (Environment::getInstance().isDebugAndVerbose() && v != nullptr && v->variableType() == NDARRAY &&
+      v->getNDArray() != nullptr) {
+    auto array = v->getNDArray();
+    std::string shape_ = ShapeUtils::shapeAsString(array);
+    auto type = DataTypeUtils::asString(array->dataType());
+    float m = std::numeric_limits<float>::quiet_NaN();
+    if (!array->isEmpty()) {
+      LongType maxLen = sd::math::sd_min<LongType>(16, array->lengthOf() - 1);
+
+      sd_printf("Debug info for node_%i input[%i]; shape: %s; ews: [%i]; order: [%c]; dtype: [%s];\n",
+                this->_nodeId, idx, shape_.c_str(),array->ews(), array->ordering(), type.c_str());
+      std::vector<sd::LongType> shapeLen = {array->lengthOf()};
+      NDArray *raveled = array->reshape(array->ordering(), shapeLen);
+      sd_printf("Values: [ ",0);
+      for (LongType i = 0; i < maxLen; i++) {
+        auto v2 = raveled->e<float>(i);
+        sd_printf("%f, ", v2);
+      }
+
+      delete raveled;
+      sd_printf("]\n",0);
+
+    } else {
+      sd_printf("Debug info for node_%i input[%i]; shape: %s; ews: [%i]; order: [%c]; dtype: [%s]; mean value: [%f]\n",
+                this->_nodeId, idx, shape_.c_str(), (int)array->ews(), array->ordering(), type.c_str(), m);
+    }
+  }
+
+  return v;
+}
+
+Variable *Context::variable(int idx) { return getVariable(idx); }
+
+Variable *Context::variable(std::initializer_list<int> p) {
+  if (p.size() != 2) THROW_EXCEPTION("Variable address should have size of 2");
+
+  std::vector<int> vec(p);
+  std::pair<int, int> pair(vec[0], vec[1]);
+  return variable(pair);
+}
+
+Variable *Context::variable(int node, int idx) {
+  std::pair<int, int> pair(node, idx);
+  return variable(pair);
+}
+
+Variable *Context::variable(std::pair<int, int> &p) {
+  if (_variableSpace == nullptr) {
+    std::string errorMessage;
+    errorMessage += "Node ";
+    errorMessage += std::to_string(this->_nodeId);
+    errorMessage += "; VariableSpace is null when trying to get variable: [";
+    errorMessage += std::to_string(p.first);
+    errorMessage += ":";
+    errorMessage += std::to_string(p.second);
+    errorMessage += "]. This usually means the Context was not properly initialized or fastpath was expected but failed.";
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+
+#ifdef __cpp_exceptions
+  try {
+    return _variableSpace->getVariable(p);
+  } catch (std::exception &e) {
+    std::string errorMessage;
+    errorMessage += "Node ";
+    errorMessage += std::to_string(this->_nodeId);
+    errorMessage += "; Non-existent variable requested: [";
+    errorMessage += std::to_string(p.first);
+    errorMessage += ":";
+    errorMessage += std::to_string(p.second);
+    errorMessage += "]";
+    errorMessage += "\n";
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+#else
+  return _variableSpace->getVariable(p);
+#endif
+
+  return nullptr;
+}
+
+void Context::pushNDArrayToVariableSpace(int nodeId, int index, NDArray *array, bool removable) {
+  std::pair<int, int> pair(nodeId, index);
+  pushNDArrayToVariableSpace(pair, array, removable);
+}
+
+void Context::pushNDArrayToVariableSpace(std::pair<int, int> &pair, NDArray *array, bool removable) {
+  if (_variableSpace != nullptr) {
+    if (!_variableSpace->hasVariable(pair)) {
+      auto var = new Variable(array, nullptr, pair.first, pair.second);
+      _variableSpace->putVariable(pair, var);
+      var->markRemovable(removable);
+    } else {
+      sd_debug("Context: Getting variable in push ndarray\n",0);
+      auto var = _variableSpace->getVariable(pair);
+      sd_debug("Context: After getting variable in push ndarray to variable space\n",0);
+      if (var->hasNDArray()) {
+        if (var->getNDArray() != array) {
+          if (var->isRemovable() && var->hasNDArray() && !var->getNDArray()->isView()) {
+            delete var->getNDArray();
+          }
+          var->setNDArray(array);
+          var->markRemovable(removable);
+        }
+      } else {
+        var->setNDArray(array);
+        var->markRemovable(removable);
+      }
+    }
+  }
+}
+
+void Context::pushNDArrayListToVariableSpace(int nodeId, int index, NDArrayList *list, bool track) {
+  std::pair<int, int> pair(nodeId, index);
+  pushNDArrayListToVariableSpace(pair, list, track);
+}
+
+void Context::pushNDArrayListToVariableSpace(std::pair<int, int> &pair, NDArrayList *list, bool track) {
+  sd_debug("Pre push variable list\n",0);
+  if (_variableSpace == nullptr) {
+    THROW_EXCEPTION("Context::pushNDArrayListToVariableSpace: VariableSpace is null. Context was not properly initialized.");
+  }
+  if (!_variableSpace->hasVariable(pair)) {
+    sd_debug("Context::pushNDArrayListToVariableSpace: Pre create variable when none exists\n",0);
+    auto var = new Variable(nullptr, nullptr, pair.first, pair.second);
+    sd_debug("Context::pushNDArrayListToVariableSpace: Created when none exists\n",0);
+    var->setNDArrayList(list);
+    _variableSpace->putVariable(pair, var);
+    sd_debug("Context::pushNDArrayListToVariableSpace: Put variable\n",0);
+  } else {
+    sd_debug("Context::pushNDArrayListToVariableSpace: In else: Getting variable\n",0);
+    auto var = _variableSpace->getVariable(pair);
+    sd_debug("Context::pushNDArrayListToVariableSpace: Got variable setting list\n",0);
+    var->setNDArrayList(list);
+  }
+
+  sd_debug("Context::pushNDArrayListToVariableSpace: pre tracking\n",0);
+
+  if (track) _variableSpace->trackList(list);
+}
+
+Variable *Context::ensureVariable(int idx) {
+  std::pair<int, int> pair(this->nodeId(), idx);
+
+  if (_variableSpace == nullptr) THROW_EXCEPTION("Context::ensureVariable VariableSpace is NULL!");
+
+  if (!_variableSpace->hasVariable(pair)) {
+    auto var = new Variable(nullptr, nullptr, this->nodeId(), idx);
+    _variableSpace->putVariable(pair, var);
+    return var;
+  } else {
+    sd_debug("Before ensure variable",0);
+    return _variableSpace->getVariable(pair);
+  }
+}
+
+bool Context::isValueAvailable(int idx) {
+  auto var = ensureVariable(idx);
+
+  if (var->variableType() == NDARRAY) {
+    return var->hasNDArray();
+  } else if (var->variableType() == ARRAY_LIST) {
+    return var->hasNDArrayList();
+  }
+
+  return false;
+}
+
+NDArray *Context::getNDArray(int idx) { return array(idx); }
+
+
+NDArray *Context::outputArray(int idx) {
+  // we check for fastpath first
+  if (!_fastpath_out.empty() && _fastpath_out.size() > static_cast<size_t>(idx)) {
+    NDArray* result = _fastpath_out[idx];
+
+    // Validate the output NDArray to catch memory corruption early
+    if (result != nullptr) {
+      const sd::LongType* shInfo = result->shapeInfo();
+      if (shInfo == nullptr) {
+        std::string errorMessage;
+        errorMessage += "Context::outputArray(";
+        errorMessage += std::to_string(idx);
+        errorMessage += "): NDArray has null shapeInfo. This indicates severe memory corruption or use-after-free.";
+        THROW_EXCEPTION(errorMessage.c_str());
+      }
+
+      sd::LongType rank = shInfo[0];
+      if (rank < 0 || rank > 32) {
+        std::string errorMessage;
+        errorMessage += "Context::outputArray(";
+        errorMessage += std::to_string(idx);
+        errorMessage += "): NDArray has corrupted rank: ";
+        errorMessage += std::to_string(rank);
+        errorMessage += " (expected 0-32). This likely indicates:\n";
+        errorMessage += "  1. Memory corruption in the output NDArray\n";
+        errorMessage += "  2. Use-after-free (accessing deallocated memory)\n";
+        errorMessage += "  3. Uninitialized output buffer allocation failure\n";
+        errorMessage += "  4. JNI marshalling error from Java layer";
+        THROW_EXCEPTION(errorMessage.c_str());
+      }
+    }
+
+    return result;
+  }
+
+  std::string errorMessage;
+  errorMessage += std::string("Context::outputArray: Fastpath is empty");
+  errorMessage += std::string(" Index: ");
+  errorMessage += std::to_string(idx);
+  errorMessage += std::string(" Fastpath size: ");
+  errorMessage += std::to_string(_fastpath_out.size());
+
+  THROW_EXCEPTION(errorMessage.c_str());
+
+  return nullptr;
+}
+
+NDArray *Context::array(int idx) {
+  // we check for fastpath first
+  if (!_fastpath_in.empty() && _fastpath_in.size() > static_cast<size_t>(idx)) {
+    NDArray* result = _fastpath_in[idx];
+
+    // Validate the NDArray to catch memory corruption early
+    if (result != nullptr) {
+      const sd::LongType* shInfo = result->shapeInfo();
+      if (shInfo == nullptr) {
+        std::string errorMessage;
+        errorMessage += "Context::array(";
+        errorMessage += std::to_string(idx);
+        errorMessage += "): NDArray has null shapeInfo. This indicates severe memory corruption or use-after-free.";
+        THROW_EXCEPTION(errorMessage.c_str());
+      }
+
+      // Check if rank is valid (should be 0-32, not a memory address)
+      sd::LongType rank = shInfo[0];
+      if (rank < 0 || rank > 32) {
+        std::string errorMessage;
+        errorMessage += "Context::array(";
+        errorMessage += std::to_string(idx);
+        errorMessage += "): NDArray has corrupted rank: ";
+        errorMessage += std::to_string(rank);
+        errorMessage += " (expected 0-32). This likely indicates:\n";
+        errorMessage += "  1. Memory corruption in the NDArray\n";
+        errorMessage += "  2. Use-after-free (accessing deallocated memory)\n";
+        errorMessage += "  3. Uninitialized NDArray from failed operation\n";
+        errorMessage += "  4. JNI marshalling error from Java layer";
+        THROW_EXCEPTION(errorMessage.c_str());
+      }
+    }
+
+    return result;
+  }
+
+  // When using fastpath (from OpContext/Java), _variableSpace is null by design.
+  // If the operation expects more inputs than were provided via setInputArrays(),
+  // throw an informative error instead of crashing in getVariable().
+  if (!_fastpath_in.empty() && _variableSpace == nullptr) {
+    // Fastpath has some inputs but not enough for the requested index
+    std::string errorMessage;
+    errorMessage += "Context::array(";
+    errorMessage += std::to_string(idx);
+    errorMessage += "): Input index out of bounds. Fastpath has ";
+    errorMessage += std::to_string(_fastpath_in.size());
+    errorMessage += " input array(s), but input at index ";
+    errorMessage += std::to_string(idx);
+    errorMessage += " was requested.\n";
+    errorMessage += "This typically means the operation expects more inputs than were provided.\n";
+    errorMessage += "For variable-input operations (like concat), ensure all input arrays are set via setInputArrays().";
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+
+  // if no luck for fastpath - return whatever is available
+  NDArray* result = getVariable(idx)->getNDArray();
+
+  // Validate the NDArray from variable path as well
+  if (result != nullptr) {
+    const sd::LongType* shInfo = result->shapeInfo();
+    if (shInfo == nullptr) {
+      std::string errorMessage;
+      errorMessage += "Context::array(";
+      errorMessage += std::to_string(idx);
+      errorMessage += "): NDArray from variable has null shapeInfo. This indicates severe memory corruption or use-after-free.";
+      THROW_EXCEPTION(errorMessage.c_str());
+    }
+
+    sd::LongType rank = shInfo[0];
+    if (rank < 0 || rank > 32) {
+      std::string errorMessage;
+      errorMessage += "Context::array(";
+      errorMessage += std::to_string(idx);
+      errorMessage += "): NDArray from variable has corrupted rank: ";
+      errorMessage += std::to_string(rank);
+      errorMessage += " (expected 0-32). This likely indicates:\n";
+      errorMessage += "  1. Memory corruption in the NDArray\n";
+      errorMessage += "  2. Use-after-free (accessing deallocated memory)\n";
+      errorMessage += "  3. Uninitialized NDArray from failed operation\n";
+      errorMessage += "  4. JNI marshalling error from Java layer";
+      THROW_EXCEPTION(errorMessage.c_str());
+    }
+  }
+
+  return result;
+}
+
+memory::Workspace *Context::fWorkspace() { return workspace(); }
+
+memory::Workspace *Context::tWorkspace() { return nullptr; }
+
+memory::Workspace *Context::oWorkspace() { return nullptr; }
+
+LaunchContext *Context::launchContext() {
+  // FIXME: we need proper context to be shared here
+  if (_context == nullptr) {
+    return LaunchContext::defaultContext();
+  } else {
+    return _context;
+  }
+}
+
+
+unsigned long Context::outputWidth() {
+  return _fastpath_out.size();
+}
+
+unsigned long Context::width() {
+  if (!_fastpath_in.empty())
+    return _fastpath_in.size();
+  else
+    return _inputs.size();
+}
+
+void Context::setInputArray(int index, NDArray *array, bool removable) {
+  // Check for null array FIRST before any dereference
+  if (array == nullptr) {
+    std::string errorMessage;
+    errorMessage += std::string("Context::setInputArray: Array at index ");
+    errorMessage += std::to_string(index);
+    errorMessage += std::string(" is null!");
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+
+  if(array->shapeInfo() == nullptr) {
+    std::string errorMessage;
+    errorMessage += std::string("Array at index ");
+    errorMessage += std::to_string(index);
+    errorMessage += std::string(" has a null shape buffer!");
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+  if(array->dataType() != ArrayOptions::dataType(array->shapeInfo())) {
+    std::string errorMessage;
+    errorMessage += std::string("Array at index ");
+    errorMessage += std::to_string(index);
+    errorMessage += std::string(" has a different data type than the shape buffer!");
+    //add the shape info as a string to the error message
+    errorMessage += std::string(" Shape info: ");
+    errorMessage += ShapeUtils::shapeAsString(array->shapeInfo());
+    errorMessage += std::string(" Data type: ");
+    errorMessage += DataTypeUtils::asString(ArrayOptions::dataType(array->shapeInfo()));
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+
+  if (_fastpath_in.size() < static_cast<size_t>(index + 1)) _fastpath_in.resize(index + 1);
+
+  _fastpath_in[index] = array;
+  if (removable) _handles.emplace_back(array);
+}
+
+
+
+
+void Context::setOutputArray(int index, NDArray *array, bool removable) {
+  // Check for null array FIRST before any dereference
+  if (array == nullptr) {
+    std::string errorMessage;
+    errorMessage += std::string("Context::setOutputArray: Array at index ");
+    errorMessage += std::to_string(index);
+    errorMessage += std::string(" is null!");
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+
+  if (_fastpath_out.size() < static_cast<size_t>(index + 1)) _fastpath_out.resize(index + 1);
+
+  // Check for null shapeInfo before accessing it
+  if (array->shapeInfo() == nullptr) {
+    std::string errorMessage;
+    errorMessage += std::string("Context::setOutputArray: Array at index ");
+    errorMessage += std::to_string(index);
+    errorMessage += std::string(" has a null shape buffer!");
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+
+  if(array->dataType() != ArrayOptions::dataType(array->shapeInfo())) {
+    std::string errorMessage;
+    errorMessage += std::string("Array at index ");
+    errorMessage += std::to_string(index);
+    errorMessage += std::string(" has a different data type than the shape buffer!");
+    //add the shape info as a string to the error message
+    errorMessage += std::string(" Shape info: ");
+    errorMessage += ShapeUtils::shapeAsString(array->shapeInfo());
+    errorMessage += std::string(" Data type: ");
+    errorMessage += DataTypeUtils::asString(ArrayOptions::dataType(array->shapeInfo()));
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+  _fastpath_out[index] = array;
+
+  if (removable) _handles.emplace_back(array);
+}
+
+
+
+
+
+void validateBufferAndShape(InteropDataBuffer* dataBuffer, LongType * newShapeInfoCast, int index) {
+  bool errorFound = false;
+  std::string errorMessage;
+  //opaque/interop data buffers are created with int8 on purpose and therefore will be excluded from validation here.
+  //see more here: https://github.com/deeplearning4j/deeplearning4j/blob/8aa0ef12794ca40a2d00c5c80206a24a3bd6529c/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cpu-backend-common/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java#L386
+
+  bool isString = ArrayOptions::dataType(newShapeInfoCast) == UTF8 || ArrayOptions::dataType(newShapeInfoCast) == UTF16 ||
+                  ArrayOptions::dataType(newShapeInfoCast) == UTF32;
+  if(isString || shape::isEmptyConst(newShapeInfoCast) || dataBuffer->getDataBuffer()->getDataType() == INT8) return;
+  if (dataBuffer != nullptr) {
+    if (!shape::isEmptyConst(newShapeInfoCast)) {
+      if (dataBuffer->dataBuffer() != nullptr) {
+
+        //opaque/interop data buffers are created with int8 on purpose and therefore will be excluded from validation here.
+        //see more here: https://github.com/deeplearning4j/deeplearning4j/blob/8aa0ef12794ca40a2d00c5c80206a24a3bd6529c/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cpu-backend-common/src/main/java/org/nd4j/linalg/cpu/nativecpu/buffer/BaseCpuDataBuffer.java#L386
+        if (!isString && dataBuffer->getDataBuffer()->getDataType() != ArrayOptions::dataType(newShapeInfoCast)) {
+          errorMessage += "Data type mismatch between data buffer and shape buffer. ";
+          errorMessage += "Data buffer data type: " + DataTypeUtils::asString(dataBuffer->dataBuffer()->getDataType()) + ". ";
+          errorMessage += "Shape buffer data type: " + DataTypeUtils::asString(ArrayOptions::dataType(newShapeInfoCast)) + ". ";
+          errorFound = true;
+        }
+        if (!DataTypeUtils::validDataType(dataBuffer->dataBuffer()->getDataType())) {
+          errorMessage += "Invalid data type in data buffer. ";
+          errorFound = true;
+        }
+      } else {
+        errorMessage += "Data buffer is null. ";
+        errorFound = true;
+      }
+
+      if (!DataTypeUtils::validDataType(ArrayOptions::dataType(newShapeInfoCast))) {
+        errorMessage += "Invalid data type in shape buffer. ";
+        errorFound = true;
+      }
+    } else if (dataBuffer->dataBuffer() != nullptr && (dataBuffer->dataBuffer()->primary() != nullptr || dataBuffer->dataBuffer()->special() != nullptr)) {
+      errorMessage += "Shape Buffer at index " + std::to_string(index) + " is marked as empty but data buffer is not null! ";
+      errorFound = true;
+    }
+  }
+
+  if (errorFound) {
+    errorMessage += "Shape info: " + ShapeUtils::shapeAsString(newShapeInfoCast) + ". ";
+    errorMessage += "Data type: " + DataTypeUtils::asString(ArrayOptions::dataType(newShapeInfoCast)) + ". ";
+    if (dataBuffer->dataBuffer() != nullptr) {
+      errorMessage += "Data buffer: " + std::string(dataBuffer->dataBuffer()->primary() != nullptr ? "not null" : "null") + ". ";
+      errorMessage += "Special buffer: " + std::string(dataBuffer->dataBuffer()->special() != nullptr ? "not null" : "null") + ". ";
+    }
+    errorMessage += "Elements: ";
+    for(int i = 0; i < shape::shapeInfoLength(newShapeInfoCast); i++) {
+      errorMessage += std::to_string(newShapeInfoCast[i]) + ", ";
+    }
+    errorMessage += "\n";
+
+    THROW_EXCEPTION(errorMessage.c_str());
+  }
+}
+
+
+
+void Context::setTArguments(double *arguments, int numberOfArguments) {
+  _tArgs.clear();
+  _tArgs.reserve(numberOfArguments);
+  for (int e = 0; e < numberOfArguments; e++) _tArgs.push_back(arguments[e]);
+  if(Environment::getInstance().isDebug() || Environment::getInstance().isVerbose()) {
+    printf("float values set in context: ");
+    for (auto d : _bArgs) {
+      printf("%s\n, ", std::to_string(d).c_str());
+    }
+    fflush(stdout);
+  }
+}
+
+void Context::setIArguments(LongType *arguments, int numberOfArguments) {
+  _iArgs.clear();
+  _iArgs.reserve(numberOfArguments);
+  for (int e = 0; e < numberOfArguments; e++) _iArgs.push_back(arguments[e]);
+  if(Environment::getInstance().isDebug() || Environment::getInstance().isVerbose()) {
+    printf("int arguments set in context: ");
+    for (auto d : _bArgs) {
+      printf("%s\n, ", std::to_string(d).c_str());
+    }
+    fflush(stdout);
+  }
+}
+
+void Context::setBArguments(bool *arguments, int numberOfArguments) {
+  _bArgs.clear();
+  _bArgs.reserve(numberOfArguments);
+  for (int e = 0; e < numberOfArguments; e++) _bArgs.push_back(arguments[e]);
+  if(Environment::getInstance().isDebug() || Environment::getInstance().isVerbose()) {
+    printf("boolean types set in context: ");
+    for (auto d : _bArgs) {
+      printf("%s\n, ", std::to_string(d).c_str());
+    }
+    fflush(stdout);
+  }
+}
+
+void Context::setCudaContext(Pointer cudaStream, Pointer reductionPointer, Pointer allocationPointer) {
+#ifdef SD_CUDA
+  _context = new LaunchContext(cudaStream, reductionPointer, allocationPointer);
+
+  // FIXME: either pass handle from outside, or make sure outside we use the same handle
+  _context->setCublasHandle(LaunchContext::defaultContext()->getCublasHandle());
+
+  for (auto v : _fastpath_out) v->setContext(_context);
+
+  for (auto v : _fastpath_in) v->setContext(_context);
+#endif
+}
+
+void Context::allowHelpers(bool reallyAllow) { _helpersAllowed = reallyAllow; }
+
+bool Context::helpersAllowed() { return _helpersAllowed; }
+
+void Context::setTArguments(const std::vector<double> &tArgs) {
+  for (auto t : tArgs) _tArgs.emplace_back(t);
+  if(Environment::getInstance().isDebug() || Environment::getInstance().isVerbose()) {
+    printf("t argument types set in context: ");
+    for (auto d : _bArgs) {
+      printf("%s\n, ", std::to_string(d).c_str());
+    }
+    fflush(stdout);
+  }
+}
+
+void Context::setIArguments(const std::vector<LongType> &iArgs) {
+  for (auto i : iArgs) _iArgs.emplace_back(i);
+  if(Environment::getInstance().isDebug() || Environment::getInstance().isVerbose()) {
+    printf("int argument types set in context: ");
+    for (auto d : iArgs) {
+      printf("%s\n, ", std::to_string(d).c_str());
+    }
+    fflush(stdout);
+  }
+}
+
+void Context::setBArguments(const std::vector<bool> &bArgs) {
+  for (auto b : bArgs) _bArgs.push_back(b);
+  if(Environment::getInstance().isDebug() || Environment::getInstance().isVerbose()) {
+    printf("boolean types set in context: ");
+    for (auto d : _bArgs) {
+      printf("%s\n, ", std::to_string(d).c_str());
+    }
+    fflush(stdout);
+  }
+}
+
+void Context::setShapeFunctionOverride(bool reallyOverride) { _shapeFunctionOverride = reallyOverride; }
+
+bool Context::shapeFunctionOverride() { return _shapeFunctionOverride; }
+
+samediff::ExecutionMode Context::executionMode() { return _execMode; }
+
+void Context::setExecutionMode(samediff::ExecutionMode executionMode) { _execMode = executionMode; }
+
+bool Context::isTraining() { return _execMode == samediff::ExecutionMode::MODE_TRAINING; }
+
+bool Context::isInference() { return _execMode == samediff::ExecutionMode::MODE_INFERENCE; }
+
+void Context::setDArguments(DataType *arguments, int numberOfArguments) {
+  _dArgs.clear();
+  for (int e = 0; e < numberOfArguments; e++) _dArgs.emplace_back(arguments[e]);
+  if(Environment::getInstance().isDebug() || Environment::getInstance().isVerbose()) {
+    printf("data types set in context: ");
+    for (auto d : _dArgs) {
+      printf("%s\n, ", DataTypeUtils::asString(d).c_str());
+    }
+    fflush(stdout);
+  }
+}
+
+void Context::setDArguments(const std::vector<DataType> &dArgs) {
+  _dArgs.clear();
+  for (auto d : dArgs) _dArgs.emplace_back(d);
+  if(Environment::getInstance().isDebug() || Environment::getInstance().isVerbose()) {
+    printf("data types set in context: ");
+    for (auto d : dArgs) {
+      printf("%s\n, ", DataTypeUtils::asString(d).c_str());
+    }
+    fflush(stdout);
+  }
+}
+
+void Context::clearFastPath() {
+  _fastpath_in.clear();
+  _fastpath_out.clear();
+
+  // Delete arrays in _handles before clearing (fixes memory leak)
+  for (auto v : _handles) {
+    if (v != nullptr) delete v;
+  }
+  _handles.clear();
+}
+
+void Context::setInputArrays(int numArrays,NDArray** array, bool removable) {
+  for(int i = 0; i < numArrays; i++) {
+    setInputArray(i,array[i],removable);
+  }
+}
+
+void Context::setOutputArrays(int numArrays,NDArray** array, bool removable) {
+  for(int i = 0; i < numArrays; i++) {
+    setOutputArray(i,array[i],removable);
+  }
+}
+
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/ContextPrototype.cpp

@@ -0,0 +1,146 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <graph/ContextPrototype.h>
+#include <types/float16.h>
+
+namespace sd {
+namespace graph {
+ContextPrototype::ContextPrototype(ops::OpDescriptor* opDescriptor, int nodeId, bool inPlace) {
+  _nodeId = nodeId;
+  _isInplace = inPlace;
+  _opDescriptor = opDescriptor;
+}
+
+void ContextPrototype::pickInput(std::pair<int, int>& p) { this->_inputs.emplace_back(p); }
+
+void ContextPrototype::pickInput(int input, int index) {
+  std::pair<int, int> pair(input, index);
+  pickInput(pair);
+}
+
+int ContextPrototype::opNum() { return this->_opNum; }
+
+void ContextPrototype::setOpNum(int opNum) { this->_opNum = opNum; }
+
+std::vector<std::pair<int, int>>* ContextPrototype::inputs() { return &_inputs; }
+
+void ContextPrototype::fillInputs(std::vector<int>& inputs) {
+  for (size_t e = 0; e < inputs.size(); e++) {
+    auto v = inputs.at(e);
+    pickInput(v);
+  }
+}
+
+samediff::Engine ContextPrototype::engine() { return _engine; }
+
+bool ContextPrototype::hasVariablesFilled() { return this->_inputs.size() > 0; }
+
+bool ContextPrototype::isInplace() { return this->_isInplace; }
+
+std::vector<double>* ContextPrototype::getTArguments() { return &(this->_tArgs); }
+
+std::vector<LongType>* ContextPrototype::getIArguments() { return &(this->_iArgs); }
+
+std::vector<bool>* ContextPrototype::getBArguments() { return &(this->_bArgs); }
+
+std::vector<LongType>* ContextPrototype::getAxis() { return &(this->_axis); }
+
+std::vector<std::string> * ContextPrototype::getSArguments() {return &(this->_sArgs);}
+void ContextPrototype::pickInput(int input) {
+  std::pair<int, int> pair(input, 0);
+  this->_inputs.emplace_back(pair);
+}
+
+std::pair<int, int>* ContextPrototype::input(int idx) { return &(this->_inputs.at(idx)); }
+
+void ContextPrototype::fillInputs(std::initializer_list<int> inputs) {
+  for (auto v : inputs) {
+    pickInput(v);
+  }
+}
+
+int ContextPrototype::nodeId() { return getNodeId(); }
+
+DataType ContextPrototype::dataType() { return dataType(0); }
+
+DataType ContextPrototype::dataType(int index) {
+  if(numD() < 1) {
+    THROW_EXCEPTION("No data types were set for this ContextPrototype");
+  } else {
+    return _dArgs.at(index);
+  }
+
+  return DataType::UNKNOWN;
+}
+
+void ContextPrototype::setDataType(int index, DataType type) {
+  _dArgs[index] = type;
+}
+
+size_t ContextPrototype::numT() { return (int)_tArgs.size(); }
+
+size_t ContextPrototype::numI() { return (int)_iArgs.size(); }
+
+size_t ContextPrototype::numB() { return (int)_bArgs.size(); }
+
+int ContextPrototype::getNodeId() { return this->_nodeId; }
+
+/**
+ * This method returns number of inputs available in this block
+ * @return
+ */
+unsigned long ContextPrototype::width() { return this->_inputs.size(); };
+
+void ContextPrototype::markInplace(bool reallyInplace) { this->_isInplace = reallyInplace; }
+
+template <typename N>
+ContextPrototype* ContextPrototype::asT() {
+  auto clone = new ContextPrototype(_opDescriptor, _nodeId, _isInplace);
+
+  return clone;
+}
+
+void ContextPrototype::setOpDescriptor(ops::OpDescriptor* opDescriptor) { _opDescriptor = opDescriptor; }
+
+ContextPrototype* ContextPrototype::clone() {
+  auto clone = new ContextPrototype(_opDescriptor, _nodeId, _isInplace);
+  clone->_opNum = _opNum;
+
+  for (auto v : _inputs) clone->_inputs.emplace_back(v);
+
+  for (auto v : _tArgs) clone->_tArgs.emplace_back(v);
+
+  for (auto v : _iArgs) clone->_iArgs.emplace_back(v);
+
+  for(auto v : _bArgs) clone->_bArgs.emplace_back(v);
+
+  for(auto v : _dArgs) clone->_dArgs.emplace_back(v);
+
+  return clone;
+}
+
+std::vector<DataType>* ContextPrototype::getDArguments() { return &_dArgs; }
+
+size_t ContextPrototype::numD() { return _dArgs.size(); }
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/ExecutionResult.cpp

@@ -0,0 +1,100 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <graph/ExecutionResult.h>
+#include <system/common.h>
+#include <system/op_boilerplate.h>
+
+namespace sd {
+namespace graph {
+ExecutionResult::ExecutionResult(const ::graph::FlatResult* flatResult) {
+  if (flatResult->variables() != nullptr) {
+    for (size_t e = 0; e < flatResult->variables()->size(); e++) {
+      auto fv = flatResult->variables()->Get(e);
+      auto v = new Variable(fv);
+      this->emplace_back(v);
+    }
+
+    _releasable = true;
+  }
+}
+
+ExecutionResult::~ExecutionResult() {
+  if (_releasable)
+    for (auto v : _variables) delete v;
+}
+
+LongType ExecutionResult::size() { return _variables.size(); }
+
+ExecutionResult::ExecutionResult(std::initializer_list<Variable*> variables) {
+  for (auto v : variables) this->emplace_back(v);
+}
+
+void ExecutionResult::emplace_back(Variable* variable) {
+  _variables.emplace_back(variable);
+
+  if (variable->getName() != nullptr) _stringIdMap[*variable->getName()] = variable;
+
+  std::pair<int, int> p(variable->id(), variable->index());
+  _pairIdMap[p] = variable;
+}
+
+Variable* ExecutionResult::at(int position) {
+  if (static_cast<size_t>(position) >= _variables.size())
+    THROW_EXCEPTION("Position index is higher then number of variables stored");
+
+  return _variables.at(position);
+}
+
+Variable* ExecutionResult::byId(std::string& id) {
+  if (_stringIdMap.count(id) == 0) THROW_EXCEPTION("Can't find specified ID");
+
+  return _stringIdMap.at(id);
+}
+
+Variable* ExecutionResult::byId(std::pair<int, int>& id) {
+  if (_pairIdMap.count(id) == 0) THROW_EXCEPTION("Can't find specified ID");
+
+  return _pairIdMap.at(id);
+}
+
+Variable* ExecutionResult::byId(int id) {
+  std::pair<int, int> p(id, 0);
+  return byId(p);
+}
+
+Variable* ExecutionResult::byId(const char* str) {
+  std::string p(str);
+  return byId(p);
+}
+
+flatbuffers::Offset<::graph::FlatResult> ExecutionResult::asFlatResult(flatbuffers::FlatBufferBuilder& builder) {
+  std::vector<flatbuffers::Offset<::graph::FlatVariable>> vec;
+  for (Variable* v : _variables) {
+    vec.emplace_back(v->asFlatVariable(builder));
+  }
+
+  auto vecOffset = builder.CreateVector(vec);
+
+  return CreateFlatResult(builder, 0, vecOffset);
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/ExecutorConfiguration.cpp

@@ -0,0 +1,62 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+#include <graph/ExecutorConfiguration.h>
+
+namespace sd {
+namespace graph {
+ExecutorConfiguration::ExecutorConfiguration(const ::graph::FlatConfiguration *conf) {
+  if (conf != nullptr) {
+    _profilingMode = conf->profilingMode();
+    _executionMode = conf->executionMode();
+    _outputMode = conf->outputMode();
+    _timestats = conf->timestats();
+    _footprintForward = conf->footprintForward();
+    _footprintBackward = conf->footprintBackward();
+    _direction = conf->direction();
+  } else {
+    _profilingMode = ::graph::ProfilingMode_NONE;
+    _executionMode = ::graph::ExecutionMode_SEQUENTIAL;
+    _outputMode = ::graph::OutputMode_IMPLICIT;
+    _timestats = false;
+  }
+};
+
+ExecutorConfiguration *ExecutorConfiguration::clone() {
+  auto clone = new ExecutorConfiguration();
+  clone->_profilingMode = _profilingMode;
+  clone->_executionMode = _executionMode;
+  clone->_outputMode = _outputMode;
+  clone->_timestats = _timestats;
+  clone->_direction = _direction;
+  clone->_footprintForward = _footprintForward;
+  clone->_footprintBackward = _footprintBackward;
+
+  return clone;
+};
+
+flatbuffers::Offset<::graph::FlatConfiguration> ExecutorConfiguration::asFlatConfiguration(
+    flatbuffers::FlatBufferBuilder &builder) {
+  return CreateFlatConfiguration(builder, 0, _executionMode, _profilingMode, _outputMode, _timestats,
+                                 _footprintBackward, _footprintBackward);
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/FlatUtils.cpp

@@ -0,0 +1,114 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 22.11.2017.
+//
+#include <array/ByteOrder.h>
+#include <array/ByteOrderUtils.h>
+#include <array/DataTypeConversions.h>
+#include <array/DataTypeUtils.h>
+#include <array/NDArrayFactory.h>
+#include <graph/FlatUtils.h>
+
+namespace sd {
+namespace graph {
+std::pair<int, int> FlatUtils::fromIntPair(::graph::IntPair *pair) { return std::pair<int, int>(pair->first(), pair->second()); }
+
+std::pair<LongType, LongType> FlatUtils::fromLongPair(::graph::LongPair *pair) {
+  return std::pair<LongType, LongType>(pair->first(), pair->second());
+}
+
+NDArray *FlatUtils::fromFlatArray(const ::graph::FlatArray *flatArray) {
+  auto rank = static_cast<int>(flatArray->shape()->Get(0));
+  auto newShape = new LongType[shape::shapeInfoLength(rank)];
+  memcpy(newShape, flatArray->shape()->data(), shape::shapeInfoByteLength(rank));
+
+  auto length = shape::length(newShape);
+  auto dtype = DataTypeUtils::fromFlatDataType(flatArray->dtype());
+
+  // empty arrays is special case, nothing to restore here
+  if (shape::isEmptyConst(newShape)) {
+     delete[] newShape;
+    return NDArrayFactory::empty_(dtype, nullptr);
+  }
+  // TODO fix UTF16 and UTF32
+  if (dtype == UTF8) {
+
+    std::vector<std::string> substrings(length);
+    std::vector<LongType> shapeVector(rank);
+    for (int e = 0; e < rank; e++) shapeVector[e] = newShape[e + 1];
+
+    auto rawPtr = (void *)flatArray->buffer()->data();
+    auto longPtr = reinterpret_cast<LongType *>(rawPtr);
+    auto charPtr = reinterpret_cast<char *>(longPtr + length + 1);
+    auto offsets = new LongType[length + 1];
+
+    for (LongType e = 0; e <= length; e++) {
+      auto o = longPtr[e];
+      // FIXME: BE vs LE on partials
+      // auto v = canKeep ?  o : BitwiseUtils::swap_bytes<sd::LongType>(o);
+      offsets[e] = o;
+    }
+
+    for (LongType e = 0; e < length; e++) {
+      auto start = offsets[e];
+      auto end = offsets[e + 1];
+      auto len = end - start;
+
+      auto c = (char *)malloc(len + 1);
+      CHECK_ALLOC(c, "Failed temp allocation", len + 1);
+      memset(c, '\0', len + 1);
+      memcpy(c, charPtr + start, len);
+
+      std::string val(c);
+      substrings[e] = val;
+      free(c);
+    }
+
+    delete[] offsets;
+    delete[] newShape;
+    // string order always 'c'
+    return NDArrayFactory::string_(shapeVector, substrings);
+  }
+
+  auto newBuffer = new int8_t[length * DataTypeUtils::sizeOf(dtype)];
+
+  BUILD_SINGLE_SELECTOR(dtype, DataTypeConversions,
+                        ::convertType(newBuffer, (void *)flatArray->buffer()->data(), dtype,
+                                      ByteOrderUtils::fromFlatByteOrder(flatArray->byteOrder()), length),
+                        SD_COMMON_TYPES);
+
+  auto array = new NDArray(newBuffer, newShape, LaunchContext::defaultContext(), true, 0);
+
+  delete[] newShape;
+  return array;
+}
+
+flatbuffers::Offset<::graph::FlatArray> FlatUtils::toFlatArray(flatbuffers::FlatBufferBuilder &builder, NDArray &array) {
+  auto byteVector = array.asByteVector();
+
+  auto fBuffer = builder.CreateVector(byteVector);
+  auto fShape = builder.CreateVector(array.getShapeInfoAsFlatVector());
+
+  auto bo = static_cast<::graph::ByteOrder>(BitwiseUtils::asByteOrder());
+
+  return CreateFlatArray(builder, fShape, fBuffer, static_cast<::graph::DType>(array.dataType()), bo);
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/FlowPath.cpp

@@ -0,0 +1,131 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16/11/17.
+//
+#include <graph/FlowPath.h>
+
+namespace sd {
+namespace graph {
+
+void FlowPath::ensureNode(int nodeId) {
+  if (_states.count(nodeId) == 0) {
+    NodeState state(nodeId);
+    _states[nodeId] = state;
+  }
+}
+
+void FlowPath::ensureFrame(int frameId) {
+  if (_frames.count(frameId) == 0) {
+    FrameState state(frameId);
+    _frames[frameId] = state;
+  }
+}
+
+void FlowPath::setInnerTime(int nodeId, LongType time) {
+  ensureNode(nodeId);
+
+  _states[nodeId].setInnerTime(time);
+}
+
+void FlowPath::setOuterTime(int nodeId, LongType time) {
+  ensureNode(nodeId);
+
+  _states[nodeId].setOuterTime(time);
+}
+
+LongType FlowPath::innerTime(int nodeId) {
+  ensureNode(nodeId);
+
+  return _states[nodeId].innerTime();
+}
+
+LongType FlowPath::outerTime(int nodeId) {
+  ensureNode(nodeId);
+
+  return _states[nodeId].outerTime();
+}
+
+bool FlowPath::isNodeActive(int nodeId) {
+  ensureNode(nodeId);
+
+  return _states[nodeId].isActive();
+}
+
+void FlowPath::markNodeActive(int nodeId, bool isActive) {
+  ensureNode(nodeId);
+
+  _states[nodeId].markActive(isActive);
+}
+
+int FlowPath::branch(int nodeId) {
+  ensureNode(nodeId);
+
+  return _states[nodeId].branch();
+}
+
+void FlowPath::markBranch(int nodeId, int index) {
+  ensureNode(nodeId);
+
+  _states[nodeId].markBranch(index);
+}
+
+bool FlowPath::isFrameActive(LongType frameId) {
+  ensureFrame(frameId);
+
+  return _frames[frameId].wasActivated();
+}
+
+void FlowPath::markFrameActive(LongType frameId, bool isActive) {
+  ensureFrame(frameId);
+
+  _frames[frameId].markActivated(isActive);
+}
+
+bool FlowPath::isRewindPlanned(LongType frameId) { return _frames[frameId].isRewindPlanned(); }
+
+void FlowPath::planRewind(LongType frameId, bool reallyRewind) { _frames[frameId].planRewind(reallyRewind); }
+
+int FlowPath::getRewindPosition(LongType frameId) { return _frames[frameId].getRewindPosition(); }
+
+void FlowPath::setRewindPosition(LongType frameId, int position) { _frames[frameId].setRewindPosition(position); }
+
+void FlowPath::setRewindPositionOnce(LongType frameId, int position) {
+  _frames[frameId].setRewindPositionOnce(position);
+}
+
+void FlowPath::registerFrame(LongType frameId) {
+  if (_frames.count(frameId) == 0) ensureFrame(frameId);
+}
+
+void FlowPath::forgetFrame(LongType frameId) {
+  if (_frames.count(frameId) > 0) _frames.erase(frameId);
+}
+
+void FlowPath::incrementNumberOfCycles(LongType frameId) { _frames[frameId].incrementNumberOfCycles(); }
+
+LongType FlowPath::getNumberOfCycles(LongType frameId) { return _frames[frameId].getNumberOfCycles(); }
+
+bool FlowPath::wasExecuted(int nodeId) { return _states[nodeId].wasExecuted(); }
+
+void FlowPath::markExecuted(int nodeId, bool wasExecuted) { _states[nodeId].markExecuted(wasExecuted); }
+
+GraphProfile* FlowPath::profile() { return &_profile; }
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/FrameState.cpp

@@ -0,0 +1,50 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 06.02.2018.
+//
+#include <graph/FrameState.h>
+
+namespace sd {
+namespace graph {
+FrameState::FrameState(sd::LongType id) { this->_id = id; }
+
+int FrameState::getNumberOfCycles() { return _numberOfCycles; }
+
+void FrameState::incrementNumberOfCycles() { ++_numberOfCycles; }
+
+bool FrameState::wasActivated() { return _activated; }
+
+void FrameState::markActivated(bool reallyActivated) { _activated = reallyActivated; }
+
+std::string &FrameState::getFrameName() { return _name; }
+
+bool FrameState::isRewindPlanned() { return _rewindPlanned; }
+
+int FrameState::getRewindPosition() { return _rewindPosition; }
+
+void FrameState::setRewindPosition(int pos) { _rewindPosition = pos; }
+
+void FrameState::setRewindPositionOnce(int pos) {
+  if (_rewindPosition < 0) _rewindPosition = pos;
+}
+
+void FrameState::planRewind(bool reallyPlanning) { _rewindPlanned = reallyPlanning; }
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/GraphExecutioner.cpp

@@ -0,0 +1,665 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <graph/generated/graph_generated.h>
+#include <graph/generated/node_generated.h>
+#include <graph/generated/result_generated.h>
+
+#include <graph/GraphExecutioner.h>
+#include <graph/Node.h>
+#include <graph/Scope.h>
+#include <graph/TimeHolder.h>
+#include <graph/Variable.h>
+#include <graph/VariableSpace.h>
+#include <loops/pairwise_transform.h>
+#include <loops/scalar.h>
+#include <loops/transform_same.h>
+#include <memory/MemoryRegistrator.h>
+#include <ops/declarable/DeclarableOp.h>
+
+
+#include <array/DataTypeUtils.h>
+#include <exceptions/graph_execution_exception.h>
+#include <exceptions/no_results_exception.h>
+#include <fcntl.h>
+#include <graph/ExecutionResult.h>
+#include <graph/FlatUtils.h>
+#include <graph/ResultWrapper.h>
+#include <graph/execution/LogicExecutor.h>
+#include <graph/generated/array_generated.h>
+#include <helpers/BitwiseUtils.h>
+#include <helpers/ShapeUtils.h>
+
+#include <chrono>
+#include <ctime>
+#include <deque>
+
+namespace sd {
+namespace graph {
+
+/**
+ * This method executes given Node (as in Op within Node)
+ *
+ * Basically it just does DeclarableOp::execute(Block), and ops to their job. However, there are some additional
+ * functionality.
+ *
+ * @param graph - Graph instance pointer
+ * @param node - Node instance pointer, which will be executed
+ * @param variableSpace - VariableSpace instance pointer - varspace specific to current Thread/Session
+ * @return
+ */
+Status GraphExecutioner::executeFlatNode(Graph *graph, Node *node, VariableSpace *variableSpace) {
+  ::graph::OpType opType = node->opType();
+  int opNum = node->opNum();
+  //    std::string opName = *(node->getCustomOp()->getOpName());
+
+  if (opType == ::graph::OpType_BOOLEAN) {
+    sd_debug("Executing boolean graph node_%i", node->id());
+  } else if (opType == ::graph::OpType_LOGIC) {
+    sd_debug("Executing logic graph node_%i", node->id());
+  } else if (opType == ::graph::OpType_GRAPH) {
+    sd_debug("Executing embedded graph node_%i", node->id());
+  } else if (opType != ::graph::OpType_CUSTOM) {
+    sd_debug("Executing node_%i{%i}\n", node->id(), opNum);
+  } else {
+    sd_debug("Executing node_%i{%s}\n", node->id(), node->getCustomOp()->getOpName()->c_str());
+  }
+
+  Context context(node->getContextPrototype(), variableSpace);
+
+  if (Environment::getInstance().isDebugAndVerbose()) {
+    // sd_debug("Input variables: %i\n", node->input()->size());
+    printf("       Inputs: {");
+    for (size_t e = 0; e < node->input()->size(); e++) {
+      printf("[%i:%i]", node->input()->at(e).first, node->input()->at(e).second);
+
+      if (e < node->input()->size() - 1) printf(", ");
+    }
+    printf("}\n");
+    fflush(stdout);
+  }
+
+  if (node->id() == 13) sd_debug("", "");
+
+  // if true - this is special case: Graph-in-Graph.
+  if (node->hasGraphEmbedded()) {
+    auto embedded = node->getGraph();
+
+    /**
+     * basically, we should do following things here:
+     * 1) fill embedded graph with input variables from this graph, if anything should be filled in
+     * 2) invoke embedded graph
+     * 3) announce its results as corresponding output variables in current VariableSpace
+     */
+
+    // enforcing IMPLICIT mode. or not... should we try to be smarter then user?
+    // embedded->getExecutorConfiguration()->_outputMode = OutputMode_IMPLICIT;
+
+    if (node->input()->size() != static_cast<size_t>(embedded->numberOfPlaceholders())) {
+      sd_debug("Placeholders amount mismatch: %i expected, and %i available\n", node->input()->size(),
+               embedded->numberOfPlaceholders());
+      return Status::BAD_INPUT;
+    }
+
+    // we need to propagate required variables to the embedded graph
+    ResultSet deletables;
+    int cnt = 0;
+    for (Variable *v : *embedded->getPlaceholders()) {
+      if (v->getName() != nullptr && v->getName()->size() > 0) {
+        // trying symbolic lookup first
+        if (variableSpace->hasVariable(v->getName())) {
+          // symbolic feeder
+          auto array = variableSpace->getVariable(v->getName())->getNDArray();
+          auto vr = array->dup(array->ordering());
+          v->setNDArray(vr);
+        } else {
+          sd_debug("Can't find variable [%s] in parent graph...", v->getName()->c_str());
+          return Status::BAD_INPUT;
+        }
+      } else {
+        // if we're not using symbolic lookup - we'll use sequential approach then
+        auto p = node->input()->at(cnt);
+        auto array = variableSpace->getVariable(p)->getNDArray();
+        auto vr = array->dup(array->ordering());  // dup() already returns NDArray*
+        v->setNDArray(vr);
+      }
+
+      cnt++;
+    }
+
+    // executing embedded graph as independent one
+    Status status = execute(embedded);
+    if (status != Status::OK) return status;
+
+    //  now we should migrate its results to this node, as its own outputs
+    cnt = 0;
+    auto outputs = embedded->fetchOutputs();
+
+    for (auto v : *outputs) {
+      NDArray *array = v->getNDArray();
+      v->setNDArray(nullptr);
+
+      std::pair<int, int> pair(node->id(), cnt++);
+
+      auto var = variableSpace->getVariable(pair);
+
+      var->setNDArray(array);
+      var->markRemovable(true);
+    }
+    deletables.size();
+    delete outputs;
+    sd_debug("Embedded graph execution finished. %i variable(s) migrated\n", cnt);
+
+  } else if (node->hasCustomOp()) {
+    // now, if we have something to execute - lets just execute it.
+    auto status = node->getCustomOp()->execute(&context);
+    if (status != Status::OK) return status;
+
+    // propagate variables
+    if (node->hasExternalOutputs()) {
+      for (auto v : *node->output()) {
+        if (variableSpace->hasExternalVariable(v.first)) {
+          variableSpace->getVariable(v.first)->getNDArray()->assign(
+              variableSpace->getVariable(node->id())->getNDArray());
+        }
+      }
+    }
+
+    return status;
+  }
+  return Status::OK;
+}
+
+/**
+ * This method executes given Graph instance, and returns error code.
+ *
+ * @param graph
+ * @return one of error codes defined in pointercast.h
+ */
+Status GraphExecutioner::execute(Graph *graph, VariableSpace *variableSpace) {
+  auto __variableSpace = variableSpace == nullptr ? graph->getVariableSpace() : variableSpace;
+
+  bool tempFlow = false;
+  if (__variableSpace->flowPath() == nullptr) {
+    tempFlow = true;
+    __variableSpace->setFlowPath(new FlowPath());
+  }
+  auto flowPath = __variableSpace->flowPath();
+
+  LongType tb0 = Environment::getInstance().isProfiling() ? GraphProfile::currentTime() : 0L;
+  graph->buildGraph();
+
+  auto footprintForward = memory::MemoryRegistrator::getInstance().getGraphMemoryFootprint(graph->hashCode());
+  if (footprintForward > 0) {
+    if (__variableSpace->launchContext()->getWorkspace() != nullptr) {
+      // this method will work only if current workspace size is smaller then proposed value
+      sd_debug("Setting workspace to %lld bytes\n", footprintForward);
+      __variableSpace->launchContext()->getWorkspace()->expandTo(footprintForward);
+    }
+  }
+
+  // optionally saving graph build time
+  if (Environment::getInstance().isProfiling()) flowPath->profile()->setBuildTime(GraphProfile::relativeTime(tb0));
+
+  LongType timeStart = Environment::getInstance().isProfiling() ? GraphProfile::currentTime() : 0L;
+
+  bool pe = graph->getExecutorConfiguration()->_executionMode == ::graph::ExecutionMode_AUTO;
+
+  // basically if at some point code diverges, code branch might be _DISABLED_, and all nodes within that branch will be
+  // disabled as well
+
+  std::deque<LongType> frames;
+  bool leftFrame = false;
+
+  auto nodeTime = GraphProfile::currentTime();
+  int lastId = -10000000;
+  LongType exec_counter = 0;
+  // we loop through op layers here
+  for (int l = 0; l < (int)graph->getOnion()->size(); l++) {
+    int layerSize = graph->getOnion()->count(l) == 1 ? graph->getOnion()->at(l)->size() : 0;
+
+    int n = 0;
+    // this omp block will probably never be the case
+    for (; n < layerSize; n++) {
+      if (++exec_counter > 10000) {
+        l = graph->getOnion()->size();
+        return Logger::logKernelFailureMsg("Early termination hit");
+      }
+
+      Node *node = graph->getOnion()->at(l)->at(n);
+
+      if (Environment::getInstance().isProfiling()) flowPath->profile()->nodeById(node->id(), node->name()->c_str());
+
+      if (lastId != node->id() && Environment::getInstance().isProfiling()) {
+        if (lastId != -10000000)
+          flowPath->profile()->nodeById(lastId)->setTotalTime(GraphProfile::relativeTime(nodeTime));
+
+        lastId = node->id();
+        nodeTime = GraphProfile::currentTime();
+      }
+
+      sd_debug("Step: %lld; Node: %i <%s>\n", exec_counter, node->id(), node->name()->c_str());
+
+      // on first non-Exit node after loop we can rewind (if planned)
+      if (!(node->opType() == ::graph::OpType_LOGIC && node->opNum() == logic::Exit)) {
+        // VALIDATED
+
+        // if we're out of frame - let's remove it from queue
+        if (leftFrame) {
+          auto frame_id = frames.back();
+          frames.pop_back();
+          flowPath->markFrameActive(frame_id, false);
+          flowPath->forgetFrame(frame_id);
+
+          leftFrame = false;
+        }
+
+        // TODO: move inactivity check right here
+        bool shouldSkip = false;
+        if (node->opType() == ::graph::OpType_LOGIC && node->opNum() == logic::Merge) {
+          // Merge node has own checkout logic
+
+          auto inputId0 = node->input()->at(0);
+          auto inputId1 = node->input()->at(1);
+
+          // Merge node can be skipped only both inputs are inactive
+          if (!flowPath->isNodeActive(inputId0.first) && !flowPath->isNodeActive(inputId1.first)) shouldSkip = true;
+
+        } else {
+          // let's check for input nodes, if they are disabled or contain divergents
+          for (size_t e = 0; e < node->input()->size(); e++) {
+            auto inputId = node->input()->at(e);
+
+            // not a node. skipping checks
+            if (graph->getMapped()->count(inputId.first) == 0) continue;
+
+            /**
+             * We can skip current node, in two cases:
+             * 1) If previous node was disabled
+             * 2) If previous node was divergent node (i.e. IF op) and code went other way
+             */
+            Node *prevNode = graph->getMapped()->at(inputId.first);
+            if (!flowPath->isNodeActive(inputId.first)) {
+              shouldSkip = true;
+              flowPath->markNodeActive(node->id(), false);
+
+              sd_debug("Skipping Node_%i due to inactive input [%i]\n", node->id(), inputId.first);
+              break;
+
+            } else if (prevNode->isDivergencePoint()) {  // literally checking for switch here
+              if (flowPath->branch(inputId.first) != inputId.second) {
+                shouldSkip = true;
+                flowPath->markNodeActive(node->id(), false);
+                sd_debug("Skipping Node_%i due to divergent branch [%i]\n", node->id(), inputId.first);
+                break;
+              }
+            }
+          }
+        }
+
+        if (shouldSkip) continue;
+      }
+
+      // we're propagating frameId here (but only if wasn't set earlier)
+      if (frames.size() > 0 && node->getFrameId() < 0) node->setFrameId(frames.back());
+
+      flowPath->markNodeActive(node->id(), true);
+
+      if (node->opType() == ::graph::OpType_LOGIC && node->opNum() == logic::Enter) {
+        // Enter operation
+        // VALIDATED
+
+        // we expect this node to have frameId set
+        auto frame_id = node->getFrameId();
+
+        // new frame starts here
+        if (frames.size() == 0 || (frames.size() > 0 && frames.back() != frame_id)) {
+          flowPath->registerFrame(frame_id);
+          frames.emplace_back(frame_id);
+        }
+
+        auto status = LogicExecutor::processNode(graph, node);
+        if (status != Status::OK) return status;
+
+      } else if (node->opType() == ::graph::OpType_LOGIC && node->opNum() == logic::NextIteration) {
+        /**
+         * NextIteration is special case: after successful execution of this op - we're changing execution position
+         */
+        // VALIDATED
+
+        auto status = LogicExecutor::processNode(graph, node);
+        if (status != Status::OK) return status;
+
+        auto frame_id = frames.back();
+
+        flowPath->markNodeActive(node->id(), true);
+        flowPath->markExecuted(node->id(), true);
+
+        if (!flowPath->isRewindPlanned(frame_id)) {
+          auto nextLayer = node->getRewindLayer();
+
+          sd_debug("Node_%i planned rewind to Node_%i at [%i:%i]\n", node->id(), node->getRewindNode(), nextLayer.first,
+                   nextLayer.second);
+
+          flowPath->planRewind(frame_id, true);
+          flowPath->setRewindPositionOnce(frame_id, nextLayer.first - 1);
+
+          continue;
+        }
+
+      } else if (node->opType() == ::graph::OpType_LOGIC && node->opNum() == logic::Exit) {
+        // Exit node is another special case: it can rewind executioner to specific point in graph
+        // VALIDATED
+
+        auto frame_id = frames.back();
+
+        // if this loop frame wasn't activated - just skip it
+        if (!flowPath->isFrameActive(frame_id)) {
+          flowPath->markNodeActive(node->id(), false);
+
+          leftFrame = true;
+          continue;
+        }
+
+        if (flowPath->isRewindPlanned(frame_id)) {
+          // just break loop here
+          l = flowPath->getRewindPosition(frame_id);
+          flowPath->setRewindPosition(frame_id, -1);
+          flowPath->planRewind(frame_id, false);
+
+          break;
+        } else {
+          // execute Exit node otherwise
+
+          auto status = LogicExecutor::processNode(graph, node);
+          if (status != Status::OK) return status;
+
+          leftFrame = true;
+        }
+
+      } else if (node->opType() == ::graph::OpType_LOGIC) {
+        /**
+         * If this LOGIC op, we'll use another execution model here
+         */
+        auto status = LogicExecutor::processNode(graph, node);
+
+        if (status != Status::OK) return status;
+      } else {
+        auto timeStart2 = std::chrono::system_clock::now();
+
+        // actual node execution happens right here
+        Status status = executeFlatNode(graph, node, __variableSpace);
+
+        auto timeEnd = std::chrono::system_clock::now();
+
+        auto outerTime = std::chrono::duration_cast<std::chrono::nanoseconds>(timeEnd - timeStart2).count();
+
+        flowPath->setOuterTime(node->id(), outerTime);
+
+        if (status != Status::OK) return status;
+
+        // here we should handle divergent ops, and disable nodes accordingly
+        if (node->isDivergencePoint()) {
+          auto activeBranch = flowPath->branch(node->id());
+          sd_debug("Active branch at node [%i]: %i\n", node->id(), activeBranch);
+
+          // now we skip all branches except of this active one
+        }
+
+        if (Environment::getInstance().isDebugAndVerbose()) {
+          if (__variableSpace->getVariable(node->id())->hasNDArray()) {
+            auto array = __variableSpace->getVariable(node->id())->getNDArray();
+            auto shape = ShapeUtils::shapeAsString(array);
+            auto values = array->asIndexedString(16);
+            auto type = DataTypeUtils::asString(array->dataType());
+            sd_debug("node_%i finished. result shape: %s; data type: %s; first values: %s\n", node->id(), shape.c_str(),
+                     type.c_str(), values->c_str());
+          } else if (__variableSpace->getVariable(node->id())->hasNDArrayList()) {
+            auto list = __variableSpace->getVariable(node->id())->hasNDArrayList()
+                        ? __variableSpace->getVariable(node->id())->getNDArrayList()
+                        : nullptr;
+            sd_debug("node_% is ListOp, skipping evaluation", node->id());
+          } else {
+            sd_debug("node_% is Unknown: has no NDArray or NDArrayList", node->id());
+          }
+        }
+      }
+
+      // if node was executed - tag it as active
+      flowPath->markExecuted(node->id(), true);
+    }
+  }
+
+  // optionally saving execution time
+  if (Environment::getInstance().isProfiling()) {
+    flowPath->profile()->nodeById(lastId)->setTotalTime(GraphProfile::relativeTime(nodeTime));
+    flowPath->profile()->setExecutionTime(GraphProfile::relativeTime(timeStart));
+  }
+
+  // saving memory footprint for current run
+  if (__variableSpace->launchContext()->getWorkspace() != nullptr) {
+    auto m = __variableSpace->launchContext()->getWorkspace()->getAllocatedSize();
+    auto h = graph->hashCode();
+    memory::MemoryRegistrator::getInstance().setGraphMemoryFootprintIfGreater(h, m);
+  }
+
+  if (tempFlow) {
+    delete flowPath;
+    __variableSpace->setFlowPath(nullptr);
+  }
+
+  return Status::OK;
+}
+
+/**
+ * This method is provided for IPC:
+ * 1) it accepts pointer to FlatBuffers buffer
+ * 2) restores Graph from it
+ * 3) Executes this Graph
+ * 4) Packs execution results into FlatBuffers (FlatResults instance)
+ * 5) Returns pointer to FlatBuffer results buffer
+ *
+ */
+ResultWrapper *GraphExecutioner::executeFlatBuffer(Pointer pointer) {
+  uint8_t *buffer = reinterpret_cast<uint8_t *>(pointer);
+
+
+  auto restoredGraph = ::graph::GetFlatGraph(buffer);
+
+
+  // converting FlatGraph to internal representation
+  auto nativeGraph = new Graph(restoredGraph);
+
+  if (Environment::getInstance().isDebugAndVerbose()) {
+    nativeGraph->printOut();
+  }
+
+  FlowPath flowPath;
+  nativeGraph->getVariableSpace()->setFlowPath(&flowPath);
+
+  // sd_debug("Going to execute graph\n", 0);
+
+  // executing internal representation
+  auto status = execute(nativeGraph);
+  if (status != Status::OK) {
+    sd_printf("Graph execution failed with status: [%i]\n", status) return nullptr;
+  }
+
+
+  flatbuffers::FlatBufferBuilder builder(1024);
+
+  // fetching time reports
+  std::vector<flatbuffers::Offset<::graph::FlatTiming>> timings_vector;
+  for (int e = 0; e < (int)nativeGraph->getAllNodes()->size(); e++) {
+    Node *node = nativeGraph->getAllNodes()->at(e);
+
+    if (node->getContextPrototype() == nullptr) continue;
+
+    auto pair = ::graph::CreateLongPair(builder, flowPath.outerTime(node->id()), flowPath.innerTime(node->id()));
+    if (node->getName() != nullptr) {
+      auto name = builder.CreateString(node->getName()->c_str());
+      auto fr = CreateFlatTiming(builder, node->id(), name, pair);
+      timings_vector.push_back(fr);
+    } else {
+      auto fr = CreateFlatTiming(builder, node->id(), 0, pair);
+      timings_vector.push_back(fr);
+    }
+  }
+
+  // now, we'll prepare output, depending on given outputmode
+  auto outputs = nativeGraph->fetchOutputs();
+  auto size = static_cast<int>(outputs->size());
+  int arrays = 0;
+  std::vector<flatbuffers::Offset<::graph::FlatVariable>> variables_vector;
+  for (int e = 0; e < size; e++) {
+    auto var = outputs->at(e);
+
+    // FIXME: we want to export multi-output nodes as well
+    // FIXME: we want to export NDArrayList and skip nodes without outputs
+    if (!var->hasNDArray()) continue;
+
+    auto array = var->getNDArray();
+
+    auto fArray = FlatUtils::toFlatArray(builder, *array);
+
+    auto fName = builder.CreateString(*(var->getName()));
+    auto id = ::graph::CreateIntPair(builder, var->id(), var->index());
+
+    auto fv = CreateFlatVariable(builder, id, fName, static_cast<::graph::DType>(array->dataType()), 0, fArray);
+
+    variables_vector.push_back(fv);
+    arrays++;
+  }
+
+  sd_debug("Returning %i variables back\n", arrays);
+
+  auto varTimings = builder.CreateVector(timings_vector);
+  auto varVectors = builder.CreateVector(variables_vector);
+  auto result = CreateFlatResult(builder, restoredGraph->id(), varVectors, varTimings);
+  builder.Finish(result);
+
+  // we might want to keep this graph for future
+  delete outputs;
+  delete nativeGraph;
+
+  char *res = new char[builder.GetSize()];
+  memcpy(res, builder.GetBufferPointer(), builder.GetSize());
+
+  sd_debug("Buffer size: %lld\n", static_cast<sd::LongType>(builder.GetSize()));
+
+  return new ResultWrapper(builder.GetSize(), reinterpret_cast<Pointer>(res));
+}
+
+Graph *GraphExecutioner::importFromTensorFlow(const char *fileName) {
+  return nullptr;
+}
+
+/**
+ *   This function returns file size for the given file name, or -1 if something went wrong
+ */
+long getFileSize(const char *filename) {
+  struct stat stat_buf;
+  int rc = stat(filename, &stat_buf);
+  return rc == 0 ? stat_buf.st_size : -1;
+}
+
+/**
+ *   Helper function, that loads given filename into uint8_t array
+ *
+ */
+uint8_t *readFlatBuffers(const char *filename) {
+  long fileLen = getFileSize(filename);
+  if (fileLen < 0) {
+    sd_printf("File [%s] wasn't found. Please check path and permissions\n", filename);
+    THROW_EXCEPTION("File not found");
+  }
+
+  sd_debug("File length: %i\n", fileLen);
+
+  uint8_t *data = new uint8_t[fileLen];
+
+  FILE *in = fopen(filename, "rb");
+  int cnt = 0;
+  int b = 0;
+  while (cnt < fileLen) {
+    b = fread(data + cnt, 1, 1, in);
+
+    cnt += b;
+  }
+  fclose(in);
+
+  return data;
+}
+
+flatbuffers::Offset<::graph::FlatResult> GraphExecutioner::execute(Graph *graph, flatbuffers::FlatBufferBuilder &builder,
+                                                          const ::graph::FlatInferenceRequest *request) {
+  ExecutionResult result;
+  auto varSpace = graph->getVariableSpace();
+
+  if (request != nullptr && request->variables() != nullptr) {
+    auto vars = request->variables();
+    for (size_t e = 0; e < vars->size(); e++) {
+      auto fv = vars->Get(e);
+      auto v = new Variable(fv);
+      varSpace->replaceVariable(v);
+    }
+  }
+
+  if (Environment::getInstance().isDebugAndVerbose()) graph->printOut();
+
+  auto status = execute(graph);
+  if (status != Status::OK) THROW_EXCEPTION(graph_execution_exception(request->id()).what());
+
+  auto outputs = graph->fetchOutputs();
+
+  if (outputs->size() == 0) THROW_EXCEPTION(no_results_exception(request->id()).what());
+
+  for (auto v : *outputs) {
+    result.emplace_back(v);
+  }
+
+  auto t = result.asFlatResult(builder);
+
+  delete outputs;
+
+  return t;
+}
+
+/**
+ *   This method reads given FlatBuffers file, and returns Graph instance
+ *
+ *   PLEASE NOTE: This method is mostly suited for tests and debugging/profiling
+ */
+Graph *GraphExecutioner::importFromFlatBuffers(const char *filename) {
+  auto data = readFlatBuffers(filename);
+  auto restoredGraph = importFromFlatPointer(reinterpret_cast<Pointer>(data));
+   delete[] data;
+  return restoredGraph;
+}
+
+Graph *GraphExecutioner::importFromFlatPointer(Pointer ptr) {
+  auto fg = ::graph::GetFlatGraph(reinterpret_cast<uint8_t *>(ptr));
+  auto restoredGraph = new Graph(fg);
+
+  return restoredGraph;
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/GraphHolder.cpp

@@ -0,0 +1,119 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+#include <exceptions/graph_execution_exception.h>
+#include <exceptions/graph_exists_exception.h>
+#include <graph/GraphExecutioner.h>
+#include <graph/GraphHolder.h>
+
+namespace sd {
+namespace graph {
+GraphHolder& GraphHolder::getInstance() {
+  static GraphHolder instance;
+  return instance;
+};
+
+void GraphHolder::registerGraph(sd::LongType graphId, Graph* graph) {
+  if (hasGraphAny(graphId)) THROW_EXCEPTION(graph_exists_exception(graphId).what());
+
+  _graphF[graphId] = graph;
+
+  sd::SimpleReadWriteLock lock;
+  _locks[graphId] = lock;
+}
+
+Graph* GraphHolder::cloneGraph(sd::LongType graphId) {
+  if (!this->hasGraph(graphId)) {
+    sd_printf("GraphHolder doesn't have graph stored for [%lld]\n", graphId);
+    THROW_EXCEPTION("Bad argument");
+  }
+
+  auto graph = _graphF[graphId]->cloneWithProxy();
+
+  return graph;
+}
+
+Graph* GraphHolder::pullGraph(sd::LongType graphId) {
+  if (!this->hasGraph(graphId)) {
+    sd_printf("GraphHolder doesn't have graph stored for [%lld]\n", graphId);
+    THROW_EXCEPTION("Bad argument");
+  }
+
+  auto graph = _graphF[graphId];
+
+  return graph;
+}
+
+void GraphHolder::forgetGraph(sd::LongType graphId) {
+  if (this->hasGraph(graphId)) _graphF.erase(graphId);
+}
+
+void GraphHolder::dropGraph(sd::LongType graphId) {
+  if (this->hasGraph(graphId)) {
+    auto g = _graphF[graphId];
+    forgetGraph(graphId);
+    delete g;
+  }
+}
+
+void GraphHolder::dropGraphAny(sd::LongType graphId) {
+  if (!hasGraphAny(graphId)) return;
+
+  this->lockWrite(graphId);
+
+  this->dropGraph(graphId);
+
+  this->unlockWrite(graphId);
+}
+
+bool GraphHolder::hasGraphAny(sd::LongType graphId) { return this->hasGraph(graphId); }
+
+bool GraphHolder::hasGraph(sd::LongType graphId) { return _graphF.count(graphId) > 0; }
+
+void GraphHolder::replaceGraph(sd::LongType graphId, Graph* graph) {
+  if (!hasGraph(graphId)) {
+    registerGraph(graphId, graph);
+    return;
+  }
+
+  this->lockWrite(graphId);
+
+  _graphF[graphId] = graph;
+
+  this->unlockWrite(graphId);
+}
+
+flatbuffers::Offset<::graph::FlatResult> GraphHolder::execute(sd::LongType graphId, flatbuffers::FlatBufferBuilder& builder,
+                                                     const ::graph::FlatInferenceRequest* request) {
+  if (!hasGraph(graphId)) THROW_EXCEPTION(unknown_graph_exception(graphId).what());
+
+  lockRead(graphId);
+
+  auto graph = cloneGraph(graphId);
+  auto res = GraphExecutioner::execute(graph, builder, request);
+  delete graph;
+
+  unlockRead(graphId);
+
+  return res;
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/GraphState.cpp

@@ -0,0 +1,152 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 23.01.18.
+//
+#include <graph/GraphState.h>
+#include <graph/Node.h>
+
+namespace sd {
+namespace graph {
+GraphState::GraphState(sd::LongType id) {
+  _id = id;
+  _graph = new Graph(nullptr, &_variableSpace);
+};
+
+GraphState::~GraphState() {
+  // stupid. we should get rid of pointers here i think. no need to bother
+  for (const auto& v : _scopes) {
+    v.second->forgetNodes();
+    delete v.second;
+  }
+
+  // we must remove reference to VariableSpace
+  _graph->forgetVariableSpace();
+
+  delete _graph;
+};
+
+sd::Status GraphState::registerScope(int scopeId) {
+  auto scope = new Scope(scopeId);
+  _scopes[scopeId] = scope;
+
+  auto scopeWrapper = new Node(::graph::OpType_LOGIC, 10, scopeId);
+  _graph->addNode(scopeWrapper);
+
+  return sd::Status::OK;
+};
+
+sd::Status GraphState::forgetScope(int scopeId) {
+  if (_scopes.count(scopeId) > 0)
+    _scopes.erase(scopeId);
+  else
+    return Logger::logKernelFailureMsg("Non-existent scope requested");
+
+  return sd::Status::OK;
+};
+
+#ifndef __JAVACPP_HACK__
+sd::Status GraphState::attachOpToScope(int scopeId, int nodeId, ops::DeclarableOp* op, ArgumentsList inputs) {
+  if (_scopes.count(scopeId) == 0) return Logger::logKernelFailureMsg("GraphState: can't attach op to unknown scope");
+
+  auto scope = _scopes[scopeId];
+
+  // creating new Node
+  auto node = new Node(op, nodeId);
+  node->setScopeInfo(scopeId);
+
+  // mapping inputs here
+  for (int e = 0; e < inputs.size(); e++) {
+    auto p = inputs.at(e);
+
+    // each expected input is Variable in current VariableSpace
+    // it should have it's numerical and symbolic ID
+
+    if (!_variableSpace.hasVariable(p.first(), p.second())) {
+      auto var = new Variable();
+      var->setId(p.first(), p.second());
+      _variableSpace.putVariable(p.first(), p.second(), var);
+    }
+
+    node->pickInput(p.first(), p.second());
+  }
+
+  scope->push_back(node);
+
+  _graph->addNode(node);
+
+  return sd::Status::OK;
+};
+
+Graph* GraphState::graph() { return _graph; }
+
+Scope* GraphState::getScope(int scopeId) {
+  if (_scopes.count(scopeId) == 0) {
+    sd_printf("GraphState: Unknown scope requested %i\n", scopeId);
+    return nullptr;
+  }
+
+  return _scopes[scopeId];
+}
+#endif
+sd::Status GraphState::defineReturn(int scopeId, int nodeId, ArgumentsList args) {
+  if (_scopes.count(scopeId) == 0) return Logger::logKernelFailureMsg("GraphState: can't attach op to unknown scope");
+
+  auto scope = _scopes[scopeId];
+
+  // creating new Node for RETURN
+  auto node = new Node(::graph::OpType_LOGIC, 40, nodeId);
+  node->setScopeInfo(scopeId);
+
+  // mapping inputs here
+  for (int e = 0; e < args.size(); e++) {
+    auto p = args.at(e);
+
+    // each expected input is Variable in current VariableSpace
+    // it should have it's numerical and symbolic ID
+
+    if (!_variableSpace.hasVariable(p.first(), p.second())) {
+      auto var = new Variable();
+      var->setId(p.first(), p.second());
+      _variableSpace.putVariable(p.first(), p.second(), var);
+    }
+
+    node->pickInput(p.first(), p.second());
+    node->pickOutput(0, e);
+  }
+
+  scope->push_back(node);
+
+  _graph->addNode(node);
+
+  return sd::Status::OK;
+}
+
+bool GraphState::hasScope(int scopeId) { return _scopes.count(scopeId) > 0; }
+
+VariableSpace* GraphState::variableSpace() { return &_variableSpace; };
+
+sd::LongType GraphState::id() { return _id; }
+
+sd::Status GraphState::attachOpToScope(int scopeId, sd::LongType opNum, int type, ArgumentsList inputs) {
+  // we should use OpRegistrator here, to create Node and push it to specific scope
+  return sd::Status::OK;
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/InferenceRequest.cpp

@@ -0,0 +1,75 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <graph/InferenceRequest.h>
+
+namespace sd {
+namespace graph {
+InferenceRequest::InferenceRequest(LongType graphId, ExecutorConfiguration *configuration) {
+  this->_id = graphId;
+  this->_configuration = configuration;
+}
+
+InferenceRequest::~InferenceRequest() {
+  for (auto v : _deletables) delete v;
+}
+
+void InferenceRequest::appendVariable(int id, NDArray *array) { appendVariable(id, 0, array); }
+
+void InferenceRequest::appendVariable(int id, int index, NDArray *array) {
+  auto v = new Variable(array, nullptr, id, index);
+  insertVariable(v);
+}
+
+void InferenceRequest::appendVariable(std::string &id, NDArray *array) {
+  auto v = new Variable(array, id.c_str());
+  insertVariable(v);
+}
+
+void InferenceRequest::appendVariable(std::string &name, int id, int index, NDArray *array) {
+  auto v = new Variable(array, name.c_str(), id, index);
+  insertVariable(v);
+}
+
+void InferenceRequest::insertVariable(Variable *variable) {
+  variable->markRemovable(false);
+  variable->markReadOnly(true);
+  _variables.emplace_back(variable);
+  _deletables.emplace_back(variable);
+}
+
+void InferenceRequest::appendVariable(Variable *variable) { _variables.emplace_back(variable); }
+
+flatbuffers::Offset<::graph::FlatInferenceRequest> InferenceRequest::asFlatInferenceRequest(
+    flatbuffers::FlatBufferBuilder &builder) {
+  std::vector<flatbuffers::Offset<::graph::FlatVariable>> vec;
+  for (Variable *v : _variables) {
+    vec.emplace_back(v->asFlatVariable(builder));
+  }
+
+  auto confOffset = _configuration != nullptr ? _configuration->asFlatConfiguration(builder) : 0;
+
+  auto vecOffset = builder.CreateVector(vec);
+
+  return CreateFlatInferenceRequest(builder, _id, vecOffset, confOffset);
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/Intervals.cpp

@@ -0,0 +1,44 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author yurii@skymind.io
+//
+#include <graph/Intervals.h>
+
+namespace sd {
+
+// default constructor
+Intervals::Intervals() : _content({{}}) {}
+
+// constructor
+Intervals::Intervals(const std::initializer_list<std::vector<LongType>>& content) : _content(content) {}
+Intervals::Intervals(const std::vector<std::vector<LongType>>& content) : _content(content) {}
+
+//////////////////////////////////////////////////////////////////////////
+// accessing operator
+std::vector<LongType> Intervals::operator[](const LongType i) const { return *(_content.begin() + i); }
+
+//////////////////////////////////////////////////////////////////////////
+// returns size of _content
+int Intervals::size() const { return _content.size(); }
+
+//////////////////////////////////////////////////////////////////////////
+
+
+}  // namespace sd

libnd4j/include/graph/impl/Node.cpp

@@ -0,0 +1,720 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <array/NDArrayFactory.h>
+#include <graph/FlatUtils.h>
+#include <graph/Node.h>
+#include <ops/declarable/LegacyBroadcastBoolOp.h>
+#include <ops/declarable/LegacyBroadcastOp.h>
+#include <ops/declarable/LegacyIndexReduceOp.h>
+#include <ops/declarable/LegacyOp.h>
+#include <ops/declarable/LegacyPairwiseTransformBoolOp.h>
+#include <ops/declarable/LegacyPairwiseTransformOp.h>
+#include <ops/declarable/LegacyRandomOp.h>
+#include <ops/declarable/LegacyReduce3Op.h>
+#include <ops/declarable/LegacyReduceBoolOp.h>
+#include <ops/declarable/LegacyReduceFloatOp.h>
+#include <ops/declarable/LegacyReduceLongOp.h>
+#include <ops/declarable/LegacyReduceSameOp.h>
+#include <ops/declarable/LegacyScalarBoolOp.h>
+#include <ops/declarable/LegacyScalarOp.h>
+#include <ops/declarable/LegacyStatsOp.h>
+#include <ops/declarable/LegacyTransformBoolOp.h>
+#include <ops/declarable/LegacyTransformFloatOp.h>
+#include <ops/declarable/LegacyTransformSameOp.h>
+#include <ops/declarable/LegacyTransformStrictOp.h>
+#include <ops/declarable/OpRegistrator.h>
+
+namespace sd {
+namespace graph {
+void sd::graph::Node::setOuterTime(sd::LongType time) {
+}
+
+void sd::graph::Node::setInnerTime(sd::LongType time) {
+}
+
+void sd::graph::Node::setGraph(sd::graph::Graph* graph) { _graph = graph; }
+
+sd::graph::Graph* sd::graph::Node::getGraph() { return _graph; }
+
+bool sd::graph::Node::hasGraphEmbedded() { return _graph != nullptr; }
+
+void sd::graph::Node::markInplace(bool reallyInplace) {
+  _isInplace = reallyInplace;
+  if (_protoContext != nullptr) {
+    _protoContext->markInplace(reallyInplace);
+  }
+}
+
+::graph::OpClass sd::graph::Node::getOpClass() { return _opClass; }
+
+bool sd::graph::Node::hasBlockAttached() { return _protoContext != nullptr; }
+
+bool sd::graph::Node::isInplace() { return _isInplace; }
+
+bool sd::graph::Node::isDivergencePoint() {
+  if (hasCustomOp()) {
+    return _customOp->getOpDescriptor()->isDivergent();
+  } else if (opType() == ::graph::OpType_LOGIC && opNum() == 30)
+    return true;
+  else
+    return false;
+}
+
+void sd::graph::Node::setActive(bool reallyActive) { _active = reallyActive; }
+
+bool sd::graph::Node::isActive() { return _active; }
+
+sd::LongType Node::getFrameId() { return _frameId; }
+
+void Node::setFrameId(sd::LongType frameId) { _frameId = frameId; }
+
+ContextPrototype* sd::graph::Node::getContextPrototype() {
+  if (_protoContext == nullptr)
+    _protoContext = new ContextPrototype(
+        this->getCustomOp() != nullptr ? this->getCustomOp()->getOpDescriptor() : nullptr, this->id());
+  if (_protoContext->inputs()->empty()) {
+    for (size_t e = 0; e < this->input()->size(); e++) {
+      _protoContext->inputs()->emplace_back(this->input()->at(e));
+    }
+  }
+  return _protoContext;
+}
+
+void sd::graph::Node::setContextPrototype(ContextPrototype* block) {
+  if (_protoContext != nullptr) THROW_EXCEPTION("Block already exists");
+
+  _protoContext = block;
+}
+
+void sd::graph::Node::setId(int id) { _id = id; }
+
+sd::ops::DeclarableOp* sd::graph::Node::getCustomOp() { return _customOp; }
+
+void sd::graph::Node::setCustomOp(sd::ops::DeclarableOp* customOp) {
+  _customOp = customOp;
+
+  // divergent ops (Switch etc) are always inplace, they don't allocate anything
+  if (_customOp != nullptr && customOp->getOpDescriptor()->isDivergent()) _isInplace = true;
+}
+
+bool sd::graph::Node::hasCustomOp() { return _customOp != nullptr; }
+
+std::string* sd::graph::Node::name() { return this->getName(); }
+
+std::string* sd::graph::Node::getName() { return &_name; }
+
+void sd::graph::Node::setName(const std::string& name) { _name = name.c_str(); }
+
+void sd::graph::Node::setName(std::string* name) { _name = *name; }
+
+;
+
+void sd::graph::Node::pickInput(std::pair<int, int>& pair) { _input.push_back(pair); }
+
+void sd::graph::Node::pickInput(int inputId, int outputId) {
+  std::pair<int, int> p(inputId, outputId);
+  pickInput(p);
+}
+
+void sd::graph::Node::pickInput(int inputId) {
+  pickInput(inputId, 0);
+
+  if (inputId < 0)
+    _hasExternalInputs = true;
+  else
+    _hasInternalInputs = true;
+}
+
+void sd::graph::Node::pickExternalOutput(int outputId) {
+  std::pair<int, int> pair(outputId, 0);
+  _output.push_back(pair);
+
+  _hasExternalOutputs = true;
+}
+
+void sd::graph::Node::pickOutputOnce(int outputId) {
+  std::pair<int, int> pair(outputId, 0);
+  if (std::find(_output.begin(), _output.end(), pair) == _output.end()) pickOutput(outputId);
+}
+
+void sd::graph::Node::pickOutput(int nodeId, int outputId) {
+  std::pair<int, int> pair(nodeId, outputId);
+  _output.emplace_back(pair);
+}
+
+void sd::graph::Node::pickOutput(int outputId) {
+  std::pair<int, int> pair(outputId, 0);
+  _output.emplace_back(pair);
+
+  if (outputId < 0)
+    _hasExternalOutputs = true;
+  else
+    _hasInternalOutputs = true;
+}
+
+sd::LongType* sd::graph::Node::getDimensionsPtr() { return _dim; }
+
+std::vector<sd::LongType>* sd::graph::Node::getDimensions() { return &_dimensions; }
+
+int sd::graph::Node::getLayer() { return _layer; }
+
+void sd::graph::Node::setLayer(int layer) { _layer = layer; }
+
+bool sd::graph::Node::hasExternalOutputs() { return _hasExternalOutputs; }
+
+bool sd::graph::Node::hasExternalInputs() { return _hasExternalInputs; }
+
+bool sd::graph::Node::hasInternalOutputs() { return _hasInternalOutputs; }
+
+bool sd::graph::Node::hasInternalInputs() { return _hasInternalInputs; }
+
+bool sd::graph::Node::isMultiInput() { return _input.size() > 1; }
+
+bool sd::graph::Node::isMultiOutput() { return _output.size() > 1; }
+
+double* sd::graph::Node::extraParams() { return _extraParams; }
+
+int Node::totalReferences() { return _referencedBy.size(); }
+
+void Node::addReference(int nodeId) { _referencedBy.emplace_back(nodeId); }
+
+::graph::OpType sd::graph::Node::opType() { return _opType; }
+
+int sd::graph::Node::id() { return _id; }
+
+sd::LongType sd::graph::Node::opNum() { return _opNum; }
+
+std::vector<std::pair<int, int>>* sd::graph::Node::input() { return &_input; }
+
+std::vector<std::pair<int, int>>* sd::graph::Node::output() { return &_output; }
+
+bool Node::isScoped() { return _scope_id != 0; }
+
+void Node::setScopeInfo(int id, const char* name) {
+  _scope_id = id;
+
+  if (name != nullptr) _scope_name = name;
+}
+
+int Node::scopeId() { return _scope_id; }
+
+std::string* Node::scopeName() { return &_scope_name; }
+
+template <typename T>
+Node* Node::asT() {
+  auto node = this->clone();
+  node->_dataType = DataTypeUtils::fromT<T>();
+  return node;
+}
+BUILD_SINGLE_TEMPLATE( SD_LIB_EXPORT Node* Node::asT, (), SD_COMMON_TYPES);
+
+sd::graph::Node::Node(sd::ops::DeclarableOp* customOp, int id, std::initializer_list<int> input,
+                      std::initializer_list<int> output, std::initializer_list<int> dimensions, float scalar,
+                      std::initializer_list<double> tArgs, std::initializer_list<int> iArgs) {
+  this->_opType = ::graph::OpType_CUSTOM;
+  this->_id = id;
+  this->_opNum = customOp->getOpHash();
+  this->_extraParams = nullptr;
+  this->_dataType = sd::DataType::FLOAT32;  // float as default
+  this->_dim = nullptr;
+  this->_customOp = customOp;
+
+  _hasExternalInputs = false;
+  _hasExternalOutputs = false;
+  _hasInternalInputs = false;
+  _hasInternalOutputs = false;
+
+
+  for (auto i : input) pickInput(i);
+
+  for (auto o : output) pickOutput(o);
+
+  if (dimensions.size() > 0) {
+    _dim = new sd::LongType[dimensions.size()];
+    int cnt = 0;
+    for (auto d : dimensions) {
+      _dimensions.push_back(d);
+      _dim[cnt++] = d;
+    }
+  }
+
+  auto block = new ContextPrototype(this->getCustomOp()->getOpDescriptor(), this->id(), false);
+
+  for (auto v : dimensions) block->getAxis()->emplace_back(v);
+
+  for (auto v : iArgs) block->getIArguments()->emplace_back(v);
+
+  for (auto v : tArgs) block->getTArguments()->emplace_back(v);
+
+  this->setContextPrototype(block);
+}
+
+void sd::graph::Node::setOpType(::graph::OpType opType) { this->_opType = opType; }
+
+sd::graph::Node::Node(::graph::OpType opType, int opNum, int id, std::initializer_list<int> input,
+                      std::initializer_list<int> output, std::initializer_list<int> dimensions, float scalar,
+                      std::initializer_list<double> tArgs, std::initializer_list<int> iArgs) {
+  this->_opType = opType;
+  this->_id = id;
+  this->_opNum = opNum;
+  this->_extraParams = nullptr;
+  this->_dataType = sd::DataType::FLOAT32;  // float as default
+  this->_dim = nullptr;
+
+  _hasExternalInputs = false;
+  _hasExternalOutputs = false;
+  _hasInternalInputs = false;
+  _hasInternalOutputs = false;
+
+
+  for (auto i : input) pickInput(i);
+
+  for (auto o : output) pickOutput(o);
+
+  if (dimensions.size() > 0) {
+    _dim = new sd::LongType[dimensions.size()];
+    int cnt = 0;
+    for (auto d : dimensions) {
+      _dimensions.push_back(d);
+      _dim[cnt++] = d;
+    }
+  }
+
+  // these ops allow in-place execution by design
+  if (opType == ::graph::OpType_TRANSFORM_SAME || opType == ::graph::OpType_TRANSFORM_FLOAT || opType == ::graph::OpType_TRANSFORM_STRICT ||
+      opType == ::graph::OpType_TRANSFORM_BOOL || opType == ::graph::OpType_SCALAR || opType == ::graph::OpType_BROADCAST) {
+    if (_output.size() <= 1) {
+      _isInplace = true;
+    }
+    _opClass = ::graph::OpClass_TRANSFORM;
+  } else if (opType == ::graph::OpType_REDUCE_SAME || opType == ::graph::OpType_REDUCE_FLOAT || opType == ::graph::OpType_REDUCE_BOOL ||
+             opType == ::graph::OpType_REDUCE_LONG || opType == ::graph::OpType_SUMMARYSTATS) {
+    _opClass = ::graph::OpClass_REDUCTION;
+  }
+
+  if (opType == ::graph::OpType_BROADCAST || opType == ::graph::OpType_BROADCAST_BOOL || opType == ::graph::OpType_INDEX_REDUCE ||
+      opType == ::graph::OpType_SUMMARYSTATS || opType == ::graph::OpType_REDUCE_BOOL || opType == ::graph::OpType_REDUCE_SAME ||
+      opType == ::graph::OpType_REDUCE_FLOAT || opType == ::graph::OpType_REDUCE_3 || opType == ::graph::OpType_TRANSFORM_STRICT ||
+      opType == ::graph::OpType_TRANSFORM_SAME || opType == ::graph::OpType_TRANSFORM_FLOAT || opType == ::graph::OpType_TRANSFORM_BOOL ||
+      opType == ::graph::OpType_RANDOM || opType == ::graph::OpType_PAIRWISE || opType == ::graph::OpType_PAIRWISE_BOOL ||
+      opType == ::graph::OpType_SCALAR_BOOL || opType == ::graph::OpType_SCALAR) {
+    this->_isDeductable = true;
+
+    auto block = new ContextPrototype(nullptr, this->id(), false);
+
+    for (auto v : dimensions) block->getAxis()->emplace_back(v);
+
+    for (auto v : iArgs) block->getIArguments()->emplace_back(v);
+
+    for (auto v : tArgs) block->getTArguments()->emplace_back(v);
+    NDArray *_scalar = NDArrayFactory::create(0.0f);
+
+    this->setContextPrototype(block);
+    this->setCustomOp(Node::buildOpByType(opType, (int)input.size(), (int)block->getIArguments()->size(),
+                                          (int)block->getTArguments()->size(), opNum, _scalar));
+    block->setOpDescriptor(this->getCustomOp()->getOpDescriptor());
+    delete _scalar;
+  } else if (opType == ::graph::OpType_CUSTOM) {
+    if (this->getCustomOp()) {
+      auto block = new ContextPrototype(this->getCustomOp()->getOpDescriptor(), this->id(), false);
+
+      for (auto v : dimensions) block->getAxis()->emplace_back(v);
+
+      for (auto v : iArgs) block->getIArguments()->emplace_back(v);
+
+      for (auto v : tArgs) block->getTArguments()->emplace_back(v);
+
+      this->setContextPrototype(block);
+    } else
+      THROW_EXCEPTION("wrong custom operation given");
+  }
+};
+
+sd::graph::Node::Node(const ::graph::FlatNode* node) {
+  _hasExternalInputs = false;
+  _hasExternalOutputs = false;
+  _hasInternalInputs = false;
+  _hasInternalOutputs = false;
+  _extraParams = nullptr;
+  _dim = nullptr;
+  _dataType = sd::DataType::FLOAT32;  // float as default
+  if (node->scope_id() != 0) this->_scope_id = node->scope_id();
+
+  if (node->scope_name() != nullptr && node->scope_name()->size() > 0) this->_scope_name = node->scope_name()->str();
+
+
+
+  if (node != nullptr) {
+    this->_id = node->id();
+    // this->_dataType = DataTypeUtils::fromFlatDataType(node->dataType());
+    this->_opNum = node->opNum();
+    this->_opType = node->opType();
+
+    if (node->name() != nullptr && node->name()->c_str() != nullptr) {
+      this->_name = node->name()->str();
+    }
+
+    if (node->inputPaired() != nullptr && node->inputPaired()->size() > 0) {
+      for (int e = 0; e < (int)node->inputPaired()->size(); e++) {
+        auto pair = node->inputPaired()->Get(e);
+        pickInput(pair->first(), pair->second());
+      }
+    } else if (node->input() != nullptr && node->input()->size() > 0) {
+      for (int e = 0; e < (int)node->input()->size(); e++) pickInput(node->input()->Get(e));
+    } else {
+      if (this->opType() != ::graph::OpType_LOGIC) {
+        if (this->_name.size() > 0) {
+          sd_debug("Node [%i:<%s>] has no inputs defined\n", this->_id, this->_name.c_str());
+        } else {
+          sd_debug("Node [%i:<noname>] has no inputs defined\n", this->_id);
+        }
+      }
+    }
+
+
+    if (node->extraParams() != nullptr && node->extraParams()->size() > 0) {
+      _extraParams = new double[node->extraParams()->size()];
+      for (int e = 0; e < (int)node->extraParams()->size(); e++) {
+        _extraParams[e] = static_cast<double>(node->extraParams()->Get(e));
+      }
+    }
+
+    if (node->dimensions() != nullptr && node->dimensions()->size() > 0) {
+      _dim = new sd::LongType [node->dimensions()->size()];
+      for (int e = 0; e < (int)node->dimensions()->size(); e++) {
+        _dimensions.emplace_back(node->dimensions()->Get(e));
+        _dim[e] = node->dimensions()->Get(e);
+      }
+    }
+
+    if (this->opType() == ::graph::OpType_LOGIC && this->opNum() == 100L) {
+      if (node->extraInteger()->size() < 1) {
+        sd_printf("Node_%i is type of Enter, but has no FrameID defined\n", this->id());
+        THROW_EXCEPTION("Enter node must have FrameID specified");
+      }
+
+      this->setFrameId(node->extraInteger()->Get(0));
+    }
+
+    // these ops allow in-place execution by design
+    if (_opType == ::graph::OpType_BROADCAST || _opType == ::graph::OpType_BROADCAST_BOOL || _opType == ::graph::OpType_INDEX_REDUCE ||
+        _opType == ::graph::OpType_SUMMARYSTATS || _opType == ::graph::OpType_REDUCE_BOOL || _opType == ::graph::OpType_REDUCE_SAME ||
+        _opType == ::graph::OpType_REDUCE_FLOAT || _opType == ::graph::OpType_REDUCE_3 || _opType == ::graph::OpType_TRANSFORM_STRICT ||
+        _opType == ::graph::OpType_TRANSFORM_SAME || _opType == ::graph::OpType_TRANSFORM_FLOAT || _opType == ::graph::OpType_TRANSFORM_BOOL ||
+        _opType == ::graph::OpType_RANDOM || _opType == ::graph::OpType_PAIRWISE || _opType == ::graph::OpType_PAIRWISE_BOOL ||
+        _opType == ::graph::OpType_SCALAR_BOOL || _opType == ::graph::OpType_SCALAR) {
+      if (_output.size() <= 1) {
+        _isInplace = true;
+      }
+
+      if (node->input() != nullptr && node->input()->size() > 0) {
+        this->_isDeductable = true;
+
+        auto block = new ContextPrototype(nullptr, this->id(), false);
+
+        for (auto v : _dimensions) block->getAxis()->emplace_back(v);
+
+        if (node->extraParams() != nullptr && node->extraParams()->size() > 0)
+          for (int e = 0; e < (int)node->extraParams()->size(); e++) {
+            block->getTArguments()->emplace_back(static_cast<double>(node->extraParams()->Get(e)));
+          }
+
+        if (node->extraBools() != nullptr && node->extraBools()->size() > 0)
+          for (int e = 0; e < (int)node->extraBools()->size(); e++) {
+            block->getBArguments()->push_back(node->extraBools()->Get(e));
+          }
+
+        if (node->extraInteger() != nullptr && node->extraInteger()->size() > 0)
+          for (int e = 0; e < (int)node->extraInteger()->size(); e++) {
+            block->getIArguments()->emplace_back(node->extraInteger()->Get(e));
+          }
+
+        if (node->extraTypes() != nullptr && node->extraTypes()->size() > 0) {
+          for (int e = 0; e < (int)node->extraTypes()->size(); e++) {
+            block->getDArguments()->emplace_back((sd::DataType)node->extraTypes()->Get(e));
+          }
+        }
+
+        NDArray *_scalar = NDArrayFactory::create(0.0f);
+        this->setContextPrototype(block);
+        this->setCustomOp(Node::buildOpByType(_opType, (int)node->input()->size(), (int)block->getIArguments()->size(),
+                                              (int)block->getTArguments()->size(), (int)_opNum, _scalar));
+        block->setOpDescriptor(this->getCustomOp()->getOpDescriptor());
+        delete _scalar;
+      } else if (node->inputPaired() != nullptr && node->inputPaired()->size() > 0) {
+        this->_isDeductable = true;
+
+        auto block = new ContextPrototype(nullptr, this->id(), false);
+
+        for (size_t e = 0; e < this->input()->size(); e++) {
+          block->inputs()->emplace_back(this->input()->at(e));
+        }
+
+        // there's no other IArgs in legacy options, actually
+        for (auto v : _dimensions) block->getAxis()->emplace_back(v);
+
+        if (node->extraParams() != nullptr && node->extraParams()->size() > 0)
+          for (int e = 0; e < (int)node->extraParams()->size(); e++) {
+            block->getTArguments()->emplace_back(static_cast<double>(node->extraParams()->Get(e)));
+          }
+
+        if (node->extraBools() != nullptr && node->extraBools()->size() > 0)
+          for (int e = 0; e < (int)node->extraBools()->size(); e++) {
+            block->getBArguments()->push_back(node->extraBools()->Get(e));
+          }
+
+        if (node->extraInteger() != nullptr && node->extraInteger()->size() > 0)
+          for (int e = 0; e < (int)node->extraInteger()->size(); e++) {
+            block->getIArguments()->emplace_back(node->extraInteger()->Get(e));
+          }
+
+        if (node->extraTypes() != nullptr && node->extraTypes()->size() > 0) {
+          for (int e = 0; e < (int)node->extraTypes()->size(); e++) {
+            block->getDArguments()->emplace_back((sd::DataType)node->extraTypes()->Get(e));
+          }
+        }
+
+        this->setContextPrototype(block);
+        NDArray *_scalar = NDArrayFactory::create(0.0f);
+
+        this->setCustomOp(Node::buildOpByType(_opType, (int)node->inputPaired()->size(),
+                                              (int)block->getIArguments()->size(), (int)block->getTArguments()->size(),
+                                              (int)_opNum, _scalar));
+        block->setOpDescriptor(this->getCustomOp()->getOpDescriptor());
+        delete _scalar;
+      }
+    } else if (this->_opType == ::graph::OpType_CUSTOM) {
+      auto op = sd::ops::OpRegistrator::getInstance().getOperation(this->opNum());
+      if (op == nullptr) {
+        sd_verbose("Can't find operation: %lld\n", this->opNum());
+        THROW_EXCEPTION("Can't find requested operation");
+      }
+
+      auto block = new ContextPrototype(nullptr, this->id());
+
+      for (size_t e = 0; e < this->input()->size(); e++) {
+        block->inputs()->emplace_back(this->input()->at(e));
+      }
+
+      if (node->extraInteger() != nullptr)
+        for (uint32_t e = 0; e < node->extraInteger()->size(); e++) {
+          auto v = node->extraInteger()->Get(e);
+          // FIXME: remove this static_cast, iArgs should be sd::LongType
+          block->getIArguments()->emplace_back(static_cast<int>(v));
+        }
+
+      if (node->extraParams() != nullptr)
+        for (uint32_t e = 0; e < node->extraParams()->size(); e++)
+          block->getTArguments()->emplace_back(static_cast<double>(node->extraParams()->Get(e)));
+
+      if (node->extraBools() != nullptr && node->extraBools()->size() > 0)
+        for (int e = 0; e < (int)node->extraBools()->size(); e++) {
+          block->getBArguments()->push_back(node->extraBools()->Get(e));
+        }
+
+      if (node->extraTypes() != nullptr && node->extraTypes()->size() > 0) {
+        for (int e = 0; e < (int)node->extraTypes()->size(); e++) {
+          block->getDArguments()->emplace_back((sd::DataType)node->extraTypes()->Get(e));
+        }
+      }
+
+      for (auto v : _dimensions) block->getAxis()->emplace_back(v);
+
+      this->setContextPrototype(block);
+      this->setCustomOp(op);
+      block->setOpDescriptor(this->getCustomOp()->getOpDescriptor());
+    }
+  } else {
+    // empty dynamic node, tests probably
+  }
+}
+
+sd::DataType Node::dataType() { return _dataType; }
+
+ContextPrototype* Node::protoContext() { return _protoContext; }
+
+sd::graph::Node::~Node() {
+  if (_extraParams != nullptr) delete[] _extraParams;
+
+  if (_dim != nullptr) delete[] _dim;
+
+
+  if (_isDeductable && _customOp != nullptr) {
+    Node::deleteOpByType(_opType, _customOp);
+  }
+}
+
+int sd::graph::Node::getRewindNode() { return _rewindNode; }
+
+void sd::graph::Node::setRewindNode(int nodeId) { _rewindNode = nodeId; }
+
+std::pair<int, int>& sd::graph::Node::getRewindLayer() { return _rewindLayer; };
+
+void sd::graph::Node::setRewindLayer(int layerId, int stepId) {
+  _rewindLayer.first = layerId;
+  _rewindLayer.second = stepId;
+}
+
+bool sd::graph::Node::equals(Node* other) {
+  if (_opType == other->_opType && _dataType == other->_dataType && _opNum == other->_opNum) return true;
+
+  return false;
+}
+
+void sd::graph::Node::deleteOpByType(::graph::OpType opType, void* op) {
+  switch (opType) {
+    case ::graph::OpType_PAIRWISE:
+      delete reinterpret_cast<sd::ops::LegacyPairwiseTransformOp*>(op);
+      break;
+    case ::graph::OpType_PAIRWISE_BOOL:
+      delete reinterpret_cast<sd::ops::LegacyPairwiseTransformBoolOp*>(op);
+      break;
+    case ::graph::OpType_TRANSFORM_STRICT:
+      delete reinterpret_cast<sd::ops::LegacyTransformStrictOp*>(op);
+      break;
+    case ::graph::OpType_TRANSFORM_SAME:
+      delete reinterpret_cast<sd::ops::LegacyTransformSameOp*>(op);
+      break;
+    case ::graph::OpType_TRANSFORM_FLOAT:
+      delete reinterpret_cast<sd::ops::LegacyTransformFloatOp*>(op);
+      break;
+    case ::graph::OpType_TRANSFORM_BOOL:
+      delete reinterpret_cast<sd::ops::LegacyTransformBoolOp*>(op);
+      break;
+    case ::graph::OpType_SCALAR:
+      delete reinterpret_cast<sd::ops::LegacyScalarOp*>(op);
+      break;
+    case ::graph::OpType_SCALAR_BOOL:
+      delete reinterpret_cast<sd::ops::LegacyScalarBoolOp*>(op);
+      break;
+    case ::graph::OpType_REDUCE_3:
+      delete reinterpret_cast<sd::ops::LegacyReduce3Op*>(op);
+      break;
+    case ::graph::OpType_REDUCE_SAME:
+      delete reinterpret_cast<sd::ops::LegacyReduceSameOp*>(op);
+      break;
+    case ::graph::OpType_REDUCE_FLOAT:
+      delete reinterpret_cast<sd::ops::LegacyReduceFloatOp*>(op);
+      break;
+    case ::graph::OpType_REDUCE_LONG:
+      delete reinterpret_cast<sd::ops::LegacyReduceLongOp*>(op);
+      break;
+    case ::graph::OpType_REDUCE_BOOL:
+      delete reinterpret_cast<sd::ops::LegacyReduceBoolOp*>(op);
+      break;
+    case ::graph::OpType_INDEX_REDUCE:
+      delete reinterpret_cast<sd::ops::LegacyIndexReduceOp*>(op);
+      break;
+    case ::graph::OpType_SUMMARYSTATS:
+      delete reinterpret_cast<sd::ops::LegacyStatsOp*>(op);
+      break;
+    case ::graph::OpType_RANDOM:
+      delete reinterpret_cast<sd::ops::LegacyRandomOp*>(op);
+      break;
+    case ::graph::OpType_BROADCAST:
+      delete reinterpret_cast<sd::ops::LegacyBroadcastOp*>(op);
+      break;
+    case ::graph::OpType_BROADCAST_BOOL:
+      delete reinterpret_cast<sd::ops::LegacyBroadcastBoolOp*>(op);
+      break;
+    case ::graph::OpType_CUSTOM:
+      delete reinterpret_cast<sd::ops::DeclarableOp*>(op);
+      break;
+    default:
+      THROW_EXCEPTION("Bad opType passed in");
+  }
+}
+
+sd::ops::DeclarableOp* sd::graph::Node::buildOpByType(::graph::OpType opType, int numInputs, int numIArgs, int numTArgs,
+                                                      int opNum, NDArray* scalar) {
+  switch (opType) {
+    case ::graph::OpType_PAIRWISE:
+      return new sd::ops::LegacyPairwiseTransformOp(opNum);
+    case ::graph::OpType_PAIRWISE_BOOL:
+      return new sd::ops::LegacyPairwiseTransformBoolOp(opNum);
+    case ::graph::OpType_TRANSFORM_STRICT:
+      return new sd::ops::LegacyTransformStrictOp(opNum);
+    case ::graph::OpType_TRANSFORM_SAME:
+      return new sd::ops::LegacyTransformSameOp(opNum);
+    case ::graph::OpType_TRANSFORM_FLOAT:
+      return new sd::ops::LegacyTransformFloatOp(opNum);
+    case ::graph::OpType_TRANSFORM_BOOL:
+      return new sd::ops::LegacyTransformBoolOp(opNum);
+    case ::graph::OpType_SCALAR:
+      return scalar == nullptr ? new sd::ops::LegacyScalarOp(opNum) : new sd::ops::LegacyScalarOp(opNum, *scalar);
+    case ::graph::OpType_SCALAR_BOOL:
+      return scalar == nullptr ? new sd::ops::LegacyScalarBoolOp(opNum)
+                               : new sd::ops::LegacyScalarBoolOp(opNum, *scalar);
+    case ::graph::OpType_REDUCE_3:
+      return new sd::ops::LegacyReduce3Op(opNum);
+    case ::graph::OpType_REDUCE_SAME:
+      return new sd::ops::LegacyReduceSameOp(opNum);
+    case ::graph::OpType_REDUCE_FLOAT:
+      return new sd::ops::LegacyReduceFloatOp(opNum);
+    case ::graph::OpType_REDUCE_LONG:
+      return new sd::ops::LegacyReduceLongOp(opNum);
+    case ::graph::OpType_REDUCE_BOOL:
+      return new sd::ops::LegacyReduceBoolOp(opNum);
+    case ::graph::OpType_INDEX_REDUCE:
+      return new sd::ops::LegacyIndexReduceOp(opNum);
+    case ::graph::OpType_SUMMARYSTATS:
+      return new sd::ops::LegacyStatsOp(opNum);
+    case ::graph::OpType_RANDOM:
+      return new sd::ops::LegacyRandomOp(opNum);
+    case ::graph::OpType_BROADCAST:
+      return new sd::ops::LegacyBroadcastOp(opNum);
+    case ::graph::OpType_BROADCAST_BOOL:
+      return new sd::ops::LegacyBroadcastBoolOp(opNum);
+    default:
+      THROW_EXCEPTION("Bad opType passed in");
+  }
+
+  return nullptr;
+}
+
+bool Node::isDeductable() { return _isDeductable; }
+
+void Node::setDeductable(bool reallyDeductable) { _isDeductable = reallyDeductable; }
+
+Node* Node::clone() {
+  if (this->_customOp && this->_opType == ::graph::OpType_CUSTOM) {
+    auto clone = new Node(this->_customOp, _id);
+    clone->pullValues(this);
+    return clone;
+  } else {
+    auto clone = new Node(_opType, _opNum, _id);
+
+    clone->pullValues(this);
+
+    // op time
+    if (!_isDeductable)
+      clone->_customOp = _customOp;
+    else {
+      auto c = static_cast<sd::ops::LegacyOp*>(_customOp);
+      clone->_customOp = c->clone();
+    }
+
+    return clone;
+  }
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/NodeState.cpp

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16/11/17.
+//
+#include <graph/NodeState.h>
+
+namespace sd {
+namespace graph {
+NodeState::NodeState(int id) { _id = id; }
+
+void NodeState::setInnerTime(LongType time) { _inner = time; }
+
+void NodeState::setOuterTime(LongType time) { _outer = time; }
+
+LongType NodeState::innerTime() { return _inner; }
+
+LongType NodeState::outerTime() { return _outer; }
+
+void NodeState::markActive(bool isActive) { _active = isActive; }
+
+bool NodeState::isActive() { return _active; }
+
+int NodeState::branch() { return _branch; }
+
+void NodeState::markBranch(int index) { _branch = index; }
+
+bool NodeState::wasExecuted() { return _executed; }
+
+void NodeState::markExecuted(bool wasExecuted) { _executed = wasExecuted; }
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/ResultWrapper.cpp

@@ -0,0 +1,46 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 11/06/18.
+//
+#include <graph/ResultWrapper.h>
+
+#include <stdexcept>
+
+namespace sd {
+namespace graph {
+ResultWrapper::ResultWrapper(LongType size, Pointer ptr) {
+  if (size <= 0) THROW_EXCEPTION("FlatResult size should be > 0");
+
+  _size = size;
+  _pointer = ptr;
+}
+
+ResultWrapper::~ResultWrapper() {
+  if (_pointer != nullptr && _size > 0) {
+    auto ptr = reinterpret_cast<char *>(_pointer);
+    delete[] ptr;
+  }
+}
+
+LongType ResultWrapper::size() { return _size; }
+
+Pointer ResultWrapper::pointer() { return _pointer; }
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/Scope.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 14.10.2017.
+//
+#include <graph/Scope.h>
+
+namespace sd {
+namespace graph {
+Scope::Scope(int id, const char* name) {
+  _id = id;
+
+  if (name != nullptr)
+    _name = name;
+  else
+    name = "";
+}
+
+Scope::~Scope() {
+  for (auto v : _nodes) delete v;
+}
+
+void Scope::push_back(Node* node) { _nodes.emplace_back(node); }
+
+std::vector<Node*>* Scope::nodes() { return &_nodes; }
+
+int Scope::size() { return (int)_nodes.size(); }
+
+int Scope::id() { return _id; }
+
+std::string* Scope::name() { return &_name; }
+
+void Scope::forgetNodes() { _nodes.clear(); }
+
+Scope* Scope::clone() {
+  auto clone = new Scope(_id, _name.c_str());
+
+  for (auto v : _nodes) clone->_nodes.emplace_back(v->clone());
+
+  return clone;
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/SessionLocalStorage.cpp

@@ -0,0 +1,120 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <graph/SessionLocalStorage.h>
+#include <graph/Stash.h>
+#include <graph/VariableSpace.h>
+
+namespace sd {
+namespace graph {
+SessionLocalStorage::SessionLocalStorage(VariableSpace* variableSpace, Stash* stash) {
+  // we start from 1, since key 0 holds original VariableSpace
+  _sessionCounter.store(1);
+  _variableSpace = variableSpace;
+  _stash = stash;
+}
+
+VariableSpace* SessionLocalStorage::localVariableSpace(LongType sessionId) {
+  _mutex.lock();
+  auto varSpace = _threadVariableSpace.at(sessionId);
+  _mutex.unlock();
+
+  return varSpace;
+}
+
+VariableSpace* SessionLocalStorage::localVariableSpace() { return localVariableSpace(getSessionId()); }
+
+SessionLocalStorage::~SessionLocalStorage() {
+  for (const auto& v : _threadVariableSpace) {
+    delete v.second;
+  }
+}
+
+LongType SessionLocalStorage::getThreadId() {
+#ifdef __APPLE__
+  // syscall?
+#elif _WIN32
+  // some win32api
+#else
+  // syscall!
+#endif
+  auto id = std::this_thread::get_id();
+  uint64_t* ptr = (uint64_t*)&id;
+  return (*ptr);
+}
+
+int SessionLocalStorage::numberOfSessions() {
+  _mutex.lock();
+  int size = (int)_threadSession.size();
+  _mutex.unlock();
+  return size;
+}
+
+void SessionLocalStorage::endSession(LongType sessionId) {
+  // we should delete specific holders here
+  _mutex.lock();
+  auto vs = _threadVariableSpace[sessionId];
+  _threadVariableSpace.erase(sessionId);
+
+  delete vs;
+  _mutex.unlock();
+}
+
+void SessionLocalStorage::endSession() {
+  auto tid = getThreadId();
+
+  _mutex.lock();
+
+  auto ntid = _threadSession[tid];
+  _threadSession.erase(tid);
+
+  _mutex.unlock();
+
+  endSession(ntid);
+}
+
+LongType SessionLocalStorage::getSessionId() {
+  auto tid = getThreadId();
+
+  _mutex.lock();
+  auto ntid = _threadSession[tid];
+
+  _mutex.unlock();
+
+  return ntid;
+}
+
+LongType SessionLocalStorage::startSession() {
+  auto tid = getThreadId();
+
+  sd_debug("Adding ThreadId: %i;\n", (int)tid);
+  LongType ntid = _sessionCounter++;
+  _mutex.lock();
+
+  _threadSession[tid] = ntid;
+  _threadVariableSpace[ntid] = _variableSpace->clone();
+
+  _mutex.unlock();
+
+  return ntid;
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/Stash.cpp

@@ -0,0 +1,98 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <graph/Stash.h>
+
+namespace std {
+size_t hash<sd::graph::KeyPair>::operator()(const sd::graph::KeyPair &k) const {
+  using std::hash;
+  auto res = std::hash<std::string>()(k.name());
+  res ^= std::hash<int>()(k.key()) + 0x9e3779b9 + (res << 6) + (res >> 2);
+  return res;
+}
+}  // namespace std
+
+namespace sd {
+namespace graph {
+KeyPair::KeyPair(int node, const char *name) {
+  _node = node;
+  _name = std::string(name);
+}
+
+bool KeyPair::operator<(const KeyPair &other) const {
+  if (_node < other._node)
+    return true;
+  else if (_node > other._node)
+    return false;
+  else
+    return _name < other._name;
+}
+
+Stash::Stash() {
+  //
+}
+
+Stash::~Stash() {
+  if (_handles.size() > 0) this->clear();
+}
+
+/*
+bool sd::graph::Stash::checkStash(sd::graph::Block& block, const char *name) {
+    return checkStash(block.getNodeId(), name);
+}
+ */
+
+bool Stash::checkStash(int nodeId, const char *name) {
+  KeyPair kp(nodeId, name);
+  return _stash.count(kp) > 0;
+}
+
+/*
+sd::NDArray* sd::graph::Stash::extractArray(sd::graph::Block& block, const char *name) {
+    return extractArray(block.getNodeId(), name);
+}
+*/
+NDArray *Stash::extractArray(int nodeId, const char *name) {
+  KeyPair kp(nodeId, name);
+  return _stash[kp];
+}
+/*
+void sd::graph::Stash::storeArray(sd::graph::Block& block, const char *name, sd::NDArray *array) {
+    storeArray(block.getNodeId(), name, array);
+}
+*/
+
+void Stash::storeArray(int nodeId, const char *name, NDArray *array) {
+  KeyPair kp(nodeId, name);
+  _stash[kp] = array;
+
+  // storing reference to delete it once it's not needed anymore
+  _handles.push_back(array);
+}
+
+void Stash::clear() {
+  for (auto v : _handles) delete v;
+
+  _handles.clear();
+  _stash.clear();
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/TimeHolder.cpp

@@ -0,0 +1,43 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16/11/17.
+//
+#include <graph/TimeHolder.h>
+
+namespace sd {
+namespace graph {
+
+void TimeHolder::setOuterTime(int nodeId, LongType time) { _outer[nodeId] = time; }
+
+void TimeHolder::setInnerTime(int nodeId, LongType time) { _inner[nodeId] = time; }
+
+LongType TimeHolder::outerTime(int nodeId) {
+  if (_outer.count(nodeId) == 0) return 0;
+
+  return _outer[nodeId];
+}
+
+LongType TimeHolder::innerTime(int nodeId) {
+  if (_inner.count(nodeId) == 0) return 0;
+
+  return _inner[nodeId];
+}
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/Variable.cpp

@@ -0,0 +1,298 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <array/ByteOrderUtils.h>
+#include <array/DataTypeConversions.h>
+#include <array/DataTypeUtils.h>
+#include <graph/FlatUtils.h>
+#include <graph/Variable.h>
+#include <helpers/EnumUtils.h>
+#include <helpers/StringUtils.h>
+
+namespace sd {
+namespace graph {
+
+template <typename N>
+Variable *Variable::asT() {
+  auto result = new Variable(this->isPlaceholder());
+
+  result->markExternal(this->_external);
+  result->setId(this->_id);
+  result->markReadOnly(this->_readOnly);
+  result->setName(&this->_name);
+  result->setIndex(this->_index);
+
+  if (this->_ndarray != nullptr) result->setNDArray(new NDArray(this->_ndarray->template asT<N>()));
+
+  // FIXME: add support for ArrayList
+  if (this->_list != nullptr) {
+    sd_printf("ArrayList not supported yet\n", "");
+    THROW_EXCEPTION("ArrayList not supported yet for asT");
+  }
+
+  return result;
+}
+BUILD_SINGLE_TEMPLATE( SD_LIB_EXPORT Variable *Variable::asT, (), SD_COMMON_TYPES);
+
+Variable *Variable::clone() {
+  auto result = new Variable(this->isPlaceholder());
+  result->_external = this->_external;
+  result->_id = this->_id;
+  result->_readOnly = this->_readOnly;
+  result->_name = this->_name;
+  result->_index = this->_index;
+
+  if (this->_ndarray != nullptr) {
+    result->_ndarray = new NDArray(this->_ndarray->dup(this->_ndarray->ordering(), false));
+    result->_readOnly = false;
+    result->_removable = true;
+  }
+
+  if (this->_list != nullptr) result->_list = this->_list->clone();
+
+  return result;
+}
+
+void Variable::setIndex(int index) { _index = index; }
+
+bool Variable::hasNDArray() { return _ndarray != nullptr; }
+
+void Variable::setVariableType(VariableType variableType) { _variableType = variableType; }
+
+bool Variable::hasNDArrayList() { return _list != nullptr; }
+
+bool Variable::isPlaceholder() { return _placeholder; }
+
+std::string *Variable::getName() { return &_name; }
+
+void Variable::setName(std::string *name) { _name = *name; }
+
+int Variable::id() { return _id; }
+
+int Variable::index() { return _index; }
+
+void Variable::setId(int id) { _id = id; }
+
+bool Variable::isEmpty() {
+  if (_variableType == NDARRAY)
+    return _ndarray == nullptr || !_ndarray->nonNull();
+  else if (_variableType == ARRAY_LIST)
+    return _list == nullptr;
+
+  return false;
+}
+
+bool Variable::isExternal() { return _external; }
+
+bool Variable::isReadOnly() { return _readOnly; }
+
+void Variable::markExternal(bool reallyExternal) { this->_external = reallyExternal; }
+
+void Variable::markRemovable(bool reallyRemovable) {
+  if (!reallyRemovable) sd_debug("", "");
+  this->_removable = reallyRemovable;
+}
+
+void Variable::markReadOnly(bool reallyReadOnly) { this->_readOnly = reallyReadOnly; }
+
+NDArray *Variable::getNDArray() {
+  if (_variableType != NDARRAY) {
+    sd_printf("Variable[%i:%i/<%s>] is has [%s] type, but NDArray was requested\n", this->_id, this->_index,
+              this->_name.c_str(), EnumUtils::_VariableTypeToString(_variableType));
+  }
+
+  if (this->_ndarray == nullptr) {
+    if (_name.empty()) {
+      auto nodeId = StringUtils::valueToString<int>(this->id());
+      auto outputIndex = StringUtils::valueToString<int>(this->index());
+      auto msg = "Array doesn't exist for Variable <" + nodeId + ":" + outputIndex + ">";
+      THROW_EXCEPTION(msg.c_str());
+    } else {
+      auto outputIndex = StringUtils::valueToString<int>(this->index());
+      auto msg = "Array doesn't exist for Variable <" + this->_name + ":" + outputIndex + ">";
+      THROW_EXCEPTION(msg.c_str());
+    }
+  }
+
+  return this->_ndarray;
+}
+
+NDArrayList *Variable::getNDArrayList() {
+  if (_variableType != ARRAY_LIST) {
+    sd_debug("Variable[%i:%i/<%s>] is has [%s] type, but NDArrayList was requested\n", this->_id, this->_index,
+             this->_name.c_str(), EnumUtils::_VariableTypeToString(_variableType));
+  }
+  return this->_list;
+}
+
+bool Variable::isRemovable() { return _removable; }
+
+void Variable::setNDArrayList(NDArrayList *list) {
+  this->_variableType = ARRAY_LIST;
+  this->_list = list;
+}
+
+void Variable::setNDArray(NDArray *array) {
+  this->_variableType = NDARRAY;
+  this->_ndarray = array;
+}
+
+VariableType Variable::variableType() { return _variableType; }
+
+Variable::Variable(const ::graph::FlatVariable *flatVariable) {
+  auto vid = flatVariable->id();
+  this->_id = vid->first();
+  this->_index = vid->second();
+
+  if (flatVariable->name() != nullptr && flatVariable->name()->size() != 0) this->_name = flatVariable->name()->str();
+
+  _external = true;
+  _readOnly = false;
+
+  int8_t *buffer = nullptr;
+
+  switch (flatVariable->variabletype()) {
+    case ::graph::VarType_VARIABLE: {
+      // ?????
+      if (flatVariable->ndarray() != nullptr) {
+        auto ar = flatVariable->ndarray();
+        _ndarray = FlatUtils::fromFlatArray(ar);
+      }
+
+      _variableType = NDARRAY;
+    } break;
+    case ::graph::VarType_CONSTANT: {
+      if (flatVariable->ndarray() == nullptr) THROW_EXCEPTION("CONSTANT variable must have NDArray bundled");
+
+      auto ar = flatVariable->ndarray();
+      if (ar->dtype() == ::graph::DType_UTF8) {
+        _ndarray = FlatUtils::fromFlatArray(ar);
+      } else {
+        _ndarray = FlatUtils::fromFlatArray(ar);
+      }
+
+      _variableType = NDARRAY;
+    } break;
+    case ::graph::VarType_ARRAY: {
+      // ?????
+      if (flatVariable->ndarray() != nullptr) {
+        auto ar = flatVariable->ndarray();
+        _ndarray = FlatUtils::fromFlatArray(ar);
+        // _ndarray->triggerAllocationFlag(true);
+      }
+
+      _variableType = NDARRAY;
+    } break;
+    case ::graph::VarType_PLACEHOLDER: {
+      if (flatVariable->shape() == nullptr && flatVariable->ndarray() == nullptr)
+        THROW_EXCEPTION("PLACEHOLDER variable must have shape defined");
+
+      if (flatVariable->ndarray() != nullptr) {
+        auto ar = flatVariable->ndarray();
+        _ndarray = FlatUtils::fromFlatArray(ar);
+
+        _variableType = NDARRAY;
+      }
+
+      if (flatVariable->shape() != nullptr) {
+        int shapeLen = flatVariable->shape()->size();
+        for (size_t i = 0; i < flatVariable->shape()->size(); i++) _shape.emplace_back(flatVariable->shape()->Get(i));
+
+        if (_ndarray == nullptr) _variableType = PLACEHOLDER;
+      }
+    } break;
+    default:
+      THROW_EXCEPTION("Unknown variable type used");
+  }
+}
+
+std::vector<LongType> &Variable::shape() { return _shape; }
+
+Variable::Variable(bool placeholder) { _placeholder = placeholder; }
+
+Variable::Variable(NDArray *array, const char *name) {
+  _ndarray = array;
+
+  _external = false;
+  _readOnly = false;
+
+  if (name != nullptr) _name = std::string(name);
+
+  if (_ndarray != nullptr) _variableType = NDARRAY;
+}
+
+Variable::Variable(NDArray *array, const char *name, int id, int idx) : Variable(array, name) {
+  _id = id;
+  _index = idx;
+}
+
+Variable::~Variable() {
+  if (_variableType == NDARRAY) {
+    sd_debug("Removing variable <%i:%i>\n", _id, _index);
+    if (_ndarray != nullptr && _removable && !_readOnly) delete _ndarray;
+  }
+}
+
+void Variable::setId(int id, int idx) {
+  _id = id;
+  _index = idx;
+}
+
+flatbuffers::Offset<::graph::FlatVariable> Variable::asFlatVariable(flatbuffers::FlatBufferBuilder &builder) {
+  if (this->hasNDArray()) {
+    auto array = this->getNDArray();
+    auto fShape = builder.CreateVector(array->getShapeInfoAsFlatVector());
+
+    auto fBuffer = builder.CreateVector(array->asByteVector());
+
+    // packing array
+    auto fArray = CreateFlatArray(builder, fShape, fBuffer, (::graph::DType)array->dataType());
+
+    // packing id/index of this var
+    auto fVid = ::graph::CreateIntPair(builder, this->_id, this->_index);
+
+    // name is still optional
+    flatbuffers::Offset<flatbuffers::String> stringId = 0;
+    if (!this->_name.empty()) stringId = builder.CreateString(this->_name);
+
+    // returning array
+    return CreateFlatVariable(builder, fVid, stringId, static_cast<::graph::DType>(array->dataType()), 0, fArray);
+  } else {
+    THROW_EXCEPTION("Variable::asFlatVariable isn't possible for NDArrayList");
+  }
+
+  return CreateFlatVariable(builder, 0, 0, static_cast<::graph::DType>(0), 0, 0);
+}
+}  // namespace graph
+}  // namespace sd
+
+namespace std {
+
+size_t hash<std::pair<int, int>>::operator()(const std::pair<int, int> &k) const {
+  auto v = std::hash<int>()(k.first);
+  v ^= std::hash<int>()(k.second) + 0x9e3779b9 + (v << 6) + (v >> 2);
+  return v;
+}
+
+size_t hash<bfloat16>::operator()(const bfloat16 &k) const { return std::hash<float>()((float)k); }
+
+size_t hash<float16>::operator()(const float16 &k) const { return std::hash<float>()((float)k); }
+}  // namespace std

libnd4j/include/graph/impl/VariableProxy.cpp

@@ -0,0 +1,196 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <graph/VariableProxy.h>
+
+namespace sd {
+namespace graph {
+
+VariableProxy::VariableProxy(VariableSpace *ref) {
+  if (ref == nullptr) _backed = new VariableSpace();
+
+  _backed = ref;
+  _current = new VariableSpace();
+}
+
+VariableProxy::~VariableProxy() {
+  delete _current;
+}
+
+int VariableProxy::numberOfPlaceholders() { return _backed->numberOfPlaceholders(); }
+
+std::vector<Variable *> *VariableProxy::getPlaceholders() { return _backed->getPlaceholders(); }
+
+bool VariableProxy::hasExternalVariable(int it) { return _backed->hasExternalVariable(it); }
+
+bool VariableProxy::hasExternalVariable(std::pair<int, int> &pair) { return _backed->hasExternalVariable(pair); }
+
+bool VariableProxy::hasExternalVariable(std::string *symbol) { return _backed->hasExternalVariable(symbol); }
+
+bool VariableProxy::hasVariable(int id) { return _current->hasVariable(id) || _backed->hasVariable(id); }
+
+bool VariableProxy::hasVariable(int id, int idx) {
+  return _current->hasVariable(id, idx) || _backed->hasVariable(id, idx);
+}
+
+bool VariableProxy::hasVariable(std::pair<int, int> &pair) {
+  return _current->hasVariable(pair) || _backed->hasVariable(pair);
+}
+
+void VariableProxy::dropVariable(std::pair<int, int> &pair) { dropVariable(pair.first, pair.second); }
+
+void VariableProxy::dropVariable(int id, int idx) {
+  assert(_current->hasVariable(id, idx));
+
+  _current->dropVariable(id, idx);
+}
+
+std::vector<Variable *> VariableProxy::getVariables() {
+  std::vector<Variable *> result;
+
+  auto b = _backed->getVariables();
+  auto c = _current->getVariables();
+
+  for (auto v : b) result.emplace_back(v);
+
+  for (auto v : c) result.emplace_back(v);
+
+  return result;
+}
+
+bool VariableProxy::hasVariable(std::string *symbol) {
+  return _current->hasVariable(symbol) || _backed->hasVariable(symbol);
+}
+
+Variable *VariableProxy::getVariable(int id) {
+  if (_current->hasVariable(id)) return _current->getVariable(id);
+
+  if (_backed->hasVariable(id)) return _backed->getVariable(id);
+
+  sd_printf("Unable to get Variable to proxy: [%i]\n", id);
+  THROW_EXCEPTION("Bad arguments");
+  return nullptr;
+}
+
+Variable *VariableProxy::getVariable(int id, int idx) {
+  if (_current->hasVariable(id, idx)) return _current->getVariable(id, idx);
+
+  if (_backed->hasVariable(id, idx)) return _backed->getVariable(id, idx);
+
+  sd_printf("Unable to get Variable to proxy: [%i:%i]\n", id, idx);
+  THROW_EXCEPTION("Bad arguments");
+  return nullptr;
+}
+
+Variable *VariableProxy::getVariable(std::pair<int, int> &pair) {
+  if (_current->hasVariable(pair)) return _current->getVariable(pair);
+
+  if (_backed->hasVariable(pair)) return _backed->getVariable(pair);
+
+  sd_printf("Unable to get Variable to proxy: [%i:%i]\n", pair.first, pair.second);
+  THROW_EXCEPTION("Bad arguments");
+  return nullptr;
+}
+
+Variable *VariableProxy::getVariable(std::string *symbol) {
+  if (_current->hasVariable(symbol)) return _current->getVariable(symbol);
+
+  if (_backed->hasVariable(symbol)) return _backed->getVariable(symbol);
+
+  sd_printf("Unable to get Variable to proxy: [%s]\n", symbol->c_str());
+  THROW_EXCEPTION("Bad arguments");
+  return nullptr;
+}
+
+void VariableProxy::replaceVariable(Variable *variable) {
+  if (variable->getName() != nullptr && !variable->getName()->empty()) {
+    // if variable has name defined - we should resolve it via backing var space
+    if (_backed->hasVariable(variable->getName())) {
+      auto origVar = _backed->getVariable(variable->getName());
+      variable->setId(origVar->id(), origVar->index());
+      _current->replaceVariable(variable);
+    } else
+      _current->replaceVariable(variable);
+  } else  // if proxy has variable - that's one story
+    _current->replaceVariable(variable);
+}
+
+Variable *VariableProxy::putVariable(std::pair<int, int> &pair, NDArray *array) {
+  return _current->putVariable(pair, array);
+}
+
+void VariableProxy::putVariable(std::pair<int, int> &pair, Variable *variable) {
+  _current->putVariable(pair, variable);
+}
+
+void VariableProxy::putVariable(int id, Variable *variable) { _current->putVariable(id, variable); }
+
+void VariableProxy::putVariable(int id, NDArray *array) { _current->putVariable(id, array); }
+
+void VariableProxy::putVariable(int id, int idx, NDArray &array) { _current->putVariable(id, idx, array); }
+
+Variable *VariableProxy::putVariable(int id, int idx, NDArray *array) { return _current->putVariable(id, idx, array); }
+
+void VariableProxy::putVariable(int id, int idx, Variable *array) { _current->putVariable(id, idx, array); }
+
+void VariableProxy::trackList(NDArrayList *list) { _current->trackList(list); }
+
+Stash *VariableProxy::getStash() { return _current->getStash(); }
+
+void VariableProxy::setFlowPath(FlowPath *timers) { _current->setFlowPath(timers); }
+
+FlowPath *VariableProxy::flowPath() { return _current->flowPath(); }
+
+void VariableProxy::putOutputVariable(Variable *variable) { _current->putOutputVariable(variable); }
+
+LongType VariableProxy::externalMemory() { return _backed->externalMemory() + _current->externalMemory(); }
+
+LongType VariableProxy::internalMemory() { return _backed->internalMemory() + _current->internalMemory(); }
+
+LongType VariableProxy::totalMemory() { return _backed->totalMemory() + _current->totalMemory(); }
+
+int VariableProxy::externalEntries() { return _backed->externalEntries() + _current->externalEntries(); }
+
+int VariableProxy::internalEntries() { return _backed->internalEntries() + _current->internalEntries(); }
+
+int VariableProxy::totalEntries() { return _backed->totalEntries() + _current->totalEntries(); }
+
+VariableSpace *VariableProxy::clone() {
+  auto clone = new VariableProxy(_backed);
+
+  delete clone->_current;
+  clone->_current = _current->clone();
+
+  return clone;
+}
+
+VariableSpace &VariableProxy::operator=(const VariableSpace &other) {
+  if (this == &other) return *this;
+
+  sd_printf("VariableProxy = not implemented\n", "");
+
+  return *this;
+}
+
+memory::Workspace *VariableProxy::workspace() { return _workspace; }
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/VariableSpace.cpp

@@ -0,0 +1,382 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <graph/VariableSpace.h>
+#include <legacy/NativeOps.h>
+
+namespace sd {
+namespace graph {
+std::vector<sd::graph::Variable*>* sd::graph::VariableSpace::getExternalVariables() { return &_external; }
+
+sd::graph::Stash* sd::graph::VariableSpace::getStash() { return &_stash; }
+
+sd::graph::VariableSpace* sd::graph::VariableSpace::clone() {
+  auto result = new VariableSpace();
+
+  for (auto const& x : _paired) {
+    std::pair<int, int> pair(x.first.first, x.first.second);
+
+    Variable* clonedVar = x.second->clone();
+
+    result->injectVariable(pair, clonedVar);
+  }
+
+  return result;
+}
+
+void VariableSpace::setWorkspace(sd::memory::Workspace* workspace) {
+  _workspace = workspace;
+}
+
+sd::graph::VariableSpace* sd::graph::VariableSpace::asT() {
+  auto result = new VariableSpace();
+
+  for (auto const& x : _paired) {
+    std::pair<int, int> pair(x.first.first, x.first.second);
+
+
+  }
+
+  return result;
+}
+
+void sd::graph::VariableSpace::injectVariable(std::pair<int, int>& pair, Variable* variable) {
+  if (pair.second == 0) {
+    if (pair.first < 0)
+      this->_variables[pair.first] = variable;
+    else
+      this->_temporary[pair.first] = variable;
+  }
+
+  if (variable->getName() != nullptr && variable->getName()->length() > 0)
+    this->_symbolic[*(variable->getName())] = variable;
+
+  this->_paired[pair] = variable;
+
+  this->_handles->push_back(variable);
+}
+
+std::vector<sd::graph::Variable*>* sd::graph::VariableSpace::getPlaceholders() { return &_placeholders; }
+
+int sd::graph::VariableSpace ::numberOfPlaceholders() { return _placeholders.size(); }
+
+bool sd::graph::VariableSpace::hasVariable(std::string* symbol) { return _symbolic.count(*symbol) == 1; }
+
+sd::graph::Variable* sd::graph::VariableSpace::getVariable(std::string* symbol) { return _symbolic.at(*symbol); }
+
+bool sd::graph::VariableSpace::hasVariable(int id, int index) {
+  std::pair<int, int> pair(id, index);
+  return hasVariable(pair);
+}
+
+bool VariableSpace::hasExternalVariable(int id) {
+  if (!hasVariable(id)) return false;
+
+  auto var = getVariable(id);
+  return var->isExternal();
+}
+
+bool VariableSpace::hasExternalVariable(std::pair<int, int>& pair) {
+  if (!hasVariable(pair)) return false;
+
+  auto var = getVariable(pair);
+  return var->isExternal();
+}
+
+bool VariableSpace::hasExternalVariable(std::string* symbol) {
+  if (!hasVariable(symbol)) return false;
+
+  auto var = getVariable(symbol);
+  return var->isExternal();
+}
+
+sd::graph::Variable* sd::graph::VariableSpace::getVariable(int id, int index) {
+  std::pair<int, int> pair(id, index);
+  return getVariable(pair);
+}
+
+sd::graph::Variable* sd::graph::VariableSpace::getVariable(std::pair<int, int>& pair) {
+  if (pair.first < 0) {
+    return getVariable(pair.first);
+  } else {
+    return _paired.at(pair);
+  }
+  sd_printf("Unknown variable requested: [%i,%i]\n", pair.first, pair.second);
+  THROW_EXCEPTION("Unknown variable requested");
+  return nullptr;
+}
+
+bool sd::graph::VariableSpace::hasVariable(int id) { return _variables.count(id) == 1 || _temporary.count(id) == 1; }
+
+bool sd::graph::VariableSpace::hasVariable(std::pair<int, int>& id) { return _paired.count(id) > 0; }
+
+void sd::graph::VariableSpace::putOutputVariable(Variable* variable) {
+  putVariable(variable->id(), variable);
+}
+
+int sd::graph::VariableSpace::externalEntries() { return _external.size(); }
+
+int sd::graph::VariableSpace::internalEntries() { return _internal.size(); }
+
+int sd::graph::VariableSpace::totalEntries() { return externalEntries() + internalEntries(); }
+
+sd::LongType sd::graph::VariableSpace::externalMemory() {
+  sd::LongType size = 0;
+  for (auto n : _external) {
+    size += n->getNDArray()->memoryFootprint();
+  }
+
+  return size;
+}
+
+std::vector<Variable*> VariableSpace::getVariables() {
+  std::vector<Variable*> result;
+
+  for (auto v : _internal) result.emplace_back(v);
+
+  for (auto v : _external) result.emplace_back(v);
+
+  return result;
+}
+
+sd::LongType sd::graph::VariableSpace::internalMemory() {
+  sd::LongType size = 0;
+  for (auto n : _internal) {
+    size += n->getNDArray()->memoryFootprint();
+  }
+
+  return size;
+}
+
+sd::LongType sd::graph::VariableSpace::totalMemory() { return externalMemory() + internalMemory(); }
+
+Variable* sd::graph::VariableSpace::putVariable(std::pair<int, int>& pair, NDArray* array) {
+  auto variable = new Variable(array, nullptr, pair.first, pair.second);
+  this->putVariable(pair, variable);
+  return variable;
+}
+
+Variable* sd::graph::VariableSpace::putVariable(int node, int idx, NDArray* array) {
+  std::pair<int, int> pair(node, idx);
+  return this->putVariable(pair, array);
+}
+
+void sd::graph::VariableSpace::putVariable(int node, int idx, Variable* variable) {
+  std::pair<int, int> pair(node, idx);
+  this->putVariable(pair, variable);
+}
+
+void sd::graph::VariableSpace::silentPutVariable(std::pair<int, int>& pair, Variable* variable) {
+  _varmap.lock();
+
+  _paired[pair] = variable;
+
+  _varmap.unlock();
+}
+
+void sd::graph::VariableSpace::putVariable(std::pair<int, int>& pair, Variable* variable) {
+  silentPutVariable(pair, variable);
+
+  if (variable->isPlaceholder()) _placeholders.push_back(variable);
+
+  // copying duplicate for compatibility
+  if (pair.second == 0 && !this->hasVariable(pair.first)) {
+    this->putVariable(pair.first, variable);
+  } else {
+    if (variable->getName() != nullptr && variable->getName()->length() != 0) {
+      _symbolic[*(variable->getName())] = variable;
+    }
+
+    _varmap.lock();
+
+    _handles->push_back(variable);
+
+    _varmap.unlock();
+  }
+}
+
+void VariableSpace::trackList(sd::NDArrayList* list) { _lists.emplace_back(list); }
+
+void sd::graph::VariableSpace::putVariable(int id, Variable* variable) {
+  // we don't want to add variables more then once
+  if (_variables.count(id) > 0 || _temporary.count(id) > 0) {
+    auto local = id < 0 ? _variables.at(id) : _temporary.at(id);
+
+    if (!local->hasNDArray() && variable->hasNDArray()) {
+      local->setNDArray(variable->getNDArray());
+
+      // we're inheriting this from Variable
+      local->markReadOnly(variable->isReadOnly());
+      local->markRemovable(variable->isRemovable());
+    }
+
+    return;
+  }
+
+  _varmap.lock();
+
+  _handles->emplace_back(variable);
+
+  if (_auto_counter >= id) _auto_counter = id - 1;
+
+  variable->setId(id);
+
+  if (variable->getName() != nullptr && variable->getName()->length() != 0) {
+    _symbolic[*(variable->getName())] = variable;
+  }
+
+  // we have special list for external variables to ensure graph completeness
+
+  if (id < 0) {
+    _external.push_back(variable);
+
+    _variables[id] = variable;
+  } else {
+    _internal.push_back(variable);
+
+    _temporary[id] = variable;
+  }
+
+  _varmap.unlock();
+
+  std::pair<int, int> pair(id, 0);
+  if (!hasVariable(pair)) {
+    this->silentPutVariable(pair, variable);
+
+    if (variable->isPlaceholder()) _placeholders.push_back(variable);
+  }
+}
+
+void sd::graph::VariableSpace::putVariable(int id, int idx, NDArray& array) {
+  auto* var = new sd::graph::Variable(&array, "", id, idx);
+  var->markRemovable(false);
+  var->markReadOnly(true);
+
+  // let's see if this op needs
+  bool d = this->hasVariable(id, idx);
+
+  this->putVariable(id, var);
+
+  // if var for this nodeid already exists - we'll just delete variable
+  if (d) delete var;
+}
+
+void sd::graph::VariableSpace::putVariable(int id, NDArray* array) {
+  auto* var = new sd::graph::Variable(array);
+  this->putVariable(id, var);
+}
+
+sd::graph::Variable* sd::graph::VariableSpace::getVariable(int id) {
+  if (id < 0) {
+    return _variables.at(id);
+  } else {
+    return _temporary.at(id);
+  }
+}
+
+LaunchContext* sd::graph::VariableSpace::launchContext() { return LaunchContext::defaultContext(); }
+
+std::vector<Variable*>* sd::graph::VariableSpace::handles() { return _handles; }
+
+/*
+ * FIXME: this thing have nice chances to become backend-specific!
+ */
+sd::graph::VariableSpace::~VariableSpace() {
+  // loop through variables and release them
+  for (auto p : *_handles) {
+    delete p;
+  }
+
+  delete _handles;
+
+  for (auto p : _lists) delete p;
+
+  _lists.clear();
+}
+
+VariableSpace& VariableSpace::operator=(const VariableSpace& other) {
+  if (this == &other) return *this;
+
+  for (auto const& x : other._paired) {
+    std::pair<int, int> pair(x.first.first, x.first.second);
+
+    Variable* clonedVar = x.second->clone();
+
+    if (pair.second == 0) {
+      if (pair.first < 0)
+        this->_variables[pair.first] = clonedVar;
+      else
+        this->_temporary[pair.first] = clonedVar;
+    }
+
+    if (clonedVar->getName() != nullptr && clonedVar->getName()->length() > 0)
+      this->_symbolic[*(clonedVar->getName())] = clonedVar;
+
+    this->_paired[pair] = clonedVar;
+
+    this->_handles->push_back(clonedVar);
+  }
+
+  return *this;
+}
+
+void VariableSpace::replaceVariable(Variable* variable) {
+  bool replaced = false;
+  // trying name first
+  if (variable->getName() != nullptr && !variable->getName()->empty()) {
+    sd_printf("Trying to replace variable by name: [%s]\n", variable->getName()->c_str());
+    if (hasVariable(variable->getName())) {
+      sd_printf("Replacing by name: [%s]\n", variable->getName()->c_str());
+      auto vs = getVariable(variable->getName());
+      dropVariable(vs->id(), vs->index());
+      putVariable(vs->id(), vs->index(), variable);
+      delete vs;
+      replaced = true;
+    }
+  } else {
+    sd_printf("Trying to replace variable by id: [%i:%i]\n", variable->id(), variable->index());
+    if (hasVariable(variable->id(), variable->index())) {
+      sd_printf("Replacing by id: [%i:%i]\n", variable->id(), variable->index());
+      auto vs = getVariable(variable->id(), variable->index());
+      dropVariable(variable->id(), variable->index());
+      putVariable(vs->id(), vs->index(), variable);
+      delete vs;
+      replaced = true;
+    }
+  }
+
+  if (!replaced) {
+    sd_printf("wasn't able to replace variable, putting\n", "");
+    putVariable(variable->id(), variable->index(), variable);
+  }
+}
+
+void VariableSpace::dropVariable(std::pair<int, int>& pair) { dropVariable(pair.first, pair.second); }
+
+void VariableSpace::dropVariable(int id, int idx) {}
+
+void VariableSpace::setFlowPath(FlowPath* flow) { _flow = flow; }
+
+FlowPath* VariableSpace::flowPath() { return _flow; }
+
+VariableSpace::VariableSpace() { _handles = new std::vector<Variable*>; }
+}  // namespace graph
+}  // namespace sd

libnd4j/include/graph/impl/VariablesSet.cpp

@@ -0,0 +1,40 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 15/11/17.
+//
+#include <graph/VariablesSet.h>
+
+namespace sd {
+namespace graph {
+Status VariablesSet::status() { return _status; }
+
+int VariablesSet::size() { return _holder.size(); }
+
+void VariablesSet::push_back(Variable *variable) { _holder.push_back(variable); }
+
+Variable *VariablesSet::at(int index) { return _holder.at(index); }
+
+VariablesSet::VariablesSet(Status status) { _status = status; }
+
+VariablesSet::~VariablesSet() {
+  for (auto v : _holder) delete v;
+}
+}  // namespace graph
+}  // namespace sd