chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/impl/BooleanOp.cpp

@@ -0,0 +1,143 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+#include "ops/declarable/BooleanOp.h"
+
+#include <array/NDArrayFactory.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <initializer_list>
+#include <vector>
+
+namespace sd {
+namespace ops {
+BooleanOp::BooleanOp(const char *name, int numInputs, bool scalar)
+    : DeclarableOp(name, numInputs, scalar) {
+  //
+}
+
+/**
+ * Output shape of any BooleanOp is ALWAYS scalar
+ */
+ShapeList *BooleanOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(DataType::BOOL));
+}
+
+bool BooleanOp::verify(Context &block) {
+  // check if scalar or not
+
+  // validation?
+
+  Status status = this->validateNonEmptyInput(block);
+  if (status != Status::OK) {
+    THROW_EXCEPTION("Bad inputs");
+  }
+
+  status = this->validateAndExecute(block);
+  if (status == Status::EQ_TRUE)
+    return true;
+  else if (status == Status::EQ_FALSE)
+    return false;
+  else {
+    sd_printf("Got error %i during [%s] evaluation: ", (int)status, this->getOpDescriptor()->getOpName()->c_str());
+    THROW_EXCEPTION("Internal error");
+  }
+
+  return false;
+}
+
+bool BooleanOp::prepareOutputs(Context &ctx) {
+  auto variableSpace = ctx.getVariableSpace();
+  if (ctx.isFastPath()) return true;
+
+  for (int e = 0; e < this->getOpDescriptor()->getNumberOfOutputs(); e++) {
+    std::pair<int, int> pair(ctx.nodeId(), e);
+
+    if (!variableSpace->hasVariable(pair)) variableSpace->putVariable(pair, new Variable());
+
+    auto var = ctx.variable(pair);
+
+    if (!var->hasNDArray()) {
+      var->setNDArray(NDArrayFactory::create_<bool>(false, ctx.launchContext()));
+      var->markRemovable(true);
+    }
+  }
+
+  return true;
+}
+
+Status BooleanOp::execute(Context *block) {
+  // basic validation: ensure inputs are set
+  REQUIRE_OK(this->validateNonEmptyInput(*block));
+
+  // ensure number of IArgs, TArgs match our expectations
+  REQUIRE_OK(this->validateArguments(*block));
+
+  // this method will allocate output NDArrays for this op
+  this->prepareOutputs(*block);
+
+  auto timeStart = std::chrono::system_clock::now();
+
+  Status status = this->validateAndExecute(*block);
+
+  auto timeEnd = std::chrono::system_clock::now();
+  auto outerTime = std::chrono::duration_cast<std::chrono::nanoseconds>(timeEnd - timeStart).count();
+  block->setInnerTime(outerTime);
+
+  // basically we're should be putting 0.0 as FALSE, and any non-0.0 value will be treated as TRUE
+  std::pair<int, int> p(block->nodeId(), 0);
+  auto var = block->isFastPath() ? block->fastpath_out()[0] : block->variable(p)->getNDArray();
+  if(!var->isEmpty())
+    var->p(LongType(0), status == Status::EQ_TRUE ? 1.0f : 0.0f);
+
+  // for CPU backend that's nop, but for CUDA-like archs this will update special buffer
+  var->syncToDevice();
+
+  if (status == Status::EQ_FALSE || status == Status::EQ_TRUE) return Status::OK;
+
+  sd_printf("%s: node_%i got unexpected result instead of boolean: [%i]\n", this->getOpName()->c_str(), block->nodeId(),
+            status);
+
+
+  traceExecIfNeeded(*block);
+
+
+  return Status::KERNEL_FAILURE;
+}
+
+bool BooleanOp::verify(const std::vector<NDArray *> &args) {
+  VariableSpace variableSpace;
+
+  int cnt = -1;
+  std::vector<int> in;
+  for (auto v : args) {
+    auto var = new Variable(v);
+    var->markRemovable(false);
+    in.push_back(cnt);
+    variableSpace.putVariable(cnt--, var);
+  }
+
+  Context block(1, &variableSpace, false);
+  block.fillInputs(in);
+
+  return this->verify(block);
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/BroadcastableBoolOp.cpp

@@ -0,0 +1,81 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/BroadcastableBoolOp.h>
+#include <system/op_boilerplate.h>
+
+namespace sd {
+namespace ops {
+BroadcastableBoolOp::BroadcastableBoolOp(const char *name, int numTArgs, int numIArgs)
+    : DeclarableCustomOp::DeclarableCustomOp(2, 1, name, false, numTArgs, numIArgs) {
+  //
+}
+ShapeList *BroadcastableBoolOp::calculateOutputShape(ShapeList *inputShape, sd::graph::Context &block) {
+  auto shapeList = SHAPELIST();
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  sd::DataType dtype = sd::DataType::BOOL;
+
+  if (shape::isEmptyConst(x) || shape::isEmptyConst(y)) {
+    // this is edge case, [3, 4] + [] = []
+    if ((shape::isEmptyConst(x) && shape::rank(x) == 0) || (shape::isEmptyConst(y) && shape::rank(y) == 0)) {
+      shapeList->push_back(ConstantShapeHelper::getInstance().createShapeInfo(ShapeDescriptor::emptyDescriptor(dtype)));
+      return shapeList;
+    }
+
+    sd::LongType *newshape = nullptr;
+    ShapeUtils::evalBroadcastShapeInfo(x, y, true, newshape, block.workspace());
+    // Cast to boolean and let ConstantShapeHelper manage the memory
+    auto castedShape = ConstantShapeHelper::getInstance().castToDataType(newshape, dtype);
+    shapeList->push_back(castedShape);
+  } else if (shape::isScalar(x) && shape::isScalar(y)) {
+    if (shape::rank(x) >= shape::rank(y)) {
+      auto castedShape = ConstantShapeHelper::getInstance().castToDataType(x, dtype);
+      shapeList->push_back(castedShape);
+    } else {
+      auto castedShape = ConstantShapeHelper::getInstance().castToDataType(y, dtype);
+      shapeList->push_back(castedShape);
+    }
+  } else if (shape::equalsSoft(x, y)) {
+    auto castedShape = ConstantShapeHelper::getInstance().castToDataType(x, dtype);
+    shapeList->push_back(castedShape);
+  } else if (shape::isScalar(x) && !shape::isScalar(y)) {
+    auto castedShape = ConstantShapeHelper::getInstance().castToDataType(y, dtype);
+    shapeList->push_back(castedShape);
+  } else if (!shape::isScalar(x) && shape::isScalar(y)) {
+    auto castedShape = ConstantShapeHelper::getInstance().castToDataType(x, dtype);
+    shapeList->push_back(castedShape);
+  } else if (ShapeUtils::areShapesBroadcastable(x, y)) {
+    sd::LongType *newshape = nullptr;
+    ShapeUtils::evalBroadcastShapeInfo(x, y, true, newshape, block.workspace());
+    // Cast to boolean and let ConstantShapeHelper manage the memory
+    auto castedShape = ConstantShapeHelper::getInstance().castToDataType(newshape, dtype);
+    shapeList->push_back(castedShape);
+  } else {
+    auto castedShape = ConstantShapeHelper::getInstance().castToDataType(x, dtype);
+    shapeList->push_back(castedShape);
+  }
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/BroadcastableOp.cpp

@@ -0,0 +1,87 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/BroadcastableOp.h>
+#include <system/op_boilerplate.h>
+
+namespace sd {
+namespace ops {
+BroadcastableOp::BroadcastableOp(const char *name, int numTArgs, int numIArgs)
+    : DeclarableCustomOp::DeclarableCustomOp(2, 1, name, false, numTArgs, numIArgs) {
+  //
+}
+
+ShapeList *BroadcastableOp::calculateOutputShape(ShapeList *inputShape, sd::graph::Context &block) {
+  auto shapeList = SHAPELIST();
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto outputs = _descriptor->getOutputTypesForOutput(0);
+  sd::DataType dtype = block.dataType(0);
+  if (block.dataType(0) != sd::DataType::BOOL && !(outputs.size() == 1 && outputs[0] == sd::DataType::BOOL)) {
+    if (Environment::getInstance().isExperimentalBuild()) {
+      if (shape::length(y) > shape::length(x)) {
+        dtype = DataTypeUtils::pickPairwiseResultType(y, x);
+      } else {
+        dtype = DataTypeUtils::pickPairwiseResultType(x, y);
+      }
+    } else {
+      dtype = ArrayOptions::dataType(x);
+    }
+  } else
+    dtype = sd::DataType::BOOL;
+
+  if (shape::isEmptyConst(x) || shape::isEmptyConst(y)) {
+    // this is edge case, [3, 4] + [] = []
+    if ((shape::isEmptyConst(x) && shape::rank(x) == 0) || (shape::isEmptyConst(y) && shape::rank(y) == 0)) {
+      auto desc = ShapeBuilders::emptyShapeInfo(dtype);
+      shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(desc)->primary());
+      return shapeList;
+    }
+
+    sd::LongType *newshape = nullptr;
+    ShapeUtils::evalBroadcastShapeInfo(x, y, true, newshape, block.workspace());
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(newshape)->primary());
+  } else if (shape::isScalar(x) && shape::isScalar(y)) {
+    if (shape::rank(x) >= shape::rank(y)) {
+      shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(x)->primary());
+    } else {
+      shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(y)->primary());
+    }
+  } else if (shape::equalsSoft(x, y)) {
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(x)->primary());
+  } else if (shape::isScalar(x) && !shape::isScalar(y)) {
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(y)->primary());
+  } else if (!shape::isScalar(x) && shape::isScalar(y)) {
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(x)->primary());
+  } else if (ShapeUtils::areShapesBroadcastable(x, y)) {
+    sd::LongType *newshape = nullptr;
+    ShapeUtils::evalBroadcastShapeInfo(x, y, true, newshape, block.workspace());
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(newshape)->primary());
+  } else {
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(x)->primary());
+
+  }
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/DeclarableCustomOp.cpp

@@ -0,0 +1,33 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.10.2017.
+//
+#include <ops/declarable/DeclarableCustomOp.h>
+#include <ops/declarable/DeclarableOp.h>
+
+namespace sd {
+namespace ops {
+DeclarableCustomOp::DeclarableCustomOp(int numInputs, int numOutputs, const char *opName, bool allowsInplace, int tArgs,
+                                       int iArgs)
+    : DeclarableOp(numInputs, numOutputs, opName, allowsInplace, tArgs, iArgs) {
+  //
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/DeclarableListOp.cpp

@@ -0,0 +1,151 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <graph/Context.h>
+#include <graph/Variable.h>
+#include <graph/VariableSpace.h>
+#include <ops/declarable/DeclarableListOp.h>
+#include <ops/declarable/OpDescriptor.h>
+
+namespace sd {
+namespace ops {
+DeclarableListOp::DeclarableListOp(int numInputs, int numOutputs, const char* opName, int tArgs, int iArgs)
+    : DeclarableOp(numInputs, numOutputs, opName, false, tArgs, iArgs) {
+  // This kind of operations work with sets: NDArrayList
+  this->getOpDescriptor()->setInputType(InputType_NUMERIC_SET);
+}
+
+/**
+ * This method just outputs scalar buffer
+ *
+ * @tparam T
+ * @param inputShape
+ * @param block
+ * @return
+ */
+ShapeList* DeclarableListOp::calculateOutputShape(ShapeList* inputShape, Context& block) {
+  // TODO: ensure this method isn't ever called
+
+  auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(block.dataType(), 'c', {1, 1});
+  return SHAPELIST(newShape);
+}
+
+NDArray* DeclarableListOp::getZ(sd::graph::Context& block, int inputId) {
+  return nullptr;
+}
+
+void DeclarableListOp::setupResult(NDArray* array, Context& block) {
+  block.pushNDArrayToVariableSpace(block.getNodeId(), 0, array);
+}
+
+void DeclarableListOp::setupResultList(NDArrayList* arrayList, Context& block) {
+  block.pushNDArrayListToVariableSpace(block.getNodeId(), 0, arrayList);
+}
+
+ResultSet DeclarableListOp::execute(NDArrayList* list, std::initializer_list<NDArray*> inputs,
+                                    std::initializer_list<double> tArgs, std::initializer_list<int> iArgs) {
+  std::vector<NDArray*> ins(inputs);
+  std::vector<double> tas(tArgs);
+  std::vector<int> ias(iArgs);
+  return this->execute(list, ins, tas, ias);
+}
+
+Status DeclarableListOp::execute(Context* block) {
+  if (block == nullptr) THROW_EXCEPTION("Block is NULL");
+
+  sd_debug("Executing list op: [%s]\n", this->getOpName()->c_str());
+
+  // ensure number of IArgs, TArgs match our expectations
+  REQUIRE_OK(this->validateArguments(*block));
+
+  // we shouldn't call for this in ListOp
+  // this->prepareOutputs(*block);
+
+  auto timeStart = std::chrono::system_clock::now();
+
+  Status status = this->validateAndExecute(*block);
+
+  auto timeEnd = std::chrono::system_clock::now();
+  auto outerTime = std::chrono::duration_cast<std::chrono::nanoseconds>(timeEnd - timeStart).count();
+  block->setInnerTime(outerTime);
+  traceExecIfNeeded(*block);
+
+  return status;
+}
+
+ResultSet DeclarableListOp::execute(NDArrayList* list, std::vector<NDArray*>& inputs, std::vector<double>& tArgs,
+                                    std::vector<int>& iArgs) {
+  VariableSpace varSpace;
+  int nodeId = 119;
+
+  // should be never used in practice, since in-graph NDArrayList should have id set
+  int cnt = -1;
+  std::vector<int> in;
+  if (list != nullptr) {
+    if (list->id().first == 0) list->id().first = -1;
+
+    auto listVar = new Variable(nullptr, nullptr, -119, 0);
+    listVar->setNDArrayList(list);
+    varSpace.putVariable(-1, listVar);
+    in.push_back(-1);
+    cnt--;
+  }
+
+  for (auto v : inputs) {
+    auto var = new Variable(v);
+    var->markRemovable(false);
+    in.push_back(cnt);
+    varSpace.putVariable(cnt--, var);
+  }
+
+  Context block(1, &varSpace, false);
+  block.fillInputs(in);
+
+  for (size_t e = 0; e < tArgs.size(); e++) block.getTArguments()->emplace_back(tArgs.at(e));
+
+  for (size_t e = 0; e < iArgs.size(); e++) block.getIArguments()->emplace_back(iArgs.at(e));
+
+  Status result = this->validateAndExecute(block);
+  ResultSet res;
+  res.setStatus(result);
+
+  for (int e = 0; e < DataTypeUtils::max<int>(); e++) {
+    std::pair<int, int> pair(1, e);
+    if (varSpace.hasVariable(pair)) {
+      auto var = varSpace.getVariable(pair);
+      if (var->hasNDArray()) {
+        auto arr = var->getNDArray();
+        if (arr->isAttached()) {
+          auto d = arr->detach();
+          res.push_back(d);
+        } else {
+          var->markRemovable(false);
+          res.push_back(arr);
+        }
+      }
+    } else
+      break;
+  }
+
+  return res;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/DeclarableReductionOp.cpp

@@ -0,0 +1,61 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.10.2017.
+//
+#include <array/DataTypeUtils.h>
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/DeclarableOp.h>
+#include <ops/declarable/DeclarableReductionOp.h>
+
+namespace sd {
+namespace ops {
+DeclarableReductionOp::DeclarableReductionOp(int numInputs, int numOutputs, const char* opName, bool allowsInplace,
+                                             int tArgs, int iArgs)
+    : DeclarableOp(numInputs, numOutputs, opName, allowsInplace, tArgs, iArgs) {
+  //
+}
+
+ShapeList* DeclarableReductionOp::calculateOutputShape(ShapeList* inputShape, Context& block) {
+  std::vector<LongType> dims;
+  if (inputShape->size() > 1) {
+    // the second argument is axis
+    auto axis = INPUT_VARIABLE(1);
+    for (int e = 0; e < axis->lengthOf(); e++) dims.push_back(axis->e<int>(e));
+  } else if (block.getIArguments()->size())
+    for (size_t e = 0; e < block.getIArguments()->size(); e++) dims.push_back(INT_ARG(e));
+  else if (block.getAxis()->size()) {
+    dims = *block.getAxis();
+  }
+
+  if (dims.size() > 1) std::sort(dims.begin(), dims.end());
+
+  // special case - output is scalar
+  if (dims.size() == 0 || (dims.size() == 1 && dims.at(0) == DataTypeUtils::max<int>())) {
+    auto newShape = ConstantShapeHelper::getInstance().scalarShapeInfo(block.dataType());
+    return SHAPELIST(newShape);
+  }
+
+  auto newShape = ShapeUtils::evalReduceShapeInfo('c', &dims, inputShape->at(0), false, false, block.getWorkspace());
+  return SHAPELIST(newShape);
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyBroadcastBoolOp.cpp

@@ -0,0 +1,110 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 17.10.2017.
+//
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/PointersManager.h>
+
+#include <ops/declarable/LegacyBroadcastBoolOp.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+Status LegacyBroadcastBoolOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  std::vector<LongType> dims(*block.getIArguments());
+  if (dims.size() > 0) std::sort(dims.begin(), dims.end());
+
+  NDArray::prepareSpecialUse({z}, {x, y});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  auto packX = ConstantTadHelper::getInstance().tadForDimensions(x->shapeInfo(), &dims);
+
+  PointersManager manager(block.launchContext(), "LegacyBroadcastBoolOp");
+  auto pTadShape = Environment::getInstance().isCPU()
+                   ? packX->primaryShapeInfo()
+                   : packX->specialShapeInfo();
+  auto pTadOffsets = Environment::getInstance().isCPU()
+                     ? packX->primaryOffsets()
+                     : packX->specialOffsets();
+
+  REQUIRE_TRUE(shape::length(packX->primaryShapeInfo()) == y->lengthOf(), 0,
+               "Length of broadcast TAD should be equal to length of Y operand, but got [%i] vs [%i]",
+               (int)shape::length(packX->primaryShapeInfo()), (int)y->lengthOf());
+
+  if (x == z)
+    NativeOpExecutioner::execBroadcast(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                       x->specialShapeInfo(), y->buffer(), y->shapeInfo(), y->specialBuffer(),
+                                       y->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+                                       z->specialShapeInfo(), dims.data(), dims.size(), pTadShape, pTadOffsets,
+                                       pTadShape, pTadOffsets);
+  else {
+    // this is rare, but possible use case - X and Z might have different shapes/strides/orders. In this case we prepare
+    // and pass separate TAD info
+
+    auto packZ = ConstantTadHelper::getInstance().tadForDimensions(z->shapeInfo(), &dims);
+
+    auto zTadShape = Environment::getInstance().isCPU()
+                     ? packZ->primaryShapeInfo()
+                     : packZ->specialShapeInfo();
+    auto zTadOffsets = Environment::getInstance().isCPU()
+                       ? packZ->primaryOffsets()
+                       : packZ->specialOffsets();  //(sd::LongType *) manager.replicatePointer(tadZ.tadOffsets,
+
+    NativeOpExecutioner::execBroadcast(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                       x->specialShapeInfo(), y->buffer(), y->shapeInfo(), y->specialBuffer(),
+                                       y->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+                                       z->specialShapeInfo(), dims.data(), dims.size(), pTadShape, pTadOffsets,
+                                       zTadShape, zTadOffsets);
+  }
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+LegacyBroadcastBoolOp::LegacyBroadcastBoolOp() : LegacyOp(2) {
+  //
+}
+
+LegacyBroadcastBoolOp::LegacyBroadcastBoolOp(int opNum) : LegacyOp(2, opNum) {
+  //
+}
+
+LegacyOp *LegacyBroadcastBoolOp::clone() { return new LegacyBroadcastBoolOp(this->_opNum); }
+
+/**
+ *   If external NDArray wasn't specified - the same shape is returned by all broadcast ops.
+ */
+ShapeList *LegacyBroadcastBoolOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().castToDataType(inShape, BOOL));
+  return ret;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyBroadcastOp.cpp

@@ -0,0 +1,122 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 17.10.2017.
+//
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyBroadcastOp.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+Status LegacyBroadcastOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x, y});
+
+  std::vector<LongType> dims(*block.getAxis());
+  if (dims.size() == 0 && block.width() > 2) {
+    auto axis = INPUT_VARIABLE(2);
+    helpers::adjustAxis(x->rankOf(), axis, dims);
+  }
+  if (dims.size() > 0) std::sort(dims.begin(), dims.end());
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  auto packX = ConstantTadHelper::getInstance().tadForDimensions(x->shapeInfo(), &dims);
+
+  auto tadLen = shape::length(packX->primaryShapeInfo());
+  REQUIRE_TRUE(tadLen == y->lengthOf(), 0,
+               "Length of broadcast TAD should be equal to length of Y operand, but got [%i] vs [%i]", tadLen,
+               (int)y->lengthOf());
+
+  PointersManager manager(block.launchContext(), "LegacyBroadcastOp");
+  auto pTadShape = Environment::getInstance().isCPU()
+                       ? packX->primaryShapeInfo()
+                       : packX->specialShapeInfo();
+  auto pTadOffsets = Environment::getInstance().isCPU()
+                         ? packX->primaryOffsets()
+                         : packX->specialOffsets();
+
+  if (x == z)
+    NativeOpExecutioner::execBroadcast(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                       x->specialShapeInfo(), y->buffer(), y->shapeInfo(), y->specialBuffer(),
+                                       y->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+                                       z->specialShapeInfo(), dims.data(), dims.size(), pTadShape, pTadOffsets,
+                                       pTadShape, pTadOffsets);
+  else {
+    // this is rare, but possible use case - X and Z might have different shapes/strides/orders. In this case we prepare
+    // and pass separate TAD info
+    auto packZ = ConstantTadHelper::getInstance().tadForDimensions(z->shapeInfo(), &dims);
+
+    auto zTadShape = Environment::getInstance().isCPU()
+                         ? packZ->primaryShapeInfo()
+                         : packZ->specialShapeInfo();
+    auto zTadOffsets = Environment::getInstance().isCPU()
+                           ? packZ->primaryOffsets()
+                           : packZ->specialOffsets();
+
+    NativeOpExecutioner::execBroadcast(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                       x->specialShapeInfo(), y->buffer(), y->shapeInfo(), y->specialBuffer(),
+                                       y->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+                                       z->specialShapeInfo(), dims.data(), dims.size(), pTadShape, pTadOffsets,
+                                       zTadShape, zTadOffsets);
+  }
+
+  manager.synchronize();
+
+  traceExecIfNeeded(block);
+
+  STORE_RESULT(*z);
+
+  return Status::OK;
+}
+
+LegacyBroadcastOp::LegacyBroadcastOp() : LegacyOp(2) {
+  //
+}
+
+LegacyBroadcastOp::LegacyBroadcastOp(int opNum) : LegacyOp(2, opNum) {
+  //
+}
+
+LegacyOp *LegacyBroadcastOp::clone() { return new LegacyBroadcastOp(this->_opNum); }
+
+/**
+ *   If external NDArray wasn't specified - the same shape is returned by all broadcast ops.
+ */
+ShapeList *LegacyBroadcastOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+
+  // FIXME: remove memcpy
+  LongType *newShape;
+  ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(inShape), sd::LongType);
+  memcpy(newShape, inShape, shape::shapeInfoByteLength(inShape));
+
+  return SHAPELIST(CONSTANT(newShape));
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyIndexReduceOp.cpp

@@ -0,0 +1,188 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyIndexReduceOp.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+LegacyIndexReduceOp::LegacyIndexReduceOp() : LegacyOp(1) {
+  //
+}
+
+LegacyIndexReduceOp::LegacyIndexReduceOp(int opNum) : LegacyOp(1, opNum) {
+  //
+}
+
+LegacyOp *LegacyIndexReduceOp::clone() { return new LegacyIndexReduceOp(this->_opNum); }
+
+ShapeList *LegacyIndexReduceOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+
+  if (block.getAxis()->size() == 0 && block.width() == 1) {
+    LongType *newShape;
+    // in this case we just return scalar
+    ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+    newShape[0] = 2;
+    newShape[1] = 1;
+    newShape[2] = 1;
+    newShape[3] = 1;
+    newShape[4] = 1;
+    newShape[6] = 1;
+    newShape[7] = 99;
+
+    auto result = ConstantShapeHelper::getInstance().bufferForShapeInfo(newShape);
+    RELEASE(newShape, block.getWorkspace());
+    return SHAPELIST(result->primary());
+  } else if (block.getAxis()->size()) {
+    // in this case we're building proper shape for reduction
+    auto array = INPUT_VARIABLE(0);
+
+    auto newShape =
+        ShapeUtils::evalReduceShapeInfo('c', block.getAxis(), *array, INT64, false, true, block.workspace());
+    return SHAPELIST(newShape);
+  } else {
+    bool allAxes = false;
+    auto indices = INPUT_VARIABLE(1);
+    LongType rank = shape::rank(inShape);
+    if (indices->lengthOf() == rank) allAxes = true;
+
+    std::vector<LongType> axis(indices->lengthOf());
+    for (int e = 0; e < indices->lengthOf(); e++) {
+      // lol otherwise we segfault on macOS
+      int f = indices->e<int>(e);
+      axis[e] = f >= 0 ? f : f += rank;
+    }
+    if (allAxes) {
+      LongType *newShape;
+      // in this case we just return scalar
+      ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+      newShape[0] = 2;
+      newShape[1] = 1;
+      newShape[2] = 1;
+      newShape[3] = 1;
+      newShape[4] = 1;
+      newShape[6] = 1;
+      newShape[7] = 99;
+
+      auto result = ConstantShapeHelper::getInstance().bufferForShapeInfo(newShape);
+
+      RELEASE(newShape, block.getWorkspace());
+      return SHAPELIST(result->primary());
+    } else {
+      // in this case we're building proper shape for reduction
+      auto array = INPUT_VARIABLE(0);
+      return SHAPELIST(
+          ShapeUtils::evalReduceShapeInfo('c', &axis, *array, DataType::INT64, false, true, block.workspace()));
+    }
+  }
+}
+
+/**
+ *   For all reductions rules are simple: either you return scalar, or you return reduced NDArray.
+ *   It solely depends on input shape, and requested dimensions
+ */
+Status LegacyIndexReduceOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x});
+
+  if (z->dataType() != INT64) {
+    THROW_EXCEPTION("IndexReduce operations require output to be INT64");
+  }
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  bool allAxes = false;
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyIndexReduceOp");
+
+  if (block.width() == 1) {
+    if (block.getAxis()->size() == 0) {
+      // scalar
+      NativeOpExecutioner::execIndexReduceScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      std::vector<LongType> dims(block.getAxis()->size());
+      for (size_t e = 0; e < dims.size(); e++) {
+        auto axe = block.getAxis()->at(e);
+        dims[e] = axe < 0 ? axe + x->rankOf() : axe;
+      }
+      if (dims.size() > 1) std::sort(dims.begin(), dims.end());
+
+      auto tadPack = ConstantTadHelper::getInstance().tadForDimensions(x->shapeInfo(), &dims);
+
+      NativeOpExecutioner::execIndexReduce(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), reinterpret_cast<LongType *>(z->buffer()), z->shapeInfo(),
+          z->specialBuffer(), z->specialShapeInfo(), nullptr, (int)dims.size(),
+          Environment::getInstance().isCPU() ? tadPack->primaryShapeInfo() : tadPack->specialShapeInfo(),
+          Environment::getInstance().isCPU() ? tadPack->primaryOffsets() : tadPack->specialOffsets());
+    }
+  } else {
+    // TF mode
+    auto indices = INPUT_VARIABLE(1);
+    if (indices->lengthOf() == x->rankOf()) allAxes = true;
+
+    std::vector<LongType> axis(indices->lengthOf());
+    for (LongType e = 0; e < indices->lengthOf(); e++) {
+      LongType f = indices->e<LongType>(e);
+      axis[e] = f >= 0 ? f : f += x->rankOf();
+    }
+
+    if (allAxes) {
+      NativeOpExecutioner::execIndexReduceScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+
+    } else {
+      if (indices->lengthOf() > 1) std::sort(axis.begin(), axis.end());
+
+      REQUIRE_TRUE(axis.size() > 0, 0, "Some dimensions required for reduction!");
+
+      auto tadPack = ConstantTadHelper::getInstance().tadForDimensions(x->shapeInfo(), &axis);
+
+      NativeOpExecutioner::execIndexReduce(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), reinterpret_cast<LongType *>(z->buffer()), z->shapeInfo(),
+          z->specialBuffer(), z->specialShapeInfo(), nullptr, (int)axis.size(),
+          Environment::getInstance().isCPU() ? tadPack->primaryShapeInfo() : tadPack->specialShapeInfo(),
+          Environment::getInstance().isCPU() ? tadPack->primaryOffsets() : tadPack->specialOffsets());
+    }
+  }
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+
+  return Status::OK;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyOp.cpp

@@ -0,0 +1,35 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+LegacyOp::LegacyOp(int numInputs) : DeclarableOp(numInputs, 1, "LegacyOp", false) {
+  _numInputs = numInputs;
+}
+
+LegacyOp::LegacyOp(int numInputs, int opNum) : DeclarableOp(numInputs, 1, "LegacyOp", false) {
+  _opNum = opNum;
+  _numInputs = numInputs;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyPairwiseTransformBoolOp.cpp

@@ -0,0 +1,76 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/LegacyPairwiseTransformBoolOp.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+namespace sd {
+namespace ops {
+LegacyPairwiseTransformBoolOp::LegacyPairwiseTransformBoolOp() : LegacyOp(2) {
+  // just a no-op
+}
+
+LegacyPairwiseTransformBoolOp::LegacyPairwiseTransformBoolOp(int opNum) : LegacyOp(2, opNum) {
+  // just a no-op
+}
+
+LegacyOp *LegacyPairwiseTransformBoolOp::clone() { return new LegacyPairwiseTransformBoolOp(this->_opNum); }
+
+Status LegacyPairwiseTransformBoolOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x, y});
+
+  if (!x->isSameShape(y))
+    REQUIRE_TRUE(x->isSameShape(y) || y->isScalar(), 0,
+                 "Node_%i: For Pairwise transforms shapes of both operands should be equal but got %s vs %s",
+                 block.getNodeId(), ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyPairwiseTransformBoolOp");
+
+  NativeOpExecutioner::execPairwiseTransform(
+      block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(), y->buffer(),
+      y->shapeInfo(), y->specialBuffer(), y->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+      z->specialShapeInfo(), extras.argumentsAsT(x->dataType()));
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+/**
+ *   Output shape of PWT operations always the same as input[0] shape, no exclusions.
+ */
+ShapeList *LegacyPairwiseTransformBoolOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().castToDataType(inShape, BOOL));
+  return ret;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyPairwiseTransformOp.cpp

@@ -0,0 +1,78 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/LegacyPairwiseTransformOp.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+namespace sd {
+namespace ops {
+LegacyPairwiseTransformOp::LegacyPairwiseTransformOp() : LegacyOp(2) {
+  this->getOpDescriptor()->allowInplace(true);
+}
+
+LegacyPairwiseTransformOp::LegacyPairwiseTransformOp(int opNum) : LegacyOp(2, opNum) {
+  this->getOpDescriptor()->allowInplace(true);
+}
+
+LegacyOp *LegacyPairwiseTransformOp::clone() { return new LegacyPairwiseTransformOp(this->_opNum); }
+
+Status LegacyPairwiseTransformOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x, y});
+
+  if (!x->isSameShape(y))
+    REQUIRE_TRUE(x->isSameShape(y) || y->isScalar(), 0,
+                 "Node_%i: For Pairwise transforms shapes of both operands should be equal but got %s vs %s",
+                 block.getNodeId(), ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyPairwiseTransformOp");
+
+  NativeOpExecutioner::execPairwiseTransform(
+      block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(), y->buffer(),
+      y->shapeInfo(), y->specialBuffer(), y->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+      z->specialShapeInfo(), extras.argumentsAsT(z->dataType()));
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+
+  return Status::OK;
+}
+
+/**
+ *   Output shape of PWT operations always the same as input[0] shape, no exclusions.
+ */
+ShapeList *LegacyPairwiseTransformOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+
+  LongType *newShape;
+  COPY_SHAPE(inShape, newShape);
+
+  return SHAPELIST(CONSTANT(newShape));
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyRandomOp.cpp

@@ -0,0 +1,405 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <array/NDArrayFactory.h>
+#include <helpers/RandomLauncher.h>
+#include <legacy/NativeOpExecutioner.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/LegacyRandomOp.h>
+#include <ops/declarable/OpRegistrator.h>
+
+namespace sd {
+namespace ops {
+LegacyRandomOp::LegacyRandomOp() : LegacyOp(1) {
+  // just a no-op
+}
+
+LegacyRandomOp::LegacyRandomOp(int opNum) : LegacyOp(1, opNum) {
+  // just a no-op
+}
+
+LegacyOp* LegacyRandomOp::clone() { return new LegacyRandomOp(this->_opNum); }
+
+template <typename T>
+Status LegacyRandomOp::validateAndExecute_(Context& block) {
+  auto input = INPUT_VARIABLE(0);
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  switch (opNum) {
+    case random::UniformDistribution: {
+      // uniform distribution
+      T from, to;
+      if (block.width() > 2) {
+        auto arg1 = INPUT_VARIABLE(1);
+        auto arg2 = INPUT_VARIABLE(2);
+        REQUIRE_TRUE(arg1->isScalar(), 0, "Uniform: Second argument must be scalar");
+        REQUIRE_TRUE(arg2->isScalar(), 0, "Uniform: Third argument must be scalar");
+
+        from = arg1->e<T>(0);
+        to = arg2->e<T>(0);
+      } else if (block.getTArguments()->size() == 2) {
+        from = T_ARG(0);
+        to = T_ARG(1);
+      } else {
+        REQUIRE_TRUE(false, 0, "Uniform requires either TArgs or 3 arguments to be present");
+      }
+
+      auto z = OUTPUT_VARIABLE(0);  // NDArrayFactory::create_<T>('c', shape, block.getWorkspace());
+
+      RandomLauncher::fillUniform(block.launchContext(), block.randomGenerator(), z, from, to);
+
+      // FIXME:
+      // OVERWRITE_RESULT(z);
+    } break;
+    case random::DropOut: {
+      auto z = OUTPUT_VARIABLE(0);
+
+      T prob;
+      if (block.width() > 1) {
+        auto arg = INPUT_VARIABLE(1);
+        REQUIRE_TRUE(arg->isScalar(), 0, "DropOut: Second argument must be scalar");
+
+        prob = arg->e<T>(0);
+      } else if (block.getTArguments()->size() > 0) {
+        prob = T_ARG(0);
+      } else {
+        REQUIRE_TRUE(false, 0, "DropOut requires either TArgs or second argument to be present");
+      }
+
+      if (!block.isInplace()) z->assign(input);
+
+      RandomLauncher::applyDropOut(block.launchContext(), block.randomGenerator(), z, prob);
+    } break;
+#if NOT_EXCLUDED(OP_dropout)
+    case random::DropOutInverted: {
+      auto z = OUTPUT_VARIABLE(0);
+      dropout op;
+      return op.execute(&block);
+    } break;
+#endif
+    case random::GaussianDistribution: {
+      // gaussian distribution
+      T mean, stdev;
+      if (block.width() > 2) {
+        auto arg1 = INPUT_VARIABLE(1);
+        auto arg2 = INPUT_VARIABLE(2);
+        REQUIRE_TRUE(arg1->isScalar(), 0, "Gaussian: Second argument must be scalar");
+        REQUIRE_TRUE(arg2->isScalar(), 0, "Gaussian: Third argument must be scalar");
+
+        mean = arg1->e<T>(0);
+        stdev = arg2->e<T>(0);
+      } else if (block.getTArguments()->size() == 2) {
+        mean = T_ARG(0);
+        stdev = T_ARG(1);
+      } else {
+        REQUIRE_TRUE(false, 0, "Gaussian requires either TArgs or 3 arguments to be present");
+      }
+
+      REQUIRE_TRUE(input->isVector(), 0, "Gaussian requires pure shape as first argument");
+
+      std::vector<LongType> shape(input->lengthOf());
+      for (int e = 0; e < input->lengthOf(); e++) shape[e] = input->e<LongType>(e);
+
+      auto z = OUTPUT_VARIABLE(0);
+
+      RandomLauncher::fillGaussian(block.launchContext(), block.randomGenerator(), z, mean, stdev);
+
+
+    } break;
+    case random::BernoulliDistribution: {
+      // bernoulli distribution
+      T prob;
+      if (block.width() > 1) {
+        auto arg1 = INPUT_VARIABLE(1);
+        REQUIRE_TRUE(arg1->isScalar(), 0, "Bernoulli: Second argument must be scalar");
+
+        prob = arg1->e<T>(0);
+      } else if (block.getTArguments()->size() > 0) {
+        prob = T_ARG(0);
+      } else {
+        REQUIRE_TRUE(false, 0, "Bernoulli requires either 1 TArg or 2 arguments to be present");
+      }
+
+      REQUIRE_TRUE(input->isVector(), 0, "Bernoulli requires pure shape as first argument");
+
+      std::vector<LongType> shape(input->lengthOf());
+      for (int e = 0; e < input->lengthOf(); e++) shape[e] = input->e<LongType>(e);
+
+      auto z = OUTPUT_VARIABLE(0);  // NDArrayFactory::create_<T>('c', shape, block.getWorkspace());
+
+      RandomLauncher::fillBernoulli(block.launchContext(), block.randomGenerator(), z, prob);
+
+
+    } break;
+    case random::BinomialDistributionEx: {
+      // BinomialEx distribution
+      T prob;
+      int trials;
+      if (block.width() > 2) {
+        auto arg1 = INPUT_VARIABLE(1);
+        auto arg2 = INPUT_VARIABLE(2);
+        REQUIRE_TRUE(arg1->isScalar(), 0, "Binomial: Second argument must be scalar");
+        REQUIRE_TRUE(arg2->isScalar(), 0, "Binomial: Third argument must be scalar");
+
+        trials = arg1->e<int>(0);
+        prob = arg2->e<T>(0);
+      } else if (block.getTArguments()->size() == 1 && block.getIArguments()->size() == 1) {
+        trials = INT_ARG(0);
+        prob = T_ARG(0);
+      } else {
+        REQUIRE_TRUE(false, 0, "Binomial requires either TArgs/IArgs or 3 arguments to be present");
+      }
+
+      REQUIRE_TRUE(input->isVector(), 0, "Binomial requires pure shape as first argument");
+
+      std::vector<LongType> shape(input->lengthOf());
+      for (int e = 0; e < input->lengthOf(); e++) shape[e] = input->e<LongType>(e);
+
+      auto z = OUTPUT_VARIABLE(0);  // NDArrayFactory::create_<T>('c', shape, block.getWorkspace());
+
+      RandomLauncher::fillBinomial(block.launchContext(), block.randomGenerator(), z, trials, prob);
+
+    } break;
+    case random::LogNormalDistribution: {
+      // lognorm distribution
+      T mean, stdev;
+      if (block.width() > 2) {
+        auto arg1 = INPUT_VARIABLE(1);
+        auto arg2 = INPUT_VARIABLE(2);
+        REQUIRE_TRUE(arg1->isScalar(), 0, "LogNormal: Second argument must be scalar");
+        REQUIRE_TRUE(arg2->isScalar(), 0, "LogNormal: Third argument must be scalar");
+
+        mean = arg1->e<T>(0);
+        stdev = arg2->e<T>(0);
+      } else if (block.getTArguments()->size() == 2) {
+        mean = T_ARG(0);
+        stdev = T_ARG(1);
+      } else {
+        REQUIRE_TRUE(false, 0, "LogNormal requires either TArgs or 3 arguments to be present");
+      }
+
+      REQUIRE_TRUE(input->isVector(), 0, "LogNormal requires pure shape as first argument");
+
+      std::vector<LongType> shape(input->lengthOf());
+      for (int e = 0; e < input->lengthOf(); e++) shape[e] = input->e<LongType>(e);
+
+      auto z = OUTPUT_VARIABLE(0);  // NDArrayFactory::create_<T>('c', shape, block.getWorkspace());
+
+      RandomLauncher::fillLogNormal(block.launchContext(), block.randomGenerator(), z, mean, stdev);
+
+      // FIXME: !!
+      // OVERWRITE_RESULT(z);
+    } break;
+    case random::TruncatedNormalDistribution: {
+      // truncated norm distribution
+      T mean, stdev;
+      if (block.width() > 2) {
+        auto arg1 = INPUT_VARIABLE(1);
+        auto arg2 = INPUT_VARIABLE(2);
+        REQUIRE_TRUE(arg1->isScalar(), 0, "TruncatedNormal: Second argument must be scalar");
+        REQUIRE_TRUE(arg2->isScalar(), 0, "TruncatedNormal: Third argument must be scalar");
+
+        mean = arg1->e<T>(0);
+        stdev = arg2->e<T>(0);
+      } else if (block.getTArguments()->size() == 2) {
+        mean = T_ARG(0);
+        stdev = T_ARG(1);
+      } else {
+        REQUIRE_TRUE(false, 0, "TruncatedNormal requires either TArgs or 3 arguments to be present");
+      }
+
+      REQUIRE_TRUE(input->isVector(), 0, "TruncatedNormal requires pure shape as first argument");
+
+      std::vector<LongType> shape(input->lengthOf());
+      for (int e = 0; e < input->lengthOf(); e++) shape[e] = input->e<LongType>(e);
+
+      auto z = OUTPUT_VARIABLE(0);
+
+      RandomLauncher::fillTruncatedNormal(block.launchContext(), block.randomGenerator(), z, mean, stdev);
+    } break;
+    case random::AlphaDropOut: {
+      auto z = OUTPUT_VARIABLE(0);
+
+      T prob, a, b, pa;
+      if (block.width() > 4) {
+        auto arg1 = INPUT_VARIABLE(1);
+        auto arg2 = INPUT_VARIABLE(2);
+        auto arg3 = INPUT_VARIABLE(3);
+        auto arg4 = INPUT_VARIABLE(4);
+        REQUIRE_TRUE(arg1->isScalar(), 0, "AlphaDropOut: Second argument must be scalar");
+        REQUIRE_TRUE(arg2->isScalar(), 0, "AlphaDropOut: Third argument must be scalar");
+        REQUIRE_TRUE(arg3->isScalar(), 0, "AlphaDropOut: Fourth argument must be scalar");
+        REQUIRE_TRUE(arg4->isScalar(), 0, "AlphaDropOut: Fifth argument must be scalar");
+
+        prob = arg1->e<T>(0);
+        a = arg2->e<T>(0);
+        b = arg3->e<T>(0);
+        pa = arg4->e<T>(0);
+      } else if (block.getTArguments()->size() == 4) {
+        prob = T_ARG(0);
+        a = T_ARG(1);
+        b = T_ARG(2);
+        pa = T_ARG(3);
+      } else {
+        REQUIRE_TRUE(false, 0, "AlphaDropOut requires either TArgs or 5 arguments to be present");
+      }
+
+      if (!block.isInplace()) z->assign(input);
+
+      RandomLauncher::applyAlphaDropOut(block.launchContext(), block.randomGenerator(), z, prob, a, b, pa);
+    } break;
+    case random::Linspace: {
+      auto z = OUTPUT_VARIABLE(0);
+      auto start = INPUT_VARIABLE(0);
+      auto finish = INPUT_VARIABLE(1);
+      auto numOfElements = INPUT_VARIABLE(2);
+
+      z->linspace(start->e<double>(0),
+                  (finish->e<double>(0) - start->e<double>(0)) / (numOfElements->e<LongType>(0) - 1.));
+    } break;
+    default: {
+      sd_printf("Unknown random op requested: [%i]\n", opNum);
+      return Status::KERNEL_FAILURE;
+    }
+  }
+
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+Status LegacyRandomOp::validateAndExecute(Context& block) {
+  auto z = OUTPUT_VARIABLE(0);
+  BUILD_SINGLE_SELECTOR(z->dataType(), return validateAndExecute_, (block), SD_FLOAT_TYPES);
+  return sd::Status::KERNEL_FAILURE;
+}
+
+/**
+ * For transform operations, output shape always equals to input shape. With just a few exclusions, like im2col and
+ * col2im. But these ops already have CustomOp implementations.
+ *
+ */
+ShapeList* LegacyRandomOp::calculateOutputShape(ShapeList* inputShape, Context& block) {
+  auto inShape = inputShape->at(0);
+  auto xType = ArrayOptions::dataType(inShape);
+  if (DataTypeUtils::isR(xType)) {
+    return SHAPELIST(CONSTANT(inShape));
+  } else if (DataTypeUtils::isZ(xType)) {
+    auto zShapeArr = INPUT_VARIABLE(0);
+    auto zShapeVector = zShapeArr->asVectorT<LongType>();
+    auto dtype = block.dataType();
+
+    return SHAPELIST(ConstantShapeHelper::getInstance().createShapeInfo(dtype, 'c', zShapeVector));
+  } else
+    THROW_EXCEPTION("LegacyRandomOp: Unknown input data type!");
+
+  return nullptr;
+}
+
+Status LegacyRandomOp::execute(Context* block) { return DeclarableOp::execute(block); }
+
+ResultSet LegacyRandomOp::execute(RandomGenerator& rng, std::initializer_list<NDArray*> inputs,
+                                  std::initializer_list<double> tArgs, std::initializer_list<int> iArgs,
+                                  bool isInplace) {
+  std::vector<NDArray*> ins(inputs);
+  std::vector<double> tas(tArgs);
+  std::vector<int> ias(iArgs);
+  return this->execute(rng, ins, tas, ias, isInplace);
+}
+
+ResultSet LegacyRandomOp::execute(RandomGenerator& rng, std::vector<NDArray*>& inputs, std::vector<double>& tArgs,
+                                  std::vector<int>& iArgs, bool isInplace) {
+  VariableSpace variableSpace;
+  ResultSet arrayList;
+  // ResultSet arrayList;
+
+  if (isInplace) arrayList.setNonRemovable();
+
+  int cnt = -1;
+  std::vector<int> in;
+  for (auto v : inputs) {
+    if (v == nullptr) continue;
+
+    auto var = new Variable(v);
+    var->markRemovable(false);
+    in.push_back(cnt);
+    variableSpace.putVariable(cnt--, var);
+  }
+
+  Context block(1, &variableSpace, false);
+  // FIX ME: implement setRng method
+  block.setRng(rng);
+  block.fillInputs(in);
+  block.markInplace(isInplace);
+
+  for (size_t e = 0; e < tArgs.size(); e++) block.getTArguments()->emplace_back(tArgs.at(e));
+
+  for (size_t e = 0; e < iArgs.size(); e++) block.getIArguments()->emplace_back(iArgs.at(e));
+
+  Status status = this->execute(&block);
+  arrayList.setStatus(status);
+  if (status != Status::OK) return arrayList;
+
+  for (int e = 0; e < DataTypeUtils::max<int>(); e++) {
+    std::pair<int, int> pair(1, e);
+    if (variableSpace.hasVariable(pair)) {
+      auto var = variableSpace.getVariable(pair);
+      auto arr = var->getNDArray();
+      if (!arr->isAttached()) {
+        var->markRemovable(false);
+        arrayList.push_back(arr);
+      } else {
+        arrayList.push_back(arr->detach());
+      }
+    } else
+      break;
+  }
+
+  return arrayList;
+}
+
+Status LegacyRandomOp::validateDataTypes(Context& block) {
+  if (block.isFastPath()) {
+    // in this case we'll roll through pre-defined outputs
+    auto fpo = block.fastpath_out();
+    for (auto v : fpo) {
+      if (v != nullptr) {
+        if (!v->isR()) return Status::BAD_ARGUMENTS;
+      }
+    }
+  } else {
+    std::pair<int, int> pair(block.nodeId(), 0);
+    if (block.getVariableSpace()->hasVariable(pair)) {
+      auto var = block.variable(pair);
+      if (!var->hasNDArray()) return Status::BAD_ARGUMENTS;
+
+      auto arr = var->getNDArray();
+      if (!arr->isR()) return Status::BAD_ARGUMENTS;
+    }
+  }
+
+  return Status::OK;
+}
+
+BUILD_SINGLE_TEMPLATE( sd::Status LegacyRandomOp::validateAndExecute_, (Context&), SD_FLOAT_TYPES);
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyReduce3Op.cpp

@@ -0,0 +1,131 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 17.10.2017.
+//
+#include <array/DataTypeUtils.h>
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyReduce3Op.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+namespace sd {
+namespace ops {
+Status LegacyReduce3Op::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x, y});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  sd_debug("Executing LegacyReduce3Op: [%i]\n", opNum);
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyReduce3Op");
+
+  if (x->isSameShape(y) && (block.getIArguments()->size() == 0 ||
+                            (block.getIArguments()->size() == 1 && INT_ARG(0) == DataTypeUtils::max<int>()))) {
+    // reduce3 to scalar
+    NativeOpExecutioner::execReduce3Scalar(
+        block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+        extras.argumentsAsT(z->dataType()), y->buffer(), y->shapeInfo(), y->specialBuffer(), y->specialShapeInfo(),
+        z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+  } else {
+    std::vector<LongType> dims(*block.getAxis());
+    for (size_t e = 0; e < dims.size(); e++)
+      if (dims[e] < 0) dims[e] += x->rankOf();
+
+    auto packX = ConstantTadHelper::getInstance().tadForDimensions(x->shapeInfo(), &dims);
+    auto packZ = ConstantTadHelper::getInstance().tadForDimensions(z->shapeInfo(), &dims);
+
+    REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions requuired for reduction!");
+
+    auto xTadShape = Environment::getInstance().isCPU()
+                         ? packX->primaryShapeInfo()
+                         : packX->specialShapeInfo();
+    auto xTadOffsets = Environment::getInstance().isCPU()
+                           ? packX->primaryOffsets()
+                           : packX->specialOffsets();
+
+    auto yTadShape = Environment::getInstance().isCPU()
+                         ? packZ->primaryShapeInfo()
+                         : packZ->specialOffsets();
+    auto yTadOffsets = Environment::getInstance().isCPU()
+                           ? packZ->primaryOffsets()
+                           : packZ->specialOffsets();
+
+    NativeOpExecutioner::execReduce3(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                     x->specialShapeInfo(), extras.argumentsAsT(z->dataType()), y->buffer(),
+                                     y->shapeInfo(), y->specialBuffer(), y->specialShapeInfo(), z->buffer(),
+                                     z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo(), dims.data(),
+                                     dims.size(), xTadShape, xTadOffsets, yTadShape, yTadOffsets);
+  }
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+  return Status::OK;
+}
+
+
+LegacyReduce3Op::LegacyReduce3Op() : LegacyOp(2) {
+  //
+}
+
+LegacyReduce3Op::LegacyReduce3Op(int opNum) : LegacyOp(2, opNum) {
+  //
+}
+
+LegacyOp *LegacyReduce3Op::clone() { return new LegacyReduce3Op(this->_opNum); }
+
+/**
+ *   For all reductions rules are simple: either you return scalar, or you return reduced NDArray.
+ *   It solely depends on input shape, and requested dimensions
+ */
+ShapeList *LegacyReduce3Op::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto xShape = inputShape->at(0);
+  auto yShape = inputShape->at(1);
+
+  LongType *zShape = nullptr;
+
+  if (shape::equalsSoft(xShape, yShape) &&
+      (block.getIArguments()->size() == 0 ||
+       (block.getIArguments()->size() == 1 && INT_ARG(0) == DataTypeUtils::max<int>()))) {
+    // reduce3 to scalar case
+    ALLOCATE(zShape, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+    zShape[0] = 2;
+    zShape[1] = 1;
+    zShape[2] = 1;
+    zShape[3] = 1;
+    zShape[4] = 1;
+    zShape[5] = 0;
+    zShape[6] = 1;
+    zShape[7] = 99;
+  } else {
+    sd::LongType *xShape2 = ShapeUtils::evalReduceShapeInfo('c', block.getIArguments(), xShape, false, true);
+    return SHAPELIST(xShape2);
+  }
+
+  return SHAPELIST(zShape);
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyReduceBoolOp.cpp

@@ -0,0 +1,164 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <array/DataTypeUtils.h>
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyReduceBoolOp.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+namespace sd {
+namespace ops {
+LegacyReduceBoolOp::LegacyReduceBoolOp() : LegacyOp(1) {
+  //
+}
+
+LegacyReduceBoolOp::LegacyReduceBoolOp(int opNum) : LegacyOp(1, opNum) {
+}
+
+LegacyOp* LegacyReduceBoolOp::clone() { return new LegacyReduceBoolOp(this->_opNum); }
+
+Status LegacyReduceBoolOp::validateAndExecute(Context& block) {
+  auto x = INPUT_VARIABLE(0);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+  sd_debug("Executing LegacyReduceFloatOp: [%i]\n", opNum);
+
+  auto axis = *block.getAxis();
+
+  bool allAxes = false;
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyReduceBoolOp");
+
+  if (block.width() == 1) {
+    if (axis.size() == static_cast<size_t>(x->rankOf())) allAxes = true;
+
+    if ((axis.empty()) || (axis.size() == 1 && axis[0] == DataTypeUtils::max<int>()) || allAxes) {
+      // scalar
+      NativeOpExecutioner::execReduceBoolScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      std::vector<LongType> dims = {axis};
+
+      for (size_t e = 0; e < dims.size(); e++)
+        if (dims[e] < 0) dims[e] += x->rankOf();
+
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+      const LongType* zShapeInfoH = z->shapeInfo();
+      const LongType* zShapeInfoD = z->specialShapeInfo();
+
+      if (x->rankOf() - dims.size() != static_cast<size_t>(z->rankOf())) {
+        auto zPack = ConstantShapeHelper::getInstance().createShapeInfoWithNoUnitiesForReduce(
+            z->shapeInfo(), &dims, z->getContext()->getWorkspace());
+        zShapeInfoH = reinterpret_cast<LongType const*>(zPack->primary());
+        zShapeInfoD = reinterpret_cast<LongType const*>(zPack->special());
+      }
+
+      std::vector<LongType> *dims2 = ShapeUtils::evalDimsForReduceOp(x->rankOf(), &dims);
+      NativeOpExecutioner::execReduceBool(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                          x->specialShapeInfo(), nullptr, z->buffer(), zShapeInfoH, z->specialBuffer(),
+                                          zShapeInfoD, dims2->data(), dims2->size());
+      delete dims2;
+
+    }
+
+    STORE_RESULT(*z);
+  } else {
+    auto indices = INPUT_VARIABLE(1);
+    if (indices->lengthOf() == x->rankOf()) allAxes = true;
+
+
+    std::vector<LongType> dims(indices->lengthOf());
+    for (LongType e = 0; e < indices->lengthOf(); e++) {
+      //segfault on macOS
+      int f = indices->e<int>(e);
+      dims[e] = f >= 0 ? f : f += x->rankOf();
+    }
+
+    if ((block.getIArguments()->size() == 1 && INT_ARG(0) == DataTypeUtils::max<int>()) || allAxes) {
+      // scalar
+      NativeOpExecutioner::execReduceBoolScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      if (indices->lengthOf() > 1) std::sort(dims.begin(), dims.end());
+
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+      const LongType* zShapeInfoH = z->shapeInfo();
+      const LongType* zShapeInfoD = z->specialShapeInfo();
+
+      if (x->rankOf() - dims.size() != static_cast<size_t>(z->rankOf())) {
+        auto zPack = ConstantShapeHelper::getInstance().createShapeInfoWithNoUnitiesForReduce(
+            z->shapeInfo(), &dims, z->getContext()->getWorkspace());
+        zShapeInfoH = reinterpret_cast<LongType const*>(zPack->primary());
+        zShapeInfoD = reinterpret_cast<LongType const*>(zPack->special());
+      }
+
+      std::vector<LongType> *dims2 = ShapeUtils::evalDimsForReduceOp(x->rankOf(), &dims);
+      NativeOpExecutioner::execReduceBool(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                          x->specialShapeInfo(), nullptr, z->buffer(), zShapeInfoH, z->specialBuffer(),
+                                          zShapeInfoD, dims2->data(), dims2->size());
+
+      delete dims2;
+
+    }
+  }
+
+  manager.synchronize();
+  traceExecIfNeeded(block);
+
+
+  return Status::OK;
+}
+
+/**
+ *   For all reductions rules are simple: either you return scalar, or you return reduced NDArray.
+ *   It solely depends on input shape, and requested dimensions
+ */
+ShapeList* LegacyReduceBoolOp::calculateOutputShape(ShapeList* inputShape, Context& block) {
+  auto inShape = inputShape->at(0);
+
+
+
+  auto keepDims = block.numB() > 0 ? B_ARG(0) : false;
+  auto newFormat = block.numB() > 1 ? B_ARG(1) : true;
+
+  auto axis = block.width() > 1 ? INPUT_VARIABLE(1)->asVectorT<LongType>() : *block.getAxis();
+
+
+  // in this case we're building proper shape for reduction
+  auto info = ShapeUtils::evalReduceShapeInfo(shape::order(inShape), &axis, inShape, BOOL, keepDims,
+                                              !newFormat, block.workspace());
+  return SHAPELIST(info);
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyReduceFloatOp.cpp

@@ -0,0 +1,167 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <array/DataTypeUtils.h>
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyReduceFloatOp.h>
+
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+LegacyReduceFloatOp::LegacyReduceFloatOp() : LegacyOp(1) {
+  //
+}
+
+LegacyReduceFloatOp::LegacyReduceFloatOp(int opNum) : LegacyOp(1, opNum) {
+}
+
+LegacyOp* LegacyReduceFloatOp::clone() { return new LegacyReduceFloatOp(this->_opNum); }
+
+Status LegacyReduceFloatOp::validateAndExecute(Context& block) {
+  auto x = INPUT_VARIABLE(0);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+  sd_debug("Executing LegacyReduceFloatOp: [%i]\n", opNum);
+
+  bool allAxes = false;
+  auto axis = *block.getAxis();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyReduceFloatOp");
+
+  if (block.width() == 1) {
+    if (axis.size() == static_cast<size_t>(x->rankOf())) allAxes = true;
+
+    if (block.getAxis()->empty() || allAxes) {
+      // scalar
+      NativeOpExecutioner::execReduceFloatScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(z->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      std::vector<LongType> dims(*block.getAxis());
+
+      for (size_t e = 0; e < dims.size(); e++)
+        if (dims[e] < 0) dims[e] += x->rankOf();
+
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+
+      const LongType* zShapeInfoH = z->shapeInfo();
+      const LongType* zShapeInfoD = z->specialShapeInfo();
+
+      if (x->rankOf() == z->rankOf()) {
+        auto zPack = ConstantShapeHelper::getInstance().createShapeInfoWithNoUnitiesForReduce(
+            z->shapeInfo(), &dims, z->getContext()->getWorkspace());
+        zShapeInfoH = reinterpret_cast<LongType const*>(zPack->primary());
+        zShapeInfoD = reinterpret_cast<LongType const*>(zPack->special());
+      }
+
+      std::vector<LongType> *dims2 = ShapeUtils::evalDimsForReduceOp(x->rankOf(), &dims);
+
+      NativeOpExecutioner::execReduceFloat(block.launchContext(), opNum, x->buffer(), x->shapeInfo(),
+                                           x->specialBuffer(), x->specialShapeInfo(),
+                                           extras.argumentsAsT(z->dataType()), z->buffer(), zShapeInfoH,
+                                           z->specialBuffer(), zShapeInfoD, dims2->data(), dims2->size());
+
+      delete dims2;
+    }
+
+    STORE_RESULT(*z);
+  } else {
+    auto indices = INPUT_VARIABLE(1);
+    if (indices->lengthOf() == x->rankOf()) allAxes = true;
+
+
+    std::vector<LongType> dims(indices->lengthOf());
+    for (int e = 0; e < indices->lengthOf(); e++) {
+      // segfault on macOS if not like this
+      int f = indices->e<int>(e);
+      dims[e] = f >= 0 ? f : f += x->rankOf();
+    }
+
+    if ((block.getIArguments()->size() == 1 && INT_ARG(0) == DataTypeUtils::max<int>()) || allAxes) {
+      // scalar
+      NativeOpExecutioner::execReduceFloatScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+
+      const LongType* zShapeInfoH = z->shapeInfo();
+      const LongType* zShapeInfoD = z->specialShapeInfo();
+
+      if (x->rankOf() == z->rankOf()) {
+        auto zPack = ConstantShapeHelper::getInstance().createShapeInfoWithNoUnitiesForReduce(
+            z->shapeInfo(), &dims, z->getContext()->getWorkspace());
+        zShapeInfoH = reinterpret_cast<LongType const*>(zPack->primary());
+        zShapeInfoD = reinterpret_cast<LongType const*>(zPack->special());
+      }
+
+      std::vector<LongType> *dims2 = ShapeUtils::evalDimsForReduceOp(x->rankOf(), &dims);
+
+      NativeOpExecutioner::execReduceFloat(block.launchContext(), opNum, x->buffer(), x->shapeInfo(),
+                                           x->specialBuffer(), x->specialShapeInfo(),
+                                           extras.argumentsAsT(z->dataType()), z->buffer(), zShapeInfoH,
+                                           z->specialBuffer(), zShapeInfoD, dims2->data(), dims2->size());
+      delete dims2;
+    }
+  }
+
+  manager.synchronize();
+
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+/**
+ *   For all reductions rules are simple: either you return scalar, or you return reduced NDArray.
+ *   It solely depends on input shape, and requested dimensions
+ */
+ShapeList* LegacyReduceFloatOp::calculateOutputShape(ShapeList* inputShape, Context& block) {
+  auto inShape = inputShape->at(0);
+
+
+  auto keepDims = block.numB() > 0 ? B_ARG(0) : false;
+  auto newFormat = block.numB() > 1 ? B_ARG(1) : true;
+
+  auto axis = block.width() > 1 ? INPUT_VARIABLE(1)->asVectorT<LongType>() : *block.getAxis();
+
+
+  // in this case we're building proper shape for reduction
+  auto newShape =
+      ShapeUtils::evalReduceShapeInfo(shape::order(inShape), &axis, inShape, keepDims, !newFormat, block.workspace());
+
+  return SHAPELIST(newShape);
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyReduceLongOp.cpp

@@ -0,0 +1,167 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <array/DataTypeUtils.h>
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyReduceLongOp.h>
+
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+LegacyReduceLongOp::LegacyReduceLongOp() : LegacyOp(1) {
+  //
+}
+
+LegacyReduceLongOp::LegacyReduceLongOp(int opNum) : LegacyOp(1, opNum) {
+}
+
+LegacyOp* LegacyReduceLongOp::clone() { return new LegacyReduceLongOp(this->_opNum); }
+
+Status LegacyReduceLongOp::validateAndExecute(Context& block) {
+  auto x = INPUT_VARIABLE(0);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+  sd_debug("Executing LegacyReduceFloatOp: [%i]\n", opNum);
+
+  auto axis = *block.getAxis();
+  bool allAxes = false;
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyReduceLongOp");
+
+  if (block.width() == 1) {
+    if (axis.size() == static_cast<size_t>(x->rankOf())) allAxes = true;
+
+    if ((axis.empty()) || (axis.size() == 1 && axis[0] == DataTypeUtils::max<int>()) || allAxes) {
+      // scalar
+      NativeOpExecutioner::execReduceLongScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      std::vector<LongType> dims(axis);
+
+      for (size_t e = 0; e < dims.size(); e++)
+        if (dims[e] < 0) dims[e] += x->rankOf();
+
+      if (dims.size() > 1) std::sort(dims.begin(), dims.end());
+
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+      const LongType* zShapeInfoH = z->shapeInfo();
+      const LongType* zShapeInfoD = z->specialShapeInfo();
+
+      if (x->rankOf() - dims.size() != static_cast<size_t>(z->rankOf())) {
+        auto zPack = ConstantShapeHelper::getInstance().createShapeInfoWithNoUnitiesForReduce(
+            z->shapeInfo(), &dims, z->getContext()->getWorkspace());
+        zShapeInfoH = reinterpret_cast<LongType const*>(zPack->primary());
+        zShapeInfoD = reinterpret_cast<LongType const*>(zPack->special());
+      }
+
+      std::vector<LongType> *dims2 = ShapeUtils::evalDimsForReduceOp(x->rankOf(), &dims);
+      NativeOpExecutioner::execReduceLong(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                          x->specialShapeInfo(), nullptr, z->buffer(), zShapeInfoH, z->specialBuffer(),
+                                          zShapeInfoD, dims2->data(), dims2->size());
+
+      delete dims2;
+
+    }
+
+    STORE_RESULT(*z);
+  } else {
+    auto indices = INPUT_VARIABLE(1);
+    if (indices->lengthOf() == x->rankOf()) allAxes = true;
+
+
+    std::vector<LongType> dims(indices->lengthOf());
+    for (int e = 0; e < indices->lengthOf(); e++) {
+      // segfault on macOS if not like this
+      int f = indices->e<int>(e);
+      dims[e] = f >= 0 ? f : f += x->rankOf();
+    }
+
+    if ((block.getIArguments()->size() == 1 && INT_ARG(0) == DataTypeUtils::max<int>()) || allAxes) {
+      // scalar
+      NativeOpExecutioner::execReduceLongScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(x->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+      const LongType* zShapeInfoH = z->shapeInfo();
+      const LongType* zShapeInfoD = z->specialShapeInfo();
+
+      if (x->rankOf() - dims.size() != static_cast<size_t>(z->rankOf())) {
+        auto zPack = ConstantShapeHelper::getInstance().createShapeInfoWithNoUnitiesForReduce(
+            z->shapeInfo(), &dims, z->getContext()->getWorkspace());
+        zShapeInfoH = reinterpret_cast<LongType const*>(zPack->primary());
+        zShapeInfoD = reinterpret_cast<LongType const*>(zPack->special());
+      }
+
+      std::vector<LongType> *dims2 = ShapeUtils::evalDimsForReduceOp(x->rankOf(), &dims);
+      NativeOpExecutioner::execReduceLong(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                          x->specialShapeInfo(), nullptr, z->buffer(), zShapeInfoH, z->specialBuffer(),
+                                          zShapeInfoD, dims2->data(), dims2->size());
+      delete  dims2;
+
+    }
+  }
+
+  manager.synchronize();
+
+
+  traceExecIfNeeded(block);
+
+
+  return Status::OK;
+}
+
+/**
+ *   For all reductions rules are simple: either you return scalar, or you return reduced NDArray.
+ *   It solely depends on input shape, and requested dimensions
+ */
+ShapeList* LegacyReduceLongOp::calculateOutputShape(ShapeList* inputShape, Context& block) {
+  auto inShape = inputShape->at(0);
+
+  LongType* newShape;
+
+
+  auto keepDims = block.numB() > 0 ? B_ARG(0) : false;
+  auto newFormat = block.numB() > 1 ? B_ARG(1) : true;
+
+  auto axis = block.width() > 1 ? INPUT_VARIABLE(1)->asVectorT<LongType>() : *block.getAxis();
+
+
+  // in this case we're building proper shape for reduction
+  return SHAPELIST(ShapeUtils::evalReduceShapeInfo(shape::order(inShape), &axis, inShape, DataType::INT64, keepDims,
+                                                   !newFormat, block.workspace()));
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyReduceOp.cpp

@@ -0,0 +1,182 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyReduceOp.h>
+#include <ops/declarable/OpRegistrator.h>
+
+#ifdef LEGACY_REDUCE_SAME_ONLY
+namespace sd {
+namespace ops {
+LegacyReduceOp::LegacyReduceOp() : LegacyOp::LegacyOp(1) {
+  //
+}
+
+LegacyReduceOp::LegacyReduceOp(int opType) : LegacyOp::LegacyOp(1, opType) {
+}
+
+LegacyOp *LegacyReduceOp::clone() { return new LegacyReduceOp(this->_opNum); }
+
+sd::Status LegacyReduceOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+
+  int opType = block.opType() < 0 ? this->_opNum : block.opType();
+  sd_debug("Executing LegacyReduceOp: [%i]\n", opType);
+
+  bool allAxes = false;
+
+  if (block.width() == 1) {
+    auto z = OUTPUT_VARIABLE(0);
+
+    if (block.getIArguments()->size() == x->rankOf()) allAxes = true;
+
+    if ((block.getIArguments()->size() == 0) || (block.getIArguments()->size() == 1 && INT_ARG(0) == SD_MAX_INT) ||
+        allAxes) {
+      // scalar
+      NativeOpExcutioner::execReduceFloatScalar(opType, x->buffer(), x->shapeInfo(), block.getTArguments()->data(),
+                                                z->buffer(), z->shapeInfo());
+    } else {
+      // TAD
+      std::vector<int> dims(*block.getIArguments());
+
+      for (int e = 0; e < dims.size(); e++)
+        if (dims[e] < 0) dims[e] += x->rankOf();
+
+      std::sort(dims.begin(), dims.end());
+
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+      shape::TAD tad(x->shapeInfo(), dims.data(), dims.size());
+      tad.createTadOnlyShapeInfo();
+      tad.createOffsets();
+
+      NativeOpExcutioner::execReduceFloat(opType, x->buffer(), x->shapeInfo(), block.getTArguments()->data(),
+                                          z->buffer(), z->shapeInfo(), dims.data(), (int)dims.size(),
+                                          tad.tadOnlyShapeInfo, tad.tadOffsets);
+    }
+
+    STORE_RESULT(*z);
+  } else {
+    auto indices = INPUT_VARIABLE(1);
+    if (indices->lengthOf() == x->rankOf()) allAxes = true;
+
+    std::vector<int> axis(indices->lengthOf());
+    for (int e = 0; e < indices->lengthOf(); e++) {
+      // lol otherwise we segfault on macOS
+      int f = indices->e<int>(e);
+      axis[e] = f >= 0 ? f : f += x->rankOf();
+    }
+
+    if ((block.getIArguments()->size() == 1 && INT_ARG(0) == SD_MAX_INT) || allAxes) {
+      auto z = OUTPUT_VARIABLE(0);
+
+      auto b = x->buffer();
+      auto s = x->shapeInfo();
+      auto e = block.numT() > 0 ? block.getTArguments()->data() : nullptr;
+
+
+      // scalar
+      NativeOpExcutioner::execReduceFloatScalar(opType, b, s, e, z->buffer(), z->shapeInfo());
+    } else {
+      // TAD
+      if (indices->lengthOf() > 1) std::sort(axis.begin(), axis.end());
+
+      REQUIRE_TRUE(axis.size() > 0, 0, "Some dimensions required for reduction!");
+
+      shape::TAD tad(x->shapeInfo(), axis.data(), axis.size());
+      tad.createTadOnlyShapeInfo();
+      tad.createOffsets();
+
+      auto newShape = ShapeUtils::evalReduceShapeInfo(x->ordering(), axis, *x);
+      auto z = new NDArray(newShape, x->getWorkspace());
+
+      NativeOpExcutioner::execReduceFloat(opType, x->buffer(), x->shapeInfo(), block.getTArguments()->data(),
+                                          z->buffer(), z->shapeInfo(), axis.data(), (int)axis.size(),
+                                          tad.tadOnlyShapeInfo, tad.tadOffsets);
+
+      // keepDims processing, for TF compatibility
+      if (block.getIArguments()->size() > 0 && block.getIArguments()->at(0) == 1) {
+        std::vector<sd::LongType> newshape(z->getShapeAsVector());
+        for (int e = 0; e < axis.size(); e++) {
+          auto a = axis.at(e);
+          newshape.insert(newshape.begin() + a, 1);
+        }
+        z->reshapei(z->ordering(), newshape);
+      }
+
+      OVERWRITE_RESULT(z);
+    }
+  }
+
+  traceExecIfNeeded(block);
+
+  return sd::Status::OK;
+}
+
+/**
+ *   For all reductions rules are simple: either you return scalar, or you return reduced NDArray.
+ *   It solely depends on input shape, and requested dimensions
+ */
+ShapeList *LegacyReduceOp::calculateOutputShape(ShapeList *inputShape, sd::graph::Context &block) {
+  auto inShape = inputShape->at(0);
+
+  sd::LongType *newShape;
+
+  bool allAxes = false;
+
+  if (block.getIArguments()->size() == shape::rank(inShape)) allAxes = true;
+
+  if (block.getIArguments()->size() == 0 || (block.getIArguments()->size() == 1 && INT_ARG(0) == SD_MAX_INT) ||
+      allAxes) {
+    if (block.getIArguments()->size() > 0 && block.getIArguments()->at(0) == 1) {
+      // in this case we just return legacy scalar
+      ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+      newShape[0] = 2;
+      newShape[1] = 1;
+      newShape[2] = 1;
+      newShape[3] = 1;
+      newShape[4] = 1;
+      newShape[5] = 0;
+      newShape[6] = 1;
+      newShape[7] = 99;
+    } else {
+      ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(0), sd::LongType);
+      newShape[0] = 0;
+      newShape[1] = 0;
+      newShape[2] = 1;
+      newShape[3] = 99;
+    }
+  } else {
+    // in this case we're building proper shape for reduction
+    auto array = new NDArray(nullptr, inShape, block.getWorkspace());
+
+    newShape = ShapeUtils::evalReduceShapeInfo(shape::order(inShape), *block.getIArguments(), *array, false, false,
+                                               block.workspace());
+
+    delete array;
+  }
+
+  return SHAPELIST(newShape);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/impl/LegacyReduceSameOp.cpp

@@ -0,0 +1,172 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <array/DataTypeUtils.h>
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyReduceSameOp.h>
+
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+LegacyReduceSameOp::LegacyReduceSameOp() : LegacyOp(1) {
+  //
+}
+
+LegacyReduceSameOp::LegacyReduceSameOp(int opNum) : LegacyOp(1, opNum) {
+}
+
+LegacyOp* LegacyReduceSameOp::clone() { return new LegacyReduceSameOp(this->_opNum); }
+
+Status LegacyReduceSameOp::validateAndExecute(Context& block) {
+  auto x = INPUT_VARIABLE(0);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+  sd_debug("Executing LegacyReduceSameOp: [%i]\n", opNum);
+
+  auto axis = *block.getAxis();
+  bool allAxes = false;
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyReduceSameOp");
+
+  if (block.width() == 1) {
+    if (axis.size() == static_cast<size_t>(x->rankOf())) allAxes = true;
+
+    if (axis.empty() || allAxes) {
+      // scalar
+      NativeOpExecutioner::execReduceSameScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(z->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      std::vector<LongType> dims(axis);
+
+      for (size_t e = 0; e < dims.size(); e++)
+        if (dims[e] < 0) dims[e] += x->rankOf();
+
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+      const LongType* zShapeInfoH = z->shapeInfo();
+      const LongType* zShapeInfoD = z->specialShapeInfo();
+
+      if (x->rankOf() - dims.size() != static_cast<size_t>(z->rankOf())) {
+        auto zPack = ConstantShapeHelper::getInstance().createShapeInfoWithNoUnitiesForReduce(
+            z->shapeInfo(), &dims, z->getContext()->getWorkspace());
+        zShapeInfoH = reinterpret_cast<LongType const*>(zPack->primary());
+        zShapeInfoD = reinterpret_cast<LongType const*>(zPack->special());
+      }
+
+      std::vector<LongType> *dims2 = ShapeUtils::evalDimsForReduceOp(x->rankOf(), &dims);
+      NativeOpExecutioner::execReduceSame(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                          x->specialShapeInfo(), nullptr, z->buffer(), zShapeInfoH, z->specialBuffer(),
+                                          zShapeInfoD, dims2->data(), dims2->size());
+
+    }
+
+    STORE_RESULT(*z);
+  } else {
+    auto indices = INPUT_VARIABLE(1);
+    if (indices->lengthOf() == x->rankOf()) allAxes = true;
+
+
+    std::vector<LongType> dims(indices->lengthOf());
+    for (int e = 0; e < indices->lengthOf(); e++) {
+      // segfault on macOS if not like this
+      int f = indices->e<LongType>(e);
+      dims[e] = f >= 0 ? f : f += x->rankOf();
+    }
+
+    if ((block.getIArguments()->size() == 1 && INT_ARG(0) == DataTypeUtils::max<int>()) || allAxes) {
+      // scalar
+      NativeOpExecutioner::execReduceSameScalar(
+          block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+          extras.argumentsAsT(z->dataType()), z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo());
+    } else {
+      // TAD
+      REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions required for reduction!");
+
+      const LongType* zShapeInfoH = z->shapeInfo();
+      const LongType* zShapeInfoD = z->specialShapeInfo();
+
+      if (x->rankOf() - dims.size() != static_cast<size_t>(z->rankOf())) {
+        auto zPack = ConstantShapeHelper::getInstance().createShapeInfoWithNoUnitiesForReduce(
+            z->shapeInfo(), &dims, z->getContext()->getWorkspace());
+        zShapeInfoH = reinterpret_cast<LongType const*>(zPack->primary());
+        zShapeInfoD = reinterpret_cast<LongType const*>(zPack->special());
+      }
+
+      std::vector<LongType> *dims2 = ShapeUtils::evalDimsForReduceOp(x->rankOf(), &dims);
+      NativeOpExecutioner::execReduceSame(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                          x->specialShapeInfo(), nullptr, z->buffer(), zShapeInfoH, z->specialBuffer(),
+                                          zShapeInfoD, dims2->data(), dims2->size());
+
+      delete dims2;
+
+    }
+  }
+
+  manager.synchronize();
+  if(OpRegistrator::getInstance().traceOps()) {
+    std::vector<const LongType*> *inputShapeBuffers = new std::vector<const LongType*>();
+    for(size_t i = 0; i < block.width(); i++) {
+      inputShapeBuffers->push_back(block.variable(i)->getNDArray()->shapeInfo());
+    }
+    std::vector<const LongType*> *outputShapeBuffers = new std::vector<const LongType*>();
+    for(size_t i = 0; i < block.outputWidth(); i++) {
+      outputShapeBuffers->push_back(getZ(block,i)->shapeInfo());
+    }
+
+    OpExecTrace *opExecTrace = new OpExecTrace(inputShapeBuffers,outputShapeBuffers,this->getOpName());
+    OpRegistrator::getInstance().registerOpExec(opExecTrace);
+  }
+  return Status::OK;
+}
+
+/**
+ *   For all reductions rules are simple: either you return scalar, or you return reduced NDArray.
+ *   It solely depends on input shape, and requested dimensions
+ */
+ShapeList* LegacyReduceSameOp::calculateOutputShape(ShapeList* inputShape, Context& block) {
+  auto inShape = inputShape->at(0);
+
+
+  auto keepDims = block.numB() > 0 ? B_ARG(0) : false;
+  auto newFormat = block.numB() > 1 ? B_ARG(1) : true;
+
+  auto axis = block.width() > 1 ? INPUT_VARIABLE(1)->asVectorT<LongType>() : *block.getAxis();
+
+
+  // in this case we're building proper shape for reduction
+  auto newShape =
+      ShapeUtils::evalReduceShapeInfo(shape::order(inShape), &axis, inShape, keepDims, !newFormat, block.workspace());
+
+  return SHAPELIST(newShape);
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyScalarBoolOp.cpp

@@ -0,0 +1,93 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <array/NDArrayFactory.h>
+#include <ops/declarable/LegacyScalarBoolOp.h>
+
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+LegacyScalarBoolOp::LegacyScalarBoolOp() : LegacyOp(1) {
+  // no-op
+}
+
+LegacyScalarBoolOp::LegacyScalarBoolOp(int opNum) : LegacyOp(1, opNum) {
+  // no-op
+}
+
+LegacyOp *LegacyScalarBoolOp::clone() { return new LegacyScalarBoolOp(this->_opNum, *this->_scalar); }
+
+LegacyScalarBoolOp::LegacyScalarBoolOp(int opNum, NDArray &scalar) : LegacyOp(1, opNum) {
+  _scalar = new NDArray(scalar.dup(scalar.ordering(), false));
+}
+
+ShapeList *LegacyScalarBoolOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+  return SHAPELIST(CONSTANT(inShape));
+}
+
+Status LegacyScalarBoolOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyScalarBoolOp");
+
+  if (block.width() > 1) {
+    auto y = INPUT_VARIABLE(1);
+
+    NDArray::prepareSpecialUse({z}, {x, y});
+
+    NativeOpExecutioner::execScalarBool(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                        x->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+                                        z->specialShapeInfo(), y->buffer(), y->shapeInfo(), y->specialBuffer(),
+                                        y->specialShapeInfo(), extras.argumentsAsT(x->dataType()));
+  } else if (block.getTArguments()->size() > 0) {
+    auto y = NDArrayFactory::create(T_ARG(0), block.launchContext());
+
+    NDArray::prepareSpecialUse({z}, {x, &y});
+
+    NativeOpExecutioner::execScalarBool(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                        x->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+                                        z->specialShapeInfo(), y.buffer(), y.shapeInfo(), y.specialBuffer(),
+                                        y.specialShapeInfo(), extras.argumentsAsT(x->dataType(), 1));
+
+    manager.synchronize();
+  } else {
+    NDArray::prepareSpecialUse({z}, {x, _scalar});
+
+    NativeOpExecutioner::execScalarBool(
+        block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+        z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo(), _scalar->buffer(), _scalar->shapeInfo(),
+        _scalar->specialBuffer(), _scalar->specialShapeInfo(), extras.argumentsAsT(x->dataType()));
+  }
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyScalarOp.cpp

@@ -0,0 +1,93 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <array/NDArrayFactory.h>
+#include <ops/declarable/LegacyScalarOp.h>
+
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+LegacyScalarOp::LegacyScalarOp() : LegacyOp(1) { this->getOpDescriptor()->allowInplace(true); }
+
+LegacyScalarOp::LegacyScalarOp(int opNum) : LegacyOp(1, opNum) {
+  this->getOpDescriptor()->allowInplace(true);
+}
+
+LegacyOp *LegacyScalarOp::clone() { return new LegacyScalarOp(this->_opNum, *this->_scalar); }
+
+LegacyScalarOp::LegacyScalarOp(int opNum, NDArray &scalar) : LegacyOp(1, opNum) {
+  _scalar = new NDArray(scalar.dup(scalar.ordering(), false));
+}
+
+ShapeList *LegacyScalarOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+
+  LongType *newShape;
+  COPY_SHAPE(inShape, newShape);
+
+  return SHAPELIST(CONSTANT(newShape));
+}
+
+Status LegacyScalarOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyScalarOp");
+
+  if (block.width() > 1) {
+    auto y = INPUT_VARIABLE(1);
+
+    NDArray::prepareSpecialUse({z}, {x, y});
+
+    NativeOpExecutioner::execScalar(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                    x->specialShapeInfo(), z->buffer(), z->shapeInfo(), z->specialBuffer(),
+                                    z->specialShapeInfo(), y->buffer(), y->shapeInfo(), y->specialBuffer(),
+                                    y->specialShapeInfo(), extras.argumentsAsT(z->dataType()));
+
+    NDArray::registerSpecialUse({z}, {x, y});
+  } else if (block.getTArguments()->size() > 0) {
+    auto y = NDArrayFactory::create(x->dataType(), T_ARG(0), block.launchContext());
+
+    x->applyScalarArr(static_cast<scalar::Ops>(opNum), &y, z);
+    manager.synchronize();
+  } else {
+    NDArray::prepareSpecialUse({z}, {x, _scalar});
+
+    NativeOpExecutioner::execScalar(
+        block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(), x->specialShapeInfo(),
+        z->buffer(), z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo(), _scalar->buffer(), _scalar->shapeInfo(),
+        _scalar->specialBuffer(), _scalar->specialShapeInfo(), extras.argumentsAsT(z->dataType()));
+
+    NDArray::registerSpecialUse({z}, {x, _scalar});
+  }
+
+
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyStatsOp.cpp

@@ -0,0 +1,125 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 17.10.2017.
+//
+#include <array/DataTypeUtils.h>
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/ShapeUtils.h>
+
+#include <ops/declarable/LegacyStatsOp.h>
+#include <ops/declarable/OpRegistrator.h>
+#include <legacy/NativeOpExecutioner.h>
+
+namespace sd {
+namespace ops {
+Status LegacyStatsOp::validateAndExecute(Context &block) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {x});
+
+  // we assume that opNuk is either stored in block, or was provided via op constructor
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  // bias goes as first argument, unlike all other reductions
+  bool biasCorrected = false;
+  if (block.getIArguments()->size() > 0) biasCorrected = INT_ARG(0) > 0;
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyStatsOp");
+
+  if (block.getIArguments()->size() == 1 ||
+      (block.getIArguments()->size() == 2 && INT_ARG(1) == DataTypeUtils::max<int>())) {
+    // scalar
+    NativeOpExecutioner::execSummaryStatsScalar(block.launchContext(), opNum, x->buffer(), x->shapeInfo(),
+                                                x->specialBuffer(), x->specialShapeInfo(),
+                                                extras.argumentsAsT(z->dataType()), z->buffer(), z->shapeInfo(),
+                                                z->specialBuffer(), z->specialShapeInfo(), biasCorrected);
+  } else {
+    // dimensions for TAD
+    // we should skip first argument here, because it's addressing bias correction
+    std::vector<LongType> dims(*block.getIArguments());
+    for (size_t e = 0; e < dims.size(); e++)
+      if (dims[e] < 0) dims[e] += x->rankOf();
+
+    REQUIRE_TRUE(dims.size() > 0, 0, "Some dimensions requuired for reduction!");
+
+    auto packX = ConstantTadHelper::getInstance().tadForDimensions(x->shapeInfo(), &dims);
+
+    auto pTadShape = Environment::getInstance().isCPU()
+                         ? packX->primaryShapeInfo()
+                         : packX->specialShapeInfo();
+
+    auto pTadOffsets = Environment::getInstance().isCPU()
+                           ? packX->primaryOffsets()
+                           : packX->specialOffsets();
+
+    NativeOpExecutioner::execSummaryStats(block.launchContext(), opNum, x->buffer(), x->shapeInfo(), x->specialBuffer(),
+                                          x->specialShapeInfo(), extras.argumentsAsT(z->dataType()), z->buffer(),
+                                          z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo(), dims.data(),
+                                          (int)dims.size(), pTadShape, pTadOffsets, biasCorrected);
+  }
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+LegacyStatsOp::LegacyStatsOp() : LegacyOp(1) {
+  //
+}
+
+LegacyStatsOp::LegacyStatsOp(int opNum) : LegacyOp(1, opNum) {
+  //
+}
+
+LegacyOp *LegacyStatsOp::clone() { return new LegacyStatsOp(this->_opNum); }
+
+/**
+ *   For all reductions rules are simple: either you return scalar, or you return reduced NDArray.
+ *   It solely depends on input shape, and requested dimensions
+ */
+ShapeList *LegacyStatsOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+
+  LongType *newShape;
+  if (block.getIArguments()->size() == 0 ||
+      (block.getIArguments()->size() == 1 && INT_ARG(0) == DataTypeUtils::max<int>())) {
+    // in this case we just return scalar
+    ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+    newShape[0] = 2;
+    newShape[1] = 1;
+    newShape[2] = 1;
+    newShape[3] = 1;
+    newShape[4] = 1;
+    newShape[5] = 0;
+    newShape[6] = 1;
+    newShape[7] = 99;
+  } else {
+    sd::LongType *xShape2 = ShapeUtils::evalReduceShapeInfo('c', block.getIArguments(), inShape, false, true);
+    return SHAPELIST(xShape2);
+  }
+
+  return SHAPELIST(CONSTANT(newShape));
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyTransformAnyOp.cpp

@@ -0,0 +1,72 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <legacy/NativeOpExecutioner.h>
+#include <ops/declarable/LegacyTransformAnyOp.h>
+
+#include <ops/declarable/OpRegistrator.h>
+
+namespace sd {
+namespace ops {
+LegacyTransformAnyOp::LegacyTransformAnyOp() : LegacyOp(1) {
+  // just a no-op
+}
+
+LegacyTransformAnyOp::LegacyTransformAnyOp(int opNum) : LegacyOp(1, opNum) {
+  // just a no-op
+}
+
+LegacyOp *LegacyTransformAnyOp::clone() { return new LegacyTransformAnyOp(this->_opNum); }
+
+Status LegacyTransformAnyOp::validateAndExecute(Context &block) {
+  auto input = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {input});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyTransformAnyOp");
+
+  NativeOpExecutioner::execTransformAny(block.launchContext(), opNum, input->buffer(), input->shapeInfo(),
+                                        input->specialBuffer(), input->specialShapeInfo(), z->buffer(), z->shapeInfo(),
+                                        z->specialBuffer(), z->specialShapeInfo(), extras.argumentsAsT(z->dataType()),
+                                        false);
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+/**
+ * For transform operations, output shape always equals to input shape. With just a few exclusions, like im2col and
+ * col2im. But these ops already have CustomOp implementations.
+ *
+ */
+ShapeList *LegacyTransformAnyOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+  return SHAPELIST(CONSTANT(inShape));
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyTransformBoolOp.cpp

@@ -0,0 +1,73 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <legacy/NativeOpExecutioner.h>
+#include <ops/declarable/LegacyTransformBoolOp.h>
+
+#include <ops/declarable/OpRegistrator.h>
+
+namespace sd {
+namespace ops {
+LegacyTransformBoolOp::LegacyTransformBoolOp() : LegacyOp(1) {
+  // just a no-op
+}
+
+LegacyTransformBoolOp::LegacyTransformBoolOp(int opNum) : LegacyOp(1, opNum) {
+  // just a no-op
+}
+
+LegacyOp *LegacyTransformBoolOp::clone() { return new LegacyTransformBoolOp(this->_opNum); }
+
+Status LegacyTransformBoolOp::validateAndExecute(Context &block) {
+  auto input = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {input});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyTransformBoolOp");
+
+  NativeOpExecutioner::execTransformBool(block.launchContext(), opNum, input->buffer(), input->shapeInfo(),
+                                         input->specialBuffer(), input->specialShapeInfo(), z->buffer(), z->shapeInfo(),
+                                         z->specialBuffer(), z->specialShapeInfo(),
+                                         extras.argumentsAsT(input->dataType()));
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+/**
+ * For transform operations, output shape always equals to input shape. With just a few exclusions, like im2col and
+ * col2im. But these ops already have CustomOp implementations.
+ *
+ */
+ShapeList *LegacyTransformBoolOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().castToDataType(inShape, BOOL));
+  return ret;
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyTransformFloatOp.cpp

@@ -0,0 +1,71 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <legacy/NativeOpExecutioner.h>
+#include <ops/declarable/LegacyTransformFloatOp.h>
+#include <ops/declarable/OpRegistrator.h>
+
+namespace sd {
+namespace ops {
+LegacyTransformFloatOp::LegacyTransformFloatOp() : LegacyOp(1) {
+  // just a no-op
+}
+
+LegacyTransformFloatOp::LegacyTransformFloatOp(int opNum) : LegacyOp(1, opNum) {
+  // just a no-op
+}
+
+LegacyOp *LegacyTransformFloatOp::clone() { return new LegacyTransformFloatOp(this->_opNum); }
+
+Status LegacyTransformFloatOp::validateAndExecute(Context &block) {
+  auto input = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {input});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyTransformFloatOp");
+
+  NativeOpExecutioner::execTransformFloat(block.launchContext(), opNum, input->buffer(), input->shapeInfo(),
+                                          input->specialBuffer(), input->specialShapeInfo(), z->buffer(),
+                                          z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo(),
+                                          extras.argumentsAsT(z->dataType()));
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+/**
+ * For transform operations, output shape always equals to input shape. With just a few exclusions, like im2col and
+ * col2im. But these ops already have CustomOp implementations.
+ *
+ */
+ShapeList *LegacyTransformFloatOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+  return SHAPELIST(CONSTANT(inShape));
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyTransformOp.cpp

@@ -0,0 +1,65 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <legacy/NativeOpExecutioner.h>
+#include <ops/declarable/LegacyTransformOp.h>
+#include <ops/declarable/OpRegistrator.h>
+
+#ifdef ONLY_SAME_TRANSFORM
+namespace sd {
+namespace ops {
+LegacyTransformOp::LegacyTransformOp() : LegacyOp::LegacyOp(1) {
+  // just a no-op
+}
+
+LegacyTransformOp::LegacyTransformOp(int opType) : LegacyOp::LegacyOp(1, opType) {
+  // just a no-op
+}
+
+LegacyOp *LegacyTransformOp::clone() { return new LegacyTransformOp(this->_opNum); }
+
+sd::Status LegacyTransformOp::validateAndExecute(Context &block) {
+  auto input = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  int opType = block.opType() < 0 ? this->_opNum : block.opType();
+
+  NativeOpExcutioner::execTransformSame(opType, input->buffer(), input->shapeInfo(), z->buffer(), z->shapeInfo(),
+                                        block.getTArguments()->data(), nullptr, nullptr);
+
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return sd::Status::OK;
+}
+
+/**
+ * For transform operations, output shape always equals to input shape. With just a few exclusions, like im2col and
+ * col2im. But these ops already have CustomOp implementations.
+ *
+ */
+ShapeList *LegacyTransformOp::calculateOutputShape(ShapeList *inputShape, sd::graph::Context &block) {
+  auto inShape = inputShape->at(0);
+  return SHAPELIST(CONSTANT(inShape));
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/impl/LegacyTransformSameOp.cpp

@@ -0,0 +1,73 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <legacy/NativeOpExecutioner.h>
+#include <ops/declarable/LegacyTransformSameOp.h>
+#include <ops/declarable/OpRegistrator.h>
+
+namespace sd {
+namespace ops {
+LegacyTransformSameOp::LegacyTransformSameOp() : LegacyOp(1) { this->getOpDescriptor()->allowInplace(true); }
+
+LegacyTransformSameOp::LegacyTransformSameOp(int opNum) : LegacyOp(1, opNum) {
+  this->getOpDescriptor()->allowInplace(true);
+}
+
+LegacyOp *LegacyTransformSameOp::clone() { return new LegacyTransformSameOp(this->_opNum); }
+
+Status LegacyTransformSameOp::validateAndExecute(Context &block) {
+  auto input = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {input});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyTransformSameOp");
+
+  NativeOpExecutioner::execTransformSame(block.launchContext(), opNum, input->buffer(), input->shapeInfo(),
+                                         input->specialBuffer(), input->specialShapeInfo(), z->buffer(), z->shapeInfo(),
+                                         z->specialBuffer(), z->specialShapeInfo(), extras.argumentsAsT(z->dataType()),
+                                         nullptr, nullptr);
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+/**
+ * For transform operations, output shape always equals to input shape. With just a few exclusions, like im2col and
+ * col2im. But these ops already have CustomOp implementations.
+ *
+ */
+ShapeList *LegacyTransformSameOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+
+  LongType *newShape;
+  COPY_SHAPE(inShape, newShape);
+
+  return SHAPELIST(CONSTANT(newShape));
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LegacyTransformStrictOp.cpp

@@ -0,0 +1,71 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+#include <legacy/NativeOpExecutioner.h>
+#include <ops/declarable/LegacyTransformStrictOp.h>
+#include <ops/declarable/OpRegistrator.h>
+
+namespace sd {
+namespace ops {
+LegacyTransformStrictOp::LegacyTransformStrictOp() : LegacyOp(1) {
+  this->getOpDescriptor()->allowInplace(true);
+}
+
+LegacyTransformStrictOp::LegacyTransformStrictOp(int opNum) : LegacyOp(1, opNum) {
+  this->getOpDescriptor()->allowInplace(true);
+}
+
+LegacyOp *LegacyTransformStrictOp::clone() { return new LegacyTransformStrictOp(this->_opNum); }
+
+Status LegacyTransformStrictOp::validateAndExecute(Context &block) {
+  auto input = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  NDArray::prepareSpecialUse({z}, {input});
+
+  int opNum = block.opNum() < 0 ? this->_opNum : block.opNum();
+
+  ExtraArguments extras(*block.getTArguments());
+  PointersManager manager(block.launchContext(), "LegacyTransformStrictOp");
+
+  NativeOpExecutioner::execTransformStrict(block.launchContext(), opNum, input->buffer(), input->shapeInfo(),
+                                           input->specialBuffer(), input->specialShapeInfo(), z->buffer(),
+                                           z->shapeInfo(), z->specialBuffer(), z->specialShapeInfo(),
+                                           extras.argumentsAsT(z->dataType()));
+
+  manager.synchronize();
+  STORE_RESULT(*z);
+  traceExecIfNeeded(block);
+
+  return Status::OK;
+}
+
+/**
+ * For transform operations, output shape always equals to input shape. With just a few exclusions, like im2col and
+ * col2im. But these ops already have CustomOp implementations.
+ *
+ */
+ShapeList *LegacyTransformStrictOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  auto inShape = inputShape->at(0);
+  return SHAPELIST(CONSTANT(inShape));
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/LogicOp.cpp

@@ -0,0 +1,41 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 15.10.2017.
+//
+#include "ops/declarable/LogicOp.h"
+
+namespace sd {
+namespace ops {
+LogicOp::LogicOp(const char *name) : DeclarableOp(name, true) {
+  // just using DeclarableOp constructor
+  // this->_descriptor->
+}
+
+Status LogicOp::validateAndExecute(Context &block) {
+  sd_logger("WARNING: LogicOps should NOT be ever called\n", "");
+  return Status::BAD_INPUT;
+}
+
+ShapeList *LogicOp::calculateOutputShape(ShapeList *inputShape, Context &block) {
+  // FIXME: we probably want these ops to evaluate scopes
+  return SHAPELIST();
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/OpDescriptor.cpp

@@ -0,0 +1,273 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+#include <ops/declarable/OpDescriptor.h>
+
+namespace sd {
+namespace ops {
+
+OpDescriptor::OpDescriptor(const char* opName, bool isLogic) {
+  _logic = isLogic;
+  _opName = opName;
+}
+
+OpDescriptor::OpDescriptor(int numInputs, const char* opName, bool isScalar) {
+  _numInputs = numInputs;
+  _numOutputs = 1;
+
+  _opName = opName;
+  _hash = HashHelper::getInstance().getLongHash(_opName);
+
+  _scalar = isScalar;
+}
+
+OpDescriptor::OpDescriptor(int numInputs, std::string opName, bool isScalar) {
+  _numInputs = numInputs;
+  _numOutputs = 1;
+
+  _opName = opName;
+  _hash = HashHelper::getInstance().getLongHash(_opName);
+
+  _scalar = isScalar;
+}
+
+void OpDescriptor::allowInplace(bool reallyAllow) { _allowsInplace = reallyAllow; }
+
+bool OpDescriptor::operator==(const OpDescriptor& other) const {
+  if (_hash == -1 && other._hash == -1)
+    return this->_opNum == other._opNum;
+  else
+    return this->_hash == other._hash;
+}
+
+OpDescriptor::OpDescriptor(int numInputs, int numOutputs, std::string opName, bool allowsInplace)
+    : OpDescriptor(numInputs, numOutputs, opName.c_str(), allowsInplace) {
+  //
+}
+
+void OpDescriptor::setHash(LongType hash) { _hash = hash; }
+
+// default constructor
+OpDescriptor::OpDescriptor(int numInputs, int numOutputs, const char* opName, bool allowsInplace) {
+  _numInputs = numInputs;
+  _numOutputs = numOutputs;
+
+  std::string tmp(opName);
+  _opName = tmp;
+  _allowsInplace = allowsInplace;
+  _hash = HashHelper::getInstance().getLongHash(tmp);
+  _divergent = false;
+
+  // just default value
+}
+
+// constructor for configurable op
+OpDescriptor::OpDescriptor(int numInputs, int numOutputs, const char* opName, bool allowsInplace, int tArgs, int iArgs)
+    : OpDescriptor(numInputs, numOutputs, opName, allowsInplace) {
+  _tArgs = tArgs;
+  _iArgs = iArgs;
+}
+
+// constructor for non-configurable divergent op
+OpDescriptor::OpDescriptor(int numInputs, int numOutputs, std::string opName, bool allowsInplace, bool divergent)
+    : OpDescriptor(numInputs, numOutputs, opName.c_str(), allowsInplace, divergent) {}
+
+// constructor for non-configurable divergent op
+OpDescriptor::OpDescriptor(int numInputs, int numOutputs, const char* opName, bool allowsInplace, bool divergent)
+    : OpDescriptor(numInputs, numOutputs, opName, allowsInplace) {
+  _divergent = divergent;
+}
+
+// constructor for configurable divergent op
+OpDescriptor::OpDescriptor(int numInputs, int numOutputs, const char* opName, bool allowsInplace, bool divergent,
+                           int tArgs, int iArgs)
+    : OpDescriptor(numInputs, numOutputs, opName, allowsInplace, tArgs, iArgs) {
+  _divergent = divergent;
+}
+
+
+int OpDescriptor::getNumberOfTArgs() { return _tArgs; }
+
+int OpDescriptor::getNumberOfIArgs() { return _iArgs; }
+
+int OpDescriptor::getNumberOfInputs() { return _numInputs; }
+
+LongType OpDescriptor::getHash() { return _hash; }
+
+int OpDescriptor::getNumberOfOutputs() { return _numOutputs; }
+
+std::string* OpDescriptor::getOpName() { return &_opName; }
+
+bool OpDescriptor::isDivergent() { return _divergent; }
+
+void OpDescriptor::setOpNum(int opNum) { _opNum = opNum; }
+
+bool OpDescriptor::allowsInplace() { return _allowsInplace; }
+
+int OpDescriptor::getOpNum() { return _opNum; }
+
+OpDescriptor* OpDescriptor::setInputType(const InputType type) {
+  _inputType = type;
+  return this;
+}
+
+InputType OpDescriptor::inputType() { return _inputType; }
+
+OpDescriptor* OpDescriptor::setAllowedInputTypes(const std::initializer_list<DataType>& dtypes) {
+  _allowedIns = dtypes;
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setAllowedOutputTypes(const std::initializer_list<DataType>& dtypes) {
+  _allowedOuts = dtypes;
+  return this;
+}
+
+OpDescriptor* OpDescriptor::allowOverride(bool allowOverride) {
+  _dtypeOverride = allowOverride;
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setAllowedInputTypes(const DataType dtype) {
+  _allowedIns.clear();
+  _allowedIns.emplace_back(dtype);
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setAllowedOutputTypes(const DataType dtype) {
+  _allowedOuts.clear();
+  _allowedOuts.emplace_back(dtype);
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setInputType(const int idx, const DataType dtype) {
+  _inputTypes[idx] = {dtype};
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setOutputType(const int idx, const DataType dtype) {
+  _outputTypes[idx] = {dtype};
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setSameMode(const bool reallySame) {
+  _sameMode = reallySame;
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setAllowedInputTypes(int index, const std::vector<DataType>& dtype) {
+  _inputTypes[index] = dtype;
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setAllowedOutputTypes(int index, const std::vector<DataType>& dtype) {
+  _outputTypes[index] = dtype;
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setAllowedInputTypes(int index, DataType dtype) {
+  if (_inputTypes.count(index) == 0)
+    _inputTypes[index] = {dtype};
+  else
+    _inputTypes[index].emplace_back(dtype);
+
+  return this;
+}
+
+OpDescriptor* OpDescriptor::setAllowedOutputTypes(int index, DataType dtype) {
+  if (_outputTypes.count(index) == 0)
+    _outputTypes[index] = {dtype};
+  else
+    _outputTypes[index].emplace_back(dtype);
+
+  return this;
+}
+
+bool OpDescriptor::checkDataTypesMatch(DataType needle, std::vector<DataType>& haystack) const {
+  // if haystack is empty - INHERIT is occurs - any type is perfect?
+  if (haystack.empty()) return true;
+
+  // first we're checking for direct input type match
+  if (std::find(haystack.begin(), haystack.end(), needle) == haystack.end()) {
+    // if direct input match failed - we're checking for ANY as allowed input
+    if (std::find(haystack.begin(), haystack.end(), ANY) == haystack.end())
+      return false;
+    else
+      return true;
+  } else {
+    return true;
+  }
+}
+
+bool OpDescriptor::checkInputMatch(int index, DataType dataType) {
+  // we check for per-input types first
+  if (_inputTypes.empty() || _inputTypes.count(index) == 0) {
+    // checking global input types
+    return checkDataTypesMatch(dataType, _allowedIns);
+  } else {
+    // checking data type for specified input
+    auto& allowed = _inputTypes[index];
+    return checkDataTypesMatch(dataType, allowed);
+  }
+  return true;
+}
+
+bool OpDescriptor::checkOutputMatch(int index, DataType dataType) {
+  // we check for per-output types first
+  if (_outputTypes.empty() || _outputTypes.count(index) == 0) {
+    // checking global output types
+    return checkDataTypesMatch(dataType, _allowedOuts);
+  } else {
+    // checking data type for specified output
+    auto allowed = _outputTypes[index];
+    return checkDataTypesMatch(dataType, allowed);
+  }
+  return true;
+}
+
+bool OpDescriptor::isSameMode() { return _sameMode; }
+
+bool OpDescriptor::isInherit(int index) {
+  if (std::find(_allowedOuts.begin(), _allowedOuts.end(), INHERIT) != _allowedOuts.end()) return true;
+  if (_outputTypes.count(index) > 0) {
+    auto vec = _outputTypes[index];
+
+    if (std::find(vec.begin(), vec.end(), INHERIT) != vec.end()) return true;
+  }
+
+  return false;
+}
+
+std::vector<DataType> OpDescriptor::getOutputTypesForOutput(int index) {
+  if (_outputTypes.count(index) > 0)
+    return _outputTypes.at(index);
+  else
+    return std::vector<DataType>();
+}
+
+std::vector<DataType> OpDescriptor::getInputTypesForInput(int index) {
+  if (_inputTypes.count(index) > 0)
+    return _inputTypes.at(index);
+  else
+    return std::vector<DataType>();
+}
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/impl/OpRegistrator.cpp

@@ -0,0 +1,273 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.10.2017.
+//
+
+#include <ops/declarable/OpRegistrator.h>
+
+#include <sstream>
+
+namespace sd {
+namespace ops {
+
+///////////////////////////////
+
+template <typename OpName>
+__registrator<OpName>::__registrator() {
+  auto ptr = new OpName();
+  OpRegistrator::getInstance().registerOperation(ptr);
+}
+
+template <typename OpName>
+__registratorSynonym<OpName>::__registratorSynonym(const char* name, const char* oname) {
+  auto ptr = reinterpret_cast<OpName*>(OpRegistrator::getInstance().getOperation(oname));
+  if (ptr == nullptr) {
+    std::string newName(name);
+    std::string oldName(oname);
+
+    OpRegistrator::getInstance().updateMSVC(HashHelper::getInstance().getLongHash(newName), oldName);
+    return;
+  }
+  OpRegistrator::getInstance().registerOperation(name, ptr);
+}
+
+///////////////////////////////
+
+OpRegistrator& OpRegistrator::getInstance() {
+  static OpRegistrator instance;
+  return instance;
+}
+
+void OpRegistrator::updateMSVC(LongType newHash, std::string& oldName) {
+  std::pair<LongType, std::string> pair(newHash, oldName);
+  _msvc.insert(pair);
+}
+
+template <typename T>
+std::string OpRegistrator::local_to_string(T value) {
+  // create an output string stream
+  std::ostringstream os;
+
+  // throw the value into the string stream
+  os << value;
+
+  // convert the string stream into a string and return
+  return os.str();
+}
+
+template <>
+std::string OpRegistrator::local_to_string(int value) {
+  // create an output string stream
+  std::ostringstream os;
+
+  // throw the value into the string stream
+  os << value;
+
+  // convert the string stream into a string and return
+  return os.str();
+}
+
+OpRegistrator::~OpRegistrator() {
+#ifndef _RELEASE
+  _msvc.clear();
+
+  for (auto x : _uniqueD) delete x;
+
+  for (auto x : _uniqueH) delete x;
+
+  _uniqueD.clear();
+
+  _uniqueH.clear();
+
+  _declarablesD.clear();
+
+  _declarablesLD.clear();
+
+#endif
+}
+
+const char* OpRegistrator::getAllCustomOperations() {
+  _locker.lock();
+
+  if (!isInit) {
+    for (SD_MAP_IMPL<std::string, DeclarableOp*>::iterator it = _declarablesD.begin();
+         it != _declarablesD.end(); ++it) {
+      std::string op = it->first + ":" + local_to_string(it->second->getOpDescriptor()->getHash()) + ":" +
+                       local_to_string(it->second->getOpDescriptor()->getNumberOfInputs()) + ":" +
+                       local_to_string(it->second->getOpDescriptor()->getNumberOfOutputs()) + ":" +
+                       local_to_string(it->second->getOpDescriptor()->allowsInplace()) + ":" +
+                       local_to_string(it->second->getOpDescriptor()->getNumberOfTArgs()) + ":" +
+                       local_to_string(it->second->getOpDescriptor()->getNumberOfIArgs()) + ":" + ";";
+      _opsList += op;
+    }
+
+    isInit = true;
+  }
+
+  _locker.unlock();
+
+  return _opsList.c_str();
+}
+
+bool OpRegistrator::registerOperation(const char* name, DeclarableOp* op) {
+  std::string str(name);
+  std::pair<std::string, DeclarableOp*> pair(str, op);
+  _declarablesD.insert(pair);
+
+  auto hash = HashHelper::getInstance().getLongHash(str);
+  std::pair<LongType, DeclarableOp*> pair2(hash, op);
+  _declarablesLD.insert(pair2);
+  return true;
+}
+
+void OpRegistrator::registerOpExec(OpExecTrace *opExecTrace) {
+  this->opexecTrace.push_back(opExecTrace);
+}
+
+bool OpRegistrator::traceOps() {
+  return this->isTrace;
+}
+
+void OpRegistrator::toggleTraceOps(bool traceOps) {
+  this->isTrace = traceOps;
+}
+
+void OpRegistrator::purgeOpExecs() {
+  this->opexecTrace.clear();
+}
+
+std::vector<OpExecTrace *>  * OpRegistrator::execTrace() {
+  return &(this->opexecTrace);
+}
+
+/**
+ * This method registers operation
+ *
+ * @param op
+ */
+bool OpRegistrator::registerOperation(DeclarableOp* op) {
+  _uniqueD.emplace_back(op);
+  return registerOperation(op->getOpName()->c_str(), op);
+}
+
+void OpRegistrator::registerHelper(platforms::PlatformHelper* op) {
+  std::pair<LongType, samediff::Engine> p = {op->hash(), op->engine()};
+  if (_helpersLH.count(p) > 0) THROW_EXCEPTION("Tried to double register PlatformHelper");
+
+  _uniqueH.emplace_back(op);
+
+  sd_debug("Adding helper for op \"%s\": [%lld - %i]\n", op->name().c_str(), op->hash(), (int)op->engine());
+
+  std::pair<std::pair<std::string, samediff::Engine>, platforms::PlatformHelper*> pair(
+      {op->name(), op->engine()}, op);
+  _helpersH.insert(pair);
+
+  std::pair<std::pair<LongType, samediff::Engine>, platforms::PlatformHelper*> pair2(p, op);
+  _helpersLH.insert(pair2);
+}
+
+
+
+DeclarableOp* OpRegistrator::getOperation(const char* name) {
+  std::string str(name);
+  return getOperation(str);
+}
+
+/**
+ * This method returns registered Op by name
+ *
+ * @param name
+ * @return
+ */
+DeclarableOp* OpRegistrator::getOperation(LongType hash) {
+  if (!_declarablesLD.count(hash)) {
+    if (!_msvc.count(hash)) {
+      sd_printf("Unknown D operation requested by hash: [%lld]\n", hash);
+      return nullptr;
+    } else {
+      _locker.lock();
+
+      auto str = _msvc.at(hash);
+      auto op = _declarablesD.at(str);
+      auto oHash = op->getOpDescriptor()->getHash();
+
+      std::pair<LongType, DeclarableOp*> pair(oHash, op);
+      _declarablesLD.insert(pair);
+
+      _locker.unlock();
+    }
+  }
+
+  return _declarablesLD.at(hash);
+}
+
+DeclarableOp* OpRegistrator::getOperation(std::string& name) {
+  if (!_declarablesD.count(name)) {
+    sd_debug("Unknown operation requested: [%s]\n", name.c_str());
+    return nullptr;
+  }
+
+  return _declarablesD.at(name);
+}
+
+platforms::PlatformHelper* OpRegistrator::getPlatformHelper(LongType hash, samediff::Engine engine) {
+  std::pair<LongType, samediff::Engine> p = {hash, engine};
+  if (_helpersLH.count(p) == 0) THROW_EXCEPTION("Requested helper can't be found");
+
+  return _helpersLH[p];
+}
+
+
+bool OpRegistrator::hasHelper(LongType hash, samediff::Engine engine) {
+  std::pair<LongType, samediff::Engine> p = {hash, engine};
+  return _helpersLH.count(p) > 0;
+}
+
+int OpRegistrator::numberOfOperations() { return (int)_declarablesLD.size(); }
+
+std::vector<LongType> OpRegistrator::getAllHashes() {
+  std::vector<LongType> result;
+
+  for (auto& v : _declarablesLD) {
+    result.emplace_back(v.first);
+  }
+
+  return result;
+}
+}  // namespace ops
+}  // namespace sd
+
+namespace std {
+size_t hash<std::pair<sd::LongType, samediff::Engine>>::operator()(
+    const std::pair<sd::LongType, samediff::Engine>& k) const {
+  using std::hash;
+  auto res = std::hash<sd::LongType>()(k.first);
+  res ^= std::hash<sd::LongType>()((sd::LongType)k.second) + 0x9e3779b9 + (res << 6) + (res >> 2);
+  return res;
+}
+
+size_t hash<std::pair<std::string, samediff::Engine>>::operator()(
+    const std::pair<std::string, samediff::Engine>& k) const {
+  using std::hash;
+  auto res = std::hash<std::string>()(k.first);
+  res ^= std::hash<sd::LongType>()((sd::LongType)k.second) + 0x9e3779b9 + (res << 6) + (res >> 2);
+  return res;
+}
+}  // namespace std

libnd4j/include/ops/declarable/impl/OpTuple.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 11.10.2017.
+//
+#include "ops/declarable/OpTuple.h"
+
+sd::ops::OpTuple::OpTuple(const char *opName) { _opName = opName; }
+
+sd::ops::OpTuple::OpTuple(const char *opName, std::initializer_list<NDArray *> &&inputs,
+                          std::initializer_list<double> &&tArgs, std::initializer_list<LongType> &&iArgs) {
+  _opName = opName;
+  _inputs = inputs;
+  _iArgs = iArgs;
+  _tArgs = tArgs;
+}
+
+sd::ops::OpTuple::~OpTuple() {
+  for (auto v : _inputs) delete v;
+}
+
+sd::ops::OpTuple *sd::ops::OpTuple::addInput(NDArray *array) {
+  _inputs.emplace_back(array);
+  return this;
+}
+
+sd::ops::OpTuple *sd::ops::OpTuple::addOutput(NDArray *array) {
+  _outputs.emplace_back(array);
+  return this;
+}
+
+sd::ops::OpTuple *sd::ops::OpTuple::setTArgs(std::initializer_list<double> tArgs) {
+  _tArgs = tArgs;
+  return this;
+}
+
+sd::ops::OpTuple *sd::ops::OpTuple::setIArgs(std::initializer_list<LongType> iArgs) {
+  _iArgs = iArgs;
+  return this;
+}

libnd4j/include/ops/declarable/impl/PlatformHelper.cpp

@@ -0,0 +1,93 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include "../PlatformHelper.h"
+
+#include <graph/Variable.h>
+
+namespace sd {
+namespace ops {
+namespace platforms {
+PlatformHelper::PlatformHelper(const char* name, samediff::Engine engine) {
+  // we just store name/hash of target operation
+  _name = std::string(name);
+  _hash = HashHelper::getInstance().getLongHash(_name);
+  _engine = engine;
+}
+
+NDArray* PlatformHelper::getNullifiedZ(graph::Context& block, int inputId) {
+  auto result = getZ(block, inputId);
+  if (result != nullptr && !block.isInplace()) result->nullify();
+
+  return result;
+}
+
+NDArray* PlatformHelper::getZ(graph::Context& ctx, int inputId) {
+  NDArray* z = nullptr;
+
+  if (ctx.isFastPath()) {
+    if (ctx.fastpath_out().size() <= static_cast<size_t>(inputId)) {
+      if (ctx.isInplace()) {
+        z = ctx.fastpath_in()[inputId];
+      } else
+        THROW_EXCEPTION("fastpath_out: unresolved output array");
+    } else {
+      z = ctx.fastpath_out()[inputId];
+    }
+  } else {
+    std::pair<int, int> pair(ctx.nodeId(), inputId);
+
+    if (ctx.isInplace()) {
+      z = ctx.variable(inputId)->getNDArray();
+
+      // hypothetically it's possible to have no variable. chances are low, but who knows. let's just create it for now
+      if (!ctx.getVariableSpace()->hasVariable(pair)) {
+        auto var = new graph::Variable();
+        ctx.getVariableSpace()->putVariable(pair, var);
+      }
+
+      // now we're saving input array as output array
+      auto var = ctx.getVariableSpace()->getVariable(pair);
+      var->markRemovable(false);
+      var->setNDArray(z);
+    } else if (!ctx.isInplace()) {
+      auto var = ctx.variable(pair);
+      if (var->getNDArray() != nullptr && var->getNDArray()->nonNull()) {
+        z = var->getNDArray();
+      } else {
+        sd_printf("Can't get Z variable for node_%i!\n", ctx.nodeId());
+      }
+    } else {
+      THROW_EXCEPTION("Failed execution after attempting to get result outside of fast_path. This should not happen.\n");
+    }
+  }
+
+  return z;
+}
+
+samediff::Engine PlatformHelper::engine() { return _engine; }
+
+std::string PlatformHelper::name() { return _name; }
+
+LongType PlatformHelper::hash() { return _hash; }
+}  // namespace platforms
+}  // namespace ops
+}  // namespace sd