chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/generic/reduce/argamax.cpp

@@ -0,0 +1,95 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+// Created by Abdelrauf 2020 (based on argmax)
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_argamax)
+
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/reductions.h>
+
+namespace sd {
+namespace ops {
+DECLARE_TYPES(argamax) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setAllowedOutputTypes({ALL_INTS});
+}
+
+CUSTOM_OP_IMPL(argamax, 1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (output->isEmpty()) return Status::OK;
+
+  auto axis = *block.getIArguments();
+
+  // axis might be dynamic (i.e. tf mode)
+  if (block.width() > 1 && axis.size() == 0) {
+    auto axisVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axisVector, axis);
+    helpers::argAbsMax(*input, *output, axis);
+  } else {
+    helpers::argAbsMax(*input, *output, axis);
+  }
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(argamax) {
+  std::vector<LongType> dims;
+
+  if (block.width() == 1) {
+    dims = *block.getIArguments();
+  } else {
+    auto y = INPUT_VARIABLE(1);
+    dims = y->template asVectorT<LongType>();
+  }
+
+  auto keepDims = block.numB() ? B_ARG(0) : false;
+  auto dtype = block.numD() ? D_ARG(0) : INT64;
+
+  // we're resolving negative axis here
+  helpers::adjustAxis(shape::rank(inputShape->at(0)), dims);
+
+  auto in = inputShape->at(0);
+  for (auto d : dims) {
+    // we have special case here
+    if (d == DataTypeUtils::max<int>()) continue;
+
+    REQUIRE_TRUE(d < shape::rank(in), 0, "ArgAmax: axis can't be above rank")
+    REQUIRE_TRUE(in[d + 1] != 0, 0, "ArgAmax: you can't reduce along axis with 0 in shape");
+  }
+
+  // special case - output is scalar
+  if (dims.empty() || (dims.size() == 1 && dims.at(0) == DataTypeUtils::max<int>())) {
+    return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(dtype));
+  }
+
+  return SHAPELIST(
+      ShapeUtils::evalReduceShapeInfo('c', &dims, inputShape->at(0), dtype, keepDims, false, block.getWorkspace()));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/argamin.cpp

@@ -0,0 +1,95 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+// Created by Abdelrauf 2020 (based on argmax)
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_argamin)
+
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/reductions.h>
+
+namespace sd {
+namespace ops {
+DECLARE_TYPES(argamin) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setAllowedOutputTypes({ALL_INTS});
+}
+
+CUSTOM_OP_IMPL(argamin, 1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (output->isEmpty()) return Status::OK;
+
+  auto axis = *block.getIArguments();
+
+  // axis might be dynamic (i.e. tf mode)
+  if (block.width() > 1 && axis.size() == 0) {
+    auto axisVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axisVector, axis);
+    helpers::argAbsMin(*input, *output, axis);
+  } else {
+    helpers::argAbsMin(*input, *output, axis);
+  }
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(argamin) {
+  std::vector<LongType> dims;
+
+  if (block.width() == 1) {
+    dims = *block.getIArguments();
+  } else {
+    auto y = INPUT_VARIABLE(1);
+    dims = y->template asVectorT<LongType>();
+  }
+
+  auto keepDims = block.numB() ? B_ARG(0) : false;
+  auto dtype = block.numD() ? D_ARG(0) : INT64;
+
+  // we're resolving negative axis here
+  helpers::adjustAxis(shape::rank(inputShape->at(0)), dims);
+
+  auto in = inputShape->at(0);
+  for (auto d : dims) {
+    // we have special case here
+    if (d == DataTypeUtils::max<int>()) continue;
+
+    REQUIRE_TRUE(d < shape::rank(in), 0, "ArgAmin: axis can't be above rank")
+    REQUIRE_TRUE(in[d + 1] != 0, 0, "ArgAmin: you can't reduce along axis with 0 in shape");
+  }
+
+  // special case - output is scalar
+  if (dims.empty() || (dims.size() == 1 && dims.at(0) == DataTypeUtils::max<int>())) {
+    return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(dtype));
+  }
+
+  return SHAPELIST(
+      ShapeUtils::evalReduceShapeInfo('c', &dims, inputShape->at(0), dtype, keepDims, false, block.getWorkspace()));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/argmax.cpp

@@ -0,0 +1,102 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// Created by raver119 on 01.11.2017.
+// Modified by GS <sgazeos@gmail.com> 4/5/2018
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_argmax)
+
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/reductions.h>
+
+namespace sd {
+namespace ops {
+DECLARE_TYPES(argmax) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setAllowedOutputTypes({ANY});
+}
+
+CUSTOM_OP_IMPL(argmax, 1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (output->isEmpty() || output->lengthOf() < 1) return Status::OK;
+
+  auto axis = *block.getIArguments();
+
+  // axis might be dynamic (i.e. tf mode)
+  if (block.width() > 1 && axis.size() == 0) {
+    auto axisVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axisVector, axis);
+    helpers::argMax(*input, *output, axis);
+  } else {
+    helpers::argMax(*input, *output, axis);
+  }
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(argmax) {
+  auto firstInputShape = inputShape->at(0);
+  if(shape::isScalar(firstInputShape)) {
+    return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(DataType::INT64));
+  }
+  std::vector<LongType> dims;
+
+  if (block.width() == 1) {
+    dims = *block.getIArguments();
+  } else {
+    auto y = INPUT_VARIABLE(1)->cast(INT64);
+    dims = y->template asVectorT<LongType>();
+  }
+
+  auto keepDims = block.numB() ? B_ARG(0) : false;
+  auto dtype = block.numD() ? D_ARG(0) : INT64;
+
+  // we're resolving negative axis here
+  helpers::adjustAxis(shape::rank(inputShape->at(0)), dims);
+
+
+  for (auto d : dims) {
+    // we have special case here
+    if (d == DataTypeUtils::max<int>()) continue;
+
+    REQUIRE_TRUE(d < shape::rank(firstInputShape), 0, "ArgMax: axis can't be above rank")
+    REQUIRE_TRUE(firstInputShape[d + 1] != 0, 0, "ArgMax: you can't reduce along axis with 0 in shape");
+  }
+
+  // special case - output is scalar
+  if (dims.empty() || (dims.size() == 1 && dims.at(0) == DataTypeUtils::max<int>())) {
+    auto ret = ConstantShapeHelper::getInstance().scalarShapeInfo(dtype);
+    return SHAPELIST(ret);
+  }
+
+  auto ret = ShapeUtils::evalReduceShapeInfo('c', &dims, firstInputShape, dtype, keepDims, false, block.getWorkspace());
+  return SHAPELIST(ret);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/argmin.cpp

@@ -0,0 +1,107 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 01.11.2017.
+// Modified by GS <sgazeos@gmail.com> 4/5/2018.
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_argmin)
+
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/reductions.h>
+
+namespace sd {
+namespace ops {
+
+DECLARE_TYPES(argmin) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setAllowedOutputTypes({ALL_INTS});
+}
+
+CUSTOM_OP_IMPL(argmin, 1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto axis = *block.getIArguments();
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (output->isEmpty()) return Status::OK;
+
+  // axis might be dynamic (i.e. tf mode)
+  if (block.width() > 1 && axis.size() == 0) {
+    auto axisVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axisVector, axis);
+    helpers::argMin(*input, *output, axis);
+  } else {
+    helpers::argMin(*input, *output, axis);
+  }
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(argmin) {
+  auto firstInputShape = inputShape->at(0);
+  if(shape::isScalar(firstInputShape)) {
+    return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(DataType::INT64));
+  }
+
+  std::vector<LongType> dims;
+
+  if (block.width() == 1) {
+    dims = *block.getIArguments();
+  } else {
+    auto y = INPUT_VARIABLE(1);
+    dims = y->template asVectorT<LongType>();
+  }
+
+
+
+  auto keepDims = block.numB() ? B_ARG(0) : false;
+  auto dtype = block.numD() ? D_ARG(0) : INT64;
+
+  // we're resolving negative axis here
+  helpers::adjustAxis(shape::rank(inputShape->at(0)), dims);
+
+  auto in = inputShape->at(0);
+
+
+
+  for (auto d : dims) {
+    // we have special case here
+    if (d == DataTypeUtils::max<int>()) continue;
+
+    REQUIRE_TRUE(d < shape::rank(in), 0, "ArgMin: axis can't be above rank")
+    REQUIRE_TRUE(in[d + 1] != 0, 0, "ArgMin: you can't reduce along axis with 0 in shape");
+  }
+
+  // special case - output is scalar
+  if (dims.empty() || (dims.size() == 1 && dims.at(0) == DataTypeUtils::max<int>())) {
+    return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(dtype));
+  }
+
+  return SHAPELIST(
+      ShapeUtils::evalReduceShapeInfo('c', &dims, inputShape->at(0), dtype, keepDims, false, block.getWorkspace()));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/norm.cpp

@@ -0,0 +1,99 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_norm)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+namespace sd {
+namespace ops {
+REDUCTION_OP_IMPL(norm, 1, 1, false, 1, -2) {
+  auto input = INPUT_VARIABLE(0);
+  NDArray *output = OUTPUT_VARIABLE(0);
+
+  auto mode = (int)T_ARG(0);
+  std::vector<sd::LongType> dims = *block.getIArguments();
+  bool overwrite = false;
+
+  if (block.width() == 1) {
+    output = OUTPUT_VARIABLE(0);
+  } else {
+    auto axisVector = INPUT_VARIABLE(1);
+    dims.resize(axisVector->lengthOf());
+    helpers::adjustAxis(input->rankOf(), axisVector, dims);
+    auto shape = ShapeUtils::evalReduceShapeInfo(input->ordering(), &dims, *input, false, false);
+    if (!shape::equalsStrict(shape, output->shapeInfo())) {
+      output = new NDArray(shape, false, block.launchContext());
+      overwrite = true;
+    }
+  }
+  switch (mode) {
+    case 0: {
+      REQUIRE_TRUE(dims.size() == 2 || (input->rankOf() == 2 && dims.size() == 0), 0,
+                   "Norm: Frobenius is defined for 2D matrices or TADS only");
+      // fro
+      input->reduceAlongDimension(reduce::NormFrobenius, output, &dims, false, output->rankOf() == 2);
+    } break;
+    case 1: {
+      // euclidean
+      if ((input->rankOf() == 2 && dims.size() == 0) || dims.size() == 2) {
+        input->reduceAlongDimension(reduce::NormFrobenius, output, &dims, false, output->rankOf() == 2);
+      } else {
+        input->reduceAlongDimension(reduce::Norm2, output, &dims, false, output->rankOf() == 2);
+      }
+    } break;
+    case 2: {
+      // 1
+      input->reduceAlongDimension(reduce::Norm1, output, &dims, false, output->rankOf() == 2);
+    } break;
+    case 3: {
+      // 2
+      input->reduceAlongDimension(reduce::Norm2, output, &dims, false, output->rankOf() == 2);
+    } break;
+    case 4: {
+      // inf-norm
+      input->reduceAlongDimension(reduce::NormMax, output, &dims, false, output->rankOf() == 2);
+    } break;
+    default: {
+      // p-norm
+      REQUIRE_TRUE(block.getIArguments()->size() > 1, 0,
+                   "P-Norm reductions requires 2 TArguments, but only 1 was provided");
+      // FIXME: p is required here
+      // T p = T_ARG(1);
+      input->reduceAlongDimension(reduce::NormP, output, &dims, false, output->rankOf() == 2);
+    }
+  }
+
+  if (overwrite) {
+    OVERWRITE_RESULT(output);
+  }
+
+  return sd::Status::OK;
+};
+
+DECLARE_TYPES(norm) { getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduceMean.cpp

@@ -0,0 +1,193 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 01.06.2018
+//
+#include <ops/declarable/CustomOperations.h>
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_reduce_mean)
+#include <ops/declarable/helpers/axis.h>
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_mean, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_MEAN OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto &item : dimensions) {
+    REQUIRE_TRUE(item >= -input->rankOf() && item < input->rankOf(), 0,
+                 "REDUCE_MEAN OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+                 input->rankOf(), input->rankOf(), item);
+  }
+
+  input->reduceAlongDimension(reduce::Mean, output, &dimensions, keepDims);
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_mean) {
+  auto dimensions = *block.getIArguments();
+  auto in = inputShape->at(0);
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(shape::rank(in), axesVector, dimensions);
+  }
+
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(in[0]), 0,
+      "REDUCE_MEAN OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto &item : dimensions)
+  REQUIRE_TRUE(item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+               "REDUCE_MEAN OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+               inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  auto outShapeInfo =
+      ShapeUtils::evalReduceShapeInfo(shape::order(in), &dimensions, in, keepDims, false, block.getWorkspace());
+
+  return SHAPELIST(outShapeInfo);
+}
+
+DECLARE_TYPES(reduce_mean) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_mean_bp, -2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_MEAN_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  auto dimLength = 1.0;
+  for (const auto &item : dimensions) {
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_MEAN_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+    dimLength *= input->sizeAt(item);
+  }
+
+  if (gradO->isScalar()) {
+    if (dimensions.size() > 0) {
+      NDArray *assign = gradO->e(0) / (static_cast<double>(dimLength));
+      gradI->assign(assign);
+      delete assign;
+    } else {
+      NDArray *assign = gradO->e(0) / (static_cast<double>(input->lengthOf()));
+      gradI->assign(assign);
+      delete assign;
+    }
+
+  } else {
+    auto val = (static_cast<double>(gradO->lengthOf() < 1 ? 1.0 : gradO->lengthOf()) )
+               / (static_cast<double>(input->lengthOf() < 1 ? 1.0 : input->lengthOf()));
+    if(val == 0.0)
+      val = SD_EPSILON;
+    gradI->assign(val);
+    if (!keepDims) {
+      auto gradOShapeKeepDims =
+          ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *input, true, false, block.getWorkspace());
+
+      std::vector<sd::LongType> shape =  ShapeUtils::pullShapeFromShapeInfo(
+          gradOShapeKeepDims);
+      NDArray *reshapedGradO = gradO->reshape(gradO->ordering(), shape);
+      *gradI *= *reshapedGradO;
+      delete reshapedGradO;
+    } else {
+      gradI->applyTrueBroadcast(sd::BroadcastOpsTuple::Multiply(), gradO, gradI);
+    }
+  }
+
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_mean_bp) {
+  auto in = inputShape->at(0);
+  auto dimensions = *block.getIArguments();
+  auto rank = shape::rank(in);
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(rank, axesVector, dimensions);
+  }
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(rank), 0,
+      "REDUCE_MEAN_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto &item : dimensions)
+  REQUIRE_TRUE(
+      item >= -rank || item < rank, 0,
+      "REDUCE_MEAN_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !", rank,
+      rank, item);
+
+  sd::LongType *gradIshapeInfo = new sd::LongType[shape::shapeInfoLength(rank)];
+  memcpy(gradIshapeInfo, in, shape::shapeInfoByteLength(in));
+  auto ret =  SHAPELIST(CONSTANT(gradIshapeInfo));
+  delete[] gradIshapeInfo;
+  return ret;
+}
+
+DECLARE_TYPES(reduce_mean_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduceStDev.cpp

@@ -0,0 +1,208 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 04.06.2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/reductions.h>
+#if NOT_EXCLUDED(OP_reduce_stdev)
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_stdev, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  //numpy compat: default is 1 for 0 length arrays https://stackoverflow.com/questions/66746566/numpy-explanation-of-numpy-prod
+  if(input->lengthOf() == 0) {
+    int one = 1;
+    output->assign(one);
+    return sd::Status::OK;
+  }
+  bool biasCorrected = false;  // block.getTArguments()->size() > 1 ? (bool)T_ARG(1) : false;
+
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size()) {
+    if (block.getBArguments()->size() > 1) biasCorrected = B_ARG(1);
+  } else if (block.getTArguments()->size()) {
+    if (block.getTArguments()->size() > 1) biasCorrected = (bool)T_ARG(1);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_STDEV OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_STDEV OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  sd::ops::helpers::standardDeviation(*input, *output, dimensions, biasCorrected);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_stdev) {
+  auto in = inputShape->at(0);
+  auto rank = shape::rank(in);
+  bool keepDims = false;  // block.getTArguments()->size() > 0 ? (bool)T_ARG(0) : false;
+  auto dimensions = *block.getIArguments();
+
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(rank, axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size()) {
+    keepDims = B_ARG(0);
+  } else if (block.getTArguments()->size()) {
+    keepDims = (bool)T_ARG(0);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(rank), 0,
+      "REDUCE_STDEV OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_STDEV OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  auto outShapeInfo =
+      ShapeUtils::evalReduceShapeInfo(shape::order(in), &dimensions, in, keepDims, false, block.getWorkspace());
+
+  return SHAPELIST(outShapeInfo);
+}
+
+DECLARE_TYPES(reduce_stdev) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_stdev_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  bool keepDims = false;       // block.getTArguments()->size() > 0 ? (bool)T_ARG(0) : false;
+  bool biasCorrected = false;  // block.getTArguments()->size() > 1 ? (bool)T_ARG(1) : false;
+
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size()) {
+    keepDims = B_ARG(0);
+    if (block.getBArguments()->size() > 1) biasCorrected = B_ARG(1);
+  } else if (block.getTArguments()->size()) {
+    keepDims = (bool)T_ARG(0);
+    if (block.getTArguments()->size() > 1) biasCorrected = (bool)T_ARG(1);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_STDEV_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_STDEV_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  auto gradOLen = gradO->lengthOf() < 1 ? 1 : gradO->lengthOf();
+  const sd::LongType N = input->lengthOf() / gradOLen;
+  const sd::LongType NminusOne = biasCorrected ? N - 1 : N;
+
+  auto* mean = input->reduceAlongDimension(reduce::Mean, &dimensions, true);
+
+  NDArray variance(mean->shapeInfo(), true,
+                   block.launchContext());  // create empty array with shape matching shape of mean array
+  input->varianceAlongDimension(variance::SummaryStatsStandardDeviation, variance, biasCorrected, &dimensions);
+
+  sd::ops::divide_no_nan divideNoNan;
+  auto* inputMinusMean = (*input) - (*mean);
+  delete mean;
+  auto* varianceTimesNMinusOne = variance * NminusOne;
+  divideNoNan.execute({inputMinusMean, varianceTimesNMinusOne}, {gradI});
+  delete inputMinusMean;
+  delete varianceTimesNMinusOne;
+
+  if (!keepDims) {
+    auto gradOShapeKeepDims =
+        ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *input, true, false, block.getWorkspace());
+    if (!gradO->isScalar()) {
+      std::vector<sd::LongType> shape =  ShapeUtils::pullShapeFromShapeInfo(
+          gradOShapeKeepDims);
+      auto* reshaped = gradO->reshape(gradO->ordering(), shape);
+      *gradI *= (*reshaped);  // for example could be something like [a,b] -> [1,a,1,b]
+      delete reshaped;
+    } else {
+      *gradI *= (*gradO);  // for example could be something like [a,b] -> [1,a,1,b]
+    }
+  } else {
+    *gradI *= (*gradO);  // automatic broadcasting happens here
+  }
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_stdev_bp) {
+  auto in = inputShape->at(0);
+  auto rank = shape::rank(in);
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(rank, axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(rank), 0,
+      "REDUCE_STDEV_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_STDEV_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  return SHAPELIST(CONSTANT(in));
+}
+
+DECLARE_TYPES(reduce_stdev_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduceVariance.cpp

@@ -0,0 +1,196 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 04.06.2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/reductions.h>
+#if NOT_EXCLUDED(OP_reduce_variance)
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_variance, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  bool biasCorrected = false;  // block.getTArguments()->size() > 1 ? (bool)T_ARG(1) : false;
+
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size()) {
+    if (block.getBArguments()->size() > 1) biasCorrected = B_ARG(1);
+  } else if (block.getTArguments()->size()) {
+    if (block.getTArguments()->size() > 1) biasCorrected = (bool)T_ARG(1);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_VARIANCE OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+  REQUIRE_TRUE(
+      item >= -input->rankOf() && item < input->rankOf(), 0,
+      "REDUCE_VARIANCE OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+      input->rankOf(), input->rankOf(), item);
+
+  sd::ops::helpers::variance(*input, *output, dimensions, biasCorrected);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_variance) {
+  bool keepDims = false;  // block.getTArguments()->size() > 0 ? (bool)T_ARG(0) : false;
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size()) {
+    keepDims = B_ARG(0);
+  } else if (block.getTArguments()->size()) {
+    keepDims = (bool)T_ARG(0);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(INPUT_VARIABLE(0)->rankOf()), 0,
+      "REDUCE_VARIANCE OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+  REQUIRE_TRUE(
+      item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+      "REDUCE_VARIANCE OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+      inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  auto outShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &dimensions, inputShape->at(0),
+                                                      keepDims, false, block.getWorkspace());
+
+  return SHAPELIST(outShapeInfo);
+}
+
+DECLARE_TYPES(reduce_variance) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_variance_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+
+  auto gradI = OUTPUT_VARIABLE(0);
+  bool keepDims = true;
+  bool biasCorrected = false;
+
+
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size()) {
+    keepDims = B_ARG(0);
+    if (block.getBArguments()->size() > 1) biasCorrected = B_ARG(1);
+  } else if (block.getTArguments()->size()) {
+    keepDims = (bool)T_ARG(0);
+    if (block.getTArguments()->size() > 1) biasCorrected = (bool)T_ARG(1);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_VARIANCE_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions) {
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_VARIANCE_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+    sd_debug("Dimension item is %d\n", item);
+  }
+
+  auto inputLen = input->lengthOf();
+  //avoid divide by zero
+  auto grad0Length = gradO->isScalar() || gradO->lengthOf() < 1 ? 1  :  gradO->lengthOf();
+  const sd::LongType N = inputLen / grad0Length;
+  const sd::LongType NminusOne = biasCorrected ? N - 1 : N;
+  
+  // Break down: (*input - mean) * (2.0f / NminusOne)
+  auto* mean = input->reduceAlongDimension(reduce::Mean, &dimensions, true);
+  auto* inputMinusMean = (*input) - (*mean);
+  delete mean;
+  auto* assign = (*inputMinusMean) * (2.0f / NminusOne);
+  delete inputMinusMean;
+  
+  gradI->assign(assign);  // automatic broadcasting happens here
+  delete assign;
+  
+  if (!keepDims) {
+    auto gradOShapeKeepDims = ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *input, true, false, block.getWorkspace());
+    auto grad0Shape = ShapeUtils::pullShapeFromShapeInfo(gradOShapeKeepDims);
+    auto* reshaped = !gradO->isScalar() ? gradO->reshape(gradO->ordering(), grad0Shape) : gradO;  // for example could be something like [a,b] -> [1,a,1,b];
+    *gradI *= (*reshaped);  // for example could be something like [a,b] -> [1,a,1,b]
+    //reshape can vary and may have the same buffer as the original
+    if(reshaped != gradO && reshaped->buffer() != gradO->buffer() && reshaped->specialBuffer() != gradI->specialBuffer())
+      delete reshaped;
+
+  } else {
+    *gradI *= (*gradO);  // automatic broadcasting happens here
+  }
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_variance_bp) {
+  auto in = inputShape->at(0);
+  auto rank = shape::rank(in);
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(rank, axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(rank), 0,
+      "REDUCE_VARIANCE_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+  REQUIRE_TRUE(
+      item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+      "REDUCE_VARIANCE_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+      inputShape->at(0)[0], inputShape->at(0)[0], item);
+  return SHAPELIST(CONSTANT(in));
+}
+
+DECLARE_TYPES(reduce_variance_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduce_dot.cpp

@@ -0,0 +1,147 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/1/2018.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+#if NOT_EXCLUDED(OP_reduce_dot_bp)
+
+namespace sd {
+namespace ops {
+
+////////////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_dot_bp, -1, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto gradO = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  // L(x,y) = SUM(x_i * y_i)
+  // dL/dx_i = y_i
+
+  REQUIRE_TRUE(x->isSameShape(y), 0,
+               "REDUCE_DOT_BP OP: both input arrays x and y should have same shapes, but got %s and %s correspondingly",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+  if (gradO->lengthOf() == 1) {  // scalar of reduced to scalar with keep dimensions
+    auto* assign1 = (*y) * (*gradO);
+    gradX->assign(assign1);
+    delete assign1;
+    
+    auto* assign2 = (*x) * (*gradO);
+    gradY->assign(assign2);
+    delete assign2;
+  } else {
+    bool keepDims = false;
+    auto dimensions = *block.getIArguments();
+
+    if (block.width() > 3) {
+      auto axesVector = INPUT_VARIABLE(3);
+      helpers::adjustAxis(x->rankOf(), axesVector, dimensions);
+    }
+
+    if (block.getBArguments()->size())
+      keepDims = B_ARG(0);
+    else if (block.getTArguments()->size())
+      keepDims = (bool)T_ARG(0);
+
+    REQUIRE_TRUE(
+        dimensions.size() <= static_cast<size_t>(x->rankOf()), 0,
+        "REDUCE_DOT_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+        dimensions.size());
+
+    for (const auto& item : dimensions)
+      REQUIRE_TRUE(
+          item >= -x->rankOf() && item < x->rankOf(), 0,
+          "REDUCE_DOT_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+          x->rankOf(), x->rankOf(), item);
+
+    if (!keepDims) {
+      auto gradOShapeKeepDims =
+          ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *x, true, false, block.getWorkspace());
+      std::vector<sd::LongType> shape =  ShapeUtils::pullShapeFromShapeInfo(
+          gradOShapeKeepDims);
+      auto* r = gradO->reshape(gradO->ordering(), shape);  // for example could be something like [a,b] -> [1,a,1,b]
+
+      // First case - for gradX
+      auto* gradXTemp1 = (*y) * (*r);
+      gradX->assign(gradXTemp1);
+      delete gradXTemp1;
+
+      // First case - for gradY
+      auto* gradYTemp1 = (*x) * (*r);
+      gradY->assign(gradYTemp1);
+      delete gradYTemp1;
+      
+      delete r;
+
+    } else {
+      // Second case - for gradX
+      auto* gradXTemp2 = (*y) * (*gradO);
+      gradX->assign(gradXTemp2);
+      delete gradXTemp2;
+
+      // Second case - for gradY
+      auto* gradYTemp2 = (*x) * (*gradO);
+      gradY->assign(gradYTemp2);
+      delete gradYTemp2;
+    }
+  }
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_dot_bp) {
+  if (shape::length(inputShape->at(2)) > 1) {
+    auto dimensions = *block.getIArguments();
+
+    if (block.width() > 3) {
+      auto axesVector = INPUT_VARIABLE(3);
+      helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+    }
+
+
+    REQUIRE_TRUE(
+        dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+        "REDUCE_DOT_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+        dimensions.size());
+
+    for (const auto& item : dimensions)
+      REQUIRE_TRUE(
+          item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+          "REDUCE_DOT_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+          inputShape->at(0)[0], inputShape->at(0)[0], item);
+  }
+
+  return SHAPELIST(CONSTANT(inputShape->at(0)), CONSTANT(inputShape->at(1)));
+}
+
+DECLARE_TYPES(reduce_dot_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduce_logsumexp.cpp

@@ -0,0 +1,80 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 11/13/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+namespace sd {
+namespace ops {
+#if NOT_EXCLUDED(OP_reduce_logsumexp)
+
+CUSTOM_OP_IMPL(reduce_logsumexp, -1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  std::vector<sd::LongType> axes;  // = *block.getIArguments();
+  if (block.width() > 1) {
+    auto axisVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axisVector, axes);
+  } else if (block.getIArguments()->size() > 0) {
+    axes = *block.getIArguments();
+  }
+
+  for (const auto& item : axes)
+    REQUIRE_TRUE(
+        item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+        "REDUCE_LOGSUMEXP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  const bool keepDims = block.getTArguments()->size() > 0 ? (bool)T_ARG(0) : false;
+  sd::LongType maxI = input->argMax();
+  auto maxVals = input->e(maxI);
+  // void* whereMax = (void*)();
+  auto internal = (*input);
+  internal -= maxVals;
+  internal.applyTransform(transform::Exp, &internal);
+  internal.reduceAlongDimension(reduce::Sum, output, &axes, keepDims, false);  //, (void*)&maxVals);
+  output->applyTransform(transform::Log, output);
+  (*output) += maxVals;
+  return sd::Status::OK;
+}
+DECLARE_TYPES(reduce_logsumexp) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS});
+}
+DECLARE_SHAPE_FN(reduce_logsumexp) {
+  const bool keepDims = block.getTArguments()->size() > 0 ? (bool)T_ARG(0) : false;
+  auto input = INPUT_VARIABLE(0);
+
+  std::vector<sd::LongType> axes;  // = *block.getIArguments();
+  if (block.width() > 1) {
+    auto axisVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axisVector, axes);
+  } else if (block.getIArguments()->size() > 0) {
+    axes = *block.getIArguments();
+  }
+
+  auto outShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &axes, inputShape->at(0),
+                                                      keepDims, false, block.getWorkspace());
+
+  return SHAPELIST(outShapeInfo);
+}
+#endif
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/generic/reduce/reduce_max.cpp

@@ -0,0 +1,181 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/1/2018.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/transforms.h>
+
+#if NOT_EXCLUDED(OP_reduce_max)
+
+namespace sd {
+namespace ops {
+
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_max, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  //numpy compat: default is 1 for 0 length arrays https://stackoverflow.com/questions/66746566/numpy-explanation-of-numpy-prod
+  if(input->lengthOf() == 0) {
+    int one = 1;
+    output->assign(one);
+    return sd::Status::OK;
+  }
+  std::vector<sd::LongType> dimensions = *block.getIArguments();
+
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_MAX OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+                 "REDUCE_MAX OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+                 input->rankOf(), input->rankOf(), item);
+
+  bool keepDims = false;  //: false;
+  if (block.getBArguments()->size() > 0)
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size() > 0)
+    keepDims = (bool)T_ARG(0);
+
+  input->reduceAlongDimension(reduce::Max, output, &dimensions, keepDims);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_max) {
+  bool keepDims = false;  //: false;
+
+  if (block.getBArguments()->size() > 0)
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size() > 0)
+    keepDims = (bool)T_ARG(0);
+
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_MAX OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+                 "REDUCE_MAX OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+                 inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  auto outShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &dimensions, inputShape->at(0),
+                                                      keepDims, false, block.getWorkspace());
+
+  return SHAPELIST(outShapeInfo);
+}
+
+DECLARE_TYPES(reduce_max) { getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setSameMode(true); }
+
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_max_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  std::vector<sd::LongType> dimensions = *block.getIArguments();
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_MAX_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+        "REDUCE_MAX_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  // *** calculations *** //
+
+  *gradI = 0;
+
+  if (gradO->lengthOf() == 1) {
+    auto indOfMaxElem = input->indexReduceNumber(sd::indexreduce::IndexMax);
+    NDArray right2 = gradO->e(0);
+    gradI->p(indOfMaxElem->t<sd::LongType>(0),&right2);
+    delete indOfMaxElem;
+
+  } else {
+    auto indicesArr = input->applyIndexReduce(sd::indexreduce::IndexMax, &dimensions);
+  auto vec = ShapeUtils::evalDimsToExclude(gradI->rankOf(), dimensions.size(),dimensions.data());
+    helpers::scatterSimple(
+        block.launchContext(), 6, *gradI, *gradO, *indicesArr,
+        *vec);  // 6 corresponds to copy operation
+    delete vec;
+    delete indicesArr;
+  }
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_max_bp) {
+  std::vector<sd::LongType> dimensions = *block.getIArguments();
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_MAX_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_MAX_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+  return SHAPELIST(CONSTANT(inputShape->at(0)));
+}
+
+DECLARE_TYPES(reduce_max_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduce_min.cpp

@@ -0,0 +1,174 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/6/2018.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/transforms.h>
+
+#if NOT_EXCLUDED(OP_reduce_min)
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_min, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  std::vector<sd::LongType> dimensions = *block.getIArguments();
+
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_MIN OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+                 "REDUCE_MIN OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+                 input->rankOf(), input->rankOf(), item);
+
+  bool keepDims = false;  //: false;
+  if (block.getBArguments()->size() > 0)
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size() > 0)
+    keepDims = (bool)T_ARG(0);
+
+  input->reduceAlongDimension(reduce::Min, output, &dimensions, keepDims);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_min) {
+  bool keepDims = false;  //: false;
+
+  if (block.getBArguments()->size() > 0)
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size() > 0)
+    keepDims = (bool)T_ARG(0);
+
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_MIN OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+                 "REDUCE_MIN OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+                 inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  auto outShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &dimensions, inputShape->at(0),
+                                                      keepDims, false, block.getWorkspace());
+
+  return SHAPELIST(outShapeInfo);
+}
+
+DECLARE_TYPES(reduce_min) { getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setSameMode(true); }
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_min_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  std::vector<sd::LongType> dimensions = *block.getIArguments();
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_MIN_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+        "REDUCE_MIN_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  // *** calculations *** //
+
+  *gradI = 0;
+
+  if (gradO->lengthOf() == 1) {
+    auto indOfMaxElem = input->indexReduceNumber(sd::indexreduce::IndexMin);
+    auto right = gradO->e(0);
+    gradI->p(indOfMaxElem->e<sd::LongType>(0),&right);
+    delete indOfMaxElem;
+
+  } else {
+    auto indicesArr = input->applyIndexReduce(sd::indexreduce::IndexMin, &dimensions);
+    auto vec = ShapeUtils::evalDimsToExclude(gradI->rankOf(), dimensions.size(),dimensions.data());
+    helpers::scatterSimple(
+        block.launchContext(), 6, *gradI, *gradO, *indicesArr,
+        *vec);  // 6 corresponds to copy operation
+    delete vec;
+    delete indicesArr;
+  }
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_min_bp) {
+  std::vector<sd::LongType> dimensions = *block.getIArguments();
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_MIN_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_MIN_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+  return SHAPELIST(CONSTANT(inputShape->at(0)));
+}
+
+DECLARE_TYPES(reduce_min_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+
+}  // namespace ops
+}  // namespace sd
+
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduce_norm1.cpp

@@ -0,0 +1,183 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/4/2018.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+namespace sd {
+namespace ops {
+#if NOT_EXCLUDED(OP_reduce_norm1)
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_norm1, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_NORM1 OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+        "REDUCE_NORM1 OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  input->reduceAlongDimension(reduce::Norm1, output, &dimensions, keepDims);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_norm1) {
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_NORM1 OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_NORM1 OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  return SHAPELIST(ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &dimensions, inputShape->at(0),
+                                                   keepDims, false, block.getWorkspace()));
+}
+
+DECLARE_TYPES(reduce_norm1) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+#endif
+#if NOT_EXCLUDED(OP_reduce_norm1_bp)
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_norm1_bp, -1, 1, false, 0, 0) {
+  // L = Sum abs(x_i) for all i = 1 to N
+  // dL/dx_i = 1 if x_i >= 0 and -1 when x_i < 0
+  // out_i = epsilon_i if x_i > 0 and -epsilon_i when x_i < 0
+  // when gradO is non a scalar, using dimensions to split output onto gradO like parts
+  // and use LAMBDA with that formula for it.
+
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  input->applyTransform(sd::transform::Sign, gradI);
+
+  bool keepDims = false;
+  auto dimensions = *block.getIArguments();
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_NORM1_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_NORM1_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  // *** calculations *** //
+
+  if (!keepDims && gradO->lengthOf() > 1) {
+    auto gradOShapeKeepDims =
+        ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *input, true, false, block.getWorkspace());
+    std::vector<sd::LongType> shape =  ShapeUtils::pullShapeFromShapeInfo(
+        gradOShapeKeepDims);
+    auto reshaped = gradO->reshape(gradO->ordering(),
+                                   shape);
+    *gradI *= *reshaped;  // for example could be something like [a,b] -> [1,a,1,b]
+    delete reshaped;
+  } else
+    *gradI *= *gradO;
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_norm1_bp) {
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_NORM1_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_NORM1_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  return SHAPELIST(CONSTANT(inputShape->at(0)));
+}
+
+DECLARE_TYPES(reduce_norm1_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+#endif
+
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/generic/reduce/reduce_norm2.cpp

@@ -0,0 +1,184 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/4/2018.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+namespace sd {
+namespace ops {
+#if NOT_EXCLUDED(OP_reduce_norm2)
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_norm2, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_NORM2 OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+        "REDUCE_NORM2 OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  input->reduceAlongDimension(reduce::Norm2, output, &dimensions, keepDims);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_norm2) {
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_NORM2 OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_NORM2 OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  return SHAPELIST(ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &dimensions, inputShape->at(0),
+                                                   keepDims, false, block.getWorkspace()));
+}
+
+DECLARE_TYPES(reduce_norm2) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+#endif
+
+#if NOT_EXCLUDED(OP_reduce_norm2_bp)
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_norm2_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  gradI->assign(input);
+
+  bool keepDims = false;
+  auto dimensions = *block.getIArguments();
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_NORM2_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_NORM2_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  // *** calculations *** //
+
+  auto* norm2 = input->reduceAlongDimension(reduce::Norm2, &dimensions, true);
+  *gradI /= (*norm2);
+  delete norm2;
+
+  if (!keepDims && gradO->lengthOf() > 1) {
+    auto gradOShapeKeepDims =
+        ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *input, true, false, block.getWorkspace());
+    std::vector<sd::LongType> shape =  ShapeUtils::pullShapeFromShapeInfo(
+        gradOShapeKeepDims);
+    auto* reshaped = gradO->reshape(gradO->ordering(), shape);
+    *gradI *= (*reshaped);  // for example could be something like [a,b] -> [1,a,1,b]
+    delete reshaped;
+  } else
+    *gradI *= (*gradO);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_norm2_bp) {
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_NORM2_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_NORM2_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  sd::LongType* outShapeInfo;
+  COPY_SHAPE(inputShape->at(0), outShapeInfo);
+
+  return SHAPELIST(CONSTANT(outShapeInfo));
+}
+
+DECLARE_TYPES(reduce_norm2_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+#endif
+
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/generic/reduce/reduce_norm_max.cpp

@@ -0,0 +1,185 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/4/2018.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/transforms.h>
+
+#if NOT_EXCLUDED(OP_reduce_norm_max)
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_norm_max, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_NORM_MAX OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+        "REDUCE_NORM_MAX OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  input->reduceAlongDimension(reduce::NormMax, output, &dimensions, keepDims);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_norm_max) {
+  auto in = inputShape->at(0);
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_NORM_MAX OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_NORM_MAX OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  return SHAPELIST(
+      ShapeUtils::evalReduceShapeInfo(shape::order(in), &dimensions, in, keepDims, false, block.getWorkspace()));
+}
+
+DECLARE_TYPES(reduce_norm_max) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_norm_max_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  std::vector<sd::LongType> dimensions = *block.getIArguments();
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_NORM_MAX_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+        "REDUCE_NORM_MAX_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  // *** calculations *** //
+
+  *gradI = 0.0;
+
+  if (gradO->lengthOf() == 1) {
+    auto* indOfAbsMaxElem = input->indexReduceNumber(sd::indexreduce::IndexAbsoluteMax);
+    const sd::LongType ind = indOfAbsMaxElem->t<sd::LongType>(0);
+    delete indOfAbsMaxElem;
+    
+    const int sign = input->e<float>(ind) >= 0 ? 1 : -1;
+    auto put = sign * gradO->e(0);
+    gradI->p(ind, put);
+    delete put;
+
+  } else {
+    auto indicesArr = input->applyIndexReduce(sd::indexreduce::IndexAbsoluteMax, &dimensions);
+    auto* vec = ShapeUtils::evalDimsToExclude(gradI->rankOf(), dimensions.size(), dimensions.data());
+    helpers::scatterSimple(
+        block.launchContext(), 6, *gradI, *gradO, *indicesArr,
+        *vec);  // 6 corresponds to copy operation
+    delete vec;
+   delete indicesArr;
+    
+    auto* signArr = input->transform(sd::transform::Sign);
+    *gradI *= (*signArr);
+    delete signArr;
+  }
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_norm_max_bp) {
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_NORM_MAX_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_NORM_MAX_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+
+  return SHAPELIST(CONSTANT(inputShape->at(0)));
+}
+
+DECLARE_TYPES(reduce_norm_max_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduce_prod.cpp

@@ -0,0 +1,200 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/1/2018.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+#if NOT_EXCLUDED(OP_reduce_prod)
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_prod, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  //numpy compat: default is 1 for 0 length arrays https://stackoverflow.com/questions/66746566/numpy-explanation-of-numpy-prod
+  if(input->lengthOf() == 0) {
+    int one = 1;
+    output->assign(one);
+    return sd::Status::OK;
+  }
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_PROD OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+  REQUIRE_TRUE(item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+               "REDUCE_PROD OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+               input->rankOf(), input->rankOf(), item);
+
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  input->reduceAlongDimension(reduce::Prod, output, &dimensions, keepDims);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_prod) {
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_PROD OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+  REQUIRE_TRUE(item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+               "REDUCE_PROD OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+               inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  return SHAPELIST(ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &dimensions, inputShape->at(0),
+                                                   keepDims, false, block.getWorkspace()));
+}
+
+DECLARE_TYPES(reduce_prod) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_prod_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  if (gradO->lengthOf() == 1) {
+    auto* assign = input->reduceNumber(sd::reduce::Prod);
+    gradI->assign(assign);
+    // FIXED: Check if view before deletion
+    if (assign != nullptr && !assign->isView()) {
+      delete assign;
+    }
+    *gradI /= *input;
+    *gradI *= gradO->e(0);
+  } else {
+    bool keepDims = false;
+    auto dimensions = *block.getIArguments();
+
+    if (block.width() > 2) {
+      auto axesVector = INPUT_VARIABLE(2);
+      helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+    }
+
+    if (block.getBArguments()->size())
+      keepDims = B_ARG(0);
+    else if (block.getTArguments()->size())
+      keepDims = (bool)T_ARG(0);
+
+    REQUIRE_TRUE(
+        dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+        "REDUCE_NORM1_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+        dimensions.size());
+
+    for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_NORM1_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+    // *** calculations *** //
+
+    auto* products = input->reduceAlongDimension(reduce::Prod, &dimensions, true);
+    gradI->applyTrueBroadcast(sd::BroadcastOpsTuple::Assign(), products, gradI);
+    // FIXED: Check if view before deletion
+    if (products != nullptr && !products->isView()) {
+      delete products;
+    }
+    *gradI /= *input;
+
+    if (!keepDims) {
+      auto gradOShapeKeepDims =
+          ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *input, true, false, block.getWorkspace());
+      std::vector<sd::LongType> shape =  ShapeUtils::pullShapeFromShapeInfo(
+          gradOShapeKeepDims);
+      auto* reshaped = gradO->reshape(gradO->ordering(), shape);
+      *gradI *= (*reshaped);  // for example could be something like [a,b] -> [1,a,1,b]
+      // FIXED: reshape() may return view - check before deletion
+      if (reshaped != nullptr && !reshaped->isView()) {
+        delete reshaped;
+      }
+    } else
+      *gradI *= (*gradO);
+  }
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_prod_bp) {
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_PROD_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+  REQUIRE_TRUE(
+      item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+      "REDUCE_PROD_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+      inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  return SHAPELIST(CONSTANT(inputShape->at(0)));
+}
+
+DECLARE_TYPES(reduce_prod_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduce_sqnorm.cpp

@@ -0,0 +1,199 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/4/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+
+#if NOT_EXCLUDED(OP_reduce_sqnorm)
+
+namespace sd {
+namespace ops {
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_sqnorm, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  bool keepDims = false;
+
+  auto dimensions = *block.getIArguments();
+
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_SQNORM OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_SQNORM OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  input->reduceAlongDimension(reduce::SquaredNorm, gradI, &dimensions, keepDims);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_sqnorm) {
+  auto dimensions = *block.getIArguments();
+  bool keepDims = false;
+
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_SQNORM OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_SQNORM OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  auto outShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &dimensions, inputShape->at(0),
+                                                      keepDims, false, block.getWorkspace());
+
+  return SHAPELIST(outShapeInfo);
+}
+
+DECLARE_TYPES(reduce_sqnorm) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_sqnorm_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  if (gradO->lengthOf() == 1) {
+    auto* temp1 = (*input) * 2.0;
+    auto* assign = (*temp1) * gradO->e(0);
+    delete temp1;
+    gradI->assign(assign);
+    delete assign;
+  } else {
+    bool keepDims = false;
+    auto dimensions = *block.getIArguments();
+
+    if (block.width() > 2) {
+      auto axesVector = INPUT_VARIABLE(2);
+      helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+    }
+
+    if (block.getBArguments()->size())
+      keepDims = B_ARG(0);
+    else if (block.getTArguments()->size())
+      keepDims = (bool)T_ARG(0);
+
+    REQUIRE_TRUE(
+        dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+        "REDUCE_SQNORM_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+        dimensions.size());
+
+    for (const auto& item : dimensions)
+      REQUIRE_TRUE(
+          item >= -input->rankOf() && item < input->rankOf(), 0,
+          "REDUCE_SQNORM_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+          input->rankOf(), input->rankOf(), item);
+
+    // *** calculations *** //
+
+    if (!keepDims) {
+      auto gradOShapeKeepDims =
+          ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *input, true, false, block.getWorkspace());
+      std::vector<sd::LongType> shape =  ShapeUtils::pullShapeFromShapeInfo(
+          gradOShapeKeepDims);
+      auto* reshaped = gradO->reshape(gradO->ordering(), shape);
+      
+      // Break down: 2. * (*input) * *reshaped
+      auto* temp1 = (*input) * 2.0;
+      auto* gradITemp1 = (*temp1) * (*reshaped);
+      delete temp1;
+      delete reshaped;
+      
+      gradI->assign(gradITemp1);
+      delete gradITemp1;
+    } else {
+      // Break down: 2. * (*input) * *gradO
+      auto* temp2 = (*input) * 2.0;
+      auto* gradITemp2 = (*temp2) * (*gradO);
+      delete temp2;
+      
+      gradI->assign(gradITemp2);
+      delete gradITemp2;
+    }
+  }
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_sqnorm_bp) {
+  if (shape::length(inputShape->at(1)) > 1) {
+    auto dimensions = *block.getIArguments();
+    if (block.width() > 2) {
+      auto axesVector = INPUT_VARIABLE(2);
+      helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+    }
+
+    REQUIRE_TRUE(
+        dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+        "REDUCE_SQNORM_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+        dimensions.size());
+
+    for (const auto& item : dimensions)
+      REQUIRE_TRUE(
+          item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+          "REDUCE_SQNORM_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+          inputShape->at(0)[0], inputShape->at(0)[0], item);
+  }
+
+  return SHAPELIST(CONSTANT(inputShape->at(0)));
+}
+
+DECLARE_TYPES(reduce_sqnorm_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/reduce/reduce_sum.cpp

@@ -0,0 +1,170 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 6/1/2018.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+#if NOT_EXCLUDED(OP_reduce_sum)
+namespace sd {
+namespace ops {
+
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_sum, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_SUM OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(item >= -input->shapeInfo()[0] && item < input->shapeInfo()[0], 0,
+                 "REDUCE_SUM OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+                 input->rankOf(), input->rankOf(), item);
+
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  input->reduceAlongDimension(reduce::Sum, output, &dimensions, keepDims);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_sum) {
+  bool keepDims = false;
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  std::vector<sd::LongType> dimensions;
+  if (block.width() > 1) {
+    auto axesVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  } else if (block.getIArguments()->size())
+    dimensions = *block.getIArguments();
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_SUM OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+                 "REDUCE_SUM OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+                 inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+  return SHAPELIST(ShapeUtils::evalReduceShapeInfo(shape::order(inputShape->at(0)), &dimensions, inputShape->at(0),
+                                                   keepDims, false, block.getWorkspace()));
+}
+
+DECLARE_TYPES(reduce_sum) { getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setSameMode(true); }
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(reduce_sum_bp, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  bool keepDims = false;
+  auto dimensions = *block.getIArguments();
+
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(input->rankOf(), axesVector, dimensions);
+  }
+
+  if (block.getBArguments()->size())
+    keepDims = B_ARG(0);
+  else if (block.getTArguments()->size())
+    keepDims = (bool)T_ARG(0);
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(input->rankOf()), 0,
+      "REDUCE_SUM_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -input->rankOf() && item < input->rankOf(), 0,
+        "REDUCE_SUM_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        input->rankOf(), input->rankOf(), item);
+
+  // *** calculations *** //
+
+  if (!keepDims) {
+    auto gradOShapeKeepDims =
+        ShapeUtils::evalReduceShapeInfo(gradO->ordering(), &dimensions, *input, true, false, block.getWorkspace());
+    std::vector<sd::LongType> shape =  ShapeUtils::pullShapeFromShapeInfo(
+        gradOShapeKeepDims);
+    auto r = gradO->reshape(gradO->ordering(),
+                            shape);  // for example could be something like [a,b] -> [1,a,1,b]
+    gradI->applyTrueBroadcast(sd::BroadcastOpsTuple::Assign(), r, gradI);
+    delete r;
+  } else
+    gradI->applyTrueBroadcast(sd::BroadcastOpsTuple::Assign(), gradO, gradI);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(reduce_sum_bp) {
+  auto dimensions = *block.getIArguments();
+  if (block.width() > 2) {
+    auto axesVector = INPUT_VARIABLE(2);
+    helpers::adjustAxis(INPUT_VARIABLE(0)->rankOf(), axesVector, dimensions);
+  }
+
+  REQUIRE_TRUE(
+      dimensions.size() <= static_cast<size_t>(inputShape->at(0)[0]), 0,
+      "REDUCE_SUM_BP OP: the number of dimensions to reduce along must be <= input array rank, but got %i instead",
+      dimensions.size());
+
+  for (const auto& item : dimensions)
+    REQUIRE_TRUE(
+        item >= -inputShape->at(0)[0] && item < inputShape->at(0)[0], 0,
+        "REDUCE_SUM_BP OP: the input dimension to reduce along must be in range [-%i, %i), but got %i instead !",
+        inputShape->at(0)[0], inputShape->at(0)[0], item);
+
+
+  return SHAPELIST(CONSTANT(inputShape->at(0)));
+}
+
+DECLARE_TYPES(reduce_sum_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+
+}  // namespace ops
+}  // namespace sd
+#endif