chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/generic/broadcastable/add.cpp

@@ -0,0 +1,109 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_add)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(add, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Add(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z && !tZ->isEmpty()) {
+    OVERWRITE_RESULT(tZ);
+  }
+  return Status::OK;
+}
+
+DECLARE_TYPES(add) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(ANY);
+}
+
+DECLARE_TYPES(add_bp) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
+
+CUSTOM_OP_IMPL(add_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+    gradY->assign(epsNext);
+    gradX->assign(epsNext);
+  } else if (y->isScalar()) {
+    // scalar case
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(tmp);
+    gradX->assign(epsNext);
+    delete tmp;
+  } else {
+    // broadcast case
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else
+      gradX->assign(epsNext);
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(epsNext);
+  }
+
+
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(add_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/assign.cpp

@@ -0,0 +1,123 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#include <helpers/StringUtils.h>
+#if NOT_EXCLUDED(OP_assign)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(assign, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = block.width() < 2 ? x: INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  // Check if any array is of string type
+  if (x->isS() || y->isS() || z->isS()) {
+    // Handle string broadcast at high level
+    StringUtils::broadcastStringAssign(x,z);
+    return Status::OK;
+  }
+
+  NDArray *castedX = x->dataType() == z->dataType() ? x : x->cast(z->dataType());
+  NDArray *castedY = y->dataType() == z->dataType() ? y : y->cast(z->dataType());
+
+  ArrayOptions::validateSingleDataType(ArrayOptions::dataType(castedX->shapeInfo()));
+  ArrayOptions::validateSingleDataType(ArrayOptions::extra(castedY->shapeInfo()));
+  ArrayOptions::validateSingleDataType(ArrayOptions::extra(z->shapeInfo()));
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Assign(), castedX, castedY, z);
+
+  if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  // Cleanup casted arrays if they were allocated
+  if (castedX != x) delete castedX;
+  if (castedY != y) delete castedY;
+
+  return Status::OK;
+}
+DECLARE_SYN(set, assign);
+DECLARE_SYN(copy, assign);
+
+DECLARE_TYPES(assign) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL})
+      ->setAllowedInputTypes(1, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL})
+      ->setAllowedOutputTypes(0, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL});
+}
+
+DECLARE_TYPES(assign_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_INTS,ALL_FLOATS,ALL_STRINGS});
+}
+
+CUSTOM_OP_IMPL(assign_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = block.width() < 2 ? x->dup(x->ordering(), false) : INPUT_VARIABLE(1);  // dup() already returns NDArray*
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  float zero = 0.0f;
+  gradX->assign(zero);
+
+  if (x->isSameShape(y)) {
+    gradY->assign(epsNext);
+  } else if (y->isScalar()) {
+    auto sum = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(sum);
+    delete sum;
+  } else {
+    // broadcastable
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(epsNext);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(assign_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/atan2.cpp

@@ -0,0 +1,55 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com, created on 10.02.18.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_tf_atan2)
+
+#include <ops/declarable/headers/broadcastable.h>
+
+namespace sd {
+namespace ops {
+
+BROADCASTABLE_OP_IMPL(tf_atan2, 0, 0) {
+  auto y = INPUT_VARIABLE(0);
+  auto x = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(BroadcastOpsTuple::custom(scalar::Atan2, pairwise::Atan2, broadcast::Atan2), y, z, true);
+
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(tf_atan2) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_and.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_or)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_and, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalAnd, pairwise::LogicalAnd, broadcast::LogicalAnd), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_and: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_and) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_or.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_or)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_or, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalOr, pairwise::LogicalOr, broadcast::LogicalOr), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_and: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_or) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_xor.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_xor)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_xor, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalXor, pairwise::LogicalXor, broadcast::LogicalXor), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_xor: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_xor) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/divide.cpp

@@ -0,0 +1,150 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_divide)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(divide, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "DIVIDE OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Divide(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Div, divide);
+
+DECLARE_TYPES(divide) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(divide_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(divide_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    NDArray *gradXTemp = (*epsNext) / (*y);
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    // Y gradient
+    NDArray *numerator = (*epsNext) * (*x);
+    NDArray *denominator = (*y) * (*y);
+    NDArray *gradYTemp = (*numerator) / (*denominator);
+    delete numerator;
+    delete denominator;
+    gradY->assign(gradYTemp);
+    gradY->applyTransform(transform::Neg, gradY);
+
+  } else if (y->isScalar()) {
+    // scalar case
+
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    auto tmpX = x->reduceNumber(reduce::Sum);
+
+    NDArray *temp1 = *tmp * *tmpX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*temp1) / (*ySquared);
+    delete temp1;
+    delete ySquared;
+    gradY->assign(gradYTemp);
+    gradY->applyTransform(transform::Neg, gradY);
+
+    epsNext->applyScalarArr(scalar::Divide, y, gradX);
+  } else {
+    // broadcast case
+
+    auto preX = *epsNext / *y;
+
+    NDArray negX(*x);
+    x->applyTransform(transform::Neg, &negX);
+    NDArray *negXMulEps = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    auto preY = (*negXMulEps) / (*ySquared);
+    delete negXMulEps;
+    delete ySquared;
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else {
+      // FIXED: preX is stack-allocated from operator/, don't delete
+      gradX->assign(preX);
+    }
+
+    if (axisY.size() > 0) {
+      auto sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      // FIXED: preY is stack-allocated from operator/, don't delete
+      gradY->assign(preY);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(divide_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/divide_no_nan.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author George A. Shulinok <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_divide_no_nan)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(divide_no_nan, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "DIVIDE_NO_NAN OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::DivideNoNan(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Div, divide);
+
+DECLARE_TYPES(divide_no_nan) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/equals.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/BroadcastBoolOpsTuple.h>
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_equals)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(equals, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastBoolOpsTuple::custom(scalar::EqualTo, pairwise::EqualTo, broadcast::EqualTo), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SYN(equal, equals);
+
+DECLARE_TYPES(equals) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/floordiv.cpp

@@ -0,0 +1,89 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_floordiv)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(floordiv, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "FLOORDIV OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::FloorDiv, pairwise::FloorDiv, broadcast::FloorDiv), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(floordiv) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(floordiv_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(floordiv_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  float zero = 0.0f;
+  gradY->assign(zero);
+  gradX->assign(zero);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(floordiv_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/floormod.cpp

@@ -0,0 +1,101 @@
+/* ******************************************************************************
+*
+*
+* This program and the accompanying materials are made available under the
+* terms of the Apache License, Version 2.0 which is available at
+* https://www.apache.org/licenses/LICENSE-2.0.
+*
+*  See the NOTICE file distributed with this work for additional
+*  information regarding copyright ownership.
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations
+* under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//  modified by sgazeos@gmail.com with backprop implementation.
+//
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_floormod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(floormod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "FLOORMOD OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(FloorMod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(floormod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(floormod_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(floormod_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  gradX->assign(epsNext);
+
+  NDArray temp(*epsNext);
+  BroadcastHelper::broadcastApply(BROADCAST(FloorMod), x, y, &temp);
+  if (gradY->rankOf() == gradX->rankOf()) {
+    epsNext->applyPairwiseTransform(pairwise::Multiply, &temp, gradY);
+  } else  { // epsNext is greater than gradY
+    std::vector<LongType> dims(epsNext->rankOf() * 2);
+    LongType gap = epsNext->rankOf() - gradY->rankOf();
+    for (LongType d = 0; d < gap; d++) {
+      dims[d * 2 + 1] = 1;
+    }
+    auto tempIn((temp)(dims));
+    NDArray negTempIn = -*tempIn;
+    auto get=  (*epsNext)(dims);
+    get->applyPairwiseTransform(pairwise::Multiply, &negTempIn, gradY);
+    delete get;
+    delete tempIn;
+  }
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(floormod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/greater.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_greater)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(greater, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(GreaterThan), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(greater) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/greater_equal.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_greater_equal)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(greater_equal, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(GreaterThanOrEqual), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(greater_equal) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/igamma.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_igamma)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(igamma, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::IGamma(), x, y, z);
+
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(igamma) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/igammac.cpp

@@ -0,0 +1,57 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_igammac)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(igammac, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::IGammac(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(igammac) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/less.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/CustomOperations.h>
+#if NOT_EXCLUDED(OP_less)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(less, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(LessThan), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(less) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/less_equal.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_less_equal)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(less_equal, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(LessThanOrEqual), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(less_equal) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/maximum.cpp

@@ -0,0 +1,83 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 12.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_maximum)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/helpers/minimax.h>
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(maximum, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(MaxPairwise), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(maximum) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(maximum_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(maximum_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  helpers::maximumBPFunctor(block.launchContext(), x, y, epsNext, gradX, gradY);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(maximum_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/meshgrid.cpp

@@ -0,0 +1,89 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 05.02.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_meshgrid)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/meshgrid.h>
+
+#include <numeric>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(meshgrid, -1, -1, false, 0, 0) {
+  int rank = block.width();
+
+  if (rank == 1) {
+    OUTPUT_VARIABLE(0)->assign(INPUT_VARIABLE(0));
+    return Status::OK;
+  }
+
+  bool swapFirst2Dims = block.getIArguments()->size() > 0 ? (bool)INT_ARG(0) : true;
+
+  std::vector<NDArray*> inArrs(rank);
+  std::vector<NDArray*> outArrs(rank);
+
+  for (int i = 0; i < rank; ++i) {
+    inArrs[i] = INPUT_VARIABLE(i);
+    outArrs[i] = OUTPUT_VARIABLE(i);
+  }
+
+  helpers::meshgrid(block.launchContext(), inArrs, outArrs, swapFirst2Dims);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(meshgrid) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(INHERIT)->setSameMode(true);
+}
+
+DECLARE_SHAPE_FN(meshgrid) {
+  bool swapFirst2Dims = block.getIArguments()->size() > 0 ? (bool)INT_ARG(0) : true;
+
+  int rank = block.width();
+  LongType* outShapeInfo = nullptr;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+  outShapeInfo[0] = rank;
+  for (int i = 1; i <= rank; ++i) outShapeInfo[i] = (LongType)shape::length(inputShape->at(i - 1));
+
+  if (swapFirst2Dims && rank > 1) math::sd_swap<LongType>(outShapeInfo[1], outShapeInfo[2]);
+
+  auto in = inputShape->at(0);
+  ShapeUtils::updateStridesAndType(outShapeInfo, in, shape::order(in));
+
+  auto shapes = SHAPELIST();
+  auto resultShape = CONSTANT(outShapeInfo);
+  shapes->push_back(resultShape);
+
+  for (int i = 2; i <= rank; ++i) {
+    shapes->push_back(resultShape);
+  }
+
+  return shapes;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/minimum.cpp

@@ -0,0 +1,83 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 12.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_minimum)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/helpers/minimax.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(minimum, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(MinPairwise), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(minimum) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(minimum_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(minimum_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  helpers::minimumBPFunctor(block.launchContext(), x, y, epsNext, gradX, gradY);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(minimum_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/mod.cpp

@@ -0,0 +1,87 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_mod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(mod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(Mod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(mod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(mod_bp) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
+
+CUSTOM_OP_IMPL(mod_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  float zero = 0.0f;
+  gradY->assign(zero);
+  gradX->assign(zero);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(mod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/multiply.cpp

@@ -0,0 +1,165 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//  @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_multiply)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+BROADCASTABLE_OP_IMPL(multiply, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+   LongType* zShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, zShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0, "MULTIPLY OP: the shapes of x %s and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Multiply(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("multiply: result was replaced");
+
+  return Status::OK;
+}
+DECLARE_SYN(Mul, multiply);
+
+DECLARE_TYPES(multiply) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(multiply_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+///////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(multiply_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto dLdz = INPUT_VARIABLE(2);
+
+  auto dLdx = OUTPUT_VARIABLE(0);
+  auto dLdy = OUTPUT_VARIABLE(1);
+
+   LongType* dLdzShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, dLdzShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0,
+               "MULTIPLY_BP OP: the shapes of x %s and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+
+  const LongType xLen = x->lengthOf();
+  const LongType yLen = y->lengthOf();
+
+  if (x->isScalar() && y->isScalar()) {  // both are scalars
+    y->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+    x->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+
+  }else if (x->isScalar()) {  // x is scalar and y is not
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceNumber(reduce::Sum);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+    dLdz->applyScalarArr(scalar::Multiply, x, dLdy);
+  } else if (y->isScalar()) {  // y is scalar and x is not
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceNumber(reduce::Sum);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+    dLdz->applyScalarArr(scalar::Multiply, y, dLdx);
+  } else if (x->isSameShape(y)) {
+    x->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+    y->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+  } else if (x->isSameShape(dLdz)) {
+    auto yTiled = NDArray(dLdz, false, block.launchContext());
+    y->tile(yTiled);
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+    yTiled.applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+  } else if (y->isSameShape(dLdz)) {
+    auto xTiled = NDArray(dLdz, false, block.launchContext());
+    x->tile(xTiled);
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    // FIXED: Clean up intermediate result from operator*
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+    xTiled.applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+  } else {
+    auto xTiled = NDArray(dLdz, false, block.launchContext());
+    auto yTiled = NDArray(dLdz, false, block.launchContext());
+    x->tile(xTiled);
+    y->tile(yTiled);
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+
+    // For dLdx
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+
+    // For dLdy
+    // FIXED: Clean up intermediate result from operator*
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(multiply_bp) {
+  auto xShapeInfo = inputShape->at(0);
+  auto yShapeInfo = inputShape->at(1);
+  return SHAPELIST(CONSTANT(xShapeInfo), CONSTANT(yShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/not_equals.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_not_equals)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(not_equals, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(NotEqualTo), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(not_equals) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/percentile.cpp

@@ -0,0 +1,108 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 17.05.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_percentile)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/percentile.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(percentile, 1, 1, false, 1, -2) {
+  auto input = INPUT_VARIABLE(0);    // tensor with rank > 0
+  auto output = OUTPUT_VARIABLE(0);  // [bS, oD, oH, oW, iC] (NDHWC) or [bS, iC, oD, oH, oW] (NCDHW)
+
+  const auto q = T_ARG(0);  // percentile
+  const int interpolation = block.getTArguments()->size() > 1 ? T_ARG(1) : 2.;  // 0-"lower", 1-"higher", 2-"nearest"(default)
+  const int keepDims = block.getTArguments()->size() > 2 ? T_ARG(2) : 0.;  // false is default
+
+  const int axisArrRank = block.getIArguments()->size();
+  const int inputArrRank = input->rankOf();
+
+  REQUIRE_TRUE(inputArrRank > 0, 0, "PERCENTILE OP: rank of input array must be positive (>0), but got %i instead !",
+               inputArrRank);
+  REQUIRE_TRUE(0.f <= q && q <= 100.f, 0,
+               "PERCENTILE OP: percentile parameter must be within [0, 100] range, but got %f instead !", q);
+  REQUIRE_TRUE(interpolation == 0 || interpolation == 1 || interpolation == 2, 0,
+               "PERCENTILE OP: the correct values for interpolation parameter are 0, 1, 2, but got %i instead !",
+               interpolation);
+  REQUIRE_TRUE(
+      axisArrRank <= inputArrRank, 0,
+      "PERCENTILE OP: the rank of axis array must be <= rank of input array, but got %i and %i correspondingly !",
+      axisArrRank, inputArrRank);
+
+  for (int i = 0; i < axisArrRank; ++i) {
+    int dim = INT_ARG(i) >= 0 ? INT_ARG(i) : INT_ARG(i) + inputArrRank;
+    REQUIRE_TRUE(dim < inputArrRank, 0,
+                 "PERCENTILE OP: element (dimension) of axis array at position %i is >= rank of input array (%i >= "
+                 "%i), which is unacceptable !",
+                 i, dim, inputArrRank);
+  }
+
+  std::vector<LongType> axises = *block.getIArguments();
+  helpers::percentile(block.launchContext(), *input, *output, axises, q, interpolation);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(percentile) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedOutputTypes(0, INHERIT)
+      ->setSameMode(true);
+}
+
+DECLARE_SHAPE_FN(percentile) {
+  auto inputShapeInfo = inputShape->at(0);
+  const int keepDims = block.getTArguments()->size() > 2 ? T_ARG(2) : 0.;  // false is default
+
+  const int axisArrRank = block.getIArguments()->size();
+  const int inputArrRank = inputShapeInfo[0];
+
+  REQUIRE_TRUE(inputArrRank > 0, 0, "PERCENTILE OP: rank of input array must be positive (>0), but got %i instead !",
+               inputArrRank);
+  REQUIRE_TRUE(
+      axisArrRank <= inputArrRank, 0,
+      "PERCENTILE OP: the rank of axis array must be <= rank of input array, but got %i and %i correspondingly !",
+      axisArrRank, inputArrRank);
+
+  for (int i = 0; i < axisArrRank; ++i) {
+    int dim = INT_ARG(i) >= 0 ? INT_ARG(i) : INT_ARG(i) + inputArrRank;
+    REQUIRE_TRUE(dim < inputArrRank, 0,
+                 "PERCENTILE OP: element (dimension) of axis array at position %i is >= rank of input array (%i >= "
+                 "%i), which is unacceptable !",
+                 i, dim, inputArrRank);
+  }
+
+  std::vector<LongType> axises = *block.getIArguments();
+  auto outputShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(inputShapeInfo), &axises, inputShapeInfo, keepDims,
+                                                         false, block.getWorkspace());
+
+  return SHAPELIST(outputShapeInfo);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp

@@ -0,0 +1,122 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_Pow)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(Pow, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  // REQUIRE_TRUE(!y->isB(), 0, "Pairwise OP: you can't divide by bool array!");
+
+  auto tZ = BroadcastHelper::broadcastApply({scalar::Pow, pairwise::Pow, broadcast::Pow}, x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(Pow) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS, ALL_INTS});
+}
+
+CUSTOM_OP_IMPL(Pow_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto dLdz = INPUT_VARIABLE(2);
+
+  auto dLdx = OUTPUT_VARIABLE(0);
+  auto dLdy = OUTPUT_VARIABLE(1);
+
+   LongType* dLdzShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, dLdzShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0,
+               "POW_BP OP: the shapes of x %s"
+               " and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+  REQUIRE_TRUE(shape::equalsSoft(dLdz->shapeInfo(), dLdzShapeInfo), 0,
+               "POW_BP OP: wrong shape of next epsilon array (dLdOut),"
+               " expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(dLdzShapeInfo).c_str(), ShapeUtils::shapeAsString(dLdz).c_str());
+
+  // dL/dy = x^y * log(x) * dL/dz
+  auto temp = x->applyTrueBroadcast(BroadcastOpsTuple::Pow(), y);  // a = x^y
+  x->applyTransform(transform::Log, dLdx);                         // b = log(x)
+  dLdx->applyScalar(scalar::ReplaceNans, 0, dLdx);
+  *temp *= *dLdx;  // c = b*a
+  *temp *= *dLdz;  // dL/dy = c * dL/dz
+  if (dLdy->isSameShape(*dLdz)) {
+    dLdy->assign(temp);
+  } else {
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+    NDArray *dLdyTemp = temp->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);  // dL/dy = sum(c * dL/dz)
+    delete dLdyTemp;
+  }
+
+  // dL/dx = y*x^(y-1) * dL/dz
+  x->applyTrueBroadcast(BroadcastOpsTuple::PowDerivative(), y, temp);  // a = y*x^(y-1)
+  *temp *= *dLdz;                                                        // dLdx = a*dL/dz
+
+  if (dLdx->isSameShape(*dLdz)) {
+    dLdx->assign(temp);  // dLdx = a*dL/dz
+  } else {
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    NDArray *dLdxTemp = temp->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);  // dLdx = a*dL/dz
+    delete dLdxTemp;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(Pow_bp) {
+  auto xShapeInfo = inputShape->at(0);
+  auto yShapeInfo = inputShape->at(1);
+  return SHAPELIST(CONSTANT(xShapeInfo), CONSTANT(yShapeInfo));
+}
+
+DECLARE_TYPES(Pow_bp) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/realdiv.cpp

@@ -0,0 +1,150 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_realdiv)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(realdiv, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Divide(), x, y, z);
+  if (tZ == nullptr) {
+    return Status::KERNEL_FAILURE;
+  }
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(RealDiv, realdiv);
+
+DECLARE_TYPES(realdiv) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, {FLOAT32, HALF, DOUBLE});
+}
+
+DECLARE_TYPES(realdiv_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(realdiv_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    epsNext->applyPairwiseTransform(pairwise::Divide, y, gradX);
+
+    // Y gradient
+
+    // First case
+    NDArray negX = -(*x);
+    NDArray *epsNextMulNegX = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*epsNextMulNegX) / (*ySquared);
+    gradY->assign(gradYTemp);
+    delete epsNextMulNegX;
+    delete ySquared;
+    delete gradYTemp;
+
+  } else if (y->isScalar()) {
+    // scalar case
+
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    auto tmpX = x->reduceNumber(reduce::Sum);
+
+    NDArray negTmpX = -*tmpX;
+    NDArray *tmpMulNegTmpX = (*tmp) * negTmpX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*tmpMulNegTmpX) / (*ySquared);
+    gradY->assign(gradYTemp);
+    delete tmpMulNegTmpX;
+    delete ySquared;
+    delete gradYTemp;
+
+    epsNext->applyScalarArr(scalar::Divide, y, gradX);
+  } else {
+    // broadcast case
+
+    auto preX = *epsNext / *y;
+
+    NDArray negX(*x);
+    x->applyTransform(transform::Neg, &negX);
+    NDArray *epsNextMulNegX = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *preY = (*epsNextMulNegX) / (*ySquared);
+    delete epsNextMulNegX;
+    delete ySquared;
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+    } else
+      gradX->assign(preX);
+
+    if (axisY.size() > 0) {
+      auto sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(preY);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(realdiv_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_divide.cpp

@@ -0,0 +1,147 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversedivide)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversedivide, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!x->isB(), 0, "REVERSEDIVIDE OP: you can't divide by bool array!");
+  x->applyTrueBroadcast(BROADCAST(ReverseDivide), y, z, true);
+
+  return Status::OK;
+}
+DECLARE_SYN(RDiv, reversedivide);
+
+DECLARE_TYPES(reversedivide) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(reversedivide_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(reversedivide_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* gradXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    gradX->applyTransform(transform::Neg, gradX);
+
+    // Y gradient
+    auto* gradYTemp = (*epsNext) / (*x);
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+  } else if (y->isScalar()) {
+    // scalar case
+    auto* tmp = epsNext->reduceNumber(reduce::Sum);
+    auto* tmpX = x->reduceNumber(reduce::Sum);
+    // For gradY
+    auto* gradYTemp = (*tmp) / (*tmpX);
+    delete tmp;
+    delete tmpX;
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+
+    // For gradX
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* gradXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    gradX->applyTransform(transform::Neg, gradX);
+  } else {
+    // broadcast case
+
+    auto* preY = (*epsNext) / (*x);
+
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* preXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    preXTemp->applyTransform(transform::Neg, preXTemp);
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto* sum = preXTemp->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else {
+      gradX->assign(preXTemp);
+    }
+    delete preXTemp;
+
+    if (axisY.size() > 0) {
+      auto* sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      gradY->assign(preY);
+    }
+    delete preY;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversedivide_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_mod.cpp

@@ -0,0 +1,88 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversemod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversemod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(ReverseMod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(reversemod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(reversemod_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(reversemod_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  float assign = 0.0f;
+  gradY->assign(assign);
+  gradX->assign(assign);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversemod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_subtract.cpp

@@ -0,0 +1,117 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversesubtract)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversesubtract, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(ReverseSubtract), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(RSub, reversesubtract);
+
+DECLARE_TYPES(reversesubtract) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(reversesubtract_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+    epsNext->applyTransform(transform::Neg, gradX);
+    gradY->assign(epsNext);
+  } else if (y->isScalar()) {
+    // scalar case
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(tmp);
+    delete tmp;
+    epsNext->applyTransform(transform::Neg, gradX);
+  } else {
+    // broadcastable
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisX);
+      sum->applyTransform(transform::Neg, gradX);
+      delete sum;
+    } else {
+      epsNext->applyTransform(transform::Neg, gradX);
+    }
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      gradY->assign(epsNext);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversesubtract_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+
+DECLARE_TYPES(reversesubtract_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/squared_subtract.cpp

@@ -0,0 +1,174 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 23.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_squaredsubtract)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(squaredsubtract, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(SquaredSubtract), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(squareddifference, squaredsubtract);
+
+DECLARE_TYPES(squaredsubtract) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(squaredsubtract_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+
+  auto* ts = NDArrayFactory::create(x->dataType(), 2, block.launchContext());
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    auto* diff1 = (*x) - (*y);
+    auto* temp1 = (*ts) * (*diff1);
+    delete diff1;
+    auto* gradXTemp = (*epsNext) * (*temp1);
+    delete temp1;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+
+    // Y gradient
+    auto* diff2 = (*y) - (*x);
+    auto* temp2 = (*ts) * (*diff2);
+    delete diff2;
+    auto* gradYTemp = (*epsNext) * (*temp2);
+    delete temp2;
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+
+  } else if (y->isScalar()) {
+    // scalar case
+    auto* tmpX = x->reduceNumber(reduce::Sum);
+    gradY->assign(tmpX);
+    delete tmpX;
+    
+    // X gradient
+    auto* diff3 = (*x) - (*y);
+    auto* temp3 = (*ts) * (*diff3);
+    delete diff3;
+    auto* gradXTemp = (*epsNext) * (*temp3);
+    delete temp3;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+  } else {
+    // broadcast case
+
+    auto* preX = x->dup(x->ordering());
+    auto* preY = y->dup(y->ordering());
+
+    auto* targetShape = epsNext->getShapeAsVector();
+
+    preX->tileToShape(*targetShape, *preX);
+    preY->tileToShape(*targetShape, *preY);
+    delete targetShape;
+    
+    auto* diff4 = (*x) - (*y);
+    auto* temp4 = (*ts) * (*diff4);
+    delete diff4;
+    auto* resX = (*epsNext) * (*temp4);
+    delete temp4;
+    preX->assign(resX);
+    delete resX;
+    
+    auto* diff5 = (*y) - (*x);
+    auto* temp5 = (*ts) * (*diff5);
+    delete diff5;
+    auto* resY = (*epsNext) * (*temp5);
+    delete temp5;
+    preY->assign(resY);
+    delete resY;
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto* sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else
+      gradX->assign(preX);
+
+    if (axisY.size() > 0) {
+      auto* sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(preY);
+
+    delete preX;
+    delete preY;
+  }
+
+  delete ts;
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(squaredsubtract_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+
+DECLARE_TYPES(squaredsubtract_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/subtract.cpp

@@ -0,0 +1,118 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_subtract)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(subtract, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Subtract(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Sub, subtract);
+DECLARE_SYN(sub, subtract);
+
+DECLARE_TYPES(subtract) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(subtract_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+    epsNext->applyTransform(transform::Neg, gradY);
+    gradX->assign(epsNext);
+  } else if (y->isScalar()) {
+    // scalar case
+    auto reduce = epsNext->reduceNumber(reduce::Sum);
+    auto tmp = -(*reduce);
+    gradY->assign(&tmp);
+    gradX->assign(epsNext);
+    delete reduce;
+  } else {
+    // broadcastable
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else
+      gradX->assign(epsNext);
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      sum->applyTransform(transform::Neg, gradY);
+      delete sum;
+    } else {
+      epsNext->applyTransform(transform::Neg, gradY);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(subtract_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(subtract_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/truncatediv.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_truncatediv)
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(truncatediv, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(TruncateDiv), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(truncatediv) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+#endif