chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/generic/CustomOperations.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// This is special snowflake. This file builds bindings for ops availability tests
+//
+// @author raver119@gmail.com
+//
+
+#include <helpers/OpTracker.h>
+#include <loops/legacy_ops.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+
+_loader::_loader() {
+  //
+  OpTracker::getInstance();
+
+  BUILD_TRACKER(::graph::OpType_TRANSFORM_SAME, TRANSFORM_FLOAT_OPS);
+  BUILD_TRACKER(::graph::OpType_TRANSFORM_SAME, TRANSFORM_SAME_OPS);
+  BUILD_TRACKER(::graph::OpType_TRANSFORM_SAME, TRANSFORM_BOOL_OPS);
+  BUILD_TRACKER(::graph::OpType_BROADCAST, BROADCAST_OPS);
+  BUILD_TRACKER(::graph::OpType_PAIRWISE, PAIRWISE_TRANSFORM_OPS);
+  BUILD_TRACKER(::graph::OpType_RANDOM, RANDOM_OPS);
+  BUILD_TRACKER(::graph::OpType_REDUCE_FLOAT, REDUCE_FLOAT_OPS);
+  BUILD_TRACKER(::graph::OpType_REDUCE_SAME, REDUCE_SAME_OPS);
+  BUILD_TRACKER(::graph::OpType_REDUCE_BOOL, REDUCE_BOOL_OPS);
+  BUILD_TRACKER(::graph::OpType_REDUCE_3, REDUCE3_OPS);
+  BUILD_TRACKER(::graph::OpType_INDEX_REDUCE, INDEX_REDUCE_OPS);
+  BUILD_TRACKER(::graph::OpType_SCALAR, SCALAR_OPS);
+  BUILD_TRACKER(::graph::OpType_SUMMARYSTATS, SUMMARY_STATS_OPS);
+};
+
+static sd::_loader loader;
+}  // namespace sd

libnd4j/include/ops/declarable/generic/README.md

@@ -0,0 +1,4 @@
+## Generic ops
+
+Put any cross platform operations here. Anything platform specific should be put in:
+in ops/helpers.

libnd4j/include/ops/declarable/generic/bitwise/bits_hamming_distance.cpp

@@ -0,0 +1,60 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bits_hamming_distance)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/hamming.h>
+#include <ops/declarable/helpers/helpers.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(bits_hamming_distance, 2, 1, true, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto output = OUTPUT_NULLIFIED(0);
+
+  REQUIRE_TRUE(x->lengthOf() == y->lengthOf(), 0, "bits_hamming_distance: both arguments must have the same length");
+  REQUIRE_TRUE(x->dataType() == y->dataType(), 0, "bits_hamming_distance: both arguments must have the same data type");
+
+  helpers::hamming(block.launchContext(), *x, *y, *output);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(bits_hamming_distance) {
+  return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(sd::DataType::INT64));
+}
+
+DECLARE_TYPES(bits_hamming_distance) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes(0, {ALL_INDICES});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/bitwise_and.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bitwise_and)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(bitwise_and, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::IntOps::IntAnd, pairwise::IntOps::IntAnd, broadcast::IntOps::IntAnd), y, z,
+      false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(bitwise_and) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/bitwise_or.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bitwise_or)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(bitwise_or, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::IntOps::IntOr, pairwise::IntOps::IntOr, broadcast::IntOps::IntOr), y, z,
+      false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(bitwise_or) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/bitwise_xor.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bitwise_xor)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(bitwise_xor, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::IntOps::IntXor, pairwise::IntOps::IntXor, broadcast::IntOps::IntXor), y, z,
+      false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(bitwise_xor) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/cyclic_rshift.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_cyclic_rshift_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(cyclic_rshift_bits, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::CyclicShiftRight, pairwise::CyclicShiftRight, broadcast::CyclicShiftRight),
+      y, z, false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(cyclic_rshift_bits) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/cyclic_shift.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_cyclic_shift_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(cyclic_shift_bits, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::CyclicShiftLeft, pairwise::CyclicShiftLeft, broadcast::CyclicShiftLeft), y,
+      z, false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(cyclic_shift_bits) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/rshift.cpp

@@ -0,0 +1,51 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_rshift_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(rshift_bits, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(BroadcastIntOpsTuple::custom(scalar::ShiftRight, pairwise::ShiftRight, broadcast::ShiftRight),
+                        y, z, false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(rshift_bits) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/shift.cpp

@@ -0,0 +1,51 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_shift_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(shift_bits, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(BroadcastIntOpsTuple::custom(scalar::ShiftLeft, pairwise::ShiftLeft, broadcast::ShiftLeft), y,
+                        z, false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(shift_bits) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/toggle_bits.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 23.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_toggle_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/toggle_bits.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(toggle_bits, -1, -1, true) {
+  for (size_t i = 0; i < block.width(); i++) {
+    auto x = INPUT_VARIABLE(i);
+    auto z = OUTPUT_VARIABLE(i);
+
+    REQUIRE_TRUE(x->dataType() == z->dataType(), 0, "Toggle bits requires input and output to have same type");
+    REQUIRE_TRUE(x->isZ(), 0, "Toggle bits requires input and output to be integer type (int8, int16, int32, int64)");
+
+    helpers::__toggle_bits(block.launchContext(), x, z);
+  }
+  return Status::OK;
+}
+
+DECLARE_TYPES(toggle_bits) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setAllowedOutputTypes({ALL_INTS})->setSameMode(false);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/blas/axpy.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_axpy)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(axpy, 2, 1, false, -2, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(x->isSameShape(y), 0, "Axpy: both arguments should have the same shape");
+  REQUIRE_TRUE(x->dataType() == y->dataType() && x->dataType() == z->dataType(), 0,
+               "Axpy: all arguments must have the same data type");
+
+  double a = 1.0;
+
+  if (block.width() > 2) {
+    auto alpha = INPUT_VARIABLE(2);
+    REQUIRE_TRUE(alpha->isScalar(), 0, "Axpy: alpha argument should be scalar or TArg");
+  } else if (block.getTArguments()->size() > 0) {
+    a = T_ARG(0);
+  }
+
+  ExtraArguments arguments({a});
+
+  y->applyPairwiseTransform(pairwise::Axpy, x, z, &arguments);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(axpy) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/blas/batched_gemm.cpp

@@ -0,0 +1,355 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//   @author Adam Gibson
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_batched_gemm)
+
+#include <ops/declarable/headers/blas.h>
+#include <ops/declarable/helpers/batched_gemm.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(batched_gemm, -1, -1, false, 0, 2) {
+  // Only require 2 IArgs: transposeA, transposeB
+  // Everything else will be inferred from the input matrices
+  int transA = INT_ARG(0);
+  int transB = INT_ARG(1);
+  
+  // Get alpha and beta
+  auto alpha = INPUT_VARIABLE(0);
+  auto beta = INPUT_VARIABLE(1);
+  
+  // Calculate batch size from number of inputs
+  // Total inputs = alpha + beta + batchSize*A + batchSize*B
+  // So batchSize = (total - 2) / 2
+  int batchSize = (block.width() - 2) / 2;
+  
+  REQUIRE_TRUE(batchSize > 0, 0, "BatchedGemm: Invalid batch size calculated: %d", batchSize);
+  REQUIRE_TRUE((block.width() - 2) % 2 == 0, 0, "BatchedGemm: Number of matrix inputs must be even");
+  
+  // Get first matrices to infer dimensions
+  auto firstA = INPUT_VARIABLE(2);
+  auto firstB = INPUT_VARIABLE(2 + batchSize);
+  
+  REQUIRE_TRUE(firstA->rankOf() == 2, 0, "BatchedGemm: A matrices must be rank 2");
+  REQUIRE_TRUE(firstB->rankOf() == 2, 0, "BatchedGemm: B matrices must be rank 2");
+  
+  // Infer dimensions from first matrices
+  int M = transA ? firstA->sizeAt(1) : firstA->sizeAt(0);
+  int K = transA ? firstA->sizeAt(0) : firstA->sizeAt(1);
+  int N = transB ? firstB->sizeAt(0) : firstB->sizeAt(1);
+  int K_B = transB ? firstB->sizeAt(1) : firstB->sizeAt(0);
+  
+  REQUIRE_TRUE(K == K_B, 0, "BatchedGemm: Incompatible dimensions - K from A is %d, K from B is %d", K, K_B);
+  
+  // Infer leading dimensions
+  int ldA = firstA->sizeAt(0);
+  int ldB = firstB->sizeAt(0);
+  int ldC = M;
+  
+  // Validate leading dimensions
+  if(transA == 0) {
+    int ldaComp = M > 1 ? M : 1;
+    if(ldA < ldaComp) THROW_EXCEPTION("LDA must be >= max(1,m) when transa == false");
+  } else {
+    int ldaComp = K > 1 ? K : 1;
+    if(ldA < ldaComp)
+      THROW_EXCEPTION("LDA must be >= max(1,k) when transa == true");
+  }
+
+  if(transB == 0) {
+    int ldBComp = K > 1 ? K : 1;
+    if(ldB < ldBComp) {
+      THROW_EXCEPTION("LDB must be >= max(1,k) when transb == false");
+    }
+  } else {
+    int ldbComp = N > 1 ? N : 1;
+    if(ldB < ldbComp)
+      THROW_EXCEPTION("LDB must be >= max(1,N) when transb == true");
+  }
+
+  int ldcComp = M > 1 ? M : 1;
+  if(ldC < ldcComp) {
+    THROW_EXCEPTION("LDC must be >= max(1,M)");
+  }
+
+  // Convert transpose flags to BLAS format
+  int transABlas = transA ? 112 : 111;  // 112 = CblasTrans, 111 = CblasNoTrans
+  int transBBlas = transB ? 112 : 111;
+
+  REQUIRE_TRUE((transA == 0 || transA == 1) && (transB == 0 || transB == 1), 0,
+               "BatchedGemm: valid values for transA and transB are: 0/1 for NoTrans/Trans respectively")
+  REQUIRE_TRUE(M > 0 && N > 0 && K > 0 && ldA > 0 && ldB > 0 && ldC > 0, 0, 
+               "BatchedGemm: Invalid dimensions M=%d, N=%d, K=%d, ldA=%d, ldB=%d, ldC=%d", M, N, K, ldA, ldB, ldC);
+
+  // Handle alpha and beta
+  NDArray *alphaInput = nullptr;
+  if(alpha->isScalar()) {
+    std::vector<sd::LongType> shape = {batchSize};
+    alphaInput = new NDArray('c',shape,alpha->dataType());
+    alphaInput->assign(alpha);
+  } else {
+    alphaInput = alpha;
+  }
+
+  NDArray *betaInput = nullptr;
+  if(beta->isScalar()) {
+    std::vector<LongType> shape = {batchSize};
+    betaInput = new NDArray('c',shape,beta->dataType());
+    betaInput->assign(beta);
+  } else {
+    betaInput = beta;
+  }
+
+  std::vector<NDArray*> vA(batchSize);
+  std::vector<NDArray*> vB(batchSize);
+  std::vector<NDArray*> vC(batchSize);
+
+  // Check data types - matrices should all match, alpha/beta can be different
+  auto firstMatrixType = firstA->dataType();
+  for (int e = 0; e < batchSize; e++) {
+    vA[e] = INPUT_VARIABLE(e + 2);
+    vB[e] = INPUT_VARIABLE(e + 2 + batchSize);
+    vC[e] = OUTPUT_VARIABLE(e);
+
+    REQUIRE_TRUE(firstMatrixType == vA[e]->dataType(), 0, 
+                 "BatchedGemm: all A matrices must have same data type");
+    REQUIRE_TRUE(firstMatrixType == vB[e]->dataType(), 0, 
+                 "BatchedGemm: all B matrices must have same data type");
+    REQUIRE_TRUE(firstMatrixType == vC[e]->dataType(), 0, 
+                 "BatchedGemm: all output matrices must have same data type as input matrices");
+    
+    REQUIRE_TRUE(vA[e]->rankOf() == 2, 0, "BatchedGemm: batch %i, rank of A should be equal to 2", e);
+    REQUIRE_TRUE(vB[e]->rankOf() == 2, 0, "BatchedGemm: batch %i, rank of B should be equal to 2", e);
+    REQUIRE_TRUE(vC[e]->rankOf() == 2, 0, "BatchedGemm: batch %i, rank of C should be equal to 2", e);
+
+    // Verify dimensions are consistent across batch
+    int currM = transABlas == 111 ? vA[e]->sizeAt(0) : vA[e]->sizeAt(1);
+    int currK_A = transABlas == 111 ? vA[e]->sizeAt(1) : vA[e]->sizeAt(0);
+    int currK_B = transBBlas == 111 ? vB[e]->sizeAt(0) : vB[e]->sizeAt(1);
+    int currN = transBBlas == 111 ? vB[e]->sizeAt(1) : vB[e]->sizeAt(0);
+    
+    REQUIRE_TRUE(currM == M, 0, "BatchedGemm: batch %i, inconsistent M dimension: expected %d, got %d", e, M, currM);
+    REQUIRE_TRUE(currK_A == K, 0, "BatchedGemm: batch %i, inconsistent K dimension in A: expected %d, got %d", e, K, currK_A);
+    REQUIRE_TRUE(currK_B == K, 0, "BatchedGemm: batch %i, inconsistent K dimension in B: expected %d, got %d", e, K, currK_B);
+    REQUIRE_TRUE(currN == N, 0, "BatchedGemm: batch %i, inconsistent N dimension: expected %d, got %d", e, N, currN);
+  }
+
+  helpers::bgemm(vA, vB, vC, alphaInput, betaInput, transABlas, transBBlas, M, N, K, ldA, ldB, ldC);
+
+  if(alphaInput != alpha) {
+    delete alphaInput;
+  }
+
+  if(betaInput != beta) {
+    delete betaInput;
+  }
+
+  return Status::OK;
+};
+
+DECLARE_SHAPE_FN(batched_gemm) {
+  // Only require 2 IArgs: transposeA, transposeB
+  int transA = INT_ARG(0);
+  int transB = INT_ARG(1);
+  
+  // Calculate batch size from inputs
+  int batchSize = (block.width() - 2) / 2;
+  
+  if (batchSize <= 0) {
+    auto shapeList = SHAPELIST();
+    shapeList->push_back(
+        ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inputShape->at(0)), 'c', {1, 1}));
+    return shapeList;
+  }
+  
+  // Get dimensions from first matrices
+  auto firstA = inputShape->at(2);
+  auto firstB = inputShape->at(2 + batchSize);
+  
+  int M = transA ? shape::sizeAt(firstA, 1) : shape::sizeAt(firstA, 0);
+  int N = transB ? shape::sizeAt(firstB, 0) : shape::sizeAt(firstB, 1);
+  
+  // Get data type from first matrix, not from alpha/beta
+  auto firstMatrixType = ArrayOptions::dataType(firstA);
+  
+  // Check that all matrices have the same type (skip alpha and beta)
+  for (int e = 2; e < block.width(); e++) {
+    REQUIRE_TRUE(firstMatrixType == ArrayOptions::dataType(inputShape->at(e)), 0,
+                 "BatchedGemm: all matrices must have same data type");
+  }
+
+  auto shapeList = SHAPELIST();
+  std::vector<LongType> shape({M, N});
+
+  for (int e = 0; e < batchSize; e++) {
+    auto newShape =
+        ConstantShapeHelper::getInstance().createShapeInfo(firstMatrixType, 'f', shape);
+    shapeList->push_back(newShape);
+  }
+
+  return shapeList;
+}
+
+DECLARE_TYPES(batched_gemm) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+
+
+CUSTOM_OP_IMPL(batched_gemm_bp, -1, -1, false, 0, 2) {
+  // Only require 2 IArgs: transposeA, transposeB
+  int transA = INT_ARG(0);
+  int transB = INT_ARG(1);
+  
+  // Calculate batch size
+  // Inputs: alpha, beta, batchSize*A, batchSize*B, batchSize*dLdC
+  // So batchSize = (total - 2) / 3
+  int batchSize = (block.width() - 2) / 3;
+  
+  REQUIRE_TRUE(batchSize > 0, 0, "BatchedGemmBp: Invalid batch size calculated: %d", batchSize);
+  
+  // Get dimensions from first matrices
+  auto firstA = INPUT_VARIABLE(2);
+  auto firstB = INPUT_VARIABLE(2 + batchSize);
+  auto firstDlDOut = INPUT_VARIABLE(2 + batchSize * 2);
+  
+  int M = transA ? firstA->sizeAt(1) : firstA->sizeAt(0);
+  int K = transA ? firstA->sizeAt(0) : firstA->sizeAt(1);
+  int N = transB ? firstB->sizeAt(0) : firstB->sizeAt(1);
+  
+  // Infer leading dimensions
+  int ldA = firstA->sizeAt(0);
+  int ldB = firstB->sizeAt(0);
+  int ldC = M;
+
+  std::vector<NDArray *> matricesA;
+  std::vector<NDArray *> matricesB;
+  std::vector<NDArray *> dlDOut;
+  std::vector<NDArray *> dldXOutputs;
+  std::vector<NDArray *> dldYOutputs;
+
+  for (int e = 0; e < batchSize; e++) {
+    matricesA.push_back(INPUT_VARIABLE(e + 2));
+    matricesB.push_back(INPUT_VARIABLE(e + 2 + batchSize));
+    dlDOut.push_back(INPUT_VARIABLE(e + 2 + batchSize * 2));
+    //alphas and betas are also set for outputs even though they're zero,every input needs a gradient
+    dldXOutputs.push_back(OUTPUT_VARIABLE(e + 2));
+    dldYOutputs.push_back(OUTPUT_VARIABLE(e + 2 + batchSize));
+  }
+
+  auto alpha = INPUT_VARIABLE(0);
+  NDArray *alphaInput = nullptr;
+  if(alpha->lengthOf() != batchSize) {
+    std::vector<sd::LongType> shape = {batchSize};
+    alphaInput = new NDArray('c',shape,alpha->dataType());
+    alphaInput->assign(alpha);
+  } else {
+    alphaInput = alpha;
+  }
+
+  auto beta = INPUT_VARIABLE(1);
+  NDArray *betaInput = nullptr;
+  if(beta->lengthOf() != batchSize) {
+    std::vector<sd::LongType> shape = {batchSize};
+    betaInput = new NDArray('c',shape,beta->dataType());
+    betaInput->assign(beta);
+  } else {
+    betaInput = beta;
+  }
+
+  // Convert transpose flags to BLAS format for helper function
+  int transABlas = transA ? 112 : 111;
+  int transBBlas = transB ? 112 : 111;
+
+  // First gradient computation: dL/dA = dL/dC @ B^T (or B if transB)
+  int transA1 = 0;
+  int transB1 = transB;
+  int M1 = dlDOut[0]->sizeAt(0);
+  int N1 = transB ? matricesB[0]->sizeAt(0) : matricesB[0]->sizeAt(1);
+  int k1 = dlDOut[0]->sizeAt(1);
+  int lda1 = dlDOut[0]->sizeAt(0);
+  int ldb1 = matricesB[0]->sizeAt(0);
+  int ldc1 = dldXOutputs[0]->sizeAt(0);
+  
+  helpers::bgemm(dlDOut, matricesB, dldXOutputs, alphaInput, betaInput, 
+                 transA1 ? 112 : 111, transB1 ? 112 : 111, M1, N1, k1, lda1, ldb1, ldc1);
+
+  // Second gradient computation: dL/dB = A^T @ dL/dC (or A if transA)
+  int transA2 = transA ? 0 : 1;
+  int transB2 = 0;
+  int M2 = transA ? matricesA[0]->sizeAt(1) : matricesA[0]->sizeAt(0);
+  int N2 = dlDOut[0]->sizeAt(1);
+  int k2 = transA ? matricesA[0]->sizeAt(0) : matricesA[0]->sizeAt(1);
+  int lda2 = matricesA[0]->sizeAt(0);
+  int ldb2 = dlDOut[0]->sizeAt(0);
+  int ldc2 = dldYOutputs[0]->sizeAt(0);
+  
+  helpers::bgemm(matricesA, dlDOut, dldYOutputs, alphaInput, betaInput, 
+                 transA2 ? 112 : 111, transB2 ? 112 : 111, M2, N2, k2, lda2, ldb2, ldc2);
+
+  if(alphaInput != alpha) {
+    delete alphaInput;
+  }
+
+  if(betaInput != beta) {
+    delete betaInput;
+  }
+
+  return Status::OK;
+};
+
+DECLARE_SHAPE_FN(batched_gemm_bp) {
+  // Calculate batch size
+  int batchSize = (block.width() - 2) / 3;
+  
+  auto xConstant = CONSTANT(inputShape->at(2));
+  auto yConstant = CONSTANT(inputShape->at(2 + batchSize));
+  auto ret = SHAPELIST();
+  //alpha
+  ret->push_back(xConstant);
+  //beta
+  ret->push_back(yConstant);
+  for(int i = 0; i < batchSize; i++) {
+    ret->push_back(xConstant);
+  }
+
+  for(int i = 0; i < batchSize; i++) {
+    ret->push_back(yConstant);
+  }
+  return ret;
+}
+
+DECLARE_TYPES(batched_gemm_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/blas/matmul.cpp

@@ -0,0 +1,352 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com, created on 07.10.2017.
+// @author GS <sgazeos@gmail.com>, modified
+// @author Yurii Shyrma (iuriish@yahoo.com), fully rewritten
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_matmul)
+
+#include <helpers/MmulHelper.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(matmul, 2, 1, false, 0, -2) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+  if(x->isEmpty() || y->isEmpty())
+    return Status::OK;
+  int iSize = (int)block.getIArguments()->size();
+  int transX = iSize > 0 ? INT_ARG(0) : 0;
+  int transY = iSize > 1 ? INT_ARG(1) : 0;
+  const int transZ = iSize > 2 ? INT_ARG(2) : 0;
+  // optional use alpha nad beta
+  iSize = (int)block.getTArguments()->size();
+  double alpha = iSize > 0 ? T_ARG(0) : 1.0;
+  double beta = iSize > 1 ? T_ARG(1) : 0.0;
+
+  if (transZ) {
+    x = INPUT_VARIABLE(1);
+    y = INPUT_VARIABLE(0);
+    bool temp = transX;
+    transX = !transY;
+    transY = !temp;
+  }
+
+  // Compute ranks AFTER potential transZ swap
+  const int xRank = x->rankOf();
+  const int yRank = y->rankOf();
+  const int zRank = z->rankOf();
+
+  const int xLastDim = transX ? -2 : -1;
+  const int yLastDim = transY ? -2 : -1;
+  const int xLastButOneDim = transX ? -1 : -2;
+  const int yLastButOneDim = transY ? -1 : -2;
+
+  // ******* input validation ******* //
+  REQUIRE_TRUE(xRank > 0 && yRank > 0, 0,
+               "MATMUL OP: input arrays must have rank bigger than 0 (should not be scalars), but got instead: x rank "
+               "= %i, y rank = %i !",
+               xRank, yRank);
+
+  // Handle rank mismatch when one input has singleton leading dimensions
+  // This supports ONNX Gemm patterns like [1,1,1,768] x [768,768] -> [1,1,1,768]
+  NDArray* xReshaped = nullptr;
+  NDArray* yReshaped = nullptr;
+  NDArray* zReshaped = nullptr;
+
+  if (xRank != yRank && xRank > 2 && yRank == 2) {
+    // Check if x has all singleton leading dims
+    bool allLeadingSingleton = true;
+    for (int i = 0; i < xRank - 2; ++i) {
+      if (x->sizeAt(i) != 1) {
+        allLeadingSingleton = false;
+        break;
+      }
+    }
+    if (allLeadingSingleton) {
+      // Reshape x from [1,1,...,M,K] to [M,K] for matmul
+      std::vector<LongType> newXShape = {x->sizeAt(-2), x->sizeAt(-1)};
+      xReshaped = new NDArray(x->reshape(x->ordering(), newXShape));
+      // Reshape z from [1,1,...,M,N] to [M,N]
+      std::vector<LongType> newZShape = {z->sizeAt(-2), z->sizeAt(-1)};
+      zReshaped = new NDArray(z->reshape(z->ordering(), newZShape));
+      x = xReshaped;
+      z = zReshaped;
+    }
+  } else if (xRank != yRank && yRank > 2 && xRank == 2) {
+    // Check if y has all singleton leading dims
+    bool allLeadingSingleton = true;
+    for (int i = 0; i < yRank - 2; ++i) {
+      if (y->sizeAt(i) != 1) {
+        allLeadingSingleton = false;
+        break;
+      }
+    }
+    if (allLeadingSingleton) {
+      // Reshape y from [1,1,...,K,N] to [K,N] for matmul
+      std::vector<LongType> newYShape = {y->sizeAt(-2), y->sizeAt(-1)};
+      yReshaped = new NDArray(y->reshape(y->ordering(), newYShape));
+      // Reshape z from [1,1,...,M,N] to [M,N]
+      std::vector<LongType> newZShape = {z->sizeAt(-2), z->sizeAt(-1)};
+      zReshaped = new NDArray(z->reshape(z->ordering(), newZShape));
+      y = yReshaped;
+      z = zReshaped;
+    }
+  }
+
+  // Update ranks after potential reshaping
+  const int xRankFinal = x->rankOf();
+  const int yRankFinal = y->rankOf();
+  const int zRankFinal = z->rankOf();
+
+  if (xRankFinal == 1 && yRankFinal == 1) {  // dot case, output is scalar (or vector with length = 1)
+    REQUIRE_TRUE(x->lengthOf() == y->lengthOf(), 0,
+                 "MATMUL OP: since input arrays are vectors they must have the same length, but got x length = %i, y "
+                 "length = %i !",
+                 x->lengthOf(), y->lengthOf());
+  } else if (xRankFinal == 1 && yRankFinal == 2) {  // vector x matrix, i.e. [4] x [4,5] = [5], output is vector
+    REQUIRE_TRUE(x->lengthOf() == y->sizeAt(yLastButOneDim), 0,
+                 "MATMUL OP: input arrays have inconsistent shapes for vector-matrix product: x %s, y %s !",
+                 ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+  } else if (xRankFinal == 2 && yRankFinal == 1) {  // matrix x vector , i.e. [4,5] x [5] = [4], output is vector
+    REQUIRE_TRUE(x->sizeAt(xLastDim) == y->lengthOf(), 0,
+                 "MATMUL OP: input arrays have inconsistent shapes for matrix-vector product: x %s, y %s !",
+                 ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+  } else {
+    REQUIRE_TRUE(xRankFinal == yRankFinal && yRankFinal == zRankFinal, 0,
+                 "MATMUL OP: input and output arrays must have the same rank, but got instead: x rank = %i, y rank = "
+                 "%i, z rank = %i !",
+                 xRankFinal, yRankFinal, zRankFinal);
+    REQUIRE_TRUE(x->sizeAt(xLastDim) == y->sizeAt(yLastButOneDim) && x->sizeAt(xLastButOneDim) == z->sizeAt(-2) &&
+                 y->sizeAt(yLastDim) == z->sizeAt(-1),
+                 0, "MATMUL OP: input/output arrays have inconsistent shapes for matrix product: x %s, y %s, z %s !",
+                 ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str(),
+                 ShapeUtils::shapeAsString(z).c_str());
+
+    if (xRankFinal > 2)  // outer dims must be the same
+      for (int i = 0; i < xRankFinal - 2; ++i)
+    REQUIRE_TRUE(x->sizeAt(i) == y->sizeAt(i) && y->sizeAt(i) == z->sizeAt(i), 0,
+                 "MATMUL OP: input/output arrays have inconsistent shapes for matrix product: x %s, y %s, z %s !",
+                 ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str(),
+                 ShapeUtils::shapeAsString(z).c_str());
+  }
+  // ******* end of input validation ******* //
+
+  MmulHelper::matmul(x, y, z, transX, transY, alpha, beta, z);
+
+  // Clean up reshaped arrays
+  delete xReshaped;
+  delete yReshaped;
+  delete zReshaped;
+
+  return Status::OK;
+}
+
+DECLARE_SYN(mMul, matmul);
+
+DECLARE_SYN(mmul, matmul);
+
+DECLARE_SYN(gemm, matmul);
+
+DECLARE_SYN(gemv, matmul);
+
+DECLARE_SYN(dot, matmul);
+
+//////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(matmul) {
+  auto xShapeInfo = inputShape->at(0);
+  auto yShapeInfo = inputShape->at(1);
+
+
+  const int iSize = (int)block.getIArguments()->size();
+  int transX = iSize > 0 ? INT_ARG(0) : 0;
+  int transY = iSize > 1 ? INT_ARG(1) : 0;
+  const int transZ = iSize > 2 ? INT_ARG(2) : 0;
+
+  if (transZ) {
+    xShapeInfo = inputShape->at(1);
+    yShapeInfo = inputShape->at(0);
+    bool temp = transX;
+    transX = !transY;
+    transY = !temp;
+  }
+
+  auto zShapeOnly = ShapeUtils::evalShapeForMatmul(xShapeInfo, yShapeInfo, transX, transY);
+
+  auto dtypeX = ArrayOptions::dataType(xShapeInfo);
+  auto dtypeY = ArrayOptions::dataType(yShapeInfo);
+
+  auto xOrder = shape::order(xShapeInfo);
+  auto yOrder = shape::order(yShapeInfo);
+  auto zOrder = xOrder == 'c' && yOrder == 'c' ? 'c' : 'f';
+
+  // we just pick the higher data type out of X and Y
+  auto dtypeZ = dtypeX > dtypeY ? dtypeX : dtypeY;
+  if(shape::isEmptyConst(xShapeInfo) || shape::isEmptyConst(yShapeInfo)) {
+    return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(ArrayOptions::dataType(xShapeInfo),zShapeOnly));
+  }
+
+  auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(dtypeZ, zOrder, zShapeOnly);
+  return SHAPELIST(newShape);
+}
+
+//////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(matmul) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS, ALL_INTS});
+}
+
+//////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(matmul_bp, 3, 2, false, 0, -2) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto eps = INPUT_VARIABLE(2);
+  auto dldx = OUTPUT_VARIABLE(0);
+  auto dldy = OUTPUT_VARIABLE(1);
+
+  int iSize = (int)block.getIArguments()->size();
+  int transX = iSize > 0 ? INT_ARG(0) : 0;
+  int transY = iSize > 1 ? INT_ARG(1) : 0;
+  const int transZ = iSize > 2 ? INT_ARG(2) : 0;
+
+  // optional use alpha nad beta
+  iSize = (int)block.getTArguments()->size();
+
+  double alpha = iSize > 0 ? T_ARG(0) : 1.0;
+  double beta = iSize > 1 ? T_ARG(1) : 0.0;
+
+  /*
+  In: x=[a,b], y=[b,c]
+  tX  tY  tZ  x       y       z       dz          dLdx                                    dLdy
+  F   F   F   [a,b]   [b,c]   [a,c]   [a,c]       [a,c]*[b,c]T = [a,b]        x*yT        [a,b]T*[a,c] = [b,c] xT*y T F
+  F   [b,a]   [b,c]   [a,c]   [a,c]       ([a,c]*[b,c]T)T = [b,a]     (x*yT)T     [b,a]*[a,c] = [b,c]         x*y F   T
+  F   [a,b]   [c,b]   [a,c]   [a,c]       ([a,c]*[c,b]) = [a,b]       x*y         [a,b]T*[a,c] = [b,c] ->T    xT*y T   T
+  F   [b,a]   [c,b]   [a,c]   [a,c]       ([a,c]*[c,b])T = [b,a]      (x*y)T      [b,a]*[a,c] = [b,c]  ->T    x*y F   F
+  T   [a,b]   [b,c]   [c,a]   [c,a]
+  */
+  // special case for scalar value
+  if (eps->isScalar()) {
+    if (x->isVector() && y->isVector()) {
+      if (x->isRowVector() && y->isRowVector()) {
+        float ySum = y->sumNumber().e<float>(0);
+        NDArray *dldxTemp = (*eps) * ySum;
+        dldx->assign(dldxTemp);
+        delete dldxTemp;
+
+        float xSum = x->sumNumber().e<float>(0);
+        NDArray *dldyTemp = (*eps) * xSum;
+        dldy->assign(dldyTemp);
+        delete dldyTemp;
+      } else if (x->isColumnVector() && y->isColumnVector()) {
+        float ySum = y->sumNumber().e<float>(0);
+        NDArray *dldxTemp = (*eps) * ySum;
+        dldx->assign(dldxTemp);
+        delete dldxTemp;
+        float xSum = x->sumNumber().e<float>(0);
+        NDArray *dldyTemp = (*eps) * xSum;
+        dldy->assign(dldyTemp);
+        delete dldyTemp;
+      } else {
+        NDArray *dldxTemp = (*eps) * (*y);
+        dldx->assign(dldxTemp);
+        delete dldxTemp;
+        NDArray *dldyTemp = (*eps) * (*x);
+        dldy->assign(dldyTemp);
+        delete dldyTemp;
+      }
+    } else {
+      // assign all ones to shape as baseline
+      auto alphaBetaBase = 1.0f;
+      if (alpha > 0.0f) {
+        alphaBetaBase *= alpha;
+      }
+
+      if (beta > 0.0f) {
+        alphaBetaBase += beta;
+      }
+
+      dldx->assign(alphaBetaBase);
+      dldy->assign(alphaBetaBase);
+      
+      // match the dimensions for reduction for matrix multiply: columns on first input, rows on second input
+      // the dimensions should match the matching dimensions to compute proper gradients wrt each input
+      // core gradient for each is sum(input) * eps as scalar
+      std::vector<LongType> axesZero({0});
+      NDArray *xSum = x->reduceAlongDimension(reduce::Sum, &axesZero);
+      NDArray *xSumScaled = *xSum * (*eps);
+      std::vector<sd::LongType> xSumShape = {xSumScaled->lengthOf(), 1};
+      NDArray* xSumRow = xSumScaled->reshape(xSumScaled->ordering(), xSumShape);
+      
+      std::vector<LongType> axes({1});
+      NDArray *ySum = y->reduceAlongDimension(reduce::Sum, &axes);
+      NDArray *ySumScaled = *ySum * (*eps);
+      std::vector<sd::LongType> ySumShape = {1, ySumScaled->lengthOf()};
+      NDArray* ySumRow = ySumScaled->reshape(ySumScaled->ordering(), ySumShape);
+
+      // execute proper multiplication: rows for first input, columns for second
+      dldx->mulRowVector(ySumRow, dldx);
+      dldy->muliColumnVector(xSumRow);
+
+      // FIXED: Proper cleanup - delete each allocated array once, add missing cleanup
+      delete xSumRow;
+      delete xSumScaled;
+      delete xSum;
+      delete ySumRow;
+      delete ySumScaled;
+      delete ySum;
+    }
+
+    return Status::OK;
+  }
+
+  matmul op;
+  op.execute({eps, y}, {dldx}, {alpha, beta}, {transZ, !transY, transX}, {});
+  op.execute({x, eps}, {dldy}, {alpha, beta}, {!transX, transZ, transY}, {});
+
+  return Status::OK;
+}
+
+//////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(matmul_bp) {
+  return SHAPELIST(CONSTANT(inputShape->at(0)), CONSTANT(inputShape->at(1)));
+}
+
+//////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(matmul_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedInputTypes(2, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/blas/tensormmul.cpp

@@ -0,0 +1,268 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_tensormmul)
+
+#include <helpers/MmulHelper.h>
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+#include <numeric>
+
+namespace sd {
+namespace ops {
+
+////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(tensormmul, 2, 1, false, 0, -1) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+
+  auto c = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(a->dataType() == b->dataType(), 0, "tensormmul: A, B and C data types must be the same");
+
+  // building axes
+  LongType axe0_size = INT_ARG(0);
+  LongType axe1_size = INT_ARG(axe0_size + 1);
+  std::vector<LongType> axes_0(axe0_size), axes_1(axe1_size);
+  for (LongType e = 0; e < axe0_size; e++) axes_0[e] = INT_ARG(e + 1);
+  for (LongType e = 0; e < axe1_size; e++) axes_1[e] = INT_ARG(e + axe0_size + 2);
+
+
+  std::vector<sd::LongType> permuteC = {};
+  MmulHelper::tensorDot(a, b, c, axes_0, axes_1,permuteC);
+  return Status::OK;
+}
+DECLARE_SYN(tensordot, tensormmul);
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(tensormmul) {
+  auto aShapeInfo = inputShape->at(0);
+  auto bShapeInfo = inputShape->at(1);
+
+  REQUIRE_TRUE(ArrayOptions::dataType(aShapeInfo) == ArrayOptions::dataType(bShapeInfo), 0,
+               "tensormmul: A and B data types must be the same");
+
+  // building axes
+  LongType axe0_size = INT_ARG(0);
+  LongType axe1_size = INT_ARG(axe0_size + 1);
+  std::vector<LongType> axes_0(axe0_size), axes_1(axe1_size);
+  for (LongType e = 0; e < axe0_size; e++) axes_0[e] = INT_ARG(e + 1);
+
+  for (LongType e = 0; e < axe1_size; e++) axes_1[e] = INT_ARG(e + axe0_size + 2);
+
+  sd_verbose("axe0: %i; axe1: %i;\n", axes_0.size(), axes_1.size());
+  // evaluate shapes
+  std::vector<LongType> permutAt, permutBt;
+  std::vector<LongType> shapeAt, shapeBt;
+  auto outShape =
+      ShapeUtils::evalShapeForTensorDot(aShapeInfo, bShapeInfo, axes_0, axes_1, permutAt, permutBt,
+                                                        shapeAt, shapeBt);
+
+  auto desc = new  ShapeDescriptor(ArrayOptions::dataType(aShapeInfo), 'c', outShape);
+  auto result = SHAPELIST(ConstantShapeHelper::getInstance().createShapeInfo(desc));
+  delete desc;
+  return result;
+}
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(tensormmul) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedInputTypes(2, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+
+// Comparator for sorting indices vector based on comparison of array values
+struct IndexComparator
+{
+  const std::vector<LongType>& array;
+
+  IndexComparator(const std::vector<LongType>& arr): array(arr) {}
+
+  bool operator() (LongType i1, LongType i2)
+  {
+    return array[i1] < array[i2];
+  }
+};
+
+
+std::vector<LongType> argsort(const std::vector<LongType>& array)
+{
+  std::vector<LongType> indices(array.size());
+  for (size_t i = 0; i < array.size(); ++i) indices[i] = i;
+
+  std::sort(indices.begin(), indices.end(), IndexComparator(array));
+
+  return indices;
+}
+
+////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(tensormmul_bp, 4, 2, false, 0, -1) {
+  auto A = INPUT_VARIABLE(0);
+  auto B = INPUT_VARIABLE(1);
+  auto C = INPUT_VARIABLE(2);
+  auto dC = INPUT_VARIABLE(3);
+  auto originalDC = dC;
+
+  //scalar case, tile value to be whatever the c value is. common when directly attached to the loss
+  if(dC->isScalar()) {
+    auto newVec = const_cast<NDArray *>(C);
+    auto* newShapeVec = newVec->getShapeAsVector();
+    dC = new NDArray('c', *newShapeVec, dC->dataType(), dC->getContext());
+    delete newShapeVec;
+  }
+
+
+  auto gradA = OUTPUT_VARIABLE(0);
+  auto gradB = OUTPUT_VARIABLE(1);
+
+  LongType axe0_size = INT_ARG(0);
+  LongType axe1_size = INT_ARG(axe0_size + 1);
+  std::vector<LongType> axes0Sum(axe0_size), axes1Sum(axe1_size);
+
+  //find the passed in axes for the feed forward
+  for (LongType e = 0; e < axe0_size; e++) axes0Sum[e] = INT_ARG(e + 1);
+  for (LongType e = 0; e < axe1_size; e++) axes1Sum[e] = INT_ARG(e + axe0_size + 2);
+
+
+  auto Arank = A->rankOf();
+  auto Brank = B->rankOf();
+  auto dCrank = dC->rankOf();
+
+
+  //part of the permtue axes before matrix multiply happens
+  std::vector<LongType> axes_a_grad;
+  for (LongType i = 0; i < Arank; ++i)
+    axes_a_grad.push_back(i);
+
+  for (size_t i = 0; i < axes0Sum.size(); ++i)
+    axes_a_grad.erase(std::remove(axes_a_grad.begin(), axes_a_grad.end(), axes0Sum[i]), axes_a_grad.end());
+
+
+
+  //part of matrix multiply axes before matrix multiply happens
+  std::vector<LongType> axes_b_grad;
+  for (LongType i = 0; i < Brank; ++i)
+    axes_b_grad.push_back(i);
+
+  for (size_t i = 0; i < axes1Sum.size(); ++i)
+    axes_b_grad.erase(std::remove(axes_b_grad.begin(), axes_b_grad.end(), axes1Sum[i]), axes_b_grad.end());
+
+  //used for post result permute to reshape result to be expected output
+  std::vector<LongType> grad_a_axes;
+  grad_a_axes.insert(grad_a_axes.end(), axes_a_grad.begin(), axes_a_grad.end());
+  grad_a_axes.insert(grad_a_axes.end(), axes1Sum.begin(), axes1Sum.end());
+
+  //used for post result permute to reshape result to be expected output
+  std::vector<LongType> grad_b_axes;
+  grad_b_axes.insert(grad_b_axes.end(), axes0Sum.begin(), axes0Sum.end());
+  grad_b_axes.insert(grad_b_axes.end(), axes_b_grad.begin(), axes_b_grad.end());
+
+  LongType starting = dCrank - axes_a_grad.size();
+  std::vector<LongType> axes_a_gradA;
+  for (LongType i = starting; i < dCrank; i++) {
+    axes_a_gradA.push_back(i);
+  }
+
+  std::vector<LongType> axes_b_gradA;
+  for (size_t i = 0; i < axes_b_grad.size(); i++) {
+    axes_b_gradA.push_back(i);
+  }
+
+  std::vector<LongType> axes_a_gradB;
+  for (size_t i = 0; i < axes_a_grad.size(); i++) {
+    axes_a_gradB.push_back(i);
+  }
+
+  LongType start = dCrank - axes_a_gradA.size();
+  std::vector<LongType> axes_b_gradB;
+  for (LongType i = start; i < dCrank; i++) {
+    axes_b_gradB.push_back(i);
+  }
+
+  //create final axes before for matrix multiply
+  std::vector<LongType> aPermuteAxesBefore;
+  aPermuteAxesBefore.insert(aPermuteAxesBefore.end(), axes_a_grad.begin(), axes_a_grad.end());
+  aPermuteAxesBefore.insert(aPermuteAxesBefore.end(), axes0Sum.begin(), axes0Sum.end());
+
+
+
+  //create final axes before for matrix multiply
+  std::vector<LongType> bPermuteAxesBefore;
+  bPermuteAxesBefore.insert(bPermuteAxesBefore.end(), axes_b_grad.begin(), axes_b_grad.end());
+  bPermuteAxesBefore.insert(bPermuteAxesBefore.end(), axes1Sum.begin(), axes1Sum.end());
+
+  auto aPermArgsAfter = argsort(grad_a_axes);
+  auto bPermArgsAfter = argsort(grad_b_axes);
+  auto newA = A->permute(aPermuteAxesBefore, false, false);
+  std::vector<LongType> empty;
+  auto newB = B->permute(bPermuteAxesBefore, false, false);
+
+
+  //perform the actual matrix multiplication
+  MmulHelper::tensorDot2(dC, newB, gradA, axes_a_gradA, axes_b_gradA, empty, empty, aPermArgsAfter, gradA);
+  MmulHelper::tensorDot2(newA, dC, gradB, axes_a_gradB, axes_b_gradB, empty, empty, bPermArgsAfter, gradB);
+
+  // FIXED: permute() with copyToNewBuff=false returns view - only delete if not view
+  if (newA != nullptr && !newA->isView()) {
+    delete newA;
+  }
+  if (newB != nullptr && !newB->isView()) {
+    delete newB;
+  }
+  if(dC != originalDC) {
+    delete dC;
+  }
+
+  return Status::OK;
+}
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(tensormmul_bp) {
+  auto aShapeInfo = inputShape->at(0);
+  auto bShapeInfo = inputShape->at(1);
+  auto cShapeInfo = inputShape->at(2);
+  auto dLShapeInfo = inputShape->at(3);
+
+  REQUIRE_TRUE((ArrayOptions::dataType(aShapeInfo) == ArrayOptions::dataType(bShapeInfo) &&
+                (ArrayOptions::dataType(dLShapeInfo) == ArrayOptions::dataType(aShapeInfo))),
+               0, "tensormmul_bp: A, B and dLdC data types must be the same");
+
+  return SHAPELIST(CONSTANT(aShapeInfo), CONSTANT(bShapeInfo));
+}
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(tensormmul_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {FLOAT32, DOUBLE, HALF})  // maybe better ALL_FLOATS
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedInputTypes(2, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(1, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/boolean_not.cpp

@@ -0,0 +1,45 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_not)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(boolean_not, 1, 1, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  x->applyTransform(transform::Not, z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_not) {
+  getOpDescriptor()->setAllowedInputTypes(0, BOOL)->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/choose.cpp

@@ -0,0 +1,87 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author Adam Gibson
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_choose)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/choose.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(choose, -1, 2, false, -2, -1) {
+  int mode = INT_ARG(0);
+  auto result = OUTPUT_VARIABLE(0);
+  auto numResults = OUTPUT_VARIABLE(1);
+
+  if (block.width() > 1) {
+    auto arg = INPUT_VARIABLE(0);
+    auto comp = INPUT_VARIABLE(1);
+
+    helpers::chooseFunctorArray(block.launchContext(), arg, comp, mode, result, numResults);
+
+  }  // scalar case
+  else {
+    double scalar = T_ARG(0);
+    auto arg = INPUT_VARIABLE(0);
+    helpers::chooseFunctorScalar(block.launchContext(), arg, scalar, mode, result, numResults);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(choose) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS});
+}
+
+DECLARE_SHAPE_FN(choose) {
+  int mode = INT_ARG(0);
+  auto numResults = NDArrayFactory::create<LongType>(0L);
+  if (block.width() > 1) {
+    auto first = INPUT_VARIABLE(0);
+    auto second = INPUT_VARIABLE(1);
+
+
+    helpers::chooseFunctorArray(block.launchContext(), first, second, mode, nullptr, numResults);
+  } else {
+    auto first = INPUT_VARIABLE(0);
+    double scalar = T_ARG(0);
+
+    helpers::chooseFunctorScalar(block.launchContext(), first, scalar, mode, nullptr, numResults);
+  }
+
+  auto newShape = ConstantShapeHelper::getInstance().vectorShapeInfo(numResults->e<LongType>(0),
+                                                                     ArrayOptions::dataType(inputShape->at(0)));
+
+  auto shapeScalar = ConstantShapeHelper::getInstance().scalarShapeInfo(INT64);
+  return SHAPELIST(newShape, shapeScalar);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/eq_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_eq_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(eq_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<double>(0) == y->e<double>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+DECLARE_SYN(Equals, eq_scalar);
+// DECLARE_SYN(equals, eq_scalar);
+
+DECLARE_TYPES(eq_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/gt_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_gt_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(gt_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) > y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+// DECLARE_SYN(Greater, gt_scalar);
+// DECLARE_SYN(greater, gt_scalar);
+
+DECLARE_TYPES(gt_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/gte_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_gte_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(gte_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) >= y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+DECLARE_SYN(GreaterOrEquals, gte_scalar);
+DECLARE_SYN(greaterOrEquals, gte_scalar);
+
+DECLARE_TYPES(gte_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/is_non_decreasing.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author @cpuheater
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_is_non_decreasing)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/compare_elem.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(is_non_decreasing, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+
+  // in case of empty input there's nothing to do
+  if (input->isEmpty()) return Status::EQ_TRUE;
+
+  bool isNonDecreasing = true;
+
+  helpers::compare_elem(block.launchContext(), input, false, isNonDecreasing);
+
+  if (isNonDecreasing)
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+
+DECLARE_TYPES(is_non_decreasing) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/is_numeric_tensor.cpp

@@ -0,0 +1,43 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author @cpuheater
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_is_numeric_tensor)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/compare_elem.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(is_numeric_tensor, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+
+  return input->isR() || input->isZ() ? Status::EQ_TRUE : Status::EQ_FALSE;
+}
+
+DECLARE_TYPES(is_numeric_tensor) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/is_strictly_increasing.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author @cpuheater
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_is_strictly_increasing)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/compare_elem.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(is_strictly_increasing, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+
+  // in case of empty input there's nothing to do
+  if (input->isEmpty()) return Status::EQ_TRUE;
+
+  bool isStrictlyIncreasing = true;
+
+  helpers::compare_elem(block.launchContext(), input, true, isStrictlyIncreasing);
+
+  if (isStrictlyIncreasing)
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+
+DECLARE_TYPES(is_strictly_increasing) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/lt_scalar.cpp

@@ -0,0 +1,50 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_lt_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(lt_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) < y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+
+
+DECLARE_TYPES(lt_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/lte_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_lte_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(lte_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) <= y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+DECLARE_SYN(LessOrEquals, lte_scalar);
+DECLARE_SYN(lessorequals, lte_scalar);
+
+DECLARE_TYPES(lte_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/neq_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_neq_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(neq_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) != y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+DECLARE_SYN(NotEquals, neq_scalar);
+DECLARE_SYN(notequals, neq_scalar);
+
+DECLARE_TYPES(neq_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/select.cpp

@@ -0,0 +1,127 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_select)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(select, 3, 1, false, 0, 0) {
+  auto cond = INPUT_VARIABLE(0);
+  auto x = INPUT_VARIABLE(1);
+  auto y = INPUT_VARIABLE(2);
+  //TODO: for some reason y being empty
+  //should not necessarily yield an empty result
+  //the loss test I'm currently dealing with seems
+  //to need to output a value yet it ends up being empty.
+  //we need to figure out why
+  if(x->isEmpty() || y->isEmpty() || cond->isEmpty()) {
+    return Status::OK;
+  }
+  if (x->isScalar()) {
+    REQUIRE_TRUE(cond->isScalar(), 0,
+                 "Select: Condition should gave either equal shape to X/Y first dimension or to be scalar");
+
+    auto z = OUTPUT_VARIABLE(0);
+
+    if (y->isR()) {
+#ifdef HAS_DOUBLE
+      auto v = !cond->e<bool>(0) ? y->e<double>(0) : x->e<double>(0);
+#elif defined(HAS_FLOAT32)
+      auto v = !cond->e<bool>(0) ? y->e<float>(0) : x->e<float>(0);
+#else
+#error "No floating-point type available for select operation"
+#endif
+      z->p(0, v);
+    } else {
+      auto v = !cond->e<bool>(0) ? y->e<LongType>(0) : x->e<LongType>(0);
+      z->p(0, v);
+    }
+  } else {
+    bool same = cond->isSameShape(x);
+    REQUIRE_TRUE(cond->isScalar() || cond->lengthOf() == x->sizeAt(0) || same, 0,
+                 "Select: Condition should gave either equal shape to X/Y first dimension or to be scalar");
+    if (same) {
+      auto z = OUTPUT_VARIABLE(0);
+
+      for (int e = 0; e < cond->lengthOf(); e++) {
+        if (y->isR()) {
+#ifdef HAS_DOUBLE
+          auto r = !cond->e<bool>(e) ? y->e<double>(e) : x->e<double>(e);
+#elif defined(HAS_FLOAT32)
+          auto r = !cond->e<bool>(e) ? y->e<float>(e) : x->e<float>(e);
+#else
+#error "No floating-point type available for select operation"
+#endif
+          z->p(e, r);
+        } else {
+          auto r = !cond->e<bool>(e) ? y->e<LongType>(e) : x->e<LongType>(e);
+          z->p(e, r);
+        }
+      }
+    } else {
+      REQUIRE_TRUE(cond->lengthOf() == x->sizeAt(0), 0,
+                   "Condition length should be equal to the dim0 of x/y to act as TAD-mask, but got %d instead",
+                   cond->lengthOf());
+
+      auto z = OUTPUT_VARIABLE(0);
+      std::vector<LongType> idxs;
+      idxs.push_back(0);
+      auto dims = ShapeUtils::evalDimsToExclude(x->rankOf() ,1,idxs.data());
+      auto tadsX = x->allTensorsAlongDimension(*dims);
+      auto tadsY = y->allTensorsAlongDimension(*dims);
+      auto tadsZ = z->allTensorsAlongDimension(*dims);
+
+      for (int e = 0; e < tadsX.size(); e++) {
+        if (!cond->e<bool>(e)) {
+          tadsZ.at(e)->assign(tadsY.at(e));
+        } else {
+          tadsZ.at(e)->assign(tadsX.at(e));
+        }
+      }
+
+      delete dims;
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(select) {
+  auto inShape = inputShape->at(1);
+  return SHAPELIST(CONSTANT(inShape));
+}
+
+DECLARE_TYPES(select) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, BOOL)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedInputTypes(2, ANY)
+      ->setAllowedOutputTypes(1, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/where.cpp

@@ -0,0 +1,339 @@
+/* ******************************************************************************
+*
+*
+* This program and the accompanying materials are made available under the
+* terms of the Apache License, Version 2.0 which is available at
+* https://www.apache.org/licenses/LICENSE-2.0.
+*
+*  See the NOTICE file distributed with this work for additional
+*  information regarding copyright ownership.
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations
+* under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_Where)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/where.h>
+
+namespace sd {
+namespace ops {
+
+// Helper function to evaluate condition regardless of underlying data type
+inline bool evaluateCondition(NDArray* condition, int index) {
+ switch(condition->dataType()) {
+#if defined(HAS_BOOL)
+   case DataType::BOOL:
+     return condition->e<bool>(index);
+#endif
+#if defined(HAS_INT8)
+   case DataType::INT8:
+     return condition->e<int8_t>(index) != 0;
+#endif
+#if defined(HAS_INT16)
+   case DataType::INT16:
+     return condition->e<int16_t>(index) != 0;
+#endif
+#if defined(HAS_INT32)
+   case DataType::INT32:
+     return condition->e<int32_t>(index) != 0;
+#endif
+#if defined(HAS_LONG)
+   case DataType::INT64:
+     return condition->e<sd::LongType>(index) != 0;
+#endif
+#if defined(HAS_UINT8)
+   case DataType::UINT8:
+     return condition->e<uint8_t>(index) != 0;
+#endif
+#if defined(HAS_UINT16)
+   case DataType::UINT16:
+     return condition->e<uint16_t>(index) != 0;
+#endif
+#if defined(HAS_UINT32)
+   case DataType::UINT32:
+     return condition->e<uint32_t>(index) != 0;
+#endif
+#if defined(HAS_UNSIGNEDLONG)
+   case DataType::UINT64:
+     return condition->e<sd::UnsignedLong>(index) != 0;
+#endif
+#if defined(HAS_FLOAT16)
+   case DataType::HALF:
+     return condition->e<float16>(index) != static_cast<float16>(0.0f);
+#endif
+#if defined(HAS_BFLOAT16)
+   case DataType::BFLOAT16:
+     return condition->e<bfloat16>(index) != static_cast<bfloat16>(0.0f);
+#endif
+#if defined(HAS_FLOAT32)
+   case DataType::FLOAT32:
+     return condition->e<float>(index) != 0.0f;
+#endif
+#if defined(HAS_DOUBLE)
+   case DataType::DOUBLE:
+     return condition->e<double>(index) != 0.0;
+#endif
+   default:
+     // Fallback: try to interpret as int32 and check if non-zero
+#if defined(HAS_INT32)
+#ifdef __cpp_exceptions
+     try {
+       return condition->e<int32_t>(index) != 0;
+     } catch (...) {
+       // Last resort: assume false to maintain safe behavior
+       return false;
+     }
+#else
+     return condition->e<int32_t>(index) != 0;
+#endif
+#else
+     // If INT32 is not available, return false as safe default
+     return false;
+#endif
+ }
+}
+
+// Helper function to perform element-wise where with proper broadcasting
+void performBroadcastedWhere(NDArray* condition, NDArray* x, NDArray* y, NDArray* z) {
+ // We'll process each element of the output array z
+ // and determine the appropriate indices for condition, x, and y based on broadcasting rules
+
+ auto* zShape = z->getShapeAsVector();
+ auto* condShape = condition->getShapeAsVector();
+ auto* xShape = x->getShapeAsVector();
+ auto* yShape = y->getShapeAsVector();
+
+ // For each element in the output array
+ for (LongType i = 0; i < z->lengthOf(); i++) {
+   // Convert linear index to multi-dimensional indices for output array
+   std::vector<LongType> zIndices(z->rankOf());
+   LongType remainder = i;
+   for (int dim = z->rankOf() - 1; dim >= 0; dim--) {
+     zIndices[dim] = remainder % z->sizeAt(dim);
+     remainder /= z->sizeAt(dim);
+   }
+
+   // Calculate corresponding indices in condition, x, and y arrays using broadcasting rules
+   auto getLinearIndex = [](const std::vector<LongType>& multiIndices, const std::vector<LongType>& shape, NDArray* array) -> LongType {
+     LongType linearIndex = 0;
+     LongType stride = 1;
+     int srcDim = shape.size() - 1;
+
+     for (int dim = multiIndices.size() - 1; dim >= 0; dim--) {
+       LongType srcIndex = 0;
+       if (srcDim >= 0) {
+         if (shape[srcDim] == 1) {
+           srcIndex = 0; // Broadcast dimension
+         } else {
+           srcIndex = multiIndices[dim];
+         }
+         srcDim--;
+       }
+       linearIndex += srcIndex * stride;
+       if (srcDim >= 0) {
+         stride *= shape[srcDim + 1];
+       }
+     }
+     return linearIndex;
+   };
+
+   LongType condIndex = condition->lengthOf() == 1 ? 0 : getLinearIndex(zIndices, *condShape, condition);
+   LongType xIndex = x->lengthOf() == 1 ? 0 : getLinearIndex(zIndices, *xShape, x);
+   LongType yIndex = y->lengthOf() == 1 ? 0 : getLinearIndex(zIndices, *yShape, y);
+
+   // Apply the where logic
+   if (z->isR()) {
+#ifdef HAS_DOUBLE
+     auto result = evaluateCondition(condition, condIndex) ?
+                                                           x->e<double>(xIndex) : y->e<double>(yIndex);
+#elif defined(HAS_FLOAT32)
+     auto result = evaluateCondition(condition, condIndex) ?
+                                                           x->e<float>(xIndex) : y->e<float>(yIndex);
+#else
+#error "No floating-point type available for where operation"
+#endif
+     z->p(i, result);
+   } else{
+     auto result = evaluateCondition(condition, condIndex) ?
+                                                           x->e<LongType>(xIndex) : y->e<LongType>(yIndex);
+     z->p(i, result);
+   }
+ }
+
+ delete zShape;
+ delete condShape;
+ delete xShape;
+ delete yShape;
+}
+
+CUSTOM_OP_IMPL(Where, 1, 1, false, 0, 0) {
+ auto condition = INPUT_VARIABLE(0);
+ auto z = OUTPUT_VARIABLE(0);
+ if (z->isEmpty()) return Status::OK;
+
+ if (block.width() == 3) {
+   auto x = INPUT_VARIABLE(1);
+   auto y = INPUT_VARIABLE(2);
+
+   // Check if x and y can be broadcast together (instead of requiring exact same shape)
+   REQUIRE_TRUE(x->isSameShape(y) || ShapeUtils::areShapesBroadcastable(*x, *y), 0,
+                "X and Y must have equal shapes or be broadcastable. X shape: %s, Y shape: %s",
+                ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+   // Case 1: All arrays have exact shape matching (element-wise operation)
+   if (condition->isSameShape(x) && x->isSameShape(y)) {
+     // FIXME: for perf it might be better to issue memcpy here, and fill only mismatched values from either X or Y
+     for (int e = 0; e < condition->lengthOf(); e++) {
+       if (z->isR()) {
+#ifdef HAS_DOUBLE
+         auto r = !evaluateCondition(condition, e) ? y->e<double>(e) : x->e<double>(e);
+#elif defined(HAS_FLOAT32)
+         auto r = !evaluateCondition(condition, e) ? y->e<float>(e) : x->e<float>(e);
+#else
+#error "No floating-point type available for where operation"
+#endif
+         z->p(e, r);
+       } else {
+         auto r = !evaluateCondition(condition, e) ? y->e<LongType>(e) : x->e<LongType>(e);
+         z->p(e, r);
+       }
+     }
+   }
+   // Case 2: Broadcasting is possible (most flexible case)
+   else if (ShapeUtils::areShapesBroadcastable(*condition, *x) &&
+            ShapeUtils::areShapesBroadcastable(*condition, *y) &&
+            ShapeUtils::areShapesBroadcastable(*x, *y)) {
+     performBroadcastedWhere(condition, x, y, z);
+   }
+   // Case 3: TAD-mask operation (legacy behavior for specific cases)
+   else if (condition->rankOf() == 1 && condition->lengthOf() == x->sizeAt(0)) {
+     std::vector<LongType> zero({0});
+     auto dims = ShapeUtils::evalDimsToExclude(x->rankOf(), 1, zero.data());
+     auto tadsX = x->allTensorsAlongDimension(*dims);
+     auto tadsY = y->allTensorsAlongDimension(*dims);
+     auto tadsZ = z->allTensorsAlongDimension(*dims);
+
+     for (int e = 0; e < tadsX.size(); e++) {
+       if (!evaluateCondition(condition, e)) {
+         tadsZ.at(e)->assign(tadsY.at(e));
+       } else {
+         tadsZ.at(e)->assign(tadsX.at(e));
+       }
+     }
+
+     delete dims;
+   }
+   // Case 4: Invalid shapes - provide detailed error message
+   else {
+     std::string condShape = ShapeUtils::shapeAsString(condition);
+     std::string xShape = ShapeUtils::shapeAsString(x);
+     std::string yShape = ShapeUtils::shapeAsString(y);
+
+     REQUIRE_TRUE(false, 0,
+                  "Where operation: Invalid shapes for broadcasting. "
+                  "Condition shape: %s, X shape: %s, Y shape: %s. "
+                  "Condition must either: (1) match X/Y shapes exactly, "
+                  "(2) be broadcastable with X/Y shapes, or "
+                  "(3) be 1D with length equal to first dimension of X/Y for TAD-mask operation.",
+                  condShape.c_str(), xShape.c_str(), yShape.c_str());
+   }
+ } else {
+   // in this case we return 2D matrix, which basically contains coordinates fo true
+   REQUIRE_TRUE(block.width() == 1, 0, "Where op takes either 1 or 3 operands, But got %d operands instead",
+                block.width());
+   auto output = OUTPUT_VARIABLE(0);
+   std::vector<LongType> zero({0});
+
+   int width = condition->rankOf();
+   if (z->isEmpty()) return Status::OK;
+
+   std::vector<LongType> *dims = ShapeUtils::evalDimsToExclude(width,1,zero.data());
+
+   helpers::_where(block.launchContext(), *condition, *output, block.workspace());
+   delete dims;
+ }
+ return Status::OK;
+}
+
+DECLARE_SHAPE_FN(Where) {
+ if (block.width() == 3) {
+   auto x = INPUT_VARIABLE(1);
+   auto y = INPUT_VARIABLE(2);
+
+   // Calculate the broadcast result shape for x and y
+   LongType* resultShapeInfo = nullptr;
+   bool canBroadcast = ShapeUtils::evalBroadcastShapeInfo(*x, *y, true, resultShapeInfo, block.getWorkspace());
+
+   if (canBroadcast && resultShapeInfo != nullptr) {
+     return SHAPELIST(CONSTANT(resultShapeInfo));
+   } else {
+     // Fallback to x's shape if broadcasting fails (should have been caught in validation)
+     auto inShape = inputShape->at(1);
+     return SHAPELIST(CONSTANT(inShape));
+   }
+ } else {
+   // FIXME: we can't estimate result here in this case
+   // output shape is the 2D tensor num_true x rankOf (inShape)
+   auto condition = INPUT_VARIABLE(0);
+   auto inShape = inputShape->at(0);
+   LongType numOfTrue = 0;  // condition->reduceNumber(reduce::CountNonZero, nullptr).e<sd::LongType>(0);
+   for (LongType i = 0; i < condition->lengthOf(); i++)
+     if (evaluateCondition(condition, i)) numOfTrue++;
+
+   LongType * theNewShape;
+   if (numOfTrue > 0) {
+     LongType* newShape;
+     ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+     newShape[0] = 2;
+     newShape[1] = numOfTrue;
+     newShape[2] = shape::rank(inShape);
+     newShape[3] = 1;
+     newShape[4] = 1;
+     newShape[5] = 0;
+     newShape[6] = 1;
+     newShape[7] = 99;
+#if defined(HAS_LONG)
+     ShapeUtils::updateStridesAndType(newShape, INT64, 'c');
+#else
+     // Fallback to INT32 if INT64 is not available
+     ShapeUtils::updateStridesAndType(newShape, INT32, 'c');
+#endif
+
+     theNewShape = CONSTANT(newShape);
+     RELEASE(newShape, block.getWorkspace());
+   } else {
+#if defined(HAS_LONG)
+     theNewShape = ConstantShapeHelper::getInstance().emptyShapeInfo(INT64);
+#else
+     // Fallback to INT32 if INT64 is not available
+     theNewShape = ConstantShapeHelper::getInstance().emptyShapeInfo(INT32);
+#endif
+   }
+
+   return SHAPELIST(theNewShape);
+ }
+}
+
+DECLARE_TYPES(Where) {
+ getOpDescriptor()
+     ->setAllowedInputTypes(0, ANY)  // bool
+     ->setAllowedInputTypes(1, ANY)
+     ->setAllowedInputTypes(2, ANY)
+     ->setAllowedOutputTypes(0, {ALL_INTS, ALL_FLOATS,BOOL});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/where_np.cpp

@@ -0,0 +1,179 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author Adam Gibson
+//
+
+#include <ops/declarable/headers/boolean.h>
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_where_np)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(where_np, -1, 1, false, 0, 0) {
+  auto condition = INPUT_VARIABLE(0);
+
+  if (block.width() == 3) {
+    auto x = INPUT_VARIABLE(1);
+    auto y = INPUT_VARIABLE(2);
+
+    auto z = OUTPUT_VARIABLE(0);
+    int numMatches = 0;
+    // if cond matches x/y shape - we have per-element mask
+    if (condition->isSameShape(x)) {
+      // FIXME: for perf it might be better to issue memcpy here, and fill only mismatched values from either X or Y
+      if (y->isScalar()) {
+        if (y->isR()) {
+          for (int e = 0; e < condition->lengthOf(); e++) {
+#ifdef HAS_DOUBLE
+            auto r = condition->e<bool>(e) ? y->e<double>(0) : x->e<double>(e);
+#elif defined(HAS_FLOAT32)
+            auto r = condition->e<bool>(e) ? y->e<float>(0) : x->e<float>(e);
+#else
+#error "No floating-point type available for where_np operation"
+#endif
+            z->p(e, r);
+          }
+        } else{
+          for (int e = 0; e < condition->lengthOf(); e++) {
+            auto r = condition->e<bool>(e) ? y->e<LongType>(0) : x->e<LongType>(e);
+            z->p(e, r);
+          }
+        }
+      } else {
+        if (y->isR()) {
+          for (int e = 0; e < condition->lengthOf(); e++) {
+            if (condition->e<bool>(e)) {
+#ifdef HAS_DOUBLE
+              auto r = y->e<double>(numMatches);
+#elif defined(HAS_FLOAT32)
+              auto r = y->e<float>(numMatches);
+#else
+#error "No floating-point type available for where_np operation"
+#endif
+              z->p(e, r);
+              numMatches++;
+            } else {
+#ifdef HAS_DOUBLE
+              auto r = x->e<double>(e);
+#elif defined(HAS_FLOAT32)
+              auto r = x->e<float>(e);
+#else
+#error "No floating-point type available for where_np operation"
+#endif
+              z->p(e, r);
+            }
+          }
+        } else {
+          for (int e = 0; e < condition->lengthOf(); e++) {
+            if (condition->e<bool>(e)) {
+              auto r = y->e<LongType>(numMatches);
+              z->p(e, r);
+              numMatches++;
+            } else {
+              auto r = x->e<LongType>(e);
+              z->p(e, r);
+            }
+          }
+        }
+      }
+    } else {
+      REQUIRE_TRUE(condition->lengthOf() == x->sizeAt(0), 0,
+                   "Condition length should be equal to the dim0 of x/y to act as TAD-mask, but got %d instead",
+                   condition->lengthOf());
+
+      std::vector<LongType> idxs;
+      idxs.push_back(0);
+      auto dims = ShapeUtils::evalDimsToExclude(x->rankOf(), 1,idxs.data());
+      auto tadsX = x->allTensorsAlongDimension(*dims);
+      auto tadsY = y->allTensorsAlongDimension(*dims);
+      auto tadsZ = z->allTensorsAlongDimension(*dims);
+
+      for (int e = 0; e < tadsX.size(); e++) {
+        if (!condition->e<bool>(e))
+          tadsZ.at(e)->assign(tadsY.at(e));
+        else
+          tadsZ.at(e)->assign(tadsX.at(e));
+      }
+
+      delete dims;
+    }
+  } else {
+    // in this case we return 2D matrix, which basically contains coordinates fo true
+
+    REQUIRE_TRUE(block.width() == 1, 0, "Where op takes either 1 or 3 operands, But got %d operands instead",
+                 block.width());
+    LongType width = condition->rankOf();
+
+    Where op;
+    auto res(op.evaluate({condition}));
+    REQUIRE_OK(res.status());
+    NDArray* whereTrue = res.at(0);
+
+    if (whereTrue->isEmpty()) return Status::OK;
+    for (LongType outNext = 0; outNext < width; ++outNext) {
+      auto output = OUTPUT_VARIABLE(outNext);
+      for (LongType e = 0; e < output->lengthOf(); ++e) {
+        output->p<LongType>(e, whereTrue->e<LongType>(e, outNext));
+      }
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(where_np) {
+  auto shapes = SHAPELIST();
+  if (block.width() == 3) {
+    auto inShape = inputShape->at(1);
+    shapes->push_back(CONSTANT(inShape));
+  } else {
+    auto condition = INPUT_VARIABLE(0);
+
+    LongType numOfTrue = 0LL;  // condition->reduceNumber(reduce::CountNonZero).e<sd::LongType>(0);
+    for (LongType i = 0; i < condition->lengthOf(); ++i)
+      if (condition->e<bool>(i)) numOfTrue++;
+
+    // output shape - a tuple of rank(inShape) 1D tensors with numOfTrue len
+    if (numOfTrue) {
+      for (LongType e = 0; e < condition->rankOf(); ++e) {
+        shapes->push_back(ConstantShapeHelper::getInstance().vectorShapeInfo(numOfTrue, INT64));
+      }
+    } else {
+      shapes->push_back(ConstantShapeHelper::getInstance().emptyShapeInfo(INT64));
+    }
+  }
+  return shapes;
+}
+
+DECLARE_TYPES(where_np) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, BOOL)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedInputTypes(2, ANY)
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/add.cpp

@@ -0,0 +1,109 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_add)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(add, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Add(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z && !tZ->isEmpty()) {
+    OVERWRITE_RESULT(tZ);
+  }
+  return Status::OK;
+}
+
+DECLARE_TYPES(add) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(ANY);
+}
+
+DECLARE_TYPES(add_bp) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
+
+CUSTOM_OP_IMPL(add_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+    gradY->assign(epsNext);
+    gradX->assign(epsNext);
+  } else if (y->isScalar()) {
+    // scalar case
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(tmp);
+    gradX->assign(epsNext);
+    delete tmp;
+  } else {
+    // broadcast case
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else
+      gradX->assign(epsNext);
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(epsNext);
+  }
+
+
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(add_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/assign.cpp

@@ -0,0 +1,123 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#include <helpers/StringUtils.h>
+#if NOT_EXCLUDED(OP_assign)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(assign, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = block.width() < 2 ? x: INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  // Check if any array is of string type
+  if (x->isS() || y->isS() || z->isS()) {
+    // Handle string broadcast at high level
+    StringUtils::broadcastStringAssign(x,z);
+    return Status::OK;
+  }
+
+  NDArray *castedX = x->dataType() == z->dataType() ? x : x->cast(z->dataType());
+  NDArray *castedY = y->dataType() == z->dataType() ? y : y->cast(z->dataType());
+
+  ArrayOptions::validateSingleDataType(ArrayOptions::dataType(castedX->shapeInfo()));
+  ArrayOptions::validateSingleDataType(ArrayOptions::extra(castedY->shapeInfo()));
+  ArrayOptions::validateSingleDataType(ArrayOptions::extra(z->shapeInfo()));
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Assign(), castedX, castedY, z);
+
+  if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  // Cleanup casted arrays if they were allocated
+  if (castedX != x) delete castedX;
+  if (castedY != y) delete castedY;
+
+  return Status::OK;
+}
+DECLARE_SYN(set, assign);
+DECLARE_SYN(copy, assign);
+
+DECLARE_TYPES(assign) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL})
+      ->setAllowedInputTypes(1, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL})
+      ->setAllowedOutputTypes(0, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL});
+}
+
+DECLARE_TYPES(assign_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_INTS,ALL_FLOATS,ALL_STRINGS});
+}
+
+CUSTOM_OP_IMPL(assign_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = block.width() < 2 ? x->dup(x->ordering(), false) : INPUT_VARIABLE(1);  // dup() already returns NDArray*
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  float zero = 0.0f;
+  gradX->assign(zero);
+
+  if (x->isSameShape(y)) {
+    gradY->assign(epsNext);
+  } else if (y->isScalar()) {
+    auto sum = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(sum);
+    delete sum;
+  } else {
+    // broadcastable
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(epsNext);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(assign_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/atan2.cpp

@@ -0,0 +1,55 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com, created on 10.02.18.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_tf_atan2)
+
+#include <ops/declarable/headers/broadcastable.h>
+
+namespace sd {
+namespace ops {
+
+BROADCASTABLE_OP_IMPL(tf_atan2, 0, 0) {
+  auto y = INPUT_VARIABLE(0);
+  auto x = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(BroadcastOpsTuple::custom(scalar::Atan2, pairwise::Atan2, broadcast::Atan2), y, z, true);
+
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(tf_atan2) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_and.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_or)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_and, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalAnd, pairwise::LogicalAnd, broadcast::LogicalAnd), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_and: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_and) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_or.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_or)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_or, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalOr, pairwise::LogicalOr, broadcast::LogicalOr), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_and: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_or) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_xor.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_xor)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_xor, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalXor, pairwise::LogicalXor, broadcast::LogicalXor), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_xor: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_xor) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/divide.cpp

@@ -0,0 +1,150 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_divide)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(divide, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "DIVIDE OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Divide(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Div, divide);
+
+DECLARE_TYPES(divide) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(divide_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(divide_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    NDArray *gradXTemp = (*epsNext) / (*y);
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    // Y gradient
+    NDArray *numerator = (*epsNext) * (*x);
+    NDArray *denominator = (*y) * (*y);
+    NDArray *gradYTemp = (*numerator) / (*denominator);
+    delete numerator;
+    delete denominator;
+    gradY->assign(gradYTemp);
+    gradY->applyTransform(transform::Neg, gradY);
+
+  } else if (y->isScalar()) {
+    // scalar case
+
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    auto tmpX = x->reduceNumber(reduce::Sum);
+
+    NDArray *temp1 = *tmp * *tmpX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*temp1) / (*ySquared);
+    delete temp1;
+    delete ySquared;
+    gradY->assign(gradYTemp);
+    gradY->applyTransform(transform::Neg, gradY);
+
+    epsNext->applyScalarArr(scalar::Divide, y, gradX);
+  } else {
+    // broadcast case
+
+    auto preX = *epsNext / *y;
+
+    NDArray negX(*x);
+    x->applyTransform(transform::Neg, &negX);
+    NDArray *negXMulEps = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    auto preY = (*negXMulEps) / (*ySquared);
+    delete negXMulEps;
+    delete ySquared;
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else {
+      // FIXED: preX is stack-allocated from operator/, don't delete
+      gradX->assign(preX);
+    }
+
+    if (axisY.size() > 0) {
+      auto sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      // FIXED: preY is stack-allocated from operator/, don't delete
+      gradY->assign(preY);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(divide_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/divide_no_nan.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author George A. Shulinok <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_divide_no_nan)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(divide_no_nan, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "DIVIDE_NO_NAN OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::DivideNoNan(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Div, divide);
+
+DECLARE_TYPES(divide_no_nan) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/equals.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/BroadcastBoolOpsTuple.h>
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_equals)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(equals, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastBoolOpsTuple::custom(scalar::EqualTo, pairwise::EqualTo, broadcast::EqualTo), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SYN(equal, equals);
+
+DECLARE_TYPES(equals) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/floordiv.cpp

@@ -0,0 +1,89 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_floordiv)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(floordiv, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "FLOORDIV OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::FloorDiv, pairwise::FloorDiv, broadcast::FloorDiv), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(floordiv) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(floordiv_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(floordiv_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  float zero = 0.0f;
+  gradY->assign(zero);
+  gradX->assign(zero);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(floordiv_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/floormod.cpp

@@ -0,0 +1,101 @@
+/* ******************************************************************************
+*
+*
+* This program and the accompanying materials are made available under the
+* terms of the Apache License, Version 2.0 which is available at
+* https://www.apache.org/licenses/LICENSE-2.0.
+*
+*  See the NOTICE file distributed with this work for additional
+*  information regarding copyright ownership.
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations
+* under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//  modified by sgazeos@gmail.com with backprop implementation.
+//
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_floormod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(floormod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "FLOORMOD OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(FloorMod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(floormod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(floormod_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(floormod_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  gradX->assign(epsNext);
+
+  NDArray temp(*epsNext);
+  BroadcastHelper::broadcastApply(BROADCAST(FloorMod), x, y, &temp);
+  if (gradY->rankOf() == gradX->rankOf()) {
+    epsNext->applyPairwiseTransform(pairwise::Multiply, &temp, gradY);
+  } else  { // epsNext is greater than gradY
+    std::vector<LongType> dims(epsNext->rankOf() * 2);
+    LongType gap = epsNext->rankOf() - gradY->rankOf();
+    for (LongType d = 0; d < gap; d++) {
+      dims[d * 2 + 1] = 1;
+    }
+    auto tempIn((temp)(dims));
+    NDArray negTempIn = -*tempIn;
+    auto get=  (*epsNext)(dims);
+    get->applyPairwiseTransform(pairwise::Multiply, &negTempIn, gradY);
+    delete get;
+    delete tempIn;
+  }
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(floormod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/greater.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_greater)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(greater, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(GreaterThan), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(greater) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/greater_equal.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_greater_equal)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(greater_equal, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(GreaterThanOrEqual), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(greater_equal) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/igamma.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_igamma)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(igamma, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::IGamma(), x, y, z);
+
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(igamma) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/igammac.cpp

@@ -0,0 +1,57 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_igammac)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(igammac, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::IGammac(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(igammac) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/less.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/CustomOperations.h>
+#if NOT_EXCLUDED(OP_less)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(less, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(LessThan), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(less) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/less_equal.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_less_equal)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(less_equal, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(LessThanOrEqual), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(less_equal) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/maximum.cpp

@@ -0,0 +1,83 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 12.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_maximum)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/helpers/minimax.h>
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(maximum, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(MaxPairwise), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(maximum) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(maximum_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(maximum_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  helpers::maximumBPFunctor(block.launchContext(), x, y, epsNext, gradX, gradY);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(maximum_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/meshgrid.cpp

@@ -0,0 +1,89 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 05.02.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_meshgrid)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/meshgrid.h>
+
+#include <numeric>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(meshgrid, -1, -1, false, 0, 0) {
+  int rank = block.width();
+
+  if (rank == 1) {
+    OUTPUT_VARIABLE(0)->assign(INPUT_VARIABLE(0));
+    return Status::OK;
+  }
+
+  bool swapFirst2Dims = block.getIArguments()->size() > 0 ? (bool)INT_ARG(0) : true;
+
+  std::vector<NDArray*> inArrs(rank);
+  std::vector<NDArray*> outArrs(rank);
+
+  for (int i = 0; i < rank; ++i) {
+    inArrs[i] = INPUT_VARIABLE(i);
+    outArrs[i] = OUTPUT_VARIABLE(i);
+  }
+
+  helpers::meshgrid(block.launchContext(), inArrs, outArrs, swapFirst2Dims);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(meshgrid) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(INHERIT)->setSameMode(true);
+}
+
+DECLARE_SHAPE_FN(meshgrid) {
+  bool swapFirst2Dims = block.getIArguments()->size() > 0 ? (bool)INT_ARG(0) : true;
+
+  int rank = block.width();
+  LongType* outShapeInfo = nullptr;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+  outShapeInfo[0] = rank;
+  for (int i = 1; i <= rank; ++i) outShapeInfo[i] = (LongType)shape::length(inputShape->at(i - 1));
+
+  if (swapFirst2Dims && rank > 1) math::sd_swap<LongType>(outShapeInfo[1], outShapeInfo[2]);
+
+  auto in = inputShape->at(0);
+  ShapeUtils::updateStridesAndType(outShapeInfo, in, shape::order(in));
+
+  auto shapes = SHAPELIST();
+  auto resultShape = CONSTANT(outShapeInfo);
+  shapes->push_back(resultShape);
+
+  for (int i = 2; i <= rank; ++i) {
+    shapes->push_back(resultShape);
+  }
+
+  return shapes;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/minimum.cpp

@@ -0,0 +1,83 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 12.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_minimum)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/helpers/minimax.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(minimum, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(MinPairwise), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(minimum) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(minimum_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(minimum_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  helpers::minimumBPFunctor(block.launchContext(), x, y, epsNext, gradX, gradY);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(minimum_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/mod.cpp

@@ -0,0 +1,87 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_mod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(mod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(Mod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(mod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(mod_bp) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
+
+CUSTOM_OP_IMPL(mod_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  float zero = 0.0f;
+  gradY->assign(zero);
+  gradX->assign(zero);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(mod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/multiply.cpp

@@ -0,0 +1,165 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//  @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_multiply)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+BROADCASTABLE_OP_IMPL(multiply, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+   LongType* zShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, zShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0, "MULTIPLY OP: the shapes of x %s and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Multiply(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("multiply: result was replaced");
+
+  return Status::OK;
+}
+DECLARE_SYN(Mul, multiply);
+
+DECLARE_TYPES(multiply) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(multiply_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+///////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(multiply_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto dLdz = INPUT_VARIABLE(2);
+
+  auto dLdx = OUTPUT_VARIABLE(0);
+  auto dLdy = OUTPUT_VARIABLE(1);
+
+   LongType* dLdzShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, dLdzShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0,
+               "MULTIPLY_BP OP: the shapes of x %s and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+
+  const LongType xLen = x->lengthOf();
+  const LongType yLen = y->lengthOf();
+
+  if (x->isScalar() && y->isScalar()) {  // both are scalars
+    y->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+    x->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+
+  }else if (x->isScalar()) {  // x is scalar and y is not
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceNumber(reduce::Sum);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+    dLdz->applyScalarArr(scalar::Multiply, x, dLdy);
+  } else if (y->isScalar()) {  // y is scalar and x is not
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceNumber(reduce::Sum);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+    dLdz->applyScalarArr(scalar::Multiply, y, dLdx);
+  } else if (x->isSameShape(y)) {
+    x->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+    y->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+  } else if (x->isSameShape(dLdz)) {
+    auto yTiled = NDArray(dLdz, false, block.launchContext());
+    y->tile(yTiled);
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+    yTiled.applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+  } else if (y->isSameShape(dLdz)) {
+    auto xTiled = NDArray(dLdz, false, block.launchContext());
+    x->tile(xTiled);
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    // FIXED: Clean up intermediate result from operator*
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+    xTiled.applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+  } else {
+    auto xTiled = NDArray(dLdz, false, block.launchContext());
+    auto yTiled = NDArray(dLdz, false, block.launchContext());
+    x->tile(xTiled);
+    y->tile(yTiled);
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+
+    // For dLdx
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+
+    // For dLdy
+    // FIXED: Clean up intermediate result from operator*
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(multiply_bp) {
+  auto xShapeInfo = inputShape->at(0);
+  auto yShapeInfo = inputShape->at(1);
+  return SHAPELIST(CONSTANT(xShapeInfo), CONSTANT(yShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/not_equals.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_not_equals)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(not_equals, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(NotEqualTo), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(not_equals) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/percentile.cpp

@@ -0,0 +1,108 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 17.05.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_percentile)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/percentile.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(percentile, 1, 1, false, 1, -2) {
+  auto input = INPUT_VARIABLE(0);    // tensor with rank > 0
+  auto output = OUTPUT_VARIABLE(0);  // [bS, oD, oH, oW, iC] (NDHWC) or [bS, iC, oD, oH, oW] (NCDHW)
+
+  const auto q = T_ARG(0);  // percentile
+  const int interpolation = block.getTArguments()->size() > 1 ? T_ARG(1) : 2.;  // 0-"lower", 1-"higher", 2-"nearest"(default)
+  const int keepDims = block.getTArguments()->size() > 2 ? T_ARG(2) : 0.;  // false is default
+
+  const int axisArrRank = block.getIArguments()->size();
+  const int inputArrRank = input->rankOf();
+
+  REQUIRE_TRUE(inputArrRank > 0, 0, "PERCENTILE OP: rank of input array must be positive (>0), but got %i instead !",
+               inputArrRank);
+  REQUIRE_TRUE(0.f <= q && q <= 100.f, 0,
+               "PERCENTILE OP: percentile parameter must be within [0, 100] range, but got %f instead !", q);
+  REQUIRE_TRUE(interpolation == 0 || interpolation == 1 || interpolation == 2, 0,
+               "PERCENTILE OP: the correct values for interpolation parameter are 0, 1, 2, but got %i instead !",
+               interpolation);
+  REQUIRE_TRUE(
+      axisArrRank <= inputArrRank, 0,
+      "PERCENTILE OP: the rank of axis array must be <= rank of input array, but got %i and %i correspondingly !",
+      axisArrRank, inputArrRank);
+
+  for (int i = 0; i < axisArrRank; ++i) {
+    int dim = INT_ARG(i) >= 0 ? INT_ARG(i) : INT_ARG(i) + inputArrRank;
+    REQUIRE_TRUE(dim < inputArrRank, 0,
+                 "PERCENTILE OP: element (dimension) of axis array at position %i is >= rank of input array (%i >= "
+                 "%i), which is unacceptable !",
+                 i, dim, inputArrRank);
+  }
+
+  std::vector<LongType> axises = *block.getIArguments();
+  helpers::percentile(block.launchContext(), *input, *output, axises, q, interpolation);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(percentile) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedOutputTypes(0, INHERIT)
+      ->setSameMode(true);
+}
+
+DECLARE_SHAPE_FN(percentile) {
+  auto inputShapeInfo = inputShape->at(0);
+  const int keepDims = block.getTArguments()->size() > 2 ? T_ARG(2) : 0.;  // false is default
+
+  const int axisArrRank = block.getIArguments()->size();
+  const int inputArrRank = inputShapeInfo[0];
+
+  REQUIRE_TRUE(inputArrRank > 0, 0, "PERCENTILE OP: rank of input array must be positive (>0), but got %i instead !",
+               inputArrRank);
+  REQUIRE_TRUE(
+      axisArrRank <= inputArrRank, 0,
+      "PERCENTILE OP: the rank of axis array must be <= rank of input array, but got %i and %i correspondingly !",
+      axisArrRank, inputArrRank);
+
+  for (int i = 0; i < axisArrRank; ++i) {
+    int dim = INT_ARG(i) >= 0 ? INT_ARG(i) : INT_ARG(i) + inputArrRank;
+    REQUIRE_TRUE(dim < inputArrRank, 0,
+                 "PERCENTILE OP: element (dimension) of axis array at position %i is >= rank of input array (%i >= "
+                 "%i), which is unacceptable !",
+                 i, dim, inputArrRank);
+  }
+
+  std::vector<LongType> axises = *block.getIArguments();
+  auto outputShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(inputShapeInfo), &axises, inputShapeInfo, keepDims,
+                                                         false, block.getWorkspace());
+
+  return SHAPELIST(outputShapeInfo);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp

@@ -0,0 +1,122 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_Pow)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(Pow, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  // REQUIRE_TRUE(!y->isB(), 0, "Pairwise OP: you can't divide by bool array!");
+
+  auto tZ = BroadcastHelper::broadcastApply({scalar::Pow, pairwise::Pow, broadcast::Pow}, x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(Pow) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS, ALL_INTS});
+}
+
+CUSTOM_OP_IMPL(Pow_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto dLdz = INPUT_VARIABLE(2);
+
+  auto dLdx = OUTPUT_VARIABLE(0);
+  auto dLdy = OUTPUT_VARIABLE(1);
+
+   LongType* dLdzShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, dLdzShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0,
+               "POW_BP OP: the shapes of x %s"
+               " and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+  REQUIRE_TRUE(shape::equalsSoft(dLdz->shapeInfo(), dLdzShapeInfo), 0,
+               "POW_BP OP: wrong shape of next epsilon array (dLdOut),"
+               " expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(dLdzShapeInfo).c_str(), ShapeUtils::shapeAsString(dLdz).c_str());
+
+  // dL/dy = x^y * log(x) * dL/dz
+  auto temp = x->applyTrueBroadcast(BroadcastOpsTuple::Pow(), y);  // a = x^y
+  x->applyTransform(transform::Log, dLdx);                         // b = log(x)
+  dLdx->applyScalar(scalar::ReplaceNans, 0, dLdx);
+  *temp *= *dLdx;  // c = b*a
+  *temp *= *dLdz;  // dL/dy = c * dL/dz
+  if (dLdy->isSameShape(*dLdz)) {
+    dLdy->assign(temp);
+  } else {
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+    NDArray *dLdyTemp = temp->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);  // dL/dy = sum(c * dL/dz)
+    delete dLdyTemp;
+  }
+
+  // dL/dx = y*x^(y-1) * dL/dz
+  x->applyTrueBroadcast(BroadcastOpsTuple::PowDerivative(), y, temp);  // a = y*x^(y-1)
+  *temp *= *dLdz;                                                        // dLdx = a*dL/dz
+
+  if (dLdx->isSameShape(*dLdz)) {
+    dLdx->assign(temp);  // dLdx = a*dL/dz
+  } else {
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    NDArray *dLdxTemp = temp->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);  // dLdx = a*dL/dz
+    delete dLdxTemp;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(Pow_bp) {
+  auto xShapeInfo = inputShape->at(0);
+  auto yShapeInfo = inputShape->at(1);
+  return SHAPELIST(CONSTANT(xShapeInfo), CONSTANT(yShapeInfo));
+}
+
+DECLARE_TYPES(Pow_bp) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/realdiv.cpp

@@ -0,0 +1,150 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_realdiv)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(realdiv, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Divide(), x, y, z);
+  if (tZ == nullptr) {
+    return Status::KERNEL_FAILURE;
+  }
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(RealDiv, realdiv);
+
+DECLARE_TYPES(realdiv) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, {FLOAT32, HALF, DOUBLE});
+}
+
+DECLARE_TYPES(realdiv_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(realdiv_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    epsNext->applyPairwiseTransform(pairwise::Divide, y, gradX);
+
+    // Y gradient
+
+    // First case
+    NDArray negX = -(*x);
+    NDArray *epsNextMulNegX = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*epsNextMulNegX) / (*ySquared);
+    gradY->assign(gradYTemp);
+    delete epsNextMulNegX;
+    delete ySquared;
+    delete gradYTemp;
+
+  } else if (y->isScalar()) {
+    // scalar case
+
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    auto tmpX = x->reduceNumber(reduce::Sum);
+
+    NDArray negTmpX = -*tmpX;
+    NDArray *tmpMulNegTmpX = (*tmp) * negTmpX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*tmpMulNegTmpX) / (*ySquared);
+    gradY->assign(gradYTemp);
+    delete tmpMulNegTmpX;
+    delete ySquared;
+    delete gradYTemp;
+
+    epsNext->applyScalarArr(scalar::Divide, y, gradX);
+  } else {
+    // broadcast case
+
+    auto preX = *epsNext / *y;
+
+    NDArray negX(*x);
+    x->applyTransform(transform::Neg, &negX);
+    NDArray *epsNextMulNegX = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *preY = (*epsNextMulNegX) / (*ySquared);
+    delete epsNextMulNegX;
+    delete ySquared;
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+    } else
+      gradX->assign(preX);
+
+    if (axisY.size() > 0) {
+      auto sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(preY);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(realdiv_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_divide.cpp

@@ -0,0 +1,147 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversedivide)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversedivide, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!x->isB(), 0, "REVERSEDIVIDE OP: you can't divide by bool array!");
+  x->applyTrueBroadcast(BROADCAST(ReverseDivide), y, z, true);
+
+  return Status::OK;
+}
+DECLARE_SYN(RDiv, reversedivide);
+
+DECLARE_TYPES(reversedivide) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(reversedivide_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(reversedivide_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* gradXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    gradX->applyTransform(transform::Neg, gradX);
+
+    // Y gradient
+    auto* gradYTemp = (*epsNext) / (*x);
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+  } else if (y->isScalar()) {
+    // scalar case
+    auto* tmp = epsNext->reduceNumber(reduce::Sum);
+    auto* tmpX = x->reduceNumber(reduce::Sum);
+    // For gradY
+    auto* gradYTemp = (*tmp) / (*tmpX);
+    delete tmp;
+    delete tmpX;
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+
+    // For gradX
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* gradXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    gradX->applyTransform(transform::Neg, gradX);
+  } else {
+    // broadcast case
+
+    auto* preY = (*epsNext) / (*x);
+
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* preXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    preXTemp->applyTransform(transform::Neg, preXTemp);
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto* sum = preXTemp->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else {
+      gradX->assign(preXTemp);
+    }
+    delete preXTemp;
+
+    if (axisY.size() > 0) {
+      auto* sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      gradY->assign(preY);
+    }
+    delete preY;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversedivide_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_mod.cpp

@@ -0,0 +1,88 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversemod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversemod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(ReverseMod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(reversemod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(reversemod_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(reversemod_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  float assign = 0.0f;
+  gradY->assign(assign);
+  gradX->assign(assign);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversemod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_subtract.cpp

@@ -0,0 +1,117 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversesubtract)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversesubtract, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(ReverseSubtract), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(RSub, reversesubtract);
+
+DECLARE_TYPES(reversesubtract) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(reversesubtract_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+    epsNext->applyTransform(transform::Neg, gradX);
+    gradY->assign(epsNext);
+  } else if (y->isScalar()) {
+    // scalar case
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(tmp);
+    delete tmp;
+    epsNext->applyTransform(transform::Neg, gradX);
+  } else {
+    // broadcastable
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisX);
+      sum->applyTransform(transform::Neg, gradX);
+      delete sum;
+    } else {
+      epsNext->applyTransform(transform::Neg, gradX);
+    }
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      gradY->assign(epsNext);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversesubtract_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+
+DECLARE_TYPES(reversesubtract_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/squared_subtract.cpp

@@ -0,0 +1,174 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 23.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_squaredsubtract)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(squaredsubtract, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(SquaredSubtract), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(squareddifference, squaredsubtract);
+
+DECLARE_TYPES(squaredsubtract) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(squaredsubtract_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+
+  auto* ts = NDArrayFactory::create(x->dataType(), 2, block.launchContext());
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    auto* diff1 = (*x) - (*y);
+    auto* temp1 = (*ts) * (*diff1);
+    delete diff1;
+    auto* gradXTemp = (*epsNext) * (*temp1);
+    delete temp1;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+
+    // Y gradient
+    auto* diff2 = (*y) - (*x);
+    auto* temp2 = (*ts) * (*diff2);
+    delete diff2;
+    auto* gradYTemp = (*epsNext) * (*temp2);
+    delete temp2;
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+
+  } else if (y->isScalar()) {
+    // scalar case
+    auto* tmpX = x->reduceNumber(reduce::Sum);
+    gradY->assign(tmpX);
+    delete tmpX;
+    
+    // X gradient
+    auto* diff3 = (*x) - (*y);
+    auto* temp3 = (*ts) * (*diff3);
+    delete diff3;
+    auto* gradXTemp = (*epsNext) * (*temp3);
+    delete temp3;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+  } else {
+    // broadcast case
+
+    auto* preX = x->dup(x->ordering());
+    auto* preY = y->dup(y->ordering());
+
+    auto* targetShape = epsNext->getShapeAsVector();
+
+    preX->tileToShape(*targetShape, *preX);
+    preY->tileToShape(*targetShape, *preY);
+    delete targetShape;
+    
+    auto* diff4 = (*x) - (*y);
+    auto* temp4 = (*ts) * (*diff4);
+    delete diff4;
+    auto* resX = (*epsNext) * (*temp4);
+    delete temp4;
+    preX->assign(resX);
+    delete resX;
+    
+    auto* diff5 = (*y) - (*x);
+    auto* temp5 = (*ts) * (*diff5);
+    delete diff5;
+    auto* resY = (*epsNext) * (*temp5);
+    delete temp5;
+    preY->assign(resY);
+    delete resY;
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto* sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else
+      gradX->assign(preX);
+
+    if (axisY.size() > 0) {
+      auto* sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(preY);
+
+    delete preX;
+    delete preY;
+  }
+
+  delete ts;
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(squaredsubtract_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+
+DECLARE_TYPES(squaredsubtract_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/subtract.cpp

@@ -0,0 +1,118 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_subtract)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(subtract, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Subtract(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Sub, subtract);
+DECLARE_SYN(sub, subtract);
+
+DECLARE_TYPES(subtract) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(subtract_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+    epsNext->applyTransform(transform::Neg, gradY);
+    gradX->assign(epsNext);
+  } else if (y->isScalar()) {
+    // scalar case
+    auto reduce = epsNext->reduceNumber(reduce::Sum);
+    auto tmp = -(*reduce);
+    gradY->assign(&tmp);
+    gradX->assign(epsNext);
+    delete reduce;
+  } else {
+    // broadcastable
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else
+      gradX->assign(epsNext);
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      sum->applyTransform(transform::Neg, gradY);
+      delete sum;
+    } else {
+      epsNext->applyTransform(transform::Neg, gradY);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(subtract_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(subtract_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/truncatediv.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_truncatediv)
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(truncatediv, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(TruncateDiv), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(truncatediv) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/compat/README.md

@@ -0,0 +1 @@
+This folder contains operations required for compatibility with TF and other frameworks.

libnd4j/include/ops/declarable/generic/compat/compat_sparse_to_dense.cpp

@@ -0,0 +1,76 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_compat_sparse_to_dense)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/sparse_to_dense.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(compat_sparse_to_dense, 4, 1, false, 0, 0) {
+  auto indices = INPUT_VARIABLE(0);
+  auto shape = INPUT_VARIABLE(1);
+  auto values = INPUT_VARIABLE(2);
+  NDArray *def = nullptr;
+
+  auto output = OUTPUT_NULLIFIED(0);
+
+  if (block.width() > 3) def = INPUT_VARIABLE(3);
+
+  sd::ops::helpers::compat_sparse_to_dense(*values, *indices, def, *output);
+
+  return sd::Status::OK;
+};
+
+DECLARE_SHAPE_FN(compat_sparse_to_dense) {
+  auto indices = INPUT_VARIABLE(0);
+  auto shape = INPUT_VARIABLE(1);
+  auto values = INPUT_VARIABLE(2);
+
+  if (block.width() > 3) {
+    auto def = INPUT_VARIABLE(3);
+
+    REQUIRE_TRUE(def->dataType() == values->dataType() && def->isScalar(), 0,
+                 "compat_sparse_to_dense: default value must be a scalar of the same data type as actual values")
+  };
+
+  auto dtype = values->dataType();
+
+  // basically output shape is defined by the type of input, and desired shape input
+  return SHAPELIST(
+      ConstantShapeHelper::getInstance().createShapeInfo(dtype, 'c', shape->getBufferAsVector<sd::LongType>()));
+}
+
+DECLARE_TYPES(compat_sparse_to_dense) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS})         // indices
+      ->setAllowedInputTypes(1, {ALL_INTS})         // shape
+      ->setAllowedInputTypes(2, sd::DataType::ANY)  // sparse values
+      ->setAllowedInputTypes(3, sd::DataType::ANY)  // default value
+      ->setAllowedOutputTypes(sd::DataType::ANY);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/compat/compat_string_split.cpp

@@ -0,0 +1,148 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_split_string)
+
+#include <helpers/StringUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(compat_string_split, 2, 2, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto delim = INPUT_VARIABLE(1);
+
+  auto indices = OUTPUT_VARIABLE(0);
+  auto values = OUTPUT_VARIABLE(1);
+
+  auto d = delim->e<std::string>(0);
+
+  NDArray::preparePrimaryUse({values},{indices});
+
+  // output rank N+1 wrt input rank
+  std::vector<LongType> icoords(input->rankOf());
+
+  // getting buffer lengths
+  auto outputLength = StringUtils::byteLength(*input);
+  LongType ic = 0L;
+  int len = input->isScalar() ? 1 : input->lengthOf();
+
+  sd::LongType inputRank = input->rankOf();
+  sd::LongType *inputShape = shape::shapeOf(input->shapeInfo());
+
+  // loop through each string within tensor
+  for (LongType e = 0L; e < len; e++) {
+    // now we should map substring to indices
+    auto s = input->e<std::string>(e);
+
+    // getting base index
+    INDEX2COORDS(e, inputRank, inputShape, icoords.data());
+
+    // getting number of substrings
+    auto cnt = StringUtils::countSubarrays(s.c_str(), s.length(), d.c_str(), d.length());
+    // filling output indices
+    for (LongType f = 0; f < cnt; f++) {
+      for (auto v : icoords) {
+        indices->p(ic++, v);
+      }
+
+      // last index
+      indices->p(ic++, f);
+    }
+
+  }
+
+  // process strings now
+  std::vector<std::string> strings;
+  for (auto e = 0L; e < input->lengthOf(); e++) {
+    auto split = StringUtils::split(input->e<std::string>(e), d);
+
+    for (const auto& s : split) strings.emplace_back(s);
+  }
+
+  // now once we have all strings in single vector time to fill
+  auto nonConst = const_cast<NDArray*>(values);
+  auto* valuesShapeVec = nonConst->getShapeAsVector();
+  auto tmp = NDArrayFactory::string(*valuesShapeVec, strings);
+  delete valuesShapeVec;
+  auto blen = StringUtils::byteLength(*tmp) + ShapeUtils::stringBufferHeaderRequirements(strings.size());
+  values->dataBuffer()->expand(blen);
+  memcpy(values->buffer(), tmp->buffer(), blen);
+  values->tickWriteHost();
+
+  // special case, for future use
+  indices->syncToDevice();
+  values->syncToDevice();
+
+  NDArray::registerPrimaryUse({values});
+  // we have to tick buffers
+  values->dataBuffer()->writePrimary();
+  values->dataBuffer()->readSpecial();
+
+  delete tmp;
+
+  return Status::OK;
+};
+
+DECLARE_SHAPE_FN(compat_string_split) {
+  auto input = INPUT_VARIABLE(0);
+  auto delim = INPUT_VARIABLE(1);
+
+
+  auto d = delim->e<std::string>(0);
+
+  // count number of delimiter substrings in all strings within input tensor
+  LongType cnt = 0;
+  int len = input->isScalar() ? 1 : input->lengthOf();
+  for (auto e = 0L; e < len; e++) {
+    auto s = input->e<std::string>(e);
+
+    // each substring we see in haystack, splits string in two parts. so we should add 1 to the number of subarrays
+    cnt += StringUtils::countSubarrays(s.c_str(), s.length(), d.c_str(), d.length());
+  }
+  cnt++;
+
+  // shape calculations
+  // virtual tensor rank will be N+1, for N rank input array, where data will be located at the biggest dimension
+  // values tensor is going to be vector always
+  // indices tensor is going to be vector with length equal to values.length * output rank
+
+  sd_printf("compat_string_split: Assigning number of values: %d\n",cnt);
+
+  auto valuesShape = ConstantShapeHelper::getInstance().vectorShapeInfo(cnt, UTF8);
+  auto indicesShape =
+      ConstantShapeHelper::getInstance().vectorShapeInfo(cnt * (input->rankOf() + 1), INT64);
+
+  return SHAPELIST(indicesShape, valuesShape);
+}
+
+DECLARE_TYPES(compat_string_split) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_STRINGS})
+      ->setAllowedOutputTypes(0, {ALL_INDICES})
+      ->setAllowedOutputTypes(1, {ALL_STRINGS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/bitcast.cpp

@@ -0,0 +1,107 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author George A. Shulinok <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bitcast)
+
+#include <array/DataTypeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(bitcast, 1, 1, false, 0, 1) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  // when empty - nothing to do
+  DataType newType = DataTypeUtils::fromInt(INT_ARG(0));
+  DataType oldType = input->dataType();
+  // correct output shape to conform with output data type
+  auto inputSize = DataTypeUtils::sizeOf(oldType);
+  auto outputSize = DataTypeUtils::sizeOf(newType);
+  auto lastSize = outputSize / inputSize;
+  if (inputSize < outputSize) {
+    REQUIRE_TRUE(static_cast<size_t>(input->sizeAt(-1)) == lastSize, 0,
+                 "BITCAST: %llu > %llu. So last dimension should be %i, but %i given.", inputSize, outputSize, lastSize,
+                 input->sizeAt(-1));
+  }
+  if (input->isEmpty()) {
+    REQUIRE_TRUE(output->isEmpty(), 0, "BITCAST: If input is empty, output array must also be empty.");
+    return sd::Status::OK;
+  }
+
+  // just memcpy data
+  DataBuffer::memcpy(output->dataBuffer(), input->dataBuffer(), 0, 0);
+
+  return sd::Status::OK;
+}
+DECLARE_SYN(BitCast, bitcast);
+
+DECLARE_SHAPE_FN(bitcast) {
+  auto inShape = inputShape->at(0);
+  auto inputRank = shape::rank(inShape);
+  auto it = INT_ARG(0);
+  DataType newType = DataTypeUtils::fromInt(it);
+  DataType oldType = ArrayOptions::dataType(inShape);
+  // correct output shape to conform with output data type
+  auto inputSize = DataTypeUtils::sizeOf(oldType);
+  auto outputSize = DataTypeUtils::sizeOf(newType);
+
+  if (shape::length(inShape) == 0) {
+    auto ret = SHAPELIST(ConstantShapeHelper::getInstance().castToDataType(inShape, newType));
+    return ret;
+  }
+  if (inputSize == outputSize) {
+    // only type should be changed
+    auto ret = SHAPELIST(ConstantShapeHelper::getInstance().castToDataType(inShape, newType));
+    return ret;
+  } else if (inputSize > outputSize) {
+    // range of output increased by 1 with inputSize / outputSize as last dimension
+    std::vector<sd::LongType> shapeOf(inputRank + 1);
+    int i;
+    for (i = 0; i < inputRank; ++i) {
+      shapeOf[i] = inShape[i + 1];
+    }
+    shapeOf[i] = inputSize / outputSize;
+    auto outputShape = ConstantShapeHelper::getInstance().bufferForShapeInfo(newType, shape::order(inShape), shapeOf)->primary();
+    return SHAPELIST(outputShape);
+  }
+  REQUIRE_TRUE(shape::sizeAt(inShape, static_cast<size_t>(static_cast<sd::LongType>(-1))) ==
+                   static_cast<sd::LongType>(outputSize / inputSize), 0,
+               "BITCAST: %llu > %llu. So last dimension should be %i, but %i given.", inputSize, outputSize,
+               outputSize / inputSize, shape::sizeAt(inShape, static_cast<sd::LongType>(-1)));
+  std::vector<sd::LongType> shapeOf(inputRank - 1);
+
+  for (size_t i = 0; i < shapeOf.size(); ++i) {
+    shapeOf[i] = inShape[i + 1];
+  }
+
+  auto outputShape = ConstantShapeHelper::getInstance().createShapeInfo(newType, shape::order(inShape), shapeOf);
+  return SHAPELIST(outputShape);
+}
+
+DECLARE_TYPES(bitcast) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes(sd::DataType::ANY);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/cast.cpp

@@ -0,0 +1,87 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_cast)
+
+#include <array/DataTypeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/assign.h>
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(cast, 1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (input->isEmpty()) {
+    REQUIRE_TRUE(output->isEmpty(), 0, "If input is empty, output array must also be empty");
+    return sd::Status::OK;
+  }
+
+  if (!block.isInplace()) {
+    helpers::assign(block.launchContext(), output, input);
+  }
+
+  STORE_RESULT(output);
+  return sd::Status::OK;
+}
+DECLARE_SYN(Cast, cast);
+
+DECLARE_SHAPE_FN(cast) {
+  auto inShape = inputShape->at(0);
+  if(!block.getDArguments()->empty()) {
+    DataType newType = block.dataType(0);
+    auto desc = new ShapeDescriptor(inShape, newType, true);
+    auto newShapeInfo = ConstantShapeHelper::getInstance().createShapeInfo(desc);
+    
+    // FIX: Clean up the ShapeDescriptor after use
+    delete desc;
+    
+    auto compDataType = ArrayOptions::dataType(newShapeInfo);
+    if(compDataType != newType) {
+      std::string errorMessage;
+      errorMessage += "cast: new data type is ";
+      errorMessage += DataTypeUtils::asString(newType);
+      errorMessage += " data type from new constant created data type ";
+      errorMessage += DataTypeUtils::asString(compDataType);
+      errorMessage += "\n";
+      THROW_EXCEPTION(errorMessage.c_str());
+    }
+    auto ret =  SHAPELIST(newShapeInfo);
+    return ret;
+
+  } else {
+    auto it = INT_ARG(0);
+    DataType newType = DataTypeUtils::fromInt(it);
+    auto ret =  SHAPELIST(ConstantShapeHelper::getInstance().castToDataType(inShape,newType));
+    return ret;
+  }
+}
+
+DECLARE_TYPES(cast) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes(sd::DataType::ANY);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/min_max_datatype.cpp

@@ -0,0 +1,207 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Adam Gibson
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_min_max_datatype)
+
+#include <array/DataTypeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(min_max_datatype, -2, 1, false, 0, 2) {
+  auto output = OUTPUT_VARIABLE(0);
+  auto dataType = INT_ARG(0);
+  DataType type = DataTypeUtils::fromInt(dataType);
+  auto minOrMax = INT_ARG(1);
+  if (minOrMax == 0) {
+    switch (type) {
+#ifdef HAS_UINT8
+      case UINT8:
+        output->p(0, DataTypeUtils::min<uint8_t>());
+        break;
+#endif
+#ifdef HAS_INT8
+      case INT8:
+        output->p(0, DataTypeUtils::min<int8_t>());
+        break;
+#endif
+#ifdef HAS_BOOL
+      case BOOL:
+        output->p(0, DataTypeUtils::min<bool>());
+        break;
+#endif
+#ifdef HAS_BFLOAT16
+      case BFLOAT16:
+        output->p(0, DataTypeUtils::min<bfloat16>());
+        break;
+#endif
+#ifdef HAS_FLOAT16
+      case HALF:
+        output->p(0, DataTypeUtils::min<float16>());
+        break;
+#endif
+#ifdef HAS_INT16
+      case INT16:
+        output->p(0, DataTypeUtils::min<int16_t>());
+        break;
+#endif
+#ifdef HAS_UINT16
+      case UINT16:
+        output->p(0, DataTypeUtils::min<uint16_t>());
+        break;
+#endif
+#ifdef HAS_INT32
+      case INT32:
+        output->p(0, DataTypeUtils::min<int>());
+        break;
+#endif
+#ifdef HAS_UINT32
+      case UINT32:
+        output->p(0, DataTypeUtils::min<uint32_t>());
+        break;
+#endif
+#ifdef HAS_FLOAT32
+      case FLOAT32:
+        output->p(0, DataTypeUtils::min<float>());
+        break;
+#endif
+#ifdef HAS_UNSIGNEDLONG
+      case UINT64:
+        output->p(0, DataTypeUtils::min<sd::UnsignedLong>());
+        break;
+#endif
+#ifdef HAS_LONG
+      case INT64:
+        output->p(0, DataTypeUtils::min<LongType>());
+        break;
+#endif
+#ifdef HAS_DOUBLE
+      case DOUBLE:
+        output->p(0, DataTypeUtils::min<double>());
+        break;
+#endif
+      default: {
+        std::string errorMessage;
+        errorMessage += "Min: Unknown type requested: " + DataTypeUtils::asString(type);
+        THROW_EXCEPTION(errorMessage.c_str());
+#ifndef __CUDA_ARCH__
+        THROW_EXCEPTION("Unknown DataType requested");
+#endif
+      }
+    }
+  } else {
+    switch (type) {
+#ifdef HAS_UINT8
+      case UINT8:
+        output->p(0, DataTypeUtils::max<uint8_t>());
+        break;
+#endif
+#ifdef HAS_INT8
+      case INT8:
+        output->p(0, DataTypeUtils::max<int8_t>());
+        break;
+#endif
+#ifdef HAS_BOOL
+      case BOOL:
+        output->p(0, DataTypeUtils::max<bool>());
+        break;
+#endif
+#ifdef HAS_BFLOAT16
+      case BFLOAT16:
+        output->p(0, DataTypeUtils::max<bfloat16>());
+        break;
+#endif
+#ifdef HAS_FLOAT16
+      case HALF:
+        output->p(0, DataTypeUtils::max<float16>());
+        break;
+#endif
+#ifdef HAS_INT16
+      case INT16:
+        output->p(0, DataTypeUtils::max<int16_t>());
+        break;
+#endif
+#ifdef HAS_UINT16
+      case UINT16:
+        output->p(0, DataTypeUtils::max<uint16_t>());
+        break;
+#endif
+#ifdef HAS_INT32
+      case INT32:
+        output->p(0, DataTypeUtils::max<int>());
+        break;
+#endif
+#ifdef HAS_UINT32
+      case UINT32:
+        output->p(0, DataTypeUtils::max<uint32_t>());
+        break;
+#endif
+#ifdef HAS_FLOAT32
+      case FLOAT32:
+        output->p(0, DataTypeUtils::max<float>());
+        break;
+#endif
+#ifdef HAS_UNSIGNEDLONG
+      case UINT64:
+        output->p(0, DataTypeUtils::max<sd::UnsignedLong>());
+        break;
+#endif
+#ifdef HAS_LONG
+      case INT64:
+        output->p(0, DataTypeUtils::max<LongType>());
+        break;
+#endif
+#ifdef HAS_DOUBLE
+      case DOUBLE:
+        output->p(0, DataTypeUtils::max<double>());
+        break;
+#endif
+      default: {
+        sd_printf("Unknown DataType used: [%i]\n", DataTypeUtils::asInt(type));
+#ifndef __CUDA_ARCH__
+        std::string errorMessage;
+        errorMessage += "Unknown data type requested min max:";
+        errorMessage += DataTypeUtils::asString(type);
+        THROW_EXCEPTION(errorMessage.c_str());
+#endif
+      }
+    }
+
+  }
+
+  return Status::OK;
+
+}
+
+DECLARE_SHAPE_FN(min_max_datatype) {
+  DataType newType = DataTypeUtils::fromInt(INT_ARG(0));
+  return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(newType));
+}
+
+DECLARE_TYPES(min_max_datatype) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/to_double.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_to_double)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(to_double, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(to_double) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(DOUBLE);
+}
+
+DECLARE_SHAPE_FN(to_double) {
+  auto outShape = ShapeBuilders::copyShapeInfoAndType(inputShape->at(0), DOUBLE, true, block.workspace());
+  return SHAPELIST(CONSTANT(outShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/to_float16.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_to_float16)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(to_float16, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(to_float16) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(HALF);
+}
+
+DECLARE_SHAPE_FN(to_float16) {
+  auto outShape = ShapeBuilders::copyShapeInfoAndType(inputShape->at(0), HALF, true, block.workspace());
+  return SHAPELIST(CONSTANT(outShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/to_float32.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_to_float32)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(to_float32, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(to_float32) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(FLOAT32);
+}
+
+DECLARE_SHAPE_FN(to_float32) {
+  auto outShape = ShapeBuilders::copyShapeInfoAndType(inputShape->at(0), FLOAT32, true, block.workspace());
+  return SHAPELIST(CONSTANT(outShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/to_int32.cpp

@@ -0,0 +1,52 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_to_int32)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(to_int32, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(to_int32) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(INT32);
+}
+DECLARE_SHAPE_FN(to_int32) {
+  auto outShape = ShapeBuilders::copyShapeInfoAndType(inputShape->at(0), INT32, true, block.workspace());
+  return SHAPELIST(CONSTANT(outShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/to_int64.cpp

@@ -0,0 +1,52 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_to_int64)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(to_int64, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(to_int64) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(INT64);
+}
+DECLARE_SHAPE_FN(to_int64) {
+  auto outShape = ShapeBuilders::copyShapeInfoAndType(inputShape->at(0), INT64, true, block.workspace());
+  return SHAPELIST(CONSTANT(outShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/to_uint32.cpp

@@ -0,0 +1,52 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_to_uint32)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(to_uint32, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(to_uint32) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(INT32);
+}
+DECLARE_SHAPE_FN(to_uint32) {
+  auto outShape = ShapeBuilders::copyShapeInfoAndType(inputShape->at(0), UINT32, true, block.workspace());
+  return SHAPELIST(CONSTANT(outShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/datatypes/to_uint64.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_to_uint64)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(to_uint64, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(to_uint64) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(INT8);
+}
+DECLARE_SHAPE_FN(to_uint64) {
+  auto outShape = ShapeBuilders::copyShapeInfoAndType(inputShape->at(0), UINT64, true, block.workspace());
+  return SHAPELIST(CONSTANT(outShape));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/decoder/ctc_beam_op.cpp

@@ -0,0 +1,145 @@
+/*******************************************************************************
+ * Copyright (c) 2021 Deeplearning4j Contributors
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *******************************************************************************/
+
+//
+// @author AbdelRauf
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/ctc.h>
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_ctc_beam)
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(ctc_beam, 2, 3, false, 0, -2) {
+  auto logit = INPUT_VARIABLE(0);
+  auto sequence_length = INPUT_VARIABLE(1);
+  auto result_sequences = OUTPUT_VARIABLE(0);
+  auto result_probs = OUTPUT_VARIABLE(1);
+  auto result_sequences_length = OUTPUT_VARIABLE(2);
+  auto arg_size = block.getIArguments()->size();
+  auto normalize_logits = block.numB() > 0 ? B_ARG(0) : false;
+
+  int blank_index = arg_size > 0 ? INT_ARG(0) : -1;
+  int beam_width = arg_size > 1 ? INT_ARG(1) : 25;
+  int nbest_len = arg_size > 2 ? INT_ARG(2) : 1;
+
+  REQUIRE_TRUE(
+      logit->rankOf() == 3, 0,
+      "Ctc Beam Search: logit Input fails to meet rank requirement {BATCH_LEN, MAX_FRAME_LEN, CLASS_LEN }: %i == 3 ",
+      logit->rankOf());
+  REQUIRE_TRUE(sequence_length->rankOf() == 1, 0,
+               "Ctc Beam Search: sequence frame length (sequence_length) Input fails to meet rank requirement "
+               "{BATCH_LEN}: %i == 1 ",
+               sequence_length->rankOf());
+
+  REQUIRE_TRUE(result_sequences->rankOf() == 3, 0,
+               "Ctc Beam Search: result_sequences Output fails to meet rank requirement {BATCH_LEN, NBEST_LEN, "
+               "MAX_FRAME_LEN }: %i == 3 ",
+               result_sequences->rankOf());
+  REQUIRE_TRUE(result_probs->rankOf() == 2, 0,
+               "Ctc Beam Search: result_probs Output fails to meet rank requirement {BATCH_LEN, NBEST_LEN}: %i == 2 ",
+               result_probs->rankOf());
+  REQUIRE_TRUE(
+      result_sequences_length->rankOf() == 2, 0,
+      "Ctc Beam Search: result_sequences_length Output fails to meet rank requirement {BATCH_LEN, NBEST_LEN}: %i == 2 ",
+      result_sequences_length->rankOf());
+
+  auto batchSize0 = logit->sizeAt(0);
+  auto batchSize1 = sequence_length->sizeAt(0);
+  auto batchSize2 = result_sequences->sizeAt(0);
+  auto batchSize3 = result_probs->sizeAt(0);
+  auto batchSize4 = result_sequences_length->sizeAt(0);
+
+  bool check_batches = (batchSize0 == batchSize1) && (batchSize2 == batchSize3);
+  check_batches = check_batches && (batchSize0 == batchSize4) && (batchSize0 == batchSize2);
+
+  REQUIRE_TRUE(nbest_len > 0 && nbest_len <= beam_width, 0, "Ctc Beam Search: nbest_len %i should be > 0 and <= %i",
+               nbest_len, beam_width);
+  REQUIRE_TRUE(check_batches, 0, "Ctc Beam Search: All batch sizes should be %i", batchSize0);
+  auto max_t = logit->sizeAt(1);
+  REQUIRE_TRUE(result_sequences->sizeAt(1) == nbest_len && result_sequences->sizeAt(2) == max_t, 0,
+               "Ctc Beam Search: shape of the result_sequences should be {%i, %i, %i} but got { %i, %i, %i}",
+               batchSize0, nbest_len, max_t, batchSize1, result_sequences->sizeAt(1), result_sequences->sizeAt(2));
+  REQUIRE_TRUE(result_probs->sizeAt(1) == nbest_len, 0,
+               "Ctc Beam Search: shape of the result_probs should be {%i, %i} but got { %i, %i}", batchSize0, nbest_len,
+               batchSize3, result_sequences->sizeAt(1));
+  REQUIRE_TRUE(result_sequences_length->sizeAt(1) == nbest_len, 0,
+               "Ctc Beam Search: shape of the result_sequences_length should be {%i, %i} but got { %i, %i}", batchSize0,
+               nbest_len, batchSize4, result_sequences_length->sizeAt(1));
+  REQUIRE_TRUE(result_sequences->ews() == 1 && result_sequences->ordering() == 'c', 0,
+               "Ctc Beam Search: result_sequences output should be ews()==1 and c order: %d == ews(1) %c == order(c) ",
+               result_sequences->ews(), result_sequences->ordering());
+  REQUIRE_TRUE(result_probs->ews() == 1 && result_probs->ordering() == 'c', 0,
+               "Ctc Beam Search: result_probs output should be ews()==1 and c order: %d == ews(1) %c == order(c) ",
+               result_probs->ews(), result_probs->ordering());
+  REQUIRE_TRUE(
+      result_sequences_length->ews() == 1 && result_sequences_length->ordering() == 'c', 0,
+      "Ctc Beam Search: result_sequences_length output should be ews()==1 and c order: %d == ews(1) %c == order(c) ",
+      result_sequences_length->ews(), result_sequences_length->ordering());
+
+  helpers::beamSearch(*logit, *sequence_length, *result_sequences, *result_probs, *result_sequences_length,
+                               blank_index, beam_width, nbest_len, normalize_logits);
+
+  return Status::OK;
+}
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(ctc_beam) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INDICES})
+      ->setAllowedOutputTypes(0, {ALL_INDICES})
+      ->setAllowedOutputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(2, {ALL_INDICES});
+}
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(ctc_beam) {
+  auto logitShapeInfo = inputShape->at(0);
+  auto sequenceShapeInfo = inputShape->at(1);
+  auto arg_size = block.getIArguments()->size();
+
+  auto nbest_len = arg_size > 2 ? INT_ARG(2) : 1;
+
+  REQUIRE_TRUE(
+      logitShapeInfo[0] == 3, 0,
+      "Ctc Beam Search: logit Input fails to meet rank requirement {BATCH_LEN, MAX_FRAME_LEN, CLASS_LEN }: %i == 3",
+      logitShapeInfo[0]);
+
+  auto batch_size = shape::shapeOf(logitShapeInfo)[0];
+  auto max_t = shape::shapeOf(logitShapeInfo)[1];
+
+  auto dtype_float = ArrayOptions::dataType(logitShapeInfo);
+  auto dtype_index = ArrayOptions::dataType(sequenceShapeInfo);
+  auto desc =  ShapeBuilders::createShapeInfo(dtype_index, 'c', {batch_size, nbest_len, max_t});
+  auto output0 = ConstantShapeHelper::getInstance().bufferForShapeInfo(desc);
+  auto desc2 = ShapeBuilders::createShapeInfo(dtype_float, 'c', {batch_size, nbest_len});
+  auto output1 =
+      ConstantShapeHelper::getInstance().bufferForShapeInfo(desc2);
+  auto desc3 = ShapeBuilders::createShapeInfo(dtype_index, 'c', {batch_size, nbest_len});
+  auto output2 = ConstantShapeHelper::getInstance().bufferForShapeInfo(desc3);
+  return SHAPELIST(output0->primary(), output1->primary(), output2->primary());
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/flow/flow_control_ops.cpp

@@ -0,0 +1,98 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(op_Switch)
+namespace sd {
+namespace ops {
+/**
+ * This operation is, basically IF statement
+ *
+ * arg_0 is our "signal"
+ * arg_1 is condition that will determine transition
+ */
+// TODO: make this op a placeholder too
+DIVERGENT_OP_IMPL(Switch, 2, 2, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto condition = INPUT_VARIABLE(1);
+
+  // we'll store signal to both ends
+  // STORE_2_RESULTS(*input, *input);
+
+  // but we'll ensure only one node is active, and other is disabled
+  auto out0 = OUTPUT_VARIABLE(0);
+  auto out1 = OUTPUT_VARIABLE(1);
+
+  if (condition->e<int>(0) == 0) {
+    block.setBranch(0);
+    if (!out0) {
+      this->storeResult(block, 0, input->dup());
+    } else {
+      out0->assign(input);
+    }
+  } else {
+    block.setBranch(1);
+    if (!out1) {
+      this->storeResult(block, 1,input->dup());
+    } else {
+      out1->assign(input);
+    }
+  }
+
+  return sd::Status::OK;
+}
+DECLARE_SYN(switch, Switch);
+DECLARE_SYN(if, Switch);
+
+/**
+ *  This op is a placeholder.
+ *  Actual WHILE implementation is in GraphExecutioner
+ */
+LOGIC_OP_IMPL(While);
+DECLARE_SYN(while, While);
+
+/**
+ *  This op is a placeholder.
+ *  Actual OpScope implementation is in Graph and GraphExecutioner
+ */
+LOGIC_OP_IMPL(OpScope);
+DECLARE_SYN(scope, OpScope);
+
+/**
+ *  This op is a placeholder.
+ *  Actual Conditional implementation is in Graph and GraphExecutioner
+ */
+LOGIC_OP_IMPL(Conditional);
+DECLARE_SYN(cond, Conditional);
+
+/**
+ * This op is a placeholder
+ * Actual implementation is in LogicReturn class
+ */
+LOGIC_OP_IMPL(Return);
+DECLARE_SYN(return, Return);
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/grad/broadcast_gradient_args.cpp

@@ -0,0 +1,44 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_broadcastgradientargs)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+/**
+ * PLEASE NOTE: This op is disabled atm, and reserved for future releases.
+ */
+OP_IMPL(broadcastgradientargs, 2, 2, true) {
+  sd_printf("BroadcastGradientArgs: Not implemented yet\n", "");
+
+  return Status::KERNEL_FAILURE;
+}
+DECLARE_SYN(BroadcastGradientArgs, broadcastgradientargs);
+
+DECLARE_TYPES(broadcastgradientargs) { getOpDescriptor()->setAllowedInputTypes(ANY); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/helpers/BroadcastHelper.h

@@ -0,0 +1,145 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#ifndef LIBND4J_BROADCAST_HELPER_H
+#define LIBND4J_BROADCAST_HELPER_H
+
+#include <array/NDArray.h>
+#include <array/NDArrayFactory.h>
+#include <helpers/ShapeUtils.h>
+#include <ops/BroadcastBoolOpsTuple.h>
+#include <ops/BroadcastOpsTuple.h>
+
+namespace sd {
+namespace ops {
+class BroadcastHelper {
+ public:
+  static SD_INLINE NDArray* broadcastApply(BroadcastOpsTuple op, NDArray* x, NDArray* y, NDArray* z,
+                                           ExtraArguments* extraArgs = nullptr) {
+    if (x->isEmpty() || y->isEmpty()) {
+      return z;
+    }
+
+    if (x->lengthOf() > 1 && y->lengthOf() > 1 && x->isSameShape(y)) {
+      x->applyPairwiseTransform(op.p, y, z, extraArgs);
+    } else if (x->lengthOf() > 1 && y->lengthOf() <= 1) {
+      x->applyScalarArr(op.s, y, z);
+    } else if (x->lengthOf() <= 1 && y->lengthOf() > 1) {
+      NDArray &xRef = *x;
+      NDArray &yRef = *y;
+      if (z->isSameShape(y)) {
+        if (op.s == scalar::Add || op.s == scalar::Multiply) {
+          y->applyScalarArr(op.s, x, z);
+        } else if (op.s == scalar::SquaredSubtract) {
+          y->applyScalarArr(scalar::SquaredReverseSubtract, x, z);
+        } else if (op.s == scalar::Subtract) {
+          y->applyScalarArr(scalar::ReverseSubtract, x, z);
+        } else if (op.s == scalar::Divide) {
+          y->applyScalarArr(scalar::ReverseDivide, x, z);
+        } else if (op.s == scalar::Pow) {
+          y->applyScalarArr(scalar::ReversePow, x, z);
+        } else if (op.s == scalar::ReverseSubtract) {
+          y->applyScalarArr(scalar::Subtract, x, z);
+        } else if (op.s == scalar::ReverseDivide) {
+          y->applyScalarArr(scalar::Divide, x, z);
+        } else if (op.s == scalar::MaxPairwise || op.s == scalar::MinPairwise || op.s == scalar::AMaxPairwise ||
+                   op.s == scalar::AMinPairwise) {
+          y->applyScalarArr(op.s, x, z);
+        } else if (op.s == scalar::CopyPws) {
+          z->assign(&yRef);
+        } else {
+          z->assign(&xRef);
+          z->applyPairwiseTransform(op.p, y, extraArgs);
+        }
+        return z;
+      } else {
+        auto v = y->getShapeAsVector();
+        auto tZ = NDArrayFactory::valueOf(v, y, y->ordering());
+        tZ->applyPairwiseTransform(op.p, y, extraArgs);
+        return tZ;
+      }
+    } else if (x->lengthOf() <= 1 && y->lengthOf() <= 1) {
+      x->applyScalarArr(op.s, y, z);
+    } else if (ShapeUtils::areShapesBroadcastable(*x, *y)) {
+      x->applyTrueBroadcast(op, y, z, true, extraArgs);
+      return z;
+    } else {
+      auto sx = ShapeUtils::shapeAsString(x);
+      auto sy = ShapeUtils::shapeAsString(y);
+      sd_printf("Broadcast: shapes should be equal, or broadcastable. But got %s vs %s instead\n", sx.c_str(),
+                sy.c_str());
+      return nullptr;
+    }
+
+    return z;
+  }
+
+  static SD_INLINE NDArray* broadcastApply(BroadcastBoolOpsTuple op, NDArray* x, NDArray* y, NDArray* z,
+                                           ExtraArguments* extraArgs = nullptr) {
+    if (x->isEmpty() || y->isEmpty()) {
+      if (!z->isEmpty()) {
+        std::string errorMessage;
+        errorMessage += "BroadcastHelper::broadcastApply: when some of input arrays (or both) is empty, output array must be empty as well !";
+        errorMessage += "X is empty: ";
+        errorMessage += std::to_string(x->isEmpty());
+        errorMessage += "Y is empty: ";
+        errorMessage += std::to_string(y->isEmpty());
+        THROW_EXCEPTION(
+            errorMessage.c_str());
+      }
+      return z;
+    }
+
+    if (!x->isScalar() && !y->isScalar() && x->isSameShape(y)) {
+      x->applyPairwiseTransform(op.p, y, z);
+    } else if (ShapeUtils::areShapesBroadcastable(*x, *y)) {
+      x->applyTrueBroadcast(op, y, z, true, extraArgs);
+      return z;
+    } else if (!x->isScalar() && y->isScalar()) {
+      x->applyScalarArr(op.s, y, z);
+    } else if (x->isScalar() && !y->isScalar()) {
+      if (z->isSameShape(y)) {
+        x->applyPairwiseTransform(op.p, y, z, extraArgs);
+        return z;
+      } else {
+        auto v = y->getShapeAsVector();
+        auto tZ = NDArrayFactory::valueOf(v, y, y->ordering());
+        return tZ;
+      }
+    } else if (x->isScalar() && y->isScalar()) {  // x->isScalar() && y->isScalar()
+      x->applyScalarArr(op.s, y, z);
+    } else if (ShapeUtils::areShapesBroadcastable(*x, *y)) {
+      x->applyTrueBroadcast(op, y, z, true, extraArgs);
+      return z;
+    } else {
+      auto sx = ShapeUtils::shapeAsString(x);
+      auto sy = ShapeUtils::shapeAsString(y);
+      return nullptr;
+    }
+
+    return z;
+  }
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/helpers/ScatterHelper.h

@@ -0,0 +1,38 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#ifndef LIBND4J_SCATTERHELPER_H
+#define LIBND4J_SCATTERHELPER_H
+
+#include <array/NDArray.h>
+#include <ops/declarable/helpers/scatter.h>
+#include <system/common.h>
+#include <system/op_boilerplate.h>
+
+#include <numeric>
+
+namespace sd {
+namespace ops {}
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/adjust_contrast.cpp

@@ -0,0 +1,140 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author George A. Shulinok <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_adjust_contrast)
+
+#include <array/NDArrayFactory.h>
+#include <ops/declarable/headers/parity_ops.h>
+
+namespace sd {
+namespace ops {
+
+////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(adjust_contrast, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  // just skip op if input is empty
+  if (input->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(block.numT() > 0 || block.width() > 1, 0, "ADJUST_CONTRAST: Scale factor required");
+  REQUIRE_TRUE(input->rankOf() > 2, 0, "ADJUST_CONTRAST: op expects rank of input array to be >= 3, but got %i instead",
+               input->rankOf());
+
+
+  NDArray* factor = nullptr;
+
+  if (block.width() > 1)
+    factor = INPUT_VARIABLE(1);
+  else {
+    factor = new NDArray(output->dataType(), block.launchContext());
+#ifdef HAS_DOUBLE
+    factor->p(0, static_cast<double>(T_ARG(0)));
+#elif defined(HAS_FLOAT32)
+    factor->p(0, static_cast<float>(T_ARG(0)));
+#else
+#error "No floating-point type available for adjust_contrast operation"
+#endif
+  }
+
+  // fill up axes vector first
+  std::vector<LongType> axes(input->rankOf() - 1);
+  for (size_t i = 0; i < axes.size(); ++i) axes[i] = i;
+  // mean as reduction for last dimension set
+  auto* mean = input->reduceAlongDimension(reduce::Mean, &axes);
+  auto* part1 = (*input) - (*mean);
+  auto* part2 = (*part1) * (*factor);
+  delete part1;
+  auto* part3 = (*part2) + (*mean);
+  delete part2;
+  delete mean;
+  // this is contrast calculation
+  output->assign(part3);
+  delete part3;
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(adjust_contrast) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS})->setSameMode(true);
+}
+
+////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(adjust_contrast_v2, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  // just skip op if input is empty
+  if (input->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(input->rankOf() > 2, 0,
+               "ADJUST_CONTRAST_V2: op expects rank of input array to be >= 3, but got %i instead", input->rankOf());
+  REQUIRE_TRUE(block.numT() > 0 || block.width() > 1, 0, "ADJUST_CONTRAST_V2: Scale factor required");
+
+  NDArray* factor = nullptr;
+  auto size = input->sizeAt(-2) * input->sizeAt(-3);
+  auto channels = input->sizeAt(-1);
+  int sizeChannels = sd::math::sd_max<int>(1,size * channels);
+  auto batch = input->lengthOf() / sizeChannels;
+  std::vector<LongType> shape = {batch, size, channels};
+  auto* input3D = input->reshape(input->ordering(), shape);
+  auto* output3D = input->reshape(input->ordering(), shape);
+
+  if (block.width() > 1) {
+    factor = INPUT_VARIABLE(1);
+  }
+  else {
+    factor = new NDArray(output->dataType(), block.launchContext());
+#ifdef HAS_DOUBLE
+    factor->p(0, static_cast<double>(T_ARG(0)));
+#elif defined(HAS_FLOAT32)
+    factor->p(0, static_cast<float>(T_ARG(0)));
+#else
+#error "No floating-point type available for adjust_contrast_v2 operation"
+#endif
+  }
+
+  std::vector<LongType> axes({1});  // dim 1 of pseudoresult
+  // mean as reduction for last dimension set over size (dim 1) of result3D
+  auto* mean = input3D->reduceAlongDimension(reduce::Mean, &axes);
+  // result as (x - mean) * factor + mean
+  auto* temp = input3D->ulike();
+  std::vector<LongType> zeroTwo = {0, 2};
+  input3D->applyBroadcast(broadcast::Subtract,&zeroTwo, mean, temp);
+  temp->applyScalarArr(scalar::Multiply, factor, temp);
+  temp->applyBroadcast(broadcast::Add, &zeroTwo, mean, output3D);
+  output->assign(output3D);
+  delete temp;
+  delete mean;
+  delete input3D;
+  delete output3D;
+  return Status::OK;
+}
+
+DECLARE_TYPES(adjust_contrast_v2) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS})->setSameMode(true);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/adjust_hue.cpp

@@ -0,0 +1,92 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_adjust_hue)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/adjust_hue.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(adjust_hue, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  // just skip op if input is empty
+  if (input->isEmpty()) return Status::OK;
+
+  const int rank = input->rankOf();
+  const int arg_size = block.getIArguments()->size();
+  const LongType dimC = arg_size > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + rank) : rank - 1;
+
+  REQUIRE_TRUE(block.numT() > 0 || block.width() > 1, 0, "ADJUST_HUE: delta factor is required !");
+  REQUIRE_TRUE(rank >= 3, 0, "ADJUST_HUE: op expects rank of input array to be >= 3, but got %i instead", rank);
+  if (arg_size > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < rank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -rank, rank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0,
+               "ADJUST_HUE: operation expects image with 3 channels (R, G, B), but got %i instead",
+               input->sizeAt(dimC));
+
+  NDArray* delta = nullptr;
+
+  if (block.width() > 1)
+    delta = INPUT_VARIABLE(1);
+  else {
+    delta = new NDArray(output->dataType(), block.launchContext());
+#ifdef HAS_DOUBLE
+    delta->p(0, static_cast<double>(T_ARG(0)));
+#elif defined(HAS_FLOAT32)
+    delta->p(0, static_cast<float>(T_ARG(0)));
+#else
+#error "No floating-point type available for adjust_hue operation"
+#endif
+  }
+
+#ifdef HAS_DOUBLE
+  auto deltaVal = delta->e<double>(0);
+#elif defined(HAS_FLOAT32)
+  auto deltaVal = delta->e<float>(0);
+#else
+#error "No floating-point type available for adjust_hue operation"
+#endif
+
+  REQUIRE_TRUE(-1. <= deltaVal && deltaVal <= 1., 0,
+               "ADJUST_HUE: parameter delta must be within [-1, 1] interval, but got %f instead", delta);
+
+  helpers::adjustHue(block.launchContext(), input, delta, output, dimC);
+
+  if (block.width() == 1) delete delta;
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(adjust_hue) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/adjust_saturation.cpp

@@ -0,0 +1,81 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_adjust_saturation)
+
+#include <array/NDArrayFactory.h>
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/adjust_saturation.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(adjust_saturation, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  // just skip op if input is empty
+  if (input->isEmpty()) return Status::OK;
+
+  const int rank = input->rankOf();
+  const int arg_size = block.getIArguments()->size();
+  const int dimC = arg_size > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + rank) : rank - 1;
+
+  REQUIRE_TRUE(rank >= 3, 0, "ADJUST_SATURATION: op expects rank of input array to be >= 3, but got %i instead", rank);
+  if (arg_size > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < rank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -rank, rank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0,
+               "ADJUST_SATURATION: operation expects image with 3 channels (R, G, B), but got %i instead",
+               input->sizeAt(dimC));
+  REQUIRE_TRUE(block.numT() > 0 || block.width() > 1, 0, "ADJUST_SATURATION: scale factor is required !");
+
+  NDArray* factor = nullptr;
+
+  if (block.width() > 1)
+    factor = INPUT_VARIABLE(1);
+  else {
+    factor = new NDArray(output->dataType(), block.launchContext());
+#ifdef HAS_DOUBLE
+    factor->p(0, static_cast<double>(T_ARG(0)));
+#elif defined(HAS_FLOAT32)
+    factor->p(0, static_cast<float>(T_ARG(0)));
+#else
+#error "No floating-point type available for adjust_saturation operation"
+#endif
+  }
+
+  helpers::adjustSaturation(block.launchContext(), input, factor, output, dimC);
+
+  if (block.width() == 1) delete factor;
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(adjust_saturation) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/crop_and_resize.cpp

@@ -0,0 +1,109 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_crop_and_resize)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/crop_and_resize.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(crop_and_resize, 4, 1, false, 0, 0) {
+  auto image = INPUT_VARIABLE(0);
+  auto boxes = INPUT_VARIABLE(1);
+  auto boxIndexes = INPUT_VARIABLE(2);
+
+  auto output = OUTPUT_VARIABLE(0);
+  int width;
+  int height;
+  int method = 0;  // bilinear
+#ifdef HAS_DOUBLE
+  double extrapolationVal = 0.;
+#elif defined(HAS_FLOAT32)
+  float extrapolationVal = 0.0f;
+#else
+#error "No floating-point type available for crop_and_resize operation"
+#endif
+
+  auto newImageSize = INPUT_VARIABLE(3);
+  REQUIRE_TRUE(output->dataType() == image->dataType(), 0,
+               "crop_and_resize: Source images and output should have the same data type.");
+  REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0, "crop_and_resize: Resize params is a pair of values, not %i.",
+               newImageSize->lengthOf());
+  // REQUIRE_TRUE(block.numI() <= 1, 0, "crop_and_resize: Resize params already given by the second param. Int params
+  // are expensive."); width = int(newImageSize->getScalar(0)); height = int(newImageSize->getScalar(1));
+  if (block.numI() == 1) {
+    method = INT_ARG(0);
+  }
+
+  if (block.numT() == 1) {
+#ifdef HAS_DOUBLE
+    extrapolationVal = static_cast<double>(T_ARG(0));
+#elif defined(HAS_FLOAT32)
+    extrapolationVal = static_cast<float>(T_ARG(0));
+#else
+#error "No floating-point type available for crop_and_resize operation"
+#endif
+  }
+
+  helpers::cropAndResizeFunctor(block.launchContext(), image, boxes, boxIndexes, newImageSize, method, extrapolationVal,
+                                output);
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(crop_and_resize) {
+  auto in = inputShape->at(0);
+  auto boxShape = inputShape->at(1);
+
+  sd::LongType outputShape[4];
+
+  int width;
+  int height;
+  auto newImageSize = INPUT_VARIABLE(3);
+  REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0, "crop_and_resize: Resize params is a pair of values, not %i.",
+               newImageSize->lengthOf());
+  // REQUIRE_TRUE(block.numI() <= 1, 0, "crop_and_resize: Resize params already given by the second param. Int params
+  // are expensive.");
+  width = newImageSize->e<int>(0);
+  height = newImageSize->e<int>(1);
+
+  outputShape[0] = boxShape[1];
+  outputShape[1] = width;
+  outputShape[2] = height;
+  outputShape[3] = in[4];
+  auto desc = new  ShapeDescriptor(ArrayOptions::dataType(in), shape::order(in), outputShape, 4);
+  return SHAPELIST(ConstantShapeHelper::getInstance().createShapeInfo(desc));
+}
+
+DECLARE_TYPES(crop_and_resize) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(2, {ALL_INTS})
+      ->setAllowedInputTypes(3, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});  // as TF
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/draw_bounding_boxes.cpp

@@ -0,0 +1,74 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author George A. Shulinok <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_draw_bounding_boxes)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/image_draw_bounding_boxes.h>
+namespace sd {
+namespace ops {
+OP_IMPL(draw_bounding_boxes, 3, 1, true) {
+  auto images = INPUT_VARIABLE(0);
+  auto boxes = INPUT_VARIABLE(1);
+
+  auto colors = (NDArray*)nullptr;
+  if (block.width() > 2)         // TF v.1.x ommits color set for boxes, and use color 1.0 for fill up
+    colors = INPUT_VARIABLE(2);  // but v.2.y require color set
+
+  auto output = OUTPUT_VARIABLE(0);
+  REQUIRE_TRUE(images->dataType() == output->dataType(), 0,
+               "draw_bounding_boxes: Input and Output types "
+               "should be equals, but %d and %d occured.",
+               (int)images->dataType(), (int)output->dataType());
+  REQUIRE_TRUE(images->rankOf() == 4, 0, "draw_bounding_boxes: Images input should be 4D tensor, but %i occured.",
+               images->rankOf());
+  REQUIRE_TRUE(boxes->rankOf() == 3, 0, "draw_bounding_boxes: Boxes should be 3D tensor, but %i occured.",
+               boxes->rankOf());
+  if (colors) {
+    REQUIRE_TRUE(colors->rankOf() == 2, 0, "draw_bounding_boxes: Color set should be 2D matrix, but %i occured.",
+                 colors->rankOf());
+    REQUIRE_TRUE(colors->sizeAt(1) >= images->sizeAt(3), 0,
+                 "draw_bounding_boxes: Color set last dim "
+                 "should be not less than images depth, but "
+                 "%lld and %lld occured.",
+                 colors->sizeAt(1), images->sizeAt(3));
+  }
+  REQUIRE_TRUE(boxes->sizeAt(0) == images->sizeAt(0), 0,
+               "draw_bounding_boxes: Batches for images and boxes "
+               "should be the same, but %lld and %lld occured.",
+               images->sizeAt(0), boxes->sizeAt(0));
+  helpers::drawBoundingBoxesFunctor(block.launchContext(), images, boxes, colors, output);
+  return Status::OK;
+}
+
+DECLARE_TYPES(draw_bounding_boxes) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {HALF, FLOAT32})  // TF allows HALF and FLOAT32 only
+      ->setAllowedInputTypes(1, {FLOAT32})        // as TF
+      ->setAllowedInputTypes(2, {FLOAT32})        // as TF
+      ->setAllowedOutputTypes({HALF, FLOAT32});   // TF allows HALF and FLOAT32 only
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/extract_image_patches.cpp

@@ -0,0 +1,96 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> at 3/30/2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/extract_patches.h>
+#if NOT_EXCLUDED(OP_extract_image_patches)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(extract_image_patches, 1, 1, false, 0, 7) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  int ksizeRows = INT_ARG(0);
+  int ksizeCols = INT_ARG(1);
+  int kstrideRows = INT_ARG(2);
+  int kstrideCols = INT_ARG(3);
+  int krateRows = INT_ARG(4);
+  int krateCols = INT_ARG(5);
+  bool isSame = INT_ARG(6) != 0;
+
+  REQUIRE_TRUE(input->rankOf() == 4, 0, "extract_image_patches: The rank of input array should be 4, but %i is given",
+               input->rankOf());
+
+  if (output->isSameShape(input))
+    output->assign(input);
+  else {
+    output->nullify();
+    helpers::extractPatches(block.launchContext(), input, output, ksizeRows, ksizeCols, kstrideRows, kstrideCols,
+                            krateRows, krateCols, isSame);
+  }
+  return Status::OK;
+}
+
+DECLARE_TYPES(extract_image_patches) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+DECLARE_SHAPE_FN(extract_image_patches) {
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType *outputShape = nullptr;
+
+  int ksizeRowsEffective = INT_ARG(0) + (INT_ARG(0) - 1) * (INT_ARG(4) - 1);
+  int ksizeColsEffective = INT_ARG(1) + (INT_ARG(1) - 1) * (INT_ARG(5) - 1);
+
+  auto batchSizeDim = shape::sizeAt(in, static_cast<LongType>(0));
+  auto inputRowsDim = shape::sizeAt(in, static_cast<LongType>(1));
+  auto inputColsDim = shape::sizeAt(in, static_cast<LongType>(2));
+  auto outputDepthDim = shape::sizeAt(in, static_cast<LongType>(3)) * INT_ARG(0) * INT_ARG(1);  // last dim * ksizeRows * ksizeCols
+
+  auto inputRowSize = inputRowsDim;
+  auto inputColSize = inputColsDim;
+  LongType outRowSize;
+  LongType outColSize;
+  if (INT_ARG(6) == 0) {
+    // Padding is "VALID":
+    outRowSize = (inputRowSize - ksizeRowsEffective + INT_ARG(2)) / INT_ARG(2);
+    outColSize = (inputColSize - ksizeColsEffective + INT_ARG(3)) / INT_ARG(3);
+  } else {
+    // Padding is "SAME":
+    outRowSize = (inputRowSize + INT_ARG(2) - 1) / INT_ARG(2);
+    outColSize = (inputColSize + INT_ARG(3) - 1) / INT_ARG(3);
+  }
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+
+  outputShape[0] = outRank;
+  outputShape[1] = batchSizeDim;
+  outputShape[2] = outRowSize;
+  outputShape[3] = outColSize;
+  outputShape[4] = outputDepthDim;
+
+  ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/hsvToRgb.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author AbdelRauf    (rauf@konduit.ai)
+//
+
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/images.h>
+
+#if NOT_EXCLUDED(OP_hsv_to_rgb)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(hsv_to_rgb, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (input->isEmpty()) return Status::OK;
+
+  const int rank = input->rankOf();
+  const int argSize = block.getIArguments()->size();
+  const int dimC = argSize > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + rank) : rank - 1;
+
+  REQUIRE_TRUE(rank >= 1, 0, "HSVtoRGB: Fails to meet the rank requirement: %i >= 1 ", rank);
+  if (argSize > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < rank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -rank, rank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0, "HSVtoRGB: operation expects 3 channels (H, S, V), but got %i instead",
+               input->sizeAt(dimC));
+
+  helpers::transformHsvRgb(block.launchContext(), input, output, dimC);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(hsv_to_rgb) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/images/image_resize.cpp

@@ -0,0 +1,158 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_image_resize)
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/image_resize.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(image_resize, 2, 1, false, -2, -2) {
+  auto image = INPUT_VARIABLE(0);
+  auto size = INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  int width;
+  int height;
+  bool antialias = false;
+  REQUIRE_TRUE(size->lengthOf() == 2, 0, "image_resize: Resize params is a pair of values, not %lld.",
+               size->lengthOf());
+  width = size->e<int>(1);
+  height = size->e<int>(0);
+  if (block.numB() >= 2) {
+    antialias = B_ARG(1);
+  }
+  bool exclude_outside = true;
+  double bicubicCoefficient = helpers::KeysCubicKernelFunc<double>::KEYS_CUBIC_COEF;
+  auto method = helpers::ImageResizeMethods::kResizeBilinear;
+  helpers::CoordinateTransformationMode coorMode = helpers::CoordinateTransformationMode::HALF_PIXEL;
+
+  if (block.numB() >= 3) {
+    exclude_outside = B_ARG(2);
+  }
+  if (block.numT() > 0) {
+    bicubicCoefficient = T_ARG(0);
+  }
+  if (block.numI() >= 1) {
+    method = (helpers::ImageResizeMethods)INT_ARG(0);
+  }
+  if (block.numI() >= 2) {
+    coorMode = static_cast<helpers::CoordinateTransformationMode>(INT_ARG(1));
+  } else if (method == helpers::ImageResizeMethods::kResizeNearest) {
+    // retain old behavour
+    coorMode = helpers::CoordinateTransformationMode::HALF_PIXEL_NN;
+  }
+  helpers::NearestMode nearestMode = helpers::NearestMode::FLOOR;
+  if (method == helpers::ImageResizeMethods::kResizeNearest && block.numI() == 3) {
+    nearestMode = static_cast<helpers::NearestMode>(INT_ARG(2));
+    REQUIRE_TRUE(nearestMode >= helpers::NearestMode::FLOOR && nearestMode <= helpers::NearestMode::CEIL, 0,
+                 "image_resize: nearest Mode should be between %i and %i, but %i was given.",
+                 (int)helpers::NearestMode::FLOOR, (int)helpers::NearestMode::CEIL, (int)nearestMode);
+  }
+  REQUIRE_TRUE(method == helpers::ImageResizeMethods::kResizeNearest || output->dataType() == DataType::FLOAT32, 0,
+               "image_resize: Output data type should be FLOAT32 for this method %i", (int)method);
+  REQUIRE_TRUE(
+      method >= helpers::ImageResizeMethods::kResizeFirst && method <= helpers::ImageResizeMethods::kResizeLast, 0,
+      "image_resize: Resize method should be between %i and %i, but %i was given.",
+      (int)helpers::ImageResizeMethods::kResizeFirst, (int)helpers::ImageResizeMethods::kResizeLast, (int)method);
+  auto inRank = image->rankOf();
+  REQUIRE_TRUE(inRank >= 3 && inRank <= 4, 0, "image_resize: Input rank should be 4 or 3, but %i given.",
+               image->rankOf());
+  std::vector<sd::LongType> imageShape1 = {image->sizeAt(0), image->sizeAt(1), image->sizeAt(2),image->sizeAt(3)};
+  std::vector<sd::LongType> imageShape2 = {1, image->sizeAt(0), image->sizeAt(1), image->sizeAt(2)};
+  auto source =
+      inRank == 4
+          ? image->reshape(image->ordering(), imageShape1)
+          : image->reshape(image->ordering(),imageShape2);
+
+  std::vector<sd::LongType> outputShape1 = {output->sizeAt(0), output->sizeAt(1), output->sizeAt(2),output->sizeAt(3)};
+  std::vector<sd::LongType> outputShape2 = {1, output->sizeAt(0), output->sizeAt(1), output->sizeAt(2)};
+
+  auto target =
+      inRank == 4
+          ? output->reshape(output->ordering(),
+                            outputShape1, false)
+          : output->reshape(output->ordering(), outputShape2, false);
+
+  // inform the user about the current state of the implementation
+  if (antialias && method != helpers::ImageResizeMethods::kResizeNearest) {
+    REQUIRE_TRUE(coorMode == helpers::CoordinateTransformationMode::HALF_PIXEL && exclude_outside, 0,
+                 "antialiasing is effective only with HALF_PIXEL and exclude_outside being set true");
+  }
+  //
+  if ((method != helpers::ImageResizeMethods::kResizeBicubic &&
+       method != helpers::ImageResizeMethods::kResizeNearest)) {
+    REQUIRE_TRUE(coorMode == helpers::CoordinateTransformationMode::HALF_PIXEL && exclude_outside, 0,
+                 "this method supports only HALF_PIXEL and exclude_outside being set true");
+  }
+
+  auto ret =  resizeFunctor(block.launchContext(), image, width, height, method, coorMode, exclude_outside,
+                                nearestMode, bicubicCoefficient, antialias, output);
+  delete target;
+  return ret;
+}
+
+DECLARE_SHAPE_FN(image_resize) {
+  auto in = inputShape->at(0);
+
+  LongType* outputShape;
+  auto method = helpers::ImageResizeMethods::kResizeBilinear;
+  if (block.numI() >= 1) {
+    method = (helpers::ImageResizeMethods)INT_ARG(0);
+  }
+
+  int width;
+  int height;
+  double ratio = shape::sizeAt(in, static_cast<LongType>(1)) / (0.0 + shape::sizeAt(in, static_cast<LongType>(2)));
+  auto newImageSize = INPUT_VARIABLE(1);
+  REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0, "resize_bilinear: Resize params is a pair of values, not %i.",
+               newImageSize->lengthOf());
+
+  width = newImageSize->e<int>(1);
+  height = newImageSize->e<int>(0);
+  if (block.numB() > 0) {
+    if (B_ARG(0)) {
+      width = math::sd_ceil<double, int>(height / ratio);
+    }
+  }
+  auto dtype = FLOAT32;
+  if (method == helpers::ImageResizeMethods::kResizeNearest) dtype = ArrayOptions::dataType(in);
+  auto shape = ConstantShapeHelper::getInstance().createShapeInfo(
+      dtype, 'c',
+      shape::rank(in) == 4 ? std::vector<LongType>{in[1], height, width, in[4]}
+                           : std::vector<LongType>{height, width, in[4]});
+
+  return SHAPELIST(shape);
+}
+DECLARE_TYPES(image_resize) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/resize_area.cpp

@@ -0,0 +1,143 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_resize_area)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/image_resize.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(resize_area, 1, 1, false, 0, -2) {
+  auto image = INPUT_VARIABLE(0);
+  int width;
+  int height;
+
+  if (block.width() == 2) {
+    auto size = INPUT_VARIABLE(1);  // integer vector with shape {2} and content (new_height, new_width)
+    REQUIRE_TRUE(size->rankOf() == 1, size->lengthOf() == 2, 0,
+                 "resize_area: Resize params is a pair of values, not %i.", size->lengthOf());
+    size->syncToHost();
+    width = size->e<int>(1);
+    height = size->e<int>(0);
+  } else {
+    REQUIRE_TRUE(block.numI() == 2, 0,
+                 "resize_area: Resize params already given by the second param. Int params are expensive.");
+    width = INT_ARG(1);
+    height = INT_ARG(0);
+  }
+
+  auto output = OUTPUT_VARIABLE(0);
+  if (output->isEmpty()) return Status::OK;
+  auto inRank = image->rankOf();
+
+  REQUIRE_TRUE(inRank == 3 || inRank == 4, 0, "resize_area: Source tensor should have rank 4, but %i given.", inRank);
+  REQUIRE_TRUE(output->rankOf() == inRank, 0,
+               "resize_area: Source tensor and output should have the same rank, but  %i and %i given.", inRank,
+               output->rankOf());
+  REQUIRE_TRUE(width > 0, 0, "resize_area: picture width should be positive 32 bit integer, but %i given", width);
+  REQUIRE_TRUE(height > 0, 0, "resize_area: picture height should be positive 32 bit integer, but %i given", height);
+  REQUIRE_TRUE(image->lengthOf() > 0, 0, "resize_area: Only non-zero images allowed to processing.");
+
+  auto alignCorners = false;
+  if (block.numB() > 0) {
+    alignCorners = B_ARG(0);
+  }
+  std::vector<sd::LongType> imageShape1 = {image->sizeAt(0), image->sizeAt(1), image->sizeAt(2),image->sizeAt(3)};
+  std::vector<sd::LongType> imageShape2 = {1, image->sizeAt(0), image->sizeAt(1), image->sizeAt(2)};
+
+  auto source =
+      inRank == 4
+      ? image->reshape(image->ordering(), imageShape1)
+      : image->reshape(image->ordering(), imageShape2);
+
+
+  std::vector<sd::LongType> outputShape1 = {output->sizeAt(0), output->sizeAt(1), output->sizeAt(2),output->sizeAt(3)};
+  std::vector<sd::LongType> outputShape2 = {1, output->sizeAt(0), output->sizeAt(1), output->sizeAt(2)};
+
+  auto target =
+      inRank == 4
+      ? output->reshape(output->ordering(),
+                        outputShape1, false)
+      : output->reshape(output->ordering(), outputShape2, false);
+
+  auto ret =  helpers::resizeAreaFunctor(block.launchContext(), source, width, height, alignCorners, target);
+  delete source;
+  delete target;
+  return ret;
+}
+
+DECLARE_SHAPE_FN(resize_area) {
+  auto shapeList = SHAPELIST();
+  auto in = inputShape->at(0);
+
+  LongType* outputShape;
+  auto inRank = shape::rank(in);
+  int width;
+  int height;
+  if (block.width() == 2) {
+    auto newImageSize = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0, "resize_area: Resize params is a pair of values, not %i.",
+                 newImageSize->lengthOf());
+    REQUIRE_TRUE(block.numI() <= 1, 0,
+                 "resize_area: Resize params already given by the second param. Int params are expensive.");
+    width = newImageSize->e<int>(1);
+    height = newImageSize->e<int>(0);
+  } else {
+    REQUIRE_TRUE(block.numI() == 2, 0, "resize_area: Resize params ommited as pair ints nor int tensor.");
+    width = INT_ARG(1);
+    height = INT_ARG(0);
+  }
+
+  REQUIRE_TRUE(inRank == 4 || inRank == 3, 0, "resize_area: Source tensor should have rank 4, but %i given.", inRank);
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(inRank), sd::LongType);
+  outputShape[0] = inRank;
+  if (inRank == 4) {
+    outputShape[1] = in[1];
+    outputShape[2] = height;
+    outputShape[3] = width;
+    outputShape[4] = in[4];
+  } else {
+    outputShape[1] = height;
+    outputShape[2] = width;
+    outputShape[3] = in[3];
+  }
+  ShapeUtils::updateStridesAndType(outputShape, FLOAT32, shape::order(in));
+
+  shapeList->push_back(CONSTANT(outputShape));
+  return shapeList;
+}
+DECLARE_TYPES(resize_area) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, INT32)
+      ->setAllowedOutputTypes({FLOAT32});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/resize_bicubic.cpp

@@ -0,0 +1,150 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_resize_bicubic)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/image_resize.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(resize_bicubic, 2, 1, false, 0, 0) {
+  auto image = INPUT_VARIABLE(0);
+  auto size = INPUT_VARIABLE(1);  // integer vector with shape {2} and content (new_height, new_width)
+  size->syncToHost();
+  auto output = OUTPUT_VARIABLE(0);
+  int width;
+  int height;
+  auto inRank = image->rankOf();
+  if (output->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(inRank == 3 || inRank == 4, 0, "resize_bicubic: Source tensor should have rank 4, but %i given.",
+               inRank);
+  REQUIRE_TRUE(output->rankOf() == inRank, 0,
+               "resize_bicubic: Source tensor and output should have the same rank, but  %i and %i given.", inRank,
+               output->rankOf());
+  REQUIRE_TRUE(size->rankOf() == 1, size->lengthOf() == 2, 0,
+               "resize_bicubic: Resize params is a pair of values, not %i.", size->lengthOf());
+  REQUIRE_TRUE(block.numI() <= 1, 0,
+               "resize_bicubic: Resize params already given by the second param. Int params are expensive.");
+  width = size->e<int>(1);
+  height = size->e<int>(0);
+  REQUIRE_TRUE(width > 0, 0, "resize_bicubic: picture width should be positive 32 bit integer, but %i given", width);
+  REQUIRE_TRUE(height > 0, 0, "resize_bicubic: picture height should be positive 32 bit integer, but %i given", height);
+  // REQUIRE_TRUE(image->sizeAt(1) > 3 && image->sizeAt(2) > 3, 0, "resize_cubic: To use bicubic algorithm need at least
+  // 16 pixels as source.");
+  REQUIRE_TRUE(width > 3 && height > 3, 0,
+               "resize_bicubic: To use bicubic algorithm need at least 16 pixels as target.");
+  REQUIRE_TRUE(image->lengthOf() > 0, 0, "resize_bicubic: Only non-zero images allowed to processing.");
+  //            auto method = 1; //kResizeBilinear;
+  //            if (block.numI() == 1) {
+  //                method = INT_ARG(0);
+  //            }
+  auto alignCorners = false;
+  auto halfPixelAlign = false;
+  if (block.numB() > 0) {
+    alignCorners = block.getBArguments()->at(0);
+    if (block.numB() > 1) halfPixelAlign = block.getBArguments()->at(1);
+  }
+  REQUIRE_TRUE(!halfPixelAlign || (halfPixelAlign && !alignCorners), 0,
+               "resize_bicubic: `half_pixel_centers' should be false or true only when `align_corners' is false");
+  std::vector<sd::LongType> imageShape1 = {image->sizeAt(0), image->sizeAt(1), image->sizeAt(2),image->sizeAt(3)};
+  std::vector<sd::LongType> imageShape2 = {1, image->sizeAt(0), image->sizeAt(1), image->sizeAt(2)};
+
+  auto source =
+      inRank == 4
+          ? image->reshape(image->ordering(), imageShape1)
+          : image->reshape(image->ordering(), imageShape2);
+
+
+  std::vector<sd::LongType> outputShape1 = {output->sizeAt(0), output->sizeAt(1), output->sizeAt(2),output->sizeAt(3)};
+  std::vector<sd::LongType> outputShape2 = {1, output->sizeAt(0), output->sizeAt(1), output->sizeAt(2)};
+
+  auto target =
+      inRank == 4
+          ? output->reshape(output->ordering(),
+                            outputShape1, false)
+          : output->reshape(output->ordering(), outputShape2, false);
+
+  // retain old behaviour
+  helpers::CoordinateTransformationMode coorMode = halfPixelAlign ? helpers::CoordinateTransformationMode::HALF_PIXEL
+                                                                  : helpers::CoordinateTransformationMode::ASYMMETRIC;
+  bool exclude_outside = halfPixelAlign;
+  double coef = halfPixelAlign ? helpers::KeysCubicKernelFunc<double>::KEYS_CUBIC_COEF
+                               : helpers::KeysCubicKernelFunc<double>::ORDINARY_COEF;
+  auto ret =  resizeBicubicFunctorA(block.launchContext(), source, width, height, alignCorners, coorMode,
+                                        exclude_outside, coef, target);
+  delete source;
+  delete target;
+  return ret;
+}
+
+DECLARE_SHAPE_FN(resize_bicubic) {
+  auto shapeList = SHAPELIST();
+  auto in = inputShape->at(0);
+
+  LongType* outputShape;
+  auto inRank = shape::rank(in);
+  int width;
+  int height;
+  auto newImageSize = INPUT_VARIABLE(1);
+  REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0, "resize_bicubic: Resize params is a pair of values, not %i.",
+               newImageSize->lengthOf());
+  REQUIRE_TRUE(block.numI() <= 1, 0,
+               "resize_bicubic: Resize params already given by the second param. Int params are expensive.");
+  width = newImageSize->e<int>(0);
+  height = newImageSize->e<int>(1);
+
+  REQUIRE_TRUE(inRank == 4 || inRank == 3, 0, "resize_bicubic: Source tensor should have rank 4, but %i given.",
+               inRank);
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(inRank), sd::LongType);
+  outputShape[0] = inRank;
+  if (inRank == 4) {
+    outputShape[1] = in[1];
+    outputShape[2] = width;
+    outputShape[3] = height;
+    outputShape[4] = in[4];
+  } else {
+    outputShape[1] = width;
+    outputShape[2] = height;
+    outputShape[3] = in[3];
+  }
+  ShapeUtils::updateStridesAndType(outputShape, FLOAT32, shape::order(in));
+
+  shapeList->push_back(CONSTANT(outputShape));
+  return shapeList;
+}
+DECLARE_TYPES(resize_bicubic) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, INT32)
+      ->setAllowedOutputTypes({FLOAT32, DOUBLE});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/resize_images.cpp

@@ -0,0 +1,154 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_resize_images)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/image_resize.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(resize_images, 1, 1, false, 0, 0) {
+  auto image = INPUT_VARIABLE(0);
+
+  auto output = OUTPUT_VARIABLE(0);
+  int width = output->sizeAt(2);
+  int height = output->sizeAt(1);
+  int method = helpers::ImageResizeMethods::kResizeBilinear;
+  if (block.width() > 1) {
+    auto size = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(size->lengthOf() == 2, 0, "resize_images: Resize params is a pair of values, not %lld.",
+                 size->lengthOf());
+    //                width = size->e<int>(1);
+    //                height = size->e<int>(0);
+    if (block.width() > 2) {
+      auto methodT = INPUT_VARIABLE(2);
+
+      REQUIRE_TRUE(methodT->isZ() && methodT->isScalar(), 0,
+                   "resize_images: Method tensor should be integer scalar, but rank of %i tensor given.",
+                   methodT->rankOf());
+      method = methodT->e<int>(0);
+    } else if (block.numI() == 1) {
+      method = I_ARG(0);
+    }
+  } else {
+    REQUIRE_TRUE(block.numI() > 1 && block.numI() < 4, 0, "resize_images: Method and size should be given properly.");
+    if (block.numI() == 3) {  // full stack of args
+                              //                    height = I_ARG(0);
+                              //                    width = I_ARG(1);
+      method = I_ARG(2);
+    } else if (block.numI() == 2) {
+      //                    height = I_ARG(0);
+      //                    width = I_ARG(1);
+    }
+  }
+  bool alignCorners = false;
+  if (block.numB()) {
+    alignCorners = B_ARG(0);
+  }
+  REQUIRE_TRUE(
+      method >= helpers::ImageResizeMethods::kResizeFirst && method <= helpers::ImageResizeMethods::kResizeOldLast, 0,
+      "resize_images: Resize method should be between %i and %i, but %i was given.",
+      (int)helpers::ImageResizeMethods::kResizeFirst, (int)helpers::ImageResizeMethods::kResizeOldLast, (int)method);
+  REQUIRE_TRUE(method == helpers::ImageResizeMethods::kResizeNearest || output->dataType() == DataType::FLOAT32, 0,
+               "image_resize: Output data type should be FLOAT32 for this method %i", (int)method);
+
+  auto inRank = image->rankOf();
+  REQUIRE_TRUE(inRank >= 3 && inRank <= 4, 0, "image_resize: Input rank should be 4 or 3, but %i given.", inRank);
+  std::vector<sd::LongType> imageShape1 = {image->sizeAt(0), image->sizeAt(1), image->sizeAt(2),image->sizeAt(3)};
+  std::vector<sd::LongType> imageShape2 = {1, image->sizeAt(0), image->sizeAt(1), image->sizeAt(2)};
+
+  auto source =
+      inRank == 4
+          ? image->reshape(image->ordering(), imageShape1)
+          : image->reshape(image->ordering(), imageShape2);
+
+
+
+  std::vector<sd::LongType> outputShape1 = {output->sizeAt(0), output->sizeAt(1), output->sizeAt(2),output->sizeAt(3)};
+  std::vector<sd::LongType> outputShape2 = {1, output->sizeAt(0), output->sizeAt(1), output->sizeAt(2)};
+
+  auto target =
+      inRank == 4
+          ? output->reshape(output->ordering(),
+                            outputShape1, false)
+          : output->reshape(output->ordering(), outputShape2, false);
+
+  auto ret =  resizeImagesFunctor(block.launchContext(), source, width, height,
+                                      (helpers::ImageResizeMethods)method, alignCorners, target);
+  delete source;
+  delete target;
+  return ret;
+}
+
+DECLARE_SHAPE_FN(resize_images) {
+  auto in = inputShape->at(0);
+
+  LongType* outputShape;
+
+  int width;
+  int height;
+  if (block.width() > 1) {
+    auto size = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(size->lengthOf() == 2, 0, "resize_images: Resize params is a pair of values, not %lld.",
+                 size->lengthOf());
+    width = size->e<int>(1);
+    height = size->e<int>(0);
+  } else {
+    REQUIRE_TRUE(block.numI() > 1 && block.numI() < 4, 0, "resize_images: Method and size should be given properly.");
+    if (block.numI() == 3) {  // full stack of args
+      height = I_ARG(0);
+      width = I_ARG(1);
+    } else if (block.numI() == 2) {
+      height = I_ARG(0);
+      width = I_ARG(1);
+    }
+  }
+
+  double ratio = shape::sizeAt(in, static_cast<LongType>(1)) / (0.0 + shape::sizeAt(in, static_cast<LongType>(2)));
+  if (block.numB() > 1) {
+    if (B_ARG(1)) {
+      width = math::sd_ceil<double, int>(height / ratio);
+    }
+  }
+
+  std::vector<LongType> shape;
+  if (shape::rank(in) == 4)
+    shape = {in[1], height, width, in[4]};
+  else if (shape::rank(in) == 3)
+    shape = {height, width, in[3]};
+
+  auto outShape = ConstantShapeHelper::getInstance().createShapeInfo(FLOAT32, shape::order(in), shape);
+  return SHAPELIST(outShape);
+}
+DECLARE_TYPES(resize_images) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes({FLOAT32});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/resize_linear.cpp

@@ -0,0 +1,150 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_resize_bilinear)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/image_resize.h>
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(resize_bilinear, 1, 1, false, 0, -2) {
+  NDArray* image = INPUT_VARIABLE(0);
+  NDArray* output = OUTPUT_VARIABLE(0);
+  int width;
+  int height;
+  bool alignCorners = false;  // - default value
+  auto inRank = image->rankOf();
+  if (output->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(inRank == 4 || inRank == 3, 0,
+               "resize_bilinear: input image should be 4D "
+               "tensor, but input has rank %i",
+               image->rankOf());
+  REQUIRE_TRUE(inRank == output->rankOf(), 0,
+               "resize_bilinear: Input and output ranks should be equals, but %i and %i occured.", inRank,
+               output->rankOf());
+  std::vector<sd::LongType> imageShape1 = {image->sizeAt(0), image->sizeAt(1), image->sizeAt(2),image->sizeAt(3)};
+  std::vector<sd::LongType> imageShape2 = {1, image->sizeAt(0), image->sizeAt(1), image->sizeAt(2)};
+
+  auto source =
+      inRank == 4
+          ? image->reshape(image->ordering(),imageShape1)
+          : image->reshape(image->ordering(), imageShape2);
+
+
+  std::vector<sd::LongType> outputShape1 = {output->sizeAt(0), output->sizeAt(1), output->sizeAt(2),output->sizeAt(3)};
+  std::vector<sd::LongType> outputShape2 = {1, output->sizeAt(0), output->sizeAt(1), output->sizeAt(2)};
+
+  auto target =
+      inRank == 4
+          ? output->reshape(output->ordering(),
+                            outputShape1, false)
+          : output->reshape(output->ordering(), outputShape2, false);
+
+  if (block.width() > 1) {
+    auto newImageSize = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0, "resize_bilinear: Resize params is a pair of values, not %i.",
+                 newImageSize->lengthOf());
+    REQUIRE_TRUE(block.numI() <= 1, 0,
+                 "resize_bilinear: Resize params already given by the second param. Int params are expensive.");
+    height = newImageSize->e<int>(0);
+    width = newImageSize->e<int>(1);
+  } else {
+    REQUIRE_TRUE(block.numI() > 1, 0, "resize_bilinear: Neither resize width nor height are provided.");
+    height = INT_ARG(0);
+    width = INT_ARG(1);
+  }
+
+  if (block.numB() > 0) alignCorners = B_ARG(0);
+  bool halfPixelCenter = false;
+
+  if (block.numB() > 1) halfPixelCenter = B_ARG(1);
+
+  REQUIRE_TRUE(!halfPixelCenter || (halfPixelCenter && !alignCorners), 0,
+               "resize_bilinear: `half_pixel_centers' should be false or true only when `align_corners' is false");
+
+  auto ret =  helpers::resizeBilinearFunctor(block.launchContext(), inRank == 4 ? image : source, width, height,
+                                        alignCorners, halfPixelCenter, inRank == 4 ? output : target);
+  delete source;
+  delete target;
+  return ret;
+}
+
+DECLARE_SHAPE_FN(resize_bilinear) {
+  auto shapeList = SHAPELIST();
+  auto in = inputShape->at(0);
+
+  LongType* outputShape;
+  auto inRank = shape::rank(in);
+  REQUIRE_TRUE(inRank == 4 || inRank == 3, 0,
+               "resize_bilinear: input image should be 4D "
+               "tensor, but input has rank %i",
+               inRank);
+
+  int width;
+  int height;
+  if (block.width() > 1) {
+    auto newImageSize = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0, "resize_bilinear: Resize params is a pair of values, not %i.",
+                 newImageSize->lengthOf());
+    REQUIRE_TRUE(block.numI() <= 1, 0,
+                 "resize_bilinear: Resize params already given by the second param. Int params are expensive.");
+    width = newImageSize->e<int>(0);
+    height = newImageSize->e<int>(1);
+  } else {
+    REQUIRE_TRUE(block.numI() == 2, 0, "resize_bilinear: Neither resize width nor height are provided.");
+    width = INT_ARG(0);
+    height = INT_ARG(1);
+  }
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(inRank), sd::LongType);
+  outputShape[0] = inRank;
+  if (inRank == 4) {
+    outputShape[1] = in[1];
+    outputShape[2] = width;
+    outputShape[3] = height;
+    outputShape[4] = in[4];
+  } else {  // input shape is 3D, so result also should be 3D
+    outputShape[1] = width;
+    outputShape[2] = height;
+    outputShape[3] = in[3];
+  }
+  if (DataTypeUtils::isR(ArrayOptions::dataType(in))) {
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+  } else {
+    ShapeUtils::updateStridesAndType(outputShape, FLOAT32, shape::order(in));
+  }
+
+  shapeList->push_back(CONSTANT(outputShape));
+  return shapeList;
+}
+DECLARE_TYPES(resize_bilinear) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/resize_neighbor.cpp

@@ -0,0 +1,154 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_resize_nearest_neighbor)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/image_resize.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(resize_nearest_neighbor, 1, 1, false, 0, -2) {
+  auto image = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  auto inRank = image->rankOf();
+  int width;
+  int height;
+  bool alignCorners = false;  // - default value
+  if (output->isEmpty()) return Status::OK;
+  if (block.width() > 1) {
+    auto newImageSize = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0,
+                 "resize_nearest_neighbor: Resize params is a pair of values, not %i.", newImageSize->lengthOf());
+    REQUIRE_TRUE(block.numI() <= 1, 0,
+                 "resize_nearest_neighbor: Resize params already given by the second param. Int params are expensive.");
+    height = newImageSize->e<int>(0);
+    width = newImageSize->e<int>(1);
+  } else {
+    REQUIRE_TRUE(block.numI() >= 2, 0, "resize_nearest_neighbor: Neither resize width nor height are provided.");
+    height = INT_ARG(0);
+    width = INT_ARG(1);
+  }
+  if (block.numB() > 0) alignCorners = B_ARG(0);
+
+  bool halfPixelCenter = false;
+
+  if (block.numB() > 1) halfPixelCenter = B_ARG(1);
+  REQUIRE_TRUE(width <= (1 << 24) || height <= (1 << 24), 0,
+               "resize_nearest_neighbor: the image resize should be limited to 2^24 pixels both for height and width, "
+               "but %d and %d were given.",
+               height, width);
+  REQUIRE_TRUE(inRank == 4 || inRank == 3, 0,
+               "resize_nearest_neighbor: Input should be 4D tensor, but rank %i occured");
+  REQUIRE_TRUE(inRank == output->rankOf(), 0,
+               "resize_nearest_neighbor: Input and output ranks should be equals, but %i and %i occured.", inRank,
+               output->rankOf());
+  REQUIRE_TRUE(image->dataType() == output->dataType(), 0,
+               "resize_nearest_neighbor: Input and output types should be the same, but `%s' occured instead.",
+               DataTypeUtils::asString(output->dataType()).c_str());
+  REQUIRE_TRUE(
+      !halfPixelCenter || (halfPixelCenter && !alignCorners), 0,
+      "resize_nearest_neighbor: `half_pixel_centers' should be false or true only when `align_corners' is false");
+  REQUIRE_TRUE(((alignCorners && height > 2) || (height > 0)) && ((alignCorners && width > 1) || (width > 0)), 0,
+               "resize_nearest_neighbor: Wrong input or output size to resize (width = %d, height = %d)", width,
+               height);
+
+  std::vector<sd::LongType> imageShape = {image->sizeAt(0), image->sizeAt(1), image->sizeAt(2)};
+  auto source = inRank == 4
+                    ? image
+                    : image->reshape(image->ordering(), imageShape);
+  std::vector<sd::LongType> outputShape = {1, output->sizeAt(0), output->sizeAt(1),output->sizeAt(2)};
+  auto target = inRank == 4 ? output
+                            : output->reshape(output->ordering(),
+                                              outputShape, false);
+
+  helpers::NearestMode nearestMode = helpers::NearestMode::FLOOR;
+  if (alignCorners) {
+    nearestMode = helpers::NearestMode::ROUND_PREFER_CEIL;
+  }
+  helpers::CoordinateTransformationMode coorMode = halfPixelCenter
+                                                       ? helpers::CoordinateTransformationMode::HALF_PIXEL_NN
+                                                       : helpers::CoordinateTransformationMode::ASYMMETRIC;
+
+  auto ret =  resizeNeighborFunctor(block.launchContext(), inRank == 4 ? image : source, width, height, coorMode,
+                                        nearestMode, alignCorners, inRank == 4 ? output : target);
+  delete source;
+  delete target;
+  return ret;
+
+}
+
+DECLARE_SHAPE_FN(resize_nearest_neighbor) {
+  auto shapeList = SHAPELIST();
+  auto in = inputShape->at(0);
+  auto inRank = shape::rank(in);
+  LongType* outputShape;
+
+  REQUIRE_TRUE(inRank == 4 || inRank == 3, 0,
+               "resize_nearest_neighbor: input image should be 4D "
+               "tensor, but input has rank %i",
+               inRank);
+
+  int width;
+  int height;
+  if (block.width() > 1) {
+    auto newImageSize = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(newImageSize->lengthOf() == 2, 0,
+                 "resize_nearest_neighbor: Resize params is a pair of values, not %i.", newImageSize->lengthOf());
+    REQUIRE_TRUE(block.numI() <= 1, 0,
+                 "resize_nearest_neighbor: Resize params already given by the second param. Int params are expensive.");
+    width = newImageSize->e<int>(0);
+    height = newImageSize->e<int>(1);
+  } else {
+    REQUIRE_TRUE(block.numI() <= 3, 0, "resize_nearest_neighbor: Neither resize width nor height are provided.");
+    width = INT_ARG(0);
+    height = INT_ARG(1);
+  }
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(inRank), sd::LongType);
+  outputShape[0] = inRank;
+  if (inRank == 4) {
+    outputShape[1] = in[1];
+    outputShape[2] = width;
+    outputShape[3] = height;
+    outputShape[4] = in[4];
+  } else {  // input shape is 3D, so result also should be 3D
+    outputShape[1] = width;
+    outputShape[2] = height;
+    outputShape[3] = in[3];
+  }
+  ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  shapeList->push_back(CONSTANT(outputShape));
+  return shapeList;
+}
+DECLARE_TYPES(resize_nearest_neighbor) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/rgbToGrs.cpp

@@ -0,0 +1,80 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/images.h>
+#if NOT_EXCLUDED(OP_rgb_to_grs)
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(rgb_to_grs, 1, 1, false, 0, 0) {
+  const auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const int inRank = input->rankOf();
+  const int argSize = block.getIArguments()->size();
+  const int dimC = argSize > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + inRank) : inRank - 1;
+
+  REQUIRE_TRUE(inRank >= 1, 0, "RGBtoGrayScale: Fails to meet the inRank requirement: %i >= 1 ", inRank);
+  if (argSize > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < inRank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -inRank, inRank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0,
+               "RGBGrayScale: operation expects 3 channels (R, G, B) in last dimention, but received %i instead",
+               input->sizeAt(dimC));
+
+  helpers::transformRgbGrs(block.launchContext(), *input, *output, dimC);
+  return Status::OK;
+}
+
+DECLARE_TYPES(rgb_to_grs) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})->setSameMode(true); }
+
+DECLARE_SHAPE_FN(rgb_to_grs) {
+  const auto input = INPUT_VARIABLE(0);
+  const int inRank = input->rankOf();
+
+  const int argSize = block.getIArguments()->size();
+  const int dimC = argSize > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + inRank) : inRank - 1;
+
+  REQUIRE_TRUE(inRank >= 1, 0, "RGBtoGrayScale: Fails to meet the inRank requirement: %i >= 1 ", inRank);
+  if (argSize > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < inRank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -inRank, inRank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0,
+               "RGBtoGrayScale: operation expects 3 channels (R, B, G) in last dimention, but received %i", dimC);
+
+  auto* inputShapeVec = input->getShapeAsVector();
+  (*inputShapeVec)[dimC] = 1;
+
+  auto result = SHAPELIST(ConstantShapeHelper::getInstance().createShapeInfo(input->dataType(), input->ordering(), *inputShapeVec));
+  delete inputShapeVec;
+  return result;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/rgbToHsv.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author AbdelRauf    (rauf@konduit.ai)
+//
+
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/images.h>
+#if NOT_EXCLUDED(OP_rgb_to_hsv)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(rgb_to_hsv, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (input->isEmpty()) return Status::OK;
+
+  const int rank = input->rankOf();
+  const int argSize = block.getIArguments()->size();
+  const int dimC = argSize > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + rank) : rank - 1;
+
+  REQUIRE_TRUE(rank >= 1, 0, "RGBtoHSV: Fails to meet the rank requirement: %i >= 1 ", rank);
+  if (argSize > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < rank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -rank, rank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0, "RGBtoHSV: operation expects 3 channels (R, G, B), but got %i instead",
+               input->sizeAt(dimC));
+
+  helpers::transformRgbHsv(block.launchContext(), input, output, dimC);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(rgb_to_hsv) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/rgbToYiq.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author AbdelRauf    (rauf@konduit.ai)
+//
+
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/images.h>
+
+#if NOT_EXCLUDED(OP_rgb_to_yiq)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(rgb_to_yiq, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (input->isEmpty()) return Status::OK;
+
+  const int rank = input->rankOf();
+  const int arg_size = block.getIArguments()->size();
+  const int dimC = arg_size > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + rank) : rank - 1;
+
+  REQUIRE_TRUE(rank >= 1, 0, "RGBtoYIQ: Fails to meet the rank requirement: %i >= 1 ", rank);
+  if (arg_size > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < rank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -rank, rank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0, "RGBtoYIQ: operation expects 3 channels (R, G, B), but got %i instead",
+               input->sizeAt(dimC));
+
+  helpers::transformRgbYiq(block.launchContext(), input, output, dimC);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(rgb_to_yiq) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/rgbToYuv.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/images.h>
+#if NOT_EXCLUDED(OP_rgb_to_yuv)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(rgb_to_yuv, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  // just skip op if input is empty
+  if (input->isEmpty()) return Status::OK;
+
+  const int rank = input->rankOf();
+  const int argSize = block.getIArguments()->size();
+  const int dimC = argSize > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + rank) : rank - 1;
+
+  REQUIRE_TRUE(rank >= 1, 0, "RGBtoYUV: Fails to meet the rank requirement: %i >= 1 ", rank);
+  if (argSize > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < rank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -rank, rank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0, "RGBtoYUV: operation expects 3 channels (R, G, B), but got %i instead",
+               input->sizeAt(dimC));
+
+  helpers::transformRgbYuv(block.launchContext(), *input, *output, dimC);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(rgb_to_yuv) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/images/yiqToRgb.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author AbdelRauf    (rauf@konduit.ai)
+//
+
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/images.h>
+#if NOT_EXCLUDED(OP_yiq_to_rgb)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(yiq_to_rgb, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (input->isEmpty()) return Status::OK;
+
+  const int rank = input->rankOf();
+  const int arg_size = block.getIArguments()->size();
+  const int dimC = arg_size > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + rank) : rank - 1;
+
+  REQUIRE_TRUE(rank >= 1, 0, "YIQtoRGB: Fails to meet the rank requirement: %i >= 1 ", rank);
+  if (arg_size > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < rank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -rank, rank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0, "YIQtoRGB: operation expects 3 channels (Y, I, Q), but got %i instead",
+               input->sizeAt(dimC));
+
+  helpers::transformYiqRgb(block.launchContext(), input, output, dimC);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(yiq_to_rgb) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/images/yuvToRgb.cpp

@@ -0,0 +1,60 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/images.h>
+#if NOT_EXCLUDED(OP_yuv_to_rgb)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(yuv_to_rgb, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  // just skip op if input is empty
+  if (input->isEmpty()) return Status::OK;
+
+  const int rank = input->rankOf();
+  const int argSize = block.getIArguments()->size();
+  const int dimC = argSize > 0 ? (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + rank) : rank - 1;
+
+  REQUIRE_TRUE(rank >= 1, 0, "YUVtoRGB: Fails to meet the rank requirement: %i >= 1 ", rank);
+  if (argSize > 0) {
+    REQUIRE_TRUE(dimC >= 0 && dimC < rank, 0, "Index of the Channel dimension out of range: %i not in [%i,%i) ",
+                 INT_ARG(0), -rank, rank);
+  }
+  REQUIRE_TRUE(input->sizeAt(dimC) == 3, 0, "YUVtoRGB: operation expects 3 channels (Y, U, V), but got %i instead",
+               input->sizeAt(dimC));
+
+  helpers::transformYuvRgb(block.launchContext(), *input, *output, dimC);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(yuv_to_rgb) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/kernels/knn_mindistance.cpp

@@ -0,0 +1,62 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_knn_mindistance)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/knn.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(knn_mindistance, 3, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto lowest = INPUT_VARIABLE(1);
+  auto highest = INPUT_VARIABLE(2);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->lengthOf() == lowest->lengthOf() && input->lengthOf() == highest->lengthOf(), 0,
+               "knn_mindistance: all input arrays must have same length");
+  REQUIRE_TRUE(input->dataType() == lowest->dataType() && input->dataType() == highest->dataType() &&
+                   input->dataType() == output->dataType(),
+               0, "knn_mindistance: all inputs must have the same data type");
+
+  helpers::knn_mindistance(*input, *lowest, *highest, *output);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(knn_mindistance) {
+  auto input = inputShape->at(0);
+
+  // always return scalar here
+  return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(input)));
+}
+
+DECLARE_TYPES(knn_mindistance) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/betaInc.cpp

@@ -0,0 +1,74 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 12.12.2017
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_betainc)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/betaInc.h>
+
+namespace sd {
+namespace ops {
+
+DECLARE_TYPES(betainc) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+CONFIGURABLE_OP_IMPL(betainc, 3, 1, false, 0, 0) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+  auto x = INPUT_VARIABLE(2);
+
+  // just skip op if input is empty
+  if (x->isEmpty()) {
+    *x = DataTypeUtils::nanOrZero<float>();
+    return Status::OK;
+  }
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(a->isSameShape(b) && a->isSameShape(x), 0,
+               "CONFIGURABLE_OP betainc: all three input arrays must have the same shapes, bit got a=%s, b=%s and x=%s "
+               "instead !",
+               ShapeUtils::shapeAsString(a).c_str(), ShapeUtils::shapeAsString(b).c_str(),
+               ShapeUtils::shapeAsString(x).c_str());
+
+  LongType arrLen = a->lengthOf();
+
+  // FIXME: this stuff should be single op call. No sense rolling over couple of arrays twice
+  for (LongType i = 0; i < arrLen; ++i) {
+    REQUIRE_TRUE(a->e<float>(i) > 0.f, 0, "BETAINC op: arrays a array must contain only elements > 0 !");
+    REQUIRE_TRUE(b->e<float>(i) > 0.f, 0, "BETAINC op: arrays b array must contain only elements > 0 !");
+    REQUIRE_TRUE(0.f <= x->e<float>(i) && x->e<float>(i) <= 1.f, 0,
+                 "BETAINC op: all elements of x array must be within [0, 1] range!");
+  }
+
+  helpers::betaInc(block.launchContext(), *a, *b, *x, *output);
+
+  return Status::OK;
+}
+
+DECLARE_SYN(BetaInc, betainc);
+DECLARE_SYN(betaInc, betainc);
+
+}  // namespace ops
+}  // namespace sd
+
+#endif