chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/generic/transforms/batch_to_space.cpp

@@ -0,0 +1,141 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_batch_to_space)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/s_t_b.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(batch_to_space, 2, 1, false, 0, 1) {
+  // [bS*blockSize*blockSize, H/blockSize, W/blockSize, iC] is rearranged/permuted to [bS, oH, oW, iC]
+  // oH = H - cropTop  - cropBottom
+  // oW = W - cropLeft - cropRight
+
+  auto input = INPUT_VARIABLE(0);
+  auto crop = INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  const LongType blockSize = INT_ARG(0);
+  REQUIRE_TRUE(blockSize >= 2, 0, "BatchToSpace: integer parameter block_size must be >= 2, but got %i instead",
+               blockSize);
+
+  const int rank = input->rankOf();
+  const int dim0 = input->sizeAt(0);
+  REQUIRE_TRUE(rank == 4, 0, "BatchToSpace: rank of input array must be equal 4, but got %i instead", rank);
+  REQUIRE_TRUE(dim0 % (blockSize * blockSize) == 0, 0,
+               "BatchToSpace: first dimension of input array must be divisible by blockSize * blockSize (that is by "
+               "%i), but got first dimension equal to %i",
+               blockSize * blockSize, dim0);
+
+  if (crop->sizeAt(0) != 2 || crop->sizeAt(1) != 2)
+  REQUIRE_TRUE(false, 0, "BatchToSpace: operation expects crop shape to be {2, 2}, but got %s instead",
+               ShapeUtils::shapeAsString(crop).c_str());
+
+  const LongType cropBottom = crop->e<LongType>(0, 0);
+  const LongType cropTop = crop->e<LongType>(0, 1);
+  const LongType cropLeft = crop->e<LongType>(1, 0);
+  const LongType cropRight = crop->e<LongType>(1, 1);
+
+  const int oH = input->sizeAt(1) * blockSize - cropBottom - cropTop;  // top and bottom
+  const int oW = input->sizeAt(2) * blockSize - cropLeft - cropRight;  // left and right
+  REQUIRE_TRUE(oH >= 0, 0,
+               "BatchToSpace: crop top/bottom values are too big and cause negative output height dimension !");
+  REQUIRE_TRUE(oW >= 0, 0,
+               "BatchToSpace: crop left/right values are too big and cause negative output width dimension !");
+
+  if (shape::strideDescendingCAscendingF(input->shapeInfo()))
+    helpers::batchToSpace(block.launchContext(), *input, *output, cropBottom, cropTop, cropLeft, cropRight, blockSize);
+  else {
+    auto dupped = input->dup(input->ordering());
+    helpers::batchToSpace(block.launchContext(), *dupped, *output, cropBottom, cropTop, cropLeft, cropRight, blockSize);
+    delete dupped;
+  }
+  return Status::OK;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(batch_to_space) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedInputTypes(1, {ALL_INTS})->setSameMode(true);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(batch_to_space) {
+  auto inputShapeInfo = inputShape->at(0);
+  auto cropShapeInfo = inputShape->at(1);
+
+  const LongType blockSize = INT_ARG(0);
+  REQUIRE_TRUE(blockSize >= 2, 0, "BatchToSpace: integer parameter block_size must be >= 2, but got %i instead",
+               blockSize);
+
+  const int rank = inputShapeInfo[0];
+  const int dim0 = inputShapeInfo[1];
+  REQUIRE_TRUE(rank == 4, 0, "BatchToSpace: rank of input array must be equal 4, but got %i instead", rank);
+  REQUIRE_TRUE(dim0 % (blockSize * blockSize) == 0, 0,
+               "BatchToSpace: first dimension of input array must be divisible by blockSize * blockSize (that is by "
+               "%i), but got first dimension equal to %i",
+               blockSize * blockSize, dim0);
+
+  if (cropShapeInfo[1] != 2 || cropShapeInfo[2] != 2)
+  REQUIRE_TRUE(false, 0, "BatchToSpace: operation expects crop shape to be {2, 2}, but got %s instead",
+               ShapeUtils::shapeAsString(cropShapeInfo).c_str());
+
+  const LongType cropBottom = INPUT_VARIABLE(1)->e<LongType>(0, 0);
+  const LongType cropTop = INPUT_VARIABLE(1)->e<LongType>(0, 1);
+  const LongType cropLeft = INPUT_VARIABLE(1)->e<LongType>(1, 0);
+  const LongType cropRight = INPUT_VARIABLE(1)->e<LongType>(1, 1);
+
+  const int oH = inputShapeInfo[2] * blockSize - cropTop - cropBottom;  // top and bottom
+  const int oW = inputShapeInfo[3] * blockSize - cropLeft - cropRight;  // left and right
+  REQUIRE_TRUE(oH >= 0, 0,
+               "BatchToSpace: crop top/bottom values are too big and cause negative output height dimension !");
+  REQUIRE_TRUE(oW >= 0, 0,
+               "BatchToSpace: crop left/right values are too big and cause negative output width dimension !");
+
+  // we always give out C order here
+  return SHAPELIST(ConstantShapeHelper::getInstance().createShapeInfo(
+      ArrayOptions::dataType(inputShapeInfo), 'c', {dim0 / (blockSize * blockSize), oH, oW, inputShapeInfo[4]}));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/batch_to_space_nd.cpp

@@ -0,0 +1,148 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_batch_to_space_nd)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/s_t_b.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(batch_to_space_nd, 3, 1, false, 0, 0) {
+  // 4D example, numOfSpatialDims = 2 - two spatial dimensions
+  // [bS*blockShape[0]*blockShape[1], iH, iW, iC] is rearranged/permuted to [bS, iH*blockShape[0] - cropTop  -
+  // cropBottom, iW*blockShape[1] - cropLeft - cropRight, iC]
+
+  auto input = INPUT_VARIABLE(0);
+  auto blockShape = INPUT_VARIABLE(1);
+  auto crop = INPUT_VARIABLE(2);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(blockShape->rankOf() == 1, 0,
+               "BatchToSpaceND: rank of blockShape array must be equal to one, but got %i instead !",
+               blockShape->rankOf());
+
+  const sd::LongType numOfSpatialDims = blockShape->sizeAt(0);
+
+  auto prod = blockShape->reduceNumber(sd::reduce::Prod);
+  const auto product = prod->e<sd::LongType>(0);
+  delete prod;
+  REQUIRE_TRUE(input->sizeAt(0) % product == 0, 0,
+               "BatchToSpaceND: first dimension of input array must be divisible by product of blockShape array "
+               "elements (= %lld), but got first dimension equal to %i",
+               product, input->sizeAt(0));
+
+  if (crop->sizeAt(0) != numOfSpatialDims || crop->sizeAt(1) != 2) {
+    const std::string expectedCropShape = "[" + std::to_string(numOfSpatialDims) + ", 2]";  // [numOfSpatialDims, 2]
+    REQUIRE_TRUE(false, 0, "BatchToSpaceND: operation expects padding shape to be %s, but got %s instead",
+                 expectedCropShape.c_str(), ShapeUtils::shapeAsString(crop).c_str());
+  }
+
+  // FIXME - should we use this time-consuming validation ?
+  for (sd::LongType i = 0; i < numOfSpatialDims; ++i) {
+    const auto cropLeft = crop->e<sd::LongType>(i, 0);
+    const auto cropRight = crop->e<sd::LongType>(i, 1);
+    const auto outSpatialDim = input->sizeAt(i + 1) * blockShape->e<sd::LongType>(i) - cropLeft - cropRight;
+    REQUIRE_TRUE(
+        outSpatialDim >= 0, 0,
+        "BatchToSpaceND: crop left/right values are too big and cause negative output spatial dimension/dimensions !");
+  }
+
+  if (shape::strideDescendingCAscendingF(input->shapeInfo()))
+    helpers::batchToSpaceND(block.launchContext(), *input, *blockShape, *crop, *output);
+  else {
+    auto dupped = input->dup();
+    helpers::batchToSpaceND(block.launchContext(), *dupped, *blockShape, *crop, *output);
+    delete dupped;
+  }
+  return sd::Status::OK;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(batch_to_space_nd) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, sd::DataType::ANY)
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS})
+      ->setSameMode(true);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(batch_to_space_nd) {
+  auto inputShapeInfo = inputShape->at(0);
+  auto blockShapeInfo = inputShape->at(1);
+  auto cropShapeInfo = inputShape->at(2);
+
+  REQUIRE_TRUE(blockShapeInfo[0] == 1, 0,
+               "BatchToSpaceND: rank of blockShape array must be equal to one, but got %i instead !",
+               blockShapeInfo[0]);
+
+  auto prod = INPUT_VARIABLE(1)->reduceNumber(sd::reduce::Prod);
+  const auto product = prod->e<sd::LongType>(0);
+  delete prod;
+  REQUIRE_TRUE(inputShapeInfo[1] % product == 0, 0,
+               "BatchToSpaceND: first dimension of input array must be divisible by product of blockShape array "
+               "elements (= %lld), but got first dimension equal to %i",
+               product, inputShapeInfo[1]);
+
+  const auto numOfSpatialDims = blockShapeInfo[1];
+
+  if (cropShapeInfo[1] != numOfSpatialDims || cropShapeInfo[2] != 2) {
+    const std::string expectedCropShape = "[" + std::to_string(numOfSpatialDims) + ", 2]";  // [numOfSpatialDims, 2]
+    REQUIRE_TRUE(false, 0, "BatchToSpaceND: operation expects padding shape to be %s, but got %s instead",
+                 expectedCropShape.c_str(), ShapeUtils::shapeAsString(cropShapeInfo).c_str());
+  }
+
+  std::vector<sd::LongType> outShape(inputShapeInfo + 1, inputShapeInfo + 1 + inputShapeInfo[0]);
+
+  outShape[0] /= product;
+
+  for (sd::LongType i = 0; i < numOfSpatialDims; ++i)
+    outShape[i + 1] = outShape[i + 1] * INPUT_VARIABLE(1)->e<sd::LongType>(i) -
+                      INPUT_VARIABLE(2)->e<sd::LongType>(i, 0) - INPUT_VARIABLE(2)->e<sd::LongType>(i, 1);
+
+  return SHAPELIST(
+      ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inputShapeInfo), 'c', outShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/clip_by_averaged_norm.cpp

@@ -0,0 +1,92 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_clipbyavgnorm)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(clipbyavgnorm, -1, 1, true, -2, 0) {
+  if (block.inputs()->size() > 1) {
+    auto input = INPUT_VARIABLE(0);
+    auto clipNorm = INPUT_VARIABLE(1);
+    auto output = OUTPUT_VARIABLE(0);
+    const bool isInplace = block.isInplace();
+    helpers::clipByNorm(block.launchContext(), input, output, *block.getIArguments(), clipNorm, isInplace, true);
+  } else {
+    auto input = INPUT_VARIABLE(0);
+    auto output = OUTPUT_VARIABLE(0);
+
+    const bool isInplace = block.isInplace();
+    auto clipNorm = NDArrayFactory::create(T_ARG(0), block.launchContext());
+
+    helpers::clipByNorm(block.launchContext(), input, output, *block.getIArguments(), clipNorm, isInplace, true);
+    delete clipNorm;
+  }
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(clipbyavgnorm) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(clipbyavgnorm_bp, -2, 1, false, -1, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+
+  auto gradI = OUTPUT_VARIABLE(0);
+  if (block.inputs()->size() > 2) {
+    const auto clipNorm = INPUT_VARIABLE(2);
+    helpers::clipByNormBp(block.launchContext(), input, gradO, gradI, *block.getIArguments(), clipNorm, true);
+  } else {
+    auto clipNorm = NDArrayFactory::create(gradI->dataType(), T_ARG(0), block.launchContext());
+    helpers::clipByNormBp(block.launchContext(), input, gradO, gradI, *block.getIArguments(), clipNorm, true);
+    delete clipNorm;
+  }
+
+  return sd::Status::OK;
+}
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(clipbyavgnorm_bp) {
+  return SHAPELIST(CONSTANT(inputShape->at(1)));
+}
+
+DECLARE_TYPES(clipbyavgnorm_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, DataType::ANY)
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/clip_by_global_norm.cpp

@@ -0,0 +1,69 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_clip_by_global_norm)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(clip_by_global_norm, 1, 2, true, 1, 0) {
+  std::vector<NDArray*> inputs(block.width());
+  std::vector<NDArray*> outputs(block.width() + 1);
+  for (size_t i = 0; i < inputs.size(); ++i) {
+    inputs[i] = INPUT_VARIABLE(i);
+    outputs[i] = OUTPUT_VARIABLE(i);
+  }
+  outputs[inputs.size()] = OUTPUT_VARIABLE(inputs.size());
+  double clipNorm = T_ARG(0);
+  bool isInplace = block.isInplace();
+  helpers::clipByGlobalNorm(block.launchContext(), inputs, clipNorm, block.workspace(), outputs, isInplace);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(clip_by_global_norm) {
+  auto shapeList = SHAPELIST();
+
+  for (size_t e = 0; e < block.width(); e++) {
+    auto in = inputShape->at(e);
+
+    sd::LongType* newShape;
+    COPY_SHAPE(in, newShape);
+    shapeList->push_back(CONSTANT(newShape));
+  }
+
+  shapeList->push_back(ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inputShape->at(0))));
+  return shapeList;
+}
+
+DECLARE_TYPES(clip_by_global_norm) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/clip_by_norm.cpp

@@ -0,0 +1,88 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_clipbynorm)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(clipbynorm, 1, 1, true, 1, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  if (block.numT() > 0) {
+    auto clipNorm = NDArrayFactory::create(output->dataType(), T_ARG(0), block.launchContext());
+    const bool isInplace = block.isInplace();
+    helpers::clipByNorm(block.launchContext(), input, output, *block.getIArguments(), clipNorm, isInplace, false);
+    delete clipNorm;
+  } else {
+    auto clipNorm = INPUT_VARIABLE(1);
+    const bool isInplace = block.isInplace();
+    helpers::clipByNorm(block.launchContext(), input, output, *block.getIArguments(), clipNorm, isInplace, false);
+  }
+
+  return Status::OK;
+}
+
+CUSTOM_OP_IMPL(clipbynorm_bp, 2, 1, false, 1, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+
+  auto gradI = OUTPUT_VARIABLE(0);
+  if (block.numT() > 0) {
+    auto clipNorm = NDArrayFactory::create(gradI->dataType(), T_ARG(0), block.launchContext());
+    helpers::clipByNormBp(block.launchContext(), input, gradO, gradI, *block.getIArguments(), clipNorm, false);
+    delete clipNorm;
+  } else {
+    const auto clipNorm = INPUT_VARIABLE(1);
+    helpers::clipByNormBp(block.launchContext(), input, gradO, gradI, *block.getIArguments(), clipNorm, false);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(clipbynorm_bp) {
+  auto inShapeInfo = inputShape->at(1);
+
+  LongType *newShape = nullptr;
+  COPY_SHAPE(inShapeInfo, newShape);
+
+  return SHAPELIST(CONSTANT(newShape));
+}
+
+DECLARE_TYPES(clipbynorm) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+DECLARE_TYPES(clipbynorm_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/clip_by_value.cpp

@@ -0,0 +1,163 @@
+/* ******************************************************************************
+*
+*
+* This program and the accompanying materials are made available under the
+* terms of the Apache License, Version 2.0 which is available at
+* https://www.apache.org/licenses/LICENSE-2.0.
+*
+*  See the NOTICE file distributed with this work for additional
+*  information regarding copyright ownership.
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations
+* under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//  @author Adam Gibson
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_clipbyvalue)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(clipbyvalue, -2, 1, true, -2, 0) {
+ auto input = INPUT_VARIABLE(0);
+ auto output = OUTPUT_VARIABLE(0);
+
+ if (block.getTArguments()->size() > 0) {
+   auto left = T_ARG(0);
+   auto right = T_ARG(1);
+
+   REQUIRE_TRUE(left < right, 0, "clip_by_value: left bound should be lesser than right. But %f >= %f given.", left,
+                right);
+   helpers::clipByValue(block.launchContext(), input, left, right, output);
+
+ } else {
+   auto left = INPUT_VARIABLE(1);
+   auto right = INPUT_VARIABLE(2);
+
+   switch (input->dataType()) {
+#if defined(HAS_DOUBLE)
+     case DOUBLE: {
+       auto leftValueDouble = left->e<double>(0);
+       auto rightValueDouble = right->e<double>(0);
+       helpers::clipByValue(block.launchContext(), input, leftValueDouble, rightValueDouble, output);
+       break;
+     }
+#endif
+#if defined(HAS_FLOAT32)
+     case FLOAT32: {
+       auto leftValueFloat = left->e<float>(0);
+       auto rightValueFloat = right->e<float>(0);
+       helpers::clipByValue(block.launchContext(), input, leftValueFloat, rightValueFloat, output);
+       break;
+     }
+#endif
+#if defined(HAS_FLOAT16)
+     case HALF: {
+       auto leftValueFloat16 = left->e<float16>(0);
+       auto rightValueFloat16 = right->e<float16>(0);
+       helpers::clipByValue(block.launchContext(), input, leftValueFloat16, rightValueFloat16, output);
+       break;
+     }
+#endif
+#if defined(HAS_BFLOAT16)
+     case BFLOAT16: {
+       auto leftValueBFloat16 = left->e<bfloat16>(0);
+       auto rightValueBFloat16 = right->e<bfloat16>(0);
+       helpers::clipByValue(block.launchContext(), input, leftValueBFloat16, rightValueBFloat16, output);
+       break;
+     }
+#endif
+     // Non-floating point types that might be present but not used in clip_by_value
+     case INHERIT:
+       break;
+#if defined(HAS_BOOL)
+     case BOOL:
+       break;
+#endif
+     case FLOAT8:
+       break;
+     case HALF2:
+       break;
+#if defined(HAS_INT8)
+     case INT8:
+       break;
+#endif
+#if defined(HAS_INT16)
+     case INT16:
+       break;
+#endif
+#if defined(HAS_INT32)
+     case INT32:
+       break;
+#endif
+#if defined(HAS_LONG)
+     case INT64:
+       break;
+#endif
+#if defined(HAS_UINT8)
+     case UINT8:
+       break;
+#endif
+#if defined(HAS_UINT16)
+     case UINT16:
+       break;
+#endif
+#if defined(HAS_UINT32)
+     case UINT32:
+       break;
+#endif
+#if defined(HAS_UNSIGNEDLONG)
+     case UINT64:
+       break;
+#endif
+     case QINT8:
+       break;
+     case QINT16:
+       break;
+#if defined(HAS_UTF8)
+     case UTF8:
+       break;
+#endif
+#if defined(HAS_UTF16)
+     case UTF16:
+       break;
+#endif
+#if defined(HAS_UTF32)
+     case UTF32:
+       break;
+#endif
+     case ANY:
+       break;
+     case AUTO:
+       break;
+     case UNKNOWN:
+       break;
+     default:
+       // Handle any other types that might not be covered
+       break;
+   }
+ }
+
+ return Status::OK;
+}
+
+DECLARE_SYN(ClipByValue, clipbyvalue);
+
+DECLARE_TYPES(clipbyvalue) {
+ getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/concat.cpp

@@ -0,0 +1,366 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//  @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+#include <array>
+#if NOT_EXCLUDED(OP_concat)
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(concat, -1, 1, false, 0, 0) {
+  REQUIRE_TRUE(block.width() > 0, 0, "CONCAT op: No input arrays were provided");
+
+  const bool isAxisInLastArr = block.getBArguments()->size() == 0 ? false : B_ARG(0);
+
+  const int numOfInArrs = isAxisInLastArr ? block.width() - 1 : block.width();
+
+  // first of all take into account possible presence of empty arrays
+  // also if scalar is present -> copy its value to vector with length=1
+  std::vector<NDArray*> nonEmptyArrs;
+  std::vector<NDArray*> arrsToDelete;  // Track allocated arrays for cleanup
+  LongType index = 0;
+  bool allOfSameType = true;
+  auto rankOfFirstArr = block.width() > 0 ? INPUT_VARIABLE(0)->rankOf() : 0;
+  auto typeOfFirstArr = block.width() > 0 ? INPUT_VARIABLE(0)->dataType() : block.dataType();
+
+  for (LongType i = 0; i < numOfInArrs; ++i) {
+    auto input = INPUT_VARIABLE(i);
+    if (!input->isEmpty()) {
+      allOfSameType &= (typeOfFirstArr == input->dataType());
+
+      if (input->rankOf() == 0) {
+        std::vector<sd::LongType> shape = {1};
+        NDArray* vec = nullptr;
+#ifdef __cpp_exceptions
+        try {
+          vec = new NDArray('c', shape, input->dataType(), block.launchContext());
+          vec->assign(input);
+          nonEmptyArrs.push_back(vec);
+          arrsToDelete.push_back(vec);  // Mark for cleanup
+        } catch (...) {
+          // If allocation fails, clean up what we've created so far
+          if (vec) delete vec;
+          for (auto arr : arrsToDelete) {
+            delete arr;
+          }
+          throw;
+        }
+#else
+        vec = new NDArray('c', shape, input->dataType(), block.launchContext());
+        vec->assign(input);
+        nonEmptyArrs.push_back(vec);
+        arrsToDelete.push_back(vec);  // Mark for cleanup
+#endif
+      } else {
+        nonEmptyArrs.push_back(input);
+      }
+      ++index;
+    }
+  }
+
+  const LongType numOfNonEmptyArrs = nonEmptyArrs.size();
+
+  if (numOfNonEmptyArrs == 0) {
+    // Clean up allocated temporary arrays before returning
+    for (auto arr : arrsToDelete) {
+      if(arr != nullptr) {
+        delete arr;
+
+      }
+    }
+    // All inputs are empty arrays -> return empty, mainly for TF import compatibility (no op)
+    REQUIRE_TRUE(OUTPUT_VARIABLE(0)->isEmpty(), 0, "CONCAT op: If all input variables are empty, output must be empty");
+    return Status::OK;
+  }
+
+  const LongType rank = nonEmptyArrs[0]->rankOf();  //  look up to first non-empty array
+  LongType axis = isAxisInLastArr ? INPUT_VARIABLE(block.width() - 1)->e<LongType>(0) : INT_ARG(0);
+  if (axis < 0) {
+    axis += rank;
+  }
+
+  // ******** input validation ******** //
+  if (!allOfSameType) {
+    for (auto arr : arrsToDelete) delete arr;
+    REQUIRE_TRUE(false, 0, "CONCAT op: all of input arrays must have same type !");
+  }
+  
+  if (nonEmptyArrs[0]->dataType() != OUTPUT_VARIABLE(0)->dataType()) {
+    for (auto arr : arrsToDelete) delete arr;
+    REQUIRE_TRUE(false, 0, "CONCAT op: output array should have the same type as inputs arrays !");
+  }
+  
+  if (!(0 <= axis && (axis < rank || (axis == 0 && rank == 0)))) {
+    for (auto arr : arrsToDelete) delete arr;
+    REQUIRE_TRUE(false, 0, "CONCAT op: input axis must be in range [0, %i], but got %i instead!", rank - 1, axis);
+  }
+
+  for (LongType i = 1; i < numOfNonEmptyArrs; ++i) {
+    if (nonEmptyArrs[i]->rankOf() != rank) {
+      std::string error;
+      error += "CONCAT op: array at index ";
+      error += std::to_string(i);
+      error += " did not have same rank. Expected rank: ";
+      error += std::to_string(rank);
+      error += " but was: ";
+      error += std::to_string(nonEmptyArrs[i]->rankOf());
+      
+      // Cleanup before throwing
+      for (auto arr : arrsToDelete) delete arr;
+      REQUIRE_TRUE(false, 0, error.c_str());
+    }
+
+    for (LongType dim = 0; dim < rank; ++dim) {
+      if (dim != axis) {
+        if (nonEmptyArrs[i]->sizeAt(dim) != nonEmptyArrs[0]->sizeAt(dim)) {
+          std::string error;
+          error += "CONCAT op: array at index ";
+          error += std::to_string(i);
+          error += " did not have same dimension at position ";
+          error += std::to_string(dim);
+          error += ". Expected dimension: ";
+          error += std::to_string(nonEmptyArrs[0]->sizeAt(dim));
+          error += " but was: ";
+          error += std::to_string(nonEmptyArrs[i]->sizeAt(dim));
+          
+          // Cleanup before throwing
+          for (auto arr : arrsToDelete) delete arr;
+          REQUIRE_TRUE(false, 0, error.c_str());
+        }
+      }
+    }
+  }
+
+  // ******** end of input validation ******** //
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  helpers::concat(block.launchContext(), nonEmptyArrs, *output, axis);
+
+  // Clean up allocated temporary arrays
+  for (auto arr : arrsToDelete) {
+    delete arr;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SYN(ParallelConcat, concat);
+DECLARE_SYN(concat_v2, concat);
+DECLARE_SYN(concatv2, concat);
+
+DECLARE_TYPES(concat) {
+  getOpDescriptor()->setAllowedInputTypes(ANY);
+}
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(concat) {
+  REQUIRE_TRUE(block.width() > 0, 0, "CONCAT op: No input arrays were provided");
+
+  const bool isAxisInLastArr = block.getBArguments()->size() == 0 ? false : B_ARG(0);
+
+  //used for copying shape later if we have a mix of empty and non empty
+  //all arrays but empty should fit same pattern
+  int firstNonEmptyShapeIdx = -1;
+  const LongType numOfInArrs = isAxisInLastArr ? block.width() - 1 : block.width();
+  // first of all take into account possible presence of empty arrays
+  // also if scalar is present -> use the shape of vector with length=1 instead
+  ShapeList arrShapes;
+  std::vector<LongType> shapesToDelete;
+  LongType numOfNonEmptyArrs = 0;
+  const LongType rank = shape::rank(INPUT_VARIABLE(0)->shapeInfo());
+  LongType newDim = 0;
+  LongType axis = isAxisInLastArr ? INPUT_VARIABLE(block.width() - 1)->e<LongType>(0) : INT_ARG(0);
+  if (axis < 0) {
+    axis += rank;
+  }
+
+  for (LongType i = 0; i < numOfInArrs; i++) {
+    if (shape::rank(inputShape->at(i)) <= 1) {
+      if (shape::isEmptyConst(inputShape->at(i))) {
+        int isScalar = shape::isScalar(inputShape->at(i));
+        int len = isScalar ? 1 : shape::length(inputShape->at(i));
+        newDim += len;
+        arrShapes.push_back(inputShape->at(i));
+      } else {
+        int isScalar = shape::isScalar(inputShape->at(i));
+        int len = isScalar ? 1 : shape::length(inputShape->at(i));
+        newDim += len;
+        arrShapes.push_back(ConstantShapeHelper::getInstance().vectorShapeInfo(len, INPUT_VARIABLE(0)->dataType()));
+
+        if (firstNonEmptyShapeIdx < 0)
+          firstNonEmptyShapeIdx = i;
+        numOfNonEmptyArrs++;
+      }
+    } else {
+      if (!shape::isEmptyConst(inputShape->at(i))) {
+        numOfNonEmptyArrs++;
+        if (firstNonEmptyShapeIdx < 0)
+          firstNonEmptyShapeIdx = i;
+        auto currShape = shape::shapeOf(inputShape->at(i));
+        newDim += currShape[axis];
+      } else {
+        //empty arrays can still have a shape and should be accounted for
+        auto currShape = shape::shapeOf(inputShape->at(i));
+        newDim += currShape[axis];
+      }
+
+      arrShapes.push_back(inputShape->at(i));
+    }
+  }
+
+  if (numOfNonEmptyArrs < 1) {
+    //this case is all empty arrays
+    //in this case we need to set the shape to be
+    //whatever the number of empty arrays is
+    //plus the shape of whatever the rest of the array is
+    //for example if empty shape is 1,2,1,0 and we have 3
+    // arrays a concat at axis 0 would be 3,2,1,0
+    LongType* outShapeInfo(nullptr);
+    COPY_SHAPE(arrShapes.at(0), outShapeInfo);
+    auto currShape = shape::shapeOf(outShapeInfo);
+    currShape[axis] = newDim;
+    std::vector<LongType> shapeVec;
+    for (int i = 0; i < rank; i++) {
+      shapeVec.push_back(currShape[i]);
+    }
+
+    // All inputs are empty arrays -> return empty, mainly for TF import compatibility (no op)
+    auto newShape = ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(INPUT_VARIABLE(0)->dataType(), shapeVec);
+    delete[] outShapeInfo;
+
+    // Clean up allocated vectors
+    for (auto idx : shapesToDelete) {
+      delete[] const_cast<LongType*>(arrShapes.at(idx));
+    }
+
+    return SHAPELIST(newShape);
+  }
+
+  // ******** input validation ******** //
+  //axis needs to be flexible between 0 and 1
+  if (axis > 1)
+    REQUIRE_TRUE(0 <= axis && axis < rank, 0, "CONCAT op: input axis must be in range [0, %i], but got %i instead!",
+                 rank - 1, axis);
+
+  // ******** end of input validation ******** //
+
+  if (shape::isScalar(arrShapes.at(firstNonEmptyShapeIdx))) {
+    //concat of scalar should be  a 1d vector
+    auto newShape = ConstantShapeHelper::getInstance().vectorShapeInfo(newDim, INPUT_VARIABLE(0)->dataType());
+    return SHAPELIST(CONSTANT(newShape));
+  } else {
+    LongType* outShapeInfo(nullptr);
+    COPY_SHAPE(arrShapes.at(firstNonEmptyShapeIdx), outShapeInfo);
+    //reset flags: if an array is empty we can have unintended side effects from the flags
+    //in our case by this point we handled empty and should only need the data type.
+    ArrayOptions::resetFlags(outShapeInfo);
+    // case when we have only one input array
+    if (numOfNonEmptyArrs == 1) {
+      ShapeUtils::updateStridesAndType(outShapeInfo, arrShapes.at(firstNonEmptyShapeIdx), shape::order(arrShapes.at(firstNonEmptyShapeIdx)));
+      auto result = CONSTANT(outShapeInfo);
+      delete[] outShapeInfo;
+      return SHAPELIST(result);
+    }
+
+    auto currShape = shape::shapeOf(outShapeInfo);
+    currShape[axis] = newDim;
+    ShapeUtils::updateStridesAndType(outShapeInfo, arrShapes.at(firstNonEmptyShapeIdx), shape::order(arrShapes.at(firstNonEmptyShapeIdx)));
+
+    //note: always ensure that the constant shape helper is used, otherwise we could end up with
+    //some modification of pre existing cache values.
+    auto result = ConstantShapeHelper::getInstance().createFromExisting(outShapeInfo);
+    delete[] outShapeInfo;
+    return SHAPELIST(result);
+  }
+}
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(concat_bp, -1, -1, false, 0, 0) {
+  const bool isAxisInLastArr = block.getBArguments()->size() == 0 ? false : B_ARG(0);
+
+  const LongType numOfInArrs = isAxisInLastArr ? block.width() - 1 : block.width();
+
+  auto epsilonNext = INPUT_VARIABLE(numOfInArrs - 1);
+
+  auto first = INPUT_VARIABLE(0);
+
+  const LongType axis = isAxisInLastArr ? INPUT_VARIABLE(block.width() - 1)->e<int>(0)
+                                        : (INT_ARG(0) >= 0 ? INT_ARG(0) : INT_ARG(0) + INPUT_VARIABLE(0)->rankOf());
+
+  LongType startPos = 0;
+
+  for (LongType e = 0; e < numOfInArrs - 1; e++) {
+    auto originalChunk = INPUT_VARIABLE(e);
+    auto epsilonChunk = OUTPUT_VARIABLE(e);
+    std::vector<LongType> indices(2 * epsilonNext->rankOf());
+
+    int width = originalChunk->sizeAt(axis);
+
+    for (LongType e2 = 0; e2 < epsilonNext->rankOf(); e2++) {
+      if (e2 == axis)
+        indices[2 * e2 + 1] = (indices[2 * e2] = startPos) + width;
+      else
+        indices[2 * e2 + 1] = indices[2 * e2] = 0;
+    }
+
+    auto subarray = (*epsilonNext)(indices, true);
+    epsilonChunk->assign(subarray);
+
+    delete subarray;
+    startPos += width;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(concat_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(concat_bp) {
+  const bool isAxisInLastArr = block.getBArguments()->size() == 0 ? false : B_ARG(0);
+
+  const LongType numOfInArrs = isAxisInLastArr ? block.width() - 1 : block.width();
+
+  auto shapeList = SHAPELIST();
+
+  for (int e = 0; e < numOfInArrs - 1; e++) {
+    auto inShape = inputShape->at(e);
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(ArrayOptions::dataType(inShape),
+                                                                               shape::order(inShape),
+                                                                               shape::rank(inShape),
+                                                                               shape::shapeOf(inShape))->primary());
+  }
+
+  return shapeList;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/cumprod.cpp

@@ -0,0 +1,149 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_cumprod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/prefix.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(cumprod, 1, 1, true, 0, 2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->dataType() == output->dataType(), 0, "CumSum: input and output data types must be equal");
+
+  if (input->isEmpty()) {
+    // No-op
+    return sd::Status::OK;
+  }
+
+  const bool exclusive = INT_ARG(0) == 1;
+  const bool reverse = INT_ARG(1) == 1;
+
+  if (block.getIArguments()->size() == 2 && block.width() == 1) {
+    // all at once case
+    sd::ops::helpers::prefix(block.launchContext(), scalar::Multiply, input, output, exclusive, reverse);
+  } else {
+    std::vector<sd::LongType> dims(block.numI() - 2);
+
+    if (block.width() == 1) {
+      for (size_t e = 0; e < block.numI() - 2; e++) dims[e] = INT_ARG(e + 2);
+    } else {
+      auto ax = INPUT_VARIABLE(1);
+      dims = ax->template asVectorT<sd::LongType>();
+    }
+
+    for (size_t e = 0; e < dims.size(); e++)
+      if (dims[e] < 0) dims[e] += input->rankOf();
+
+    sd::ops::helpers::prefix(block.launchContext(), scalar::Multiply, input, output, dims, exclusive, reverse);
+  }
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(cumprod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, sd::DataType::ANY)
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_FLOATS})
+      ->setSameMode(true);
+}
+
+DECLARE_TYPES(cumprod_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, sd::DataType::ANY)
+      ->setAllowedInputTypes(1, {ALL_INTS, ALL_FLOATS})  // there is a case when axes given as IArgs
+      ->setAllowedInputTypes(2, {ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_FLOATS})
+      ->setSameMode(true);
+}
+
+CUSTOM_OP_IMPL(cumprod_bp, 2, 1, false, 0, 2) {
+  auto input = INPUT_VARIABLE(0);
+  auto axis = block.width() == 3 ? INPUT_VARIABLE(1) : nullptr;
+  auto gradOut = block.width() == 3 ? INPUT_VARIABLE(2) : INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const bool exclusive = INT_ARG(0) == 1;
+  const bool reverse = INT_ARG(1) == 1;
+
+  std::vector<sd::LongType> dims;
+
+  if (block.width() > 2) {
+    dims = axis->template asVectorT<sd::LongType>();
+    float one = 1.f;
+    OUTPUT_VARIABLE(1)->assign(one);
+  } else if (int newSize = (block.numI() - 2)) {
+    dims.resize(newSize);
+
+    for (int e = 0; e < newSize; e++) dims[e] = INT_ARG(e + 2);
+  }
+
+  sd::ops::helpers::prefix(block.launchContext(), scalar::Multiply, input, output, dims, exclusive, reverse);
+  NDArray *val = output->dup();
+
+  gradOut->applyPairwiseTransform(pairwise::Multiply, output, val);
+  val->applyPairwiseTransform(pairwise::Divide, input, val);
+  if (!exclusive && !reverse) {
+    if (dims.size())
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, val, output, dims, true, false);
+    else
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, val, output, false, true);
+
+  } else if (!exclusive && reverse) {
+    if (dims.size())
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, val, output, dims, false, false);
+    else
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, val, output, false, false);
+  } else if (exclusive && !reverse) {
+    if (dims.size())
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, val, output, dims, true, true);
+    else
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, val, output, true, true);
+  } else {
+    if (dims.size())
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, val, output, dims, true, false);
+    else
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, val, output, true, false);
+  }
+
+  delete val;
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(cumprod_bp) {
+  auto inp = inputShape->at(0);
+  if (block.width() == 2) {
+    return SHAPELIST(CONSTANT(inp));
+  } else {
+    return SHAPELIST(CONSTANT(inp), CONSTANT(inputShape->at(1)));
+  }
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/cumsum.cpp

@@ -0,0 +1,149 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_cumsum)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/prefix.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(cumsum, 1, 1, true, 0, 2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const bool exclusive = INT_ARG(0) == 1;
+  const bool reverse = INT_ARG(1) == 1;
+
+  REQUIRE_TRUE(input->dataType() == output->dataType(), 0, "CumSum: input and output data types must be equal");
+
+  if (input->isEmpty()) {
+    // No-op
+    return sd::Status::OK;
+  }
+
+  if (block.getIArguments()->size() == 2 && block.width() == 1) {
+    // all at once case
+    sd::ops::helpers::prefix(block.launchContext(), scalar::Add, input, output, exclusive, reverse);
+  } else {
+    std::vector<sd::LongType> dims(block.numI() - 2);
+
+    if (block.width() == 1) {
+      for (size_t e = 0; e < block.numI() - 2; e++) dims[e] = INT_ARG(e + 2);
+    } else {
+      auto ax = INPUT_VARIABLE(1);
+      dims = ax->template asVectorT<sd::LongType>();
+    }
+
+    for (size_t e = 0; e < dims.size(); e++)
+      if (dims[e] < 0) dims[e] += input->rankOf();
+
+    sd::ops::helpers::prefix(block.launchContext(), scalar::Add, input, output, dims, exclusive, reverse);
+  }
+
+  return sd::Status::OK;
+}
+DECLARE_TYPES(cumsum) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS,ALL_INTS})
+      ->setAllowedOutputTypes({ALL_FLOATS})
+      ->setSameMode(false);
+}
+
+CUSTOM_OP_IMPL(cumsum_bp, 2, -1, true, 0, 2) {
+  auto input = INPUT_VARIABLE(0);
+  auto axis = block.width() == 3 ? INPUT_VARIABLE(1) : nullptr;
+  auto gradOut = block.width() == 3 ? INPUT_VARIABLE(2) : INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+  const bool exclusive = INT_ARG(0) == 1;
+  const bool reverse = INT_ARG(1) == 1;
+
+  std::vector<sd::LongType> dims;
+
+  if (block.width() > 2) {
+    dims = axis->template asVectorT<sd::LongType>();
+    float one = 1.f;
+    OUTPUT_VARIABLE(1)->assign(one);
+  } else if (int newSize = (block.numI() - 2)) {
+    dims.resize(newSize);
+
+    for (int e = 0; e < newSize; e++) dims[e] = INT_ARG(e + 2);
+  }
+  if (!exclusive && !reverse) {
+    if (dims.size())
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, gradOut, output, dims, false, true);
+    else
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, gradOut, output, false, true);
+
+  } else if (!exclusive && reverse) {
+    if (dims.size())
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, gradOut, output, dims, false, false);
+    else
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, gradOut, output, false, false);
+  } else if (exclusive && !reverse) {
+    if (dims.size())
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, gradOut, output, dims, true, true);
+    else
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, gradOut, output, true, true);
+  } else {
+    if (dims.size())
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, gradOut, output, dims, true, false);
+    else
+      sd::ops::helpers::prefix(block.launchContext(), scalar::Add, gradOut, output, true, false);
+  }
+
+  return sd::Status::OK;
+}
+DECLARE_TYPES(cumsum_bp) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS});
+  getOpDescriptor()->setAllowedInputTypes(1, {ALL_FLOATS, ALL_INTS});  // axes can be set as the second param
+  getOpDescriptor()->setAllowedInputTypes(2, {ALL_FLOATS});
+  getOpDescriptor()->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(cumsum_bp) {
+  auto inp = inputShape->at(0);
+  sd::LongType *newShapeX = nullptr;
+  COPY_SHAPE(inp, newShapeX);
+
+  if (block.width() == 2) {
+    auto result = CONSTANT(newShapeX);
+    delete[] newShapeX;
+    return SHAPELIST(result);
+  } else {
+    sd::LongType *newShapeA = nullptr;
+    COPY_SHAPE(inputShape->at(1), newShapeA);
+
+    auto resultX = CONSTANT(newShapeX);
+    auto resultA = CONSTANT(newShapeA);
+    delete[] newShapeX;
+    delete[] newShapeA;
+    return SHAPELIST(resultX, resultA);
+  }
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/depth_to_space.cpp

@@ -0,0 +1,101 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_depth_to_space)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/d_t_s.h>
+
+#include <array>
+
+namespace sd {
+namespace ops {
+    CUSTOM_OP_IMPL(depth_to_space, 1, 1, false, 0, 2) {
+        int block_size = INT_ARG(0);
+        REQUIRE_TRUE(block_size > 0,0, "DepthToSpace: block_size should be > 0");
+        bool isNHWC = INT_ARG(1) == 1;
+
+        auto input = INPUT_VARIABLE(0);
+
+        REQUIRE_TRUE(input->rankOf() == 4, 0, "DepthToSpace: input should be 4D array, but got %f instead", input->rankOf());
+
+        int bS = input->sizeAt(0);
+        int iD = isNHWC ? input->sizeAt(3) : input->sizeAt(1);
+        int iH = isNHWC ? input->sizeAt(1) : input->sizeAt(2);
+        int iW = isNHWC ? input->sizeAt(2) : input->sizeAt(3);
+
+        REQUIRE_TRUE(iD % (block_size * block_size) == 0, 0, "DepthToSpace: input number of channels should be divisible by square(block_size)");
+
+        auto output = OUTPUT_VARIABLE(0);
+
+        if (shape::strideDescendingCAscendingF(input->shapeInfo()))
+            helpers::_depthToSpace(block.launchContext(), *input, output, block_size, isNHWC);
+        else {
+          NDArray *dup = input->dup();
+          helpers::_depthToSpace(block.launchContext(), *dup, output, block_size, isNHWC);
+          delete dup;
+        }
+        STORE_RESULT(output);     
+
+        return Status::OK;
+    }
+
+    DECLARE_TYPES(depth_to_space) {
+        getOpDescriptor()
+                ->setAllowedInputTypes(sd::DataType::ANY)
+                ->setSameMode(true);
+    }
+    
+
+    DECLARE_SHAPE_FN(depth_to_space) {
+        auto in = inputShape->at(0);
+        auto block_size = INT_ARG(0);
+        REQUIRE_TRUE(block_size > 0,0, "DepthToSpace: input should be > 0");
+
+        bool isNHWC = INT_ARG(1) == 1;
+
+        int bS = shape::sizeAt(in, static_cast<sd::LongType>(0));
+        int iD = isNHWC ? shape::sizeAt(in, static_cast<sd::LongType>(3)) : shape::sizeAt(in, static_cast<sd::LongType>(1));
+        int iH = isNHWC ? shape::sizeAt(in, static_cast<sd::LongType>(1)) : shape::sizeAt(in, static_cast<sd::LongType>(2));
+        int iW = isNHWC ? shape::sizeAt(in, static_cast<sd::LongType>(2)) : shape::sizeAt(in, static_cast<sd::LongType>(3));
+
+        int oD = iD / (block_size * block_size);
+        int oH = iH * block_size;
+        int oW = iW * block_size;
+
+        
+        std::array<sd::LongType, 4> shape;
+        if (isNHWC) 
+            shape = {{bS, oH, oW, oD }};
+        else 
+            shape = {{bS, oD, oH, oW }};
+        
+        auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(in), 'c', 4, shape.data(),0);
+        return SHAPELIST(newShape);
+    }
+
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/dynamic_parititon.cpp

@@ -0,0 +1,139 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author GS <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_dynamic_partition)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/dynamic.h>
+
+#include <array>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(dynamic_partition, 2, 1, false, 0, 1) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+
+  REQUIRE_TRUE(input->rankOf() >= indices->rankOf(), 0,
+               "dynamic_partition: data tensor rank should be non-lesser than indices\' tensor, but %i < %i given,",
+               input->rankOf(), indices->rankOf());
+  for (int dim = 0; dim < indices->rankOf(); dim++) {
+    REQUIRE_TRUE(
+        input->sizeAt(dim) == indices->sizeAt(dim), 0,
+        "dynamic_partition: dimensions should be equals for data and indices tensors, but at axis[%i] %i != %i given",
+        dim, input->sizeAt(dim), indices->sizeAt(dim));
+  }
+
+  auto numPartition = INT_ARG(0);
+  std::vector<NDArray *> outputList(numPartition);
+  for (int o = 0; o < numPartition; ++o) {
+    outputList[o] = OUTPUT_VARIABLE(o);
+  }
+  helpers::dynamicPartitionFunctor(block.launchContext(), input, indices, outputList);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(dynamic_partition) {
+  auto numPartition = INT_ARG(0);
+  auto indices = INPUT_VARIABLE(1);
+  std::vector<sd::LongType> partitionSizes(numPartition, 0);
+  auto in = inputShape->at(0);
+  auto idx = inputShape->at(1);
+  for (int i = 0; i < numPartition; i++) {
+    for (int e = 0; e < indices->lengthOf(); ++e)
+      if (indices->e<sd::LongType>(e) == i) partitionSizes[i]++;
+  }
+
+  auto shapes = SHAPELIST();
+  sd::LongType outRank = shape::rank(in) - shape::rank(idx) + 1;
+  for (sd::LongType e = 0; e < numPartition; e++) {
+    sd::LongType *newShape;
+    ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    newShape[0] = outRank;
+    newShape[1] = partitionSizes[e];
+    for (sd::LongType i = 1; i < outRank; ++i) newShape[i + 1] = shape::sizeAt(in, outRank + i - 1);
+
+    shape::updateStrides(newShape, shape::order(in), false);
+    ArrayOptions::setDataType(newShape, ArrayOptions::dataType(in));
+    shapes->push_back(CONSTANT(newShape));
+  }
+
+  return shapes;
+}
+
+DECLARE_TYPES(dynamic_partition) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS});
+}
+
+DECLARE_TYPES(dynamic_partition_bp) { getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setSameMode(true); }
+
+CUSTOM_OP_IMPL(dynamic_partition_bp, 3, 2, false, 0, 1) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto numPartition = INT_ARG(0);
+
+  std::vector<NDArray *> outputList(2);  // only for output
+  std::vector<NDArray *> gradOutList(numPartition);
+  for (sd::LongType e = 0; e < numPartition; e++) {
+    gradOutList[e] = INPUT_VARIABLE(e + 2);
+  }
+  outputList[0] = OUTPUT_VARIABLE(0);
+  outputList[1] = OUTPUT_VARIABLE(1);
+  NDArray originalIndices(*indices);
+  originalIndices.linspace(0);
+  ops::dynamic_partition op;
+  auto res = op.evaluate({&originalIndices, indices}, {numPartition});
+  REQUIRE_TRUE(res.status() == sd::Status::OK, 0, "dynamic_partition_bp: Error with dynamic partitioning.");
+  ops::dynamic_stitch stitchOp;
+  std::vector<NDArray *> partitions(numPartition * 2);
+  for (int i = 0; i < res.size(); i++) {
+    partitions[i] = res.at(i);
+    partitions[i + numPartition] = gradOutList[i];
+  }
+
+  auto result = stitchOp.evaluate(partitions, {numPartition});
+  REQUIRE_TRUE(result.status() == sd::Status::OK, 0, "dynamic_partition_bp: Error with dynamic partitioning.");
+  outputList[1]->assign(indices);
+  outputList[0]->assign(result.at(0));
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(dynamic_partition_bp) {
+  auto numPartition = INT_ARG(0);
+  auto indices = INPUT_VARIABLE(1);
+  std::vector<sd::LongType> partitionSizes(numPartition, 0);
+
+  auto shapes = SHAPELIST();
+  // just copy shape info from input and indices to output
+  for (sd::LongType i = 0; i < 2; i++) {
+    shapes->push_back(CONSTANT(inputShape->at(i)));
+  }
+
+  return shapes;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/dynamic_stitch.cpp

@@ -0,0 +1,91 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author GS <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_dynamic_stitch)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/dynamic.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(dynamic_stitch, 2, 1, false, 0, 0) {
+  int numOfData = block.width();
+  //        int k = 0;
+  // checking input data size
+  REQUIRE_TRUE(numOfData % 2 == 0, 0,
+               "dynamic_stitch: The input params should contains"
+               " both indeces and data lists with same length.");
+  // split input data list on two equal parts
+  numOfData /= 2;
+
+  // form input lists to use with helpers - both indices and float data inputs
+  auto output = OUTPUT_VARIABLE(0);
+  std::vector<NDArray*> inputs(numOfData);
+  std::vector<NDArray*> indices(numOfData);
+
+  for (int e = 0; e < numOfData; e++) {
+    auto data = INPUT_VARIABLE(numOfData + e);
+    auto index = INPUT_VARIABLE(e);
+
+    inputs[e] = data;
+    indices[e] = index;
+  }
+  // run helper
+  return helpers::dynamicStitchFunctor(block.launchContext(), inputs, indices, output);
+}
+
+DECLARE_TYPES(dynamic_stitch) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(dynamic_stitch) {
+  sd::LongType maxValue = 0;
+  auto numOfData = block.width();
+  numOfData /= 2;  // only index part it's needed to review
+  auto restShape = inputShape->at(numOfData);
+  auto firstShape = inputShape->at(0);
+  // check up inputs to avoid non-int indices and calculate max value from indices to output shape length
+  for (size_t i = 0; i < numOfData; i++) {
+    auto input = INPUT_VARIABLE(i);
+    REQUIRE_TRUE(input->isZ(), 0, "dynamic_stitch: Indices should be integer, but %d type given.",
+                 (int)input->dataType());
+    auto maxV = input->reduceNumber(reduce::Max);
+    if (maxV->e<sd::LongType>(0) > maxValue) maxValue = maxV->e<sd::LongType>(0);
+    delete maxV;
+  }
+  // calculate output rank - difference between indices shape and data shape
+  int outRank = shape::rank(restShape) - shape::rank(firstShape) + 1;  // at least 1D tensor
+  std::vector<sd::LongType> outShape(outRank);
+  // fill up output shape template: the first to max index, and rests - to vals from the first data input
+  outShape[0] = maxValue + 1;
+  for (sd::LongType i = 1; i < outRank; ++i) outShape[i] = shape::sizeAt(restShape, i);
+
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().bufferForShapeInfo(ArrayOptions::dataType(restShape),
+                                                                             shape::order(firstShape),
+                                                                             outShape)->primary());
+  return ret;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/floor.cpp

@@ -0,0 +1,46 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_Floor)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(Floor, 1, 1, true) {
+  auto first = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  first->applyTransform(transform::Floor, z);
+
+  STORE_RESULT(*z);
+
+  return sd::Status::OK;
+}
+DECLARE_SYN(floor, Floor);
+
+DECLARE_TYPES(Floor) { getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/gather.cpp

@@ -0,0 +1,175 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Shyrma Yurii (iuriish@yahoo.com), created on 16.11.2017
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_gather)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/gather.h>
+#include <ops/declarable/helpers/scatter.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(gather, 1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = block.width() > 1 ? INPUT_VARIABLE(1) : nullptr;
+  auto output = OUTPUT_VARIABLE(0);
+
+  const bool checkIndices = block.getBArguments()->empty() ? true : B_ARG(0);
+
+  // Edge case: empty indices -> empty output
+  if (indices != nullptr && indices->isEmpty()) {
+    REQUIRE_TRUE(output->isEmpty(), 0, "Gather op: If indices are empty, output must also be empty");
+    return sd::Status::OK;  // No op
+  }
+
+  const sd::LongType numOfIntArgs = block.numI();
+
+  std::vector<sd::LongType> intArgs;
+  if (block.width() > 2) {
+    intArgs = INPUT_VARIABLE(2)->template asVectorT<sd::LongType>();
+  } else {
+    if (numOfIntArgs == 0)
+      intArgs.emplace_back(0);
+    else
+      for (sd::LongType i = 0; i < numOfIntArgs; i++) intArgs.emplace_back(block.getIArguments()->at(i));
+  }
+
+  const sd::LongType inputRank = input->rankOf();
+  if (intArgs[0] < 0) intArgs[0] += inputRank;
+
+  // input validation
+  REQUIRE_TRUE(intArgs[0] < inputRank, 0,
+               "GATHER op: input axis must be smaller than input array rank, but got %i and %i correspondingly!",
+               intArgs[0], inputRank);
+  REQUIRE_TRUE(indices != nullptr || numOfIntArgs > 1, 0,
+               "GATHER op: indices should be provided either as additional input array or as IntArguments !");
+
+  if (checkIndices) {
+    NDArray* pIndices = indices;
+    bool ownsIndices = false;
+    
+    if (indices == nullptr) {
+      std::vector<sd::LongType> shape = {static_cast<sd::LongType>(intArgs.size()) - 1};
+      std::vector<double> inputVec = std::vector<double>(intArgs.begin() + 1, intArgs.end());
+      pIndices = new NDArray(input->ordering(), shape, inputVec, DataType::INT64, block.launchContext());
+      ownsIndices = true;
+    }
+    
+    const sd::LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *pIndices, *input, intArgs[0]);
+
+    // FIXED: Cleanup BEFORE checking condition (REQUIRE_TRUE can throw)
+    if (ownsIndices) {
+      delete pIndices;
+      pIndices = nullptr;
+    }
+
+    // Check condition after cleanup
+    REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                 "GATHER OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                 numOfBadIndx);
+  }
+
+  helpers::gather(block.launchContext(), input, indices, output, intArgs);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(gather) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS});
+  getOpDescriptor()->setAllowedInputTypes(1, {ALL_INTS,ALL_FLOATS});
+  getOpDescriptor()->setAllowedOutputTypes(0, {ALL_INTS, ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(gather) {
+  // check shape of paddings
+  auto inputShapeInfo = inputShape->at(0);
+  sd::LongType* outputShapeInfo = nullptr;
+
+  sd::LongType axis = 0;
+
+  if (block.width() > 2) {
+    axis = INPUT_VARIABLE(2)->e<sd::LongType>(0);
+  } else
+    axis = block.numI() > 0 ? block.getIArguments()->at(0) : 0;
+
+  sd::LongType inputRank = shape::rank(inputShapeInfo);
+  if (axis < 0) axis += inputRank;
+
+  REQUIRE_TRUE(axis < inputRank, 0,
+               "GATHER op: input axis must be smaller than input array rank, but got %i and %i correspondingly!", axis,
+               inputRank);
+
+  bool isEmpty = false;
+
+  if (block.width() > 1) {
+    auto indicesShapeInfo = inputShape->at(1);
+
+    sd::LongType indicesRank = shape::rank(indicesShapeInfo);
+
+    sd::LongType outputRank = inputRank + indicesRank - 1;
+
+    ALLOCATE(outputShapeInfo, block.getWorkspace(), shape::shapeInfoLength(outputRank), sd::LongType);
+
+    // fill output shapeInfo
+    outputShapeInfo[0] = outputRank;
+    sd::LongType shapeIdx = 1;
+
+    for (sd::LongType i = 0; i < axis; ++i) outputShapeInfo[shapeIdx++] = inputShapeInfo[i + 1];
+
+    for (sd::LongType i = 0; i < indicesRank; ++i) outputShapeInfo[shapeIdx++] = indicesShapeInfo[i + 1];
+
+    for (sd::LongType i = axis + 1; i < inputRank; ++i) outputShapeInfo[shapeIdx++] = inputShapeInfo[i + 1];
+  } else if (block.numI() > 1) {
+    int indicesRank = block.numI() == 2 ? 0 : 1;
+
+    sd::LongType outputRank = inputRank + indicesRank - 1;
+    ALLOCATE(outputShapeInfo, block.getWorkspace(), shape::shapeInfoLength(outputRank), sd::LongType);
+
+    // building shape manually
+    outputShapeInfo[0] = outputRank;
+    int shapeIdx = 1;
+    for (sd::LongType i = 0; i < axis; ++i) outputShapeInfo[shapeIdx++] = inputShapeInfo[i + 1];
+
+    if (block.numI() > 2) outputShapeInfo[shapeIdx++] = block.numI() - 1;
+
+    for (sd::LongType i = axis + 1; i < inputRank; ++i) outputShapeInfo[shapeIdx++] = inputShapeInfo[i + 1];
+  } else
+    REQUIRE_TRUE(false, 0,
+                 "GATHER op: indices should be provided either as additional input array or as IntArguments !");
+
+  ShapeUtils::updateStridesAndType(outputShapeInfo, inputShapeInfo, shape::order(inputShapeInfo));
+
+  if (isEmpty) {
+    ArrayOptions::setPropertyBit(outputShapeInfo, ARRAY_EMPTY);
+  }
+
+  auto result = ConstantShapeHelper::getInstance().bufferForShapeInfo(outputShapeInfo)->primary();
+  RELEASE(outputShapeInfo, block.getWorkspace());
+  return SHAPELIST(result);
+}
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/transforms/gatherNd.cpp

@@ -0,0 +1,108 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Shyrma Yurii (iuriish@yahoo.com), created on 23.01.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_gather_nd)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/scatter.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(gather_nd, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const bool checkIndices = block.getBArguments()->empty() ? false : B_ARG(0);
+
+  const int rankIn = input->rankOf();
+  const int rankInd = indices->rankOf();
+
+  REQUIRE_TRUE(
+      rankInd > 0, 0,
+      "GATHER_ND op: array of indexes can't be single scalar, the requirement is: rank > 0, but got rank = %i instead!",
+      rankInd);
+  int lastIndDim = indices->sizeAt(-1);
+  REQUIRE_TRUE(lastIndDim <= rankIn, 0,
+               "GATHER_ND op: the last dimension of indices array must be <= rank of input array but got %i and %i "
+               "correspondingly!",
+               lastIndDim, rankIn);
+
+  if (checkIndices) {
+    const sd::LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *input);
+    REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                 "GATHER_ND OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                 numOfBadIndx);
+  }
+
+  helpers::gatherND(block.launchContext(), *input, *indices, *output);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(gather_nd) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(gather_nd) {
+  auto inShapeInfoIn = inputShape->at(0);
+  auto inShapeInfoInd = inputShape->at(1);
+
+  const int rankIn = inShapeInfoIn[0];
+  const int rankInd = inShapeInfoInd[0];
+  REQUIRE_TRUE(
+      rankInd > 0, 0,
+      "GATHER_ND op: array of indexes can't be single scalar, the requirement is: rank > 0, but got rank = %i instead!",
+      rankInd);
+  const int lastIndDim = inShapeInfoInd[rankInd];
+  REQUIRE_TRUE(lastIndDim <= rankIn, 0,
+               "GATHER_ND op: the last dimension of indices array must be <= rank of input array but got %i and %i "
+               "correspondingly!",
+               lastIndDim, rankIn);
+
+  int outRank = (rankInd - 1) + (rankIn - lastIndDim);
+
+  sd::LongType* outShapeInfo = nullptr;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+
+  outShapeInfo[0] = outRank;
+
+  for (int i = 1; i <= rankInd - 1; ++i) outShapeInfo[i] = inShapeInfoInd[i];
+
+  for (int i = 0; i < rankIn - lastIndDim; ++i) outShapeInfo[rankInd + i] = inShapeInfoIn[lastIndDim + i + 1];
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, inShapeInfoIn, 'c');
+  // ArrayOptions::setDataType(outShapeInfo, ArrayOptions::dataType(inShapeInfoIn));
+  return SHAPELIST(CONSTANT(outShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/hashcode.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_hashcode)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/hashcode.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(hashcode, 1, 1, false, 0, 0) {
+  REQUIRE_TRUE(block.width() == 1, 0, "hashcode: this op can't be applied along dimension");
+
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(output->isScalar(), 0, "hashcode: this op requires scalar output");
+
+  helpers::hashCode(block.launchContext(), *input, *output);
+
+  return sd::Status::OK;
+};
+
+DECLARE_SHAPE_FN(hashcode) {
+  return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(sd::DataType::INT64));
+}
+
+DECLARE_TYPES(hashcode) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ANY})
+      ->setAllowedInputTypes(1, {ANY})
+      ->setAllowedOutputTypes({sd::DataType::INT64});
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/histogram.cpp

@@ -0,0 +1,57 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_histogram)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/histogram.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(histogram, 1, 1, false, 0, 1) {
+  auto input = INPUT_VARIABLE(0);
+  auto numBins = INT_ARG(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(numBins == output->lengthOf(), 0, "Histogram: numBins must match output length")
+
+  output->nullify();
+  helpers::histogramHelper(block.launchContext(), *input, *output);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(histogram) {
+  auto numBins = INT_ARG(0);
+
+  return SHAPELIST(ConstantShapeHelper::getInstance().vectorShapeInfo(numBins, sd::DataType::INT64));
+}
+
+DECLARE_TYPES(histogram) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})->setAllowedOutputTypes({ALL_INTS});
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/histogram_fixed_width.cpp

@@ -0,0 +1,71 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 31.08.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_histogram_fixed_width)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/histogramFixedWidth.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(histogram_fixed_width, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto range = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const int nbins =
+      block.width() == 3 ? INPUT_VARIABLE(2)->e<int>(0) : block.getIArguments()->empty() ? 100 : INT_ARG(0);
+
+  const double leftEdge = range->e<double>(0);
+  const double rightEdge = range->e<double>(1);
+
+  REQUIRE_TRUE(leftEdge < rightEdge, 0,
+               "HISTOGRAM_FIXED_WIDTH OP: wrong content of range input array, bottom_edge must be smaller than "
+               "top_edge, but got %f and %f correspondingly !",
+               leftEdge, rightEdge);
+  REQUIRE_TRUE(nbins >= 1, 0,
+               "HISTOGRAM_FIXED_WIDTH OP: wrong nbins value, expected value should be >= 1, however got %i instead !",
+               nbins);
+
+  helpers::histogramFixedWidth(block.launchContext(), *input, *range, *output);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(histogram_fixed_width) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_INDICES});
+}
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(histogram_fixed_width) {
+  const int nbins =
+      block.width() == 3 ? INPUT_VARIABLE(2)->e<int>(0) : block.getIArguments()->empty() ? 100 : INT_ARG(0);
+  auto outShapeInfo = ConstantShapeHelper::getInstance().vectorShapeInfo(nbins, DataType::INT64);
+  return SHAPELIST(outShapeInfo);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/invertPermutation.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author, Yurii Shyrma (iuriish@yahoo.com), created on 06.12.2017
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_invert_permutation)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+////////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(invert_permutation, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->isVector(), 0, "INVERT_PERMUTATION op: input array must be vector, but got shape %s instead !",
+               ShapeUtils::shapeAsString(input).c_str());
+
+  helpers::invertPermutation(block.launchContext(), *input, *output);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SYN(InvertPermutation, invert_permutation);
+
+DECLARE_TYPES(invert_permutation) { getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/merge_add.cpp

@@ -0,0 +1,102 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_mergeadd)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(mergeadd, -1, 1, false) {
+  REQUIRE_OK(this->validateInputDimensionsMatch(block));
+
+  auto output = OUTPUT_VARIABLE(0);
+  if (output->isEmpty()) {
+    return Status::OK;
+  }
+
+  int nonEmpty = 0;
+  for (size_t i = 0; i < block.width(); i++)
+    if (!INPUT_VARIABLE(i)->isEmpty()) nonEmpty++;
+
+
+  std::vector<NDArray*> inArrs(nonEmpty);
+  int numNonEmptyAdded = 0;
+  if(nonEmpty > 0)
+  for (size_t i = 0; i < block.width(); ++i) {
+     if(!INPUT_VARIABLE(i)->isEmpty())inArrs[numNonEmptyAdded++] = INPUT_VARIABLE(i);
+  }
+
+  helpers::mergeAdd(block.launchContext(), inArrs, *output);
+
+  return Status::OK;
+}
+
+DECLARE_SYN(mergesum, mergeadd);
+DECLARE_SYN(add_n, mergeadd);
+DECLARE_SYN(addn, mergeadd);
+DECLARE_SYN(accumulaten, mergeadd);
+DECLARE_SYN(accumulate_n, mergeadd);
+
+DECLARE_TYPES(mergeadd) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY); }
+
+CUSTOM_OP_IMPL(mergeadd_bp, 2, 1, false, 0, 0) {
+  auto inSize = block.width() - 1;
+
+  REQUIRE_OK(this->validateInputDimensionsMatch(block));
+
+  std::vector<NDArray*> outArrs(inSize);
+
+  const auto gradient = INPUT_VARIABLE(inSize);
+
+  for (size_t i = 0; i < inSize; ++i) {
+    outArrs[i] = OUTPUT_VARIABLE(i);
+  }
+  helpers::mergeAddBp(block.launchContext(), *gradient, outArrs);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(mergeadd_bp) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY); }
+DECLARE_SHAPE_FN(mergeadd_bp) {
+  const int numOfInArrs = block.width() - 1;
+
+  auto shapeList = SHAPELIST();
+
+  for (int e = 0; e < numOfInArrs; e++) {
+    auto inShape = inputShape->at(e);
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(ArrayOptions::dataType(inShape),
+                                                                               shape::order(inShape),
+                                                                               shape::rank(inShape),
+                                                                               shape::shapeOf(inShape))->primary());
+  }
+
+  return shapeList;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/merge_avg.cpp

@@ -0,0 +1,94 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_mergeavg)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(mergeavg, -1, 1, false) {
+  REQUIRE_OK(this->validateInputDimensionsMatch(block));
+
+  auto output = OUTPUT_VARIABLE(0);
+  if (output->isEmpty()) {
+    return Status::OK;
+  }
+
+  int nonEmpty = 0;
+  for (size_t i = 0; i < block.width(); i++)
+    if (!INPUT_VARIABLE(i)->isEmpty()) nonEmpty++;
+
+  std::vector<NDArray*> inArrs(nonEmpty);
+  int numNonEmptyAdded = 0;
+  if(nonEmpty > 0)
+  for (size_t i = 0; i < block.width(); ++i) if(!INPUT_VARIABLE(i)->isEmpty())inArrs[numNonEmptyAdded++] = INPUT_VARIABLE(i);
+
+  helpers::mergeAvg(block.launchContext(), inArrs, *output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(mergeavg) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS}); }
+
+CUSTOM_OP_IMPL(mergeavg_bp, 2, 1, false, 0, 0) {
+  auto inSize = block.width() - 1;
+
+  REQUIRE_OK(this->validateInputDimensionsMatch(block));
+
+  std::vector<NDArray*> outArrs(inSize);
+
+  const auto gradient = INPUT_VARIABLE(inSize);
+
+  for (size_t i = 0; i < inSize; ++i) {
+    outArrs[i] = OUTPUT_VARIABLE(i);
+  }
+  helpers::mergeAvgBp(block.launchContext(), *gradient, outArrs);
+  return Status::OK;
+}
+
+DECLARE_TYPES(mergeavg_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY);
+}
+DECLARE_SHAPE_FN(mergeavg_bp) {
+  const int numOfInArrs = block.width() - 1;
+
+  auto shapeList = SHAPELIST();
+
+  for (int e = 0; e < numOfInArrs; e++) {
+    auto inShape = inputShape->at(e);
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(ArrayOptions::dataType(inShape),
+                                                                               shape::order(inShape),
+                                                                               shape::rank(inShape),
+                                                                               shape::shapeOf(inShape))->primary());
+  }
+
+  return shapeList;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/merge_max.cpp

@@ -0,0 +1,100 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_mergemax)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(mergemax, -1, 1, false) {
+  REQUIRE_OK(this->validateInputDimensionsMatch(block));
+
+  auto output = OUTPUT_VARIABLE(0);
+  if (output->isEmpty()) {
+    return Status::OK;
+  }
+  int nonEmpty = 0;
+  for (size_t i = 0; i < block.width(); i++)
+    if (!INPUT_VARIABLE(i)->isEmpty()) nonEmpty++;
+
+  std::vector<NDArray*> inArrs(nonEmpty);
+
+  int numNonEmptyAdded = 0;
+  if(nonEmpty > 0)
+  for (size_t i = 0; i < block.width(); ++i) if(!INPUT_VARIABLE(i)->isEmpty())inArrs[numNonEmptyAdded++] = INPUT_VARIABLE(i);
+
+  helpers::mergeMax(block.launchContext(), inArrs, *output);
+
+  return Status::OK;
+}
+DECLARE_SYN(MergeMax, mergemax);
+
+DECLARE_TYPES(mergemax) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY);
+}
+
+CUSTOM_OP_IMPL(mergemax_bp, 2, 1, false, 0, 0) {
+  auto inSize = block.width();
+
+  REQUIRE_OK(this->validateInputDimensionsMatch(block));
+
+  std::vector<NDArray*> inArrs(inSize);
+  std::vector<NDArray*> outArrs(inSize - 1);
+
+  for (size_t i = 0; i < inSize; ++i) inArrs[i] = INPUT_VARIABLE(i);
+
+  for (size_t i = 0; i < (inSize - 1); ++i) {
+    outArrs[i] = OUTPUT_NULLIFIED(i);
+  }
+
+  helpers::mergeMaxBp(block.launchContext(), inArrs, outArrs);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(mergemax_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY);
+}
+DECLARE_SHAPE_FN(mergemax_bp) {
+  const int numOfInArrs = block.width() - 1;
+
+  auto shapeList = SHAPELIST();
+
+  for (int e = 0; e < numOfInArrs; e++) {
+    auto inShape = inputShape->at(e);
+    shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(ArrayOptions::dataType(inShape),
+                                                                               shape::order(inShape),
+                                                                               shape::rank(inShape),
+                                                                               shape::shapeOf(inShape))->primary());
+  }
+
+  return shapeList;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/merge_max_idx.cpp

@@ -0,0 +1,61 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_mergemaxindex)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(mergemaxindex, -1, 1, false, 0, 0) {
+  REQUIRE_OK(this->validateInputDimensionsMatch(block));
+  auto output = OUTPUT_VARIABLE(0);
+
+  std::vector<NDArray*> inArrs(block.width());
+
+  for (size_t i = 0; i < block.width(); ++i) inArrs[i] = INPUT_VARIABLE(i);
+
+  helpers::mergeMaxIndex(block.launchContext(), inArrs, *output);
+
+  return Status::OK;
+}
+
+DECLARE_SYN(MergeMaxIndex, mergemaxindex);
+
+DECLARE_TYPES(mergemaxindex) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})->setAllowedOutputTypes({ALL_INDICES});
+}
+}  // namespace ops
+DECLARE_SHAPE_FN(mergemaxindex) {
+  auto in = inputShape->at(0);
+  auto dtype = INT32;
+  if (block.getIArguments()->size() > 0) dtype = (DataType)INT_ARG(0);
+
+  auto resShape = ShapeBuilders::copyShapeInfoAndType(in, dtype, block.workspace());
+  return SHAPELIST(CONSTANT(resShape));
+}
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/mirrorPad.cpp

@@ -0,0 +1,86 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 07.06.2018
+//
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_mirror_pad)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(mirror_pad, 2, 1, false, 0, 1) {
+  auto input = INPUT_VARIABLE(0);
+  auto paddings = INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  const int mode = INT_ARG(0);  // 0 - REFLECT, else - SYMMETRIC
+  const int includeBorder = mode ? 0 : 1;
+  helpers::mirrorPad(block.launchContext(), *input, *paddings, *output, mode);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(mirror_pad) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_FLOATS});
+  getOpDescriptor()->setAllowedInputTypes(1, {DataType::INT32, DataType::INT64});  // to conform with TF
+  getOpDescriptor()->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(mirror_pad) {
+  auto input = INPUT_VARIABLE(0);
+  auto paddings = INPUT_VARIABLE(1);
+
+  const int includeBorder = static_cast<bool>(INT_ARG(0)) ? 0 : 1;
+
+  if (input->isScalar()) {
+    sd::LongType len = input->isScalar() ? 1 + paddings->e<sd::LongType>(0)  + paddings->e<sd::LongType>(1) : input->lengthOf() + paddings->e<sd::LongType>(0) + paddings->e<sd::LongType>(1);
+    return SHAPELIST(ConstantShapeHelper::getInstance().vectorShapeInfo(len, input->dataType()));
+  }
+
+  sd::LongType* outShapeInfo(nullptr);
+  int rank = input->rankOf();
+
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+  outShapeInfo[0] = rank;
+  if(paddings->isVector()) {
+    for (int i = 0; i < rank; ++i) {
+      outShapeInfo[i + 1] = input->sizeAt(i) + paddings->e<sd::LongType>(0) + paddings->e<sd::LongType>(1);
+
+    }
+  } else {
+    for (int i = 0; i < rank; ++i) {
+      outShapeInfo[i + 1] = input->sizeAt(i) + paddings->e<sd::LongType>(i, 0) + paddings->e<sd::LongType>(i, 1);
+
+    }
+  }
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, input->shapeInfo(), input->ordering());
+
+  return SHAPELIST(CONSTANT(outShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/pad.cpp

@@ -0,0 +1,128 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Shyrma Yurii (iuriish@yahoo.com), created on 06.11.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_pad)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+#include <numeric>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(pad, 2, 1, false, 0, 1) {
+  auto input = INPUT_VARIABLE(0);
+  auto paddings = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const int rank = input->rankOf();
+
+  // input validation
+  std::vector<sd::LongType> expectedPaddingsShape = {rank, 2};
+  std::vector<sd::LongType> *currentPaddingsShape = paddings->getShapeAsVector();
+  REQUIRE_TRUE(expectedPaddingsShape == *currentPaddingsShape, 0,
+               "PAD op: wrong shape of paddings array, expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(expectedPaddingsShape).c_str(),
+               ShapeUtils::shapeAsString(*currentPaddingsShape).c_str());
+
+
+  NDArray padValue(input->dataType(), block.launchContext());
+
+  // in case of REFLECT and SYMMETRIC modes paddings must obey additional shape requirements
+  if (INT_ARG(0) == 0) {  // CONSTANT mode
+    if (block.width() > 2) {
+      REQUIRE_TRUE(
+          input->dataType() == INPUT_VARIABLE(2)->dataType(), 0,
+          "PAD op: data types of input and padValue arrays should be the same but got %i and %i correspondingly !",
+          input->dataType(), INPUT_VARIABLE(2)->dataType());
+      auto get = INPUT_VARIABLE(2)->e(0);
+      padValue.assign(&get);
+    } else if (!block.getTArguments()->empty())
+      padValue = T_ARG(0);
+  } else if (INT_ARG(0) == 1) {  // REFLECT mode
+    for (int dim = 0; dim < rank; ++dim)
+    REQUIRE_TRUE(paddings->e<sd::LongType>(dim, 0) <= (input->shapeOf()[dim] - 1) &&
+                 paddings->e<sd::LongType>(dim, 1) <= (input->shapeOf()[dim] - 1),
+                 0, "PAD op: wrong content of paddings array for REFLECT mode !");
+  }
+  if (INT_ARG(0) == 2) {  // SYMMETRIC mode
+    for (int dim = 0; dim < rank; ++dim)
+    REQUIRE_TRUE(paddings->e<sd::LongType>(dim, 0) <= input->shapeOf()[dim] &&
+                 paddings->e<sd::LongType>(dim, 1) <= input->shapeOf()[dim],
+                 0, "PAD op: wrong content of paddings array for SYMMETRIC mode !");
+  }
+
+  // CONSTANT->0, REFLECT->1, SYMMETRIC->2
+  REQUIRE_TRUE(
+      INT_ARG(0) >= 0 && INT_ARG(0) <= 2, 0,
+      "PAD op: unknown padding mode, there are only three possible legal values -> 0,1,2, but got %i instead !",
+      INT_ARG(0));
+
+  helpers::pad(block.launchContext(), INT_ARG(0), *input, *paddings, *output, padValue);
+
+  delete currentPaddingsShape;
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(pad) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, sd::DataType::ANY)
+      ->setAllowedInputTypes(1, {DataType::INT32, DataType::INT64})  // INT32 with TF
+      ->setSameMode(true);
+}
+
+DECLARE_SHAPE_FN(pad) {
+  // check shape of paddings
+  auto inputShapeInfo = inputShape->at(0);
+  auto paddings = INPUT_VARIABLE(1);
+  const int rank = inputShapeInfo[0];
+  if(rank < 0 || rank > SD_MAX_RANK) {
+    THROW_EXCEPTION("PAD op: Bad shape buffer. Likely corrupt. Please ensure buffer was not deallocated.");
+  }
+  // paddings validation
+  const std::vector<sd::LongType> expectedPaddingsShape = {rank, 2};
+  const std::vector<sd::LongType> *currentPaddingsShape = paddings->getShapeAsVector();
+  REQUIRE_TRUE(expectedPaddingsShape == *currentPaddingsShape, 0,
+               "PAD op: wrong shape of paddings array, expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(expectedPaddingsShape).c_str(),
+               ShapeUtils::shapeAsString(*currentPaddingsShape).c_str());
+
+  delete currentPaddingsShape;
+  sd::LongType* outShapeInfo = nullptr;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+  outShapeInfo[0] = rank;
+  for (int i = 1; i <= rank; ++i)
+    outShapeInfo[i] = inputShapeInfo[i] + paddings->e<sd::LongType>(i - 1, 0) + paddings->e<sd::LongType>(i - 1, 1);
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, inputShapeInfo, shape::order(inputShapeInfo));
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().bufferForShapeInfo(outShapeInfo)->primary());
+  RELEASE(outShapeInfo, block.getWorkspace());
+  return ret;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/parallelStack.cpp

@@ -0,0 +1,73 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 01.11.2017
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_parallel_stack)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/stack.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(parallel_stack, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  // check whether shapes of all input array are the same
+  for (int i = 0; i < (int)block.width() - 1; ++i)
+    REQUIRE_TRUE(shape::equalsSoft((INPUT_VARIABLE(i))->shapeInfo(), (INPUT_VARIABLE(i + 1))->shapeInfo()), 0,
+                 "PARALLEL_STACK op: the shapes of all input arrays must be the same !");
+
+  std::vector<NDArray*> inArrs(block.width());
+  for (size_t i = 0; i < block.width(); ++i) inArrs[i] = INPUT_VARIABLE(i);
+
+  const int dim = 0;
+  helpers::stack(block.launchContext(), inArrs, *output, dim);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(parallel_stack) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(parallel_stack) {
+  auto inShapeInfo = inputShape->at(0);
+  int rank = inShapeInfo[0];
+
+  sd::LongType* outShapeInfo = nullptr;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank + 1), sd::LongType);
+
+  outShapeInfo[0] = rank + 1;
+  outShapeInfo[1] = block.width();
+  for (int i = 1; i <= rank; ++i) outShapeInfo[i + 1] = inShapeInfo[i];
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, inShapeInfo, shape::order(inShapeInfo));
+
+  return SHAPELIST(CONSTANT(outShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/repeat.cpp

@@ -0,0 +1,78 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_repeat)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+// here iArgs is int vector of repeats at the beginning and last element in iArgs is dimension
+CUSTOM_OP_IMPL(repeat, 1, 1, true, 0, -1) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  std::vector<LongType> repeats = *block.getIArguments();
+
+  const int axis = repeats.back() < 0 ? repeats.back() + input->rankOf() : repeats.back();
+
+  repeats.pop_back();
+
+  REQUIRE_TRUE(0 <= axis && axis < input->rankOf(), 0,
+               "CUSTOM REPEAT OP: wrong axis argument it should be less then input array rank %i, but got %i instead !",
+               input->rankOf(), axis);
+
+  REQUIRE_TRUE(repeats.size() == 1 || repeats.size() == static_cast<size_t>(input->sizeAt(axis)), 0,
+               "CUSTOM REPEAT OP: wrong axis argument, size of repeats vector must be 1 or equal to dimension at given "
+               "axis, but got repeats.size = %i and axis = %i !",
+               repeats.size(), axis);
+
+  input->repeat(axis, repeats, *output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(repeat) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+DECLARE_SHAPE_FN(repeat) {
+  auto input = INPUT_VARIABLE(0);
+
+  std::vector<LongType> repeats = *block.getIArguments();
+
+  const int axis = repeats.back() < 0 ? repeats.back() + input->rankOf() : repeats.back();
+
+  repeats.pop_back();
+
+  auto outShape = ShapeUtils::evalRepeatShape(axis, repeats, *input);
+
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().bufferForShapeInfo(input->dataType(),
+                                                                             input->ordering(),
+                                                                             outShape)->primary());
+  return ret;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/reverse.cpp

@@ -0,0 +1,106 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 02.11.2017
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reverse)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/reverse.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(reverse, 1, 1, true, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (output->isEmpty()) {
+    // No-op
+    return Status::OK;
+  }
+
+  std::vector<LongType> axis;
+
+  if (block.width() > 1)
+    axis = INPUT_VARIABLE(1)->template asVectorT<LongType>();
+  else if (block.numI() > 0)
+    axis = *block.getIArguments();
+
+  if (axis.empty()) {  // do not perform reversion
+    if (!block.isInplace()) output->assign(input);
+  } else {
+    // check the consistency of input dimensions to reverse along
+    shape::checkDimensions(input->rankOf(), &axis);
+    helpers::reverse(block.launchContext(), input, output, &axis);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SYN(reverse_v2, reverse);
+
+DECLARE_TYPES(reverse) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY);
+  getOpDescriptor()->setAllowedInputTypes(1, {INT32, INT64});
+  getOpDescriptor()->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(reverse_bp, 2, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto eps = block.width() == 3 ? INPUT_VARIABLE(2) : INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+  std::vector<LongType> axis;
+
+  if (block.width() == 3)
+    axis = INPUT_VARIABLE(1)->template asVectorT<LongType>();
+  else if (block.numI() > 0)
+    axis = *block.getIArguments();
+
+  if (axis.empty()) {  // reversion is not performed in this case
+    output->assign(eps);
+  } else {
+    // check the consistency of input dimensions to reverse along
+    shape::checkDimensions(input->rankOf(), &axis);
+    // we just reverse back original array
+    helpers::reverse(block.launchContext(), eps, output, &axis);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(reverse_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(reverse_bp) {
+  auto in = inputShape->at(0);
+  LongType *out;
+  COPY_SHAPE(in, out);
+
+  return SHAPELIST(CONSTANT(out));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/reverseSequence.cpp

@@ -0,0 +1,111 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  Created by Yurii Shyrma on 25.01.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reverse_sequence)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/reverse.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(reverse_sequence, 2, 1, false, 0, 2) {
+  auto input = INPUT_VARIABLE(0);
+  auto seqLengths = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+
+  int seqDim = INT_ARG(0);
+  int batchDim = block.numI() > 1 ? INT_ARG(1) : 0;
+
+  REQUIRE_TRUE(input->rankOf() > 1, 0,
+               "REVERSE_SEQUENSE operation: input array must have rank > 1, but got %i instead !", input->rankOf());
+  REQUIRE_TRUE(seqLengths->rankOf() == 1, 0,
+               "REVERSE_SEQUENSE operation: input array seqLengths must be 1D vector, that is it must have rank == 1, "
+               "but got %i instead !",
+               seqLengths->rankOf());
+  REQUIRE_TRUE(seqLengths->lengthOf() == input->sizeAt(batchDim), 0,
+               "REVERSE_SEQUENSE custom operation: the length of array seqLengths must be equal to the value of "
+               "batchDim dimension of input array, but got %i and %i correspondingly !",
+               seqLengths->lengthOf(), input->sizeAt(batchDim));
+  REQUIRE_TRUE(seqDim != batchDim, 0,
+               "REVERSE_SEQUENSE operation: input integer parameters seqDim and batchDim must be different, but they "
+               "both are equal to %i !",
+               batchDim);
+  REQUIRE_TRUE(batchDim < input->rankOf(), 0,
+               "REVERSE_SEQUENSE operation: input integer parameter batchDim must be smaller than input array rank, "
+               "but got %i and %i correspondingly !",
+               batchDim, input->rankOf());
+  REQUIRE_TRUE(seqDim < input->rankOf(), 0,
+               "REVERSE_SEQUENSE operation: input integer parameter seqDim must be smaller than input array rank, but "
+               "got %i  and %i correspondingly !",
+               seqDim, input->rankOf());
+
+  auto maxElem = seqLengths->reduceNumber(reduce::Max);
+  REQUIRE_TRUE(maxElem->e<sd::LongType>(0) <= input->sizeAt(seqDim), 0,
+               "REVERSE_SEQUENSE operation: max element in seqLengths array must be not greater than value of seqDim "
+               "dimension of input array !");
+
+  helpers::reverseSequence(block.launchContext(), input, seqLengths, output, seqDim, batchDim);
+  delete maxElem;
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(reverse_sequence) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS});
+  getOpDescriptor()->setAllowedInputTypes(1, {DataType::INT32, DataType::INT64});
+  getOpDescriptor()->setAllowedOutputTypes(0, DataType::INHERIT);
+}
+
+DECLARE_SHAPE_FN(reverse_sequence) {
+  auto inShapeInfo = inputShape->at(0);
+  auto seqLenShapeInfo = inputShape->at(1);
+
+  int seqDim = INT_ARG(0);
+  int batchDim = block.numI() > 1 ? INT_ARG(1) : 0;
+
+  REQUIRE_TRUE(batchDim < inShapeInfo[0], 0,
+               "REVERSE_SEQUENSE operation: input integer parameter batchDim must be smaller than input array rank, "
+               "but got %i and %i correspondingly !",
+               batchDim, inShapeInfo[0]);
+  REQUIRE_TRUE(seqDim < inShapeInfo[0], 0,
+               "REVERSE_SEQUENSE operation: input integer parameter seqDim must be smaller than input array rank, but "
+               "got %i  and %i correspondingly !",
+               seqDim, inShapeInfo[0]);
+  REQUIRE_TRUE(inShapeInfo[0] > 1, 0,
+               "REVERSE_SEQUENSE operation: input array must have rank > 1, but got %i instead !", inShapeInfo[0]);
+  REQUIRE_TRUE(seqLenShapeInfo[0] == 1, 0,
+               "REVERSE_SEQUENSE operation: input array seqLengths must be 1D vector, that is it must have rank == 1, "
+               "but got %i instead !",
+               seqLenShapeInfo[0]);
+  REQUIRE_TRUE(seqLenShapeInfo[1] == inShapeInfo[batchDim + 1], 0,
+               "REVERSE_SEQUENSE custom operation: the length of array seqLengths must be equal to the value of "
+               "batchDim dimension of input array, but got %i and %i correspondingly !",
+               seqLenShapeInfo[1], inShapeInfo[batchDim + 1]);
+
+  return SHAPELIST(CONSTANT(inShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_add.cpp

@@ -0,0 +1,108 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com, created on 24.11.17.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_add)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(scatter_add, 3, 1, true) {
+    auto input = INPUT_VARIABLE(0);
+    auto indices = INPUT_VARIABLE(1);
+    auto updates = INPUT_VARIABLE(2);
+    if(indices->isEmpty())
+      return Status::OK;
+
+    auto output = OUTPUT_VARIABLE(0);
+
+    if (!block.isInplace())
+        output->assign(input);
+
+    const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+    const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+    const int inRank  = input->rankOf();
+    const int indRank = indices->rankOf();
+    const int updRank = updates->rankOf();
+    const LongType indLen = indices->lengthOf();
+
+    REQUIRE_TRUE(inRank > 0, 0, "SCATTER_ADD OP: input should not be scalar !");
+
+    if(inRank == 1) {
+        REQUIRE_TRUE(indices->isSameShape(updates), 0, "SCATTER_ADD OP: when input array has rank = 1 then indices and updates must have the same shapes, but got %s and %s correspondingly !", ShapeUtils::shapeAsString(indices).c_str(), ShapeUtils::shapeAsString(updates).c_str());
+    }
+    else if (inRank == updRank && indices->isVector()) {
+        auto* updShapeVec = updates->getShapeAsVector();
+        auto* inShapeVec  = input->getShapeAsVector();
+        std::vector<LongType> expectedUpdShape = {indices->lengthOf()};
+        expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin()+1, inShapeVec->end());
+        delete updShapeVec;
+        delete inShapeVec;
+
+    }
+    else {
+
+        REQUIRE_TRUE(updRank == indRank + inRank - 1, 0, "SCATTER_ADD OP: wrong rank of updates array, expected is %i, but got %i instead !", indRank + inRank - 1 , updRank);
+
+        auto* updShapeVec = updates->getShapeAsVector();
+        auto* inShapeVec  = input->getShapeAsVector();
+        auto* indShapeVec = indices->getShapeAsVector();
+        std::vector<LongType> expectedUpdShape = *indShapeVec;
+        expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + LongType(1L), inShapeVec->end());
+        delete updShapeVec;
+        delete inShapeVec;
+        delete indShapeVec;
+
+    }
+
+    if (!indices->isEmpty()) {
+
+        if(checkIndices) {
+            const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output, 0);
+            REQUIRE_TRUE(numOfBadIndx == 0, 0, "SCATTER_ADD OP: please check elements of indices-array, total number of wrong elements is %lld!", numOfBadIndx);
+        }
+
+        helpers::scatter(block.launchContext(), pairwise::Add, *indices, *updates, *output, lock);
+    }
+
+    return Status::OK;
+}
+
+DECLARE_SYN(ScatterAdd, scatter_add);
+
+DECLARE_TYPES(scatter_add) {
+    getOpDescriptor()
+        ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+        ->setAllowedInputTypes(1, {ALL_INTS})
+        ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+        ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+}
+}
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_div.cpp

@@ -0,0 +1,108 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Created by raver119 on 24.11.17.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_div)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(scatter_div, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  REQUIRE_TRUE(inRank > 0, 0, "SCATTER_DIV OP: input should not be scalar !");
+
+  if (inRank == 1) {
+    REQUIRE_TRUE(indices->isSameShape(updates), 0,
+                 "SCATTER_DIV OP: when input array has rank = 1 then indices and updates must have the same shapes, "
+                 "but got %s and %s correspondingly !",
+                 ShapeUtils::shapeAsString(indices).c_str(), ShapeUtils::shapeAsString(updates).c_str());
+  } else if (inRank == updRank && indices->isVector()) {
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = {indices->lengthOf()};
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_DIV OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+  } else {
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    auto* indShapeVec = indices->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = *indShapeVec;
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_DIV OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+    delete indShapeVec;
+  }
+
+  if (!indices->isEmpty()) {
+    if (checkIndices) {
+      const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output, 0);
+      REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                   "SCATTER_DIV OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                   numOfBadIndx);
+    }
+
+    helpers::scatter(block.launchContext(), pairwise::Divide, *indices, *updates, *output, lock);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(ScatterDiv, scatter_div);
+
+DECLARE_TYPES(scatter_div) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_max.cpp

@@ -0,0 +1,113 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 1.08.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_max)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(scatter_max, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  REQUIRE_TRUE(inRank > 0, 0, "SCATTER_MAX OP: input should not be scalar !");
+
+  if (inRank == 1) {
+    REQUIRE_TRUE(indices->isSameShape(updates), 0,
+                 "SCATTER_MAX OP: when input array has rank = 1 then indices and updates must have the same shapes, "
+                 "but got %s and %s correspondingly !",
+                 ShapeUtils::shapeAsString(indices).c_str(), ShapeUtils::shapeAsString(updates).c_str());
+  } else if (inRank == updRank && indices->isVector()) {
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = {indices->lengthOf()};
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_MAX OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+  } else {
+    REQUIRE_TRUE(updRank == indRank + inRank - 1, 0,
+                 "SCATTER_MAX OP: wrong rank of updates array, expected is %i, but got %i instead !",
+                 indRank + inRank - 1, updRank);
+
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    auto* indShapeVec = indices->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = *indShapeVec;
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_MAX OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+    delete indShapeVec;
+  }
+
+  if (!indices->isEmpty()) {
+    if (checkIndices) {
+      const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output, 0);
+      REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                   "SCATTER_MAX OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                   numOfBadIndx);
+    }
+
+    helpers::scatter(block.launchContext(), pairwise::MaxPairwise, *indices, *updates, *output, lock);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(ScatterMax, scatter_max);
+
+DECLARE_TYPES(scatter_max) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_min.cpp

@@ -0,0 +1,106 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 1.08.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_min)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(scatter_min, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  REQUIRE_TRUE(inRank > 0, 0, "SCATTER_MIN OP: input should not be scalar !");
+
+  if (inRank <= 1) {
+    REQUIRE_TRUE(indices->isSameShape(updates), 0,
+                 "SCATTER_MIN OP: when input array has rank = 1 then indices and updates must have the same shapes, "
+                 "but got %s and %s correspondingly !",
+                 ShapeUtils::shapeAsString(indices).c_str(), ShapeUtils::shapeAsString(updates).c_str());
+  } else if (inRank == updRank && indices->isVector()) {
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = {indices->lengthOf()};
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_MIN OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+  } else {
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    auto* indShapeVec = indices->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = *indShapeVec;
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+    delete updShapeVec;
+    delete inShapeVec;
+    delete indShapeVec;
+
+  }
+
+  if (!indices->isEmpty()) {
+    if (checkIndices) {
+      const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output, 0);
+      REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                   "SCATTER_MIN OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                   numOfBadIndx);
+    }
+
+    helpers::scatter(block.launchContext(), pairwise::MinPairwise, *indices, *updates, *output, lock);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(ScatterMin, scatter_min);
+
+DECLARE_TYPES(scatter_min) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_mul.cpp

@@ -0,0 +1,112 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Created by raver119 on 24.11.17.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_mul)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(scatter_mul, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+
+  auto output = OUTPUT_VARIABLE(0);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  if (!block.isInplace()) output->assign(input);
+
+  REQUIRE_TRUE(inRank > 0, 0, "SCATTER_MUL OP: input should not be scalar !");
+
+  if (inRank == 1) {
+    REQUIRE_TRUE(indices->isSameShape(updates), 0,
+                 "SCATTER_MUL OP: when input array has rank = 1 then indices and updates must have the same shapes, "
+                 "but got %s and %s correspondingly !",
+                 ShapeUtils::shapeAsString(indices).c_str(), ShapeUtils::shapeAsString(updates).c_str());
+  } else if (inRank == updRank && indices->isVector()) {
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = {indices->lengthOf()};
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_MUL OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+  } else {
+    REQUIRE_TRUE(updRank == indRank + inRank - 1, 0,
+                 "SCATTER_MUL OP: wrong rank of updates array, expected is %i, but got %i instead !",
+                 indRank + inRank - 1, updRank);
+
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    auto* indShapeVec = indices->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = *indShapeVec;
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_MUL OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+    delete indShapeVec;
+  }
+
+  if (!indices->isEmpty()) {
+    if (checkIndices) {
+      const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output, 0);
+      REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                   "SCATTER_MUL OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                   numOfBadIndx);
+    }
+
+    helpers::scatter(block.launchContext(), pairwise::Multiply, *indices, *updates, *output, lock);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(ScatterMul, scatter_mul);
+
+DECLARE_TYPES(scatter_mul) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_nd.cpp

@@ -0,0 +1,118 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 21.08.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_nd)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(scatter_nd, 3, 1, false, 0, 0) {
+  auto indices = INPUT_VARIABLE(0);
+  auto updates = INPUT_VARIABLE(1);
+  auto shape = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+  const int shapeRank = shape->rankOf();
+  const LongType shapeLen = shape->lengthOf();
+
+  REQUIRE_TRUE(shapeRank == 1, 0, "SCATTER_ND OP: the rank of shape array must be 1, but got %i instead !", shapeRank);
+  REQUIRE_TRUE(indices->sizeAt(-1) <= shapeLen, 0,
+               "SCATTER_ND OP: last dimension of indices array must be <= length of shape array, but got %i and %i "
+               "correspondingly !",
+               indices->sizeAt(-1), shapeLen);
+
+  REQUIRE_TRUE(
+      updRank == (indRank - 1 + shapeLen - indices->sizeAt(-1)), 0,
+      "SCATTER_ND OP: the equality updates_rank = (indices_rank - 1 + shape_length - last_indices_dimension) must be "
+      "true for input arrays, but got instead: updates_rank = %i, shape_length = %i, last_indices_dimension = %i !",
+      updRank, shapeLen, indices->sizeAt(-1));
+
+  std::vector<LongType> outShape = shape->getBufferAsVector<LongType>();
+  auto* updShapePtr = updates->getShapeAsVector();
+  std::vector<LongType> updShape = *updShapePtr;
+  delete updShapePtr;
+  auto* indShapePtr = indices->getShapeAsVector();
+  std::vector<LongType> indShape = *indShapePtr;
+  delete indShapePtr;
+  std::vector<LongType> expectedUpdShape(std::begin(indShape), std::end(indShape) - 1);
+  std::move(std::begin(outShape) + indices->sizeAt(-1), std::end(outShape), std::back_inserter(expectedUpdShape));
+  REQUIRE_TRUE(expectedUpdShape == updShape, 0,
+               "SCATTER_ND OP: wrong shape of updates array, expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(updShape).c_str());
+
+  if (checkIndices) {
+    const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output);
+    REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                 "SCATTER_ND OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                 numOfBadIndx);
+  }
+
+  // initial zeroing of output
+  *output = 0;
+
+  helpers::scatterND(block.launchContext(), pairwise::Add, *indices, *updates, *output, lock);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(scatter_nd) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(2, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(scatter_nd) {
+  auto shape = INPUT_VARIABLE(2);
+  auto updShapeInfo = inputShape->at(1);
+
+  LongType *outShapeInfo;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(shape->lengthOf()), sd::LongType);
+
+  outShapeInfo[0] = shape->lengthOf();
+  for (int i = 0; i < outShapeInfo[0]; ++i) outShapeInfo[i + 1] = shape->e<LongType>(i);
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, updShapeInfo, shape::order(updShapeInfo));
+
+  auto result = SHAPELIST(CONSTANT(outShapeInfo));
+  RELEASE(outShapeInfo, block.getWorkspace());
+  return result;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_nd_add.cpp

@@ -0,0 +1,101 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 22.08.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_nd_add)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(scatter_nd_add, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  const LongType indLastDim = indices->sizeAt(-1);
+
+  REQUIRE_TRUE(
+      indLastDim <= inRank, 0,
+      "SCATTER_ND_ADD OP: the last dimension of indices array must be <= input_array_rank, but got %i instead !",
+      indLastDim);
+  REQUIRE_TRUE(
+      updRank == (indRank - 1 + inRank - indLastDim), 0,
+      "SCATTER_ND_ADD OP: the equality updates_rank = (indices_rank - 1 + input_rank - last_indices_dimension) must be "
+      "true for input arrays, but got instead: updates_rank = %i, indices_rank = %i, last_indices_dimension = %i !",
+      updRank, indRank, indLastDim);
+
+  auto* inShapePtr = input->getShapeAsVector();
+  std::vector<LongType> inShape = *inShapePtr;
+  delete inShapePtr;
+  auto* updShapePtr = updates->getShapeAsVector();
+  std::vector<LongType> updShape = *updShapePtr;
+  delete updShapePtr;
+  auto* indShapePtr = indices->getShapeAsVector();
+  std::vector<LongType> indShape = *indShapePtr;
+  delete indShapePtr;
+  std::vector<LongType> expectedUpdShape(std::begin(indShape), std::end(indShape) - 1);
+  if (inRank > indLastDim)
+    std::move(std::begin(inShape) + indLastDim, std::end(inShape), std::back_inserter(expectedUpdShape));
+  REQUIRE_TRUE(expectedUpdShape == updShape, 0,
+               "SCATTER_ND_ADD OP: wrong shape of updates array, expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(updShape).c_str());
+
+  if (checkIndices) {
+    const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output);
+    REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                 "SCATTER_ND_ADD OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                 numOfBadIndx);
+  }
+
+  if (!block.isInplace()) output->assign(input);
+
+  helpers::scatterND(block.launchContext(), pairwise::Add, *indices, *updates, *output, lock);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(scatter_nd_add) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_nd_sub.cpp

@@ -0,0 +1,101 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 24.08.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_nd_sub)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(scatter_nd_sub, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  const LongType indLastDim = indices->sizeAt(-1);
+
+  REQUIRE_TRUE(
+      indLastDim <= inRank, 0,
+      "SCATTER_ND_SUB OP: the last dimension of indices array must be <= input_array_rank, but got %i instead !",
+      indLastDim);
+  REQUIRE_TRUE(
+      updRank == (indRank - 1 + inRank - indLastDim), 0,
+      "SCATTER_ND_SUB OP: the equality updates_rank = (indices_rank - 1 + input_rank - last_indices_dimension) must be "
+      "true for input arrays, but got instead: updates_rank = %i, indices_rank = %i, last_indices_dimension = %i !",
+      updRank, indRank, indLastDim);
+
+  auto* inShapePtr = input->getShapeAsVector();
+  std::vector<LongType> inShape = *inShapePtr;
+  delete inShapePtr;
+  auto* updShapePtr = updates->getShapeAsVector();
+  std::vector<LongType> updShape = *updShapePtr;
+  delete updShapePtr;
+  auto* indShapePtr = indices->getShapeAsVector();
+  std::vector<LongType> indShape = *indShapePtr;
+  delete indShapePtr;
+  std::vector<LongType> expectedUpdShape(std::begin(indShape), std::end(indShape) - 1);
+  if (inRank > indLastDim)
+    std::move(std::begin(inShape) + indLastDim, std::end(inShape), std::back_inserter(expectedUpdShape));
+  REQUIRE_TRUE(expectedUpdShape == updShape, 0,
+               "SCATTER_ND_SUB OP: wrong shape of updates array, expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(updShape).c_str());
+
+  if (checkIndices) {
+    const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output);
+    REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                 "SCATTER_ND_SUB OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                 numOfBadIndx);
+  }
+
+  if (!block.isInplace()) output->assign(input);
+
+  helpers::scatterND(block.launchContext(), pairwise::Subtract, *indices, *updates, *output, lock);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(scatter_nd_sub) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_nd_update.cpp

@@ -0,0 +1,102 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 24.08.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_nd_update)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(scatter_nd_update, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  const bool lock = block.getBArguments()->empty() ? true : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  const LongType indLastDim = indices->sizeAt(-1);
+
+  REQUIRE_TRUE(
+      indLastDim <= inRank, 0,
+      "SCATTER_ND_UPDATE OP: the last dimension of indices array must be <= input_array_rank, but got %i instead !",
+      indLastDim);
+  REQUIRE_TRUE(
+      updRank == (indRank - 1 + inRank - indLastDim), 0,
+      "SCATTER_ND_UPDATE OP: the equality updates_rank = (indices_rank - 1 + input_rank - last_indices_dimension) must "
+      "be true for input arrays, but got instead: updates_rank = %i, indices_rank = %i, last_indices_dimension = %i !",
+      updRank, indRank, indLastDim);
+
+  auto* inShapePtr = input->getShapeAsVector();
+  std::vector<LongType> inShape = *inShapePtr;
+  delete inShapePtr;
+  auto* updShapePtr = updates->getShapeAsVector();
+  std::vector<LongType> updShape = *updShapePtr;
+  delete updShapePtr;
+  auto* indShapePtr = indices->getShapeAsVector();
+  std::vector<LongType> indShape = *indShapePtr;
+  delete indShapePtr;
+  std::vector<LongType> expectedUpdShape(std::begin(indShape), std::end(indShape) - 1);
+  if (inRank > indLastDim)
+    std::move(std::begin(inShape) + indLastDim, std::end(inShape), std::back_inserter(expectedUpdShape));
+  REQUIRE_TRUE(expectedUpdShape == updShape, 0,
+               "SCATTER_ND_UPDATE OP: wrong shape of updates array, expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(updShape).c_str());
+
+  if (checkIndices) {
+    const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output);
+    REQUIRE_TRUE(
+        numOfBadIndx == 0, 0,
+        "SCATTER_ND_UPDATE OP: please check elements of indices-array, total number of wrong elements is %lld!",
+        numOfBadIndx);
+  }
+
+  if (!block.isInplace()) output->assign(input);
+
+  helpers::scatterND(block.launchContext(), pairwise::CopyPws, *indices, *updates, *output, lock);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(scatter_nd_update) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_sub.cpp

@@ -0,0 +1,115 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Created by raver119 on 24.11.17.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_sub)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(scatter_sub, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  const bool lock = block.getBArguments()->empty() ? false : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  REQUIRE_TRUE(inRank > 0, 0, "SCATTER_SUB OP: input should not be scalar !");
+
+  if (inRank == 1) {
+    REQUIRE_TRUE(indices->isSameShape(updates), 0,
+                 "SCATTER_SUB OP: when input array has rank = 1 then indices and updates must have the same shapes, "
+                 "but got %s and %s correspondingly !",
+                 ShapeUtils::shapeAsString(indices).c_str(), ShapeUtils::shapeAsString(updates).c_str());
+  } else if (inRank == updRank && indices->isVector()) {
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = {indices->lengthOf()};
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_SUB OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+  }
+
+  else {
+    REQUIRE_TRUE(updRank == indRank + inRank - 1, 0,
+                 "SCATTER_SUB OP: wrong rank of updates array, expected is %i, but got %i instead !",
+                 indRank + inRank - 1, updRank);
+
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    auto* indShapeVec = indices->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = *indShapeVec;
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_SUB OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+    delete indShapeVec;
+  }
+
+  if (!indices->isEmpty()) {
+    if (checkIndices) {
+      const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output, 0);
+      REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                   "SCATTER_SUB OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                   numOfBadIndx);
+    }
+
+    // ScatterHelper<T>::template scatterApply<simdOps::Subtract<T>>(output, indices, updates);
+    helpers::scatter(block.launchContext(), pairwise::Subtract, *indices, *updates, *output, lock);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(ScatterSub, scatter_sub);
+
+DECLARE_TYPES(scatter_sub) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_upd.cpp

@@ -0,0 +1,111 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_upd)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/ScatterHelper.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(scatter_upd, 3, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto updates = INPUT_VARIABLE(2);
+  if(indices->isEmpty())
+    return Status::OK;
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) output->assign(input);
+
+  const bool lock = block.getBArguments()->empty() ? true : B_ARG(0);
+  const bool checkIndices = block.getBArguments()->size() <= 1 ? false : B_ARG(1);
+
+  const int inRank = input->rankOf();
+  const int indRank = indices->rankOf();
+  const int updRank = updates->rankOf();
+
+  REQUIRE_TRUE(inRank > 0, 0, "SCATTER_UPD OP: input should not be scalar !");
+
+  if (inRank == 1) {
+    REQUIRE_TRUE(indices->isSameShape(updates), 0,
+                 "SCATTER_UPD OP: when input array has rank = 1 then indices and updates must have the same shapes, "
+                 "but got %s and %s correspondingly !",
+                 ShapeUtils::shapeAsString(indices).c_str(), ShapeUtils::shapeAsString(updates).c_str());
+  } else if (inRank == updRank && indices->isVector()) {
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = {indices->lengthOf()};
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_UPD OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+  } else {
+    REQUIRE_TRUE(updRank == indRank + inRank - 1, 0,
+                 "SCATTER_UPD OP: wrong rank of updates array, expected is %i, but got %i instead !",
+                 indRank + inRank - 1, updRank);
+
+    auto* updShapeVec = updates->getShapeAsVector();
+    auto* inShapeVec = input->getShapeAsVector();
+    auto* indShapeVec = indices->getShapeAsVector();
+    std::vector<LongType> expectedUpdShape = *indShapeVec;
+    expectedUpdShape.insert(expectedUpdShape.end(), inShapeVec->begin() + 1, inShapeVec->end());
+
+    REQUIRE_TRUE(expectedUpdShape == *updShapeVec, 0,
+                 "SCATTER_UPD OP: wrong shape of updates array, expected is %s, but got %s instead !",
+                 ShapeUtils::shapeAsString(expectedUpdShape).c_str(), ShapeUtils::shapeAsString(*updShapeVec).c_str());
+    delete updShapeVec;
+    delete inShapeVec;
+    delete indShapeVec;
+  }
+
+  if (!indices->isEmpty()) {
+    if (checkIndices) {
+      const LongType numOfBadIndx = helpers::checkIndices(block.launchContext(), *indices, *output, 0);
+      REQUIRE_TRUE(numOfBadIndx == 0, 0,
+                   "SCATTER_UPD OP: please check elements of indices-array, total number of wrong elements is %lld!",
+                   numOfBadIndx);
+    }
+
+    helpers::scatter(block.launchContext(), pairwise::CopyPws, *indices, *updates, *output, lock);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(ScatterUpdate, scatter_upd);
+
+DECLARE_TYPES(scatter_upd) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/scatter_update.cpp

@@ -0,0 +1,60 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_scatter_update)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+/**
+ * scatter update operation
+ *
+ * IArgs map:
+ * IArgs[0] - update operation: 0 - add; 1 - sub; 2 - mul; 3 - div; 4 - rsub; 5 - rdiv; 6 - assign
+ * IArgs[1] - number of dimensions
+ * IArgs[...] - dimensions
+ * IArgs[...] - number of indices
+ * IArgs[...] - indices
+ *
+ * @tparam T
+ */
+CONFIGURABLE_OP_IMPL(scatter_update, -2, 1, true, 0, -2) {
+  //NOTE: DO NOT USE. USE scatter_upd instead.
+  auto operand = INPUT_VARIABLE(0);
+  auto updates = INPUT_VARIABLE(1);
+  if(updates->isEmpty())
+    return Status::OK;
+
+  helpers::scatterUpdate(block.launchContext(), *operand, *updates, block.getIArguments());
+
+  return Status::OK;
+}
+DECLARE_SYN(scatterupdate, scatter_update);
+
+DECLARE_TYPES(scatter_update) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/slice.cpp

@@ -0,0 +1,273 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 02.11.2017.
+//
+
+#include <system/op_boilerplate.h>
+#include <legacy/NativeOpExecutioner.h>
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+#if NOT_EXCLUDED(OP_slice)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(slice, 1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  int x_rank = input->rankOf();
+
+  std::vector<LongType> begin;
+  std::vector<LongType> sz;
+
+  if (block.width() == 3) {
+    auto b = INPUT_VARIABLE(1);
+    auto e = INPUT_VARIABLE(2);
+
+    begin = b->template asVectorT<LongType>();
+    sz = e->template asVectorT<LongType>();
+  } else {
+    REQUIRE_TRUE(block.numI() >= static_cast<size_t>(x_rank * 2), 0, "Number of IArgs should be equal to [%i] but got [%i] instead",
+                 x_rank * 2, block.numI());
+
+    ShapeUtils::copyVectorPart(begin, *(block.getIArguments()), x_rank, 0);
+    ShapeUtils::copyVectorPart(sz, *(block.getIArguments()), x_rank, x_rank);
+  }
+
+  REQUIRE_TRUE(begin.size() == static_cast<size_t>(x_rank), 0, "begin array should have length of [%i] but got [%i] instead", x_rank,
+               begin.size());
+  REQUIRE_TRUE(sz.size() == static_cast<size_t>(x_rank), 0, "size array should have length of [%i] but got [%i] instead", x_rank, sz.size());
+
+  std::vector<LongType> indices(2 * x_rank);
+  auto empty = false;
+  for (int e = 0; e < x_rank; e++) {
+    int size = sz[e];
+    int start = begin[e];
+
+    REQUIRE_TRUE(start >= 0, 0, "Slice: start index should not be negative");
+
+    REQUIRE_TRUE(start <= input->sizeAt(e), 0, "Index %i is invalid for dimension %i with size %i.", start, e,
+                 input->shapeInfo()[e + 1]);
+    if (size == -1) {
+      size = input->sizeAt(e) - start;
+    }
+    REQUIRE_TRUE(size >= 0, 0, "Slice: interval for dimension %i is less then 1");
+    REQUIRE_TRUE(start + size <= input->sizeAt(e), 0,
+                 "Slice: interval [%i, %i] is out of bounds for dimension %i with size %i", start, start + size, e,
+                 input->sizeAt(e));
+
+    if (start == input->sizeAt(e) || size == 0) {
+      empty = true;
+      // Don't break to perform input validation on other dims
+    }
+
+    indices[2 * e] = start;
+    indices[2 * e + 1] = start + size;
+  }
+
+  if (empty) {
+    REQUIRE_TRUE(output->isEmpty(), 0, "Slice: empty array indices requested, but output array is not empty");
+    return Status::OK;
+  }
+
+  LongType* subArrShapeInfo = nullptr;
+  ALLOCATE(subArrShapeInfo, block.getWorkspace(), shape::shapeInfoLength(input->rankOf()), sd::LongType);
+
+  LongType offset;
+
+  shape::calcSubArrShapeInfoAndOffset(indices.data(), input->shapeInfo(), subArrShapeInfo, offset, true);
+
+  auto subArrShapeInfoPack = ConstantShapeHelper::getInstance().bufferForShapeInfo(subArrShapeInfo);
+
+  NDArray::prepareSpecialUse({output}, {input});
+
+  NativeOpExecutioner::execTransformAny(block.launchContext(), transform::Assign, input->bufferWithOffset(offset),
+                                        subArrShapeInfoPack->primary(), input->specialBufferWithOffset(offset),
+                                        subArrShapeInfoPack->special(), output->buffer(), output->shapeInfo(),
+                                        output->specialBuffer(), output->specialShapeInfo(), nullptr, true);
+
+  NDArray::registerSpecialUse({output}, {input});
+
+  RELEASE(subArrShapeInfo, block.getWorkspace());
+
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(slice) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+DECLARE_SHAPE_FN(slice) {
+  auto inShape = inputShape->at(0);
+  if(shape::isEmptyConst(inShape)) {
+    std::vector<LongType> emptyShape = {0};
+    return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(ArrayOptions::dataType(inShape), emptyShape));
+  }
+  auto x_rank = shape::rank(inShape);
+
+  std::vector<LongType> begin;
+  std::vector<LongType> sz;
+
+  if (block.width() == 3) {
+    auto b = INPUT_VARIABLE(1);
+    auto e = INPUT_VARIABLE(2);
+
+    // Check if begin/end are empty - this can happen during graph construction
+    if (b->isEmpty() || e->isEmpty()) {
+      // For slicing a 1D shape tensor to extract a single element, return scalar
+      if (x_rank == 1) {
+        auto scalarShape = ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inShape));
+        return SHAPELIST(scalarShape);
+      }
+      // Otherwise cannot determine shape at compile time
+      std::vector<LongType> unknownShape(x_rank, -1);
+      auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), 'c', unknownShape);
+      return SHAPELIST(newShape);
+    }
+
+    begin = b->template asVectorT<LongType>();
+    sz = e->template asVectorT<LongType>();
+  } else {
+    REQUIRE_TRUE(block.numI() >= static_cast<size_t>(x_rank) * 2, 0, "Number of IArgs should be equal to [%i] but got [%i] instead",
+                 x_rank * 2, block.numI());
+
+    ShapeUtils::copyVectorPart(begin, *(block.getIArguments()), x_rank, 0);
+    ShapeUtils::copyVectorPart(sz, *(block.getIArguments()), x_rank, x_rank);
+  }
+
+  REQUIRE_TRUE(begin.size() == static_cast<size_t>(x_rank), 0, "Begin array should have length of [%i] but got [%i] instead", x_rank,
+               begin.size());
+  REQUIRE_TRUE(sz.size() == static_cast<size_t>(x_rank), 0, "Size array should have length of [%i] but got [%i] instead", x_rank, sz.size());
+
+  std::vector<LongType> shape;
+  auto empty = false;
+  for (int e = 0; e < x_rank; e++) {
+    auto size = sz[e];
+    auto start = begin[e];
+
+    // Handle unknown/dynamic dimensions
+    if (inShape[e + 1] < 0) {
+      shape.emplace_back(-1);
+      continue;
+    }
+
+    if (size == -1) {
+      size = inShape[e + 1] - start;
+    }
+
+    // Bounds checking. Note that begin[i] == size[i] means empty array
+    REQUIRE_TRUE(
+        start >= 0 && start <= inShape[e + 1], 0,
+        "Invalid begin[%i] value: Begin must satisfy 0 <= begin <= size[i], got begin=%i for dimension size %i", e,
+        start, inShape[e + 1]);
+    REQUIRE_TRUE(size == -1 || size >= 0, 0,
+                 "Invalid size[%i] value: must be positive (or -1 for 'all remaining'), got %i", e, size,
+                 inShape[e + 1]);
+    REQUIRE_TRUE(
+        start >= 0 && start <= inShape[e + 1], 0,
+        "Invalid begin[%i] value: Begin must satisfy 0 <= begin <= size[i], got begin=%i for dimension size %i", e,
+        start, inShape[e + 1]);
+    REQUIRE_TRUE(start + size <= inShape[e + 1], 0,
+                 "Slice: interval [%i, %i] is out of bounds for dimension %i with size %i", start, start + size, e,
+                 inShape[e + 1]);
+    if (start == inShape[e + 1]) {
+      size = 0;
+    }
+
+    shape.emplace_back(size);
+  }
+
+  // Special case: slicing a 1D tensor with size 1 should produce a scalar
+  if (x_rank == 1 && shape.size() == 1 && shape[0] == 1) {
+    auto scalarShape = ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inShape));
+    return SHAPELIST(scalarShape);
+  }
+
+  if(shape.size() == 1 && shape[0] == 0) {
+    std::vector<LongType> emptyShape = {0};
+    return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(ArrayOptions::dataType(inShape), emptyShape));
+  }
+
+  auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), 'c', shape);
+  return SHAPELIST(newShape);
+}
+
+DECLARE_TYPES(slice_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(slice_bp, 2, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsNext = block.width() == 4 ? INPUT_VARIABLE(3) : INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+  double zero = 0.;
+  output->assign(zero);
+  int x_rank = input->rankOf();
+
+  std::vector<LongType> begin;
+  std::vector<LongType> end;
+
+  if (block.width() == 4) {
+    auto b = INPUT_VARIABLE(1);
+    auto e = INPUT_VARIABLE(2);
+
+    begin = b->template asVectorT<LongType>();
+    end = e->template asVectorT<LongType>();
+  } else {
+    REQUIRE_TRUE(block.numI() >= static_cast<size_t>(x_rank) * 2, 0, "Number of IArgs should be equal to [%i] but got [%i] instead",
+                 x_rank * 2, block.numI());
+
+    ShapeUtils::copyVectorPart(begin, *(block.getIArguments()), x_rank, 0);
+    ShapeUtils::copyVectorPart(end, *(block.getIArguments()), x_rank, x_rank);
+  }
+
+  REQUIRE_TRUE(begin.size() == static_cast<size_t>(x_rank), 0, "begin array should have length of [%i] but got [%i] instead", x_rank,
+               begin.size());
+  REQUIRE_TRUE(end.size() == static_cast<size_t>(x_rank), 0, "end array should have length of [%i] but got [%i] instead", x_rank,
+               end.size());
+
+  std::vector<LongType> indices(2 * x_rank);
+  for (int e = 0; e < x_rank; e++) {
+    int size = end[e];
+    int start = begin[e];
+
+    if (size == -1) {  //-1 means all remaining values
+      size = input->sizeAt(e) - start;
+    }
+    REQUIRE_TRUE(size > 0, 0, "Slice: interval for dimension %i is less then 1", e);
+
+    indices[2 * e] = start;
+    indices[2 * e + 1] = start + size;
+  }
+  auto sub = (*output)(indices, true);
+  sub->assign(epsNext);
+
+  delete sub;
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(slice_bp) {
+  auto inShape = inputShape->at(0);
+  return SHAPELIST(CONSTANT(inShape));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/space_to_batch.cpp

@@ -0,0 +1,117 @@
+// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_space_to_batch)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/s_t_b.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(space_to_batch, 2, 1, false, 0, 1) {
+  // [bS, iH, iW, iC] is rearranged/permuted to [bS*blockSize*blockSize, (iH + padBottom + padTop)/blockSize, (iW +
+  // padLeft + padRight)/blockSize, iC]
+
+  auto input = INPUT_VARIABLE(0);
+  auto padding = INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  const LongType blockSize = INT_ARG(0);
+  REQUIRE_TRUE(blockSize >= 2, 0, "SpaceToBatch: integer parameter block_size must be >= 2, but got %i instead",
+               blockSize);
+
+  REQUIRE_TRUE(input->rankOf() == 4, 0, "SpaceToBatch: rank of input array must be equal 4, but got %i instead",
+               input->rankOf());
+  REQUIRE_TRUE(output->rankOf() == 4, 0, "SpaceToBatch: rank of output array must be equal 4, but got %i instead",
+               output->rankOf());
+
+  if (padding->sizeAt(0) != 2 || padding->sizeAt(1) != 2)
+    REQUIRE_TRUE(false, 0, "SpaceToBatch: operation expects padding shape to be {2, 2}, but got %s instead",
+                 ShapeUtils::shapeAsString(padding).c_str());
+
+  const LongType padBottom = padding->e<LongType>(0, 0);
+  const LongType padTop = padding->e<LongType>(0, 1);
+  const LongType padLeft = padding->e<LongType>(1, 0);
+  const LongType padRight = padding->e<LongType>(1, 1);
+
+  REQUIRE_TRUE(
+      (input->sizeAt(1) + padBottom + padTop) % blockSize == 0 &&
+          (input->sizeAt(2) + padLeft + padRight) % blockSize == 0,
+      0, "SpaceToBatch: after padding, second and third dimensions of input array must be divisible by blockSize !");
+
+  if (shape::strideDescendingCAscendingF(input->shapeInfo()))
+    helpers::spaceToBatch(block.launchContext(), *input, *output, padBottom, padTop, padLeft, padRight, blockSize);
+  else {
+    NDArray *inputDup = input->dup(input->ordering());
+    helpers::spaceToBatch(block.launchContext(), *inputDup, *output, padBottom, padTop, padLeft, padRight,
+                          blockSize);
+  }
+  return Status::OK;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(space_to_batch) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedInputTypes(1, {ALL_INTS})->setSameMode(true);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(space_to_batch) {
+  auto inputShapeInfo = inputShape->at(0);
+  auto paddingShapeInfo = inputShape->at(1);
+
+  const LongType blockSize = INT_ARG(0);
+  REQUIRE_TRUE(blockSize >= 2, 0, "SpaceToBatch: integer parameter block_size must be >= 2, but got %i instead",
+               blockSize);
+
+  const int rank = inputShapeInfo[0];
+  REQUIRE_TRUE(rank == 4, 0, "SpaceToBatch: rank of input array must be equal 4, but got %i instead", rank);
+
+  if (paddingShapeInfo[1] != 2 || paddingShapeInfo[1] != 2)
+    REQUIRE_TRUE(false, 0, "SpaceToBatch: operation expects padding shape to be {2, 2}, but got %s instead",
+                 ShapeUtils::shapeAsString(paddingShapeInfo).c_str());
+
+  const LongType padBottom = INPUT_VARIABLE(1)->e<LongType>(0, 0);
+  const LongType padTop = INPUT_VARIABLE(1)->e<LongType>(0, 1);
+  const LongType padLeft = INPUT_VARIABLE(1)->e<LongType>(1, 0);
+  const LongType padRight = INPUT_VARIABLE(1)->e<LongType>(1, 1);
+
+  REQUIRE_TRUE(
+      (inputShapeInfo[2] + padBottom + padTop) % blockSize == 0 &&
+          (inputShapeInfo[3] + padLeft + padRight) % blockSize == 0,
+      0, "SpaceToBatch: after padding, second and third dimensions of input array must be divisible by blockSize !");
+
+  return SHAPELIST(ConstantShapeHelper::getInstance().createShapeInfo(
+      ArrayOptions::dataType(inputShapeInfo), 'c',
+      {inputShapeInfo[1] * blockSize * blockSize, (inputShapeInfo[2] + padBottom + padTop) / blockSize,
+       (inputShapeInfo[3] + padLeft + padRight) / blockSize, inputShapeInfo[4]}));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/space_to_batch_nd.cpp

@@ -0,0 +1,123 @@
+
+// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_space_to_batch_nd)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/s_t_b.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(space_to_batch_nd, 3, 1, false, 0, 0) {
+  // 4D example, numOfSpatialDims = 2 - two spatial dimensions
+  // [bS, iH, iW, iC] is rearranged/permuted to [bS*blockShape[0]*blockShape[1], (iH + padBottom + padTop)/blockSize[0],
+  // (iW + padLeft + padRight)/blockSize[1], iC]
+
+  auto input = INPUT_VARIABLE(0);
+  auto blockShape = INPUT_VARIABLE(1);
+  auto padding = INPUT_VARIABLE(2);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(blockShape->rankOf() == 1, 0,
+               "SpaceToBatchND: rank of blockShape array must be equal to one, but got %i instead !",
+               blockShape->rankOf());
+
+  const LongType numOfSpatialDims = blockShape->sizeAt(0);
+
+  REQUIRE_TRUE(input->rankOf() == output->rankOf(), 0,
+               "SpaceToBatchND: rank of input and output array must be the same, but got %i and %i correspondingly !",
+               input->rankOf(), output->rankOf());
+
+  if (padding->sizeAt(0) != numOfSpatialDims || padding->sizeAt(1) != 2) {
+    const std::string expectedpaddingShape = "[" + std::to_string(numOfSpatialDims) + ", 2]";  // [numOfSpatialDims, 2]
+    REQUIRE_TRUE(false, 0, "SpaceToBatchND: operation expects padding shape to be %s, but got %s instead",
+                 expectedpaddingShape.c_str(), ShapeUtils::shapeAsString(padding).c_str());
+  }
+
+  // FIXME - should we use this time-consuming validation ?
+  for (LongType i = 0; i < numOfSpatialDims; ++i) {
+    const LongType padLeft = padding->e<LongType>(i, 0);
+    const LongType padRight = padding->e<LongType>(i, 1);
+    const LongType blockSize = blockShape->e<LongType>(i);
+    REQUIRE_TRUE((input->sizeAt(i + 1) + padLeft + padRight) % blockSize == 0, 0,
+                 "SpaceToBatchND: after padding, spatial dimensions of input array must be divisible by blockSize !");
+  }
+
+  if (shape::strideDescendingCAscendingF(input->shapeInfo()))
+    helpers::spaceToBatchND(block.launchContext(), *input, *blockShape, *padding, *output);
+  else {
+    NDArray *inputDup = input->dup(input->ordering());
+    helpers::spaceToBatchND(block.launchContext(), *inputDup, *blockShape, *padding, *output);
+    delete inputDup;
+  }
+  return Status::OK;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(space_to_batch_nd) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS})
+      ->setSameMode(true);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(space_to_batch_nd) {
+  auto inputShapeInfo = inputShape->at(0);
+  auto blockShapeInfo = inputShape->at(1);
+  auto paddingShapeInfo = inputShape->at(2);
+
+  REQUIRE_TRUE(blockShapeInfo[0] == 1, 0,
+               "SpaceToBatchND: rank of blockShape array must be equal to one, but got %i instead !",
+               blockShapeInfo[0]);
+
+  const LongType numOfSpatialDims = blockShapeInfo[1];
+
+  if (paddingShapeInfo[1] != numOfSpatialDims || paddingShapeInfo[2] != 2) {
+    const std::string expectedpaddingShape = "[" + std::to_string(numOfSpatialDims) + ", 2]";  // [numOfSpatialDims, 2]
+    REQUIRE_TRUE(false, 0, "SpaceToBatchND: operation expects padding shape to be %s, but got %s instead",
+                 expectedpaddingShape.c_str(), ShapeUtils::shapeAsString(paddingShapeInfo).c_str());
+  }
+
+  std::vector<LongType> outShape(inputShapeInfo + 1, inputShapeInfo + 1 + inputShapeInfo[0]);
+  auto prod = INPUT_VARIABLE(1)->reduceNumber(reduce::Prod);
+  outShape[0] *= prod->e<LongType>(0);
+  delete prod;
+  for (LongType i = 0; i < numOfSpatialDims; ++i)
+    outShape[i + 1] =
+        (outShape[i + 1] + INPUT_VARIABLE(2)->e<LongType>(i, 0) + INPUT_VARIABLE(2)->e<LongType>(i, 1)) /
+        INPUT_VARIABLE(1)->e<LongType>(i);
+
+  return SHAPELIST(
+      ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inputShapeInfo), 'c', outShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/space_to_depth.cpp

@@ -0,0 +1,98 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_space_to_depth)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/s_t_d.h>
+
+#include <array>
+
+namespace sd {
+namespace ops {
+
+
+    DECLARE_TYPES(space_to_depth) {
+        getOpDescriptor()
+                ->setAllowedInputTypes(sd::DataType::ANY)
+                ->setSameMode(true);
+    }
+
+    CUSTOM_OP_IMPL(space_to_depth, 1, 1, false, 0, 2) {
+        int block_size = INT_ARG(0);
+        REQUIRE_TRUE(block_size > 0,0, "SpaceToDepth: input should be > 0");
+
+        bool isNHWC = INT_ARG(1) == 1;
+
+        auto input = INPUT_VARIABLE(0);
+
+        REQUIRE_TRUE(input->rankOf() == 4, 0, "SpaceToDepth: input should be 4D array, but got %f instead", input->rankOf());
+
+        int bS = input->sizeAt(0);
+        int iD = isNHWC ? input->sizeAt(3) : input->sizeAt(1);
+        int iH = isNHWC ? input->sizeAt(1) : input->sizeAt(2);
+        int iW = isNHWC ? input->sizeAt(2) : input->sizeAt(3);
+
+        REQUIRE_TRUE(iH % block_size == 0 && iW % block_size == 0, 0, "SpaceToDepth: input Height & Width should be divisible by block_size");
+
+        auto output = OUTPUT_VARIABLE(0);
+
+        if (shape::strideDescendingCAscendingF(input->shapeInfo()))
+            helpers::_spaceTodepth(block.launchContext(), *input, output, block_size, isNHWC);
+        else {
+          NDArray *inputDup = input->dup(input->ordering());
+          helpers::_spaceTodepth(block.launchContext(), *inputDup, output, block_size, isNHWC);
+        }
+        return Status::OK;
+    }
+    
+
+    DECLARE_SHAPE_FN(space_to_depth) {
+        auto in = inputShape->at(0);
+        int block_size = INT_ARG(0);
+        REQUIRE_TRUE(block_size > 0,0, "SpaceToDepth: input should be > 0");
+        bool isNHWC = INT_ARG(1) == 1;
+
+        int bS = shape::sizeAt(in, static_cast<sd::LongType>(0));
+        int iD = isNHWC ? shape::sizeAt(in, static_cast<sd::LongType>(3)) : shape::sizeAt(in, static_cast<sd::LongType>(1));
+        int iH = isNHWC ? shape::sizeAt(in, static_cast<sd::LongType>(1)) : shape::sizeAt(in, static_cast<sd::LongType>(2));
+        int iW = isNHWC ? shape::sizeAt(in, static_cast<sd::LongType>(2)) : shape::sizeAt(in, static_cast<sd::LongType>(3));
+
+        int oD = iD * block_size * block_size;
+        int oH = iH / block_size;
+        int oW = iW / block_size;
+        
+        std::array<sd::LongType, 4> shape;
+        if (isNHWC) 
+            shape = {{bS, oH, oW, oD }};
+        else 
+            shape = {{bS, oD, oH, oW }};
+
+        auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(in), 'c', 4, shape.data(),0);
+        return SHAPELIST(newShape);
+    }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/split.cpp

@@ -0,0 +1,149 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_split)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/transforms.h>
+
+#include <array>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(split, 1, -1, false, 0, 1) {
+  NDArray *input = nullptr;
+  int num_splits = INT_ARG(0);
+
+  // axis is 0 by default
+  sd::LongType axis = 0;
+
+  if (block.width() == 1) {
+    input = INPUT_VARIABLE(0);
+  } else {
+    auto a = INPUT_VARIABLE(0);
+    auto b = INPUT_VARIABLE(1);
+
+    if (a->isScalar()) {
+      // axis goes first
+      axis = a->e<sd::LongType>(0);
+      input = b;
+    } else if (b->isScalar()) {
+      axis = b->e<sd::LongType>(0);
+      input = a;
+    }
+  }
+
+  // Edge case: splitting empty array (mainly for TF import compatibility) -> return N empty arrays
+  if (input->isEmpty()) {
+    for (int i = 0; i < num_splits; i++) {
+      REQUIRE_TRUE(OUTPUT_VARIABLE(i)->isEmpty(), 0,
+                   "Split: When input array is empty, all output arrays must be empty");
+    }
+    // No op
+    return sd::Status::OK;
+  }
+
+  if (block.numI() == 2) axis = INT_ARG(1);
+
+  if (axis < 0) axis += input->rankOf();
+
+  REQUIRE_TRUE(input->sizeAt(axis) % num_splits == 0, 0,
+               "Split: num_splits has wrong value, remainder of division should be 0, but it's %i",
+               input->sizeAt(axis) % num_splits);
+
+  std::vector<NDArray *> outArrs(num_splits);
+  for (int e = 0; e < num_splits; e++) {
+    outArrs[e] = OUTPUT_VARIABLE(e);
+  }
+
+  helpers::split(block.launchContext(), *input, outArrs, axis);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(split) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(split) {
+  int num_splits = INT_ARG(0);
+  auto input = inputShape->at(0);
+  sd::DataType dataType = ArrayOptions::dataType(input);
+
+  // axis is 0 by default
+  int axis = 0;
+
+  int inputVar = 0;
+  if (inputShape->size() != 1) {
+    auto shape0 = inputShape->at(0);
+    auto shape1 = inputShape->at(1);
+
+    if (shape::isScalar(shape0)) {
+      input = shape1;
+      auto _a = INPUT_VARIABLE(0);
+      axis = _a->e<sd::LongType>(0);
+      dataType = ArrayOptions::dataType(shape1);
+      inputVar = 1;
+    } else if (shape::isScalar(shape1)) {
+      input = shape0;
+      auto _a = INPUT_VARIABLE(1);
+      axis = _a->e<sd::LongType>(0);
+      dataType = ArrayOptions::dataType(shape0);
+      inputVar = 0;
+    }
+  }
+
+  auto shapes = SHAPELIST();
+
+  // Edge case: splitting empty array (mainly for TF import compatibility) -> return N empty arrays
+   if(INPUT_VARIABLE(inputVar)->isEmpty()) {
+       for (int e = 0; e < num_splits; e++) {
+                 auto empty = ConstantShapeHelper::getInstance().emptyShapeInfo(dataType);
+           shapes->push_back(empty);
+       }
+       return shapes;
+   }
+
+  if (block.numI() == 2) axis = INT_ARG(1);
+
+  if (axis < 0) axis += shape::rank(input);
+
+  std::vector<sd::LongType> shape(shape::rank(input));
+
+  for (sd::LongType e = 0; e < shape::rank(input); e++)
+    if (e == axis)
+      shape[e] = shape::sizeAt(input, e) / num_splits;
+    else
+      shape[e] = shape::sizeAt(input, e);
+
+  for (int e = 0; e < num_splits; e++) {
+    auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(dataType, shape::order(input), shape);
+    shapes->push_back(newShape);
+  }
+
+  return shapes;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/split_v.cpp

@@ -0,0 +1,128 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_split_v)
+
+#include <ops/declarable/headers/parity_ops.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(split_v, 2, -1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto sizes = INPUT_VARIABLE(1);
+
+  int axis = 0;
+
+  if (block.getIArguments()->size() > 0) {
+    axis = INT_ARG(0);
+  } else if (block.width() > 2) {
+    auto _a = INPUT_VARIABLE(2);
+    axis = _a->e<int>(0);
+  }
+
+  if (axis < 0) axis += input->rankOf();
+
+  std::vector<sd::LongType> axisVec = {axis};
+
+  int pos = 0;
+  std::vector<sd::LongType> indices(2 * input->rankOf());
+
+  for (sd::LongType e = 0; e < sizes->lengthOf(); e++) {
+    int c_size = sizes->e<int>(e);
+
+    for (int d = 0; d < input->rankOf(); d++) {
+      if (d == axis)
+        indices[2 * d + 1] = (indices[2 * d] = pos) + c_size;
+      else
+        indices[2 * d] = indices[2 * d + 1] = 0;
+    }
+
+    auto output = OUTPUT_VARIABLE(e);
+    REQUIRE_TRUE(output->dataType() == input->dataType(), 0, "SplitV: all outputs must have same data type as input");
+
+    auto sub = (*input)(indices);
+
+    output->assign(sub);
+
+    delete sub;
+    pos += c_size;
+  }
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(split_v) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(split_v) {
+  auto input = inputShape->at(0);
+  // auto sizes = inputShape->at(1);
+
+  auto shapeList = SHAPELIST();
+  int rank = shape::rank(input);
+
+  // 0 is just default axis
+  int axis = 0;
+
+  if (block.getIArguments()->size() > 0)
+    axis = INT_ARG(0);
+  else if (block.width() > 2) {
+    auto _a = INPUT_VARIABLE(2);
+    axis = _a->e<int>(0);
+  }
+
+  if (axis < 0) axis += shape::rank(input);
+
+  // this op assumes we have sizes defined
+  auto sizes = INPUT_VARIABLE(1);
+
+  auto length = sizes->lengthOf();
+  int pos = 0;
+  for (sd::LongType e = 0; e < length; e++) {
+    int c_size = sizes->e<int>(e);
+
+    std::vector<sd::LongType> shape(rank);
+
+    for (sd::LongType d = 0; d < rank; d++) {
+      if (d != axis)
+        shape[d] = shape::sizeAt(input, d);
+      else
+        shape[d] = c_size;
+    }
+
+    auto newShape =
+        ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(input), shape::order(input), shape);
+    shapeList->push_back(newShape);
+  }
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/stack.cpp

@@ -0,0 +1,96 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 01.11.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_stack)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/stack.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(stack, -1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  int dim = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+  if (dim < 0) dim += input->rankOf() + 1;
+
+  // no-op in case of empty output array
+  if (output->isEmpty()) return Status::OK;
+
+  // input validation
+  // check whether shapes of all input array are the same
+  for (size_t i = 0; i < block.width() - 1; ++i)
+  REQUIRE_TRUE(shape::equalsSoft((INPUT_VARIABLE(i))->shapeInfo(), (INPUT_VARIABLE(i + 1))->shapeInfo()), 0,
+               "STACK op: the shapes of all input arrays must be the same !");
+
+  REQUIRE_TRUE(
+      dim <= input->rankOf(), 0,
+      "STACK op: the input dimension parameter must be <= rank of input arrays shapes (rank=%i), but got %i instead !",
+      input->shapeOf(), dim);
+
+  std::vector<NDArray*> inArrs(block.width());
+  for (size_t i = 0; i < block.width(); ++i) inArrs[i] = INPUT_VARIABLE(i);
+
+  //empty arrays are a no op
+  if(block.width() >= 1 && !inArrs[0]->isEmpty())
+    helpers::stack(block.launchContext(), inArrs, *output, dim);
+
+  return Status::OK;
+}
+DECLARE_SYN(pack, stack);
+DECLARE_SYN(Pack, stack);
+
+DECLARE_TYPES(stack) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY);
+}
+
+DECLARE_SHAPE_FN(stack) {
+  // check whether input dimension is within rank range
+  auto inShapeInfo = inputShape->at(0);
+
+  int rank = shape::rank(inShapeInfo);
+  int dim = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+  if (dim < 0) dim += rank + 1;
+
+  REQUIRE_TRUE(
+      dim <= inShapeInfo[0], 0,
+      "STACK op: the input dimension parameter must be <= rank of input arrays shapes (rank=%i), but got %i instead !",
+      inShapeInfo[0], dim);
+
+
+  // the rank of output ShapeInfo is larger by one compared to input ShapeInfo
+  std::vector<LongType> outShape(inShapeInfo + 1, inShapeInfo + 1 + rank);
+
+  // insert (int) block.width() at dim position of input shape to get output shape
+  outShape.insert(outShape.begin() + LongType(dim), (LongType)block.width());
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().bufferForShapeInfo(ArrayOptions::dataType(inShapeInfo),
+                                                                             shape::order(inShapeInfo),
+                                                                             outShape)->primary());
+  return ret;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/standardize.cpp

@@ -0,0 +1,175 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Paul Dubs
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_standardize)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/reverse.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(standardize, 1, 1, true, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  output->nullify();
+  std::vector<sd::LongType> axis;
+
+  if (block.width() > 1)
+    axis = INPUT_VARIABLE(1)->template asVectorT<sd::LongType>();
+  else if (block.numI() > 0)
+    axis = *block.getIArguments();
+
+  REQUIRE_TRUE(!axis.empty(), 0, "STANDARDIZE OP: axis has to be non-empty")
+
+  shape::checkDimensions(input->rankOf(), &axis);
+
+  // Compute mean with keepDims=true for broadcasting
+  auto means = input->reduceAlongDimension(reduce::Mean, &axis, true);
+
+  // Compute VARIANCE (not stdev) - uses Welford's algorithm internally
+  // biasCorrected=false gives population variance (divide by N, not N-1)
+  auto varianceRaw = input->varianceAlongDimension(variance::SummaryStatsVariance, false, &axis);
+
+  // Reshape variance to match means shape for broadcasting (use reshape instead of reshapei)
+  auto meansShape = means->getShapeAsVector();
+  auto variance = varianceRaw->reshape(varianceRaw->ordering(), *meansShape);
+  delete meansShape;
+  delete varianceRaw;
+
+  // Add epsilon BEFORE sqrt: stdev = sqrt(variance + epsilon)
+  // This is the numerically stable formula for LayerNorm
+  NDArray* varPlusEps = *variance + 1e-5;
+  NDArray* stdev = varPlusEps->transform(transform::Sqrt);
+
+  // output = (input - mean) / stdev
+  input->applyTrueBroadcast(sd::BroadcastOpsTuple::Subtract(), means, output, false);
+  output->applyTrueBroadcast(sd::BroadcastOpsTuple::Divide(), stdev, output, false);
+
+  delete means;
+  delete variance;
+  delete varPlusEps;
+  delete stdev;
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(standardize) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_FLOATS});
+  getOpDescriptor()->setAllowedInputTypes(1, {DataType::INT32, DataType::INT64});
+  getOpDescriptor()->setAllowedOutputTypes(0, DataType::INHERIT);
+}
+
+CUSTOM_OP_IMPL(standardize_bp, 2, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto eps = block.width() == 3 ? INPUT_VARIABLE(2) : INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+  std::vector<sd::LongType> axis;
+
+  if (block.width() == 3)
+    axis = INPUT_VARIABLE(1)->template asVectorT<sd::LongType>();
+  else if (block.numI() > 0)
+    axis = *block.getIArguments();
+
+  REQUIRE_TRUE(!axis.empty(), 0, "STANDARDIZE OP: axis has to be non-empty")
+
+  shape::checkDimensions(input->rankOf(), &axis);
+  auto longAxis = ArrayUtils::toLongVector(axis);
+
+  auto means = input->reduceAlongDimension(reduce::Mean, &axis, true);
+  REQUIRE_TRUE(means != nullptr, 0, "STANDARDIZE_BP OP: failed to compute mean along dimension");
+
+  auto stdevRaw = input->varianceAlongDimension(variance::SummaryStatsStandardDeviation, false, &axis);
+  REQUIRE_TRUE(stdevRaw != nullptr, 0, "STANDARDIZE_BP OP: failed to compute standard deviation along dimension");
+
+  // Reshape stdev to match means shape for broadcasting (use reshape instead of reshapei)
+  auto meansShape = means->getShapeAsVector();
+  auto stdev = stdevRaw->reshape(stdevRaw->ordering(), *meansShape);
+  delete meansShape;
+  delete stdevRaw;
+
+  eps->applyTrueBroadcast(sd::BroadcastOpsTuple::Divide(), stdev, output, false);
+
+  auto sum = output->reduceAlongDimension(reduce::Sum, &axis, true);
+  REQUIRE_TRUE(sum != nullptr, 0, "STANDARDIZE_BP OP: failed to compute sum along dimension");
+
+  NDArray dldu_sum = -(*sum);
+
+  NDArray dldx_u(input->shapeInfo(), false, block.launchContext());
+  std::vector<NDArray *> meanBpArgs = {input, &dldu_sum};
+  std::vector<NDArray *> meanBpOutput = {&dldx_u};
+  std::vector<double> meanBpTArgs = {};
+  std::vector<bool> meanBpBArgs = {};
+
+  sd::ops::reduce_mean_bp meanBp;
+  meanBp.execute(meanBpArgs, meanBpOutput, meanBpTArgs, longAxis, meanBpBArgs);
+  *output += dldx_u;
+
+  // (eps * (means - input) / (stdev * stdev))
+  NDArray tmp(eps->shapeInfo(), false, block.launchContext());
+  means->applyTrueBroadcast(sd::BroadcastOpsTuple::Subtract(), input, &tmp, false);
+  tmp.applyPairwiseTransform(sd::pairwise::Multiply, eps, &tmp);
+  stdev->applyPairwiseTransform(sd::pairwise::Multiply, stdev, stdev);
+  tmp.applyTrueBroadcast(sd::BroadcastOpsTuple::Divide(), stdev, &tmp, false);
+
+  auto dlds_sum = tmp.reduceAlongDimension(reduce::Sum, &axis, true);
+  REQUIRE_TRUE(dlds_sum != nullptr, 0, "STANDARDIZE_BP OP: failed to compute dlds_sum along dimension");
+
+  NDArray dldx_s(input->shapeInfo(), false, block.launchContext());
+  std::vector<NDArray *> stdevBpArgs = {input, dlds_sum};
+  std::vector<NDArray *> stdevBpOutput = {&dldx_s};
+  std::vector<double> stdevBpTArgs = {};
+  std::vector<bool> stdevBpBArgs = {};
+  sd::ops::reduce_stdev_bp stdevBp;
+  stdevBp.execute(stdevBpArgs, stdevBpOutput, stdevBpTArgs, longAxis, stdevBpBArgs);
+  *output += dldx_s;
+
+  output->applyScalar(sd::scalar::ReplaceNans, 0, output);
+  delete sum;
+  delete means;
+  delete stdev;
+  delete dlds_sum;
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(standardize_bp) {
+  getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(standardize_bp) {
+  auto in = inputShape->at(0);
+  sd::LongType *out;
+  COPY_SHAPE(in, out);
+  auto result = CONSTANT(out);
+  delete[] out;
+
+  return SHAPELIST(result);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/tear.cpp

@@ -0,0 +1,80 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 12.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_tear)
+
+#include <helpers/ConstantTadHelper.h>
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(tear, 1, -1, false, 0, -1) {
+  auto input = INPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(!block.getIArguments()->empty(), 0, "At least 1 dimension should be specified for Tear");
+
+  std::vector<sd::LongType> dims(*block.getIArguments());
+
+  for (auto &v : dims)
+    REQUIRE_TRUE(v >= 0 && v < input->rankOf(), 0,
+                 "Tear dimensions should be non-negative values, and lower then input rank. Got %i instead", v);
+
+  auto tads = input->allTensorsAlongDimension(dims);
+  for (sd::LongType e = 0; e < tads.size(); e++) {
+    auto outE = OUTPUT_VARIABLE(e);
+    outE->assign(tads.at(e));
+
+    // just for debugging purposes
+    this->storeResult(block, e, *outE);
+  }
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(tear) {
+  auto inShape = inputShape->at(0);
+
+  std::vector<sd::LongType> dims(*block.getIArguments());
+
+  if (dims.size() > 1) std::sort(dims.begin(), dims.end());
+
+  auto tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(inShape, &dims);
+  auto numTads = tadPack->numberOfTads();
+
+  auto result = SHAPELIST();
+  for (sd::LongType e = 0; e < numTads; e++) {
+    auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(block.dataType(), shape::order(inShape),
+                                                                       shape::rank(tadPack->primaryShapeInfo()),
+                                                                       shape::shapeOf(tadPack->primaryShapeInfo()),0);
+    result->push_back(newShape);
+  }
+
+  return result;
+}
+
+DECLARE_TYPES(tear) { getOpDescriptor()->setAllowedInputTypes(sd::DataType::ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/tile.cpp

@@ -0,0 +1,185 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_tile)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(tile, 1, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const int inRank = input->rankOf();
+  std::vector<sd::LongType> reps;
+
+  if (block.getIArguments()->size() == static_cast<size_t>(inRank)) {
+    reps = ArrayUtils::toLongVector(*(block.getIArguments()));
+  } else if (block.width() > 1) {
+    auto reps_vector = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(reps_vector->lengthOf() == inRank, 0,
+                 "TILE op: repeats vector length should be equal to input rank, but got %i and %i correspondingly !",
+                 reps_vector->lengthOf(), inRank);
+
+    reps = reps_vector->template asVectorT<sd::LongType>();
+  } else {
+    REQUIRE_TRUE(false, 0,
+                 "TILE op: this op requires repeats vector, either as IArgs or second array with length equal to rank "
+                 "of input array to be tiled !");
+  }
+
+  auto repProd = shape::prodLong(reps.data(), reps.size());
+  REQUIRE_TRUE(repProd > 0, 0, "TILE op: reps can't contain 0s");
+
+  input->tile(reps, *output);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(tile) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, sd::DataType::ANY)
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes(sd::DataType::ANY);
+}
+
+DECLARE_SHAPE_FN(tile) {
+  auto inShape = inputShape->at(0);
+  const int inRank = inShape[0];
+  std::vector<sd::LongType> reps;
+
+  if (block.getIArguments()->size() == static_cast<size_t>(inRank)) {
+    reps = ArrayUtils::toLongVector(*(block.getIArguments()));
+  } else if (block.width() > 1) {
+    auto reps_vector = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(reps_vector->lengthOf() == inRank, 0,
+                 "TILE op: repeats vector length should be equal to input rank, but got %i and %i correspondingly !",
+                 reps_vector->lengthOf(), inRank);
+    reps = reps_vector->template asVectorT<sd::LongType>();
+  } else {
+    REQUIRE_TRUE(false, 0,
+                 "TILE op: this op requires repeats vector, either as IArgs or second array with length equal to rank "
+                 "of input array to be tiled !");
+  }
+
+  auto repProd = shape::prodLong(reps.data(), reps.size());
+  REQUIRE_TRUE(repProd > 0, 0, "TILE op: reps can't contain 0s");
+
+  std::vector<sd::LongType> shape(inRank);
+  for (sd::LongType e = 0; e < shape::rank(inShape); e++) shape[e] = shape::sizeAt(inShape, e) * reps[e];
+
+  auto newShape =
+      ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), shape::order(inShape), shape);
+  return SHAPELIST(newShape);
+}
+
+////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(tile_bp, 2, 1, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  const int inRank = input->rankOf();
+
+  std::vector<sd::LongType> reps;
+
+  if (block.getIArguments()->size() == static_cast<size_t>(inRank)) {
+    reps = ArrayUtils::toLongVector(*(block.getIArguments()));
+  } else if (block.width() > 2) {
+    auto reps_vector = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(reps_vector->lengthOf() == inRank, 0,
+                 "TILE_BP op: repeats vector length should be equal to input rank, but got %i and %i correspondingly !",
+                 reps_vector->lengthOf(), inRank);
+
+    reps = reps_vector->template asVectorT<sd::LongType>();
+    gradO = INPUT_VARIABLE(2);
+  } else {
+    REQUIRE_TRUE(false, 0,
+                 "TILE_BP op: this op requires repeats vector, either as IArgs or second array with length equal to "
+                 "rank of input array to be tiled !");
+  }
+
+  REQUIRE_TRUE(inRank == gradO->rankOf(), 0,
+               "TILE_BP op: the ranks of input array and output's gradients array (next epsilon) must be equal, but "
+               "got %i and %i correspondingly !",
+               inRank, gradO->rankOf());
+
+  for (int i = 0; i < inRank; ++i)
+    REQUIRE_TRUE(gradO->sizeAt(i) == gradI->sizeAt(i) * reps[i], 0,
+                 "TILE_BP op: shapes of input array and output's gradients array (next epsilon) are inconsistent !");
+
+  helpers::tileBP(block.launchContext(), *gradO, *gradI, reps);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(tile_bp) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_FLOATS});
+  getOpDescriptor()->setAllowedInputTypes(1, {ALL_INTS, ALL_FLOATS});
+  getOpDescriptor()->setAllowedInputTypes(2, {ALL_FLOATS});
+
+  getOpDescriptor()->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(tile_bp) {
+  auto inShape = inputShape->at(0);
+  auto gradOShape = inputShape->at(1);
+  const int inRank = inShape[0];
+
+  std::vector<sd::LongType> reps;
+
+  if (block.getIArguments()->size() == static_cast<size_t>(inRank)) {
+    reps = ArrayUtils::toLongVector(*(block.getIArguments()));
+  } else if (block.width() > 2) {
+    auto reps_vector = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(reps_vector->lengthOf() == inRank, 0,
+                 "TILE_BP op: repeats vector length should be equal to input rank, but got %i and %i correspondingly !",
+                 reps_vector->lengthOf(), inRank);
+    reps = reps_vector->template asVectorT<sd::LongType>();
+    gradOShape = inputShape->at(2);
+  } else {
+    REQUIRE_TRUE(false, 0,
+                 "TILE_BP op: this op requires repeats vector, either as IArgs or second array with length equal to "
+                 "rank of input array to be tiled !");
+  }
+
+  REQUIRE_TRUE(inRank == gradOShape[0], 0,
+               "TILE_BP op: the ranks of input array and output's gradients array (next epsilon) must be equal, but "
+               "got %i and %i correspondingly !",
+               inRank, gradOShape[0]);
+
+  for (sd::LongType i = 0; i < inRank; ++i)
+    REQUIRE_TRUE(shape::sizeAt(gradOShape, i) == shape::sizeAt(inShape, i) * reps[i], 0,
+                 "TILE_BP op: shapes of input array and output's gradients array (next epsilon) are inconsistent !");
+
+  return SHAPELIST(CONSTANT(inShape));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/transforms/unstack.cpp

@@ -0,0 +1,127 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_unstack)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/stack.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(unstack, 1, -1, false, 0, 1) {
+  auto input = INPUT_VARIABLE(0);
+  if (input->isEmpty()) return Status::OK;
+
+  auto dim = INT_ARG(0);
+  if (dim < 0) dim += input->rankOf();
+
+  REQUIRE_TRUE(dim < input->rankOf(), 0,
+               "Unstack dimension should be lower then rank of input %i, but got dimension=%i !", input->rankOf(), dim);
+  REQUIRE_TRUE(dim >= 0, 0, "Unstack dimension should be non-negative value, but got %i !", dim);
+
+
+  std::vector<NDArray*> outArrs(input->sizeAt(dim));
+  for (size_t i = 0; i < outArrs.size(); ++i) outArrs[i] = OUTPUT_VARIABLE(i);
+
+  helpers::unstack(block.launchContext(), *input, outArrs, dim);
+
+  return Status::OK;
+}
+
+DECLARE_SYN(unpack, unstack);
+
+DECLARE_SHAPE_FN(unstack) {
+  auto inShapeInfo = inputShape->at(0);
+
+  auto dim = INT_ARG(0);
+  const LongType numTads = block.numI() > 1 ? I_ARG(1) : shape::shapeOf(inShapeInfo)[dim];
+  if (dim < 0) dim += shape::rank(inShapeInfo);
+  if(!shape::isEmptyConst(inShapeInfo)) {
+    REQUIRE_TRUE(dim < inShapeInfo[0], 0,
+                 "UNSTACK op: dimension should be lower then rank of input %i, but got dimension=%i !", inShapeInfo[0],
+                 dim);
+    REQUIRE_TRUE(dim >= 0, 0, "UNSTACK op: dimension should be non-negative value, but got %i !", dim);
+
+  }
+
+
+
+
+
+  if (ArrayOptions::arrayType(inShapeInfo) == EMPTY) {
+    std::vector<LongType> outShape;
+    for (LongType i = 0; i < shape::rank(inShapeInfo); ++i)
+      if (i != dim) outShape.push_back(shape::shapeOf(inShapeInfo)[i]);
+
+    auto result = SHAPELIST();
+    for (LongType i = 0; i < numTads; ++i)
+      result->push_back(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(ArrayOptions::dataType(inShapeInfo),outShape));
+    if(numTads < 1) {
+      result->push_back(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(ArrayOptions::dataType(inShapeInfo),outShape));
+    }
+
+    return result;
+  }
+
+  std::vector<LongType> dimVec = {dim};
+  std::vector<LongType> *dims = ShapeUtils::evalDimsToExclude(inShapeInfo[0], 1,dimVec.data());
+
+  if (dims->size() == 0 && shape::rank(inShapeInfo) == 1) {  // split vector into lengthOf scalars
+    auto result = SHAPELIST();
+    for (LongType e = 0; e < numTads; e++)
+      result->push_back(ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inShapeInfo)));
+    delete dims;
+
+    return result;
+  }
+
+  std::vector<LongType> subArrShape(shape::rank(inShapeInfo) - 1);
+
+  for (LongType j = 0, i = 0; i < shape::rank(inShapeInfo); i++)
+    if (i != dim) subArrShape[j++] = shape::shapeOf(inShapeInfo)[i];
+
+  // remove leading and trailing 1
+  if (inShapeInfo[0] == 2 && subArrShape.size() == 2) {
+    if (subArrShape[0] == 1)
+      subArrShape.erase(subArrShape.begin());
+    else if (subArrShape[1] == 1)
+      subArrShape.erase(subArrShape.end());
+  }
+
+  auto result = SHAPELIST();
+  for (int e = 0; e < numTads; e++) {
+    auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShapeInfo),
+                                                                       shape::order(inShapeInfo), subArrShape);
+    result->push_back(newShape);
+  }
+  return result;
+}
+
+DECLARE_TYPES(unstack) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif