chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/generic/parity_ops/assert.cpp

@@ -0,0 +1,44 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_Assert)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(Assert, 1, 1, false) {
+  auto x = INPUT_VARIABLE(0);
+
+  if (!x->e<bool>(0)) {
+    REQUIRE_TRUE(false, 0, "Assertion failed for node [%i]\n", block.getNodeId());
+  }
+
+  return sd::Status::OK;
+}
+DECLARE_TYPES(Assert) { getOpDescriptor()->setAllowedInputTypes(DataType::ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/bincount.cpp

@@ -0,0 +1,163 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bincount)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/weights.h>
+
+namespace sd {
+namespace ops {
+DECLARE_TYPES(bincount) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_INTS})
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(bincount, 1, 1, false, 0, 0) {
+  auto values = INPUT_VARIABLE(0)->cast(INT64);
+
+  NDArray *weights = nullptr;
+
+  LongType maxLength = -1;
+  LongType minLength = 0;
+  LongType maxIndex = values->argMax();
+  maxLength = values->e<LongType>(maxIndex) + 1;
+
+  if (block.numI() > 0) {
+    minLength = math::sd_max(INT_ARG(0), (LongType) 0L);
+    if (block.numI() == 2) maxLength = math::sd_min(maxLength, INT_ARG(1));
+  }
+
+  if (block.width() == 2) {  // the second argument is weights
+    weights = INPUT_VARIABLE(1);
+    if (weights->lengthOf() < 1) {
+      auto* valuesShape = values->getShapeAsVector();
+      weights = NDArrayFactory::create_('c', *valuesShape, values->dataType());
+      delete valuesShape;
+      int one = 1;
+      weights->assign(one);
+    } else if (weights->isScalar()) {
+      auto value = weights->cast(INT64)->asVectorT<LongType>();
+      auto* valuesShape = values->getShapeAsVector();
+      weights = NDArrayFactory::create_('c', *valuesShape, values->dataType());
+      delete valuesShape;
+      weights->assign(value[0]);
+    }
+
+
+    REQUIRE_TRUE(values->isSameShape(weights), 0, "bincount: the input and weights shapes should be equals");
+  } else if (block.width() == 3) {  // the second argument is min and the third is max
+    auto min = INPUT_VARIABLE(1);
+    auto max = min;
+    if (INPUT_VARIABLE(2)->lengthOf() > 0) {
+      max = INPUT_VARIABLE(2);
+    }
+    minLength = min->e<LongType>(0);
+    maxLength = max->e<LongType>(0);
+  } else if (block.width() > 3) {
+    auto min = INPUT_VARIABLE(2);
+    auto max = INPUT_VARIABLE(3);
+    minLength = min->e<LongType>(0);
+    if (INPUT_VARIABLE(2)->lengthOf() > 0) {
+      maxLength = max->e<LongType>(0);
+    } else
+      maxLength = minLength;
+    weights = INPUT_VARIABLE(1);
+    if (weights->lengthOf() < 1) {
+      auto* valuesShape = values->getShapeAsVector();
+      weights = NDArrayFactory::create_('c', *valuesShape, values->dataType());
+      delete valuesShape;
+      int one = 1;
+      weights->assign(one);
+    } else if (weights->isScalar()) {
+      auto value = weights->asVectorT<LongType>();
+      auto* valuesShape = values->getShapeAsVector();
+      weights = NDArrayFactory::create_('c', *valuesShape, values->dataType());
+      delete valuesShape;
+      weights->assign(value[0]);
+    }
+    REQUIRE_TRUE(values->isSameShape(weights), 0, "bincount: the input and weights shapes should be equals");
+  }
+
+  minLength = math::sd_max(minLength, (LongType) 0);
+  maxLength = math::sd_min(maxLength, values->e<LongType>(maxIndex) + 1);
+
+  auto result = OUTPUT_VARIABLE(0);
+  float zero = 0.0f;
+  result->assign(zero);
+
+  helpers::adjustWeights(block.launchContext(), values, weights, result, minLength, maxLength);
+  if(weights->isScalar()) {
+    delete weights;
+  }
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(bincount) {
+  auto shapeList = SHAPELIST();
+  auto in = INPUT_VARIABLE(0);
+  DataType dtype = INT64;
+  if (block.width() > 1)
+    dtype = ArrayOptions::dataType(inputShape->at(1));
+  else if (block.numI() > 2)
+    dtype = (DataType)INT_ARG(2);
+
+  LongType maxIndex = in->argMax();
+  LongType maxLength = in->e<LongType>(maxIndex) + 1;
+  LongType outLength = maxLength;
+
+  if (block.numI() > 0) outLength = math::sd_max(maxLength, INT_ARG(0));
+
+  if (block.numI() > 1) outLength = math::sd_min(outLength, INT_ARG(1));
+
+  if (block.width() == 3) {  // the second argument is min and the third is max
+    auto min = INPUT_VARIABLE(1)->e<LongType>(0);
+    auto max = min;
+    if (INPUT_VARIABLE(2)->lengthOf() > 0) {
+      max = INPUT_VARIABLE(2)->e<LongType>(0);
+    }
+
+    outLength = math::sd_max(maxLength, min);
+    outLength = math::sd_min(outLength, max);
+  } else if (block.width() > 3) {
+    auto min = INPUT_VARIABLE(2);
+    auto max = min;
+    if (INPUT_VARIABLE(3)->lengthOf() > 0) {
+      max = INPUT_VARIABLE(3);
+    }
+    outLength = math::sd_max(maxLength, min->e<LongType>(0));
+    outLength = math::sd_min(outLength, max->e<LongType>(0));
+  }
+
+  auto newshape = ConstantShapeHelper::getInstance().vectorShapeInfo(outLength, dtype);
+
+  shapeList->push_back(newshape);
+  return shapeList;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/broadcast_dynamic_shape.cpp

@@ -0,0 +1,99 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_broadcast_dynamic_shape)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(broadcast_dynamic_shape, 2, 1, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(x->rankOf() == 1, 0,
+               "BROADCAST_DYNAMIC_SHAPE OP: the first input array must have rank = 1, but got %i instead!",
+               x->rankOf());
+  REQUIRE_TRUE(y->rankOf() == 1, 0,
+               "BROADCAST_DYNAMIC_SHAPE OP: the second input array must have rank = 1, but got %i instead!",
+               y->rankOf());
+  REQUIRE_TRUE(x->dataType() == y->dataType(), 0,
+               "BROADCAST_DYNAMIC_SHAPE OP: both input arrays must have the same integer type !");
+
+  // contract shapeInfos, neglect and don't fill strides, ews, order
+  // shapes are of interest only
+  std::vector<sd::LongType> xShapeInfo(shape::shapeInfoLength(x->lengthOf()));
+  std::vector<sd::LongType> yShapeInfo(shape::shapeInfoLength(y->lengthOf()));
+
+  // fill rank and data type
+  xShapeInfo[0] = x->lengthOf();
+  yShapeInfo[0] = y->lengthOf();
+  ArrayOptions::setDataType(
+      xShapeInfo.data(),
+      sd::DataType::INT64);  // fill with some data type, it doesn't matter what type exactly to choose
+  ArrayOptions::setDataType(yShapeInfo.data(), sd::DataType::INT64);
+
+  for (sd::LongType i = 0; i < x->lengthOf(); ++i) xShapeInfo[i + 1] = x->e<sd::LongType>(i);
+
+  for (sd::LongType i = 0; i < y->lengthOf(); ++i) yShapeInfo[i + 1] = y->e<sd::LongType>(i);
+
+   sd::LongType* poinerOnOutShapeInfo = nullptr;
+
+  const bool isBroadcastPossible = ShapeUtils::evalBroadcastShapeInfo(
+      xShapeInfo.data(), yShapeInfo.data(), true, poinerOnOutShapeInfo, block.launchContext()->getWorkspace());
+
+  REQUIRE_TRUE(
+      isBroadcastPossible, 0,
+      "BROADCAST_DYNAMIC_SHAPE OP: the shapes of two input arrays %s and %s are not suitable for broadcast operation !",
+      ShapeUtils::shapeAsString(xShapeInfo.data()).c_str(), ShapeUtils::shapeAsString(yShapeInfo.data()).c_str());
+
+  for (sd::LongType i = 0; i < z->lengthOf(); ++i) z->p<sd::LongType>(i, poinerOnOutShapeInfo[i + 1]);
+
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(broadcast_dynamic_shape) {
+  getOpDescriptor()->setAllowedOutputTypes({ALL_INTS})->setAllowedInputTypes({ALL_INTS});
+}
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(broadcast_dynamic_shape) {
+  const int xRank = INPUT_VARIABLE(0)->lengthOf();
+  const int yRank = INPUT_VARIABLE(1)->lengthOf();
+
+  const int maxRank = xRank > yRank ? xRank : yRank;
+
+  auto outputShapeInfo =
+      ConstantShapeHelper::getInstance().vectorShapeInfo(maxRank, ArrayOptions::dataType(inputShape->at(0)));
+
+  return SHAPELIST(outputShapeInfo);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/check_numerics.cpp

@@ -0,0 +1,61 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_check_numerics)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(check_numerics, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto message = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+
+  auto* allFinite = input->reduceNumber(reduce::BoolOps::IsFinite);
+  bool isFinite = allFinite->e<bool>(0);
+  delete allFinite;
+  
+  REQUIRE_TRUE(isFinite, 0, "CheckNumerics: %s", message->e<std::string>(0).c_str());
+
+  if (!block.isInplace()) output->assign(input);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(check_numerics) {
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().bufferForShapeInfo(inputShape->at(0))->primary());
+  return ret;
+}
+
+DECLARE_TYPES(check_numerics) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, sd::DataType::UTF8)
+      ->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/compare_and_bitpack.cpp

@@ -0,0 +1,63 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author sgazeos@gmail.com
+//
+
+#include <array/NDArrayFactory.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/headers/datatypes.h>
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/transforms.h>
+#if NOT_EXCLUDED(OP_compare_and_bitpack)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(compare_and_bitpack, 2, 1, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+  sd::ops::helpers::compareAndBitpack(block, *x, *y, *z);
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(compare_and_bitpack) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, DataType::ANY)
+      ->setAllowedInputTypes(1, DataType::ANY)
+      ->setAllowedOutputTypes(0, DataType::UINT8);
+}
+
+DECLARE_SHAPE_FN(compare_and_bitpack) {
+  auto inShape = inputShape->at(0);
+  auto shapes = shape::shapeOf(inShape);
+  const int rank = shape::rank(inShape);
+  REQUIRE_TRUE(!shape::isScalar(inShape), 0, "Input should not be a scalar");
+  std::vector<sd::LongType> shapeDims{shapes, shapes + rank};
+  REQUIRE_TRUE(shapeDims[rank - 1] % 8 == 0, 0, "Last dimension of the input (which is %i) should be divisible by 8 ",
+               shapeDims[rank - 1]);
+  shapeDims[rank - 1] = shapeDims[rank - 1] / 8;
+  DataType newType = DataType::UINT8;
+  auto outputShape = ConstantShapeHelper::getInstance().createShapeInfo(newType, shape::order(inShape), shapeDims);
+  return SHAPELIST(outputShape);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/confusion_matrix.cpp

@@ -0,0 +1,103 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author @cpuheater
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_confusion_matrix)
+
+#include <array/NDArray.h>
+#include <array/NDArrayList.h>
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/confusion.h>
+
+#include <array>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(confusion_matrix, 2, 1, false, 0, -2) {
+  auto labels = INPUT_VARIABLE(0);
+  auto predictions = INPUT_VARIABLE(1);
+  NDArray *weights = nullptr;
+  if (block.width() > 2) {
+    weights = INPUT_VARIABLE(2);
+    REQUIRE_TRUE(weights->isSameShape(predictions), 0,
+                 "CONFUSION_MATRIX: Weights and predictions should have equal shape");
+  }
+  auto output = OUTPUT_NULLIFIED(0);
+
+  auto* minPredictionArr = predictions->reduceNumber(reduce::Min);
+  int minPrediction = minPredictionArr->e<int>(0);
+  delete minPredictionArr;
+  
+  auto* minLabelArr = labels->reduceNumber(reduce::Min);
+  int minLabel = minLabelArr->e<int>(0);
+  delete minLabelArr;
+
+  REQUIRE_TRUE(minLabel >= 0, 0, "CONFUSION_MATRIX: Labels contains negative values !");
+  REQUIRE_TRUE(minPrediction >= 0, 0, "CONFUSION_MATRIX: Predictions contains negative values !");
+  REQUIRE_TRUE(labels->isVector(), 0, "CONFUSION_MATRIX: Labels input should be a Vector, but got %iD instead",
+               labels->rankOf());
+  REQUIRE_TRUE(predictions->isVector(), 0, "CONFUSION_MATRIX: Predictions input should be Vector, but got %iD instead",
+               predictions->rankOf());
+  REQUIRE_TRUE(labels->isSameShape(predictions), 0, "CONFUSION_MATRIX: Labels and predictions should have equal shape");
+
+  helpers::confusionFunctor(block.launchContext(), labels, predictions, weights, output);
+
+  return sd::Status::OK;
+}
+
+DECLARE_SHAPE_FN(confusion_matrix) {
+  auto labels = INPUT_VARIABLE(0);
+  auto predictions = INPUT_VARIABLE(1);
+  auto dtype = block.numD() ? D_ARG(0) : sd::DataType::INT64;
+  int numClasses = 0;
+
+  if (block.getIArguments()->size() > 0) {
+    numClasses = INT_ARG(0);
+  } else {
+    auto* maxPredictionArr = predictions->reduceNumber(reduce::Max);
+    int maxPrediction = maxPredictionArr->e<int>(0);
+    delete maxPredictionArr;
+    
+    auto* maxLabelArr = labels->reduceNumber(reduce::Max);
+    int maxLabel = maxLabelArr->e<int>(0);
+    delete maxLabelArr;
+    
+    numClasses = (maxPrediction >= maxLabel) ? maxPrediction + 1 : maxLabel + 1;
+  }
+
+  std::array<sd::LongType, 2> shape = {{numClasses, numClasses}};
+  auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(dtype, 'c', 2, shape.data(),0);
+  return SHAPELIST(newShape);
+}
+
+DECLARE_TYPES(confusion_matrix) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_INDICES})
+      ->setAllowedOutputTypes({ALL_INDICES});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/expose.cpp

@@ -0,0 +1,75 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 12/11/17.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#if NOT_EXCLUDED(OP_expose)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(expose, -2, -2, true, 0, 0) {
+  for (size_t e = 0; e < block.width(); e++) {
+    //omit for eager computation, normally array size should be equal to block size
+    if(block.getVariableSpace() == nullptr || block.getVariableSpace()->getVariables().size() != block.width()) {
+      auto in = INPUT_VARIABLE(e);
+      auto out = OUTPUT_VARIABLE(e);
+      out->assign(in);
+    } else {
+      auto inVar = block.variable(e);
+      if (inVar->variableType() == NDARRAY) {
+        auto in = INPUT_VARIABLE(e);
+        auto out = OUTPUT_VARIABLE(e);
+
+        out->assign(in);
+      } else if (inVar->variableType() == ARRAY_LIST) {
+        auto var = block.ensureVariable(e);
+        if (!var->hasNDArrayList()) {
+          auto list = inVar->getNDArrayList();
+
+          block.pushNDArrayListToVariableSpace(block.nodeId(), e, list, false);
+        }
+      }
+    }
+
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Enter, expose);
+DECLARE_SYN(enter, expose);
+
+DECLARE_TYPES(expose) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+DECLARE_SHAPE_FN(expose) {
+  auto shapeList = SHAPELIST();
+
+  for (size_t e = 0; e < block.width(); e++) {
+    auto var = block.getVariable(e);
+    if (var->variableType() == NDARRAY) {
+      auto inShape = inputShape->at(e);
+      shapeList->push_back(ConstantShapeHelper::getInstance().bufferForShapeInfo(inShape)->primary());
+    }
+  }
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars.cpp

@@ -0,0 +1,84 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author George Shulinok (sgazeos@gmail.com), created on 13.11.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_fake_quant_with_min_max_vars)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/fake_quantization.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(fake_quant_with_min_max_vars, 1, 1, true, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+
+  NDArray* min;
+  NDArray* max;
+
+  REQUIRE_TRUE(block.width() == 3 || block.getTArguments()->size() == 2, 0,
+               "fake_quant_with_min_max_vars: No minimum/maximum values provided by either input arrays or TArgs");
+
+  NDArray *m;
+  NDArray *m2;
+  if (block.width() == 3) {
+    min = INPUT_VARIABLE(1);
+    max = INPUT_VARIABLE(2);
+  } else if (block.getTArguments()->size() == 2) {
+    m = NDArrayFactory::create(x->dataType(), T_ARG(0), block.launchContext());
+    m2 = NDArrayFactory::create(x->dataType(), T_ARG(1), block.launchContext());
+    min = m;
+    max = m2;
+  }
+  auto output = OUTPUT_VARIABLE(0);
+  REQUIRE_TRUE(x->dataType() == output->dataType(), 0,
+               "fake_quant_with_min_max_vars: input and output data types must be the same");
+
+  int numBits = 8;
+  if (block.getIArguments() && block.getIArguments()->size()) numBits = INT_ARG(0);
+  bool narrowed = false;
+  if (block.getBArguments() && block.getBArguments()->size()) {
+    narrowed = B_ARG(0);
+  }
+  REQUIRE_TRUE(numBits > 1 && numBits < 17, 0,
+               "fake_quant_with_min_max_vars: Number of bits for quantization should be in between 2 and 16, but %i was given.",
+               numBits);
+  helpers::fakeQuantWithMinMaxVars(x, min, max, numBits, narrowed, output);
+
+  if(m != nullptr) {
+    delete m;
+  }
+
+  if(m2 != nullptr) {
+    delete m2;
+  }
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(fake_quant_with_min_max_vars) {
+  getOpDescriptor()->setAllowedOutputTypes({ALL_FLOATS})->setAllowedInputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+DECLARE_SYN(fake_quant_with_min_max_args, fake_quant_with_min_max_vars);
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/fake_quant_with_min_max_vars_per_channel.cpp

@@ -0,0 +1,77 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author George Shulinok <sgazeos@gmail.com>, created on 08.10.2019
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_fake_quant_with_min_max_vars_per_channel)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/fake_quantization.h>
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(fake_quant_with_min_max_vars_per_channel, 3, 1, true, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto min = INPUT_VARIABLE(1);
+  auto max = INPUT_VARIABLE(2);
+
+  auto depth = x->sizeAt(-1);
+  REQUIRE_TRUE(min->rankOf() == 1 && max->rankOf() == 1 && min->lengthOf() == max->lengthOf(), 0,
+               "fake_quant_with_min_max_vars_per_channel: Min and Max should be 1D tensors with the same length");
+  REQUIRE_TRUE(depth == min->lengthOf(), 0,
+               "fake_quant_with_min_max_vars_per_channel: Min length should be"
+               " %lld, but %lld occurs.",
+               depth, min->lengthOf());
+
+  REQUIRE_TRUE(depth == max->lengthOf(), 0,
+               "fake_quant_with_min_max_vars_per_channel: Max length should be"
+               "%lld, but %lld occurs.",
+               depth, max->lengthOf());
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(x->dataType() == output->dataType(), 0,
+               "fake_quant_with_min_max_vars_per_channel: input and output data types must be the same");
+
+  int numBits = 8;
+  if (block.getIArguments() && block.getIArguments()->size()) numBits = INT_ARG(0);
+  bool narrowed = false;
+  // INT_ARG(1);
+  if (block.getBArguments() && block.getBArguments()->size()) {
+    narrowed = B_ARG(0);
+  }
+
+  REQUIRE_TRUE(numBits > 1 && numBits < 17, 0,
+               "fake_quant_with_min_max_vars_per_channel: Number of bits"
+               " for quatization should be in between 2 and 16, but %i "
+               "was given.",
+               numBits);
+  helpers::fakeQuantWithMinMaxVarsPerChannel(block.launchContext(), x, min, max, numBits, narrowed, output);
+  return sd::Status::OK;
+}
+
+DECLARE_TYPES(fake_quant_with_min_max_vars_per_channel) {
+  getOpDescriptor()->setAllowedOutputTypes({ALL_FLOATS})->setAllowedInputTypes({ALL_INTS, ALL_FLOATS});
+}
+
+DECLARE_SYN(fake_quant_with_min_max_args_per_channel, fake_quant_with_min_max_vars_per_channel);
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/in_top_k.cpp

@@ -0,0 +1,70 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_in_top_k)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/top_k.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(in_top_k, 2, 1, true, 0, 1) {
+  auto predictions = INPUT_VARIABLE(0);
+  auto target = INPUT_VARIABLE(1);
+
+  auto result = OUTPUT_VARIABLE(0);
+  REQUIRE_TRUE(block.numI() > 0, 0, "in_top_k: Parameter k is needed to be set");
+  REQUIRE_TRUE(predictions->sizeAt(0) == target->lengthOf(), 0,
+               "in_top_k: the number of predictions rows should be equal to target array length, but got %i and %i "
+               "correspondingly !",
+               predictions->sizeAt(0), target->lengthOf());
+  REQUIRE_TRUE(predictions->rankOf() == 2, 0, "in_top_k: The predictions array should have rank 2, but %i given",
+               predictions->rankOf());
+  REQUIRE_TRUE(target->rankOf() == 1, 0, "in_top_k: The target should be a vector");
+
+  int k = INT_ARG(0);
+  return helpers::inTopKFunctor(block.launchContext(), predictions, target, result, k);
+}
+
+DECLARE_SHAPE_FN(in_top_k) {
+  auto shapeList = SHAPELIST();
+  auto in = inputShape->at(1);
+  int shapeRank = shape::rank(in);
+
+  auto aShape = ConstantShapeHelper::getInstance().createShapeInfo(sd::DataType::BOOL, shape::order(in),
+                                                                   shape::rank(in), shape::shapeOf(in),0);
+  shapeList->push_back(aShape);
+  return shapeList;
+}
+
+DECLARE_TYPES(in_top_k) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes(DataType::BOOL);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/invoke.cpp

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_invoke)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(invoke, 2, 2, false, 0, 0) {
+  sd_printf("This operation is unimplemented and is present for only metadata purposes.\n",0);
+  return Status::VALIDATION;
+};
+
+DECLARE_SHAPE_FN(invoke) {
+  sd_printf("This shape function is unimplemented and is present for only metadata purposes.\n",0);
+  return SHAPELIST();
+}
+
+DECLARE_TYPES(invoke) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(ANY)
+      ->setAllowedOutputTypes(ANY);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/listdiff.cpp

@@ -0,0 +1,75 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_listdiff)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/listdiff.h>
+
+// this op will probably never become GPU-compatible
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(listdiff, 2, 2, false, 0, 0) {
+  auto values = INPUT_VARIABLE(0);
+  auto keep = INPUT_VARIABLE(1);
+  auto output1 = OUTPUT_VARIABLE(0);
+  auto output2 = OUTPUT_VARIABLE(1);
+
+  REQUIRE_TRUE(values->rankOf() == 1, 0, "ListDiff: rank of values should be 1D, but got %iD instead",
+               values->rankOf());
+  REQUIRE_TRUE(keep->rankOf() == 1, 0, "ListDiff: rank of keep should be 1D, but got %iD instead", keep->rankOf());
+  REQUIRE_TRUE(keep->dataType() == values->dataType(), 0, "ListDiff: both inputs must have same data type");
+
+  return helpers::listDiffFunctor(block.launchContext(), values, keep, output1, output2);
+};
+
+DECLARE_SHAPE_FN(listdiff) {
+  auto values = INPUT_VARIABLE(0);
+  auto keep = INPUT_VARIABLE(1);
+
+  REQUIRE_TRUE(values->rankOf() == 1, 0, "ListDiff: rank of values should be 1D, but got %iD instead",
+               values->rankOf());
+  REQUIRE_TRUE(keep->rankOf() == 1, 0, "ListDiff: rank of keep should be 1D, but got %iD instead", keep->rankOf());
+  auto v = values->dataType();
+  auto k = keep->dataType();
+  REQUIRE_TRUE(k == v, 0, "ListDiff: both inputs must have same data type");
+
+  auto saved = helpers::listDiffCount(block.launchContext(), values, keep);
+
+  REQUIRE_TRUE(saved > 0, 0, "ListDiff: no matches found");
+
+  auto shapeX = ConstantShapeHelper::getInstance().vectorShapeInfo(saved, values->dataType());
+  auto shapeY = ConstantShapeHelper::getInstance().vectorShapeInfo(saved, DataType::INT64);
+  return SHAPELIST(shapeX, shapeY);
+}
+
+DECLARE_TYPES(listdiff) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes(0, DataType::INHERIT)
+      ->setAllowedOutputTypes(1, {ALL_INTS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/non_max_suppression.cpp

@@ -0,0 +1,229 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> at 3/30/2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/image_suppression.h>
+
+namespace sd {
+namespace ops {
+#if NOT_EXCLUDED(OP_non_max_suppression)
+CUSTOM_OP_IMPL(non_max_suppression, 2, 1, false, 0, 0) {
+  auto boxes = INPUT_VARIABLE(0);
+  auto scales = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+  int maxOutputSize;  // = INT_ARG(0);
+  if (block.width() > 2)
+    maxOutputSize = INPUT_VARIABLE(2)->e<int>(0);
+  else if (block.getIArguments()->size() == 1)
+    maxOutputSize = INT_ARG(0);
+  else
+  REQUIRE_TRUE(false, 0, "image.non_max_suppression: Max output size argument cannot be retrieved.");
+
+  double overlayThreshold = 0.5;
+  double scoreThreshold = -DataTypeUtils::infOrMax<float>();
+
+  if (block.width() > 3) {
+    overlayThreshold = INPUT_VARIABLE(3)->e<double>(0);
+  } else if (block.getTArguments()->size() > 0) {
+    overlayThreshold = T_ARG(0);
+  }
+
+  if (block.width() > 4) {
+    scoreThreshold = INPUT_VARIABLE(4)->e<double>(0);
+  } else if (block.getTArguments()->size() > 1) {
+    scoreThreshold = T_ARG(1);
+  }
+  if (boxes->isEmpty() || scales->isEmpty()) return Status::OK;
+
+  if (output->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(boxes->rankOf() == 2, 0,
+               "image.non_max_suppression: The rank of boxes array should be 2, "
+               "but %i is given",
+               boxes->rankOf());
+  REQUIRE_TRUE(boxes->sizeAt(1) == 4, 0,
+               "image.non_max_suppression: The last dimension of boxes array "
+               "should be 4, but %i is given",
+               boxes->sizeAt(1));
+  REQUIRE_TRUE(scales->rankOf() == 1 && scales->lengthOf() == boxes->sizeAt(0), 0,
+               "image.non_max_suppression: The rank of scales array should be 1, but %i is given", boxes->rankOf());
+  REQUIRE_TRUE(overlayThreshold >= 0. && overlayThreshold <= 1., 0,
+               "image.non_max_suppressio: The overlay "
+               "threashold should be in [0, 1], but "
+               "%lf is given.",
+               overlayThreshold);
+  REQUIRE_TRUE(boxes->dataType() == scales->dataType(), 0,
+               "image.non_max_suppression: Boxes and scores inputs should have the same data type, but %s and %s "
+               "were given.",
+               DataTypeUtils::asString(boxes->dataType()).c_str(), DataTypeUtils::asString(scales->dataType()).c_str());
+  helpers::nonMaxSuppression(block.launchContext(), boxes, scales, maxOutputSize, overlayThreshold, scoreThreshold,
+                             output);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(non_max_suppression) {
+  auto in = inputShape->at(0);
+  LongType *outputShape = nullptr;
+
+  int maxOutputSize;
+  if (block.width() > 2)
+    maxOutputSize = INPUT_VARIABLE(2)->e<int>(0);
+  else if (block.getIArguments()->size() == 1)
+    maxOutputSize = INT_ARG(0);
+  else
+  REQUIRE_TRUE(false, 0, "image.non_max_suppression: Max output size argument cannot be retrieved.");
+
+  if (maxOutputSize > 0) {
+    auto actualIndicesCount = shape::sizeAt(in, static_cast<LongType>(0));
+    if (block.getTArguments()->size() > 1 || block.width() > 4) {
+      auto scoreThreshold = block.getTArguments()->size() > 1 ? T_ARG(1) : INPUT_VARIABLE(4)->e<double>(0);
+      auto scales = INPUT_VARIABLE(1);
+      scales->syncToHost();
+      for (auto e = 0; e < scales->lengthOf(); e++) {
+        if (scales->e<float>(e) < (float)scoreThreshold) {
+          actualIndicesCount--;
+        }
+      }
+    }
+    if (actualIndicesCount < maxOutputSize) maxOutputSize = actualIndicesCount;
+  }
+
+
+  if(shape::isEmptyConst(in)) {
+    std::vector<LongType> shape = {maxOutputSize};
+    return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(DataType::INT32,shape));
+  }
+  outputShape = ConstantShapeHelper::getInstance().vectorShapeInfo(maxOutputSize, INT32);
+
+  return SHAPELIST(outputShape);
+}
+DECLARE_TYPES(non_max_suppression) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_INDICES});
+}
+#endif
+#if NOT_EXCLUDED(OP_non_max_suppression_v3)
+DECLARE_TYPES(non_max_suppression_v3) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_INDICES});
+}
+
+CUSTOM_OP_IMPL(non_max_suppression_v3, 2, 1, false, 0, 0) {
+  auto boxes = INPUT_VARIABLE(0);
+  auto scales = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+  int maxOutputSize;  // = INT_ARG(0);
+  if (block.width() > 2)
+    maxOutputSize = INPUT_VARIABLE(2)->e<int>(0);
+  else if (block.getIArguments()->size() == 1)
+    maxOutputSize = INT_ARG(0);
+  else
+  REQUIRE_TRUE(false, 0, "image.non_max_suppression: Max output size argument cannot be retrieved.");
+
+  double overlayThreshold = 0.5;
+  double scoreThreshold = -DataTypeUtils::infOrMax<float>();
+
+  if (block.width() > 3) {
+    overlayThreshold = INPUT_VARIABLE(3)->e<double>(0);
+  } else if (block.getTArguments()->size() > 0) {
+    overlayThreshold = T_ARG(0);
+  }
+
+  if (block.width() > 4) {
+    scoreThreshold = INPUT_VARIABLE(4)->e<double>(0);
+  } else if (block.getTArguments()->size() > 1) {
+    scoreThreshold = T_ARG(1);
+  }
+  if (boxes->isEmpty() || scales->isEmpty()) return Status::OK;
+  if (output->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(boxes->rankOf() == 2, 0,
+               "image.non_max_suppression: The rank of boxes array should be 2, but "
+               "%i is given",
+               boxes->rankOf());
+  REQUIRE_TRUE(boxes->sizeAt(1) == 4, 0,
+               "image.non_max_suppression: The last dimension of boxes array should "
+               "be 4, but %i is given",
+               boxes->sizeAt(1));
+  REQUIRE_TRUE(scales->rankOf() == 1 && scales->lengthOf() == boxes->sizeAt(0), 0,
+               "image.non_max_suppression: The rank of scales array should be 1, but %i is given", boxes->rankOf());
+  REQUIRE_TRUE(overlayThreshold >= 0. && overlayThreshold <= 1., 0,
+               "image.non_max_suppression_v3: The overlay threashold should be in [0, 1], but %lf given.",
+               overlayThreshold);
+  REQUIRE_TRUE(boxes->dataType() == scales->dataType(), 0,
+               "image.non_max_suppression_v3: Boxes and scores inputs should have the same data type, but %s and %s "
+               "were given.",
+               DataTypeUtils::asString(boxes->dataType()).c_str(), DataTypeUtils::asString(scales->dataType()).c_str());
+
+  helpers::nonMaxSuppressionV3(block.launchContext(), boxes, scales, maxOutputSize, overlayThreshold, scoreThreshold,
+                               output);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(non_max_suppression_v3) {
+  auto in = inputShape->at(0);
+  if(shape::isEmptyConst(in)) {
+    std::vector<LongType> shape = {0};
+    return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(DataType::INT32,shape));
+  }
+  int outRank = shape::rank(in);
+
+  int maxOutputSize;
+  if (block.width() > 2)
+    maxOutputSize = INPUT_VARIABLE(2)->e<int>(0);
+  else if (block.getIArguments()->size() == 1)
+    maxOutputSize = INT_ARG(0);
+  else
+  REQUIRE_TRUE(false, 0, "image.non_max_suppression: Max output size argument cannot be retrieved.");
+  auto boxes = INPUT_VARIABLE(0);
+  auto scales = INPUT_VARIABLE(1);
+
+  double overlayThreshold = 0.5;
+  double scoreThreshold = -DataTypeUtils::infOrMax<float>();
+
+  if (block.width() > 3) {
+    overlayThreshold = INPUT_VARIABLE(3)->e<double>(0);
+  } else if (block.getTArguments()->size() > 0) {
+    overlayThreshold = T_ARG(0);
+  }
+
+  if (block.width() > 4) {
+    scoreThreshold = INPUT_VARIABLE(4)->e<double>(0);
+  } else if (block.getTArguments()->size() > 1) {
+    scoreThreshold = T_ARG(1);
+  }
+
+  auto len = maxOutputSize;
+  if (len > 0)
+    len = helpers::nonMaxSuppressionV3(block.launchContext(), boxes, scales, maxOutputSize, overlayThreshold,
+                                       scoreThreshold, nullptr);
+
+  if(len == 0) {
+    std::vector<LongType> shape = {0};
+    return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(DataType::INT32,shape));
+  }
+
+  auto outputShape = ConstantShapeHelper::getInstance().vectorShapeInfo(len, INT32);
+  return SHAPELIST(outputShape);
+}
+#endif
+
+}  // namespace ops
+}  // namespace sd

libnd4j/include/ops/declarable/generic/parity_ops/non_max_suppression_overlaps.cpp

@@ -0,0 +1,103 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> at 10/17/2019
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/image_suppression.h>
+
+#if NOT_EXCLUDED(OP_non_max_suppression_overlaps)
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(non_max_suppression_overlaps, 2, 1, false, 0, 0) {
+  auto boxes = INPUT_VARIABLE(0);
+  auto scales = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+  int maxOutputSize;  // = INT_ARG(0);
+  if (block.width() > 2)
+    maxOutputSize = INPUT_VARIABLE(2)->e<LongType>(0);
+  else if (block.getIArguments()->size() == 1)
+    maxOutputSize = INT_ARG(0);
+  else
+    REQUIRE_TRUE(false, 0, "image.non_max_suppression_overlaps: Max output size argument cannot be retrieved.");
+  REQUIRE_TRUE(boxes->rankOf() == 2, 0,
+               "image.non_max_suppression_overlaps: The rank of boxes array should be 2, but %i is given",
+               boxes->rankOf());
+  REQUIRE_TRUE(boxes->sizeAt(0) == boxes->sizeAt(1), 0,
+               "image.non_max_suppression_overlaps: The boxes array should be square, but {%lld, %lld} is given",
+               boxes->sizeAt(0), boxes->sizeAt(1));
+  REQUIRE_TRUE(scales->rankOf() == 1 && scales->lengthOf() == boxes->sizeAt(0), 0,
+               "image.non_max_suppression_overlaps: The rank of scales array should be 1, but %i is given",
+               boxes->rankOf());
+
+
+  double overlapThreshold = 0.5;
+  double scoreThreshold = -DataTypeUtils::infOrMax<double>();
+  if (block.getTArguments()->size() > 0) overlapThreshold = T_ARG(0);
+  if (block.getTArguments()->size() > 1) scoreThreshold = T_ARG(1);
+
+  // TODO: refactor helpers to multithreaded facility
+  helpers::nonMaxSuppressionGeneric(block.launchContext(), boxes, scales, maxOutputSize, overlapThreshold,
+                                    scoreThreshold, output);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(non_max_suppression_overlaps) {
+  auto in = inputShape->at(0);
+
+  int maxOutputSize;
+  if (block.width() > 2)
+    maxOutputSize = INPUT_VARIABLE(2)->e<LongType>(0);
+  else if (block.getIArguments()->size() == 1)
+    maxOutputSize = INT_ARG(0);
+  else
+    REQUIRE_TRUE(false, 0, "image.non_max_suppression: Max output size argument cannot be retrieved.");
+
+  double overlapThreshold = 0.5;
+  double scoreThreshold = 0.;
+
+  LongType boxSize =
+      helpers::nonMaxSuppressionGeneric(block.launchContext(), INPUT_VARIABLE(0), INPUT_VARIABLE(1), maxOutputSize,
+                                        overlapThreshold, scoreThreshold, nullptr);
+  if (boxSize < maxOutputSize) {
+    maxOutputSize = boxSize;
+  }
+
+  if(shape::isEmptyConst(in)) {
+    std::vector<LongType> shape = {maxOutputSize};
+    return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(DataType::INT32,shape));
+  }
+
+  auto outputShape = ConstantShapeHelper::getInstance().vectorShapeInfo(maxOutputSize, INT64);
+
+  return SHAPELIST(outputShape);
+}
+DECLARE_TYPES(non_max_suppression_overlaps) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedInputTypes(2, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_INDICES});
+}
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/normalize_moments.cpp

@@ -0,0 +1,85 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 26.01.2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_normalize_moments)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(normalize_moments, 3, 2, false, 1, 0) {
+  auto counts = INPUT_VARIABLE(0);
+  auto means = INPUT_VARIABLE(1);
+  auto variances = INPUT_VARIABLE(2);
+
+  auto resMeans = OUTPUT_VARIABLE(0);
+  auto resVariances = OUTPUT_VARIABLE(1);
+
+  // FIXME: double?
+  NDArray *shift = NDArrayFactory::create<double>(0., block.launchContext());
+
+  if (block.getTArguments()->size() > 0) {
+    shift->assign(T_ARG(0));
+  }
+
+  means->applyScalarArr(scalar::Divide, counts, resMeans);
+
+  NDArray *squareMeans = resMeans->dup('c', false);
+  NDArray *tempVariances = resVariances->dup('c', false);
+
+  squareMeans->applyTransform(transform::Square, squareMeans, nullptr);
+  variances->applyScalarArr(scalar::Divide, counts, tempVariances);
+  tempVariances->applyPairwiseTransform(pairwise::Subtract, squareMeans, resVariances);
+
+  if (shift->e<double>(0) != 0) {
+    resMeans->applyScalarArr(scalar::Add, shift, resMeans);
+  }
+
+  delete shift;
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(normalize_moments) {
+  auto in = inputShape->at(1);
+
+  LongType* meanShape = nullptr;
+  LongType* varianceShape = nullptr;
+
+  COPY_SHAPE_EX(in, meanShape, block.getWorkspace());
+  COPY_SHAPE_EX(in, varianceShape, block.getWorkspace());
+
+  auto shapeList = SHAPELIST();
+  shapeList->push_back(CONSTANT(meanShape));
+  shapeList->push_back(CONSTANT(varianceShape));
+
+  return shapeList;
+}
+
+DECLARE_TYPES(normalize_moments) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/nth_element.cpp

@@ -0,0 +1,75 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> at 3/30/2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/nth_element.h>
+#if NOT_EXCLUDED(OP_nth_element)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(nth_element, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto n = INPUT_VARIABLE(1);
+  bool reverse = false;
+  if (block.getIArguments()->size() > 0) reverse = (bool)INT_ARG(0);
+
+  auto output = OUTPUT_VARIABLE(0);
+  LongType lastDim = input->sizeAt(-1);
+  int nVal = n->e<int>(0);
+  REQUIRE_TRUE(nVal < lastDim && nVal >= 0, 0,
+               "nth_element: n should be non-negative and less than last dimension size (%lld), but %i was given.",
+               lastDim, n);
+  REQUIRE_TRUE(input->rankOf() > 0, 0, "nth_element: The rank of input array should be at least 1, but %i is given",
+               input->rankOf());  //
+  if (output->lengthOf() == input->lengthOf())
+    output->assign(input);
+  else {
+    helpers::nthElementFunctor(block.launchContext(), input, nVal, output, reverse);
+  }
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(nth_element) {
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in) - 1;
+  LongType* outShape = nullptr;
+  if (outRank > 1) {
+    LongType* outputShape = nullptr;
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    outputShape[0] = outRank;
+    for (LongType e = 0; e < outRank; e++) outputShape[e + 1] = in[e + 1];
+
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+    outShape = CONSTANT(outputShape);
+  } else if (outRank == 1) {
+    outShape = ConstantShapeHelper::getInstance().vectorShapeInfo(shape::sizeAt(in, static_cast<LongType>(0)), ArrayOptions::dataType(in));
+  } else {
+    outShape = ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(in));
+  }
+  return SHAPELIST(outShape);
+}
+DECLARE_TYPES(nth_element) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY);
+}
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/onehot.cpp

@@ -0,0 +1,108 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 01/11/17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_onehot)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/one_hot.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(onehot, 1, 1, false, -2, -2) {
+  auto input = INPUT_VARIABLE(0);
+
+  // FIXME: double?
+  double on(1.0f);   // T_ARG(0);
+  double off(0.0f);  // T_ARG(1);
+
+  auto axis = -1;   // INT_ARG(0);
+  auto depth = -1;  // INT_ARG(1);
+
+  if (block.numI() > 0) axis = INT_ARG(0);
+
+  if (block.numI() > 1) {
+    depth = INT_ARG(1);
+  } else if (block.width() > 1) {
+    depth = INPUT_VARIABLE(1)->e<sd::LongType>(0);
+  }
+
+  REQUIRE_TRUE(depth > 0, 0, "OneHot: depth must be positive value");
+
+  if (block.width() > 2) {
+    on = INPUT_VARIABLE(2)->e<double>(0);
+
+    if (block.width() > 3) off = INPUT_VARIABLE(3)->e<double>(0);
+  } else if (block.numT() > 0) {
+    on = T_ARG(0);
+
+    if (block.numT() > 1) off = T_ARG(1);
+  }
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (axis < 0) axis = output->rankOf() + axis;
+
+  helpers::onehot(block.launchContext(), input, output, axis, depth, on, off);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(onehot) {
+  auto inShape = inputShape->at(0);
+
+  DataType dtype = block.numD() > 0 ? D_ARG(0) : FLOAT32;
+
+  int depth = -1;
+  LongType axis = -1;
+
+  if (block.numI() > 0) axis = INT_ARG(0);
+
+  if (block.numI() > 1) {
+    depth = INT_ARG(1);
+  } else if (block.width() > 1) {
+    depth = INPUT_VARIABLE(1)->e<sd::LongType>(0);
+  }
+
+  REQUIRE_TRUE(depth > 0, 0, "OneHot: depth must be positive value");
+
+  int rank = shape::rank(inShape);
+
+  if (axis < 0) axis = rank + 1 + axis;
+
+  std::vector<LongType> shape;
+  for (int e = 0; e < rank; e++) shape.push_back(shape::shapeOf(inShape)[e]);
+
+  shape.insert(shape.begin() + axis, depth);
+  auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(dtype, 'c', rank + 1, shape.data(), -1);
+
+  return SHAPELIST(newShape);
+}
+
+DECLARE_TYPES(onehot) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/rint.cpp

@@ -0,0 +1,43 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_rint)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(rint, 1, 1, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  x->applyTransform(transform::Rint, z);
+
+  return Status::OK;
+}
+}  // namespace ops
+
+DECLARE_TYPES(rint) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/roll.cpp

@@ -0,0 +1,130 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_roll)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/axis.h>
+#include <ops/declarable/helpers/roll.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(roll, -2, 1, true, 0, 0) {
+  auto output = OUTPUT_VARIABLE(0);
+  auto input = INPUT_VARIABLE(0);
+  int inputLen = input->lengthOf();
+
+  bool shiftIsLinear = block.width() == 1;
+  std::vector<LongType> axes;
+  std::vector<LongType> shifts;
+  if (block.width() > 1) {
+    REQUIRE_TRUE(block.width() == 3, 0, "roll: 3 arguments required for roll - input, shifts and axes. But %i given.",
+                 block.width());
+    auto axesI = INPUT_VARIABLE(2);
+    auto shiftsI = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(axesI->rankOf() == shiftsI->rankOf(), 0,
+                 "roll: shifts and axes should be the same rank, but %i and %i given.", (int)shiftsI->rankOf(),
+                 (int)axesI->rankOf());
+    REQUIRE_TRUE(axesI->lengthOf() == shiftsI->lengthOf(), 0,
+                 "roll: shifts and axes should be the same length, but %i and %i given.", (int)shiftsI->lengthOf(),
+                 (int)axesI->lengthOf());
+    helpers::adjustAxis(axesI->lengthOf(), axesI, axes);
+    shifts.resize(shiftsI->lengthOf());
+    for (LongType i = 0; i < shiftsI->lengthOf(); i++) {
+      auto shift = shiftsI->e<int>(i);
+      if (shift < 0) {
+        shift -= input->sizeAt(i) * (shift / inputLen - 1);
+      } else if (shift != 0) {
+        shift %= input->sizeAt(i);
+      }
+
+      shifts[i] = shift;
+    }
+
+  } else {
+    int shift = INT_ARG(0);
+    if (shift < 0) {
+      // convert shift to positive value between 1 and inputLen - 1
+      shift -= inputLen * (shift / inputLen - 1);
+    } else if (shift != 0)
+      // cut shift to value between 1 and inputLen - 1
+      shift %= inputLen;
+    axes.resize(block.getIArguments()->size() - 1);
+    if (axes.size())
+      shifts.resize(axes.size());  // emplace_back(shift);
+    else
+      shifts.push_back(shift);
+
+    for (auto& s : shifts) s = shift;
+
+    for (unsigned e = 0; e < axes.size(); ++e) {
+      int axis = INT_ARG(e + 1);
+      REQUIRE_TRUE(axis < input->rankOf() && axis >= -input->rankOf(), 0,
+                   "roll: axe value should be between -%i and %i, but %i was given.", input->rankOf(),
+                   input->rankOf() - 1, axis);
+      axes[e] = (axis < 0 ? (input->rankOf() + axis) : axis);
+    }
+  }
+
+  if (block.isInplace()) output = input;
+
+  shiftIsLinear = (axes.size() == 0) || (input->rankOf() == 1);
+  sd_debug("Roll: Shift is linear %d Shift is %d, first dimension is %d\n", (int)shiftIsLinear, (int)shifts[0],
+           axes.size() > 0 ? axes[0] : 0);
+  bool shiftsSumZero = false;
+  auto shiftSum = 0;
+  for (auto& s : shifts) {
+    shiftSum += s;
+    sd_debug("Roll: Shift  is %d\n", s);
+  }
+  // all zeros is no op
+  if (shiftSum < 1) {
+    sd_debug("Roll: No shift needed. Shift total was %d\n", shiftSum);
+    if (!block.isInplace()) {
+      output->assign(input);
+    }
+    return Status::OK;
+  }
+
+  if (shiftIsLinear) {
+    helpers::rollFunctorLinear(block.launchContext(), input, output, shifts[0], block.isInplace());
+  } else {
+    helpers::rollFunctorFull(block.launchContext(), input, output, shifts, axes, block.isInplace());
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(roll) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, INT32)  // TODO: all ints in future
+      ->setAllowedInputTypes(2, INT32)
+      ->setAllowedOutputTypes(ANY)
+      ->setSameMode(true);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/segment_max.cpp

@@ -0,0 +1,105 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 2/21/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_segment_max)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(segment_max, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto segmentedOutput = OUTPUT_VARIABLE(0);
+  REQUIRE_TRUE(idxSegments->isVector(), 0, "segment_max: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(idxSegments->lengthOf() == input->sizeAt(0), 0,
+               "segment_max: segment indexes array length should be equal to the input first dimension, but %i != %i.",
+               idxSegments->lengthOf(), input->sizeAt(0));
+
+  auto expected = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+  auto wrong = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+
+  REQUIRE_TRUE(helpers::segmentIndicesValidate(block.launchContext(), idxSegments, *expected, *wrong), 0,
+               "segment_max: segment indices should be arranged, but %2.1f > %2.1f", expected->e<float>(0),
+               wrong->e<float>(0));
+
+  segmentedOutput->nullify();
+  helpers::segmentMaxFunctor(block.launchContext(), input, idxSegments, segmentedOutput);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(segment_max) {
+  auto idxVector = INPUT_VARIABLE(1);
+
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  idxVector->syncToHost();
+  int val = (*idxVector).e<LongType>(idxVector->lengthOf() - 1);
+
+  int numOfClasses = val + 1;
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+
+  outputShape[0] = outRank;
+  outputShape[1] = numOfClasses;
+  for (LongType i = 1; i < outRank; ++i) outputShape[i + 1] = shape::sizeAt(in, i);
+
+  ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+
+DECLARE_TYPES(segment_max) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS})
+      ->setSameMode(false);
+}
+CUSTOM_OP_IMPL(segment_max_bp, 3, 2, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto gradOut = INPUT_VARIABLE(2);
+  auto output = OUTPUT_NULLIFIED(0);
+  auto outIndices = OUTPUT_NULLIFIED(1);
+  outIndices->assign(indices);
+  return helpers::segmentMaxFunctorBP(block.launchContext(), input, indices, gradOut, output);
+}
+DECLARE_SHAPE_FN(segment_max_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+DECLARE_TYPES(segment_max_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(ANY)
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setSameMode(true);
+}
+
+}  // namespace ops
+
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/segment_mean.cpp

@@ -0,0 +1,105 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 2/21/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_segment_mean)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(segment_mean, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto idxSegments = INPUT_VARIABLE(1)->cast(INT64);
+  auto segmentedOutput = OUTPUT_VARIABLE(0);
+  REQUIRE_TRUE(idxSegments->isVector(), 0, "segment_mean: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(idxSegments->lengthOf() == input->sizeAt(0), 0,
+               "segment_mean: segment indexes array length should be equal to the input first dimension, but %i != %i.",
+               idxSegments->lengthOf(), input->sizeAt(0));
+
+  auto expected = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+  auto wrong = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+
+  REQUIRE_TRUE(helpers::segmentIndicesValidate(block.launchContext(), idxSegments, *expected, *wrong), 0,
+               "segment_mean: segment indices should be arranged, but %2.1f > %2.1f", expected->e<float>(0),
+               wrong->e<float>(0));
+
+  segmentedOutput->nullify();
+  helpers::segmentMeanFunctor(block.launchContext(), input, idxSegments, segmentedOutput);
+
+  delete wrong;
+  delete expected;
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(segment_mean) {
+  auto idxVector = INPUT_VARIABLE(1);
+
+  auto in = inputShape->at(0);
+  LongType outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  LongType val = (*idxVector).e<LongType>(idxVector->lengthOf() - 1);
+
+  LongType numOfClasses = val + 1;
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+
+  outputShape[0] = outRank;
+  outputShape[1] = numOfClasses;
+  for (LongType i = 1; i < outRank; ++i) outputShape[i + 1] = shape::sizeAt(in, i);
+
+  ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+
+DECLARE_TYPES(segment_mean) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_FLOATS})
+      ->setSameMode(false);
+}
+
+CUSTOM_OP_IMPL(segment_mean_bp, 3, 2, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto gradOut = INPUT_VARIABLE(2);
+  auto output = OUTPUT_NULLIFIED(0);
+  auto outIndices = OUTPUT_NULLIFIED(1);
+  outIndices->assign(indices);
+  return helpers::segmentMeanFunctorBP(block.launchContext(), input, indices, gradOut, output);
+}
+DECLARE_SHAPE_FN(segment_mean_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+DECLARE_TYPES(segment_mean_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(ANY)
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setSameMode(false);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/segment_min.cpp

@@ -0,0 +1,104 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 2/21/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_segment_min)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(segment_min, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto segmentedOutput = OUTPUT_VARIABLE(0);
+  REQUIRE_TRUE(idxSegments->isVector(), 0, "segment_min: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(idxSegments->lengthOf() == input->sizeAt(0), 0,
+               "segment_min: segment indexes array length should be equal to the input first dimension, but %i != %i.",
+               idxSegments->lengthOf(), input->sizeAt(0));
+
+  auto expected = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+  auto wrong = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+
+  REQUIRE_TRUE(helpers::segmentIndicesValidate(block.launchContext(), idxSegments, *expected, *wrong), 0,
+               "segment_min: segment indices should be arranged, but %2.1f > %2.1f", expected->e<float>(0),
+               wrong->e<float>(0));
+
+  segmentedOutput->nullify();
+  helpers::segmentMinFunctor(block.launchContext(), input, idxSegments, segmentedOutput);
+
+  delete wrong;
+  delete expected;
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(segment_min) {
+  auto idxVector = INPUT_VARIABLE(1);
+
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  int val = (*idxVector).e<int>(idxVector->lengthOf() - 1);
+
+  int numOfClasses = val + 1;
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+
+  outputShape[0] = outRank;
+  outputShape[1] = numOfClasses;
+  for (LongType i = 1; i < outRank; ++i) outputShape[i + 1] = shape::sizeAt(in, i);
+
+  ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+CUSTOM_OP_IMPL(segment_min_bp, 3, 2, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto gradOut = INPUT_VARIABLE(2);
+  auto output = OUTPUT_NULLIFIED(0);
+  auto outIndices = OUTPUT_NULLIFIED(1);
+  outIndices->assign(indices);
+  return helpers::segmentMinFunctorBP(block.launchContext(), input, indices, gradOut, output);
+}
+DECLARE_SHAPE_FN(segment_min_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+
+DECLARE_TYPES(segment_min) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS})
+      ->setSameMode(false);
+}
+DECLARE_TYPES(segment_min_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(ANY)
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setSameMode(true);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/segment_prod.cpp

@@ -0,0 +1,109 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 2/21/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_segment_prod)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(segment_prod, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto segmentedOutput = OUTPUT_VARIABLE(0);
+  REQUIRE_TRUE(idxSegments->isVector(), 0, "segment_prod: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(idxSegments->lengthOf() == input->sizeAt(0), 0,
+               "segment_prod: segment indexes array length should be equal to the input first dimension, but %i != %i.",
+               idxSegments->lengthOf(), input->sizeAt(0));
+
+  auto expected = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+  auto wrong = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+
+  REQUIRE_TRUE(helpers::segmentIndicesValidate(block.launchContext(), idxSegments, *expected, *wrong), 0,
+               "segment_prod: segment indices should be arranged, but %2.1f > %2.1f", expected->e<float>(0),
+               wrong->e<float>(0));
+
+  segmentedOutput->nullify();
+  helpers::segmentProdFunctor(block.launchContext(), input, idxSegments, segmentedOutput);
+
+  delete wrong;
+  delete expected;
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(segment_prod) {
+  auto idxVector = INPUT_VARIABLE(1);
+
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  int val = (*idxVector).e<int>(idxVector->lengthOf() - 1);
+
+  int numOfClasses = val + 1;
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+
+  outputShape[0] = outRank;
+  outputShape[1] = numOfClasses;
+  for (LongType i = 1; i < outRank; ++i) outputShape[i + 1] = shape::sizeAt(in, i);
+
+  ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+
+CUSTOM_OP_IMPL(segment_prod_bp, 3, 2, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto gradOut = INPUT_VARIABLE(2);
+  auto output = OUTPUT_NULLIFIED(0);
+  auto outIndices = OUTPUT_NULLIFIED(1);
+  outIndices->assign(indices);
+  helpers::segmentProdFunctorBP(block.launchContext(), input, indices, gradOut, output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(segment_prod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS})
+      ->setSameMode(false);
+}
+
+DECLARE_SHAPE_FN(segment_prod_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(inIdx), CONSTANT(inIdx));
+}
+
+DECLARE_TYPES(segment_prod_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(ANY)
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setSameMode(false);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/segment_sum.cpp

@@ -0,0 +1,95 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 2/21/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_segment_sum)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(segment_sum, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto segmentedOutput = OUTPUT_VARIABLE(0);
+  REQUIRE_TRUE(idxSegments->isVector(), 0, "segment_sum: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(idxSegments->lengthOf() == input->sizeAt(0), 0,
+               "segment_sum: segment indexes array length should be equal to the input first dimension, but %i != %i.",
+               idxSegments->lengthOf(), input->sizeAt(0));
+
+  auto expected = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+  auto wrong = NDArrayFactory::create(input->dataType(), 0.f, block.launchContext());
+
+  REQUIRE_TRUE(helpers::segmentIndicesValidate(block.launchContext(), idxSegments, *expected, *wrong), 0,
+               "segment_sum: segment indices should be arranged, but %2.1f > %2.1f", expected->e<float>(0),
+               wrong->e<float>(0));
+
+  segmentedOutput->nullify();
+  helpers::segmentSumFunctor(block.launchContext(), input, idxSegments, segmentedOutput);
+
+  delete wrong;
+  delete expected;
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(segment_sum) {
+  auto idxVector = INPUT_VARIABLE(1);
+
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  int val = (*idxVector).e<int>(idxVector->lengthOf() - 1);
+
+  int numOfClasses = static_cast<int>(val) + 1;
+
+  ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+
+  outputShape[0] = outRank;
+  outputShape[1] = numOfClasses;
+  for (LongType i = 1; i < outRank; ++i) outputShape[i + 1] = shape::sizeAt(in, i);
+
+  ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+
+CUSTOM_OP_IMPL(segment_sum_bp, 3, 2, false, 0, 0) {
+  return helpers::segmentSumFunctorBP(block.launchContext(), INPUT_VARIABLE(0), INPUT_VARIABLE(1), INPUT_VARIABLE(2),
+                                      OUTPUT_NULLIFIED(0));
+}
+DECLARE_SHAPE_FN(segment_sum_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+
+DECLARE_TYPES(segment_sum) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+DECLARE_TYPES(segment_sum_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(ANY)
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setSameMode(false);
+}
+}  // namespace ops
+
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/sequence_mask.cpp

@@ -0,0 +1,95 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created to use with batched tensor by GS <sgazeos@gmail.com> 3/27/2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/sequence_mask.h>
+#if NOT_EXCLUDED(OP_sequence_mask)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(sequence_mask, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_NULLIFIED(0);
+  const int inRank = input->rankOf();
+
+  // REQUIRE_TRUE(inRank >= 1, 0, "sequence_mask: input array must have rank >= 1, but %i given!", inRank);
+  LongType maxInd = input->argMax();
+  float max = input->e<float>(maxInd);
+  if (block.getIArguments()->size() > 0) {
+    maxInd = INT_ARG(0);
+    if (maxInd < max) maxInd = static_cast<LongType>(max);
+  } else if (block.width() > 1) {
+    auto maxlen = INPUT_VARIABLE(1);
+    // REQUIRE_TRUE(maxlen->lengthOf() == 1, "sequence_mask: 2nd input (max length) should be a scalar array.");
+    float tmaxlen = maxlen->e<float>(0);
+    if (tmaxlen > max) maxInd = static_cast<LongType>(tmaxlen);
+  } else
+    maxInd = static_cast<LongType>(max);
+
+  helpers::sequenceMask(block.launchContext(), input, output, maxInd);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(sequence_mask) {
+  LongType* outShapeInfo = nullptr;
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in) + 1;
+  auto input = INPUT_VARIABLE(0);
+  auto dtype = BOOL;
+  auto argMaxInd = input->argMax();
+  LongType max = input->e<LongType>(argMaxInd);
+  LongType maxInd = max;
+
+  if (block.numD() > 0) dtype = D_ARG(0);
+
+  if (block.width() > 1) {
+    auto maxlen = INPUT_VARIABLE(1);
+    LongType tmaxlen = maxlen->e<LongType>(0);
+    if (tmaxlen > max) maxInd = static_cast<LongType>(tmaxlen);
+    if (block.numI() > 0) {
+      dtype = (DataType)INT_ARG(0);
+    }
+  } else {
+    if (block.numI() > 0) {
+      maxInd = INT_ARG(0);
+    }
+    if (maxInd < max) maxInd = max;
+    if (block.numI() > 1) dtype = (DataType)INT_ARG(1);  // to work with legacy code
+  }
+
+  int lastDimension = maxInd;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+  outShapeInfo[0] = outRank;
+  for (LongType i = 0; i < outRank - 1; ++i) outShapeInfo[i + 1] = shape::sizeAt(in, i);
+  outShapeInfo[outRank] = lastDimension;
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, dtype, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outShapeInfo));
+}
+
+DECLARE_TYPES(sequence_mask) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setAllowedOutputTypes(ANY);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/square.cpp

@@ -0,0 +1,44 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 01/11/17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_square)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(square, 1, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  int extras = 2;
+  input->applyScalar(scalar::Pow, extras, output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(square) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/stop_gradient.cpp

@@ -0,0 +1,47 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_stop_gradient)
+
+#include <ops/declarable/headers/parity_ops.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(stop_gradient, 1, 1, true) {
+  auto out = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) {
+    auto x = INPUT_VARIABLE(0);
+    // we hope for memcpy here
+    out->assign(x);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(StopGradient, stop_gradient);
+
+DECLARE_TYPES(stop_gradient) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/top_k.cpp

@@ -0,0 +1,98 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_top_k)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/top_k.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(top_k, 1, 2, false, 0, -1) {
+  auto x = INPUT_VARIABLE(0);
+  int k = 1;  // from params
+  bool needSort = true;
+
+  auto values = OUTPUT_VARIABLE(0);
+  auto indices = OUTPUT_VARIABLE(1);
+
+  if (block.numB() == 1) {
+      needSort = B_ARG(0);
+  }
+
+  if (block.width() == 1) {
+    if (block.numI() > 0) {
+      k = INT_ARG(0);
+    }
+  } else {
+    k = INPUT_VARIABLE(1)->e<int>(0);
+  }
+
+  REQUIRE_TRUE(k <= x->sizeAt(-1), 0, "top_k: k should not be greater than last dimension");
+  REQUIRE_TRUE(k > 0, 0, "top_k: k should be positive, but %i given.", k);
+
+  auto res = helpers::topKFunctor(block.launchContext(), x, values, indices, k, needSort);
+  return res;
+}
+
+DECLARE_SHAPE_FN(top_k) {
+  auto shapeList = SHAPELIST();
+  auto in = inputShape->at(0);
+  int shapeRank = shape::rank(in);
+  int k = 1;  // default output shape is size 1
+
+  if (block.width() == 2) {
+    k = INPUT_VARIABLE(1)->e<int>(0);
+  } else if (block.numI() > 0) {
+    k = INT_ARG(0);
+  }
+
+
+  REQUIRE_TRUE(k > 0, 0, "top_k: k should be positive, but %i given.", k);
+
+  for (int e = 0; e < 2; e++) {  // 2 element tuple at output
+    LongType* aShape;
+    ALLOCATE(aShape, block.getWorkspace(), shape::shapeInfoLength(shapeRank), sd::LongType);
+    aShape[0] = shapeRank;
+    for (LongType i = 1; i < shapeRank; ++i) aShape[i] = shape::sizeAt(in, i - 1);
+    aShape[shapeRank] = k;
+
+    shape::updateStrides(aShape, shape::order(in), false);
+    auto desc = new ShapeDescriptor(aShape, (e == 0 ? ArrayOptions::dataType(in) : INT64), false);
+    shapeList->push_back(ConstantShapeHelper::getInstance().createShapeInfo(desc));
+
+    RELEASE(aShape, block.getWorkspace());
+  }
+  return shapeList;
+}
+
+DECLARE_TYPES(top_k) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(ANY)
+      ->setAllowedOutputTypes(0, ANY)
+      ->setAllowedOutputTypes(1, {ALL_INDICES});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/unique.cpp

@@ -0,0 +1,107 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_unique)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/unique.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(unique, 1, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto values = OUTPUT_VARIABLE(0);
+  auto indices = OUTPUT_VARIABLE(1);
+
+  REQUIRE_TRUE(x->dataType() == values->dataType(), 0, "Unique: input and output data types must be the same");
+
+  return helpers::uniqueFunctor(block.launchContext(), x, values, indices, (NDArray*)nullptr);
+}
+
+DECLARE_SHAPE_FN(unique) {
+  auto in = inputShape->at(0);
+  auto source = INPUT_VARIABLE(0);
+   LongType* valuesShape;
+   LongType* indicesShape;
+
+  int uniqueCount = helpers::uniqueCount(block.launchContext(), source);
+
+  if (uniqueCount == 0) {  // empty value Shape
+    valuesShape = ConstantShapeHelper::getInstance().emptyShapeInfo(source->dataType());
+  } else {
+    // all output shapes are 1D arrays (vectors)
+    valuesShape = ConstantShapeHelper::getInstance().vectorShapeInfo(uniqueCount, ArrayOptions::dataType(in));
+  }
+  // second output is always LONG
+  indicesShape = ConstantShapeHelper::getInstance().vectorShapeInfo(shape::length(in), INT64);
+
+
+  return SHAPELIST(valuesShape, indicesShape);
+}
+
+CUSTOM_OP_IMPL(unique_with_counts, 1, 3, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto values = OUTPUT_VARIABLE(0);
+  auto indices = OUTPUT_VARIABLE(1);
+  auto counts = OUTPUT_VARIABLE(2);
+
+  return helpers::uniqueFunctor(block.launchContext(), input, values, indices, counts);
+}
+
+DECLARE_SHAPE_FN(unique_with_counts) {
+  auto in = inputShape->at(0);
+  auto source = INPUT_VARIABLE(0);
+
+  int uniqueCount = helpers::uniqueCount(block.launchContext(), source);
+  // all output shapes are 1D arrays (vectors)
+  // all output shapes are 1D arrays (vectors)
+  auto valuesShape = ConstantShapeHelper::getInstance().vectorShapeInfo(uniqueCount, source->dataType());
+
+  // second output is always LONG
+  auto indicesShape = ConstantShapeHelper::getInstance().vectorShapeInfo(source->lengthOf(), INT64);
+
+  // third one as well
+  auto countsShape = ConstantShapeHelper::getInstance().vectorShapeInfo(uniqueCount, INT64);
+
+  return SHAPELIST(valuesShape, indicesShape, countsShape);
+}
+
+DECLARE_TYPES(unique) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(ANY)
+      ->setAllowedOutputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS});
+}
+
+DECLARE_TYPES(unique_with_counts) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes(2, {ALL_INTS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_max.cpp

@@ -0,0 +1,109 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 9/6/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_unsorted_segment_max)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(unsorted_segment_max, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto reshapedSegments = idxSegments;
+  if (!idxSegments->isVector() && idxSegments->rankOf() > 1) {
+    std::vector<sd::LongType> shape = {idxSegments->lengthOf()};
+    reshapedSegments = idxSegments->reshape('c',shape, false);
+  }
+
+  auto segmentedOutput = OUTPUT_NULLIFIED(0);
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+  REQUIRE_TRUE(reshapedSegments->isVector(), 0,
+               "unsorted_segment_max: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(reshapedSegments->lengthOf() == input->sizeAt(0), 0,
+               "unsorted_segment_max: segment indexes array length should be equal to the input first dimension, but "
+               "%ld != %ld.",
+               reshapedSegments->lengthOf(), input->sizeAt(0));
+
+  LongType wrong;
+
+  REQUIRE_TRUE(helpers::unsortedSegmentIndicesValidate(block.launchContext(), reshapedSegments, numOfClasses, wrong),
+               0, "unsorted_segment_max: segment indices should be in range [0, %ld), but %ld != %ld", numOfClasses,
+               wrong, numOfClasses);
+  helpers::unsortedSegmentMaxFunctor(block.launchContext(), input, reshapedSegments, numOfClasses, segmentedOutput);
+
+  return Status::OK;
+}
+DECLARE_TYPES(unsorted_segment_max) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setSameMode(true);
+}
+DECLARE_SHAPE_FN(unsorted_segment_max) {
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+
+  if (INPUT_VARIABLE(0)->rankOf() >= 2) {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    outputShape[0] = outRank;
+    outputShape[1] = numOfClasses;
+    for (LongType i = 1; i < outRank; i++) outputShape[i + 1] = shape::sizeAt(in, i);
+
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  } else {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(1), sd::LongType);
+    outputShape[0] = 1;
+    outputShape[1] = numOfClasses;
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+  }
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+
+CUSTOM_OP_IMPL(unsorted_segment_max_bp, 3, 2, false, 0, 1) {
+  return helpers::unsortedSegmentMaxFunctorBP(block.launchContext(), INPUT_VARIABLE(0), INPUT_VARIABLE(1),
+                                              INPUT_VARIABLE(2), INT_ARG(0), OUTPUT_NULLIFIED(0));
+}
+
+DECLARE_TYPES(unsorted_segment_max_bp) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_FLOATS})
+      ->setSameMode(false);
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_max_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_mean.cpp

@@ -0,0 +1,114 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 9/6/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_unsorted_segment_mean)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(unsorted_segment_mean, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto reshapedInput = input;
+
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto reshapedSegments = idxSegments;
+  if (!idxSegments->isVector() && idxSegments->rankOf() > 1) {
+    std::vector<sd::LongType> shape = {idxSegments->lengthOf()};
+    reshapedSegments = idxSegments->reshape('c',shape, false);
+  }
+
+  auto segmentedOutput = OUTPUT_NULLIFIED(0);
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+  REQUIRE_TRUE(reshapedSegments->isVector(), 0,
+               "unsorted_segment_mean: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(reshapedSegments->lengthOf() == input->sizeAt(0), 0,
+               "unsorted_segment_mean: segment indexes array length should be equal to the input first dimension, but "
+               "%ld != %ld.",
+               reshapedSegments->lengthOf(), input->sizeAt(0));
+
+  LongType wrong;
+
+  REQUIRE_TRUE(helpers::unsortedSegmentIndicesValidate(block.launchContext(), reshapedSegments, numOfClasses, wrong),
+               0, "unsorted_segment_mean: segment indices should be in range [0, %ld), but %ld != %ld", numOfClasses,
+               wrong, numOfClasses);
+  helpers::unsortedSegmentMeanFunctor(block.launchContext(), reshapedInput, reshapedSegments, numOfClasses,
+                                      segmentedOutput);
+  delete reshapedInput;
+
+  return Status::OK;
+}
+DECLARE_TYPES(unsorted_segment_mean) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes({ALL_FLOATS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setSameMode(false);
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_mean) {
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+
+  if (INPUT_VARIABLE(0)->rankOf() >= 2) {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    outputShape[0] = outRank;
+    outputShape[1] = numOfClasses;
+    for (LongType i = 1; i < outRank; i++) outputShape[i + 1] = shape::sizeAt(in, i);
+
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  } else {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(1), sd::LongType);
+    outputShape[0] = 1;
+    outputShape[1] = numOfClasses;
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+  }
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+
+CUSTOM_OP_IMPL(unsorted_segment_mean_bp, 3, 2, false, 0, 1) {
+  return helpers::unsortedSegmentMeanFunctorBP(block.launchContext(), INPUT_VARIABLE(0), INPUT_VARIABLE(1),
+                                               INPUT_VARIABLE(2), INT_ARG(0), OUTPUT_NULLIFIED(0));
+}
+
+DECLARE_TYPES(unsorted_segment_mean_bp) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_FLOATS})
+      ->setSameMode(false);
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_mean_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_min.cpp

@@ -0,0 +1,115 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 9/6/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_unsorted_segment_min)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(unsorted_segment_min, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto reshapedInput = input;
+
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto reshapedSegments = idxSegments;
+  if (!idxSegments->isVector() && idxSegments->rankOf() > 1) {
+    std::vector<sd::LongType> shape = {idxSegments->lengthOf()};
+    reshapedSegments = idxSegments->reshape('c', shape, false);
+  }
+
+  auto segmentedOutput = OUTPUT_NULLIFIED(0);
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+  REQUIRE_TRUE(reshapedSegments->isVector(), 0,
+               "unsorted_segment_min: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(reshapedSegments->lengthOf() == input->sizeAt(0), 0,
+               "unsorted_segment_min: segment indexes array length should be equal to the input first dimension, but "
+               "%ld != %ld.",
+               reshapedSegments->lengthOf(), input->sizeAt(0));
+
+  LongType wrong;
+
+  REQUIRE_TRUE(helpers::unsortedSegmentIndicesValidate(block.launchContext(), reshapedSegments, numOfClasses, wrong),
+               0, "unsorted_segment_min: segment indices should be in range [0, %ld), but %ld != %ld", numOfClasses,
+               wrong, numOfClasses);
+  helpers::unsortedSegmentMinFunctor(block.launchContext(), reshapedInput, reshapedSegments, numOfClasses,
+                                     segmentedOutput);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_min) {
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+
+  if (INPUT_VARIABLE(0)->rankOf() >= 2) {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    outputShape[0] = outRank;
+    outputShape[1] = numOfClasses;
+    for (LongType i = 1; i < outRank; i++) outputShape[i + 1] = shape::sizeAt(in, i);
+
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  } else {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(1), sd::LongType);
+    outputShape[0] = 1;
+    outputShape[1] = numOfClasses;
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+  }
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+
+DECLARE_TYPES(unsorted_segment_min) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setSameMode(true);
+}
+
+CUSTOM_OP_IMPL(unsorted_segment_min_bp, 3, 2, false, 0, 1) {
+  return helpers::unsortedSegmentMinFunctorBP(block.launchContext(), INPUT_VARIABLE(0), INPUT_VARIABLE(1),
+                                              INPUT_VARIABLE(2), INT_ARG(0), OUTPUT_NULLIFIED(0));
+}
+
+DECLARE_TYPES(unsorted_segment_min_bp) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_FLOATS, ALL_INTS})
+      ->setSameMode(false);
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_min_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+
+}  // namespace ops
+
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_prod.cpp

@@ -0,0 +1,136 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 9/6/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_unsorted_segment_prod)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(unsorted_segment_prod, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto reshapedInput = input;
+
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto reshapedSegments = idxSegments;
+  if (!idxSegments->isVector() && idxSegments->rankOf() > 1) {
+    std::vector<sd::LongType> shape = {idxSegments->lengthOf()};
+    reshapedSegments = idxSegments->reshape('c',shape, false);
+  }
+
+  auto segmentedOutput = OUTPUT_NULLIFIED(0);
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+  REQUIRE_TRUE(reshapedSegments->isVector(), 0,
+               "unsorted_segment_prod: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(reshapedSegments->lengthOf() == input->sizeAt(0), 0,
+               "unsorted_segment_pod: segment indexes array length should be equal to the input first dimension, but "
+               "%ld != %ld.",
+               reshapedSegments->lengthOf(), input->sizeAt(0));
+
+  LongType wrong;
+
+  REQUIRE_TRUE(helpers::unsortedSegmentIndicesValidate(block.launchContext(), reshapedSegments, numOfClasses, wrong),
+               0, "unsorted_segment_pod: segment indices should be in range [0, %ld), but %ld != %ld", numOfClasses,
+               wrong, numOfClasses);
+  helpers::unsortedSegmentProdFunctor(block.launchContext(), reshapedInput, reshapedSegments, numOfClasses,
+                                      segmentedOutput);
+
+  delete reshapedSegments;
+  delete reshapedInput;
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_prod) {
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+
+  if (INPUT_VARIABLE(0)->rankOf() >= 2) {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    outputShape[0] = outRank;
+    outputShape[1] = numOfClasses;
+    for (LongType i = 1; i < outRank; i++) outputShape[i + 1] = shape::sizeAt(in, i);
+
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  } else {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(1), sd::LongType);
+    outputShape[0] = 1;
+    outputShape[1] = numOfClasses;
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+  }
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+DECLARE_TYPES(unsorted_segment_prod) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INDICES})
+      ->setSameMode(false);
+}
+
+CUSTOM_OP_IMPL(unsorted_segment_prod_bp, 3, 2, false, 0, 1) {
+  auto input = INPUT_VARIABLE(0);
+  auto indices = INPUT_VARIABLE(1);
+  auto eps = INPUT_VARIABLE(2);
+  //            auto numOfClasses = INT_ARG(0);
+  auto output = OUTPUT_NULLIFIED(0);
+
+  LongType numOfClasses = block.width() == 4 ? INPUT_VARIABLE(3)->e<LongType>(0) : INT_ARG(0);
+  REQUIRE_TRUE(indices->isVector(), 0,
+               "unsorted_segment_prod_bp: segment indexes array should be a vector, but it rank is %i.",
+               indices->rankOf());
+  REQUIRE_TRUE(indices->lengthOf() == input->sizeAt(0), 0,
+               "unsorted_segment_prod_bp: segment indexes array length should be equal to the input first dimension, "
+               "but %lld != %lld.",
+               indices->lengthOf(), input->sizeAt(0));
+
+  LongType wrong = numOfClasses;
+
+  REQUIRE_TRUE(helpers::unsortedSegmentIndicesValidate(block.launchContext(), indices, numOfClasses, wrong), 0,
+               "unsorted_segment_prod_bp: segment indices should be in range [0, %lld), but %lld > %lld", numOfClasses,
+               wrong, numOfClasses);
+
+  return helpers::unsortedSegmentProdFunctorBP(block.launchContext(), input, indices, eps, numOfClasses, output);
+}
+DECLARE_TYPES(unsorted_segment_prod_bp) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INDICES})
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INDICES})
+      ->setAllowedInputTypes(2, {ALL_FLOATS, ALL_INTS})
+      ->setSameMode(false);
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_prod_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+
+}  // namespace ops
+
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_sqrt_n.cpp

@@ -0,0 +1,117 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 9/6/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_unsorted_segment_sqrt_n)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(unsorted_segment_sqrt_n, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto reshapedInput = input;
+
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto reshapedSegments = idxSegments;
+  if (!idxSegments->isVector() && idxSegments->rankOf() > 1) {
+    std::vector<sd::LongType> shape = {idxSegments->lengthOf()};
+    reshapedSegments = idxSegments->reshape('c', shape, false);
+  }
+
+  auto segmentedOutput = OUTPUT_NULLIFIED(0);
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+  REQUIRE_TRUE(reshapedSegments->isVector(), 0,
+               "unsorted_segment_sqrt_n: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  REQUIRE_TRUE(reshapedSegments->lengthOf() == input->sizeAt(0), 0,
+               "unsorted_segment_sqrt_n: segment indexes array length should be equal to the input first dimension, "
+               "but %ld != %ld.",
+               reshapedSegments->lengthOf(), input->sizeAt(0));
+
+  LongType wrong;
+
+  REQUIRE_TRUE(helpers::unsortedSegmentIndicesValidate(block.launchContext(), reshapedSegments, numOfClasses, wrong),
+               0, "unsorted_segment_sqrt_n: segment indices should be in range [0, %ld), but %ld != %ld", numOfClasses,
+               wrong, numOfClasses);
+  helpers::unsortedSegmentSqrtNFunctor(block.launchContext(), reshapedInput, reshapedSegments, numOfClasses,
+                                       segmentedOutput);
+
+
+  delete reshapedInput;
+  delete reshapedSegments;
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_sqrt_n) {
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+
+  if (INPUT_VARIABLE(0)->rankOf() >= 2) {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    outputShape[0] = outRank;
+    outputShape[1] = numOfClasses;
+    for (LongType i = 1; i < outRank; i++) outputShape[i + 1] = shape::sizeAt(in, i);
+
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  } else {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(1), sd::LongType);
+    outputShape[0] = 1;
+    outputShape[1] = numOfClasses;
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+  }
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+DECLARE_TYPES(unsorted_segment_sqrt_n) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes({ALL_FLOATS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setSameMode(false);
+}
+
+CUSTOM_OP_IMPL(unsorted_segment_sqrt_n_bp, 3, 2, false, 0, 1) {
+  return helpers::unsortedSegmentSqrtNFunctorBP(block.launchContext(), INPUT_VARIABLE(0), INPUT_VARIABLE(1),
+                                                INPUT_VARIABLE(2), INT_ARG(0), OUTPUT_NULLIFIED(0));
+}
+DECLARE_TYPES(unsorted_segment_sqrt_n_bp) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_FLOATS})
+      ->setSameMode(false);
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_sqrt_n_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/unsorted_segment_sum.cpp

@@ -0,0 +1,117 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 9/6/2018.
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/segment.h>
+#if NOT_EXCLUDED(OP_unsorted_segment_sum)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(unsorted_segment_sum, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto reshapedInput = input;
+
+  auto idxSegments = INPUT_VARIABLE(1);
+  auto reshapedSegments = idxSegments;
+  if (!idxSegments->isVector() || idxSegments->rankOf() > 1) {
+    std::vector<sd::LongType> shape = {idxSegments->lengthOf()};
+    reshapedSegments = idxSegments->reshape('c', shape, false);
+  }
+
+  auto segmentedOutput = OUTPUT_NULLIFIED(0);
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+  REQUIRE_TRUE(reshapedSegments->isVector(), 0,
+               "unsorted_segment_sum: segment indexes array should be a vector, but it rank is %i.",
+               idxSegments->rankOf());
+  if(reshapedSegments->lengthOf() > 1)
+  REQUIRE_TRUE(reshapedSegments->lengthOf() == input->sizeAt(0), 0,
+               "unsorted_segment_sum: segment indexes array length should be equal to the input first dimension, but "
+               "%ld != %ld.",
+               reshapedSegments->lengthOf(), input->sizeAt(0));
+
+  LongType wrong;
+
+  REQUIRE_TRUE(helpers::unsortedSegmentIndicesValidate(block.launchContext(), reshapedSegments, numOfClasses, wrong),
+               0, "unsorted_segment_sum: segment indices should be in range [0, %ld), but %ld != %ld", numOfClasses,
+               wrong, numOfClasses);
+  helpers::unsortedSegmentSumFunctor(block.launchContext(), reshapedInput, reshapedSegments, numOfClasses,
+
+                                     segmentedOutput);
+
+
+
+
+  return Status::OK;
+}
+DECLARE_TYPES(unsorted_segment_sum) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setSameMode(false);
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_sum) {
+  auto in = inputShape->at(0);
+  int outRank = shape::rank(in);
+  LongType* outputShape = nullptr;
+  LongType numOfClasses = block.width() == 3 ? INPUT_VARIABLE(2)->e<LongType>(0) : INT_ARG(0);
+
+  if (INPUT_VARIABLE(0)->rankOf() >= 2) {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    outputShape[0] = outRank;
+    outputShape[1] = numOfClasses;
+    for (LongType i = 1; i < outRank; i++) outputShape[i + 1] = shape::sizeAt(in, i);
+
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+
+  } else {
+    ALLOCATE(outputShape, block.getWorkspace(), shape::shapeInfoLength(1), sd::LongType);
+    outputShape[0] = 1;
+    outputShape[1] = numOfClasses;
+    ShapeUtils::updateStridesAndType(outputShape, in, shape::order(in));
+  }
+
+  return SHAPELIST(CONSTANT(outputShape));
+}
+CUSTOM_OP_IMPL(unsorted_segment_sum_bp, 3, 2, false, 0, 1) {
+  return helpers::unsortedSegmentSumFunctorBP(block.launchContext(), INPUT_VARIABLE(0), INPUT_VARIABLE(1),
+                                              INPUT_VARIABLE(2), INT_ARG(0), OUTPUT_NULLIFIED(0));
+}
+
+DECLARE_SHAPE_FN(unsorted_segment_sum_bp) {
+  auto in = inputShape->at(0);
+  auto inIdx = inputShape->at(1);
+  return SHAPELIST(CONSTANT(in), CONSTANT(inIdx));
+}
+DECLARE_TYPES(unsorted_segment_sum_bp) {
+  getOpDescriptor()
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(ANY)
+      ->setSameMode(false);
+}
+
+}  // namespace ops
+
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/weighted_cross_entropy_with_logits.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author @shugeo
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_weighted_cross_entropy_with_logits)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(weighted_cross_entropy_with_logits, 3, 1, true) {
+  auto targets = INPUT_VARIABLE(0);
+  auto input = INPUT_VARIABLE(1);
+  auto weights = INPUT_VARIABLE(2);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(targets->isSameShape(input), 0,
+               "WEIGHTED_CROSS_ENTROPY_WITH_LOGITS op: The shape of both input params should be equal, but got "
+               "input_shape=%s and targets_shape=%s !",
+               ShapeUtils::shapeAsString(input).c_str(), ShapeUtils::shapeAsString(targets).c_str());
+  REQUIRE_TRUE(weights->isScalar() || targets->sizeAt(-1) == weights->lengthOf(), 0,
+               "WEIGHTED_CROSS_ENTROPY_WITH_LOGITS op: The weights should be scalar or vector with length equal to "
+               "size of last targets dimension, but got weights_shape=%s instead!",
+               ShapeUtils::shapeAsString(weights).c_str());
+
+  helpers::weightedCrossEntropyWithLogitsFunctor(block.launchContext(), targets, input, weights, output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(weighted_cross_entropy_with_logits) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/parity_ops/zero_fraction.cpp

@@ -0,0 +1,57 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> 31.01.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_zero_fraction)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(zero_fraction, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(output->isScalar(), 0, "Rank output should be scalar");
+
+  if (input->isEmpty()) {
+    output->p<double>(0, std::numeric_limits<double>::quiet_NaN());
+    return Status::OK;
+  }
+
+
+  auto countZero = input->reduceNumber(reduce::CountZero);
+  output->p<double>(0, countZero->e<LongType>(0) / double(input->lengthOf()));
+  delete countZero;
+  return Status::OK;
+}
+DECLARE_SHAPE_FN(zero_fraction) {
+  return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(sd::DataType::DOUBLE));
+}
+
+DECLARE_TYPES(zero_fraction) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif