chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/generic/nn/activations/crelu.cpp

@@ -0,0 +1,110 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_crelu)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+#include <ops/declarable/helpers/transforms.h>
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(crelu, 1, 1, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(x->isR(), 0, "CRELU: input must be real type");
+
+  auto tmp = x->dup(x->ordering());
+  tmp->applyTransform(transform::Neg, tmp);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  helpers::concat(block.launchContext(), {x, tmp}, *z, x->rankOf() - 1);
+
+  // TODO: make this configurable?
+  double threshold = 0.0;
+  z->applyScalar(scalar::RELU, threshold, z);
+
+  STORE_RESULT(z);
+
+  delete tmp;
+  return Status::OK;
+}
+
+DECLARE_TYPES(crelu) { getOpDescriptor()->setAllowedInputTypes(0, ANY)->setSameMode(true); }
+
+DECLARE_SHAPE_FN(crelu) {
+  auto inShape = inputShape->at(0);
+  std::vector<LongType> shape;
+  for (int e = 0; e < shape::rank(inShape); e++) shape.emplace_back(shape::shapeOf(inShape)[e]);
+
+  shape[shape.size() - 1] *= 2;
+  auto newShape =
+      ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), shape::order(inShape), shape);
+
+  return SHAPELIST(newShape);
+}
+
+CUSTOM_OP_IMPL(crelu_bp, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilonNext = INPUT_VARIABLE(1);
+  auto epsilon = OUTPUT_VARIABLE(0);
+
+  // at first step we build fwd activation
+  crelu op;
+  auto tmpResult = op.evaluate({input});
+  if (tmpResult.status() != Status::OK) return tmpResult.status();
+
+  auto actv = tmpResult.at(0);
+
+  // now we do RELU backward pass
+  helpers::reluDerivative(block.launchContext(), actv, epsilonNext);
+  // now we split updated array into 2 chunks along last dimension
+  concat_bp opc;
+  auto dec = opc.evaluate({input, input, actv}, {-1});
+  if (dec.status() != Status::OK) return dec.status();
+
+  // and now we subtract two parts of epsilons and pass result out
+  auto pos = dec.at(0);
+  auto neg = dec.at(1);
+
+  pos->applyPairwiseTransform(pairwise::Subtract, neg, epsilon);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(crelu_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+
+DECLARE_SHAPE_FN(crelu_bp) {
+  auto inShape = inputShape->at(0);
+  auto ret = SHAPELIST(ConstantShapeHelper::getInstance().bufferForShapeInfo(inShape)->primary());
+  return ret;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/cube.cpp

@@ -0,0 +1,61 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_cube)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(cube, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  input->applyTransform(transform::Cube, output);
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(cube) { getOpDescriptor()->setAllowedInputTypes(0, ANY)->setSameMode(true); }
+
+CONFIGURABLE_OP_IMPL(cube_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+  helpers::cubeDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+
+DECLARE_TYPES(cube_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/elu.cpp

@@ -0,0 +1,67 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_elu)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(elu, 1, 1, true, -2, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const auto alpha = block.numT() > 0 ? T_ARG(0) : 1.f;
+
+  input->applyScalar(scalar::ELU, alpha, output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(elu) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(elu_bp, 2, 1, true, -2, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  const auto alpha = block.numT() > 0 ? T_ARG(0) : 1.f;
+
+  helpers::eluDerivative(block.launchContext(), input, epsilon, output, alpha);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(elu_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/hardsigmoid.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_hardsigmoid)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(hardsigmoid, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  input->applyTransform(transform::HardSigmoid, output);
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(hardsigmoid) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(hardsigmoid_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  helpers::hardSigmoidDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+
+DECLARE_TYPES(hardsigmoid_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/hardtanh.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_hardtanh)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(hardtanh, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  input->applyTransform(transform::HardTanh, output);
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(hardtanh) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(hardtanh_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  helpers::hardTanhDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+
+DECLARE_TYPES(hardtanh_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/identity.cpp

@@ -0,0 +1,66 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_identity)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(identity, 1, 1, true) {
+  auto z = OUTPUT_VARIABLE(0);
+
+  if (!block.isInplace()) {
+    auto first = INPUT_VARIABLE(0);
+
+    // we hope for memcpy here
+    z->assign(first);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(linear, identity);
+
+DECLARE_TYPES(identity) { getOpDescriptor()->setAllowedInputTypes(0, ANY)->setSameMode(true); }
+
+OP_IMPL(identity_bp, 2, 1, true) {
+  auto first = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  z->assign(epsilon);
+
+  return Status::OK;
+}
+DECLARE_SYN(LinearGrad, identity_bp);
+
+DECLARE_TYPES(identity_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/identity_n.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_identity_n)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(identity_n, 1, 1, true, 0, 0) {
+  if (!block.isInplace()) {
+    for (size_t i = 0; i < block.width(); ++i) {
+      auto x = INPUT_VARIABLE(i);
+      auto z = OUTPUT_VARIABLE(i);
+
+      x->applyTransform(transform::Identity, z);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(identity_n) {
+  auto shapes = SHAPELIST();
+  for (int i = 0; i < inputShape->size(); ++i) {
+    shapes->push_back(CONSTANT(inputShape->at(i)));
+  }
+  return shapes;
+}
+
+DECLARE_TYPES(identity_n) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(ANY);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/lrelu.cpp

@@ -0,0 +1,67 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_lrelu)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(lrelu, 1, 1, true, -2, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  float alpha = block.numT() > 0 ? T_ARG(0) : 0.01f;
+
+  input->applyScalar(scalar::LeakyRELU, alpha, output);
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(lrelu) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(lrelu_bp, 2, 1, true, -2, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  float alpha = block.numT() > 0 ? T_ARG(0) : 0.01f;
+
+  helpers::leakyReluDerivative(block.launchContext(), input, epsilon, z, alpha);
+  return Status::OK;
+}
+
+DECLARE_TYPES(lrelu_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/prelu.cpp

@@ -0,0 +1,171 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 24.07.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_prelu)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/activations.h>
+
+#include <numeric>
+
+namespace sd {
+namespace ops {
+
+////////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(prelu, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto alpha = INPUT_VARIABLE(1);
+  auto output = OUTPUT_VARIABLE(0);
+
+  std::vector<LongType> sharedAxes = *block.getIArguments();
+
+  const int inputRank = input->rankOf();
+  const int numSharedAxes = sharedAxes.size();  // can be zero as well
+  const LongType inputLen = input->lengthOf();
+  const LongType alphaLen = alpha->lengthOf();
+  auto* inputShapeVec = input->getShapeAsVector();
+  auto* alphaShapeVec = alpha->getShapeAsVector();
+  const std::vector<LongType> inputShape = *inputShapeVec;
+  const std::vector<LongType> alphaShape = *alphaShapeVec;
+  delete inputShapeVec;
+  delete alphaShapeVec;
+
+  //***** input validation *****//
+  std::vector<LongType> expectedAlphaShape(&inputShape[1], &inputShape[inputRank]);
+
+  REQUIRE_TRUE(inputRank > 1, 0,
+               "PRELU OP: wrong rank of input array, expected rank should be > 1, but got %i instead !", inputRank);
+
+  for (int i = 0; i < numSharedAxes; ++i) {
+    if (sharedAxes[i] <= 0) sharedAxes[i] += inputRank - 1;
+    REQUIRE_TRUE(1 <= sharedAxes[i] && sharedAxes[i] <= inputRank - 1, 0,
+                 "PRELU OP: wrong axis value %i in sharedAxes at position %i, axis value must be within range [1, "
+                 "input_rank-1] !",
+                 sharedAxes[i], i);
+    expectedAlphaShape[sharedAxes[i] - 1] = 1;
+  }
+
+
+  NDArray *alpha2 =  alphaShape != expectedAlphaShape ? alpha->reshape(alpha->ordering(), expectedAlphaShape) : alpha;
+  helpers::prelu(block.launchContext(), input,
+                 alpha2,
+                 output);
+
+  if(alpha2 != alpha) delete alpha2;
+  return Status::OK;
+}
+
+DECLARE_TYPES(prelu) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+////////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(prelu_bp, 3, 2, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto alpha = INPUT_VARIABLE(1);
+  auto dLdO = INPUT_VARIABLE(2);
+
+  auto dLdI = OUTPUT_VARIABLE(0);
+  auto dLdA = OUTPUT_VARIABLE(1);
+
+  std::vector<LongType> sharedAxes = *block.getIArguments();
+
+  const int inputRank = input->rankOf();
+  const int numSharedAxes = sharedAxes.size();  // can be zero as well
+  const LongType inputLen = input->lengthOf();
+  const LongType alphaLen = alpha->lengthOf();
+  auto* inputShapeVec = input->getShapeAsVector();
+  auto* alphaShapeVec = alpha->getShapeAsVector();
+  const std::vector<LongType> inputShape = *inputShapeVec;
+  const std::vector<LongType> alphaShape = *alphaShapeVec;
+  delete inputShapeVec;
+  delete alphaShapeVec;
+
+  //***** input validation *****//
+
+  // temporary limitation imposed by Yurii
+  REQUIRE_TRUE(inputRank <= SD_MAX_RANK / 2, 0, "rank of input array should be <= SD_MAX_RANK/2, but got %i instead!",
+               inputRank);
+  REQUIRE_TRUE(input->lengthOf() / alpha->lengthOf() <= SD_MAX_RANK * 2, 0,
+               "the length of input array should be no more than SD_MAX_RANK*2 times the alpha array length, but got "
+               "%lld and %lld correspondingly!",
+               input->lengthOf(), alpha->lengthOf());
+
+  std::vector<LongType> expectedAlphaShape(&inputShape[1], &inputShape[inputRank]);
+
+  REQUIRE_TRUE(inputRank > 1, 0,
+               "PRELU_BP OP: wrong rank of input array, expected rank should be > 1, but got %i instead !", inputRank);
+
+  for (int i = 0; i < numSharedAxes; ++i) {
+    if (sharedAxes[i] <= 0) sharedAxes[i] += inputRank - 1;
+    REQUIRE_TRUE(1 <= sharedAxes[i] && sharedAxes[i] <= inputRank - 1, 0,
+                 "PRELU_BP OP: wrong axis value %i in sharedAxes at position %i, axis value must be within range [1, "
+                 "input_rank-1] !",
+                 sharedAxes[i], i);
+    expectedAlphaShape[sharedAxes[i] - 1] = 1;
+  }
+
+  LongType product = 1;
+  for (const auto& item : expectedAlphaShape) product *= item;
+
+  REQUIRE_TRUE(product == alphaLen, 0, "PRELU_BP OP: wrong shape of alpha array, expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(expectedAlphaShape).c_str(), ShapeUtils::shapeAsString(alphaShape).c_str());
+  // ***** end of validation ***** //
+
+  NDArray* alphaReshaped = nullptr;
+  NDArray* dLdAReshaped = nullptr;
+  
+  if (alphaShape != expectedAlphaShape) {
+    alphaReshaped = alpha->reshape(alpha->ordering(), expectedAlphaShape);
+    dLdAReshaped = dLdA->reshape(dLdA->ordering(), expectedAlphaShape);
+  }
+
+  helpers::preluBP(block.launchContext(), input, 
+                   alphaReshaped != nullptr ? alphaReshaped : alpha, 
+                   dLdO, dLdI, 
+                   dLdAReshaped != nullptr ? dLdAReshaped : dLdA);
+
+  if (alphaReshaped != nullptr) {
+    delete alphaReshaped;
+    delete dLdAReshaped;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(prelu_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedInputTypes(2, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(1, {FLOAT32, DOUBLE, HALF});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/rationaltanh.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_rationaltanh)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(rationaltanh, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  input->applyTransform(transform::RationalTanh, output);
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(rationaltanh) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(rationaltanh_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  helpers::rationalTanhDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+
+DECLARE_TYPES(rationaltanh_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/rectifiedtanh.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_rectifiedtanh)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(rectifiedtanh, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  input->applyTransform(transform::RectifiedTanh, output);
+  STORE_RESULT(output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(rectifiedtanh) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(rectifiedtanh_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  helpers::rectifiedTanhDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+
+DECLARE_TYPES(rectifiedtanh_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/relu.cpp

@@ -0,0 +1,65 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_relu)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(relu, 1, 1, true, 1, 0) {
+  auto first = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  auto scalar = block.numT() > 0 ? block.getTArguments()->at(0) : 0.0;
+
+  first->applyScalar(scalar::RELU, scalar, z);
+
+  STORE_RESULT(*z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(relu) { getOpDescriptor()->setAllowedInputTypes(0, ANY)->setSameMode(true); }
+
+CONFIGURABLE_OP_IMPL(relu_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+  helpers::reluDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+DECLARE_SYN(ReluGrad, relu_bp);
+
+DECLARE_TYPES(relu_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/relu6.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma, created on 16.02.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_relu6)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+
+////////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(relu6, 1, 1, true, 1, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  input->applyScalar(scalar::RELU6, T_ARG(0), output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(relu6) { getOpDescriptor()->setAllowedInputTypes(0, ANY)->setSameMode(true); }
+
+////////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(relu6_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);
+  auto gradI = OUTPUT_VARIABLE(0);
+
+  helpers::relu6Derivative(block.launchContext(), input, gradO, gradI);
+  return Status::OK;
+}
+
+DECLARE_TYPES(relu6_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/selu.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_selu)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(selu, 1, 1, true, 0, 0) {
+  auto first = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  first->applyTransform(transform::SELU, z);
+
+  STORE_RESULT(*z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(selu) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(selu_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+  helpers::seluDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+
+DECLARE_TYPES(selu_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/sigmoid.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_sigmoid)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(sigmoid, 1, 1, true, 0, 0) {
+  auto first = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  first->applyTransform(transform::Sigmoid, z);
+
+  STORE_RESULT(*z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(sigmoid) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(sigmoid_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  helpers::sigmoidDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+
+DECLARE_TYPES(sigmoid_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/softplus.cpp

@@ -0,0 +1,66 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_softplus)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(softplus, 1, 1, true, 0, 0) {
+  auto first = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  first->applyTransform(transform::SoftPlus, z);
+
+  STORE_RESULT(*z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(softplus) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(softplus_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+
+  helpers::softPlusDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+DECLARE_SYN(SoftplusGrad, softplus_bp);
+
+DECLARE_TYPES(softplus_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/softsign.cpp

@@ -0,0 +1,66 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_softsign)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(softsign, 1, 1, true, 0, 0) {
+  auto first = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  first->applyTransform(transform::SoftSign, z);
+
+  STORE_RESULT(*z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(softsign) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(softsign_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+  auto z = OUTPUT_VARIABLE(0);
+  helpers::softSignDerivative(block.launchContext(), input, epsilon, z);
+
+  return Status::OK;
+}
+DECLARE_SYN(SoftsignGrad, softsign_bp);
+
+DECLARE_TYPES(softsign_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/tanh.cpp

@@ -0,0 +1,66 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_tanh)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(tanh, 1, 1, true, 0, 0) {
+  auto first = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  first->applyTransform(transform::Tanh, z);
+
+  STORE_RESULT(*z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(tanh) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+CONFIGURABLE_OP_IMPL(tanh_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto epsilon = INPUT_VARIABLE(1);
+
+
+  auto z = OUTPUT_VARIABLE(0);
+  helpers::tanhDerivative(block.launchContext(), input, epsilon, z);
+  return Status::OK;
+}
+DECLARE_SYN(TanhGrad, tanh_bp);
+
+DECLARE_TYPES(tanh_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/nn/activations/thresholdedrelu.cpp

@@ -0,0 +1,69 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma, created on 24.07.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_thresholdedrelu)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/activations.h>
+#include <ops/declarable/helpers/legacy_helpers.h>
+namespace sd {
+namespace ops {
+
+////////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(thresholdedrelu, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  auto scalar = block.numT() > 0 ? block.getTArguments()->at(0) : 0.0;
+
+  helpers::thresholdRelu(block.launchContext(), input, scalar, output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(thresholdedrelu) { getOpDescriptor()->setAllowedInputTypes(0, ANY)->setSameMode(true); }
+
+////////////////////////////////////////////////////////////////////////
+CONFIGURABLE_OP_IMPL(thresholdedrelu_bp, 2, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto dLdO = INPUT_VARIABLE(1);
+
+  auto dLdI = OUTPUT_VARIABLE(0);
+  auto threshold = block.numT() > 0 ? block.getTArguments()->at(0) : 0.0;
+
+  helpers::thresholdReluDerivative(block.launchContext(), input, threshold, dLdO, dLdI);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(thresholdedrelu_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif