chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/generic/updaters/adaBeliefUpdater.cpp

@@ -0,0 +1,100 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+// @author Abdelrauf(rauf@konduit.ai)
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_adabelief_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(adabelief_updater, 3, 3, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initStateU = INPUT_VARIABLE(1);
+  const auto initStateM = INPUT_VARIABLE(2);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateU = OUTPUT_VARIABLE(1);
+  auto stateM = OUTPUT_VARIABLE(2);
+
+  // todo maybe we need an error like on Java side
+  if (gradient->isEmpty() || initStateU->isEmpty() || initStateM->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(gradient->isSameShape(initStateU), 0,
+               "ADABELIEF UPDATER OP: input state V must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateU->shapeInfo()).c_str());
+  REQUIRE_TRUE(gradient->isSameShape(initStateM), 0,
+               "ADABELIEF UPDATER OP: input state M must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateM->shapeInfo()).c_str());
+
+  bool bParamsSupply = 7 == block.width() || 4 == block.getTArguments()->size();
+
+  auto iteration = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+
+  REQUIRE_TRUE(bParamsSupply, 0, "ADABELIEF UPDATER OP: learning rate, beta 1, beta 2 and epsilon were not provided!");
+
+  double dLr, dBeta1, dBeta2, dEpsilon;
+
+  if (block.width() > 3) {
+    const auto lr = INPUT_VARIABLE(3);
+    const auto beta1 = INPUT_VARIABLE(4);
+    const auto beta2 = INPUT_VARIABLE(5);
+    const auto epsilon = INPUT_VARIABLE(6);
+
+    REQUIRE_TRUE(lr->isScalar(), 0, "ADABELIEF UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+    REQUIRE_TRUE(beta1->isScalar(), 0, "ADABELIEF UPDATER OP: beta 1 has to be a scalar, but instead got rank %i!",
+                 beta1->rankOf());
+    REQUIRE_TRUE(beta2->isScalar(), 0, "ADABELIEF UPDATER OP: beta 2 has to be a scalar, but instead got rank %i!",
+                 beta2->rankOf());
+    REQUIRE_TRUE(epsilon->isScalar(), 0, "ADABELIEF UPDATER OP: Epsilon has to be a scalar, but instead got rank %i!",
+                 epsilon->rankOf());
+
+    dLr = lr->e<double>(0);
+    dBeta1 = beta1->e<double>(0);
+    dBeta2 = beta2->e<double>(0);
+    dEpsilon = epsilon->e<double>(0);
+  } else {
+    dLr = T_ARG(0);
+    dBeta1 = T_ARG(1);
+    dBeta2 = T_ARG(2);
+    dEpsilon = T_ARG(3);
+  }
+
+  helpers::updaterAdaBelief(block.launchContext(), *gradient, *initStateU, *initStateM, *update, *stateU, *stateM, dLr,
+                            dBeta1, dBeta2, dEpsilon, iteration);
+  return Status::OK;
+}
+
+DECLARE_TYPES(adabelief_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/updaters/adaDeltaUpdater.cpp

@@ -0,0 +1,87 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_ada_delta_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(ada_delta_updater, 3, 3, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initStateMsg = INPUT_VARIABLE(1);
+  const auto initStateMsdx = INPUT_VARIABLE(2);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateMsg = OUTPUT_VARIABLE(1);
+  auto stateMsdx = OUTPUT_VARIABLE(2);
+
+  if (gradient->isEmpty() || initStateMsg->isEmpty() || initStateMsdx->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(gradient->isSameShape(initStateMsg), 0,
+               "ADA_DELTA UPDATER OP: input state Msg must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateMsg->shapeInfo()).c_str());
+  REQUIRE_TRUE(gradient->isSameShape(initStateMsdx), 0,
+               "ADA_DELTA UPDATER OP: input state Msdx must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateMsdx->shapeInfo()).c_str());
+
+  bool bParamsSupply = 5 == block.width() || 2 == block.getTArguments()->size();
+
+  REQUIRE_TRUE(bParamsSupply, 0, "ADA_DELTA UPDATER OP: Rho and epsilon were not provided!");
+
+  double dRho, dEpsilon;
+
+  if (block.width() > 3) {
+    const auto rho = INPUT_VARIABLE(3);
+    const auto epsilon = INPUT_VARIABLE(4);
+
+    REQUIRE_TRUE(rho->isScalar(), 0, "ADA_DELTA UPDATER OP: Rho has to be a scalar, but instead got rank %i!",
+                 rho->rankOf());
+    REQUIRE_TRUE(epsilon->isScalar(), 0, "ADA_DELTA UPDATER OP: Epsilon has to be a scalar, but instead got rank %i!",
+                 epsilon->rankOf());
+
+    dRho = rho->e<double>(0);
+    dEpsilon = epsilon->e<double>(0);
+  } else {
+    dRho = T_ARG(0);
+    dEpsilon = T_ARG(1);
+  }
+
+  helpers::updaterAdaDelta(block.launchContext(), *gradient, *initStateMsg, *initStateMsdx, *update, *stateMsg,
+                           *stateMsdx, dRho, dEpsilon);
+  return Status::OK;
+}
+
+DECLARE_TYPES(ada_delta_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/updaters/adaGradUpdater.cpp

@@ -0,0 +1,79 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_ada_grad_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(ada_grad_updater, 2, 2, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initState = INPUT_VARIABLE(1);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateH = OUTPUT_VARIABLE(1);
+
+  if (gradient->isEmpty() || initState->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(gradient->isSameShape(initState), 0,
+               "ADA_GRAD UPDATER OP: input state must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initState->shapeInfo()).c_str());
+
+  bool bParamsSupply = 4 == block.width() || 2 == block.getTArguments()->size();
+
+  REQUIRE_TRUE(bParamsSupply, 0, "ADA_GRAD UPDATER OP: learning rate and epsilon were not provided!");
+
+  double dLr, dEpsilon;
+
+  if (block.width() > 2) {
+    const auto lr = INPUT_VARIABLE(2);
+    const auto epsilon = INPUT_VARIABLE(3);
+
+    REQUIRE_TRUE(lr->isScalar(), 0, "ADA_GRAD UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+    REQUIRE_TRUE(epsilon->isScalar(), 0, "ADA_GRAD UPDATER OP: Epsilon has to be a scalar, but instead got rank %i!",
+                 epsilon->rankOf());
+
+    dLr = lr->e<double>(0);
+    dEpsilon = epsilon->e<double>(0);
+  } else {
+    dLr = T_ARG(0);
+    dEpsilon = T_ARG(1);
+  }
+
+  helpers::updaterAdaGrad(block.launchContext(), *gradient, *initState, *update, *stateH, dLr, dEpsilon);
+  return Status::OK;
+}
+
+DECLARE_TYPES(ada_grad_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/updaters/adaMaxUpdater.cpp

@@ -0,0 +1,100 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_ada_max_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(ada_max_updater, 3, 3, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initStateU = INPUT_VARIABLE(1);
+  const auto initStateM = INPUT_VARIABLE(2);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateU = OUTPUT_VARIABLE(1);
+  auto stateM = OUTPUT_VARIABLE(2);
+
+  // todo maybe we need an error like on Java side
+  if (gradient->isEmpty() || initStateU->isEmpty() || initStateM->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(gradient->isSameShape(initStateU), 0,
+               "ADA_MAX UPDATER OP: input state V must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateU->shapeInfo()).c_str());
+  REQUIRE_TRUE(gradient->isSameShape(initStateM), 0,
+               "ADA_MAX UPDATER OP: input state M must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateM->shapeInfo()).c_str());
+
+  bool bParamsSupply = 7 == block.width() || 4 == block.getTArguments()->size();
+
+  int iteration = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+
+  REQUIRE_TRUE(bParamsSupply, 0, "ADA_MAX UPDATER OP: learning rate, beta 1, beta 2 and epsilon were not provided!");
+
+  double dLr, dBeta1, dBeta2, dEpsilon;
+
+  if (block.width() > 3) {
+    const auto lr = INPUT_VARIABLE(3);
+    const auto beta1 = INPUT_VARIABLE(4);
+    const auto beta2 = INPUT_VARIABLE(5);
+    const auto epsilon = INPUT_VARIABLE(6);
+
+    REQUIRE_TRUE(lr->isScalar(), 0, "ADA_MAX UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+    REQUIRE_TRUE(beta1->isScalar(), 0, "ADA_MAX UPDATER OP: beta 1 has to be a scalar, but instead got rank %i!",
+                 beta1->rankOf());
+    REQUIRE_TRUE(beta2->isScalar(), 0, "ADA_MAX UPDATER OP: beta 2 has to be a scalar, but instead got rank %i!",
+                 beta2->rankOf());
+    REQUIRE_TRUE(epsilon->isScalar(), 0, "ADA_MAX UPDATER OP: Epsilon has to be a scalar, but instead got rank %i!",
+                 epsilon->rankOf());
+
+    dLr = lr->e<double>(0);
+    dBeta1 = beta1->e<double>(0);
+    dBeta2 = beta2->e<double>(0);
+    dEpsilon = epsilon->e<double>(0);
+  } else {
+    dLr = T_ARG(0);
+    dBeta1 = T_ARG(1);
+    dBeta2 = T_ARG(2);
+    dEpsilon = T_ARG(3);
+  }
+
+  helpers::updaterAdaMax(block.launchContext(), *gradient, *initStateU, *initStateM, *update, *stateU, *stateM, dLr,
+                         dBeta1, dBeta2, dEpsilon, iteration);
+  return Status::OK;
+}
+
+DECLARE_TYPES(ada_max_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/updaters/adamUpdater.cpp

@@ -0,0 +1,101 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_adam_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(adam_updater, 3, 3, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initStateU = INPUT_VARIABLE(1);
+  const auto initStateM = INPUT_VARIABLE(2);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateU = OUTPUT_VARIABLE(1);
+  auto stateM = OUTPUT_VARIABLE(2);
+
+  if (gradient->isEmpty() || initStateU->isEmpty() || initStateM->isEmpty())
+    THROW_EXCEPTION("adam_updater: Unable to apply empty update");
+
+  REQUIRE_TRUE(gradient->isSameShape(initStateU), 0,
+               "ADAM UPDATER OP: input state V must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateU->shapeInfo()).c_str());
+  REQUIRE_TRUE(gradient->isSameShape(initStateM), 0,
+               "ADAM UPDATER OP: input state M must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateM->shapeInfo()).c_str());
+
+  bool bParamsSupply = 7 == block.width() || 4 == block.getTArguments()->size();
+
+  auto iteration = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+
+  REQUIRE_TRUE(bParamsSupply, 0, "ADAM UPDATER OP: learning rate, beta 1, beta 2 and epsilon were not provided!");
+
+  double dLr, dBeta1, dBeta2, dEpsilon;
+
+  if (block.width() > 3) {
+    const auto lr = INPUT_VARIABLE(3);
+    const auto beta1 = INPUT_VARIABLE(4);
+    const auto beta2 = INPUT_VARIABLE(5);
+    const auto epsilon = INPUT_VARIABLE(6);
+
+    REQUIRE_TRUE(lr->isScalar(), 0, "ADAM UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+    REQUIRE_TRUE(beta1->isScalar(), 0, "ADAM UPDATER OP: beta 1 has to be a scalar, but instead got rank %i!",
+                 beta1->rankOf());
+    REQUIRE_TRUE(beta2->isScalar(), 0, "ADAM UPDATER OP: beta 2 has to be a scalar, but instead got rank %i!",
+                 beta2->rankOf());
+    REQUIRE_TRUE(epsilon->isScalar(), 0, "ADAM UPDATER OP: Epsilon has to be a scalar, but instead got rank %i!",
+                 epsilon->rankOf());
+
+    dLr = lr->e<double>(0);
+    dBeta1 = beta1->e<double>(0);
+    dBeta2 = beta2->e<double>(0);
+    dEpsilon = epsilon->e<double>(0);
+  } else {
+    dLr = T_ARG(0);
+    dBeta1 = T_ARG(1);
+    dBeta2 = T_ARG(2);
+    dEpsilon = T_ARG(3);
+  }
+
+
+  helpers::updaterAdam(block.launchContext(), *gradient, *initStateU, *initStateM, *update, *stateU, *stateM, dLr,
+                       dBeta1, dBeta2, dEpsilon, iteration);
+  return Status::OK;
+}
+
+DECLARE_TYPES(adam_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/updaters/amsGradUpdater.cpp

@@ -0,0 +1,109 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_ams_grad_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(ams_grad_updater, 4, 4, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initStateV = INPUT_VARIABLE(1);
+  const auto initStateM = INPUT_VARIABLE(2);
+  const auto initStateH = INPUT_VARIABLE(3);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateV = OUTPUT_VARIABLE(1);
+  auto stateM = OUTPUT_VARIABLE(2);
+  auto stateH = OUTPUT_VARIABLE(3);
+
+  // todo maybe we need an error like on Java side
+  if (gradient->isEmpty() || initStateV->isEmpty() || initStateM->isEmpty() || initStateH->isEmpty())
+    return Status::OK;
+
+  REQUIRE_TRUE(gradient->isSameShape(initStateV), 0,
+               "AMSGRAD UPDATER OP: input state Msg must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateV->shapeInfo()).c_str());
+  REQUIRE_TRUE(gradient->isSameShape(initStateM), 0,
+               "AMSGRAD UPDATER OP: input state Msdx must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateM->shapeInfo()).c_str());
+  REQUIRE_TRUE(gradient->isSameShape(initStateH), 0,
+               "AMSGRAD UPDATER OP: input state Msdx must have the same shape as gradient!,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateH->shapeInfo()).c_str());
+
+  bool bParamsSupply = 8 == block.width() || 4 == block.getTArguments()->size();
+
+  auto iteration = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+
+  REQUIRE_TRUE(bParamsSupply, 0, "AMSGRAD UPDATER OP: learning rate, beta 1, beta 2 and epsilon were not provided!");
+
+  double dLr, dBeta1, dBeta2, dEpsilon;
+
+  if (block.width() > 4) {
+    const auto lr = INPUT_VARIABLE(4);
+    const auto beta1 = INPUT_VARIABLE(5);
+    const auto beta2 = INPUT_VARIABLE(6);
+    const auto epsilon = INPUT_VARIABLE(7);
+
+    REQUIRE_TRUE(lr->isScalar(), 0, "AMSGRAD UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+    REQUIRE_TRUE(beta1->isScalar(), 0, "AMSGRAD UPDATER OP: beta 1 has to be a scalar, but instead got rank %i!",
+                 beta1->rankOf());
+    REQUIRE_TRUE(beta2->isScalar(), 0, "AMSGRAD UPDATER OP: beta 2 has to be a scalar, but instead got rank %i!",
+                 beta2->rankOf());
+    REQUIRE_TRUE(epsilon->isScalar(), 0, "AMSGRAD UPDATER OP: Epsilon has to be a scalar, but instead got rank %i!",
+                 epsilon->rankOf());
+
+    dLr = lr->e<double>(0);
+    dBeta1 = beta1->e<double>(0);
+    dBeta2 = beta2->e<double>(0);
+    dEpsilon = epsilon->e<double>(0);
+  } else {
+    dLr = T_ARG(0);
+    dBeta1 = T_ARG(1);
+    dBeta2 = T_ARG(2);
+    dEpsilon = T_ARG(3);
+  }
+
+  helpers::updaterAmsGrad(block.launchContext(), *gradient, *initStateV, *initStateM, *initStateH, *update, *stateV,
+                          *stateM, *stateH, dLr, dBeta1, dBeta2, dEpsilon, iteration);
+  return Status::OK;
+}
+
+DECLARE_TYPES(ams_grad_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/updaters/nadamUpdater.cpp

@@ -0,0 +1,100 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_nadam_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(nadam_updater, 3, 3, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initStateV = INPUT_VARIABLE(1);
+  const auto initStateM = INPUT_VARIABLE(2);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateV = OUTPUT_VARIABLE(1);
+  auto stateM = OUTPUT_VARIABLE(2);
+
+  // todo maybe we need an error like on Java side
+  if (gradient->isEmpty() || initStateV->isEmpty() || initStateM->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(gradient->isSameShape(initStateM), 0,
+               "NADAM UPDATER OP: input state M must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateM->shapeInfo()).c_str());
+  REQUIRE_TRUE(gradient->isSameShape(initStateV), 0,
+               "NADAM UPDATER OP: input state V must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initStateV->shapeInfo()).c_str());
+
+  bool bParamsSupply = 7 == block.width() || 4 == block.getTArguments()->size();
+
+  auto nIteration = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+
+  REQUIRE_TRUE(bParamsSupply, 0, "NADAM UPDATER OP: learning rate, beta 1, beta 2 and epsilon were not provided!");
+
+  double dLr, dBeta1, dBeta2, dEpsilon;
+
+  if (block.width() > 3) {
+    const auto lr = INPUT_VARIABLE(3);
+    const auto beta1 = INPUT_VARIABLE(4);
+    const auto beta2 = INPUT_VARIABLE(5);
+    const auto epsilon = INPUT_VARIABLE(6);
+
+    REQUIRE_TRUE(lr->isScalar(), 0, "NADAM UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+    REQUIRE_TRUE(beta1->isScalar(), 0, "NADAM UPDATER OP: beta 1 has to be a scalar, but instead got rank %i!",
+                 beta1->rankOf());
+    REQUIRE_TRUE(beta2->isScalar(), 0, "NADAM UPDATER OP: beta 2 has to be a scalar, but instead got rank %i!",
+                 beta2->rankOf());
+    REQUIRE_TRUE(epsilon->isScalar(), 0, "NADAM UPDATER OP: Epsilon has to be a scalar, but instead got rank %i!",
+                 epsilon->rankOf());
+
+    dLr = lr->e<double>(0);
+    dBeta1 = beta1->e<double>(0);
+    dBeta2 = beta2->e<double>(0);
+    dEpsilon = epsilon->e<double>(0);
+  } else {
+    dLr = T_ARG(0);
+    dBeta1 = T_ARG(1);
+    dBeta2 = T_ARG(2);
+    dEpsilon = T_ARG(3);
+  }
+
+  helpers::updaterNadam(block.launchContext(), *gradient, *initStateV, *initStateM, *update, *stateV, *stateM, dLr,
+                        dBeta1, dBeta2, dEpsilon, nIteration);
+  return Status::OK;
+}
+
+DECLARE_TYPES(nadam_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/updaters/nesterovsUpdater.cpp

@@ -0,0 +1,78 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_nesterovs_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(nesterovs_updater, 2, 2, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initState = INPUT_VARIABLE(1);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateV = OUTPUT_VARIABLE(1);
+
+  if (gradient->isEmpty() || initState->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(gradient->isSameShape(initState), 0,
+               "NESTEROVS UPDATER OP: input state Msg must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initState->shapeInfo()).c_str());
+
+  bool bParamsSupply = 4 == block.width() || 2 == block.getTArguments()->size();
+
+  REQUIRE_TRUE(bParamsSupply, 0, "NESTEROVS UPDATER OP: learning rate and momentum were not provided!");
+
+  double dLr, dMomentum;
+
+  if (block.width() > 2) {
+    const auto lr = INPUT_VARIABLE(2);
+    const auto momentum = INPUT_VARIABLE(3);
+
+    REQUIRE_TRUE(lr->isScalar(), 0, "NESTEROVS UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+    REQUIRE_TRUE(momentum->isScalar(), 0, "NESTEROVS UPDATER OP: Momentum has to be a scalar, but instead got rank %i!",
+                 momentum->rankOf());
+
+    dLr = lr->e<double>(0);
+    dMomentum = momentum->e<double>(0);
+  } else {
+    dLr = T_ARG(0);
+    dMomentum = T_ARG(1);
+  }
+  helpers::updaterNesterovs(block.launchContext(), *gradient, *initState, *update, *stateV, dLr, dMomentum);
+  return Status::OK;
+}
+
+DECLARE_TYPES(nesterovs_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/updaters/rmsPropUpdater.cpp

@@ -0,0 +1,84 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+#if NOT_EXCLUDED(OP_rms_prop_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(rms_prop_updater, 2, 2, true, 0, 0) {
+  const auto gradient = INPUT_VARIABLE(0);
+  const auto initState = INPUT_VARIABLE(1);
+
+  auto update = OUTPUT_VARIABLE(0);
+  auto stateG = OUTPUT_VARIABLE(1);
+
+  if (gradient->isEmpty() || initState->isEmpty()) return Status::OK;
+
+  REQUIRE_TRUE(gradient->isSameShape(initState), 0,
+               "RMS_PROB UPDATER OP: input state must have the same shape as gradient,"
+               "  expected shape %s, but got %s!",
+               ShapeUtils::shapeAsString(gradient->shapeInfo()).c_str(),
+               ShapeUtils::shapeAsString(initState->shapeInfo()).c_str());
+
+  bool bParamsSupply = 5 == block.width() || 3 == block.getTArguments()->size();
+
+  REQUIRE_TRUE(bParamsSupply, 0, "RSM_PROB UPDATER OP: learning rate, rsm decay and epsilon were not provided!");
+
+  double dLr, dRmsDecay, dEpsilon;
+
+  if (block.width() > 2) {
+    const auto lr = INPUT_VARIABLE(2);
+    const auto rmsDecay = INPUT_VARIABLE(3);
+    const auto epsilon = INPUT_VARIABLE(4);
+
+    REQUIRE_TRUE(lr->isScalar(), 0, "RSM_PROB UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+    REQUIRE_TRUE(rmsDecay->isScalar(), 0, "RSM_PROB UPDATER OP: Rms decay has to be a scalar, but instead got rank %i!",
+                 rmsDecay->rankOf());
+    REQUIRE_TRUE(epsilon->isScalar(), 0, "RSM_PROB UPDATER OP: Epsilon has to be a scalar, but instead got rank %i!",
+                 epsilon->rankOf());
+
+    dLr = lr->e<double>(0);
+    dRmsDecay = rmsDecay->e<double>(0);
+    dEpsilon = epsilon->e<double>(0);
+  } else {
+    dLr = T_ARG(0);
+    dRmsDecay = T_ARG(1);
+    dEpsilon = T_ARG(2);
+  }
+
+  helpers::updaterRmsProp(block.launchContext(), *gradient, *initState, *update, *stateG, dLr, dRmsDecay, dEpsilon);
+  return Status::OK;
+}
+
+DECLARE_TYPES(rms_prop_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/updaters/sgdUpdater.cpp

@@ -0,0 +1,62 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <array/NDArray.h>
+#include <execution/Threads.h>
+#include <helpers/ConstantTadHelper.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/headers/updaters.h>
+
+#if NOT_EXCLUDED(OP_sgd_updater)
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(sgd_updater, 1, 1, true, 0, 0) {
+  const auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  if (input->isEmpty()) return Status::OK;
+
+  bool bLearningRate = 2 == block.width() || 1 == block.getTArguments()->size();
+
+  REQUIRE_TRUE(bLearningRate, 0, "SGD UPDATER OP: Learning rate was not provided!");
+
+  if (block.width() > 1) {
+    const auto lr = INPUT_VARIABLE(1);
+    REQUIRE_TRUE(lr->isScalar(), 0, "SGD UPDATER OP: Learning rate has to be a scalar, but instead got rank %i!",
+                 lr->rankOf());
+
+    input->applyScalarArr(scalar::Multiply, lr, output);
+  } else {
+    input->applyScalar(scalar::Multiply, T_ARG(0), output);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(sgd_updater) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+#endif