chore: import upstream snapshot with attribution

libnd4j/include/ops/declarable/generic/linalg/betaInc.cpp

@@ -0,0 +1,74 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 12.12.2017
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_betainc)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/betaInc.h>
+
+namespace sd {
+namespace ops {
+
+DECLARE_TYPES(betainc) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+CONFIGURABLE_OP_IMPL(betainc, 3, 1, false, 0, 0) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+  auto x = INPUT_VARIABLE(2);
+
+  // just skip op if input is empty
+  if (x->isEmpty()) {
+    *x = DataTypeUtils::nanOrZero<float>();
+    return Status::OK;
+  }
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(a->isSameShape(b) && a->isSameShape(x), 0,
+               "CONFIGURABLE_OP betainc: all three input arrays must have the same shapes, bit got a=%s, b=%s and x=%s "
+               "instead !",
+               ShapeUtils::shapeAsString(a).c_str(), ShapeUtils::shapeAsString(b).c_str(),
+               ShapeUtils::shapeAsString(x).c_str());
+
+  LongType arrLen = a->lengthOf();
+
+  // FIXME: this stuff should be single op call. No sense rolling over couple of arrays twice
+  for (LongType i = 0; i < arrLen; ++i) {
+    REQUIRE_TRUE(a->e<float>(i) > 0.f, 0, "BETAINC op: arrays a array must contain only elements > 0 !");
+    REQUIRE_TRUE(b->e<float>(i) > 0.f, 0, "BETAINC op: arrays b array must contain only elements > 0 !");
+    REQUIRE_TRUE(0.f <= x->e<float>(i) && x->e<float>(i) <= 1.f, 0,
+                 "BETAINC op: all elements of x array must be within [0, 1] range!");
+  }
+
+  helpers::betaInc(block.launchContext(), *a, *b, *x, *output);
+
+  return Status::OK;
+}
+
+DECLARE_SYN(BetaInc, betainc);
+DECLARE_SYN(betaInc, betainc);
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/cholesky.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> at 11/12/2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/lup.h>
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_cholesky)
+
+namespace sd {
+namespace ops {
+OP_IMPL(cholesky, 1, 1, true) {
+  NDArray* input = INPUT_VARIABLE(0);
+  NDArray* output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->rankOf() >= 2, 0, "cholesky: The rank of input array should not less than 2, but %i is given",
+               input->rankOf());
+  REQUIRE_TRUE(input->sizeAt(-1) == input->sizeAt(-2), 0,
+               "cholesky: The last two dimensions should be equal, but %i and %i are given", input->sizeAt(-1),
+               input->sizeAt(-2));
+/*  REQUIRE_TRUE(helpers::checkCholeskyInput(block.launchContext(), input), 0,
+               "cholesky: The input tensor should be positive-defined and symmetric.");*/
+  return helpers::cholesky(block.launchContext(), input, output, block.isInplace());
+}
+DECLARE_TYPES(cholesky) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/cross.cpp

@@ -0,0 +1,62 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_cross)
+
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/helpers/cross.h>
+
+namespace sd {
+namespace ops {
+DECLARE_TYPES(cross) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_INTS, ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_INTS, ALL_FLOATS})
+      ->setSameMode(true);
+}
+
+OP_IMPL(cross, 2, 1, false) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+
+  REQUIRE_TRUE(a->lengthOf() == b->lengthOf(), 0, "Cross: A and B lengths should match");
+  REQUIRE_TRUE(a->rankOf() >= 1 && b->rankOf() >= 1, 0, "Cross: A and B should have rank >= 1");
+
+  // TODO: we might want to lift this restriction
+  REQUIRE_TRUE(a->isSameShape(b), 0, "Cross: A and B should have equal shape");
+  REQUIRE_TRUE(a->sizeAt(-1) == 3, 0, "Cross: outer dimension of A and B should be equal to 3");
+
+  auto o = OUTPUT_VARIABLE(0);
+
+  if (a->lengthOf() == 3) {
+    helpers::cross(block.launchContext(), a, b, o);
+  } else {
+    helpers::crossBatched(block.launchContext(), a, b, o);
+  }
+
+  return Status::OK;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/diag.cpp

@@ -0,0 +1,62 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 06.12.2017
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_diag)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/diag.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(diag, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  // input validation
+  REQUIRE_TRUE(input->rankOf() <= 3, 0, "CUSTOM_OP diag: rank of input array must be <= 3 !, but got %i instead",
+               input->rankOf());
+  int zero = 0;
+  output->assign(zero);
+
+  helpers::diagFunctor(block.launchContext(), input, output);
+
+  return Status::OK;
+}
+
+DECLARE_SYN(MatrixDiag, diag);
+
+DECLARE_TYPES(diag) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(diag) {
+  LongType* inputShapeInfo = inputShape->at(0);
+
+  return SHAPELIST(ShapeUtils::evalDiagShapeInfo(inputShapeInfo, block.workspace()));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/diagPart.cpp

@@ -0,0 +1,88 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 06.12.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_diag_part)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/diag.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(diag_part, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  const int inRank = input->rankOf();
+
+  // input validation
+  REQUIRE_TRUE(
+      inRank == 2 || inRank == 4 || inRank == 6, 0,
+      "DIAG_PART op: input array must have rank among following three possible values: 2, 4, 6, but got %i instead !",
+      inRank);
+  for (int i = 0; i < inRank - 1; ++i)
+    REQUIRE_TRUE(input->sizeAt(i) == input->sizeAt(i + 1), 0,
+                 "DIAG_PART op: wrong shape of input array %s ! All dimensions must be equal !",
+                 ShapeUtils::shapeAsString(input).c_str());
+
+  helpers::diagPartFunctor(block.launchContext(), input, output);
+
+  return Status::OK;
+}
+DECLARE_SYN(DiagPart, diag_part);
+
+DECLARE_TYPES(diag_part) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+DECLARE_SHAPE_FN(diag_part) {
+  auto inputShapeInfo = inputShape->at(0);
+
+  const int inRank = inputShapeInfo[0];
+
+  // input validation
+  REQUIRE_TRUE(
+      inRank == 2 || inRank == 4 || inRank == 6, 0,
+      "DIAG_PART op: input array must have rank among following three possible values: 2, 4, 6, but got %i instead !",
+      inRank);
+  for (int i = 1; i < inRank; ++i)
+    REQUIRE_TRUE(inputShapeInfo[i] == inputShapeInfo[i + 1], 0,
+                 "DIAG_PART op: wrong shape of input array %s ! All dimensions must be equal !",
+                 ShapeUtils::shapeAsString(inputShapeInfo).c_str());
+
+  LongType* outShapeInfo = nullptr;
+
+  int outRank = inRank / 2;
+
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+
+  outShapeInfo[0] = outRank;
+  for (int i = 1; i <= outRank; ++i) outShapeInfo[i] = inputShapeInfo[i];
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, inputShapeInfo, shape::order(inputShapeInfo));
+
+  return SHAPELIST(ConstantShapeHelper::getInstance().createFromExisting(outShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/digamma.cpp

@@ -0,0 +1,50 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_digamma)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/gammaMathFunc.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(digamma, 1, 1, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+  if (x->isEmpty()) {
+  return Status::OK;
+  }
+  helpers::diGamma(block.launchContext(), *x, *z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(digamma) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/eig.cpp

@@ -0,0 +1,79 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_eig)
+
+#include <helpers/EigenValsAndVecs.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(eig, 1, 2, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto eig_vals = OUTPUT_VARIABLE(0);
+  auto eig_vectors = OUTPUT_VARIABLE(1);
+
+  // input validation
+  REQUIRE_TRUE(input->rankOf() == 2, 0, "Eig: input is not a matrix. rank: %i == 2", input->rankOf());
+
+  auto n1 = input->sizeAt(0);
+  auto n2 = input->sizeAt(1);
+
+  REQUIRE_TRUE(n1 == n2, 0, "Eig: input is not a square matrix. rank: {%i, %i}", n1, n2);
+
+  REQUIRE_TRUE(eig_vals->rankOf() == 2 && eig_vals->sizeAt(0) == n1 && eig_vals->sizeAt(1) == 2, 0,
+               "Eig: the shape of the eigenvalue results should be {%i, 2}", n1);
+  REQUIRE_TRUE(eig_vectors->rankOf() == 3 && eig_vectors->sizeAt(0) == n1 && eig_vectors->sizeAt(1) == n1 &&
+               eig_vectors->sizeAt(2) == 2,
+               0, "Eig: the shape of the eigenvector results should be {%i, %i, 2}", n1);
+
+  helpers::eig(*input, *eig_vals, *eig_vectors);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(eig) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(eig) {
+  auto inputShapeInfo = inputShape->at(0);
+  REQUIRE_TRUE(inputShapeInfo[0] == 2, 0, "Eig: input is not a matrix. rank: %i == 2", inputShapeInfo[0]);
+
+  auto n1 = shape::shapeOf(inputShapeInfo)[0];
+  auto n2 = shape::shapeOf(inputShapeInfo)[1];
+
+  REQUIRE_TRUE(n1 == n2, 0, "Eig: input is not a square matrix. rank: {%i, %i}", n1, n2);
+
+  auto dtype_float = ArrayOptions::dataType(inputShapeInfo);
+  auto ordering = shape::order(inputShapeInfo);
+
+  auto desc = ShapeBuilders::createShapeInfo(dtype_float, ordering, {n1, 2});
+  auto output0 = ConstantShapeHelper::getInstance().bufferForShapeInfo(desc);
+  auto desc2 = ShapeBuilders::createShapeInfo(dtype_float, ordering, {n1, n1, 2});
+  auto output1 =ConstantShapeHelper::getInstance().bufferForShapeInfo(desc2);
+  return SHAPELIST(output0->primary(), output1->primary());
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/eye.cpp

@@ -0,0 +1,105 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 22.01.2018
+//
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_eye)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(eye, -2, 1, false, -2, -2) {
+  helpers::eye(block.launchContext(), *OUTPUT_VARIABLE(0));
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(eye) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_INTS});
+  getOpDescriptor()->setAllowedInputTypes(1, {INT32, INT64});
+  getOpDescriptor()->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(eye) {
+  std::vector<LongType> params;
+
+  DataType dtype = block.getTArguments()->empty() ? FLOAT32 : DataTypeUtils::fromInt(T_ARG(0));
+
+  if (block.width() == 0) {
+    params = *block.getIArguments();
+  } else {
+    for (size_t i = 0; i < block.width(); i++) {
+      auto input = INPUT_VARIABLE(i);
+      REQUIRE_TRUE(input->rankOf() == 1, 0, "Inputs to eye should be 1D");
+
+      for (int e = 0; e < input->lengthOf(); e++) params.emplace_back(input->e<LongType>(e));
+    }
+  }
+
+  REQUIRE_TRUE(params.size() > 0, 0, "Size is not provided for eye op.");
+
+  const bool ordered = (params[0] == -99 || params[0] == -102);  // -99 :'c', -102 : 'f'
+  if (!ordered) params.insert(params.begin(), -99);
+
+  REQUIRE_TRUE(params.size() > 1, 0, "Size is not provided for eye op.");
+
+  LongType* outShapeInfo(nullptr);
+
+  const int size = params.size();
+
+  switch (size) {
+    case 2:
+      ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+      outShapeInfo[0] = 2;
+      outShapeInfo[1] = params[1];
+      outShapeInfo[2] = params[1];
+      break;
+
+    case 3:
+      ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+      outShapeInfo[0] = 2;
+      outShapeInfo[1] = params[1];
+      outShapeInfo[2] = params[2];
+      break;
+
+    default:
+      int rank = size - 1;
+      ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+      outShapeInfo[0] = rank;
+      outShapeInfo[rank - 1] = params[1];
+      outShapeInfo[rank] = params[2];
+      for (int i = 1; i < rank - 1; ++i) outShapeInfo[i] = params[i + 2];
+      break;
+  }
+
+  shape::updateStrides(outShapeInfo, static_cast<char>(-params[0]), false);
+  auto desc = new ShapeDescriptor(outShapeInfo, dtype, false);
+  auto result = ConstantShapeHelper::getInstance().bufferForShapeInfo(outShapeInfo);
+  auto ret =  SHAPELIST(result->primary());
+  return ret;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/lgamma.cpp

@@ -0,0 +1,51 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author George A. Shulinok <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_lgamma)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/lgamma.h>
+
+namespace sd {
+namespace ops {
+
+OP_IMPL(lgamma, 1, 1, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+  helpers::lgamma(block.launchContext(), x, z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(lgamma) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_FLOATS})  // as TF says
+      ->setSameMode(true);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/log1p.cpp

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_Log1p)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(Log1p, 1, 1, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  x->applyTransform(transform::Log1p, z);
+
+  STORE_RESULT(z);
+
+  return Status::OK;
+}
+DECLARE_SYN(log1p, Log1p);
+}  // namespace ops
+
+DECLARE_TYPES(Log1p) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/lstsq.cpp

@@ -0,0 +1,144 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// Created by GS <sgazeos@gmail.com> at 01/28/2020
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_lstsq)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/lstsq.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(lstsq, 2, 1, false, 0, 0) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+  auto z = OUTPUT_NULLIFIED(0);
+  bool fastFlag = true;
+  double l2_factor = 0.;
+  if (block.numB() > 0) {
+    fastFlag = B_ARG(0);
+  }
+  if (block.numT() > 0) {
+    l2_factor = T_ARG(0);
+  }
+  REQUIRE_TRUE(a->rankOf() >= 2, 0, "lstsq: The rank of input left tensor should not be less than 2, but %i is given",
+               a->rankOf());
+  REQUIRE_TRUE(b->rankOf() >= 2, 0, "lstsq: The rank of input right tensor should not be less than 2, but %i is given",
+               b->rankOf());
+
+
+  REQUIRE_TRUE(
+      a->sizeAt(-2) == b->sizeAt(-2), 0,
+      "lstsq: The last dimmension of left part should be equal to prelast of right part, but %i and %i are given",
+      a->sizeAt(-1), b->sizeAt(-2));
+
+  if (a->isEmpty() || b->isEmpty() || z->isEmpty()) return Status::OK;
+
+  auto res = helpers::leastSquaresSolveFunctor(block.launchContext(), a, b, l2_factor, fastFlag, z);
+
+  return res;
+}
+
+CUSTOM_OP_IMPL(solve_ls, 2, 1, false, 0, 0) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+  auto z = OUTPUT_NULLIFIED(0);
+  bool fastFlag = true;
+  double l2_factor = 0.;
+  if (block.numB() > 0) {
+    fastFlag = B_ARG(0);
+  }
+  if (block.numT() > 0) {
+    l2_factor = T_ARG(0);
+  }
+  REQUIRE_TRUE(a->rankOf() >= 2, 0, "lstsq: The rank of input left tensor should not be less than 2, but %i is given",
+               a->rankOf());
+  REQUIRE_TRUE(b->rankOf() >= 2, 0, "lstsq: The rank of input right tensor should not be less than 2, but %i is given",
+               b->rankOf());
+
+  //            REQUIRE_TRUE(a->sizeAt(-1) == a->sizeAt(-2), 0, "lstsq: The last two dimensions should be equal, but %i
+  //            and %i are given", a->sizeAt(-1), a->sizeAt(-2));
+  REQUIRE_TRUE(
+      a->sizeAt(-2) == b->sizeAt(-2), 0,
+      "lstsq: The last dimmension of left part should be equal to prelast of right part, but %i and %i are given",
+      a->sizeAt(-1), b->sizeAt(-2));
+  // REQUIRE_TRUE(l2_factor == 0., 0, "lstsq: Implementation of operation is not finished for factor difference from
+  // 0.");
+  auto res = Status::OK;
+  if (a->isEmpty() || b->isEmpty() || z->isEmpty()) return res;
+
+  res = helpers::leastSquaresSolveFunctor(block.launchContext(), a, b, l2_factor, fastFlag, z);
+
+  return res;
+}
+
+DECLARE_SYN(MatrixSolveLs, lstsq);
+
+DECLARE_SHAPE_FN(lstsq) {
+  auto in0 = inputShape->at(0);
+  auto in1 = inputShape->at(1);
+  auto shapeOf = ShapeUtils::shapeAsVector(in1);
+  auto rank = shapeOf.size();
+  shapeOf[rank - 2] = shape::sizeAt(in0, static_cast<LongType>(-1));
+
+  if (shape::isEmptyConst(in0) || shape::isEmptyConst(in1)) {
+    shapeOf[rank - 1] = 0;  // set output shape to empty
+  }
+  auto resShape = ConstantShapeHelper::getInstance().createShapeInfo(
+      ArrayOptions::dataType(in0), shape::order(in1),
+      shapeOf);
+  if (shapeOf[rank - 1] == 0) {
+    resShape = ConstantShapeHelper::getInstance().emptyShapeInfo(ArrayOptions::dataType(in0));
+  }
+  return SHAPELIST(resShape);
+}
+
+DECLARE_TYPES(lstsq) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS})->setSameMode(false);
+}
+DECLARE_SHAPE_FN(solve_ls) {
+  auto in0 = inputShape->at(0);
+  auto in1 = inputShape->at(1);
+  auto shapeOf = ShapeUtils::shapeAsVector(in1);
+  auto rank = shapeOf.size();
+  shapeOf[rank - 2] = shape::sizeAt(in0, static_cast<LongType>(-1));
+
+  if (shape::isEmptyConst(in0) || shape::isEmptyConst(in1)) {
+    shapeOf[rank - 1] = 0;  // set output shape to empty
+  }
+  auto resShape = ConstantShapeHelper::getInstance().createShapeInfo(
+      ArrayOptions::dataType(in0), shape::order(in1),
+      shapeOf);
+
+  return SHAPELIST(resShape);
+}
+
+DECLARE_TYPES(solve_ls) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS})->setSameMode(false);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/lup.cpp

@@ -0,0 +1,90 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> at 12/10/2019
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_matrix_inverse)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/lup.h>
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(lu, 1, 2, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  if(input->isEmpty()) {
+    return Status::OK;
+  }
+  auto z = OUTPUT_VARIABLE(0);
+
+  auto p = OUTPUT_VARIABLE(1);
+  if (block.getIArguments()->size()) {
+    DataType dtype = (DataType)INT_ARG(0);
+    REQUIRE_TRUE(dtype == sd::DataType::INT32 || dtype == sd::DataType::INT64, 0,
+                 "lu: Permutation data type should be 32bit or 64bit int only, but '%s' given.",
+                 DataTypeUtils::asString(dtype).c_str());
+  }
+
+  REQUIRE_TRUE(input->rankOf() >= 2, 0, "lu: The rank of input array should not less than 2, but %i is given",
+               input->rankOf());
+  REQUIRE_TRUE(input->sizeAt(-1) == input->sizeAt(-2), 0,
+               "lu: The last two dimensions should be equal, but %i and %i are given", input->sizeAt(-1),
+               input->sizeAt(-2));
+
+  helpers::lu(block.launchContext(), input, z, p);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(lu) {
+  auto in = inputShape->at(0);
+  auto dtype = INT32;
+  if (block.getIArguments()->size()) {
+    dtype = (DataType)INT_ARG(0);
+    REQUIRE_TRUE(dtype == sd::DataType::INT32 || dtype == sd::DataType::INT64, 0,
+                 "lu: Permutation data type should be 32 bit or 64bit int only, but '%s' given.",
+                 DataTypeUtils::asString(dtype).c_str());
+  }
+
+  auto shapeVector = ShapeUtils::shapeAsVector(in);
+
+  if(shape::isEmptyConst(in)) {
+
+    auto luP = ShapeBuilders::createShapeInfo(dtype, shape::order(in), shapeVector.size() - 1, shapeVector.data(),
+                                              block.workspace(), true);
+    return SHAPELIST(in,luP);
+  }
+  auto luShape = ShapeBuilders::copyShapeInfoAndType(in, in, true, block.workspace());
+  auto luP = ShapeBuilders::createShapeInfo(dtype, shape::order(in), shapeVector.size() - 1, shapeVector.data(),
+                                            block.workspace(), false);
+
+  return SHAPELIST(CONSTANT(luShape), CONSTANT(luP));
+}
+
+DECLARE_TYPES(lu) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {INT32, INT64})
+      ->setSameMode(false);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/matrixDiagPart.cpp

@@ -0,0 +1,69 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created to use with batched tensor by GS <sgazeos@gmail.com> 3/21/2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/matrix_diag_part.h>
+
+#if NOT_EXCLUDED(OP_matrix_diag_part)
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(matrix_diag_part, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+  const int inRank = input->rankOf();
+
+  REQUIRE_TRUE(inRank >= 2, 0, "CUSTOM_OP matrix_diag_part: input array must have rank >= 2, but %i given!", inRank);
+
+  output->nullify();
+  return helpers::matrixDiagPart(block.launchContext(), input, output);
+}
+
+DECLARE_SHAPE_FN(matrix_diag_part) {
+  LongType * outShapeInfo = nullptr;
+  auto in = inputShape->at(0);
+  LongType inRank = shape::rank(in);
+
+  REQUIRE_TRUE(inRank >= 2, 0, "CUSTOM_OP matrix_diag_part: input array must have rank >= 2, but %i given!", inRank);
+
+  LongType outRank = inRank - 1;
+  LongType lastDimension = sd::math::sd_min<LongType>(shape::sizeAt(in, static_cast<LongType>(-1)), shape::sizeAt(in, static_cast<LongType>(-2)));
+  if (outRank == 1) {
+    // output shape is a vector with size min(sizeAt(0), sizeAt(1))
+    outShapeInfo = ConstantShapeHelper::getInstance().vectorShapeInfo(lastDimension, ArrayOptions::dataType(in));
+  } else {
+    LongType* anShapeInfo;
+    ALLOCATE(anShapeInfo, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+    anShapeInfo[0] = outRank;
+    for (LongType i = 0; i < outRank - 1; ++i) anShapeInfo[i + 1] = shape::sizeAt(in, i);
+    anShapeInfo[outRank] = lastDimension;
+
+    ShapeUtils::updateStridesAndType(anShapeInfo, in, shape::order(in));
+    outShapeInfo = CONSTANT(anShapeInfo);
+  }
+  return SHAPELIST(outShapeInfo);
+}
+
+DECLARE_TYPES(matrix_diag_part) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/linalg/matrixSetDiag.cpp

@@ -0,0 +1,66 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_matrix_set_diag)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/matrixSetDiag.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(matrix_set_diag, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto diagonal = INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(diagonal->rankOf() == input->rankOf() - 1, 0,
+               "MATRIX_SET_DIAG op: rank of diagonal array must be smaller by one compared to rank of input array, but "
+               "got %i and %i correspondingly !",
+               diagonal->rankOf(), input->rankOf());
+
+  for (int i = 0; i < diagonal->rankOf() - 1; ++i)
+    REQUIRE_TRUE(diagonal->sizeAt(i) == input->sizeAt(i), 0,
+                 "MATRIX_SET_DIAG op: the shapes of diagonal and input arrays must be equal till last diagonal "
+                 "dimension but one, however got diagonal=%s and input=%s instead !",
+                 ShapeUtils::shapeAsString(diagonal).c_str(), ShapeUtils::shapeAsString(input).c_str());
+
+  REQUIRE_TRUE(diagonal->sizeAt(-1) == (int)sd::math::sd_min<sd::LongType>(input->sizeAt(-1), input->sizeAt(-2)), 0,
+               "MATRIX_SET_DIAG op: the value of last dimension of diagonal array must be equal to "
+               "min(input_last_shape=%i, input_last_but_one_shape=%i), but got %i instead !",
+               input->sizeAt(-1), input->sizeAt(-2), diagonal->sizeAt(-1));
+
+  helpers::matrixSetDiag(block.launchContext(), *input, *diagonal, *output, false);
+
+  return Status::OK;
+}
+
+DECLARE_SYN(MatrixSetDiag, matrix_set_diag);
+
+DECLARE_TYPES(matrix_set_diag) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/matrix_band_part.cpp

@@ -0,0 +1,75 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author GS <sgazeos@gmail.com>, created on 8/22/2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_matrix_band_part)
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/matrix_band.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(matrix_band_part, 1, 1, true, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  LongType minLower(0LL);
+  LongType maxUpper(0LL);
+  if (block.width() == 1) {
+    REQUIRE_TRUE(block.numI() == 2, 0, "matrix_band_part: min and max band numbers should be given before.");
+    minLower = INT_ARG(0);
+    maxUpper = INT_ARG(1);
+  } else {
+    REQUIRE_TRUE(block.width() == 3, 0,
+                 "matrix_band_part: min and max band numbers should be given as scalars before.");
+    auto minLowerT = INPUT_VARIABLE(1);
+    auto maxUpperT = INPUT_VARIABLE(2);
+    REQUIRE_TRUE(minLowerT->isScalar() && maxUpperT->isScalar(), 0,
+                 "matrix_band_part: min and max should be scalars, but %i and %i ranks given", minLowerT->rankOf(),
+                 maxUpperT->rankOf());
+    minLower = minLowerT->e<LongType>(0);
+    maxUpper = maxUpperT->e<LongType>(0);
+  }
+  REQUIRE_TRUE(input->rankOf() >= 2, 0, "matrix_band_part: Input rank should be 2 or greater.");
+  LongType N = input->sizeAt(-2);
+  LongType M = input->sizeAt(-1);
+  REQUIRE_TRUE(minLower > -N && minLower < N, 0, "matrix_band_part: lower diagonal count %i should be less than %i.",
+               minLower, N);
+  REQUIRE_TRUE(maxUpper > -M && maxUpper < M, 0, "matrix_band_part: upper diagonal count %i should be less than %i.",
+               maxUpper, M);
+
+  helpers::matrixBandPart(block.launchContext(), input, output, minLower, maxUpper);
+  return Status::OK;
+}
+DECLARE_SYN(band_part, matrix_band_part);
+}  // namespace ops
+
+DECLARE_TYPES(matrix_band_part) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedInputTypes(2, {ALL_INTS})
+      ->setAllowedInputTypes({ALL_INTS, ALL_FLOATS});
+}
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/matrix_determinant.cpp

@@ -0,0 +1,152 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> at 2/26/2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/lup.h>
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_matrix_determinant)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(matrix_determinant, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->rankOf() >= 2, 0,
+               "matrix_determinant: The rank of input array should not less than 2, but %i is given", input->rankOf());
+  REQUIRE_TRUE(input->sizeAt(-1) == input->sizeAt(-2), 0,
+               "matrix_determinant: The last two dimensions should be equal, but %i and %i are given",
+               input->sizeAt(-1), input->sizeAt(-2));
+
+  return helpers::determinant(block.launchContext(), input, output);
+}
+
+DECLARE_SHAPE_FN(matrix_determinant) {
+  auto inShape = inputShape->at(0);
+
+  LongType * determinantShape;
+  int targetRank = shape::rank(inShape) - 2;  // last two dimensions will be reduced to scalar
+
+  if (targetRank == 0) {  // scalar only
+    determinantShape = ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inShape));
+  } else if (targetRank == 1) {  // vector
+    determinantShape =
+        ConstantShapeHelper::getInstance().vectorShapeInfo(shape::sizeAt(inShape, static_cast<LongType>(0)), ArrayOptions::dataType(inShape));
+  } else {  // only two last dimensions are excluded
+    determinantShape = ConstantShapeHelper::getInstance().createShapeInfo(
+        ArrayOptions::dataType(inShape), shape::order(inShape), targetRank, shape::shapeOf(inShape), -1);
+  }
+  return SHAPELIST(determinantShape);
+}
+
+DECLARE_TYPES(matrix_determinant) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif
+
+#if NOT_EXCLUDED(OP_log_matrix_determinant)
+namespace sd {
+namespace ops {
+DECLARE_TYPES(log_matrix_determinant) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(log_matrix_determinant, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->rankOf() >= 2, 0,
+               "log_matrix_determinant: The rank of input array should not less than 2, but %i is given",
+               input->rankOf());
+  REQUIRE_TRUE(input->sizeAt(-1) == input->sizeAt(-2), 0,
+               "log_matrix_determinant: The last two dimensions should be equal, but %i and %i are given",
+               input->sizeAt(-1), input->sizeAt(-2));
+
+  return helpers::logAbsDeterminant(block.launchContext(), input, output);
+}
+
+DECLARE_SHAPE_FN(log_matrix_determinant) {
+  auto inShape = inputShape->at(0);
+
+  LongType * determinantShape;
+  int targetRank = shape::rank(inShape) - 2;  // last two dimensions will be reduced to scalar
+
+  if (targetRank == 0) {  // scalar only
+    determinantShape = ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inShape));
+  } else if (targetRank == 1) {  // vector
+    determinantShape =
+        ConstantShapeHelper::getInstance().vectorShapeInfo(shape::sizeAt(inShape, static_cast<LongType>(0)), ArrayOptions::dataType(inShape));
+  } else {  // only two last dimensions are excluded
+    determinantShape = ConstantShapeHelper::getInstance().createShapeInfo(
+        ArrayOptions::dataType(inShape), shape::order(inShape), targetRank, shape::shapeOf(inShape), -1);
+  }
+  return SHAPELIST(determinantShape);
+}
+}  // namespace ops
+}  // namespace sd
+#endif
+
+#if NOT_EXCLUDED(OP_logdet)
+namespace sd {
+namespace ops {
+DECLARE_TYPES(logdet) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(logdet, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_NULLIFIED(0);
+
+  REQUIRE_TRUE(input->rankOf() >= 2, 0, "logdet: The rank of input array should not less than 2, but %i is given",
+               input->rankOf());
+  REQUIRE_TRUE(input->sizeAt(-1) == input->sizeAt(-2), 0,
+               "logdet: The last two dimensions should be equal, but %i and %i are given", input->sizeAt(-1),
+               input->sizeAt(-2));
+  REQUIRE_TRUE(helpers::checkCholeskyInput(block.launchContext(), input), 0,
+               "logdet: The input tensor should be positive-defined hermitian.");
+
+  return helpers::logdetFunctor(block.launchContext(), input, output);
+}
+
+DECLARE_SHAPE_FN(logdet) {
+  auto inShape = inputShape->at(0);
+
+  LongType* determinantShape;
+  int targetRank = shape::rank(inShape) - 2;  // last two dimensions will be reduced to scalar
+
+  if (targetRank == 0) {  // scalar only
+    determinantShape = ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inShape));
+  } else if (targetRank == 1) {  // vector
+    determinantShape =
+        ConstantShapeHelper::getInstance().vectorShapeInfo(shape::sizeAt(inShape, static_cast<LongType>(0)), ArrayOptions::dataType(inShape));
+  } else {  // only two last dimensions are excluded
+    determinantShape = ConstantShapeHelper::getInstance().createShapeInfo(
+        ArrayOptions::dataType(inShape), shape::order(inShape), targetRank, shape::shapeOf(inShape), -1);
+  }
+  return SHAPELIST(determinantShape);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/linalg/matrix_diag.cpp

@@ -0,0 +1,66 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author GS <sgazeos@gmail.com> 3/21/2018
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/matrixSetDiag.h>
+
+#if NOT_EXCLUDED(OP_matrix_diag)
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(matrix_diag, 1, 1, false, 0, 0) {
+  auto diagonal = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(!diagonal->isScalar(), 0,
+               "CUSTOM_OP matrix_diag: input diagonal array must be at list a vector, but scalar was given!");
+
+  helpers::matrixSetDiag(block.launchContext(), *output, *diagonal, *output, true);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(matrix_diag) {
+  LongType* outShapeInfo = nullptr;
+  auto in = inputShape->at(0);
+  int inRank = shape::rank(in);
+
+  //  if for example diagonal array has shape [A,B,C] then output array has shape [A,B,C,C]
+
+  int outRank = inRank + 1;
+
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(outRank), sd::LongType);
+  outShapeInfo[0] = outRank;
+  for (LongType i = 0; i < inRank; ++i) outShapeInfo[i + 1] = shape::sizeAt(in, i);
+  outShapeInfo[outRank] = shape::sizeAt(in, static_cast<LongType>(-1));
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, in, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outShapeInfo));
+}
+
+DECLARE_TYPES(matrix_diag) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/matrix_inverse.cpp

@@ -0,0 +1,47 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by GS <sgazeos@gmail.com> at 2/27/2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_matrix_inverse)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/lup.h>
+namespace sd {
+namespace ops {
+OP_IMPL(matrix_inverse, 1, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->rankOf() >= 2, 0,
+               "matrix_inverse: The rank of input array should not less than 2, but %i is given", input->rankOf());
+  REQUIRE_TRUE(input->sizeAt(-1) == input->sizeAt(-2), 0,
+               "matrix_inverse: The last two dimensions should be equal, but %i and %i are given", input->sizeAt(-1),
+               input->sizeAt(-2));
+
+  return helpers::inverse(block.launchContext(), input, output);
+}
+
+DECLARE_TYPES(matrix_inverse) { getOpDescriptor()->setAllowedInputTypes(ANY)->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/moments.cpp

@@ -0,0 +1,89 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by sgazeos@gmail.com on 26.01.2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_moments)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(moments, 1, 2, false, 0, -2) {
+  auto input = INPUT_VARIABLE(0);
+  auto means = OUTPUT_VARIABLE(0);
+  auto variances = OUTPUT_VARIABLE(1);
+
+  std::vector<LongType> axis = *block.getIArguments();
+  const bool keepDims = block.getBArguments()->size() > 0 ? (bool)B_ARG(0) : false;
+  reduce_variance varianceOp;
+
+  // axis might be dynamic (i.e. tf mode)
+  if (block.width() > 1) {
+    auto axisVector = INPUT_VARIABLE(1);
+    helpers::adjustAxis(input->rankOf(), axisVector, axis);
+    varianceOp.execute({input, axisVector}, {variances}, {}, {}, {keepDims}, {}, false);
+  } else {
+    std::vector<LongType>& dims = axis;
+    std::vector<LongType> axes;
+    for (size_t i = 0; i < dims.size(); i++) {
+      axes.push_back(dims[i]);
+    }
+
+    varianceOp.execute({input}, {variances}, {}, axes, {keepDims}, {}, false);
+  }
+
+  input->reduceAlongDimension(reduce::Mean, means, &axis, keepDims);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(moments) {
+  auto axis = *block.getIArguments();
+  auto input = INPUT_VARIABLE(0);
+
+  // axis might be dynamic (i.e. tf mode)
+  if (block.width() > 1 && axis.size() == 0) {
+    auto axisVector = INPUT_VARIABLE(1);
+
+    for (int e = 0; e < axisVector->lengthOf(); e++) {
+      int ca = axisVector->e<int>(e);
+      if (ca < 0) ca += input->rankOf();
+
+      axis.emplace_back(ca);
+    }
+  }
+  const bool keepDims = block.getBArguments()->size() > 0 ? (bool)B_ARG(0) : false;
+
+  auto meanShape = ShapeUtils::evalReduceShapeInfo('c', &axis, *input, keepDims, false, block.workspace());
+  auto varianceShape = ShapeUtils::evalReduceShapeInfo('c', &axis, *input, keepDims, false, block.workspace());
+  return SHAPELIST(meanShape, varianceShape);
+}
+
+DECLARE_TYPES(moments) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/polygamma.cpp

@@ -0,0 +1,63 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_polygamma)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/gammaMathFunc.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(polygamma, 2, 1, false, 0, 0) {
+  auto n = INPUT_VARIABLE(0);
+  auto x = INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(n->isSameShape(x), 0,
+               "POLYGAMMA op: two input arrays n and x must have the same shapes, but got n=%s and x=%s instead !",
+               ShapeUtils::shapeAsString(n).c_str(), ShapeUtils::shapeAsString(x).c_str());
+
+  void* extraArgs = nullptr;
+  auto* nNegative = n->reduceNumber(reduce::IsNegative, extraArgs);
+  auto* xPositive = x->reduceNumber(reduce::IsPositive, extraArgs);
+  bool nPositiveFlag = !nNegative->e<bool>(0);  // require all n >= 0
+  bool xPositiveFlag = xPositive->e<bool>(0);   // require all x > 0
+  delete nNegative;
+  delete xPositive;
+  REQUIRE_TRUE(nPositiveFlag, 0, "POLYGAMMA op: all elements of n array must be >= 0 !");
+  REQUIRE_TRUE(xPositiveFlag, 0, "POLYGAMMA op: all elements of x array must be > 0 !");
+
+  helpers::polyGamma(block.launchContext(), *n, *x, *output);
+  return Status::OK;
+}
+
+DECLARE_SYN(polyGamma, polygamma);
+DECLARE_SYN(PolyGamma, polygamma);
+
+DECLARE_TYPES(polygamma) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/qr.cpp

@@ -0,0 +1,90 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// Created by GS <sgazeos@gmail.com> at 12/20/2019
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/qr.h>
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_qr)
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(qr, 1, 2, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto outputQ = OUTPUT_VARIABLE(0);
+  auto outputR = OUTPUT_VARIABLE(1);
+  auto fullMatricies = false;
+  if (block.getBArguments()->size()) fullMatricies = B_ARG(0);
+  REQUIRE_TRUE(input->rankOf() >= 2, 0, "qr: The rank of input array should not be less than 2, but %i is given",
+               input->rankOf());
+  REQUIRE_TRUE(
+      (fullMatricies && outputQ->sizeAt(-1) == input->sizeAt(-2)) || (!fullMatricies && outputQ->isSameShape(input)), 0,
+      "qr: The last dimensions should be equal to result Q, but %i and %i are given", outputQ->sizeAt(-1),
+      input->sizeAt(-2));
+  REQUIRE_TRUE((fullMatricies && outputR->sizeAt(-1) == input->sizeAt(-1)) ||
+                   (!fullMatricies && outputR->sizeAt(-1) == outputR->sizeAt(-2)),
+               0, "qr: The last dimensions should be equal to result R, but %i and %i are given", outputR->sizeAt(-1),
+               input->sizeAt(-1));
+  if (!input->isEmpty() && !outputQ->isEmpty() && !outputR->isEmpty())
+    helpers::qr(block.launchContext(), input, outputQ, outputR, fullMatricies);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(qr) {
+  auto inShape = inputShape->at(0);
+
+  LongType * shapeQ;
+  LongType * shapeR;
+  int targetRank = shape::rank(inShape);  // last two dimensions will be reduced to scalar
+
+  auto fullMatricies = false;
+  if (block.getBArguments()->size()) fullMatricies = B_ARG(0);
+
+  auto shape = ShapeUtils::shapeAsVector(inShape);
+
+  if (!fullMatricies) {  // outputs are: Q is MxN and R is NxN
+    shape[targetRank - 1] = shape::sizeAt(inShape, static_cast<LongType>(-1));
+    shape[targetRank - 2] = shape[targetRank - 1];
+    shapeQ = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), shape::order(inShape),
+                                                                targetRank, shape::shapeOf(inShape), -1);
+    shapeR = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), shape::order(inShape),
+                                                                shape);
+
+  } else {  // otherwise outputs are Q is MxM and R is MxN with zero filled rows
+    shape[targetRank - 1] = shape::sizeAt(inShape, static_cast<LongType>(-2));
+    shape[targetRank - 2] = shape[targetRank - 1];
+    shapeR = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), shape::order(inShape),
+                                                                targetRank, shape::shapeOf(inShape), -1);
+    shapeQ = ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inShape), shape::order(inShape),
+                                                                shape);
+  }
+
+  return SHAPELIST(shapeQ, shapeR);
+}
+
+DECLARE_TYPES(qr) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS}); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/solve.cpp

@@ -0,0 +1,89 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// Created by GS <sgazeos@gmail.com> at 01/22/2020
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_solve)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/solve.h>
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(solve, 2, 1, false, 0, 0) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+  if(a->isEmpty() || b->isEmpty())
+    return Status::OK;
+  bool useAdjoint = false;
+
+  if (block.numB() > 0) {
+    useAdjoint = B_ARG(0);
+  }
+  REQUIRE_TRUE(shape::shapeEquals(a->rankOf() - 2, a->shapeInfo(), b->rankOf() - 2, b->shapeInfo()), 0,
+               "solve: Input shapes should be alike.");
+  REQUIRE_TRUE(a->rankOf() >= 2, 0, "solve: The rank of input left tensor should not be less than 2, but %i is given",
+               a->rankOf());
+  REQUIRE_TRUE(b->rankOf() >= 2, 0, "solve: The rank of input right tensor should not be less than 2, but %i is given",
+               b->rankOf());
+
+  REQUIRE_TRUE(a->sizeAt(-1) == a->sizeAt(-2), 0,
+               "solve: The last two dimensions should be equal, but %i and %i are given", a->sizeAt(-1),
+               a->sizeAt(-2));
+  REQUIRE_TRUE(
+      a->sizeAt(-1) == b->sizeAt(-2), 0,
+      "solve: The last dimension of left part should be equal to prelast of right part, but %i and %i are given",
+      a->sizeAt(-1), b->sizeAt(-2));
+  if (a->isEmpty() || b->isEmpty() || z->isEmpty()) return Status::OK;
+
+  auto input = a;
+  if (useAdjoint) {
+    auto adjointA = a->ulike();
+    helpers::adjointMatrix(block.launchContext(), a, adjointA);
+    input = new NDArray(adjointA);
+  }
+
+  auto res = helpers::solveFunctor(block.launchContext(), input, b, useAdjoint, z);
+  if(res != Status::OK)
+    return res;
+
+  if (input != a) delete input;
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(solve) {
+  auto in0 = inputShape->at(1);
+  auto in1 = inputShape->at(1);
+  auto luShape = ShapeBuilders::copyShapeInfoAndType(in1, in0, true, block.workspace());
+
+  return SHAPELIST(CONSTANT(luShape));
+}
+
+DECLARE_TYPES(solve) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS})->setSameMode(false);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/sqrtm.cpp

@@ -0,0 +1,55 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_sqrtm)
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/sqrtm.h>
+
+namespace sd {
+namespace ops {
+
+CONFIGURABLE_OP_IMPL(sqrtm, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->rankOf() > 1, 0,
+               "CONFIGURABLE_OP sqrtm: input array rank is required to be > 1, but got %i instead !", input->rankOf());
+  REQUIRE_TRUE(input->sizeAt(-2) == input->sizeAt(-1), 0,
+               "CONFIGURABLE_OP sqrtm: two last dimensions of input array should be square matrices, but got such "
+               "wrong shape instead: %s!",
+               ShapeUtils::shapeAsString(input).c_str());
+
+  helpers::sqrtm(block.launchContext(), input, output);
+
+  return Status::OK;
+}
+
+//////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(sqrtm) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/sufficient_statistics.cpp

@@ -0,0 +1,94 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by george@skymind.io on 2/21/2018.
+// Modified by sgazeos@gmail.com on 4/4/2018
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_sufficient_statistics)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/axis.h>
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(sufficient_statistics, 2, 3, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto axisVector = INPUT_VARIABLE(1);
+  auto dataCount = OUTPUT_VARIABLE(0);
+
+  auto sum = OUTPUT_VARIABLE(1);
+  auto squares = OUTPUT_VARIABLE(2);
+
+  std::vector<LongType> axis(axisVector->lengthOf());
+
+  // axis might be dynamic (i.e. tf mode)
+  helpers::adjustAxis(input->rankOf(), axisVector, axis);
+
+  input->reduceAlongDimension(reduce::SquaredNorm, squares, &axis);
+  input->reduceAlongDimension(reduce::Sum, sum, &axis);
+  auto count = NDArrayFactory::create(input->dataType(), input->lengthOf() / sum->lengthOf());
+  dataCount->assign(count);
+  if (block.numT() > 0) {
+    auto shift = OUTPUT_VARIABLE(3);
+#ifdef HAS_DOUBLE
+    double shiftValue = static_cast<double>(T_ARG(0));
+    shift->assign(shiftValue);
+#elif defined(HAS_FLOAT32)
+    float shiftValue = static_cast<float>(T_ARG(0));
+    shift->assign(shiftValue);
+#else
+#error "No floating-point type available for sufficient_statistics operation"
+#endif
+  }
+
+  delete count;
+  return Status::OK;
+}
+
+DECLARE_TYPES(sufficient_statistics) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_INTS, ALL_FLOATS});
+  getOpDescriptor()->setAllowedInputTypes(1, {INT32, INT64});
+  getOpDescriptor()->setAllowedOutputTypes(0, INHERIT);
+  getOpDescriptor()->setAllowedOutputTypes(1, INHERIT);
+  getOpDescriptor()->setAllowedOutputTypes(2, INHERIT);
+}
+
+DECLARE_SHAPE_FN(sufficient_statistics) {
+  auto axisVector = INPUT_VARIABLE(1);
+  std::vector<LongType> axis(axisVector->lengthOf());
+
+  auto input = INPUT_VARIABLE(0);
+  helpers::adjustAxis(input->rankOf(), axisVector, axis);
+
+  auto scalarShape = ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inputShape->at(0)));
+  auto sumShape = ShapeUtils::evalReduceShapeInfo('c', &axis, *input, false, false, block.workspace());
+
+  auto squareShape = ShapeUtils::evalReduceShapeInfo('c', &axis, *input, false, false, block.workspace());
+
+  auto shapeList = SHAPELIST(scalarShape, sumShape, squareShape);
+  if (block.numT() > 0)
+    shapeList->push_back(ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inputShape->at(0))));
+
+  return shapeList;
+}
+}  // namespace ops
+
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/svd.cpp

@@ -0,0 +1,109 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author Yurii Shyrma (iuriish@yahoo.com), created on 20.01.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_svd)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/svd.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(svd, 1, 1, false, 0, 3) {
+  auto x = INPUT_VARIABLE(0);
+
+  const int rank = x->rankOf();
+  REQUIRE_TRUE(rank >= 2, 0, "SVD OP: the rank of input array must be >=2, but got %i instead!", rank);
+
+  bool fullUV = (bool)INT_ARG(0);
+  const bool calcUV = (bool)INT_ARG(1);
+
+  if (calcUV == false) fullUV = false;
+
+  const int switchNum = INT_ARG(2);
+
+  helpers::svd(block.launchContext(), x,
+               {OUTPUT_VARIABLE(0), calcUV ? OUTPUT_VARIABLE(1) : nullptr, calcUV ? OUTPUT_VARIABLE(2) : nullptr},
+               fullUV, calcUV, switchNum);
+
+  return Status::OK;
+  ;
+}
+
+DECLARE_TYPES(svd) {
+  getOpDescriptor()->setAllowedInputTypes(0, {FLOAT32, DOUBLE, HALF})->setSameMode(true);
+}
+
+DECLARE_SHAPE_FN(svd) {
+  auto inShapeInfo = inputShape->at(0);
+  bool fullUV = (bool)INT_ARG(0);
+  bool calcUV = (bool)INT_ARG(1);
+
+  const int rank = inShapeInfo[0];
+  REQUIRE_TRUE(rank >= 2, 0, "SVD OP: the rank of input array must be >=2, but got %i instead!", rank);
+
+  const int diagSize = inShapeInfo[rank] < inShapeInfo[rank - 1] ? inShapeInfo[rank] : inShapeInfo[rank - 1];
+
+  LongType* sShapeInfo(nullptr);
+  if (rank == 2) {
+    ALLOCATE(sShapeInfo, block.getWorkspace(), shape::shapeInfoLength(1), sd::LongType);
+    sShapeInfo[0] = 1;
+    sShapeInfo[1] = diagSize;
+  } else {
+    ALLOCATE(sShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank - 1), sd::LongType);
+    sShapeInfo[0] = rank - 1;
+    for (int i = 1; i <= rank - 2; ++i) sShapeInfo[i] = inShapeInfo[i];
+    sShapeInfo[rank - 1] = diagSize;
+  }
+
+  ShapeUtils::updateStridesAndType(sShapeInfo, inShapeInfo, shape::order(inShapeInfo));
+
+  if (calcUV) {
+    LongType *uShapeInfo(nullptr), *vShapeInfo(nullptr);
+    COPY_SHAPE(inShapeInfo, uShapeInfo);
+    COPY_SHAPE(inShapeInfo, vShapeInfo);
+
+    if (fullUV) {
+      uShapeInfo[rank] = uShapeInfo[rank - 1];
+      vShapeInfo[rank - 1] = vShapeInfo[rank];
+    } else {
+      uShapeInfo[rank] = diagSize;
+      vShapeInfo[rank - 1] = vShapeInfo[rank];
+      vShapeInfo[rank] = diagSize;
+    }
+
+    shape::updateStrides(uShapeInfo, shape::order(inShapeInfo), false);
+    shape::updateStrides(vShapeInfo, shape::order(inShapeInfo), false);
+    auto result = SHAPELIST(ConstantShapeHelper::getInstance().bufferForShapeInfo(sShapeInfo)->primary(),
+                            ConstantShapeHelper::getInstance().bufferForShapeInfo(uShapeInfo)->primary(),
+                            ConstantShapeHelper::getInstance().bufferForShapeInfo(vShapeInfo)->primary());
+    return result;
+  }
+
+  return SHAPELIST(ConstantShapeHelper::getInstance().createFromExisting(sShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/trace.cpp

@@ -0,0 +1,71 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 24.01.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_trace)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(trace, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->rankOf() >= 2, 0, "TRACE op: the rank of input array must be >=2, but got %i instead!",
+               input->rankOf());
+
+  helpers::trace(block.launchContext(), *input, *output);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(trace) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_FLOATS})->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(trace) {
+  auto inShapeInfo = inputShape->at(0);
+
+  REQUIRE_TRUE(inShapeInfo[0] >= 2, 0, "TRACE op: the rank of input array must be >=2, but got %i instead!",
+               inShapeInfo[0]);
+  const int rank = inShapeInfo[0] - 2;
+
+  LongType* outShapeInfo(nullptr);
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+
+  outShapeInfo[0] = rank;
+  for (int i = 1; i <= rank; ++i) outShapeInfo[i] = inShapeInfo[i];
+
+  shape::updateStrides(outShapeInfo, shape::order(inShapeInfo), false);
+  ArrayOptions::setDataType(outShapeInfo,ArrayOptions::dataType(inShapeInfo));
+  auto result = ConstantShapeHelper::getInstance().bufferForShapeInfo(outShapeInfo)->primary();
+  RELEASE(outShapeInfo, block.getWorkspace());
+  return SHAPELIST(result);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/tri.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma, created on 31.03.2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#if NOT_EXCLUDED(OP_tri)
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(tri, -2, 1, false, 0, 1) {
+  auto output = OUTPUT_VARIABLE(0);
+
+  const int diag = block.numI() > 2 ? INT_ARG(2) : 0;
+  char direction = diag <= 0  || diag == 0 || diag > 0 ? 'l': 'u';
+  BUILD_SINGLE_SELECTOR(output->dataType(), output->fillAsTriangular,
+                        (1., diag, diag, *output, direction),
+                        SD_COMMON_TYPES);  // fill with unities lower triangular block of matrix
+
+
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(tri) { getOpDescriptor()->setAllowedOutputTypes(0, {ALL_FLOATS, ALL_INTS}); }
+
+DECLARE_SHAPE_FN(tri) {
+  const int rows = INT_ARG(0);
+  const int cols = block.numI() > 1 ? INT_ARG(1) : rows;
+
+  auto dtype = block.numD() ? D_ARG(0) : FLOAT32;
+
+  return SHAPELIST(ConstantShapeHelper::getInstance().createShapeInfo(dtype, 'c', {rows, cols}));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/triangular_solve.cpp

@@ -0,0 +1,90 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// Created by GS <sgazeos@gmail.com> at 01/14/2020
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_triangual_solve)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/triangular_solve.h>
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(triangular_solve, 2, 1, false, 0, 0) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+  bool isLower = true;
+  bool useAdjoint = false;
+
+  if (block.numB() > 0) {
+    if (block.numB() > 1) {
+      isLower = B_ARG(0);
+      useAdjoint = B_ARG(1);
+    } else {
+      isLower = B_ARG(0);
+    }
+  }
+
+  REQUIRE_TRUE(a->rankOf() >= 2, 0,
+               "triangular_solve: The rank of input left tensor should not be less than 2, but %i is given",
+               a->rankOf());
+  REQUIRE_TRUE(b->rankOf() >= 2, 0,
+               "triangular_solve: The rank of input right tensor should not be less than 2, but %i is given",
+               b->rankOf());
+
+  REQUIRE_TRUE(a->sizeAt(-1) == a->sizeAt(-2), 0,
+               "triangular_solve: The last two dimensions should be equal, but %i and %i are given", a->sizeAt(-1),
+               a->sizeAt(-2));
+  REQUIRE_TRUE(a->sizeAt(-1) == b->sizeAt(-2), 0,
+               "triangular_solve: The last dimmension of left part should be equal to prelast of right part, but %i "
+               "and %i are given",
+               a->sizeAt(-1), b->sizeAt(-2));
+  auto input = a;
+  if (useAdjoint) {
+    auto adjointA = a->ulike();
+    helpers::adjointMatrix(block.launchContext(), a, isLower, adjointA);
+    input = new NDArray(adjointA);
+    isLower = !isLower;
+  };
+
+  auto res = helpers::triangularSolveFunctor(block.launchContext(), input, b, isLower, false, z);
+  if (input != a) delete input;
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(triangular_solve) {
+  auto in0 = inputShape->at(1);
+  auto in1 = inputShape->at(1);
+  auto luShape = ShapeBuilders::copyShapeInfoAndType(in1, in0, true, block.workspace());
+
+  return SHAPELIST(CONSTANT(luShape));
+}
+
+DECLARE_TYPES(triangular_solve) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS})->setSameMode(false);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/triu.cpp

@@ -0,0 +1,120 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 31.03.2018
+//
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/transforms.h>
+
+#if NOT_EXCLUDED(OP_triu)
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(triu, 1, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(input->rankOf() > 0, 0, "TRIU OP: the rank of input array must be > 0, but got %i instead !",
+               input->rankOf());
+
+  int diag = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+  auto lower = diag;
+  auto upper = diag;
+  char direction = diag <= 0  || diag > 0 ? 'l': 'u';
+  BUILD_SINGLE_SELECTOR(input->dataType(), input->fillAsTriangular, (0, lower, upper, *output, direction,false), SD_COMMON_TYPES);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(triu) {
+  getOpDescriptor()->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(triu) {
+  auto inShapeInfo = inputShape->at(0);
+
+  REQUIRE_TRUE(inShapeInfo[0] > 0, 0, "TRIU OP: the rank of input array must be > 0, but got %i instead !",
+               inShapeInfo[0]);
+
+  int rank = (inShapeInfo[0] == 1) ? 2 : inShapeInfo[0];
+
+  LongType* outShapeInfo = nullptr;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+  memcpy(outShapeInfo, inShapeInfo, (1 + rank) * sizeof(LongType));  // copy rank and dimensions values only
+
+  if (inShapeInfo[0] == 1) {
+    outShapeInfo[0] = rank;
+    outShapeInfo[1] = inShapeInfo[1];
+    outShapeInfo[2] = inShapeInfo[1];
+  }
+
+  ShapeUtils::updateStridesAndType(outShapeInfo, inShapeInfo, shape::order(inShapeInfo));
+
+  return SHAPELIST(CONSTANT(outShapeInfo));
+}
+
+//////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(triu_bp, 2, 1, false, 0, 0) {
+  auto input = INPUT_VARIABLE(0);
+  auto gradO = INPUT_VARIABLE(1);  // dLoss/dO
+
+
+  auto gradI = OUTPUT_VARIABLE(0);  // dLoss/dI
+  if(gradI->isScalar()) {
+    gradI->p(0,0.0);
+    return Status::OK;
+  }
+
+  REQUIRE_TRUE(input->rankOf() > 0, 0, "TRIU_BP OP: the rank of input array must be > 0, but got %i instead !",
+               input->rankOf());
+
+  const int diag = block.getIArguments()->size() > 0 ? INT_ARG(0) : 0;
+
+  helpers::triuBP(block.launchContext(), *input, *gradO, *gradI, diag);
+  return Status::OK;
+}
+
+DECLARE_TYPES(triu_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+
+DECLARE_SHAPE_FN(triu_bp) {
+  auto gradOShapeInfo = inputShape->at(0);
+  int rank = gradOShapeInfo[0];
+
+  LongType* outShapeInfo = nullptr;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+  memcpy(outShapeInfo, gradOShapeInfo, (1 + rank) * sizeof(LongType));  // copy rank and dimensions values only
+
+  auto in = inputShape->at(0);
+  ShapeUtils::updateStridesAndType(outShapeInfo, in, shape::order(in));
+
+  return SHAPELIST(CONSTANT(outShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/linalg/zeta.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 12.12.2017
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_zeta)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/zeta.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(zeta, 2, 1, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto q = INPUT_VARIABLE(1);
+
+  auto output = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(x->isSameShape(q), 0, "ZETA op: two input arrays must have the same shapes, bot got x=%s and q=%s !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(q).c_str());
+
+  LongType arrLen = x->lengthOf();
+
+  // FIXME: this should NOT be loop.
+  for (LongType i = 0; i < arrLen; ++i) {
+    REQUIRE_TRUE(x->e<float>(i) > 1.f, 0, "ZETA op: all elements of x array must be > 1 !");
+    REQUIRE_TRUE(q->e<float>(i) > 0.f, 0, "ZETA op: all elements of q array must be > 0 !");
+  }
+
+  helpers::zeta(block.launchContext(), *x, *q, *output);
+
+  return Status::OK;
+}
+DECLARE_SYN(Zeta, zeta);
+
+DECLARE_TYPES(zeta) { getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS})->setAllowedOutputTypes({ALL_FLOATS}); }
+}  // namespace ops
+}  // namespace sd
+
+#endif