chore: import upstream snapshot with attribution

libnd4j/include/ops/BroadcastBoolOpsTuple.h

@@ -0,0 +1,49 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef DEV_TESTS_BROADCASTBOOLOPSTUPLE_H
+#define DEV_TESTS_BROADCASTBOOLOPSTUPLE_H
+#include <system/common.h>
+#include <system/op_enums.h>
+
+namespace sd {
+class SD_LIB_EXPORT BroadcastBoolOpsTuple {
+ private:
+ public:
+  scalar::BoolOps s;
+  pairwise::BoolOps p;
+  broadcast::BoolOps b;
+
+  BroadcastBoolOpsTuple() = default;
+  ~BroadcastBoolOpsTuple() = default;
+
+  BroadcastBoolOpsTuple(scalar::BoolOps scalar, pairwise::BoolOps pairwise, broadcast::BoolOps broadcast) {
+    s = scalar;
+    p = pairwise;
+    b = broadcast;
+  }
+
+  static BroadcastBoolOpsTuple custom(scalar::BoolOps scalar, pairwise::BoolOps pairwise, broadcast::BoolOps broadcast);
+};
+}  // namespace sd
+
+#endif  // DEV_TESTS_BROADCASTOPSTUPLE_H

libnd4j/include/ops/BroadcastIntOpsTuple.h

@@ -0,0 +1,50 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef DEV_TESTS_BROADCASTINTOPSTUPLE_H
+#define DEV_TESTS_BROADCASTINTOPSTUPLE_H
+#include <system/common.h>
+#include <system/op_enums.h>
+
+namespace sd {
+class SD_LIB_EXPORT BroadcastIntOpsTuple {
+ private:
+ public:
+  sd::scalar::IntOps s;
+  sd::pairwise::IntOps p;
+  sd::broadcast::IntOps b;
+
+  BroadcastIntOpsTuple() = default;
+  ~BroadcastIntOpsTuple() = default;
+
+  BroadcastIntOpsTuple(sd::scalar::IntOps scalar, sd::pairwise::IntOps pairwise, sd::broadcast::IntOps broadcast) {
+    s = scalar;
+    p = pairwise;
+    b = broadcast;
+  }
+
+  static BroadcastIntOpsTuple custom(sd::scalar::IntOps scalar, sd::pairwise::IntOps pairwise,
+                                     sd::broadcast::IntOps broadcast);
+};
+}  // namespace sd
+
+#endif  // DEV_TESTS_BROADCASTOPSTUPLE_H

libnd4j/include/ops/BroadcastOpsTuple.h

@@ -0,0 +1,61 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef DEV_TESTS_BROADCASTOPSTUPLE_H
+#define DEV_TESTS_BROADCASTOPSTUPLE_H
+#include <system/common.h>
+#include <system/op_enums.h>
+
+namespace sd {
+class SD_LIB_EXPORT BroadcastOpsTuple {
+ private:
+ public:
+  scalar::Ops s;
+  pairwise::Ops p;
+  broadcast::Ops b;
+
+  BroadcastOpsTuple() = default;
+  ~BroadcastOpsTuple() = default;
+
+  BroadcastOpsTuple(scalar::Ops scalar, pairwise::Ops pairwise, broadcast::Ops broadcast) {
+    s = scalar;
+    p = pairwise;
+    b = broadcast;
+  }
+
+  static BroadcastOpsTuple custom(scalar::Ops scalar, pairwise::Ops pairwise, broadcast::Ops broadcast);
+
+  static BroadcastOpsTuple Add();
+  static BroadcastOpsTuple Assign();
+  static BroadcastOpsTuple Divide();
+  static BroadcastOpsTuple DivideNoNan();
+  static BroadcastOpsTuple Multiply();
+  static BroadcastOpsTuple Subtract();
+  static BroadcastOpsTuple IGamma();
+  static BroadcastOpsTuple IGammac();
+
+  static BroadcastOpsTuple Pow();
+  static BroadcastOpsTuple PowDerivative();
+};
+}  // namespace sd
+
+#endif  // DEV_TESTS_BROADCASTOPSTUPLE_H

libnd4j/include/ops/InputType.h

@@ -0,0 +1,38 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef ND4J_INPUTTYPE_H
+#define ND4J_INPUTTYPE_H
+
+namespace sd {
+namespace ops {
+enum InputType {
+  InputType_BOOLEAN = 0,
+  InputType_NUMERIC = 1,
+  InputType_STRINGULAR = 2,
+  InputType_NUMERIC_SET = 3,
+  InputType_STRINGULAR_SET = 4,
+};
+}
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/BooleanOp.h

@@ -0,0 +1,52 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#ifndef LIBND4J_BOOLEANOP_H
+#define LIBND4J_BOOLEANOP_H
+#include <graph/Context.h>
+
+#include "DeclarableOp.h"
+#include "OpDescriptor.h"
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT BooleanOp : public DeclarableOp {
+ protected:
+  OpDescriptor* _descriptor;
+
+  bool prepareOutputs(sd::graph::Context& block);
+  Status validateAndExecute(sd::graph::Context& block) override = 0;
+
+ public:
+  BooleanOp(const char* name, int numInputs, bool scalar);
+
+  bool verify(const std::vector<NDArray*>& args);
+  bool verify(sd::graph::Context& block);
+
+  Status execute(Context* block) override;
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_BOOLEANOP_H

libnd4j/include/ops/declarable/BroadcastableBoolOp.h

@@ -0,0 +1,45 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#ifndef SD_BROADCASTABLEBOOLOP_H
+#define SD_BROADCASTABLEBOOLOP_H
+#include <graph/Context.h>
+
+#include "DeclarableCustomOp.h"
+#include "DeclarableOp.h"
+#include "OpDescriptor.h"
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT BroadcastableBoolOp : public DeclarableCustomOp {
+ protected:
+  Status validateAndExecute(Context &block) override = 0;
+
+ public:
+  BroadcastableBoolOp(const char *name, int numTArgs, int numIArgs);
+
+  ShapeList *calculateOutputShape(ShapeList *inputShape, Context &block) override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // SD_BROADCASTABLEBOOLOP_H

libnd4j/include/ops/declarable/BroadcastableOp.h

@@ -0,0 +1,42 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#ifndef LIBND4J_BROADCASTABLEOP_H
+#define LIBND4J_BROADCASTABLEOP_H
+
+#include <ops/declarable/DeclarableCustomOp.h> 
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT BroadcastableOp : public DeclarableCustomOp {
+ protected:
+  Status validateAndExecute(Context &block) override = 0;
+
+ public:
+  BroadcastableOp(const char *name, int numTArgs, int numIArgs);
+
+  ShapeList *calculateOutputShape(ShapeList *inputShape, Context &block) override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_BROADCASTABLEOP_H

libnd4j/include/ops/declarable/CustomOperations.h

@@ -0,0 +1,100 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.10.2017.
+//
+
+#ifndef LIBND4J_CUSTOMOPERATIONS_H
+#define LIBND4J_CUSTOMOPERATIONS_H
+#include <array/NDArrayFactory.h>
+#include <helpers/ArrayUtils.h>
+#include <helpers/ConstantShapeHelper.h>
+#include <helpers/ConstantTadHelper.h>
+#include <helpers/OpTracker.h>
+#include <helpers/ShapeBuilders.h>
+#include <helpers/shape.h>
+#include <ops/declarable/headers/BarnesHutTsne.h>
+#include <ops/declarable/headers/activations.h>
+#include <ops/declarable/headers/bitwise.h>
+#include <ops/declarable/headers/blas.h>
+#include <ops/declarable/headers/boolean.h>
+#include <ops/declarable/headers/broadcastable.h>
+#include <ops/declarable/headers/compat.h>
+#include <ops/declarable/headers/convo.h>
+#include <ops/declarable/headers/datatypes.h>
+#include <ops/declarable/headers/decoder.h>
+#include <ops/declarable/headers/images.h>
+#include <ops/declarable/headers/kernels.h>
+#include <ops/declarable/headers/list.h>
+#include <ops/declarable/headers/loss.h>
+#include <ops/declarable/headers/nlp.h>
+#include <ops/declarable/headers/nn.h>
+#include <ops/declarable/headers/parity_ops.h>
+#include <ops/declarable/headers/random.h>
+#include <ops/declarable/headers/recurrent.h>
+#include <ops/declarable/headers/shape.h>
+#include <ops/declarable/headers/strings.h>
+#include <ops/declarable/headers/tests.h>
+#include <ops/declarable/headers/third_party.h>
+#include <ops/declarable/headers/transforms.h>
+#include <ops/declarable/headers/updaters.h>
+#include <ops/declarable/headers/util.h>
+
+namespace sd {
+     class SD_LIB_EXPORT  _loader {
+     public:
+         _loader();
+};
+
+namespace ops {
+
+// logic ops
+#if NOT_EXCLUDED(OP_Switch)
+DECLARE_DIVERGENT_OP(Switch, 2, 2, true);
+#endif
+#if NOT_EXCLUDED(OP_While)
+DECLARE_LOGIC_OP(While);
+#endif
+#if NOT_EXCLUDED(OP_Scope)
+DECLARE_LOGIC_OP(OpScope);
+#endif
+#if NOT_EXCLUDED(OP_Conditional)
+DECLARE_LOGIC_OP(Conditional);
+#endif
+#if NOT_EXCLUDED(OP_Return)
+DECLARE_LOGIC_OP(Return);
+#endif
+
+/**
+ * This operations exposes given arguments as it's own outputs, but does it only once.
+ * Subsequent calls will be served directly by this op.
+ *
+ * PLEASE NOTE: This operation is internal graph operation, and shouldn't be used directly usually.
+ */
+#if NOT_EXCLUDED(OP_expose)
+DECLARE_CUSTOM_OP(expose, -2, -2, true, 0, 0);
+#endif
+
+#if NOT_EXCLUDED(OP_invoke)
+DECLARE_CUSTOM_OP(invoke, 2, 2, false, 0, 0);
+#endif
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_CUSTOMOPERATIONS_H

libnd4j/include/ops/declarable/DeclarableCustomOp.h

@@ -0,0 +1,44 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.10.2017.
+//
+
+#ifndef LIBND4J_DECLARABLECUSTOMOP_H
+#define LIBND4J_DECLARABLECUSTOMOP_H
+#include <ops/declarable/DeclarableOp.h>
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT DeclarableCustomOp : public DeclarableOp {
+ protected:
+  /**
+   * This method executes this Op
+   */
+  Status validateAndExecute(sd::graph::Context& block) override = 0;
+
+ public:
+  DeclarableCustomOp(int numInputs, int numOutputs, const char* opName, bool allowsInplace, int tArgs, int iArgs);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShapes, Context& block) override = 0;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_DECLARABLECUSTOMOP_H

libnd4j/include/ops/declarable/DeclarableListOp.h

@@ -0,0 +1,57 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#ifndef LIBND4J_DECLARABLE_LIST_OP_H
+#define LIBND4J_DECLARABLE_LIST_OP_H
+#include <array/ResultSet.h>
+#include <graph/Context.h>
+#include <ops/declarable/DeclarableOp.h>
+#include <ops/declarable/OpRegistrator.h>
+
+using namespace sd::graph;
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT DeclarableListOp : public DeclarableOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override = 0;
+
+  NDArray* getZ(sd::graph::Context& block, int inputId);
+  void setupResult(NDArray* array, Context& block);
+  void setupResultList(NDArrayList* arrayList, Context& block);
+
+ public:
+  DeclarableListOp(int numInputs, int numOutputs, const char* opName, int tArgs, int iArgs);
+
+  Status execute(Context* block) override;
+
+  ResultSet execute(NDArrayList* list, std::initializer_list<NDArray*> inputs, std::initializer_list<double> tArgs,
+                    std::initializer_list<int> iArgs);
+  ResultSet execute(NDArrayList* list, std::vector<NDArray*>& inputs, std::vector<double>& tArgs,
+                    std::vector<int>& iArgs);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/DeclarableOp.h

@@ -0,0 +1,229 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef LIBND4J_DECLARABLE_OPS_H
+#define LIBND4J_DECLARABLE_OPS_H
+#include <system/common.h>
+#include <array/NDArray.h>
+#include <array/ResultSet.h>
+#include <array/ShapeList.h>
+#include <helpers/OpArgsHolder.h>
+#include <helpers/helper_hash.h>
+#include <ops/declarable/EmptyHandling.h>
+#include <ops/declarable/OpDescriptor.h>
+#include <types/float16.h>
+#include <graph/Context.h>
+
+#include <chrono>
+#include <ctime>
+#include <mutex>
+#include <sstream>
+
+using namespace sd::graph;
+
+namespace sd {
+namespace ops {
+
+SD_LIB_EXPORT ErrorResult conditionHelper(const char* file, int line, int condition, int argNumber, const char* format,
+                                         ...);
+
+template <typename T>
+sd::Status resultHelper(T status, const char* func, const char* file, int line) {
+  if (status != sd::Status::OK) {
+    //  TODO: fill out error codes here
+    fprintf(stderr, "Validation error at %s:%d code=%d(%s) \"%s\" \n", file, line, static_cast<unsigned int>(status),
+            "", func);
+
+    return sd::Status::BAD_INPUT;
+  }
+
+  return sd::Status::OK;
+}
+
+/**
+ * This class is the basic building block of Graph Operations. Any CustomOp out there is built on top of this "abstract"
+ * class.
+ *
+ */
+class SD_LIB_EXPORT DeclarableOp {
+ private:
+  std::mutex _registrator;
+  bool _registered = false;
+  std::string _name;
+
+ protected:
+  OpDescriptor* _descriptor;
+  NDArray* _scalar = nullptr;
+
+  virtual void registerTypes();
+
+  /**
+   * This method executes this Op, and defined for most of individual ops separately
+   */
+  virtual sd::Status validateAndExecute(Context& block) = 0;
+
+  /**
+   * This method ensures that target variable has enough space for op execution
+   *
+   * TODO: we want workspaces support right here
+   */
+  bool allocateResult(Context& block, std::initializer_list<sd::LongType>& shape, char order = 'c');
+  bool allocateResult(Context& block, sd::LongType* shape);
+
+  /**
+   * This method overwrites existing NDArray or NDArrayList in VariableSpace
+   *
+   * PLEASE NOTE: This method is dangerous.
+   *
+   * @param block
+   * @param numOutput
+   * @param array
+   */
+  void overwriteResult(Context& block, int outputIdx, NDArray* array);
+  void overwriteResult(Context& block, int outputIdx, NDArrayList* list);
+
+  /*
+   * This method attaches array to specific Variable, identified by node ID and outputNumber (which is output index for
+   * multi-output operations)
+   */
+  void storeResult(Context& block, int outputNumber, NDArray& array);
+  void storeResult(Context& block, int outputNumber, NDArray* array);
+  sd::NDArray* getZ(Context& block, int inputId = 0);
+  sd::NDArray* getNullifiedZ(Context& block, int inputId = 0);
+
+  /**
+   *   This method pre-allocates NDArrays for Op output, in case they are not available at op execution time
+   */
+  int prepareOutputs(Context& block);
+
+  virtual samediff::EmptyHandling emptyHandling();
+
+ public:
+  // for special cases, like BooleanOps
+  DeclarableOp();
+  DeclarableOp(const char* name, int numInputs, bool scalar);
+
+  // regular constructors
+  DeclarableOp(int numInputs, int numOutputs, const char* opName, bool allowsInplace);
+  DeclarableOp(int numInputs, int numOutputs, const char* opName, bool allowsInplace, bool divergent);
+  DeclarableOp(int numInputs, int numOutputs, const char* opName, bool allowsInplace, int tArgs, int iArgs);
+
+  // for LogicalOps
+  DeclarableOp(const char* name, bool isLogical);
+
+  // default testructor
+  virtual ~DeclarableOp();
+
+  // this method returns OpDescriptor, describing this Op instance
+  OpDescriptor* getOpDescriptor();
+
+  virtual sd::Status validateDataTypes(Context& block);
+
+  /**
+   *   This method should be available in each implemented Op, and should return Op output shape(s), for a given input
+   * shape(s)
+   */
+  virtual ShapeList* calculateOutputShape(ShapeList* inputShape, sd::graph::Context& block) = 0;
+
+  /**
+   * Returns opName
+   *
+   * @return
+   */
+  std::string* getOpName();
+
+  /**
+   * Returns opHash
+   */
+  sd::LongType getOpHash();
+
+
+
+  /**
+   * This method executes given Op
+   *
+   * @param block
+   * @return 0 if OK, error code otherwise
+   */
+  virtual sd::Status execute(Context* block);
+
+  sd::Status execute(const std::vector<NDArray*>& inputs, const std::vector<NDArray*>& outputs);
+
+  template <class T, typename = std::enable_if<DataTypeUtils::scalarTypesForExecution<T>::value>>
+  sd::Status execute(const std::vector<NDArray*>& inputs, const std::vector<NDArray*>& outputs,
+                     std::initializer_list<T> tArgs);
+
+  sd::Status execute(const std::vector<NDArray*>& inputs, const std::vector<NDArray*>& outputs,
+                     const std::vector<double>& tArgs, const std::vector<sd::LongType>& iArgs,
+                     const std::vector<bool>& bArgs = std::vector<bool>(),
+                     const std::vector<sd::DataType>& dArgs = std::vector<sd::DataType>(), bool isInplace = false);
+
+  sd::ResultSet evaluate(const std::vector<NDArray*>& inputs);
+
+  template <class T, typename = std::enable_if<DataTypeUtils::scalarTypesForExecution<T>::value>>
+  sd::ResultSet evaluate(const std::vector<NDArray*>& inputs, std::initializer_list<T> args);
+
+  sd::ResultSet evaluate(const std::vector<NDArray*>& inputs, const std::vector<double>& tArgs,
+                         const std::vector<sd::LongType>& iArgs, const std::vector<bool>& bArgs = std::vector<bool>(),
+                         const std::vector<sd::DataType>& dArgs = std::vector<sd::DataType>(), bool isInplace = false);
+
+  sd::Status execute(sd::graph::RandomGenerator& rng, const std::vector<NDArray*>& inputs,
+                     const std::vector<NDArray*>& outputs, const std::vector<double>& tArgs,
+                     const std::vector<sd::LongType>& iArgs, const std::vector<bool>& bArgs,
+                     const std::vector<sd::DataType>& dArgs = std::vector<sd::DataType>(), bool isInplace = false,
+                     sd::DataType type = sd::DataType::FLOAT32);
+
+  sd::ResultSet execute(const sd::OpArgsHolder& holder, bool isInplace = false);
+
+  // There methods provide various validation options
+  sd::Status validateNonEmptyInput(Context& block);
+
+  // this method checks if all input arrays have equal lengths
+  sd::Status validateInputLengthMatch(Context& block);
+
+  // this method checks if all input arrays have the same shapes (orders/strides are NOT checked)
+  sd::Status validateInputDimensionsMatch(Context& block);
+
+  // this method check if all input arrays have the same orders
+  sd::Status validateOrdersMatch(Context& block);
+
+  // this method checks if all input arrays are 2D
+  sd::Status validateInput2D(Context& block);
+
+  // this method checks if all input arrays are 3D
+  sd::Status validateInput3D(Context& block);
+
+  // this method checks if all input arrays are 4D
+  sd::Status validateInput4D(Context& block);
+
+  // this method checks if all input arrays are ND
+  sd::Status validateInputDimensions(Context& block, int rank);
+
+  // this method checks if number of available arguments matches op expectations
+  sd::Status validateArguments(Context& block);
+  void overwriteResult(Context& block, int outputIdx, NDArray* array, bool remove);
+  void traceExecIfNeeded(Context& block);
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_DECLARABLE_OPS_H

libnd4j/include/ops/declarable/DeclarableReductionOp.h

@@ -0,0 +1,44 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.10.2017.
+//
+
+#ifndef LIBND4J_DECLARABLE_REDUCTION_OP_H
+#define LIBND4J_DECLARABLE_REDUCTION_OP_H
+#include <ops/declarable/DeclarableOp.h>
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT DeclarableReductionOp : public DeclarableOp {
+ protected:
+  /**
+   * This method executes this Op
+   */
+  Status validateAndExecute(sd::graph::Context& block) override = 0;
+
+ public:
+  DeclarableReductionOp(int numInputs, int numOutputs, const char* opName, bool allowsInplace, int tArgs, int iArgs);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_DECLARABLE_REDUCTION_OP_H

libnd4j/include/ops/declarable/EmptyHandling.h

@@ -0,0 +1,30 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SAMEDIFF_EMPTYHANDLING_H
+#define SAMEDIFF_EMPTYHANDLING_H
+
+namespace samediff {
+enum EmptyHandling { EMPTY_SKIP = 1, EMPTY_EXCEPTION = 2, EMPTY_EXECUTE = 3 };
+}
+
+#endif  // SAMEDIFF_EMPTYHANDLING_H

libnd4j/include/ops/declarable/LegacyBroadcastBoolOp.h

@@ -0,0 +1,46 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 17.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYBROADCAST_BOOL_OP_H
+#define LIBND4J_LEGACYBROADCAST_BOOL_OP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for broadcast operations.
+ */
+class SD_LIB_EXPORT LegacyBroadcastBoolOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyBroadcastBoolOp();
+  LegacyBroadcastBoolOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYBROADCASTOP_H

libnd4j/include/ops/declarable/LegacyBroadcastOp.h

@@ -0,0 +1,46 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 17.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYBROADCASTOP_H
+#define LIBND4J_LEGACYBROADCASTOP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for broadcast operations.
+ */
+class SD_LIB_EXPORT LegacyBroadcastOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyBroadcastOp();
+  LegacyBroadcastOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYBROADCASTOP_H

libnd4j/include/ops/declarable/LegacyIndexReduceOp.h

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYINDEXREDUCEOP_H
+#define LIBND4J_LEGACYINDEXREDUCEOP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for IndexAccumulation operations. i.e. IndexMax or IndexAbsoluteMin etc
+ *
+ *   TODO: eventually we want this op class to return long long instead of T
+ */
+class SD_LIB_EXPORT LegacyIndexReduceOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyIndexReduceOp();
+  LegacyIndexReduceOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYINDEXREDUCEOP_H

libnd4j/include/ops/declarable/LegacyOp.h

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYOP_H
+#define LIBND4J_LEGACYOP_H
+#include <helpers/PointersManager.h>
+#include <ops/declarable/DeclarableOp.h>
+
+namespace sd {
+namespace ops {
+
+/**
+ * This class is root abstraction for legacy XYZ ops wrappers.
+ * All wrappers for specific op groups (i.e. LegacyTransformOp for Transform ops) are inheriting this class.
+ *
+ *
+ */
+class SD_LIB_EXPORT LegacyOp : public DeclarableOp {
+ protected:
+  // this field is mainly for debugging
+  // it defines, which legacy op should be invoked on a given data
+  int _opNum = -1;
+  int _numInputs = 0;
+
+  // All Op classes provide own specific implementation for this method
+  Status validateAndExecute(sd::graph::Context& block) override = 0;
+
+ public:
+  LegacyOp(int numInputs);
+  LegacyOp(int numInputs, int opNum);
+  ~LegacyOp() = default;
+
+  // All Op classes provide own specific implementation for this method
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override = 0;
+  virtual LegacyOp* clone() = 0;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYOP_H

libnd4j/include/ops/declarable/LegacyPairwiseTransformBoolOp.h

@@ -0,0 +1,46 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYPAIRWISETRANSFORM_BOOL_OP_H
+#define LIBND4J_LEGACYPAIRWISETRANSFORM_BOOL_OP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Pairwise transform operations
+ */
+class SD_LIB_EXPORT LegacyPairwiseTransformBoolOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyPairwiseTransformBoolOp();
+  LegacyPairwiseTransformBoolOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYPAIRWISETRANSFORMOP_H

libnd4j/include/ops/declarable/LegacyPairwiseTransformOp.h

@@ -0,0 +1,46 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYPAIRWISETRANSFORMOP_H
+#define LIBND4J_LEGACYPAIRWISETRANSFORMOP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Pairwise transform operations
+ */
+class SD_LIB_EXPORT LegacyPairwiseTransformOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyPairwiseTransformOp();
+  LegacyPairwiseTransformOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYPAIRWISETRANSFORMOP_H

libnd4j/include/ops/declarable/LegacyRandomOp.h

@@ -0,0 +1,60 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYRANDOMOP_H
+#define LIBND4J_LEGACYRANDOMOP_H
+
+#include <helpers/helper_random.h>
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Random operations (i.e. linspace or Uniform)
+ */
+class SD_LIB_EXPORT LegacyRandomOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyRandomOp();
+  LegacyRandomOp(int opNum);
+  ~LegacyRandomOp() = default;
+
+  template <typename T>
+  Status validateAndExecute_(sd::graph::Context& block);
+
+  ResultSet execute(RandomGenerator& rng, std::initializer_list<NDArray*> inputs, std::initializer_list<double> tArgs,
+                    std::initializer_list<int> iArgs, bool isInplace = false);
+  ResultSet execute(RandomGenerator& rng, std::vector<NDArray*>& inputs, std::vector<double>& tArgs,
+                    std::vector<int>& iArgs, bool isInplace = false);
+
+  Status execute(Context* block) override;
+
+  Status validateDataTypes(sd::graph::Context& block) override;
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYTRANSFORMOP_H

libnd4j/include/ops/declarable/LegacyReduce3Op.h

@@ -0,0 +1,46 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 17.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYREDUCE3OP_H
+#define LIBND4J_LEGACYREDUCE3OP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Reduce3 operations (i.e. dot, cosineDistance etc)
+ */
+class SD_LIB_EXPORT LegacyReduce3Op : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyReduce3Op();
+  LegacyReduce3Op(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYREDUCE3OP_H

libnd4j/include/ops/declarable/LegacyReduceBoolOp.h

@@ -0,0 +1,43 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACY_REDUCE_BOOL_OP_H
+#define LIBND4J_LEGACY_REDUCE_BOOL_OP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT LegacyReduceBoolOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyReduceBoolOp();
+  LegacyReduceBoolOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYREDUCEOP_H

libnd4j/include/ops/declarable/LegacyReduceFloatOp.h

@@ -0,0 +1,43 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACY_REDUCE_FLOAT_OP_H
+#define LIBND4J_LEGACY_REDUCE_FLOAT_OP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT LegacyReduceFloatOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyReduceFloatOp();
+  LegacyReduceFloatOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYREDUCEOP_H

libnd4j/include/ops/declarable/LegacyReduceLongOp.h

@@ -0,0 +1,43 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACY_REDUCE_LONG_OP_H
+#define LIBND4J_LEGACY_REDUCE_LONG_OP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT LegacyReduceLongOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyReduceLongOp();
+  LegacyReduceLongOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYREDUCEOP_H

libnd4j/include/ops/declarable/LegacyReduceOp.h

@@ -0,0 +1,32 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYREDUCEOP_H
+#define LIBND4J_LEGACYREDUCEOP_H
+
+
+#include <ops/declarable/LegacyReduceBoolOp.h>
+#include <ops/declarable/LegacyReduceFloatOp.h>
+#include <ops/declarable/LegacyReduceLongOp.h>
+#include <ops/declarable/LegacyReduceSameOp.h>
+
+#endif  // LIBND4J_LEGACYREDUCEOP_H

libnd4j/include/ops/declarable/LegacyReduceSameOp.h

@@ -0,0 +1,43 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYREDUCE_SAME_OP_H
+#define LIBND4J_LEGACYREDUCE_SAME_OP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT LegacyReduceSameOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyReduceSameOp();
+  LegacyReduceSameOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYREDUCEOP_H

libnd4j/include/ops/declarable/LegacyScalarBoolOp.h

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYSCALAR_BOOL_OP_H
+#define LIBND4J_LEGACYSCALAR_BOOL_OP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for scalar transform operations, i.e. a + b = c, where either a or b is scalar
+ * primitive and other operand is NDArray
+ */
+class SD_LIB_EXPORT LegacyScalarBoolOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyScalarBoolOp();
+  LegacyScalarBoolOp(int opNum);
+  LegacyScalarBoolOp(int opNum, NDArray& scalar);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYSCALAROP_H

libnd4j/include/ops/declarable/LegacyScalarOp.h

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYSCALAROP_H
+#define LIBND4J_LEGACYSCALAROP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for scalar transform operations, i.e. a + b = c, where either a or b is scalar
+ * primitive and other operand is NDArray
+ */
+class SD_LIB_EXPORT LegacyScalarOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyScalarOp();
+  LegacyScalarOp(int opNum);
+  LegacyScalarOp(int opNum, NDArray& scalar);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYSCALAROP_H

libnd4j/include/ops/declarable/LegacyStatsOp.h

@@ -0,0 +1,46 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 17.10.2017.
+//
+
+#ifndef LIBND4J_LEGACYSTATSOP_H
+#define LIBND4J_LEGACYSTATSOP_H
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for SummaryStats operations: Variance and Standard Deviation
+ */
+class SD_LIB_EXPORT LegacyStatsOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyStatsOp();
+  LegacyStatsOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LEGACYSTATSOP_H

libnd4j/include/ops/declarable/LegacyTransformAnyOp.h

@@ -0,0 +1,47 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J__LEGACY_TRANSFORM_ANY_OP__H
+#define LIBND4J__LEGACY_TRANSFORM_ANY_OP__H
+
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Transform operations (i.e. Pow or OneMinus)
+ */
+class SD_LIB_EXPORT LegacyTransformAnyOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyTransformAnyOp();
+  LegacyTransformAnyOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J__LEGACY_TRANSFORM_FLOAT_OP__H

libnd4j/include/ops/declarable/LegacyTransformBoolOp.h

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+// Modified by GS <sgazeos@gmail.com> on 19.10.2018
+// Modified by GS <sgazeos@gmail.com> on 19.10.2018
+//
+#ifndef LIBND4J__LEGACY_TRANSFORM_BOOL_OP__H
+#define LIBND4J__LEGACY_TRANSFORM_BOOL_OP__H
+
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Transform operations (i.e. Pow or OneMinus)
+ */
+class SD_LIB_EXPORT LegacyTransformBoolOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyTransformBoolOp();
+  LegacyTransformBoolOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J__LEGACY_TRANSFORM_SAME_OP__H

libnd4j/include/ops/declarable/LegacyTransformFloatOp.h

@@ -0,0 +1,47 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J__LEGACY_TRANSFORM_FLOAT_OP__H
+#define LIBND4J__LEGACY_TRANSFORM_FLOAT_OP__H
+
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Transform operations (i.e. Pow or OneMinus)
+ */
+class SD_LIB_EXPORT LegacyTransformFloatOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyTransformFloatOp();
+  LegacyTransformFloatOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J__LEGACY_TRANSFORM_FLOAT_OP__H

libnd4j/include/ops/declarable/LegacyTransformOp.h

@@ -0,0 +1,52 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+//
+
+#ifndef LIBND4J__LEGACY_TRANSFORM_OP__H
+#define LIBND4J__LEGACY_TRANSFORM_OP__H
+
+
+#ifdef ONLY_SAME_TRANSFORM
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Transform operations (i.e. Pow or OneMinus)
+ */
+class SD_LIB_EXPORT LegacyTransformOp : public LegacyOp {
+ protected:
+  sd::Status validateAndExecute(sd::graph::Context& block);
+
+ public:
+  LegacyTransformOp();
+  LegacyTransformOp(int opType);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, sd::graph::Context& block);
+  virtual LegacyOp* clone();
+};
+}  // namespace ops
+}  // namespace sd
+#endif
+#include <ops/declarable/LegacyTransformBoolOp.h>
+#include <ops/declarable/LegacyTransformFloatOp.h>
+#include <ops/declarable/LegacyTransformSameOp.h>
+#include <ops/declarable/LegacyTransformStrictOp.h>
+
+#endif  // LIBND4J__LEGACY_TRANSFORM_OP__H

libnd4j/include/ops/declarable/LegacyTransformSameOp.h

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+// Modified by GS <sgazeos@gmail.com> on 19.10.2018
+// Modified by GS <sgazeos@gmail.com> on 19.10.2018
+//
+#ifndef LIBND4J__LEGACY_TRANSFORM_SAME_OP__H
+#define LIBND4J__LEGACY_TRANSFORM_SAME_OP__H
+
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Transform operations (i.e. Pow or OneMinus)
+ */
+class SD_LIB_EXPORT LegacyTransformSameOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyTransformSameOp();
+  LegacyTransformSameOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J__LEGACY_TRANSFORM_SAME_OP__H

libnd4j/include/ops/declarable/LegacyTransformStrictOp.h

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 16.10.2017.
+// Modified by GS <sgazeos@gmail.com> on 19.10.2018
+// Modified by GS <sgazeos@gmail.com> on 19.10.2018
+//
+#ifndef LIBND4J__LEGACY_TRANSFORM_STRICT_OP__H
+#define LIBND4J__LEGACY_TRANSFORM_STRICT_OP__H
+
+#include <ops/declarable/LegacyOp.h>
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides wrapper for Transform operations (i.e. Pow or OneMinus)
+ */
+class SD_LIB_EXPORT LegacyTransformStrictOp : public LegacyOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LegacyTransformStrictOp();
+  LegacyTransformStrictOp(int opNum);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+  LegacyOp* clone() override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J__LEGACY_TRANSFORM_SAME_OP__H

libnd4j/include/ops/declarable/LogicOp.h

@@ -0,0 +1,49 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 15.10.2017.
+//
+
+#ifndef LIBND4J_LOGICOP_H
+#define LIBND4J_LOGICOP_H
+#include "DeclarableOp.h"
+
+namespace sd {
+namespace ops {
+
+/**
+ * Logic ops are unique snowflakes in any Graph. They dramatically change Graph Execution process, by introducing loops,
+ * conditions, etc.
+ *
+ * Their code is the part of GraphExecutioner logic. But we still want them to be expressed via Graph
+ * @tparam T
+ */
+class SD_LIB_EXPORT LogicOp : public DeclarableOp {
+ protected:
+  Status validateAndExecute(sd::graph::Context& block) override;
+
+ public:
+  LogicOp(const char* name);
+
+  ShapeList* calculateOutputShape(ShapeList* inputShape, Context& block) override;
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_LOGICOP_H

libnd4j/include/ops/declarable/OpDescriptor.h

@@ -0,0 +1,270 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef LIBND4J_OPDESCRIPTOR_H
+#define LIBND4J_OPDESCRIPTOR_H
+#include <array/DataType.h>
+#include <graph/generated/node_generated.h>
+#include <helpers/helper_hash.h>
+#include <ops/InputType.h>
+
+#include <initializer_list>
+#include <string>
+#include <vector>
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT OpExecTrace {
+ public:
+  std::vector<const LongType*> *inputShapeBuffers;
+  std::vector<const LongType*> *outputShapeBuffers;
+  const std::string *opName;
+  std::vector<LongType> iArgs;
+  std::vector<double> tArgs;
+  std::vector<DataType> dArgs;
+  std::vector<bool> bArgs;
+  std::vector<std::string> sArguments;
+  int opType = -1;
+
+
+#ifndef __JAVACPP_HACK__
+  OpExecTrace(std::vector<const LongType*> *inputShapeBuffers,
+              std::vector<const LongType*> *outputShapeBuffers,
+              const std::string *opName) {
+    this->inputShapeBuffers = inputShapeBuffers;
+    this->outputShapeBuffers = outputShapeBuffers;
+    this->opName = opName;
+
+  }
+
+  OpExecTrace(std::vector<const LongType*> *inputShapeBuffers,
+              std::vector<const LongType*> *outputShapeBuffers,
+              const std::string *opName,
+              std::vector<LongType> *iArgs,
+              std::vector<double> *tArgs,
+              std::vector<bool> *bArgs,
+              std::vector<std::string> *sArgs,
+              int opType) {
+    this->inputShapeBuffers = inputShapeBuffers;
+    this->outputShapeBuffers = outputShapeBuffers;
+    this->opName = opName;
+    this->opType = opType;
+    for(size_t i = 0; i < tArgs->size(); i++) {
+      this->tArgs.push_back(tArgs->at(i));
+    }
+
+    for(size_t i = 0; i < bArgs->size(); i++) {
+      this->bArgs.push_back(bArgs->at(i));
+    }
+
+    for(size_t i = 0; i < iArgs->size(); i++) {
+      this->iArgs.push_back(iArgs->at(i));
+    }
+
+    for(size_t i = 0; i < sArgs->size(); i++) {
+      this->sArguments.push_back(sArgs->at(i));
+    }
+
+  }
+#endif
+
+  OpExecTrace() = default;
+
+  ~OpExecTrace() = default;
+
+  std::vector<const LongType*>* getInputShapeBuffers() const { return inputShapeBuffers; }
+  void setInputShapeBuffers(std::vector<const LongType*>* inputShapeBuffersIn) {
+    OpExecTrace::inputShapeBuffers = inputShapeBuffersIn;
+  }
+  std::vector<const LongType*>* getOutputShapeBuffers() const { return outputShapeBuffers; }
+  void setOutputShapeBuffers(std::vector<const LongType*>* outputShapeBuffersIn) {
+    OpExecTrace::outputShapeBuffers = outputShapeBuffersIn;
+  }
+  const std::string* getOpName() const { return opName; }
+  void setOpName(const std::string* opNameIn) { OpExecTrace::opName = opNameIn; }
+  const std::vector<LongType>& getIArgs() const { return iArgs; }
+  void setIArgs(const std::vector<LongType>& iArgsIn) { OpExecTrace::iArgs = iArgsIn; }
+  const std::vector<double>& getTArgs() const { return tArgs; }
+  void setTArgs(const std::vector<double>& tArgsIn) { OpExecTrace::tArgs = tArgsIn; }
+  const std::vector<DataType>& getDArgs() const { return dArgs; }
+  void setDArgs(const std::vector<DataType>& dArgsIn) { OpExecTrace::dArgs = dArgsIn; }
+  const std::vector<bool>& getBArgs() const { return bArgs; }
+  void setBArgs(const std::vector<bool>& bArgsIn) { OpExecTrace::bArgs = bArgsIn; }
+  const std::vector<std::string>& getSArguments() const { return sArguments; }
+  void setSArguments(const std::vector<std::string>& sArgumentsIn) { OpExecTrace::sArguments = sArgumentsIn; }
+  int getOpType() const { return opType; }
+  void setOpType(int opTypeIn) { OpExecTrace::opType = opTypeIn; }
+};
+
+/**
+ *   This class is very basic info holder for ops. bean/pojo pretty much.
+ *
+ */
+class SD_LIB_EXPORT OpDescriptor {
+ protected:
+  // opType for legacy XYZ ops
+  int _opNum = 0;
+
+  // opName for CustomOp
+  std::string _opName;
+
+  // hash is used for ops lookup in OpRegistrator
+  LongType _hash = -1;
+
+  // minimal required/expected number of inputs/outpus for this given op
+  int _numInputs = 1;
+  int _numOutputs = 1;
+
+  // special flag for divergent ops - ops that CAN and WILL modify graph behavior. Literally: IF, CASE.
+  bool _divergent = false;
+
+  // flag, if this given op allows in-place execution
+  bool _allowsInplace = true;
+
+  // minimal required number of T-type arguments.
+  // -1 as value means: not limited, variable number of arguments
+  int _tArgs = 0;
+
+  // minimal required number of Integer-type arguments.
+  // -1 as value means: not limited, variable number of arguments
+  int _iArgs = 0;
+
+  // field for BooleanOps
+  bool _scalar = false;
+
+  // field for LogicOps
+  bool _logic = false;
+
+  // default InputType is numeric
+  InputType _inputType = InputType_NUMERIC;
+
+  bool _sameMode = false;
+  std::vector<DataType> _allowedIns;
+  std::vector<DataType> _allowedOuts;
+
+  // optional per-input configuration
+  SD_MAP_IMPL<int, std::vector<DataType>> _outputTypes;
+  SD_MAP_IMPL<int, std::vector<DataType>> _inputTypes;
+
+  // field for ops that allow data type override at runtime
+  bool _dtypeOverride = false;
+
+  bool checkDataTypesMatch(DataType needle, std::vector<DataType>& haystack) const;
+
+ public:
+  // default constructor
+  OpDescriptor(int numInputs, int numOutputs, std::string opName, bool allowsInplace);
+
+  // constructor for boolean ops
+  OpDescriptor(int numInputs, std::string opName, bool isScalar);
+  OpDescriptor(int numInputs, const char* opName, bool isScalar);
+
+  // default constructor
+  OpDescriptor(int numInputs, int numOutputs, const char* opName, bool allowsInplace);
+
+  // constructor for configurable op
+  OpDescriptor(int numInputs, int numOutputs, const char* opName, bool allowsInplace, int tArgs, int iArgs);
+
+  // constructor for non-configurable divergent op
+  OpDescriptor(int numInputs, int numOutputs, std::string opName, bool allowsInplace, bool divergent);
+
+  // constructor for non-configurable divergent op
+  OpDescriptor(int numInputs, int numOutputs, const char* opName, bool allowsInplace, bool divergent);
+
+  // constructor for configurable divergent op
+  OpDescriptor(int numInputs, int numOutputs, const char* opName, bool allowsInplace, bool divergent, int tArgs,
+               int iArgs);
+
+  // constructor for logical ops (while, scope, etc)
+  OpDescriptor(const char* opName, bool isLogic);
+
+  bool operator==(const OpDescriptor& other) const;
+
+  // default destructor
+  ~OpDescriptor() = default;
+
+  // this method returns minimal expected number of T arguments
+  int getNumberOfTArgs();
+
+  // this method returns minimal expected number of Integer arguments
+  int getNumberOfIArgs();
+
+  // this method returns minimal expected number of inputs
+  int getNumberOfInputs();
+
+  // this method returns hash code for this operation
+  LongType getHash();
+
+  // this method returns minimal expected number of outputs
+  int getNumberOfOutputs();
+
+  // this method returns opName (can be empty)
+  std::string* getOpName();
+
+  // returns TRUE if this op is divergent. FALSE otherwise
+  bool isDivergent();
+
+  // returns TRUE if this op allows in-place execution
+  bool allowsInplace();
+
+  // this method allows you to enable/disable inplace call for a given op
+  void allowInplace(bool reallyAllow);
+
+  // this method returns opType (applicable for legacy XYZ ops only)
+  int getOpNum();
+
+  // this method allows to set specific opNum
+  void setOpNum(int opNum);
+
+  void setHash(LongType hash);
+
+  InputType inputType();
+
+  OpDescriptor* setInputType(InputType type);
+  OpDescriptor* setAllowedInputTypes(const std::initializer_list<DataType>& dtype);
+  OpDescriptor* setAllowedOutputTypes(const std::initializer_list<DataType>& dtype);
+  OpDescriptor* setAllowedInputTypes(int index, const std::vector<DataType>& dtype);
+  OpDescriptor* setAllowedOutputTypes(int index, const std::vector<DataType>& dtype);
+  OpDescriptor* setAllowedInputTypes(int index, DataType dtype);
+  OpDescriptor* setAllowedOutputTypes(int index, DataType dtype);
+  OpDescriptor* setAllowedInputTypes(DataType dtype);
+  OpDescriptor* setAllowedOutputTypes(DataType dtype);
+  OpDescriptor* allowOverride(bool reallyAllow);
+  OpDescriptor* setSameMode(bool reallySame);
+  OpDescriptor* setInputType(int idx, DataType dtype);
+  OpDescriptor* setOutputType(int idx, DataType dtype);
+
+  std::vector<DataType> getOutputTypesForOutput(int index);
+  std::vector<DataType> getInputTypesForInput(int index);
+
+
+
+  bool checkInputMatch(int index, DataType dataType);
+  bool checkOutputMatch(int index, DataType dataType);
+  bool isSameMode();
+
+  bool isInherit(int index);
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_OPDESCRIPTOR_H

libnd4j/include/ops/declarable/OpRegistrator.h

@@ -0,0 +1,149 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 07.10.2017.
+//
+
+#ifndef LIBND4J_OPREGISTRATOR_H
+#define LIBND4J_OPREGISTRATOR_H
+
+#include <execution/Engine.h>
+#include <ops/declarable/DeclarableOp.h>
+#include <ops/declarable/PlatformHelper.h>
+#include <ops/declarable/PlatformHelperLegacy.h>
+
+#include <mutex>
+#include <unordered_map>
+#include <vector>
+
+// handlers part
+#include <csignal>
+#include <cstdlib>
+
+#ifndef __JAVACPP_HACK__
+
+namespace std {
+
+template <>
+class hash<std::pair<sd::LongType, samediff::Engine>> {
+ public:
+  size_t operator()(const std::pair<sd::LongType, samediff::Engine>& k) const;
+};
+
+template <>
+class hash<std::pair<std::string, samediff::Engine>> {
+ public:
+  size_t operator()(const std::pair<std::string, samediff::Engine>& k) const;
+};
+};  // namespace std
+
+#endif
+
+namespace sd {
+namespace ops {
+/**
+ *   This class provides runtime ops lookup, based on opName or opHash.
+ *   To build lookup directory we use *_OP_IMPL macro, which puts static structs at compile time in .cpp files,
+ *   so once binary is executed, static objects are initialized automatically, and we get list of all ops
+ *   available at runtime via this singleton.
+ *
+ */
+class SD_LIB_EXPORT OpRegistrator {
+ private:
+  static OpRegistrator* _INSTANCE;
+  OpRegistrator() {
+    sd_debug("OpRegistrator started\n", "");
+  };
+
+  SD_MAP_IMPL<LongType, std::string> _msvc;
+
+  // pointers to our operations
+  SD_MAP_IMPL<LongType, DeclarableOp*> _declarablesLD;
+  SD_MAP_IMPL<std::string, DeclarableOp*> _declarablesD;
+  std::vector<DeclarableOp*> _uniqueD;
+
+  // pointers to platform-specific helpers
+  SD_MAP_IMPL<std::pair<LongType, samediff::Engine>, platforms::PlatformHelper*> _helpersLH;
+  SD_MAP_IMPL<std::pair<std::string, samediff::Engine>, platforms::PlatformHelper*> _helpersH;
+  std::vector<platforms::PlatformHelper*> _uniqueH;
+
+
+  std::mutex _locker;
+  std::string _opsList;
+  std::vector<OpExecTrace *> opexecTrace;
+
+  bool isInit = false;
+  bool isTrace = false;
+ public:
+  ~OpRegistrator();
+
+  void purgeOpExecs();
+  void registerOpExec(OpExecTrace *opExecTrace);
+  std::vector<OpExecTrace *> * execTrace();
+
+  static OpRegistrator& getInstance();
+
+  void updateMSVC(LongType newHash, std::string& oldName);
+
+  template <typename T>
+  std::string local_to_string(T value);
+  const char* getAllCustomOperations();
+
+  /**
+   * This method registers operation in our registry, so we can use them later
+   *
+   * @param op
+   */
+  bool registerOperation(const char* name, DeclarableOp* op);
+  bool registerOperation(DeclarableOp* op);
+  bool traceOps();
+  void toggleTraceOps(bool traceOps);
+  void registerHelper(platforms::PlatformHelper* op);
+
+
+  bool hasHelper(LongType hash, samediff::Engine engine);
+
+  DeclarableOp* getOperation(const char* name);
+  DeclarableOp* getOperation(LongType hash);
+  DeclarableOp* getOperation(std::string& name);
+
+  platforms::PlatformHelper* getPlatformHelper(LongType hash, samediff::Engine engine);
+
+  std::vector<LongType> getAllHashes();
+
+  int numberOfOperations();
+};
+
+/*
+ *  These structs are used to "register" our ops in OpRegistrator.
+ */
+template <typename OpName>
+struct __registrator {
+  __registrator();
+};
+
+template <typename OpName>
+struct __registratorSynonym {
+  __registratorSynonym(const char* name, const char* oname);
+};
+
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_OPREGISTRATOR_H

libnd4j/include/ops/declarable/OpTuple.h

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 11.10.2017.
+//
+
+#ifndef LIBND4J_OPTUPLE_H
+#define LIBND4J_OPTUPLE_H
+#include <array/NDArray.h>
+
+#include <initializer_list>
+#include <vector>
+
+namespace sd {
+namespace ops {
+class SD_LIB_EXPORT OpTuple {
+ public:
+  std::string _opName;
+  std::vector<NDArray*> _inputs;
+  std::vector<NDArray*> _outputs;
+  std::vector<double> _tArgs;
+  std::vector<LongType> _iArgs;
+
+  OpTuple(const char* opName);
+  OpTuple(const char* opName, std::initializer_list<NDArray*>&& inputs, std::initializer_list<double>&& tArgs,
+          std::initializer_list<LongType>&& iArgs);
+  ~OpTuple();
+
+  OpTuple* addInput(NDArray* array);
+  OpTuple* addOutput(NDArray* array);
+  OpTuple* setTArgs(std::initializer_list<double> tArgs);
+  OpTuple* setIArgs(std::initializer_list<LongType> iArgs);
+};
+}  // namespace ops
+}  // namespace sd
+
+#endif  // LIBND4J_OPTUPLE_H

libnd4j/include/ops/declarable/PlatformHelper.h

@@ -0,0 +1,96 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SD_PLATFORMHELPER_H
+#define SD_PLATFORMHELPER_H
+#include <execution/Engine.h>
+#include <graph/Context.h>
+#include <helpers/ShapeUtils.h>
+#include <system/RequirementsHelper.h>
+
+#include <string>
+
+namespace sd {
+namespace ops {
+namespace platforms {
+/**
+ * This abstract class defines methods used by platform-specific helpers implementations
+ */
+class SD_LIB_EXPORT PlatformHelper {
+ protected:
+  // target engine for this impl
+  samediff::Engine _engine;
+
+  // name of the operation this helper is built for
+  std::string _name;
+
+  // hash of the operation this helper is built for
+  LongType _hash;
+
+ public:
+  PlatformHelper(const char *name, samediff::Engine engine);
+
+  virtual ~PlatformHelper() = default;
+
+  std::string name();
+
+  samediff::Engine engine();
+
+  LongType hash();
+
+  /**
+   * This method checks, if given helper can be used with given input/output/configuration options
+   *
+   * @param context
+   * @return
+   */
+  virtual bool isUsable(graph::Context &context) = 0;
+
+  /**
+   * This method invokes helper. Typically this method replaces actual op execution
+   *
+   * @param context
+   * @return
+   */
+  virtual Status invokeHelper(graph::Context &context) = 0;
+
+  /**
+   * Helper method, needed for compatibility with DeclarableOp macros
+   * @param ctx
+   * @param inputId
+   * @return
+   */
+  NDArray *getZ(graph::Context &ctx, int inputId);
+
+  /**
+   * Helper method, needed for compatibility with DeclarableOp macros
+   * @param ctx
+   * @param inputId
+   * @return
+   */
+  NDArray *getNullifiedZ(graph::Context &ctx, int inputId);
+};
+}  // namespace platforms
+}  // namespace ops
+}  // namespace sd
+
+#endif  // SD_PLATFORMHELPER_H

libnd4j/include/ops/declarable/PlatformHelperLegacy.h

@@ -0,0 +1,92 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+#ifndef SD_PLATFORMHELPERLEGACY_H
+#define SD_PLATFORMHELPERLEGACY_H
+#include <array/InteropDataBuffer.h>
+#include <execution/Engine.h>
+#include <graph/Context.h>
+#include <helpers/ShapeUtils.h>
+#include <system/RequirementsHelper.h>
+#include <system/op_enums.h>
+
+#include <string>
+
+namespace sd {
+namespace ops {
+namespace platforms {
+
+struct PlatformHelperLegacyEntry {
+  // prefix for the legacy. must be constant with global scope
+  const char *prefix;
+
+  int opNum;
+  // target engine for this impl
+  samediff::Engine engine;
+
+  bool operator==(const PlatformHelperLegacyEntry &other) const {
+    return (prefix == other.prefix && opNum == other.opNum && engine == other.engine);
+  }
+};
+
+struct PlatformHelperLegacyEntryHasher {
+  std::size_t operator()(PlatformHelperLegacyEntry const &p) const noexcept {
+    auto res = std::hash<LongType>()(reinterpret_cast<LongType>(p.prefix));
+    res ^= std::hash<int>()(p.opNum) + 0x9e3779b9 + (res << 6) + (res >> 2);
+    res ^= std::hash<int>()(p.engine) + 0x9e3779b9 + (res << 6) + (res >> 2);
+    return res;
+  }
+};
+/**
+ * This abstract class defines methods used by platform-specific helpers implementations for legacy ops
+ */
+class SD_LIB_EXPORT PlatformHelperLegacy {
+ protected:
+  PlatformHelperLegacyEntry entry;
+
+ public:
+  PlatformHelperLegacy(const char *prefix, int opNum, samediff::Engine engine) : entry{prefix, opNum, engine} {}
+
+  ~PlatformHelperLegacy() = default;
+
+  PlatformHelperLegacyEntry getEntry() const { return entry; }
+
+  /**
+   * This method checks, if given helper can be used with given input/output shapes
+   *
+   * @param context
+   * @return
+   */
+  virtual bool isUsable(void *extraParams, const LongType *outShapeInfo, const LongType *inArg0ShapeInfo,
+                        const LongType *inArg1ShapeInfo) = 0;
+
+  /**
+   * This method invokes helper
+   *
+   * @param context
+   * @return
+   */
+  virtual Status invokeHelper(void *extraParams, const LongType *outShapeInfo, InteropDataBuffer *outputBuffer, const LongType *inArg0ShapeInfo,
+                                  const InteropDataBuffer *inArg0Buffer, const LongType *inArg1ShapeInfo,
+                                  const InteropDataBuffer *inArg1Buffer) = 0;
+};
+}  // namespace platforms
+}  // namespace ops
+}  // namespace sd
+
+#endif  // SD_PLATFORMHELPERLEGACY_H

libnd4j/include/ops/declarable/generic/CustomOperations.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// This is special snowflake. This file builds bindings for ops availability tests
+//
+// @author raver119@gmail.com
+//
+
+#include <helpers/OpTracker.h>
+#include <loops/legacy_ops.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+
+_loader::_loader() {
+  //
+  OpTracker::getInstance();
+
+  BUILD_TRACKER(::graph::OpType_TRANSFORM_SAME, TRANSFORM_FLOAT_OPS);
+  BUILD_TRACKER(::graph::OpType_TRANSFORM_SAME, TRANSFORM_SAME_OPS);
+  BUILD_TRACKER(::graph::OpType_TRANSFORM_SAME, TRANSFORM_BOOL_OPS);
+  BUILD_TRACKER(::graph::OpType_BROADCAST, BROADCAST_OPS);
+  BUILD_TRACKER(::graph::OpType_PAIRWISE, PAIRWISE_TRANSFORM_OPS);
+  BUILD_TRACKER(::graph::OpType_RANDOM, RANDOM_OPS);
+  BUILD_TRACKER(::graph::OpType_REDUCE_FLOAT, REDUCE_FLOAT_OPS);
+  BUILD_TRACKER(::graph::OpType_REDUCE_SAME, REDUCE_SAME_OPS);
+  BUILD_TRACKER(::graph::OpType_REDUCE_BOOL, REDUCE_BOOL_OPS);
+  BUILD_TRACKER(::graph::OpType_REDUCE_3, REDUCE3_OPS);
+  BUILD_TRACKER(::graph::OpType_INDEX_REDUCE, INDEX_REDUCE_OPS);
+  BUILD_TRACKER(::graph::OpType_SCALAR, SCALAR_OPS);
+  BUILD_TRACKER(::graph::OpType_SUMMARYSTATS, SUMMARY_STATS_OPS);
+};
+
+static sd::_loader loader;
+}  // namespace sd

libnd4j/include/ops/declarable/generic/README.md

@@ -0,0 +1,4 @@
+## Generic ops
+
+Put any cross platform operations here. Anything platform specific should be put in:
+in ops/helpers.

libnd4j/include/ops/declarable/generic/bitwise/bits_hamming_distance.cpp

@@ -0,0 +1,60 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bits_hamming_distance)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/hamming.h>
+#include <ops/declarable/helpers/helpers.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(bits_hamming_distance, 2, 1, true, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto output = OUTPUT_NULLIFIED(0);
+
+  REQUIRE_TRUE(x->lengthOf() == y->lengthOf(), 0, "bits_hamming_distance: both arguments must have the same length");
+  REQUIRE_TRUE(x->dataType() == y->dataType(), 0, "bits_hamming_distance: both arguments must have the same data type");
+
+  helpers::hamming(block.launchContext(), *x, *y, *output);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(bits_hamming_distance) {
+  return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(sd::DataType::INT64));
+}
+
+DECLARE_TYPES(bits_hamming_distance) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_INTS})
+      ->setAllowedOutputTypes(0, {ALL_INDICES});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/bitwise_and.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bitwise_and)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(bitwise_and, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::IntOps::IntAnd, pairwise::IntOps::IntAnd, broadcast::IntOps::IntAnd), y, z,
+      false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(bitwise_and) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/bitwise_or.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bitwise_or)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(bitwise_or, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::IntOps::IntOr, pairwise::IntOps::IntOr, broadcast::IntOps::IntOr), y, z,
+      false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(bitwise_or) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/bitwise_xor.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_bitwise_xor)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(bitwise_xor, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::IntOps::IntXor, pairwise::IntOps::IntXor, broadcast::IntOps::IntXor), y, z,
+      false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(bitwise_xor) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/cyclic_rshift.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_cyclic_rshift_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(cyclic_rshift_bits, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::CyclicShiftRight, pairwise::CyclicShiftRight, broadcast::CyclicShiftRight),
+      y, z, false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(cyclic_rshift_bits) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/cyclic_shift.cpp

@@ -0,0 +1,52 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_cyclic_shift_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(cyclic_shift_bits, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(
+      BroadcastIntOpsTuple::custom(scalar::CyclicShiftLeft, pairwise::CyclicShiftLeft, broadcast::CyclicShiftLeft), y,
+      z, false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(cyclic_shift_bits) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/rshift.cpp

@@ -0,0 +1,51 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_rshift_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(rshift_bits, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(BroadcastIntOpsTuple::custom(scalar::ShiftRight, pairwise::ShiftRight, broadcast::ShiftRight),
+                        y, z, false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(rshift_bits) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/shift.cpp

@@ -0,0 +1,51 @@
+/*
+ *  ******************************************************************************
+ *  *
+ *  *
+ *  * This program and the accompanying materials are made available under the
+ *  * terms of the Apache License, Version 2.0 which is available at
+ *  * https://www.apache.org/licenses/LICENSE-2.0.
+ *  *
+ *  * See the NOTICE file distributed with this work for additional
+ *  * information regarding copyright ownership.
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *  * License for the specific language governing permissions and limitations
+ *  * under the License.
+ *  *
+ *  * SPDX-License-Identifier: Apache-2.0
+ *  *****************************************************************************
+ */
+
+//
+// @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_shift_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/shift.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(shift_bits, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(BroadcastIntOpsTuple::custom(scalar::ShiftLeft, pairwise::ShiftLeft, broadcast::ShiftLeft), y,
+                        z, false);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(shift_bits) { getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setSameMode(true); }
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/bitwise/toggle_bits.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 23.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_toggle_bits)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/helpers.h>
+#include <ops/declarable/helpers/toggle_bits.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(toggle_bits, -1, -1, true) {
+  for (size_t i = 0; i < block.width(); i++) {
+    auto x = INPUT_VARIABLE(i);
+    auto z = OUTPUT_VARIABLE(i);
+
+    REQUIRE_TRUE(x->dataType() == z->dataType(), 0, "Toggle bits requires input and output to have same type");
+    REQUIRE_TRUE(x->isZ(), 0, "Toggle bits requires input and output to be integer type (int8, int16, int32, int64)");
+
+    helpers::__toggle_bits(block.launchContext(), x, z);
+  }
+  return Status::OK;
+}
+
+DECLARE_TYPES(toggle_bits) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_INTS})->setAllowedOutputTypes({ALL_INTS})->setSameMode(false);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/blas/axpy.cpp

@@ -0,0 +1,64 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_axpy)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CONFIGURABLE_OP_IMPL(axpy, 2, 1, false, -2, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(x->isSameShape(y), 0, "Axpy: both arguments should have the same shape");
+  REQUIRE_TRUE(x->dataType() == y->dataType() && x->dataType() == z->dataType(), 0,
+               "Axpy: all arguments must have the same data type");
+
+  double a = 1.0;
+
+  if (block.width() > 2) {
+    auto alpha = INPUT_VARIABLE(2);
+    REQUIRE_TRUE(alpha->isScalar(), 0, "Axpy: alpha argument should be scalar or TArg");
+  } else if (block.getTArguments()->size() > 0) {
+    a = T_ARG(0);
+  }
+
+  ExtraArguments arguments({a});
+
+  y->applyPairwiseTransform(pairwise::Axpy, x, z, &arguments);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(axpy) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/blas/batched_gemm.cpp

@@ -0,0 +1,355 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//   @author Adam Gibson
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_batched_gemm)
+
+#include <ops/declarable/headers/blas.h>
+#include <ops/declarable/helpers/batched_gemm.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(batched_gemm, -1, -1, false, 0, 2) {
+  // Only require 2 IArgs: transposeA, transposeB
+  // Everything else will be inferred from the input matrices
+  int transA = INT_ARG(0);
+  int transB = INT_ARG(1);
+  
+  // Get alpha and beta
+  auto alpha = INPUT_VARIABLE(0);
+  auto beta = INPUT_VARIABLE(1);
+  
+  // Calculate batch size from number of inputs
+  // Total inputs = alpha + beta + batchSize*A + batchSize*B
+  // So batchSize = (total - 2) / 2
+  int batchSize = (block.width() - 2) / 2;
+  
+  REQUIRE_TRUE(batchSize > 0, 0, "BatchedGemm: Invalid batch size calculated: %d", batchSize);
+  REQUIRE_TRUE((block.width() - 2) % 2 == 0, 0, "BatchedGemm: Number of matrix inputs must be even");
+  
+  // Get first matrices to infer dimensions
+  auto firstA = INPUT_VARIABLE(2);
+  auto firstB = INPUT_VARIABLE(2 + batchSize);
+  
+  REQUIRE_TRUE(firstA->rankOf() == 2, 0, "BatchedGemm: A matrices must be rank 2");
+  REQUIRE_TRUE(firstB->rankOf() == 2, 0, "BatchedGemm: B matrices must be rank 2");
+  
+  // Infer dimensions from first matrices
+  int M = transA ? firstA->sizeAt(1) : firstA->sizeAt(0);
+  int K = transA ? firstA->sizeAt(0) : firstA->sizeAt(1);
+  int N = transB ? firstB->sizeAt(0) : firstB->sizeAt(1);
+  int K_B = transB ? firstB->sizeAt(1) : firstB->sizeAt(0);
+  
+  REQUIRE_TRUE(K == K_B, 0, "BatchedGemm: Incompatible dimensions - K from A is %d, K from B is %d", K, K_B);
+  
+  // Infer leading dimensions
+  int ldA = firstA->sizeAt(0);
+  int ldB = firstB->sizeAt(0);
+  int ldC = M;
+  
+  // Validate leading dimensions
+  if(transA == 0) {
+    int ldaComp = M > 1 ? M : 1;
+    if(ldA < ldaComp) THROW_EXCEPTION("LDA must be >= max(1,m) when transa == false");
+  } else {
+    int ldaComp = K > 1 ? K : 1;
+    if(ldA < ldaComp)
+      THROW_EXCEPTION("LDA must be >= max(1,k) when transa == true");
+  }
+
+  if(transB == 0) {
+    int ldBComp = K > 1 ? K : 1;
+    if(ldB < ldBComp) {
+      THROW_EXCEPTION("LDB must be >= max(1,k) when transb == false");
+    }
+  } else {
+    int ldbComp = N > 1 ? N : 1;
+    if(ldB < ldbComp)
+      THROW_EXCEPTION("LDB must be >= max(1,N) when transb == true");
+  }
+
+  int ldcComp = M > 1 ? M : 1;
+  if(ldC < ldcComp) {
+    THROW_EXCEPTION("LDC must be >= max(1,M)");
+  }
+
+  // Convert transpose flags to BLAS format
+  int transABlas = transA ? 112 : 111;  // 112 = CblasTrans, 111 = CblasNoTrans
+  int transBBlas = transB ? 112 : 111;
+
+  REQUIRE_TRUE((transA == 0 || transA == 1) && (transB == 0 || transB == 1), 0,
+               "BatchedGemm: valid values for transA and transB are: 0/1 for NoTrans/Trans respectively")
+  REQUIRE_TRUE(M > 0 && N > 0 && K > 0 && ldA > 0 && ldB > 0 && ldC > 0, 0, 
+               "BatchedGemm: Invalid dimensions M=%d, N=%d, K=%d, ldA=%d, ldB=%d, ldC=%d", M, N, K, ldA, ldB, ldC);
+
+  // Handle alpha and beta
+  NDArray *alphaInput = nullptr;
+  if(alpha->isScalar()) {
+    std::vector<sd::LongType> shape = {batchSize};
+    alphaInput = new NDArray('c',shape,alpha->dataType());
+    alphaInput->assign(alpha);
+  } else {
+    alphaInput = alpha;
+  }
+
+  NDArray *betaInput = nullptr;
+  if(beta->isScalar()) {
+    std::vector<LongType> shape = {batchSize};
+    betaInput = new NDArray('c',shape,beta->dataType());
+    betaInput->assign(beta);
+  } else {
+    betaInput = beta;
+  }
+
+  std::vector<NDArray*> vA(batchSize);
+  std::vector<NDArray*> vB(batchSize);
+  std::vector<NDArray*> vC(batchSize);
+
+  // Check data types - matrices should all match, alpha/beta can be different
+  auto firstMatrixType = firstA->dataType();
+  for (int e = 0; e < batchSize; e++) {
+    vA[e] = INPUT_VARIABLE(e + 2);
+    vB[e] = INPUT_VARIABLE(e + 2 + batchSize);
+    vC[e] = OUTPUT_VARIABLE(e);
+
+    REQUIRE_TRUE(firstMatrixType == vA[e]->dataType(), 0, 
+                 "BatchedGemm: all A matrices must have same data type");
+    REQUIRE_TRUE(firstMatrixType == vB[e]->dataType(), 0, 
+                 "BatchedGemm: all B matrices must have same data type");
+    REQUIRE_TRUE(firstMatrixType == vC[e]->dataType(), 0, 
+                 "BatchedGemm: all output matrices must have same data type as input matrices");
+    
+    REQUIRE_TRUE(vA[e]->rankOf() == 2, 0, "BatchedGemm: batch %i, rank of A should be equal to 2", e);
+    REQUIRE_TRUE(vB[e]->rankOf() == 2, 0, "BatchedGemm: batch %i, rank of B should be equal to 2", e);
+    REQUIRE_TRUE(vC[e]->rankOf() == 2, 0, "BatchedGemm: batch %i, rank of C should be equal to 2", e);
+
+    // Verify dimensions are consistent across batch
+    int currM = transABlas == 111 ? vA[e]->sizeAt(0) : vA[e]->sizeAt(1);
+    int currK_A = transABlas == 111 ? vA[e]->sizeAt(1) : vA[e]->sizeAt(0);
+    int currK_B = transBBlas == 111 ? vB[e]->sizeAt(0) : vB[e]->sizeAt(1);
+    int currN = transBBlas == 111 ? vB[e]->sizeAt(1) : vB[e]->sizeAt(0);
+    
+    REQUIRE_TRUE(currM == M, 0, "BatchedGemm: batch %i, inconsistent M dimension: expected %d, got %d", e, M, currM);
+    REQUIRE_TRUE(currK_A == K, 0, "BatchedGemm: batch %i, inconsistent K dimension in A: expected %d, got %d", e, K, currK_A);
+    REQUIRE_TRUE(currK_B == K, 0, "BatchedGemm: batch %i, inconsistent K dimension in B: expected %d, got %d", e, K, currK_B);
+    REQUIRE_TRUE(currN == N, 0, "BatchedGemm: batch %i, inconsistent N dimension: expected %d, got %d", e, N, currN);
+  }
+
+  helpers::bgemm(vA, vB, vC, alphaInput, betaInput, transABlas, transBBlas, M, N, K, ldA, ldB, ldC);
+
+  if(alphaInput != alpha) {
+    delete alphaInput;
+  }
+
+  if(betaInput != beta) {
+    delete betaInput;
+  }
+
+  return Status::OK;
+};
+
+DECLARE_SHAPE_FN(batched_gemm) {
+  // Only require 2 IArgs: transposeA, transposeB
+  int transA = INT_ARG(0);
+  int transB = INT_ARG(1);
+  
+  // Calculate batch size from inputs
+  int batchSize = (block.width() - 2) / 2;
+  
+  if (batchSize <= 0) {
+    auto shapeList = SHAPELIST();
+    shapeList->push_back(
+        ConstantShapeHelper::getInstance().createShapeInfo(ArrayOptions::dataType(inputShape->at(0)), 'c', {1, 1}));
+    return shapeList;
+  }
+  
+  // Get dimensions from first matrices
+  auto firstA = inputShape->at(2);
+  auto firstB = inputShape->at(2 + batchSize);
+  
+  int M = transA ? shape::sizeAt(firstA, 1) : shape::sizeAt(firstA, 0);
+  int N = transB ? shape::sizeAt(firstB, 0) : shape::sizeAt(firstB, 1);
+  
+  // Get data type from first matrix, not from alpha/beta
+  auto firstMatrixType = ArrayOptions::dataType(firstA);
+  
+  // Check that all matrices have the same type (skip alpha and beta)
+  for (int e = 2; e < block.width(); e++) {
+    REQUIRE_TRUE(firstMatrixType == ArrayOptions::dataType(inputShape->at(e)), 0,
+                 "BatchedGemm: all matrices must have same data type");
+  }
+
+  auto shapeList = SHAPELIST();
+  std::vector<LongType> shape({M, N});
+
+  for (int e = 0; e < batchSize; e++) {
+    auto newShape =
+        ConstantShapeHelper::getInstance().createShapeInfo(firstMatrixType, 'f', shape);
+    shapeList->push_back(newShape);
+  }
+
+  return shapeList;
+}
+
+DECLARE_TYPES(batched_gemm) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+
+
+CUSTOM_OP_IMPL(batched_gemm_bp, -1, -1, false, 0, 2) {
+  // Only require 2 IArgs: transposeA, transposeB
+  int transA = INT_ARG(0);
+  int transB = INT_ARG(1);
+  
+  // Calculate batch size
+  // Inputs: alpha, beta, batchSize*A, batchSize*B, batchSize*dLdC
+  // So batchSize = (total - 2) / 3
+  int batchSize = (block.width() - 2) / 3;
+  
+  REQUIRE_TRUE(batchSize > 0, 0, "BatchedGemmBp: Invalid batch size calculated: %d", batchSize);
+  
+  // Get dimensions from first matrices
+  auto firstA = INPUT_VARIABLE(2);
+  auto firstB = INPUT_VARIABLE(2 + batchSize);
+  auto firstDlDOut = INPUT_VARIABLE(2 + batchSize * 2);
+  
+  int M = transA ? firstA->sizeAt(1) : firstA->sizeAt(0);
+  int K = transA ? firstA->sizeAt(0) : firstA->sizeAt(1);
+  int N = transB ? firstB->sizeAt(0) : firstB->sizeAt(1);
+  
+  // Infer leading dimensions
+  int ldA = firstA->sizeAt(0);
+  int ldB = firstB->sizeAt(0);
+  int ldC = M;
+
+  std::vector<NDArray *> matricesA;
+  std::vector<NDArray *> matricesB;
+  std::vector<NDArray *> dlDOut;
+  std::vector<NDArray *> dldXOutputs;
+  std::vector<NDArray *> dldYOutputs;
+
+  for (int e = 0; e < batchSize; e++) {
+    matricesA.push_back(INPUT_VARIABLE(e + 2));
+    matricesB.push_back(INPUT_VARIABLE(e + 2 + batchSize));
+    dlDOut.push_back(INPUT_VARIABLE(e + 2 + batchSize * 2));
+    //alphas and betas are also set for outputs even though they're zero,every input needs a gradient
+    dldXOutputs.push_back(OUTPUT_VARIABLE(e + 2));
+    dldYOutputs.push_back(OUTPUT_VARIABLE(e + 2 + batchSize));
+  }
+
+  auto alpha = INPUT_VARIABLE(0);
+  NDArray *alphaInput = nullptr;
+  if(alpha->lengthOf() != batchSize) {
+    std::vector<sd::LongType> shape = {batchSize};
+    alphaInput = new NDArray('c',shape,alpha->dataType());
+    alphaInput->assign(alpha);
+  } else {
+    alphaInput = alpha;
+  }
+
+  auto beta = INPUT_VARIABLE(1);
+  NDArray *betaInput = nullptr;
+  if(beta->lengthOf() != batchSize) {
+    std::vector<sd::LongType> shape = {batchSize};
+    betaInput = new NDArray('c',shape,beta->dataType());
+    betaInput->assign(beta);
+  } else {
+    betaInput = beta;
+  }
+
+  // Convert transpose flags to BLAS format for helper function
+  int transABlas = transA ? 112 : 111;
+  int transBBlas = transB ? 112 : 111;
+
+  // First gradient computation: dL/dA = dL/dC @ B^T (or B if transB)
+  int transA1 = 0;
+  int transB1 = transB;
+  int M1 = dlDOut[0]->sizeAt(0);
+  int N1 = transB ? matricesB[0]->sizeAt(0) : matricesB[0]->sizeAt(1);
+  int k1 = dlDOut[0]->sizeAt(1);
+  int lda1 = dlDOut[0]->sizeAt(0);
+  int ldb1 = matricesB[0]->sizeAt(0);
+  int ldc1 = dldXOutputs[0]->sizeAt(0);
+  
+  helpers::bgemm(dlDOut, matricesB, dldXOutputs, alphaInput, betaInput, 
+                 transA1 ? 112 : 111, transB1 ? 112 : 111, M1, N1, k1, lda1, ldb1, ldc1);
+
+  // Second gradient computation: dL/dB = A^T @ dL/dC (or A if transA)
+  int transA2 = transA ? 0 : 1;
+  int transB2 = 0;
+  int M2 = transA ? matricesA[0]->sizeAt(1) : matricesA[0]->sizeAt(0);
+  int N2 = dlDOut[0]->sizeAt(1);
+  int k2 = transA ? matricesA[0]->sizeAt(0) : matricesA[0]->sizeAt(1);
+  int lda2 = matricesA[0]->sizeAt(0);
+  int ldb2 = dlDOut[0]->sizeAt(0);
+  int ldc2 = dldYOutputs[0]->sizeAt(0);
+  
+  helpers::bgemm(matricesA, dlDOut, dldYOutputs, alphaInput, betaInput, 
+                 transA2 ? 112 : 111, transB2 ? 112 : 111, M2, N2, k2, lda2, ldb2, ldc2);
+
+  if(alphaInput != alpha) {
+    delete alphaInput;
+  }
+
+  if(betaInput != beta) {
+    delete betaInput;
+  }
+
+  return Status::OK;
+};
+
+DECLARE_SHAPE_FN(batched_gemm_bp) {
+  // Calculate batch size
+  int batchSize = (block.width() - 2) / 3;
+  
+  auto xConstant = CONSTANT(inputShape->at(2));
+  auto yConstant = CONSTANT(inputShape->at(2 + batchSize));
+  auto ret = SHAPELIST();
+  //alpha
+  ret->push_back(xConstant);
+  //beta
+  ret->push_back(yConstant);
+  for(int i = 0; i < batchSize; i++) {
+    ret->push_back(xConstant);
+  }
+
+  for(int i = 0; i < batchSize; i++) {
+    ret->push_back(yConstant);
+  }
+  return ret;
+}
+
+DECLARE_TYPES(batched_gemm_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes({ALL_FLOATS})
+      ->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/blas/matmul.cpp

@@ -0,0 +1,352 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com, created on 07.10.2017.
+// @author GS <sgazeos@gmail.com>, modified
+// @author Yurii Shyrma (iuriish@yahoo.com), fully rewritten
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_matmul)
+
+#include <helpers/MmulHelper.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+//////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(matmul, 2, 1, false, 0, -2) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+  if(x->isEmpty() || y->isEmpty())
+    return Status::OK;
+  int iSize = (int)block.getIArguments()->size();
+  int transX = iSize > 0 ? INT_ARG(0) : 0;
+  int transY = iSize > 1 ? INT_ARG(1) : 0;
+  const int transZ = iSize > 2 ? INT_ARG(2) : 0;
+  // optional use alpha nad beta
+  iSize = (int)block.getTArguments()->size();
+  double alpha = iSize > 0 ? T_ARG(0) : 1.0;
+  double beta = iSize > 1 ? T_ARG(1) : 0.0;
+
+  if (transZ) {
+    x = INPUT_VARIABLE(1);
+    y = INPUT_VARIABLE(0);
+    bool temp = transX;
+    transX = !transY;
+    transY = !temp;
+  }
+
+  // Compute ranks AFTER potential transZ swap
+  const int xRank = x->rankOf();
+  const int yRank = y->rankOf();
+  const int zRank = z->rankOf();
+
+  const int xLastDim = transX ? -2 : -1;
+  const int yLastDim = transY ? -2 : -1;
+  const int xLastButOneDim = transX ? -1 : -2;
+  const int yLastButOneDim = transY ? -1 : -2;
+
+  // ******* input validation ******* //
+  REQUIRE_TRUE(xRank > 0 && yRank > 0, 0,
+               "MATMUL OP: input arrays must have rank bigger than 0 (should not be scalars), but got instead: x rank "
+               "= %i, y rank = %i !",
+               xRank, yRank);
+
+  // Handle rank mismatch when one input has singleton leading dimensions
+  // This supports ONNX Gemm patterns like [1,1,1,768] x [768,768] -> [1,1,1,768]
+  NDArray* xReshaped = nullptr;
+  NDArray* yReshaped = nullptr;
+  NDArray* zReshaped = nullptr;
+
+  if (xRank != yRank && xRank > 2 && yRank == 2) {
+    // Check if x has all singleton leading dims
+    bool allLeadingSingleton = true;
+    for (int i = 0; i < xRank - 2; ++i) {
+      if (x->sizeAt(i) != 1) {
+        allLeadingSingleton = false;
+        break;
+      }
+    }
+    if (allLeadingSingleton) {
+      // Reshape x from [1,1,...,M,K] to [M,K] for matmul
+      std::vector<LongType> newXShape = {x->sizeAt(-2), x->sizeAt(-1)};
+      xReshaped = new NDArray(x->reshape(x->ordering(), newXShape));
+      // Reshape z from [1,1,...,M,N] to [M,N]
+      std::vector<LongType> newZShape = {z->sizeAt(-2), z->sizeAt(-1)};
+      zReshaped = new NDArray(z->reshape(z->ordering(), newZShape));
+      x = xReshaped;
+      z = zReshaped;
+    }
+  } else if (xRank != yRank && yRank > 2 && xRank == 2) {
+    // Check if y has all singleton leading dims
+    bool allLeadingSingleton = true;
+    for (int i = 0; i < yRank - 2; ++i) {
+      if (y->sizeAt(i) != 1) {
+        allLeadingSingleton = false;
+        break;
+      }
+    }
+    if (allLeadingSingleton) {
+      // Reshape y from [1,1,...,K,N] to [K,N] for matmul
+      std::vector<LongType> newYShape = {y->sizeAt(-2), y->sizeAt(-1)};
+      yReshaped = new NDArray(y->reshape(y->ordering(), newYShape));
+      // Reshape z from [1,1,...,M,N] to [M,N]
+      std::vector<LongType> newZShape = {z->sizeAt(-2), z->sizeAt(-1)};
+      zReshaped = new NDArray(z->reshape(z->ordering(), newZShape));
+      y = yReshaped;
+      z = zReshaped;
+    }
+  }
+
+  // Update ranks after potential reshaping
+  const int xRankFinal = x->rankOf();
+  const int yRankFinal = y->rankOf();
+  const int zRankFinal = z->rankOf();
+
+  if (xRankFinal == 1 && yRankFinal == 1) {  // dot case, output is scalar (or vector with length = 1)
+    REQUIRE_TRUE(x->lengthOf() == y->lengthOf(), 0,
+                 "MATMUL OP: since input arrays are vectors they must have the same length, but got x length = %i, y "
+                 "length = %i !",
+                 x->lengthOf(), y->lengthOf());
+  } else if (xRankFinal == 1 && yRankFinal == 2) {  // vector x matrix, i.e. [4] x [4,5] = [5], output is vector
+    REQUIRE_TRUE(x->lengthOf() == y->sizeAt(yLastButOneDim), 0,
+                 "MATMUL OP: input arrays have inconsistent shapes for vector-matrix product: x %s, y %s !",
+                 ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+  } else if (xRankFinal == 2 && yRankFinal == 1) {  // matrix x vector , i.e. [4,5] x [5] = [4], output is vector
+    REQUIRE_TRUE(x->sizeAt(xLastDim) == y->lengthOf(), 0,
+                 "MATMUL OP: input arrays have inconsistent shapes for matrix-vector product: x %s, y %s !",
+                 ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+  } else {
+    REQUIRE_TRUE(xRankFinal == yRankFinal && yRankFinal == zRankFinal, 0,
+                 "MATMUL OP: input and output arrays must have the same rank, but got instead: x rank = %i, y rank = "
+                 "%i, z rank = %i !",
+                 xRankFinal, yRankFinal, zRankFinal);
+    REQUIRE_TRUE(x->sizeAt(xLastDim) == y->sizeAt(yLastButOneDim) && x->sizeAt(xLastButOneDim) == z->sizeAt(-2) &&
+                 y->sizeAt(yLastDim) == z->sizeAt(-1),
+                 0, "MATMUL OP: input/output arrays have inconsistent shapes for matrix product: x %s, y %s, z %s !",
+                 ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str(),
+                 ShapeUtils::shapeAsString(z).c_str());
+
+    if (xRankFinal > 2)  // outer dims must be the same
+      for (int i = 0; i < xRankFinal - 2; ++i)
+    REQUIRE_TRUE(x->sizeAt(i) == y->sizeAt(i) && y->sizeAt(i) == z->sizeAt(i), 0,
+                 "MATMUL OP: input/output arrays have inconsistent shapes for matrix product: x %s, y %s, z %s !",
+                 ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str(),
+                 ShapeUtils::shapeAsString(z).c_str());
+  }
+  // ******* end of input validation ******* //
+
+  MmulHelper::matmul(x, y, z, transX, transY, alpha, beta, z);
+
+  // Clean up reshaped arrays
+  delete xReshaped;
+  delete yReshaped;
+  delete zReshaped;
+
+  return Status::OK;
+}
+
+DECLARE_SYN(mMul, matmul);
+
+DECLARE_SYN(mmul, matmul);
+
+DECLARE_SYN(gemm, matmul);
+
+DECLARE_SYN(gemv, matmul);
+
+DECLARE_SYN(dot, matmul);
+
+//////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(matmul) {
+  auto xShapeInfo = inputShape->at(0);
+  auto yShapeInfo = inputShape->at(1);
+
+
+  const int iSize = (int)block.getIArguments()->size();
+  int transX = iSize > 0 ? INT_ARG(0) : 0;
+  int transY = iSize > 1 ? INT_ARG(1) : 0;
+  const int transZ = iSize > 2 ? INT_ARG(2) : 0;
+
+  if (transZ) {
+    xShapeInfo = inputShape->at(1);
+    yShapeInfo = inputShape->at(0);
+    bool temp = transX;
+    transX = !transY;
+    transY = !temp;
+  }
+
+  auto zShapeOnly = ShapeUtils::evalShapeForMatmul(xShapeInfo, yShapeInfo, transX, transY);
+
+  auto dtypeX = ArrayOptions::dataType(xShapeInfo);
+  auto dtypeY = ArrayOptions::dataType(yShapeInfo);
+
+  auto xOrder = shape::order(xShapeInfo);
+  auto yOrder = shape::order(yShapeInfo);
+  auto zOrder = xOrder == 'c' && yOrder == 'c' ? 'c' : 'f';
+
+  // we just pick the higher data type out of X and Y
+  auto dtypeZ = dtypeX > dtypeY ? dtypeX : dtypeY;
+  if(shape::isEmptyConst(xShapeInfo) || shape::isEmptyConst(yShapeInfo)) {
+    return SHAPELIST(ConstantShapeHelper::getInstance().emptyShapeInfoWithShape(ArrayOptions::dataType(xShapeInfo),zShapeOnly));
+  }
+
+  auto newShape = ConstantShapeHelper::getInstance().createShapeInfo(dtypeZ, zOrder, zShapeOnly);
+  return SHAPELIST(newShape);
+}
+
+//////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(matmul) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS, ALL_INTS});
+}
+
+//////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(matmul_bp, 3, 2, false, 0, -2) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto eps = INPUT_VARIABLE(2);
+  auto dldx = OUTPUT_VARIABLE(0);
+  auto dldy = OUTPUT_VARIABLE(1);
+
+  int iSize = (int)block.getIArguments()->size();
+  int transX = iSize > 0 ? INT_ARG(0) : 0;
+  int transY = iSize > 1 ? INT_ARG(1) : 0;
+  const int transZ = iSize > 2 ? INT_ARG(2) : 0;
+
+  // optional use alpha nad beta
+  iSize = (int)block.getTArguments()->size();
+
+  double alpha = iSize > 0 ? T_ARG(0) : 1.0;
+  double beta = iSize > 1 ? T_ARG(1) : 0.0;
+
+  /*
+  In: x=[a,b], y=[b,c]
+  tX  tY  tZ  x       y       z       dz          dLdx                                    dLdy
+  F   F   F   [a,b]   [b,c]   [a,c]   [a,c]       [a,c]*[b,c]T = [a,b]        x*yT        [a,b]T*[a,c] = [b,c] xT*y T F
+  F   [b,a]   [b,c]   [a,c]   [a,c]       ([a,c]*[b,c]T)T = [b,a]     (x*yT)T     [b,a]*[a,c] = [b,c]         x*y F   T
+  F   [a,b]   [c,b]   [a,c]   [a,c]       ([a,c]*[c,b]) = [a,b]       x*y         [a,b]T*[a,c] = [b,c] ->T    xT*y T   T
+  F   [b,a]   [c,b]   [a,c]   [a,c]       ([a,c]*[c,b])T = [b,a]      (x*y)T      [b,a]*[a,c] = [b,c]  ->T    x*y F   F
+  T   [a,b]   [b,c]   [c,a]   [c,a]
+  */
+  // special case for scalar value
+  if (eps->isScalar()) {
+    if (x->isVector() && y->isVector()) {
+      if (x->isRowVector() && y->isRowVector()) {
+        float ySum = y->sumNumber().e<float>(0);
+        NDArray *dldxTemp = (*eps) * ySum;
+        dldx->assign(dldxTemp);
+        delete dldxTemp;
+
+        float xSum = x->sumNumber().e<float>(0);
+        NDArray *dldyTemp = (*eps) * xSum;
+        dldy->assign(dldyTemp);
+        delete dldyTemp;
+      } else if (x->isColumnVector() && y->isColumnVector()) {
+        float ySum = y->sumNumber().e<float>(0);
+        NDArray *dldxTemp = (*eps) * ySum;
+        dldx->assign(dldxTemp);
+        delete dldxTemp;
+        float xSum = x->sumNumber().e<float>(0);
+        NDArray *dldyTemp = (*eps) * xSum;
+        dldy->assign(dldyTemp);
+        delete dldyTemp;
+      } else {
+        NDArray *dldxTemp = (*eps) * (*y);
+        dldx->assign(dldxTemp);
+        delete dldxTemp;
+        NDArray *dldyTemp = (*eps) * (*x);
+        dldy->assign(dldyTemp);
+        delete dldyTemp;
+      }
+    } else {
+      // assign all ones to shape as baseline
+      auto alphaBetaBase = 1.0f;
+      if (alpha > 0.0f) {
+        alphaBetaBase *= alpha;
+      }
+
+      if (beta > 0.0f) {
+        alphaBetaBase += beta;
+      }
+
+      dldx->assign(alphaBetaBase);
+      dldy->assign(alphaBetaBase);
+      
+      // match the dimensions for reduction for matrix multiply: columns on first input, rows on second input
+      // the dimensions should match the matching dimensions to compute proper gradients wrt each input
+      // core gradient for each is sum(input) * eps as scalar
+      std::vector<LongType> axesZero({0});
+      NDArray *xSum = x->reduceAlongDimension(reduce::Sum, &axesZero);
+      NDArray *xSumScaled = *xSum * (*eps);
+      std::vector<sd::LongType> xSumShape = {xSumScaled->lengthOf(), 1};
+      NDArray* xSumRow = xSumScaled->reshape(xSumScaled->ordering(), xSumShape);
+      
+      std::vector<LongType> axes({1});
+      NDArray *ySum = y->reduceAlongDimension(reduce::Sum, &axes);
+      NDArray *ySumScaled = *ySum * (*eps);
+      std::vector<sd::LongType> ySumShape = {1, ySumScaled->lengthOf()};
+      NDArray* ySumRow = ySumScaled->reshape(ySumScaled->ordering(), ySumShape);
+
+      // execute proper multiplication: rows for first input, columns for second
+      dldx->mulRowVector(ySumRow, dldx);
+      dldy->muliColumnVector(xSumRow);
+
+      // FIXED: Proper cleanup - delete each allocated array once, add missing cleanup
+      delete xSumRow;
+      delete xSumScaled;
+      delete xSum;
+      delete ySumRow;
+      delete ySumScaled;
+      delete ySum;
+    }
+
+    return Status::OK;
+  }
+
+  matmul op;
+  op.execute({eps, y}, {dldx}, {alpha, beta}, {transZ, !transY, transX}, {});
+  op.execute({x, eps}, {dldy}, {alpha, beta}, {!transX, transZ, transY}, {});
+
+  return Status::OK;
+}
+
+//////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(matmul_bp) {
+  return SHAPELIST(CONSTANT(inputShape->at(0)), CONSTANT(inputShape->at(1)));
+}
+
+//////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(matmul_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedInputTypes(2, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/blas/tensormmul.cpp

@@ -0,0 +1,268 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_tensormmul)
+
+#include <helpers/MmulHelper.h>
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+#include <numeric>
+
+namespace sd {
+namespace ops {
+
+////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(tensormmul, 2, 1, false, 0, -1) {
+  auto a = INPUT_VARIABLE(0);
+  auto b = INPUT_VARIABLE(1);
+
+  auto c = OUTPUT_VARIABLE(0);
+
+  REQUIRE_TRUE(a->dataType() == b->dataType(), 0, "tensormmul: A, B and C data types must be the same");
+
+  // building axes
+  LongType axe0_size = INT_ARG(0);
+  LongType axe1_size = INT_ARG(axe0_size + 1);
+  std::vector<LongType> axes_0(axe0_size), axes_1(axe1_size);
+  for (LongType e = 0; e < axe0_size; e++) axes_0[e] = INT_ARG(e + 1);
+  for (LongType e = 0; e < axe1_size; e++) axes_1[e] = INT_ARG(e + axe0_size + 2);
+
+
+  std::vector<sd::LongType> permuteC = {};
+  MmulHelper::tensorDot(a, b, c, axes_0, axes_1,permuteC);
+  return Status::OK;
+}
+DECLARE_SYN(tensordot, tensormmul);
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(tensormmul) {
+  auto aShapeInfo = inputShape->at(0);
+  auto bShapeInfo = inputShape->at(1);
+
+  REQUIRE_TRUE(ArrayOptions::dataType(aShapeInfo) == ArrayOptions::dataType(bShapeInfo), 0,
+               "tensormmul: A and B data types must be the same");
+
+  // building axes
+  LongType axe0_size = INT_ARG(0);
+  LongType axe1_size = INT_ARG(axe0_size + 1);
+  std::vector<LongType> axes_0(axe0_size), axes_1(axe1_size);
+  for (LongType e = 0; e < axe0_size; e++) axes_0[e] = INT_ARG(e + 1);
+
+  for (LongType e = 0; e < axe1_size; e++) axes_1[e] = INT_ARG(e + axe0_size + 2);
+
+  sd_verbose("axe0: %i; axe1: %i;\n", axes_0.size(), axes_1.size());
+  // evaluate shapes
+  std::vector<LongType> permutAt, permutBt;
+  std::vector<LongType> shapeAt, shapeBt;
+  auto outShape =
+      ShapeUtils::evalShapeForTensorDot(aShapeInfo, bShapeInfo, axes_0, axes_1, permutAt, permutBt,
+                                                        shapeAt, shapeBt);
+
+  auto desc = new  ShapeDescriptor(ArrayOptions::dataType(aShapeInfo), 'c', outShape);
+  auto result = SHAPELIST(ConstantShapeHelper::getInstance().createShapeInfo(desc));
+  delete desc;
+  return result;
+}
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(tensormmul) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedInputTypes(2, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF});
+}
+
+// Comparator for sorting indices vector based on comparison of array values
+struct IndexComparator
+{
+  const std::vector<LongType>& array;
+
+  IndexComparator(const std::vector<LongType>& arr): array(arr) {}
+
+  bool operator() (LongType i1, LongType i2)
+  {
+    return array[i1] < array[i2];
+  }
+};
+
+
+std::vector<LongType> argsort(const std::vector<LongType>& array)
+{
+  std::vector<LongType> indices(array.size());
+  for (size_t i = 0; i < array.size(); ++i) indices[i] = i;
+
+  std::sort(indices.begin(), indices.end(), IndexComparator(array));
+
+  return indices;
+}
+
+////////////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(tensormmul_bp, 4, 2, false, 0, -1) {
+  auto A = INPUT_VARIABLE(0);
+  auto B = INPUT_VARIABLE(1);
+  auto C = INPUT_VARIABLE(2);
+  auto dC = INPUT_VARIABLE(3);
+  auto originalDC = dC;
+
+  //scalar case, tile value to be whatever the c value is. common when directly attached to the loss
+  if(dC->isScalar()) {
+    auto newVec = const_cast<NDArray *>(C);
+    auto* newShapeVec = newVec->getShapeAsVector();
+    dC = new NDArray('c', *newShapeVec, dC->dataType(), dC->getContext());
+    delete newShapeVec;
+  }
+
+
+  auto gradA = OUTPUT_VARIABLE(0);
+  auto gradB = OUTPUT_VARIABLE(1);
+
+  LongType axe0_size = INT_ARG(0);
+  LongType axe1_size = INT_ARG(axe0_size + 1);
+  std::vector<LongType> axes0Sum(axe0_size), axes1Sum(axe1_size);
+
+  //find the passed in axes for the feed forward
+  for (LongType e = 0; e < axe0_size; e++) axes0Sum[e] = INT_ARG(e + 1);
+  for (LongType e = 0; e < axe1_size; e++) axes1Sum[e] = INT_ARG(e + axe0_size + 2);
+
+
+  auto Arank = A->rankOf();
+  auto Brank = B->rankOf();
+  auto dCrank = dC->rankOf();
+
+
+  //part of the permtue axes before matrix multiply happens
+  std::vector<LongType> axes_a_grad;
+  for (LongType i = 0; i < Arank; ++i)
+    axes_a_grad.push_back(i);
+
+  for (size_t i = 0; i < axes0Sum.size(); ++i)
+    axes_a_grad.erase(std::remove(axes_a_grad.begin(), axes_a_grad.end(), axes0Sum[i]), axes_a_grad.end());
+
+
+
+  //part of matrix multiply axes before matrix multiply happens
+  std::vector<LongType> axes_b_grad;
+  for (LongType i = 0; i < Brank; ++i)
+    axes_b_grad.push_back(i);
+
+  for (size_t i = 0; i < axes1Sum.size(); ++i)
+    axes_b_grad.erase(std::remove(axes_b_grad.begin(), axes_b_grad.end(), axes1Sum[i]), axes_b_grad.end());
+
+  //used for post result permute to reshape result to be expected output
+  std::vector<LongType> grad_a_axes;
+  grad_a_axes.insert(grad_a_axes.end(), axes_a_grad.begin(), axes_a_grad.end());
+  grad_a_axes.insert(grad_a_axes.end(), axes1Sum.begin(), axes1Sum.end());
+
+  //used for post result permute to reshape result to be expected output
+  std::vector<LongType> grad_b_axes;
+  grad_b_axes.insert(grad_b_axes.end(), axes0Sum.begin(), axes0Sum.end());
+  grad_b_axes.insert(grad_b_axes.end(), axes_b_grad.begin(), axes_b_grad.end());
+
+  LongType starting = dCrank - axes_a_grad.size();
+  std::vector<LongType> axes_a_gradA;
+  for (LongType i = starting; i < dCrank; i++) {
+    axes_a_gradA.push_back(i);
+  }
+
+  std::vector<LongType> axes_b_gradA;
+  for (size_t i = 0; i < axes_b_grad.size(); i++) {
+    axes_b_gradA.push_back(i);
+  }
+
+  std::vector<LongType> axes_a_gradB;
+  for (size_t i = 0; i < axes_a_grad.size(); i++) {
+    axes_a_gradB.push_back(i);
+  }
+
+  LongType start = dCrank - axes_a_gradA.size();
+  std::vector<LongType> axes_b_gradB;
+  for (LongType i = start; i < dCrank; i++) {
+    axes_b_gradB.push_back(i);
+  }
+
+  //create final axes before for matrix multiply
+  std::vector<LongType> aPermuteAxesBefore;
+  aPermuteAxesBefore.insert(aPermuteAxesBefore.end(), axes_a_grad.begin(), axes_a_grad.end());
+  aPermuteAxesBefore.insert(aPermuteAxesBefore.end(), axes0Sum.begin(), axes0Sum.end());
+
+
+
+  //create final axes before for matrix multiply
+  std::vector<LongType> bPermuteAxesBefore;
+  bPermuteAxesBefore.insert(bPermuteAxesBefore.end(), axes_b_grad.begin(), axes_b_grad.end());
+  bPermuteAxesBefore.insert(bPermuteAxesBefore.end(), axes1Sum.begin(), axes1Sum.end());
+
+  auto aPermArgsAfter = argsort(grad_a_axes);
+  auto bPermArgsAfter = argsort(grad_b_axes);
+  auto newA = A->permute(aPermuteAxesBefore, false, false);
+  std::vector<LongType> empty;
+  auto newB = B->permute(bPermuteAxesBefore, false, false);
+
+
+  //perform the actual matrix multiplication
+  MmulHelper::tensorDot2(dC, newB, gradA, axes_a_gradA, axes_b_gradA, empty, empty, aPermArgsAfter, gradA);
+  MmulHelper::tensorDot2(newA, dC, gradB, axes_a_gradB, axes_b_gradB, empty, empty, bPermArgsAfter, gradB);
+
+  // FIXED: permute() with copyToNewBuff=false returns view - only delete if not view
+  if (newA != nullptr && !newA->isView()) {
+    delete newA;
+  }
+  if (newB != nullptr && !newB->isView()) {
+    delete newB;
+  }
+  if(dC != originalDC) {
+    delete dC;
+  }
+
+  return Status::OK;
+}
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_SHAPE_FN(tensormmul_bp) {
+  auto aShapeInfo = inputShape->at(0);
+  auto bShapeInfo = inputShape->at(1);
+  auto cShapeInfo = inputShape->at(2);
+  auto dLShapeInfo = inputShape->at(3);
+
+  REQUIRE_TRUE((ArrayOptions::dataType(aShapeInfo) == ArrayOptions::dataType(bShapeInfo) &&
+                (ArrayOptions::dataType(dLShapeInfo) == ArrayOptions::dataType(aShapeInfo))),
+               0, "tensormmul_bp: A, B and dLdC data types must be the same");
+
+  return SHAPELIST(CONSTANT(aShapeInfo), CONSTANT(bShapeInfo));
+}
+
+////////////////////////////////////////////////////////////////////////
+DECLARE_TYPES(tensormmul_bp) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {FLOAT32, DOUBLE, HALF})  // maybe better ALL_FLOATS
+      ->setAllowedInputTypes(1, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedInputTypes(2, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(0, {FLOAT32, DOUBLE, HALF})
+      ->setAllowedOutputTypes(1, {FLOAT32, DOUBLE, HALF});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/boolean_not.cpp

@@ -0,0 +1,45 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_not)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+OP_IMPL(boolean_not, 1, 1, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto z = OUTPUT_VARIABLE(0);
+
+  x->applyTransform(transform::Not, z);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_not) {
+  getOpDescriptor()->setAllowedInputTypes(0, BOOL)->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/choose.cpp

@@ -0,0 +1,87 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author Adam Gibson
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_choose)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/choose.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(choose, -1, 2, false, -2, -1) {
+  int mode = INT_ARG(0);
+  auto result = OUTPUT_VARIABLE(0);
+  auto numResults = OUTPUT_VARIABLE(1);
+
+  if (block.width() > 1) {
+    auto arg = INPUT_VARIABLE(0);
+    auto comp = INPUT_VARIABLE(1);
+
+    helpers::chooseFunctorArray(block.launchContext(), arg, comp, mode, result, numResults);
+
+  }  // scalar case
+  else {
+    double scalar = T_ARG(0);
+    auto arg = INPUT_VARIABLE(0);
+    helpers::chooseFunctorScalar(block.launchContext(), arg, scalar, mode, result, numResults);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(choose) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS})
+      ->setAllowedOutputTypes(1, {ALL_INTS});
+}
+
+DECLARE_SHAPE_FN(choose) {
+  int mode = INT_ARG(0);
+  auto numResults = NDArrayFactory::create<LongType>(0L);
+  if (block.width() > 1) {
+    auto first = INPUT_VARIABLE(0);
+    auto second = INPUT_VARIABLE(1);
+
+
+    helpers::chooseFunctorArray(block.launchContext(), first, second, mode, nullptr, numResults);
+  } else {
+    auto first = INPUT_VARIABLE(0);
+    double scalar = T_ARG(0);
+
+    helpers::chooseFunctorScalar(block.launchContext(), first, scalar, mode, nullptr, numResults);
+  }
+
+  auto newShape = ConstantShapeHelper::getInstance().vectorShapeInfo(numResults->e<LongType>(0),
+                                                                     ArrayOptions::dataType(inputShape->at(0)));
+
+  auto shapeScalar = ConstantShapeHelper::getInstance().scalarShapeInfo(INT64);
+  return SHAPELIST(newShape, shapeScalar);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/eq_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_eq_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(eq_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<double>(0) == y->e<double>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+DECLARE_SYN(Equals, eq_scalar);
+// DECLARE_SYN(equals, eq_scalar);
+
+DECLARE_TYPES(eq_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/gt_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_gt_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(gt_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) > y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+// DECLARE_SYN(Greater, gt_scalar);
+// DECLARE_SYN(greater, gt_scalar);
+
+DECLARE_TYPES(gt_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/gte_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_gte_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(gte_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) >= y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+DECLARE_SYN(GreaterOrEquals, gte_scalar);
+DECLARE_SYN(greaterOrEquals, gte_scalar);
+
+DECLARE_TYPES(gte_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/is_non_decreasing.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author @cpuheater
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_is_non_decreasing)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/compare_elem.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(is_non_decreasing, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+
+  // in case of empty input there's nothing to do
+  if (input->isEmpty()) return Status::EQ_TRUE;
+
+  bool isNonDecreasing = true;
+
+  helpers::compare_elem(block.launchContext(), input, false, isNonDecreasing);
+
+  if (isNonDecreasing)
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+
+DECLARE_TYPES(is_non_decreasing) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/is_numeric_tensor.cpp

@@ -0,0 +1,43 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author @cpuheater
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_is_numeric_tensor)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/compare_elem.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(is_numeric_tensor, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+
+  return input->isR() || input->isZ() ? Status::EQ_TRUE : Status::EQ_FALSE;
+}
+
+DECLARE_TYPES(is_numeric_tensor) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/is_strictly_increasing.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author @cpuheater
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_is_strictly_increasing)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/compare_elem.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(is_strictly_increasing, 1, true) {
+  auto input = INPUT_VARIABLE(0);
+
+  // in case of empty input there's nothing to do
+  if (input->isEmpty()) return Status::EQ_TRUE;
+
+  bool isStrictlyIncreasing = true;
+
+  helpers::compare_elem(block.launchContext(), input, true, isStrictlyIncreasing);
+
+  if (isStrictlyIncreasing)
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+
+DECLARE_TYPES(is_strictly_increasing) {
+  getOpDescriptor()->setAllowedInputTypes(0, ANY)->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/lt_scalar.cpp

@@ -0,0 +1,50 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_lt_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(lt_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) < y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+
+
+DECLARE_TYPES(lt_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/lte_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_lte_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(lte_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) <= y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+DECLARE_SYN(LessOrEquals, lte_scalar);
+DECLARE_SYN(lessorequals, lte_scalar);
+
+DECLARE_TYPES(lte_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/neq_scalar.cpp

@@ -0,0 +1,51 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 13.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_neq_scalar)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BOOLEAN_OP_IMPL(neq_scalar, 2, true) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+
+  if (x->e<float>(0) != y->e<float>(0))
+    return Status::EQ_TRUE;
+  else
+    return Status::EQ_FALSE;
+}
+DECLARE_SYN(NotEquals, neq_scalar);
+DECLARE_SYN(notequals, neq_scalar);
+
+DECLARE_TYPES(neq_scalar) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/select.cpp

@@ -0,0 +1,127 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_select)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(select, 3, 1, false, 0, 0) {
+  auto cond = INPUT_VARIABLE(0);
+  auto x = INPUT_VARIABLE(1);
+  auto y = INPUT_VARIABLE(2);
+  //TODO: for some reason y being empty
+  //should not necessarily yield an empty result
+  //the loss test I'm currently dealing with seems
+  //to need to output a value yet it ends up being empty.
+  //we need to figure out why
+  if(x->isEmpty() || y->isEmpty() || cond->isEmpty()) {
+    return Status::OK;
+  }
+  if (x->isScalar()) {
+    REQUIRE_TRUE(cond->isScalar(), 0,
+                 "Select: Condition should gave either equal shape to X/Y first dimension or to be scalar");
+
+    auto z = OUTPUT_VARIABLE(0);
+
+    if (y->isR()) {
+#ifdef HAS_DOUBLE
+      auto v = !cond->e<bool>(0) ? y->e<double>(0) : x->e<double>(0);
+#elif defined(HAS_FLOAT32)
+      auto v = !cond->e<bool>(0) ? y->e<float>(0) : x->e<float>(0);
+#else
+#error "No floating-point type available for select operation"
+#endif
+      z->p(0, v);
+    } else {
+      auto v = !cond->e<bool>(0) ? y->e<LongType>(0) : x->e<LongType>(0);
+      z->p(0, v);
+    }
+  } else {
+    bool same = cond->isSameShape(x);
+    REQUIRE_TRUE(cond->isScalar() || cond->lengthOf() == x->sizeAt(0) || same, 0,
+                 "Select: Condition should gave either equal shape to X/Y first dimension or to be scalar");
+    if (same) {
+      auto z = OUTPUT_VARIABLE(0);
+
+      for (int e = 0; e < cond->lengthOf(); e++) {
+        if (y->isR()) {
+#ifdef HAS_DOUBLE
+          auto r = !cond->e<bool>(e) ? y->e<double>(e) : x->e<double>(e);
+#elif defined(HAS_FLOAT32)
+          auto r = !cond->e<bool>(e) ? y->e<float>(e) : x->e<float>(e);
+#else
+#error "No floating-point type available for select operation"
+#endif
+          z->p(e, r);
+        } else {
+          auto r = !cond->e<bool>(e) ? y->e<LongType>(e) : x->e<LongType>(e);
+          z->p(e, r);
+        }
+      }
+    } else {
+      REQUIRE_TRUE(cond->lengthOf() == x->sizeAt(0), 0,
+                   "Condition length should be equal to the dim0 of x/y to act as TAD-mask, but got %d instead",
+                   cond->lengthOf());
+
+      auto z = OUTPUT_VARIABLE(0);
+      std::vector<LongType> idxs;
+      idxs.push_back(0);
+      auto dims = ShapeUtils::evalDimsToExclude(x->rankOf() ,1,idxs.data());
+      auto tadsX = x->allTensorsAlongDimension(*dims);
+      auto tadsY = y->allTensorsAlongDimension(*dims);
+      auto tadsZ = z->allTensorsAlongDimension(*dims);
+
+      for (int e = 0; e < tadsX.size(); e++) {
+        if (!cond->e<bool>(e)) {
+          tadsZ.at(e)->assign(tadsY.at(e));
+        } else {
+          tadsZ.at(e)->assign(tadsX.at(e));
+        }
+      }
+
+      delete dims;
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(select) {
+  auto inShape = inputShape->at(1);
+  return SHAPELIST(CONSTANT(inShape));
+}
+
+DECLARE_TYPES(select) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, BOOL)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedInputTypes(2, ANY)
+      ->setAllowedOutputTypes(1, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/where.cpp

@@ -0,0 +1,339 @@
+/* ******************************************************************************
+*
+*
+* This program and the accompanying materials are made available under the
+* terms of the Apache License, Version 2.0 which is available at
+* https://www.apache.org/licenses/LICENSE-2.0.
+*
+*  See the NOTICE file distributed with this work for additional
+*  information regarding copyright ownership.
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations
+* under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_Where)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/where.h>
+
+namespace sd {
+namespace ops {
+
+// Helper function to evaluate condition regardless of underlying data type
+inline bool evaluateCondition(NDArray* condition, int index) {
+ switch(condition->dataType()) {
+#if defined(HAS_BOOL)
+   case DataType::BOOL:
+     return condition->e<bool>(index);
+#endif
+#if defined(HAS_INT8)
+   case DataType::INT8:
+     return condition->e<int8_t>(index) != 0;
+#endif
+#if defined(HAS_INT16)
+   case DataType::INT16:
+     return condition->e<int16_t>(index) != 0;
+#endif
+#if defined(HAS_INT32)
+   case DataType::INT32:
+     return condition->e<int32_t>(index) != 0;
+#endif
+#if defined(HAS_LONG)
+   case DataType::INT64:
+     return condition->e<sd::LongType>(index) != 0;
+#endif
+#if defined(HAS_UINT8)
+   case DataType::UINT8:
+     return condition->e<uint8_t>(index) != 0;
+#endif
+#if defined(HAS_UINT16)
+   case DataType::UINT16:
+     return condition->e<uint16_t>(index) != 0;
+#endif
+#if defined(HAS_UINT32)
+   case DataType::UINT32:
+     return condition->e<uint32_t>(index) != 0;
+#endif
+#if defined(HAS_UNSIGNEDLONG)
+   case DataType::UINT64:
+     return condition->e<sd::UnsignedLong>(index) != 0;
+#endif
+#if defined(HAS_FLOAT16)
+   case DataType::HALF:
+     return condition->e<float16>(index) != static_cast<float16>(0.0f);
+#endif
+#if defined(HAS_BFLOAT16)
+   case DataType::BFLOAT16:
+     return condition->e<bfloat16>(index) != static_cast<bfloat16>(0.0f);
+#endif
+#if defined(HAS_FLOAT32)
+   case DataType::FLOAT32:
+     return condition->e<float>(index) != 0.0f;
+#endif
+#if defined(HAS_DOUBLE)
+   case DataType::DOUBLE:
+     return condition->e<double>(index) != 0.0;
+#endif
+   default:
+     // Fallback: try to interpret as int32 and check if non-zero
+#if defined(HAS_INT32)
+#ifdef __cpp_exceptions
+     try {
+       return condition->e<int32_t>(index) != 0;
+     } catch (...) {
+       // Last resort: assume false to maintain safe behavior
+       return false;
+     }
+#else
+     return condition->e<int32_t>(index) != 0;
+#endif
+#else
+     // If INT32 is not available, return false as safe default
+     return false;
+#endif
+ }
+}
+
+// Helper function to perform element-wise where with proper broadcasting
+void performBroadcastedWhere(NDArray* condition, NDArray* x, NDArray* y, NDArray* z) {
+ // We'll process each element of the output array z
+ // and determine the appropriate indices for condition, x, and y based on broadcasting rules
+
+ auto* zShape = z->getShapeAsVector();
+ auto* condShape = condition->getShapeAsVector();
+ auto* xShape = x->getShapeAsVector();
+ auto* yShape = y->getShapeAsVector();
+
+ // For each element in the output array
+ for (LongType i = 0; i < z->lengthOf(); i++) {
+   // Convert linear index to multi-dimensional indices for output array
+   std::vector<LongType> zIndices(z->rankOf());
+   LongType remainder = i;
+   for (int dim = z->rankOf() - 1; dim >= 0; dim--) {
+     zIndices[dim] = remainder % z->sizeAt(dim);
+     remainder /= z->sizeAt(dim);
+   }
+
+   // Calculate corresponding indices in condition, x, and y arrays using broadcasting rules
+   auto getLinearIndex = [](const std::vector<LongType>& multiIndices, const std::vector<LongType>& shape, NDArray* array) -> LongType {
+     LongType linearIndex = 0;
+     LongType stride = 1;
+     int srcDim = shape.size() - 1;
+
+     for (int dim = multiIndices.size() - 1; dim >= 0; dim--) {
+       LongType srcIndex = 0;
+       if (srcDim >= 0) {
+         if (shape[srcDim] == 1) {
+           srcIndex = 0; // Broadcast dimension
+         } else {
+           srcIndex = multiIndices[dim];
+         }
+         srcDim--;
+       }
+       linearIndex += srcIndex * stride;
+       if (srcDim >= 0) {
+         stride *= shape[srcDim + 1];
+       }
+     }
+     return linearIndex;
+   };
+
+   LongType condIndex = condition->lengthOf() == 1 ? 0 : getLinearIndex(zIndices, *condShape, condition);
+   LongType xIndex = x->lengthOf() == 1 ? 0 : getLinearIndex(zIndices, *xShape, x);
+   LongType yIndex = y->lengthOf() == 1 ? 0 : getLinearIndex(zIndices, *yShape, y);
+
+   // Apply the where logic
+   if (z->isR()) {
+#ifdef HAS_DOUBLE
+     auto result = evaluateCondition(condition, condIndex) ?
+                                                           x->e<double>(xIndex) : y->e<double>(yIndex);
+#elif defined(HAS_FLOAT32)
+     auto result = evaluateCondition(condition, condIndex) ?
+                                                           x->e<float>(xIndex) : y->e<float>(yIndex);
+#else
+#error "No floating-point type available for where operation"
+#endif
+     z->p(i, result);
+   } else{
+     auto result = evaluateCondition(condition, condIndex) ?
+                                                           x->e<LongType>(xIndex) : y->e<LongType>(yIndex);
+     z->p(i, result);
+   }
+ }
+
+ delete zShape;
+ delete condShape;
+ delete xShape;
+ delete yShape;
+}
+
+CUSTOM_OP_IMPL(Where, 1, 1, false, 0, 0) {
+ auto condition = INPUT_VARIABLE(0);
+ auto z = OUTPUT_VARIABLE(0);
+ if (z->isEmpty()) return Status::OK;
+
+ if (block.width() == 3) {
+   auto x = INPUT_VARIABLE(1);
+   auto y = INPUT_VARIABLE(2);
+
+   // Check if x and y can be broadcast together (instead of requiring exact same shape)
+   REQUIRE_TRUE(x->isSameShape(y) || ShapeUtils::areShapesBroadcastable(*x, *y), 0,
+                "X and Y must have equal shapes or be broadcastable. X shape: %s, Y shape: %s",
+                ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+   // Case 1: All arrays have exact shape matching (element-wise operation)
+   if (condition->isSameShape(x) && x->isSameShape(y)) {
+     // FIXME: for perf it might be better to issue memcpy here, and fill only mismatched values from either X or Y
+     for (int e = 0; e < condition->lengthOf(); e++) {
+       if (z->isR()) {
+#ifdef HAS_DOUBLE
+         auto r = !evaluateCondition(condition, e) ? y->e<double>(e) : x->e<double>(e);
+#elif defined(HAS_FLOAT32)
+         auto r = !evaluateCondition(condition, e) ? y->e<float>(e) : x->e<float>(e);
+#else
+#error "No floating-point type available for where operation"
+#endif
+         z->p(e, r);
+       } else {
+         auto r = !evaluateCondition(condition, e) ? y->e<LongType>(e) : x->e<LongType>(e);
+         z->p(e, r);
+       }
+     }
+   }
+   // Case 2: Broadcasting is possible (most flexible case)
+   else if (ShapeUtils::areShapesBroadcastable(*condition, *x) &&
+            ShapeUtils::areShapesBroadcastable(*condition, *y) &&
+            ShapeUtils::areShapesBroadcastable(*x, *y)) {
+     performBroadcastedWhere(condition, x, y, z);
+   }
+   // Case 3: TAD-mask operation (legacy behavior for specific cases)
+   else if (condition->rankOf() == 1 && condition->lengthOf() == x->sizeAt(0)) {
+     std::vector<LongType> zero({0});
+     auto dims = ShapeUtils::evalDimsToExclude(x->rankOf(), 1, zero.data());
+     auto tadsX = x->allTensorsAlongDimension(*dims);
+     auto tadsY = y->allTensorsAlongDimension(*dims);
+     auto tadsZ = z->allTensorsAlongDimension(*dims);
+
+     for (int e = 0; e < tadsX.size(); e++) {
+       if (!evaluateCondition(condition, e)) {
+         tadsZ.at(e)->assign(tadsY.at(e));
+       } else {
+         tadsZ.at(e)->assign(tadsX.at(e));
+       }
+     }
+
+     delete dims;
+   }
+   // Case 4: Invalid shapes - provide detailed error message
+   else {
+     std::string condShape = ShapeUtils::shapeAsString(condition);
+     std::string xShape = ShapeUtils::shapeAsString(x);
+     std::string yShape = ShapeUtils::shapeAsString(y);
+
+     REQUIRE_TRUE(false, 0,
+                  "Where operation: Invalid shapes for broadcasting. "
+                  "Condition shape: %s, X shape: %s, Y shape: %s. "
+                  "Condition must either: (1) match X/Y shapes exactly, "
+                  "(2) be broadcastable with X/Y shapes, or "
+                  "(3) be 1D with length equal to first dimension of X/Y for TAD-mask operation.",
+                  condShape.c_str(), xShape.c_str(), yShape.c_str());
+   }
+ } else {
+   // in this case we return 2D matrix, which basically contains coordinates fo true
+   REQUIRE_TRUE(block.width() == 1, 0, "Where op takes either 1 or 3 operands, But got %d operands instead",
+                block.width());
+   auto output = OUTPUT_VARIABLE(0);
+   std::vector<LongType> zero({0});
+
+   int width = condition->rankOf();
+   if (z->isEmpty()) return Status::OK;
+
+   std::vector<LongType> *dims = ShapeUtils::evalDimsToExclude(width,1,zero.data());
+
+   helpers::_where(block.launchContext(), *condition, *output, block.workspace());
+   delete dims;
+ }
+ return Status::OK;
+}
+
+DECLARE_SHAPE_FN(Where) {
+ if (block.width() == 3) {
+   auto x = INPUT_VARIABLE(1);
+   auto y = INPUT_VARIABLE(2);
+
+   // Calculate the broadcast result shape for x and y
+   LongType* resultShapeInfo = nullptr;
+   bool canBroadcast = ShapeUtils::evalBroadcastShapeInfo(*x, *y, true, resultShapeInfo, block.getWorkspace());
+
+   if (canBroadcast && resultShapeInfo != nullptr) {
+     return SHAPELIST(CONSTANT(resultShapeInfo));
+   } else {
+     // Fallback to x's shape if broadcasting fails (should have been caught in validation)
+     auto inShape = inputShape->at(1);
+     return SHAPELIST(CONSTANT(inShape));
+   }
+ } else {
+   // FIXME: we can't estimate result here in this case
+   // output shape is the 2D tensor num_true x rankOf (inShape)
+   auto condition = INPUT_VARIABLE(0);
+   auto inShape = inputShape->at(0);
+   LongType numOfTrue = 0;  // condition->reduceNumber(reduce::CountNonZero, nullptr).e<sd::LongType>(0);
+   for (LongType i = 0; i < condition->lengthOf(); i++)
+     if (evaluateCondition(condition, i)) numOfTrue++;
+
+   LongType * theNewShape;
+   if (numOfTrue > 0) {
+     LongType* newShape;
+     ALLOCATE(newShape, block.getWorkspace(), shape::shapeInfoLength(2), sd::LongType);
+     newShape[0] = 2;
+     newShape[1] = numOfTrue;
+     newShape[2] = shape::rank(inShape);
+     newShape[3] = 1;
+     newShape[4] = 1;
+     newShape[5] = 0;
+     newShape[6] = 1;
+     newShape[7] = 99;
+#if defined(HAS_LONG)
+     ShapeUtils::updateStridesAndType(newShape, INT64, 'c');
+#else
+     // Fallback to INT32 if INT64 is not available
+     ShapeUtils::updateStridesAndType(newShape, INT32, 'c');
+#endif
+
+     theNewShape = CONSTANT(newShape);
+     RELEASE(newShape, block.getWorkspace());
+   } else {
+#if defined(HAS_LONG)
+     theNewShape = ConstantShapeHelper::getInstance().emptyShapeInfo(INT64);
+#else
+     // Fallback to INT32 if INT64 is not available
+     theNewShape = ConstantShapeHelper::getInstance().emptyShapeInfo(INT32);
+#endif
+   }
+
+   return SHAPELIST(theNewShape);
+ }
+}
+
+DECLARE_TYPES(Where) {
+ getOpDescriptor()
+     ->setAllowedInputTypes(0, ANY)  // bool
+     ->setAllowedInputTypes(1, ANY)
+     ->setAllowedInputTypes(2, ANY)
+     ->setAllowedOutputTypes(0, {ALL_INTS, ALL_FLOATS,BOOL});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/boolean/where_np.cpp

@@ -0,0 +1,179 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author Adam Gibson
+//
+
+#include <ops/declarable/headers/boolean.h>
+#include <system/op_boilerplate.h>
+
+#if NOT_EXCLUDED(OP_where_np)
+
+#include <helpers/ShapeUtils.h>
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+CUSTOM_OP_IMPL(where_np, -1, 1, false, 0, 0) {
+  auto condition = INPUT_VARIABLE(0);
+
+  if (block.width() == 3) {
+    auto x = INPUT_VARIABLE(1);
+    auto y = INPUT_VARIABLE(2);
+
+    auto z = OUTPUT_VARIABLE(0);
+    int numMatches = 0;
+    // if cond matches x/y shape - we have per-element mask
+    if (condition->isSameShape(x)) {
+      // FIXME: for perf it might be better to issue memcpy here, and fill only mismatched values from either X or Y
+      if (y->isScalar()) {
+        if (y->isR()) {
+          for (int e = 0; e < condition->lengthOf(); e++) {
+#ifdef HAS_DOUBLE
+            auto r = condition->e<bool>(e) ? y->e<double>(0) : x->e<double>(e);
+#elif defined(HAS_FLOAT32)
+            auto r = condition->e<bool>(e) ? y->e<float>(0) : x->e<float>(e);
+#else
+#error "No floating-point type available for where_np operation"
+#endif
+            z->p(e, r);
+          }
+        } else{
+          for (int e = 0; e < condition->lengthOf(); e++) {
+            auto r = condition->e<bool>(e) ? y->e<LongType>(0) : x->e<LongType>(e);
+            z->p(e, r);
+          }
+        }
+      } else {
+        if (y->isR()) {
+          for (int e = 0; e < condition->lengthOf(); e++) {
+            if (condition->e<bool>(e)) {
+#ifdef HAS_DOUBLE
+              auto r = y->e<double>(numMatches);
+#elif defined(HAS_FLOAT32)
+              auto r = y->e<float>(numMatches);
+#else
+#error "No floating-point type available for where_np operation"
+#endif
+              z->p(e, r);
+              numMatches++;
+            } else {
+#ifdef HAS_DOUBLE
+              auto r = x->e<double>(e);
+#elif defined(HAS_FLOAT32)
+              auto r = x->e<float>(e);
+#else
+#error "No floating-point type available for where_np operation"
+#endif
+              z->p(e, r);
+            }
+          }
+        } else {
+          for (int e = 0; e < condition->lengthOf(); e++) {
+            if (condition->e<bool>(e)) {
+              auto r = y->e<LongType>(numMatches);
+              z->p(e, r);
+              numMatches++;
+            } else {
+              auto r = x->e<LongType>(e);
+              z->p(e, r);
+            }
+          }
+        }
+      }
+    } else {
+      REQUIRE_TRUE(condition->lengthOf() == x->sizeAt(0), 0,
+                   "Condition length should be equal to the dim0 of x/y to act as TAD-mask, but got %d instead",
+                   condition->lengthOf());
+
+      std::vector<LongType> idxs;
+      idxs.push_back(0);
+      auto dims = ShapeUtils::evalDimsToExclude(x->rankOf(), 1,idxs.data());
+      auto tadsX = x->allTensorsAlongDimension(*dims);
+      auto tadsY = y->allTensorsAlongDimension(*dims);
+      auto tadsZ = z->allTensorsAlongDimension(*dims);
+
+      for (int e = 0; e < tadsX.size(); e++) {
+        if (!condition->e<bool>(e))
+          tadsZ.at(e)->assign(tadsY.at(e));
+        else
+          tadsZ.at(e)->assign(tadsX.at(e));
+      }
+
+      delete dims;
+    }
+  } else {
+    // in this case we return 2D matrix, which basically contains coordinates fo true
+
+    REQUIRE_TRUE(block.width() == 1, 0, "Where op takes either 1 or 3 operands, But got %d operands instead",
+                 block.width());
+    LongType width = condition->rankOf();
+
+    Where op;
+    auto res(op.evaluate({condition}));
+    REQUIRE_OK(res.status());
+    NDArray* whereTrue = res.at(0);
+
+    if (whereTrue->isEmpty()) return Status::OK;
+    for (LongType outNext = 0; outNext < width; ++outNext) {
+      auto output = OUTPUT_VARIABLE(outNext);
+      for (LongType e = 0; e < output->lengthOf(); ++e) {
+        output->p<LongType>(e, whereTrue->e<LongType>(e, outNext));
+      }
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(where_np) {
+  auto shapes = SHAPELIST();
+  if (block.width() == 3) {
+    auto inShape = inputShape->at(1);
+    shapes->push_back(CONSTANT(inShape));
+  } else {
+    auto condition = INPUT_VARIABLE(0);
+
+    LongType numOfTrue = 0LL;  // condition->reduceNumber(reduce::CountNonZero).e<sd::LongType>(0);
+    for (LongType i = 0; i < condition->lengthOf(); ++i)
+      if (condition->e<bool>(i)) numOfTrue++;
+
+    // output shape - a tuple of rank(inShape) 1D tensors with numOfTrue len
+    if (numOfTrue) {
+      for (LongType e = 0; e < condition->rankOf(); ++e) {
+        shapes->push_back(ConstantShapeHelper::getInstance().vectorShapeInfo(numOfTrue, INT64));
+      }
+    } else {
+      shapes->push_back(ConstantShapeHelper::getInstance().emptyShapeInfo(INT64));
+    }
+  }
+  return shapes;
+}
+
+DECLARE_TYPES(where_np) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, BOOL)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedInputTypes(2, ANY)
+      ->setAllowedOutputTypes({ALL_FLOATS, ALL_INTS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/add.cpp

@@ -0,0 +1,109 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_add)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(add, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Add(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z && !tZ->isEmpty()) {
+    OVERWRITE_RESULT(tZ);
+  }
+  return Status::OK;
+}
+
+DECLARE_TYPES(add) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(ANY);
+}
+
+DECLARE_TYPES(add_bp) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
+
+CUSTOM_OP_IMPL(add_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+    gradY->assign(epsNext);
+    gradX->assign(epsNext);
+  } else if (y->isScalar()) {
+    // scalar case
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(tmp);
+    gradX->assign(epsNext);
+    delete tmp;
+  } else {
+    // broadcast case
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else
+      gradX->assign(epsNext);
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(epsNext);
+  }
+
+
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(add_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/assign.cpp

@@ -0,0 +1,123 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 24.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#include <helpers/StringUtils.h>
+#if NOT_EXCLUDED(OP_assign)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(assign, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = block.width() < 2 ? x: INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  // Check if any array is of string type
+  if (x->isS() || y->isS() || z->isS()) {
+    // Handle string broadcast at high level
+    StringUtils::broadcastStringAssign(x,z);
+    return Status::OK;
+  }
+
+  NDArray *castedX = x->dataType() == z->dataType() ? x : x->cast(z->dataType());
+  NDArray *castedY = y->dataType() == z->dataType() ? y : y->cast(z->dataType());
+
+  ArrayOptions::validateSingleDataType(ArrayOptions::dataType(castedX->shapeInfo()));
+  ArrayOptions::validateSingleDataType(ArrayOptions::extra(castedY->shapeInfo()));
+  ArrayOptions::validateSingleDataType(ArrayOptions::extra(z->shapeInfo()));
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Assign(), castedX, castedY, z);
+
+  if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  // Cleanup casted arrays if they were allocated
+  if (castedX != x) delete castedX;
+  if (castedY != y) delete castedY;
+
+  return Status::OK;
+}
+DECLARE_SYN(set, assign);
+DECLARE_SYN(copy, assign);
+
+DECLARE_TYPES(assign) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL})
+      ->setAllowedInputTypes(1, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL})
+      ->setAllowedOutputTypes(0, {ALL_INTS,ALL_FLOATS,ALL_STRINGS,BOOL});
+}
+
+DECLARE_TYPES(assign_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_INTS,ALL_FLOATS,ALL_STRINGS});
+}
+
+CUSTOM_OP_IMPL(assign_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = block.width() < 2 ? x->dup(x->ordering(), false) : INPUT_VARIABLE(1);  // dup() already returns NDArray*
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  float zero = 0.0f;
+  gradX->assign(zero);
+
+  if (x->isSameShape(y)) {
+    gradY->assign(epsNext);
+  } else if (y->isScalar()) {
+    auto sum = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(sum);
+    delete sum;
+  } else {
+    // broadcastable
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(epsNext);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(assign_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/atan2.cpp

@@ -0,0 +1,55 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com, created on 10.02.18.
+// @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_tf_atan2)
+
+#include <ops/declarable/headers/broadcastable.h>
+
+namespace sd {
+namespace ops {
+
+BROADCASTABLE_OP_IMPL(tf_atan2, 0, 0) {
+  auto y = INPUT_VARIABLE(0);
+  auto x = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  x->applyTrueBroadcast(BroadcastOpsTuple::custom(scalar::Atan2, pairwise::Atan2, broadcast::Atan2), y, z, true);
+
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(tf_atan2) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_and.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_or)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_and, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalAnd, pairwise::LogicalAnd, broadcast::LogicalAnd), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_and: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_and) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_or.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_or)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_or, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalOr, pairwise::LogicalOr, broadcast::LogicalOr), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_and: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_or) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/boolean_xor.cpp

@@ -0,0 +1,56 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver on 6/6/2018.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_boolean_xor)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(boolean_xor, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::LogicalXor, pairwise::LogicalXor, broadcast::LogicalXor), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("boolean_xor: result was overwritten");
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(boolean_xor) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/divide.cpp

@@ -0,0 +1,150 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_divide)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(divide, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "DIVIDE OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Divide(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Div, divide);
+
+DECLARE_TYPES(divide) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(divide_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(divide_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    NDArray *gradXTemp = (*epsNext) / (*y);
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    // Y gradient
+    NDArray *numerator = (*epsNext) * (*x);
+    NDArray *denominator = (*y) * (*y);
+    NDArray *gradYTemp = (*numerator) / (*denominator);
+    delete numerator;
+    delete denominator;
+    gradY->assign(gradYTemp);
+    gradY->applyTransform(transform::Neg, gradY);
+
+  } else if (y->isScalar()) {
+    // scalar case
+
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    auto tmpX = x->reduceNumber(reduce::Sum);
+
+    NDArray *temp1 = *tmp * *tmpX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*temp1) / (*ySquared);
+    delete temp1;
+    delete ySquared;
+    gradY->assign(gradYTemp);
+    gradY->applyTransform(transform::Neg, gradY);
+
+    epsNext->applyScalarArr(scalar::Divide, y, gradX);
+  } else {
+    // broadcast case
+
+    auto preX = *epsNext / *y;
+
+    NDArray negX(*x);
+    x->applyTransform(transform::Neg, &negX);
+    NDArray *negXMulEps = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    auto preY = (*negXMulEps) / (*ySquared);
+    delete negXMulEps;
+    delete ySquared;
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else {
+      // FIXED: preX is stack-allocated from operator/, don't delete
+      gradX->assign(preX);
+    }
+
+    if (axisY.size() > 0) {
+      auto sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      // FIXED: preY is stack-allocated from operator/, don't delete
+      gradY->assign(preY);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(divide_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/divide_no_nan.cpp

@@ -0,0 +1,59 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author George A. Shulinok <sgazeos@gmail.com>
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_divide_no_nan)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(divide_no_nan, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "DIVIDE_NO_NAN OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::DivideNoNan(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(Div, divide);
+
+DECLARE_TYPES(divide_no_nan) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/equals.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/BroadcastBoolOpsTuple.h>
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_equals)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(equals, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastBoolOpsTuple::custom(scalar::EqualTo, pairwise::EqualTo, broadcast::EqualTo), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SYN(equal, equals);
+
+DECLARE_TYPES(equals) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/floordiv.cpp

@@ -0,0 +1,89 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_floordiv)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(floordiv, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "FLOORDIV OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(
+      BroadcastOpsTuple::custom(scalar::FloorDiv, pairwise::FloorDiv, broadcast::FloorDiv), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(floordiv) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(floordiv_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(floordiv_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  float zero = 0.0f;
+  gradY->assign(zero);
+  gradX->assign(zero);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(floordiv_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/floormod.cpp

@@ -0,0 +1,101 @@
+/* ******************************************************************************
+*
+*
+* This program and the accompanying materials are made available under the
+* terms of the Apache License, Version 2.0 which is available at
+* https://www.apache.org/licenses/LICENSE-2.0.
+*
+*  See the NOTICE file distributed with this work for additional
+*  information regarding copyright ownership.
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations
+* under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//  modified by sgazeos@gmail.com with backprop implementation.
+//
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_floormod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(floormod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!y->isB(), 0, "FLOORMOD OP: you can't divide by bool array!");
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(FloorMod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(floormod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(floormod_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(floormod_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  gradX->assign(epsNext);
+
+  NDArray temp(*epsNext);
+  BroadcastHelper::broadcastApply(BROADCAST(FloorMod), x, y, &temp);
+  if (gradY->rankOf() == gradX->rankOf()) {
+    epsNext->applyPairwiseTransform(pairwise::Multiply, &temp, gradY);
+  } else  { // epsNext is greater than gradY
+    std::vector<LongType> dims(epsNext->rankOf() * 2);
+    LongType gap = epsNext->rankOf() - gradY->rankOf();
+    for (LongType d = 0; d < gap; d++) {
+      dims[d * 2 + 1] = 1;
+    }
+    auto tempIn((temp)(dims));
+    NDArray negTempIn = -*tempIn;
+    auto get=  (*epsNext)(dims);
+    get->applyPairwiseTransform(pairwise::Multiply, &negTempIn, gradY);
+    delete get;
+    delete tempIn;
+  }
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(floormod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/greater.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_greater)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(greater, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(GreaterThan), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(greater) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/greater_equal.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_greater_equal)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(greater_equal, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(GreaterThanOrEqual), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(greater_equal) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/igamma.cpp

@@ -0,0 +1,58 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_igamma)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(igamma, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::IGamma(), x, y, z);
+
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(igamma) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/igammac.cpp

@@ -0,0 +1,57 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author sgazeos@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_igammac)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(igammac, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::IGammac(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(igammac) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/less.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/CustomOperations.h>
+#if NOT_EXCLUDED(OP_less)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(less, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(LessThan), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(less) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/less_equal.cpp

@@ -0,0 +1,53 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_less_equal)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(less_equal, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(LessThanOrEqual), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(less_equal) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/maximum.cpp

@@ -0,0 +1,83 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 12.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_maximum)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/helpers/minimax.h>
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(maximum, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(MaxPairwise), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(maximum) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(maximum_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(maximum_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  helpers::maximumBPFunctor(block.launchContext(), x, y, epsNext, gradX, gradY);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(maximum_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/meshgrid.cpp

@@ -0,0 +1,89 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 05.02.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_meshgrid)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/meshgrid.h>
+
+#include <numeric>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(meshgrid, -1, -1, false, 0, 0) {
+  int rank = block.width();
+
+  if (rank == 1) {
+    OUTPUT_VARIABLE(0)->assign(INPUT_VARIABLE(0));
+    return Status::OK;
+  }
+
+  bool swapFirst2Dims = block.getIArguments()->size() > 0 ? (bool)INT_ARG(0) : true;
+
+  std::vector<NDArray*> inArrs(rank);
+  std::vector<NDArray*> outArrs(rank);
+
+  for (int i = 0; i < rank; ++i) {
+    inArrs[i] = INPUT_VARIABLE(i);
+    outArrs[i] = OUTPUT_VARIABLE(i);
+  }
+
+  helpers::meshgrid(block.launchContext(), inArrs, outArrs, swapFirst2Dims);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(meshgrid) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes(INHERIT)->setSameMode(true);
+}
+
+DECLARE_SHAPE_FN(meshgrid) {
+  bool swapFirst2Dims = block.getIArguments()->size() > 0 ? (bool)INT_ARG(0) : true;
+
+  int rank = block.width();
+  LongType* outShapeInfo = nullptr;
+  ALLOCATE(outShapeInfo, block.getWorkspace(), shape::shapeInfoLength(rank), sd::LongType);
+  outShapeInfo[0] = rank;
+  for (int i = 1; i <= rank; ++i) outShapeInfo[i] = (LongType)shape::length(inputShape->at(i - 1));
+
+  if (swapFirst2Dims && rank > 1) math::sd_swap<LongType>(outShapeInfo[1], outShapeInfo[2]);
+
+  auto in = inputShape->at(0);
+  ShapeUtils::updateStridesAndType(outShapeInfo, in, shape::order(in));
+
+  auto shapes = SHAPELIST();
+  auto resultShape = CONSTANT(outShapeInfo);
+  shapes->push_back(resultShape);
+
+  for (int i = 2; i <= rank; ++i) {
+    shapes->push_back(resultShape);
+  }
+
+  return shapes;
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/minimum.cpp

@@ -0,0 +1,83 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 12.10.2017.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_minimum)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+#include <ops/declarable/helpers/minimax.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(minimum, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(MinPairwise), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(minimum) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(minimum_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(minimum_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  helpers::minimumBPFunctor(block.launchContext(), x, y, epsNext, gradX, gradY);
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(minimum_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/mod.cpp

@@ -0,0 +1,87 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_mod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(mod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(Mod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(mod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(mod_bp) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
+
+CUSTOM_OP_IMPL(mod_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  float zero = 0.0f;
+  gradY->assign(zero);
+  gradX->assign(zero);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(mod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/multiply.cpp

@@ -0,0 +1,165 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//  @author Yurii Shyrma (iuriish@yahoo.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_multiply)
+
+#include <ops/declarable/CustomOperations.h>
+
+namespace sd {
+namespace ops {
+
+BROADCASTABLE_OP_IMPL(multiply, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+   LongType* zShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, zShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0, "MULTIPLY OP: the shapes of x %s and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Multiply(), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z)
+    THROW_EXCEPTION("multiply: result was replaced");
+
+  return Status::OK;
+}
+DECLARE_SYN(Mul, multiply);
+
+DECLARE_TYPES(multiply) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(multiply_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+///////////////////////////////////////////////////////////////////
+CUSTOM_OP_IMPL(multiply_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto dLdz = INPUT_VARIABLE(2);
+
+  auto dLdx = OUTPUT_VARIABLE(0);
+  auto dLdy = OUTPUT_VARIABLE(1);
+
+   LongType* dLdzShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, dLdzShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0,
+               "MULTIPLY_BP OP: the shapes of x %s and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+
+
+  const LongType xLen = x->lengthOf();
+  const LongType yLen = y->lengthOf();
+
+  if (x->isScalar() && y->isScalar()) {  // both are scalars
+    y->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+    x->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+
+  }else if (x->isScalar()) {  // x is scalar and y is not
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceNumber(reduce::Sum);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+    dLdz->applyScalarArr(scalar::Multiply, x, dLdy);
+  } else if (y->isScalar()) {  // y is scalar and x is not
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceNumber(reduce::Sum);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+    dLdz->applyScalarArr(scalar::Multiply, y, dLdx);
+  } else if (x->isSameShape(y)) {
+    x->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+    y->applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+  } else if (x->isSameShape(dLdz)) {
+    auto yTiled = NDArray(dLdz, false, block.launchContext());
+    y->tile(yTiled);
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+    yTiled.applyPairwiseTransform(pairwise::Multiply, dLdz, dLdx);
+  } else if (y->isSameShape(dLdz)) {
+    auto xTiled = NDArray(dLdz, false, block.launchContext());
+    x->tile(xTiled);
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    // FIXED: Clean up intermediate result from operator*
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+    xTiled.applyPairwiseTransform(pairwise::Multiply, dLdz, dLdy);
+  } else {
+    auto xTiled = NDArray(dLdz, false, block.launchContext());
+    auto yTiled = NDArray(dLdz, false, block.launchContext());
+    x->tile(xTiled);
+    y->tile(yTiled);
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+
+    // For dLdx
+    NDArray *yMulDldz = (*y) * (*dLdz);
+    NDArray *dLdxTemp = yMulDldz->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);
+    delete yMulDldz;
+    delete dLdxTemp;
+
+    // For dLdy
+    // FIXED: Clean up intermediate result from operator*
+    NDArray *xMulDldz = (*x) * (*dLdz);
+    NDArray *dLdyTemp = xMulDldz->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);
+    delete xMulDldz;
+    delete dLdyTemp;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(multiply_bp) {
+  auto xShapeInfo = inputShape->at(0);
+  auto yShapeInfo = inputShape->at(1);
+  return SHAPELIST(CONSTANT(xShapeInfo), CONSTANT(yShapeInfo));
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/not_equals.cpp

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <ops/declarable/headers/broadcastable.h>
+#if NOT_EXCLUDED(OP_not_equals)
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_BOOL_OP_IMPL(not_equals, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST_BOOL(NotEqualTo), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(not_equals) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, BOOL);
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/percentile.cpp

@@ -0,0 +1,108 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 17.05.2018
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_percentile)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/helpers/percentile.h>
+
+namespace sd {
+namespace ops {
+
+CUSTOM_OP_IMPL(percentile, 1, 1, false, 1, -2) {
+  auto input = INPUT_VARIABLE(0);    // tensor with rank > 0
+  auto output = OUTPUT_VARIABLE(0);  // [bS, oD, oH, oW, iC] (NDHWC) or [bS, iC, oD, oH, oW] (NCDHW)
+
+  const auto q = T_ARG(0);  // percentile
+  const int interpolation = block.getTArguments()->size() > 1 ? T_ARG(1) : 2.;  // 0-"lower", 1-"higher", 2-"nearest"(default)
+  const int keepDims = block.getTArguments()->size() > 2 ? T_ARG(2) : 0.;  // false is default
+
+  const int axisArrRank = block.getIArguments()->size();
+  const int inputArrRank = input->rankOf();
+
+  REQUIRE_TRUE(inputArrRank > 0, 0, "PERCENTILE OP: rank of input array must be positive (>0), but got %i instead !",
+               inputArrRank);
+  REQUIRE_TRUE(0.f <= q && q <= 100.f, 0,
+               "PERCENTILE OP: percentile parameter must be within [0, 100] range, but got %f instead !", q);
+  REQUIRE_TRUE(interpolation == 0 || interpolation == 1 || interpolation == 2, 0,
+               "PERCENTILE OP: the correct values for interpolation parameter are 0, 1, 2, but got %i instead !",
+               interpolation);
+  REQUIRE_TRUE(
+      axisArrRank <= inputArrRank, 0,
+      "PERCENTILE OP: the rank of axis array must be <= rank of input array, but got %i and %i correspondingly !",
+      axisArrRank, inputArrRank);
+
+  for (int i = 0; i < axisArrRank; ++i) {
+    int dim = INT_ARG(i) >= 0 ? INT_ARG(i) : INT_ARG(i) + inputArrRank;
+    REQUIRE_TRUE(dim < inputArrRank, 0,
+                 "PERCENTILE OP: element (dimension) of axis array at position %i is >= rank of input array (%i >= "
+                 "%i), which is unacceptable !",
+                 i, dim, inputArrRank);
+  }
+
+  std::vector<LongType> axises = *block.getIArguments();
+  helpers::percentile(block.launchContext(), *input, *output, axises, q, interpolation);
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(percentile) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedOutputTypes(0, INHERIT)
+      ->setSameMode(true);
+}
+
+DECLARE_SHAPE_FN(percentile) {
+  auto inputShapeInfo = inputShape->at(0);
+  const int keepDims = block.getTArguments()->size() > 2 ? T_ARG(2) : 0.;  // false is default
+
+  const int axisArrRank = block.getIArguments()->size();
+  const int inputArrRank = inputShapeInfo[0];
+
+  REQUIRE_TRUE(inputArrRank > 0, 0, "PERCENTILE OP: rank of input array must be positive (>0), but got %i instead !",
+               inputArrRank);
+  REQUIRE_TRUE(
+      axisArrRank <= inputArrRank, 0,
+      "PERCENTILE OP: the rank of axis array must be <= rank of input array, but got %i and %i correspondingly !",
+      axisArrRank, inputArrRank);
+
+  for (int i = 0; i < axisArrRank; ++i) {
+    int dim = INT_ARG(i) >= 0 ? INT_ARG(i) : INT_ARG(i) + inputArrRank;
+    REQUIRE_TRUE(dim < inputArrRank, 0,
+                 "PERCENTILE OP: element (dimension) of axis array at position %i is >= rank of input array (%i >= "
+                 "%i), which is unacceptable !",
+                 i, dim, inputArrRank);
+  }
+
+  std::vector<LongType> axises = *block.getIArguments();
+  auto outputShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(inputShapeInfo), &axises, inputShapeInfo, keepDims,
+                                                         false, block.getWorkspace());
+
+  return SHAPELIST(outputShapeInfo);
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/pow.cpp

@@ -0,0 +1,122 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+// @author Oleh Semeniv (oleg.semeniv@gmail.com)
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_Pow)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(Pow, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  // REQUIRE_TRUE(!y->isB(), 0, "Pairwise OP: you can't divide by bool array!");
+
+  auto tZ = BroadcastHelper::broadcastApply({scalar::Pow, pairwise::Pow, broadcast::Pow}, x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(Pow) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedInputTypes(1, {ALL_FLOATS, ALL_INTS})
+      ->setAllowedOutputTypes(0, {ALL_FLOATS, ALL_INTS});
+}
+
+CUSTOM_OP_IMPL(Pow_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto dLdz = INPUT_VARIABLE(2);
+
+  auto dLdx = OUTPUT_VARIABLE(0);
+  auto dLdy = OUTPUT_VARIABLE(1);
+
+   LongType* dLdzShapeInfo = nullptr;
+  const bool areShapesBroadcastable =
+      ShapeUtils::evalBroadcastShapeInfo(x->shapeInfo(), y->shapeInfo(), true, dLdzShapeInfo, block.getWorkspace());
+  REQUIRE_TRUE(areShapesBroadcastable, 0,
+               "POW_BP OP: the shapes of x %s"
+               " and y %s are not suitable for broadcast !",
+               ShapeUtils::shapeAsString(x).c_str(), ShapeUtils::shapeAsString(y).c_str());
+  REQUIRE_TRUE(shape::equalsSoft(dLdz->shapeInfo(), dLdzShapeInfo), 0,
+               "POW_BP OP: wrong shape of next epsilon array (dLdOut),"
+               " expected is %s, but got %s instead !",
+               ShapeUtils::shapeAsString(dLdzShapeInfo).c_str(), ShapeUtils::shapeAsString(dLdz).c_str());
+
+  // dL/dy = x^y * log(x) * dL/dz
+  auto temp = x->applyTrueBroadcast(BroadcastOpsTuple::Pow(), y);  // a = x^y
+  x->applyTransform(transform::Log, dLdx);                         // b = log(x)
+  dLdx->applyScalar(scalar::ReplaceNans, 0, dLdx);
+  *temp *= *dLdx;  // c = b*a
+  *temp *= *dLdz;  // dL/dy = c * dL/dz
+  if (dLdy->isSameShape(*dLdz)) {
+    dLdy->assign(temp);
+  } else {
+    std::vector<LongType> axesForY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), dLdz->shapeInfo());
+    NDArray *dLdyTemp = temp->reduceAlongDimension(reduce::Sum, &axesForY);
+    dLdy->assign(dLdyTemp);  // dL/dy = sum(c * dL/dz)
+    delete dLdyTemp;
+  }
+
+  // dL/dx = y*x^(y-1) * dL/dz
+  x->applyTrueBroadcast(BroadcastOpsTuple::PowDerivative(), y, temp);  // a = y*x^(y-1)
+  *temp *= *dLdz;                                                        // dLdx = a*dL/dz
+
+  if (dLdx->isSameShape(*dLdz)) {
+    dLdx->assign(temp);  // dLdx = a*dL/dz
+  } else {
+    std::vector<LongType> axesForX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), dLdz->shapeInfo());
+    NDArray *dLdxTemp = temp->reduceAlongDimension(reduce::Sum, &axesForX);
+    dLdx->assign(dLdxTemp);  // dLdx = a*dL/dz
+    delete dLdxTemp;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(Pow_bp) {
+  auto xShapeInfo = inputShape->at(0);
+  auto yShapeInfo = inputShape->at(1);
+  return SHAPELIST(CONSTANT(xShapeInfo), CONSTANT(yShapeInfo));
+}
+
+DECLARE_TYPES(Pow_bp) {
+  getOpDescriptor()->setAllowedInputTypes({ALL_FLOATS, ALL_INTS})->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/realdiv.cpp

@@ -0,0 +1,150 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_realdiv)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(realdiv, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+  auto tZ = BroadcastHelper::broadcastApply(BroadcastOpsTuple::Divide(), x, y, z);
+  if (tZ == nullptr) {
+    return Status::KERNEL_FAILURE;
+  }
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(RealDiv, realdiv);
+
+DECLARE_TYPES(realdiv) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, {FLOAT32, HALF, DOUBLE});
+}
+
+DECLARE_TYPES(realdiv_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(realdiv_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    epsNext->applyPairwiseTransform(pairwise::Divide, y, gradX);
+
+    // Y gradient
+
+    // First case
+    NDArray negX = -(*x);
+    NDArray *epsNextMulNegX = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*epsNextMulNegX) / (*ySquared);
+    gradY->assign(gradYTemp);
+    delete epsNextMulNegX;
+    delete ySquared;
+    delete gradYTemp;
+
+  } else if (y->isScalar()) {
+    // scalar case
+
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    auto tmpX = x->reduceNumber(reduce::Sum);
+
+    NDArray negTmpX = -*tmpX;
+    NDArray *tmpMulNegTmpX = (*tmp) * negTmpX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *gradYTemp = (*tmpMulNegTmpX) / (*ySquared);
+    gradY->assign(gradYTemp);
+    delete tmpMulNegTmpX;
+    delete ySquared;
+    delete gradYTemp;
+
+    epsNext->applyScalarArr(scalar::Divide, y, gradX);
+  } else {
+    // broadcast case
+
+    auto preX = *epsNext / *y;
+
+    NDArray negX(*x);
+    x->applyTransform(transform::Neg, &negX);
+    NDArray *epsNextMulNegX = (*epsNext) * negX;
+    NDArray *ySquared = (*y) * (*y);
+    NDArray *preY = (*epsNextMulNegX) / (*ySquared);
+    delete epsNextMulNegX;
+    delete ySquared;
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+    } else
+      gradX->assign(preX);
+
+    if (axisY.size() > 0) {
+      auto sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(preY);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(realdiv_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_divide.cpp

@@ -0,0 +1,147 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversedivide)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversedivide, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  REQUIRE_TRUE(!x->isB(), 0, "REVERSEDIVIDE OP: you can't divide by bool array!");
+  x->applyTrueBroadcast(BROADCAST(ReverseDivide), y, z, true);
+
+  return Status::OK;
+}
+DECLARE_SYN(RDiv, reversedivide);
+
+DECLARE_TYPES(reversedivide) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(reversedivide_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(reversedivide_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* gradXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    gradX->applyTransform(transform::Neg, gradX);
+
+    // Y gradient
+    auto* gradYTemp = (*epsNext) / (*x);
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+  } else if (y->isScalar()) {
+    // scalar case
+    auto* tmp = epsNext->reduceNumber(reduce::Sum);
+    auto* tmpX = x->reduceNumber(reduce::Sum);
+    // For gradY
+    auto* gradYTemp = (*tmp) / (*tmpX);
+    delete tmp;
+    delete tmpX;
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+
+    // For gradX
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* gradXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+    gradX->applyTransform(transform::Neg, gradX);
+  } else {
+    // broadcast case
+
+    auto* preY = (*epsNext) / (*x);
+
+    auto* epsY = (*epsNext) * (*y);
+    auto* xSquared = (*x) * (*x);
+    auto* preXTemp = (*epsY) / (*xSquared);
+    delete epsY;
+    delete xSquared;
+    preXTemp->applyTransform(transform::Neg, preXTemp);
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto* sum = preXTemp->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else {
+      gradX->assign(preXTemp);
+    }
+    delete preXTemp;
+
+    if (axisY.size() > 0) {
+      auto* sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      gradY->assign(preY);
+    }
+    delete preY;
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversedivide_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_mod.cpp

@@ -0,0 +1,88 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversemod)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversemod, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(ReverseMod), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+
+DECLARE_TYPES(reversemod) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+DECLARE_TYPES(reversemod_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+CUSTOM_OP_IMPL(reversemod_bp, 3, 2, false, 0, 0) {
+  // PLEASE NOTE: we're just passing eps down the line here
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+  float assign = 0.0f;
+  gradY->assign(assign);
+  gradX->assign(assign);
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversemod_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/reverse_subtract.cpp

@@ -0,0 +1,117 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+//  @author raver119@gmail.com
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_reversesubtract)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(reversesubtract, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(ReverseSubtract), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(RSub, reversesubtract);
+
+DECLARE_TYPES(reversesubtract) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(reversesubtract_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+    epsNext->applyTransform(transform::Neg, gradX);
+    gradY->assign(epsNext);
+  } else if (y->isScalar()) {
+    // scalar case
+    auto tmp = epsNext->reduceNumber(reduce::Sum);
+    gradY->assign(tmp);
+    delete tmp;
+    epsNext->applyTransform(transform::Neg, gradX);
+  } else {
+    // broadcastable
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisX);
+      sum->applyTransform(transform::Neg, gradX);
+      delete sum;
+    } else {
+      epsNext->applyTransform(transform::Neg, gradX);
+    }
+
+    if (axisY.size() > 0) {
+      auto sum = epsNext->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else {
+      gradY->assign(epsNext);
+    }
+  }
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(reversesubtract_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+  auto shapeList = SHAPELIST(CONSTANT(x), CONSTANT(y));
+
+  return shapeList;
+}
+
+DECLARE_TYPES(reversesubtract_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+}  // namespace ops
+}  // namespace sd
+
+#endif

libnd4j/include/ops/declarable/generic/broadcastable/squared_subtract.cpp

@@ -0,0 +1,174 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 23.11.17.
+//
+
+#include <system/op_boilerplate.h>
+#if NOT_EXCLUDED(OP_squaredsubtract)
+
+#include <ops/declarable/CustomOperations.h>
+#include <ops/declarable/generic/helpers/BroadcastHelper.h>
+
+namespace sd {
+namespace ops {
+BROADCASTABLE_OP_IMPL(squaredsubtract, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto z = OUTPUT_VARIABLE(0);
+
+  BROADCAST_CHECK_EMPTY(x, y, z);
+
+  auto tZ = BroadcastHelper::broadcastApply(BROADCAST(SquaredSubtract), x, y, z);
+  if (tZ == nullptr)
+    return Status::KERNEL_FAILURE;
+  else if (tZ != z) {
+    OVERWRITE_RESULT(tZ);
+  }
+
+  return Status::OK;
+}
+DECLARE_SYN(squareddifference, squaredsubtract);
+
+DECLARE_TYPES(squaredsubtract) {
+  getOpDescriptor()
+      ->setAllowedInputTypes(0, ANY)
+      ->setAllowedInputTypes(1, ANY)
+      ->setAllowedOutputTypes(0, INHERIT);
+}
+
+CUSTOM_OP_IMPL(squaredsubtract_bp, 3, 2, false, 0, 0) {
+  auto x = INPUT_VARIABLE(0);
+  auto y = INPUT_VARIABLE(1);
+  auto epsNext = INPUT_VARIABLE(2);
+
+  auto gradX = OUTPUT_VARIABLE(0);
+  auto gradY = OUTPUT_VARIABLE(1);
+
+
+  auto* ts = NDArrayFactory::create(x->dataType(), 2, block.launchContext());
+
+  if (x->isSameShape(y)) {
+    // PWT case case
+
+    // X gradient
+    auto* diff1 = (*x) - (*y);
+    auto* temp1 = (*ts) * (*diff1);
+    delete diff1;
+    auto* gradXTemp = (*epsNext) * (*temp1);
+    delete temp1;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+
+    // Y gradient
+    auto* diff2 = (*y) - (*x);
+    auto* temp2 = (*ts) * (*diff2);
+    delete diff2;
+    auto* gradYTemp = (*epsNext) * (*temp2);
+    delete temp2;
+    gradY->assign(gradYTemp);
+    delete gradYTemp;
+
+  } else if (y->isScalar()) {
+    // scalar case
+    auto* tmpX = x->reduceNumber(reduce::Sum);
+    gradY->assign(tmpX);
+    delete tmpX;
+    
+    // X gradient
+    auto* diff3 = (*x) - (*y);
+    auto* temp3 = (*ts) * (*diff3);
+    delete diff3;
+    auto* gradXTemp = (*epsNext) * (*temp3);
+    delete temp3;
+    gradX->assign(gradXTemp);
+    delete gradXTemp;
+  } else {
+    // broadcast case
+
+    auto* preX = x->dup(x->ordering());
+    auto* preY = y->dup(y->ordering());
+
+    auto* targetShape = epsNext->getShapeAsVector();
+
+    preX->tileToShape(*targetShape, *preX);
+    preY->tileToShape(*targetShape, *preY);
+    delete targetShape;
+    
+    auto* diff4 = (*x) - (*y);
+    auto* temp4 = (*ts) * (*diff4);
+    delete diff4;
+    auto* resX = (*epsNext) * (*temp4);
+    delete temp4;
+    preX->assign(resX);
+    delete resX;
+    
+    auto* diff5 = (*y) - (*x);
+    auto* temp5 = (*ts) * (*diff5);
+    delete diff5;
+    auto* resY = (*epsNext) * (*temp5);
+    delete temp5;
+    preY->assign(resY);
+    delete resY;
+
+    auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
+    auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
+
+    if (axisX.size() > 0) {
+      auto* sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
+      gradX->assign(sum);
+      delete sum;
+    } else
+      gradX->assign(preX);
+
+    if (axisY.size() > 0) {
+      auto* sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
+      gradY->assign(sum);
+      delete sum;
+    } else
+      gradY->assign(preY);
+
+    delete preX;
+    delete preY;
+  }
+
+  delete ts;
+
+  return Status::OK;
+}
+
+DECLARE_SHAPE_FN(squaredsubtract_bp) {
+  auto x = inputShape->at(0);
+  auto y = inputShape->at(1);
+  auto e = inputShape->at(2);
+
+  // eps always has shape of x
+  // grad always has shape of y
+
+  return SHAPELIST(CONSTANT(x), CONSTANT(y));
+}
+
+DECLARE_TYPES(squaredsubtract_bp) {
+  getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
+}
+
+}  // namespace ops
+}  // namespace sd
+
+#endif