chore: import upstream snapshot with attribution

libnd4j/include/execution/AffinityManager.h

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef LIBND4J_AFFINITYMANAGER_H
+#define LIBND4J_AFFINITYMANAGER_H
+
+#include <system/common.h>
+
+#include <atomic>
+#include <mutex>
+
+namespace sd {
+class SD_LIB_EXPORT AffinityManager {
+ private:
+  static std::atomic<int> _lastDevice;
+  static int _numberOfDevices;
+  static std::mutex _currentMutex;
+  static std::mutex _numberMutex;
+
+ public:
+  static int currentNativeDeviceId();
+  static int currentDeviceId();
+  static int numberOfDevices();
+  static void setCurrentDevice(int deviceId);
+  static void setCurrentNativeDevice(int deviceId);
+};
+}  // namespace sd
+
+#endif  // DEV_TESTS_AFFINITYMANAGER_H

libnd4j/include/execution/BlockingQueue.h

@@ -0,0 +1,54 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SAMEDIFF_BLOCKINGQUEUE_H
+#define SAMEDIFF_BLOCKINGQUEUE_H
+#include <atomic>
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <queue>
+
+namespace samediff {
+template <typename T>
+class BlockingQueue {
+ private:
+  std::queue<T> _queue;
+  std::mutex _lock;
+  std::atomic<int> _size;
+  std::atomic<bool> _available;
+
+  std::condition_variable _condition;
+
+ public:
+  BlockingQueue(int queueSize);
+  ~BlockingQueue() = default;
+  T poll();
+  void put(const T &t);
+
+  bool available();
+  void markAvailable();
+  void markUnavailable();
+};
+}  // namespace samediff
+
+#endif  // DEV_TESTS_BLOCKINGQUEUE_H

libnd4j/include/execution/CallableInterface.h

@@ -0,0 +1,99 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SAMEDIFF_CALLABLEINTERFACE_H
+#define SAMEDIFF_CALLABLEINTERFACE_H
+
+#include <system/common.h>
+
+#include <array>
+#include <atomic>
+#include <condition_variable>
+#include <cstdint>
+#include <functional>
+#include <mutex>
+
+namespace samediff {
+/**
+ * This class is suited for passing functions to execution threads without queues
+ */
+class CallableInterface {
+ private:
+  // parallel_for functions
+  FUNC_1D _function_1d;
+  FUNC_2D _function_2d;
+  FUNC_3D _function_3d;
+
+  // parallel function
+  FUNC_DO _function_do;
+
+  // reduction functions
+  FUNC_RL _function_rl;
+  FUNC_RD _function_rd;
+
+  std::array<int64_t, 9> _arguments;
+
+  volatile int _branch = 0;
+  volatile uint32_t _thread_id = 0;
+  volatile uint32_t _num_threads = 0;
+
+  std::atomic<bool> _finished;
+  std::atomic<bool> _filled;
+  std::atomic<bool> _available;
+
+  std::condition_variable _starter;
+  std::condition_variable _finisher;
+
+  int64_t *_lptr = nullptr;
+  double *_dptr = nullptr;
+
+  std::mutex _ms;
+  std::mutex _mf;
+
+ public:
+  CallableInterface();
+  ~CallableInterface() = default;
+
+  void waitForTask();
+  void waitForCompletion();
+
+  void fill(int thread_id, int num_threads, int64_t *lpt, FUNC_RL func, int64_t start_x, int64_t stop_x, int64_t inc_x);
+  void fill(int thread_id, int num_threads, double *dpt, FUNC_RD func, int64_t start_x, int64_t stop_x, int64_t inc_x);
+
+  void fill(int thread_id, int num_threads, FUNC_DO func);
+  void fill(int thread_id, int num_threads, FUNC_1D func, int64_t start_x, int64_t stop_x, int64_t inc_x);
+  void fill(int thread_id, int num_threads, FUNC_2D func, int64_t start_x, int64_t stop_x, int64_t inc_x,
+            int64_t start_y, int64_t stop_y, int64_t inc_y);
+  void fill(int thread_id, int num_threads, FUNC_3D func, int64_t start_x, int64_t stop_x, int64_t inc_x,
+            int64_t start_y, int64_t stop_y, int64_t inc_y, int64_t start_z, int64_t stop_z, int64_t inc_z);
+
+  bool available();
+  void markAvailable();
+  void markUnavailable();
+
+  void finish();
+
+  void execute();
+};
+}  // namespace samediff
+
+#endif  // DEV_TESTS_CALLABLEINTERFACE_H

libnd4j/include/execution/CallableWithArguments.h

@@ -0,0 +1,95 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef DEV_TESTS_CALLABLEWITHARGUMENTS_H
+#define DEV_TESTS_CALLABLEWITHARGUMENTS_H
+#include <system/op_boilerplate.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <functional>
+#include <vector>
+
+namespace samediff {
+class CallableWithArguments {
+  FUNC_DO _function_do;
+  FUNC_1D _function_1d;
+  FUNC_2D _function_2d;
+  FUNC_3D _function_3d;
+
+  std::vector<int64_t> _arguments;
+
+  std::atomic<bool> _finished;
+
+  std::condition_variable _condition;
+
+  std::mutex _lock;
+
+  int _dimensions = 0;
+
+  uint64_t _threadId;
+  uint64_t _numThreads;
+
+ public:
+  CallableWithArguments(FUNC_DO func, uint64_t thread_id, uint64_t numThreads);
+  CallableWithArguments(FUNC_1D func, uint64_t thread_id, int64_t start_x, int64_t stop_x, int64_t increment_x);
+  CallableWithArguments(FUNC_2D func, uint64_t thread_id, int64_t start_x, int64_t stop_x, int64_t increment_x,
+                        int64_t start_y, int64_t stop_y, int64_t increment_y);
+  CallableWithArguments(FUNC_3D func, uint64_t thread_id, int64_t start_x, int64_t stop_x, int64_t increment_x,
+                        int64_t start_y, int64_t stop_y, int64_t increment_y, int64_t start_z, int64_t stop_z,
+                        int64_t increment_z);
+
+  /**
+   * This method returns number of dimensions
+   * @return
+   */
+  int dimensions();
+
+  /**
+   * This method checks if this callable is finished
+   * @return
+   */
+  bool finished();
+
+  /**
+   * this method marks this Callable as finished
+   */
+  void finish();
+
+  /**
+   * This method blocks until callable is finished
+   */
+  void waitUntilFinished();
+
+  std::vector<int64_t> &arguments();
+  FUNC_DO function_do();
+  FUNC_1D function_1d();
+  FUNC_2D function_2d();
+  FUNC_3D function_3d();
+
+  uint64_t threadId();
+
+  uint64_t numThreads();
+};
+}  // namespace samediff
+
+#endif  // DEV_TESTS_CALLABLEWITHARGUMENTS_H

libnd4j/include/execution/ContextBuffers.h

@@ -0,0 +1,77 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef LIBND4J_CONTEXTBUFFERS_H
+#define LIBND4J_CONTEXTBUFFERS_H
+
+#include <execution/ErrorReference.h>
+#include <system/common.h>
+
+namespace sd {
+class SD_LIB_EXPORT ContextBuffers {
+ private:
+  void *_reductionPointer = nullptr;
+  void *_scalarPointer = nullptr;
+  void *_allocationPointer = nullptr;
+  void *_execStream = nullptr;
+  void *_specialStream = nullptr;
+  ErrorReference _errorReference;
+  bool _allocated = false;
+  bool _initialized = false;
+
+  int _deviceId = -1;
+
+  void initialize();
+
+ public:
+  ContextBuffers();
+  ContextBuffers(const ContextBuffers &other);
+  ContextBuffers(void *rPointer, void *sPointer, void *aPointer, bool isOwner = false);
+  ~ContextBuffers();
+
+  ContextBuffers &operator=(const ContextBuffers &other);
+  ContextBuffers &operator=(ContextBuffers &&other);
+
+  void release();
+
+  void *reductionBuffer();
+  void *scalarBuffer();
+  void *allocationBuffer();
+
+  void *execStream();
+  void *specialStream();
+
+  void setReductionBuffer(void *pointer);
+  void setScalarBuffer(void *pointer);
+  void setAllocationBuffer(void *pointer);
+
+  ErrorReference *errorReference();
+
+  void triggerOwnership(bool isOwner);
+
+  int deviceId();
+
+  bool isInitialized();
+};
+}  // namespace sd
+
+#endif  // DEV_TESTS_CONTEXTBUFFERS_H

libnd4j/include/execution/Engine.h

@@ -0,0 +1,33 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SD_ENGINE_H
+#define SD_ENGINE_H
+
+namespace samediff {
+enum Engine {
+  ENGINE_CPU = 0,
+  ENGINE_CUDA = 1
+};
+}
+
+#endif  // SD_ENGINE_H

libnd4j/include/execution/ErrorReference.h

@@ -0,0 +1,48 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef DEV_TESTS_ERRORREFERENCE_H
+#define DEV_TESTS_ERRORREFERENCE_H
+#include <system/common.h>
+
+#include <string>
+
+namespace sd {
+class SD_LIB_EXPORT ErrorReference {
+ private:
+  int _errorCode = 0;
+  std::string *_errorMessage;
+
+ public:
+  ErrorReference() = default;
+  ~ErrorReference() = default;
+
+  int errorCode();
+  const char *errorMessage();
+
+  void setErrorCode(int errorCode);
+  void setErrorMessage(std::string message);
+  void setErrorMessage(const char *message);
+};
+}  // namespace sd
+
+#endif  // DEV_TESTS_ERRORREFERENCE_H

libnd4j/include/execution/ExecutionMode.h

@@ -0,0 +1,34 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SD_EXECUTIONMODE_H
+#define SD_EXECUTIONMODE_H
+
+namespace samediff {
+enum ExecutionMode {
+  MODE_UNDEFINED = 0,
+  MODE_TRAINING = 1,
+  MODE_INFERENCE = 2,
+};
+}
+
+#endif  // SD_EXECUTIONMODE_H

libnd4j/include/execution/Executor.h

@@ -0,0 +1,35 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SD_EXECUTOR_H
+#define SD_EXECUTOR_H
+
+namespace sd {
+class Executor {
+ public:
+  static void execute() {
+    //
+  }
+};
+}  // namespace sd
+
+#endif  // SD_EXECUTOR_H

libnd4j/include/execution/LaunchContext.h

@@ -0,0 +1,140 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 30.11.17.
+//
+
+#ifndef LIBND4J_CUDACONTEXT_H
+#define LIBND4J_CUDACONTEXT_H
+
+
+
+#ifdef SD_CUDA
+#include <cuda.h>
+#include <cuda_device_runtime_api.h>
+#include <cuda_runtime.h>
+#include <cuda_runtime_api.h>
+
+#include "config.h"
+#endif
+
+// used for MKLDNN etc
+#if !defined(__STANDALONE_BUILD__)
+#include "config.h"
+#endif
+
+#include <execution/ContextBuffers.h>
+#include <execution/ErrorReference.h>
+#include <memory/Workspace.h>
+#include <system/common.h>
+#include <system/op_boilerplate.h>
+
+#include <memory>
+#include <mutex>
+#include <vector>
+
+namespace sd {
+
+class SD_LIB_EXPORT LaunchContext {
+ private:
+  // Previous implementation used heap-allocated pointer which caused crashes during JVM shutdown:
+  // - The vector* was allocated with new and "intentionally leaked"
+  // - But C++ static destruction still tried to destruct it, causing SIGSEGV
+  // - This SIGSEGV triggered signal handler which called abort(), resulting in SIGABRT
+  // Function-local static has well-defined destruction order and avoids this crash
+  static std::vector<LaunchContext*>& contexts();
+  static std::mutex _mutex;
+
+  static SD_MAP_IMPL<int, std::mutex*> _deviceMutexes;
+
+  // used for MKLDNN
+  void* _engine = nullptr;
+
+#ifdef SD_CUDA
+
+#ifndef __JAVACPP_HACK__
+
+  void* _cublasHandle = nullptr;
+  void* _cusolverHandle = nullptr;
+
+#endif  // JCPP
+
+  bool _isAllocated = false;
+#endif  // CUDA
+  memory::Workspace* _workspace = nullptr;
+  int _deviceID = 0;
+
+ public:
+#ifdef SD_CUDA
+
+#ifndef __JAVACPP_HACK__
+  LaunchContext(cudaStream_t* cudaStream, cudaStream_t& specialCudaStream, void* reductionPointer = nullptr,
+                void* scalarPointer = nullptr, int* allocationPointer = nullptr);
+
+  void* getReductionPointer() const;
+  void* getScalarPointer() const;
+  LongType* getAllocationPointer() const;
+  void* getCublasHandle() const;
+  void* getCusolverHandle() const;
+  void* getCuDnnHandle() const;
+  cudaStream_t* getCudaStream() const;
+  cudaStream_t* getCudaSpecialStream() const;
+
+  void setReductionPointer(void* reductionPointer);
+  void setScalarPointer(void* scalarPointer);
+  void setAllocationPointer(int* allocationPointer);
+  void setCudaStream(cudaStream_t* cudaStream);
+  void setCudaSpecialStream(cudaStream_t* cudaStream);
+  void setCublasHandle(void* handle);
+
+#endif  // JCPP
+
+#endif  // CUDA
+  LaunchContext(Pointer cudaStream, Pointer reductionPointer = nullptr, Pointer scalarPointer = nullptr,
+                Pointer allocationPointer = nullptr);
+  LaunchContext();
+  ~LaunchContext();
+  memory::Workspace* getWorkspace() const {
+    return _workspace;
+  }
+  void setWorkspace(memory::Workspace* theWorkspace) { _workspace = theWorkspace; }
+
+  void* engine();
+
+  int getDeviceID() const { return _deviceID; }
+  void setDeviceID(int deviceID) { _deviceID = deviceID; }
+  ErrorReference* errorReference();
+
+#ifndef __JAVACPP_HACK__
+  // this method returns mutex shared between all threads that use the same device
+  static std::mutex* deviceMutex();
+
+#endif
+
+  static bool isInitialized();
+  static void releaseBuffers();
+
+  static LaunchContext* defaultContext();
+
+  static void swapContextBuffers(ContextBuffers& buffers);
+};
+
+}  // namespace sd
+
+#endif  // LIBND4J_CUDACONTEXT_H

libnd4j/include/execution/ThreadPool.h

@@ -0,0 +1,74 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SAMEDIFF_THREADPOOL_H
+#define SAMEDIFF_THREADPOOL_H
+#include <execution/BlockingQueue.h>
+#include <execution/CallableInterface.h>
+#include <execution/CallableWithArguments.h>
+#include <execution/Ticket.h>
+#include <system/common.h>
+
+#include <atomic>
+#include <list>
+#include <mutex>
+#include <queue>
+#include <thread>
+#include <vector>
+
+namespace samediff {
+class SD_LIB_EXPORT ThreadPool {
+ private:
+  std::vector<std::thread> _threads;
+  std::vector<BlockingQueue<CallableWithArguments*>*> _queues;
+  std::vector<CallableInterface*> _interfaces;
+
+  std::mutex _lock;
+  std::atomic<int> _available;
+  std::queue<Ticket*> _tickets;
+
+ protected:
+  ThreadPool();
+  ~ThreadPool();
+
+ public:
+  static ThreadPool& getInstance();
+
+  /**
+   * This method returns list of pointers to threads ONLY if num_threads of threads were available upon request,
+   * returning empty list otherwise
+   * @param num_threads
+   * @return
+   */
+  Ticket* tryAcquire(int num_threads);
+
+  /**
+   * This method marks specified number of threads as released, and available for use
+   * @param num_threads
+   */
+  void release(int num_threads = 1);
+
+  void release(Ticket* ticket);
+};
+}  // namespace samediff
+
+#endif  // DEV_TESTS_THREADPOOL_H

libnd4j/include/execution/Threads.h

@@ -0,0 +1,206 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#ifndef SAMEDIFF_THREADS_H
+#define SAMEDIFF_THREADS_H
+#include <system/Environment.h>
+#include <system/common.h>
+#include <system/op_boilerplate.h>
+#include <system/op_enums.h>
+
+#include <functional>
+
+namespace samediff {
+class SD_LIB_EXPORT ThreadsHelper {
+ public:
+  static int numberOfThreads(int maxThreads, uint64_t numberOfElements);
+  static int numberOfThreads2d(int maxThreads, uint64_t iters_x, uint64_t iters_y);
+  static int numberOfThreads3d(int maxThreads, uint64_t iters_x, uint64_t iters_y, uint64_t iters_z);
+  static int pickLoop2d(int numThreads, uint64_t iters_x, uint64_t iters_y);
+  static int pickLoop3d(int numThreads, uint64_t iters_x, uint64_t iters_y, uint64_t iters_z);
+};
+
+class SD_LIB_EXPORT Span {
+ private:
+  int64_t _startX, _stopX, _incX;
+
+ public:
+  Span(int64_t start_x, int64_t stop_x, int64_t inc_x);
+  ~Span() = default;
+
+  int64_t startX() const;
+  int64_t stopX() const;
+  int64_t incX() const;
+
+  static Span build(uint64_t thread_id, uint64_t num_threads, int64_t start_x, int64_t stop_x, int64_t inc_x);
+};
+
+class SD_LIB_EXPORT Span2 {
+ private:
+  int64_t _startX, _stopX, _incX;
+  int64_t _startY, _stopY, _incY;
+
+ public:
+  Span2(int64_t start_x, int64_t stop_x, int64_t inc_x, int64_t start_y, int64_t stop_y, int64_t inc_y);
+  ~Span2() = default;
+
+  int64_t startX() const;
+  int64_t startY() const;
+
+  int64_t stopX() const;
+  int64_t stopY() const;
+
+  int64_t incX() const;
+  int64_t incY() const;
+
+  static Span2 build(int loop, uint64_t thread_id, uint64_t num_threads, int64_t start_x, int64_t stop_x, int64_t inc_x,
+                     int64_t start_y, int64_t stop_y, int64_t inc_y);
+};
+
+class SD_LIB_EXPORT Span3 {
+ private:
+  int64_t _startX, _stopX, _incX;
+  int64_t _startY, _stopY, _incY;
+  int64_t _startZ, _stopZ, _incZ;
+
+ public:
+  Span3(int64_t start_x, int64_t stop_x, int64_t inc_x, int64_t start_y, int64_t stop_y, int64_t inc_y, int64_t start_z,
+        int64_t stop_z, int64_t inc_z);
+  ~Span3() = default;
+
+  int64_t startX() const;
+  int64_t startY() const;
+  int64_t startZ() const;
+
+  int64_t stopX() const;
+  int64_t stopY() const;
+  int64_t stopZ() const;
+
+  int64_t incX() const;
+  int64_t incY() const;
+  int64_t incZ() const;
+
+  static Span3 build(int loop, uint64_t thread_id, uint64_t num_threads, int64_t start_x, int64_t stop_x, int64_t inc_x,
+                     int64_t start_y, int64_t stop_y, int64_t inc_y, int64_t start_z, int64_t stop_z, int64_t inc_z);
+};
+
+class SD_LIB_EXPORT Threads {
+#ifdef _OPENMP
+ public:
+  static std::mutex gThreadmutex;
+  static uint64_t _nFreeThreads;
+  static bool tryAcquire(int numThreads);
+  static bool freeThreads(int numThreads);
+#endif
+ public:
+  /**
+   * This function executes 1 dimensional loop for a given number of threads
+   * PLEASE NOTE: this function can use smaller number of threads than requested.
+   *
+   * @param function
+   * @param numThreads
+   * @param start
+   * @param stop
+   * @param increment
+   * @return
+   */
+  static int parallel_for(FUNC_1D function, long long int start, long long int stop, long long int increment = 1,
+                          long long int numThreads = sd::Environment::getInstance().maxMasterThreads());
+
+  /**
+   * This function executes 1 dimensional loop for a given number of threads
+   *
+   * @param function
+   * @param start
+   * @param stop
+   * @param increment
+   * @param numThreads
+   * @return
+   */
+  static int parallel_tad(FUNC_1D function, long long int start, long long int stop, long long int increment = 1,
+                          long long int numThreads = sd::Environment::getInstance().maxMasterThreads());
+
+  /**
+   * This method will execute function splitting 2 nested loops space with multiple threads
+   *
+   * @param function
+   * @param numThreads
+   * @param start_x
+   * @param stop_x
+   * @param inc_x
+   * @param start_y
+   * @param stop_y
+   * @param inc_y
+   * @return
+   */
+  static int parallel_for(FUNC_2D function, int64_t start_x, int64_t stop_x, int64_t inc_x, int64_t start_y,
+                          int64_t stop_y, int64_t inc_y,
+                          uint64_t numThreads = sd::Environment::getInstance().maxMasterThreads(), bool debug = false);
+
+  /**
+   * This method will execute function splitting 3 nested loops space with multiple threads
+   *
+   * @param function
+   * @param numThreads
+   * @param start_x
+   * @param stop_x
+   * @param inc_x
+   * @param start_y
+   * @param stop_y
+   * @param inc_y
+   * @param start_z
+   * @param stop_z
+   * @param inc_z
+   * @return
+   */
+  static int parallel_for(FUNC_3D function, int64_t start_x, int64_t stop_x, int64_t inc_x, int64_t start_y,
+                          int64_t stop_y, int64_t inc_y, int64_t start_z, int64_t stop_z, int64_t inc_z,
+                          uint64_t numThreads = sd::Environment::getInstance().maxMasterThreads());
+
+  /**
+   *
+   * @param function
+   * @param numThreads
+   * @return
+   */
+  static int parallel_do(FUNC_DO function,
+                         long long int numThreads = sd::Environment::getInstance().maxMasterThreads());
+
+  static int64_t parallel_long(FUNC_RL function, FUNC_AL aggregator, long long int start, long long int stop,
+                               long long int increment = 1,
+                               long long int numThreads = sd::Environment::getInstance().maxMasterThreads());
+
+  static double parallel_double(FUNC_RD function, FUNC_AD aggregator, int64_t start, int64_t stop,
+                                int64_t increment = 1,
+                                uint64_t numThreads = sd::Environment::getInstance().maxMasterThreads());
+
+  /**
+   * This method will execute function in parallel preserving the parts to be aligned increment size
+   * PLEASE NOTE: this function can use smaller number of threads than requested.
+   *
+   */
+  static int parallel_aligned_increment(FUNC_1D function, int64_t start, int64_t stop, int64_t increment,
+                                        size_t type_size = sizeof(float),
+                                        uint32_t req_numThreads = sd::Environment::getInstance().maxMasterThreads());
+};
+}  // namespace samediff
+
+#endif  // SAMEDIFF_THREADS_H

libnd4j/include/execution/Ticket.h

@@ -0,0 +1,74 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef SAMEDIFF_TICKET_H
+#define SAMEDIFF_TICKET_H
+#include <execution/BlockingQueue.h>
+#include <execution/CallableInterface.h>
+#include <execution/CallableWithArguments.h>
+#include <system/common.h>
+
+#include <atomic>
+#include <mutex>
+#include <vector>
+
+namespace samediff {
+class SD_LIB_EXPORT Ticket {
+ private:
+  bool _acquired = false;
+  std::vector<BlockingQueue<CallableWithArguments *> *> _queues;
+  std::vector<CallableWithArguments *> _callables;
+  std::vector<CallableInterface *> _interfaces;
+
+  uint32_t _acquiredThreads = 0;
+
+ public:
+  explicit Ticket(const std::vector<BlockingQueue<CallableWithArguments *> *> &queues);
+  Ticket();
+  ~Ticket() = default;
+
+  bool acquired();
+
+  void acquiredThreads(uint32_t threads);
+
+  void attach(uint32_t thread_id, CallableInterface *call_interface);
+
+  // deprecated one
+  void enqueue(int thread_id, CallableWithArguments *callable);
+
+  void enqueue(uint32_t thread_id, uint32_t num_threads, int64_t *lpt, FUNC_RL func, int64_t start_x, int64_t stop_x,
+               int64_t inc_x);
+  void enqueue(uint32_t thread_id, uint32_t num_threads, double *lpt, FUNC_RD func, int64_t start_x, int64_t stop_x,
+               int64_t inc_x);
+
+  void enqueue(uint32_t thread_id, uint32_t num_threads, FUNC_DO func);
+  void enqueue(uint32_t thread_id, uint32_t num_threads, FUNC_1D func, int64_t start_x, int64_t stop_x, int64_t inc_x);
+  void enqueue(uint32_t thread_id, uint32_t num_threads, FUNC_2D func, int64_t start_x, int64_t stop_x, int64_t inc_x,
+               int64_t start_y, int64_t stop_y, int64_t inc_y);
+  void enqueue(uint32_t thread_id, uint32_t num_threads, FUNC_3D func, int64_t start_x, int64_t stop_x, int64_t inc_x,
+               int64_t start_y, int64_t stop_y, int64_t inc_y, int64_t start_, int64_t stop_z, int64_t inc_z);
+
+  void waitAndRelease();
+};
+}  // namespace samediff
+
+#endif  // DEV_TESTS_TICKET_H

libnd4j/include/execution/cpu/AffinityManager.cpp

@@ -0,0 +1,38 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <execution/AffinityManager.h>
+
+namespace sd {
+int AffinityManager::currentDeviceId() { return 0; }
+
+int AffinityManager::currentNativeDeviceId() { return 0; }
+
+int AffinityManager::numberOfDevices() { return 1; }
+
+void AffinityManager::setCurrentDevice(int deviceId) {
+  // no-op
+}
+
+void AffinityManager::setCurrentNativeDevice(int deviceId) {
+  // no-op
+}
+}  // namespace sd

libnd4j/include/execution/cpu/ContextBuffers.cpp

@@ -0,0 +1,78 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <execution/AffinityManager.h>
+#include <execution/ContextBuffers.h>
+
+namespace sd {
+ContextBuffers::ContextBuffers() { _deviceId = AffinityManager::currentDeviceId(); }
+
+ContextBuffers::~ContextBuffers() {
+  // no-op
+}
+
+ContextBuffers::ContextBuffers(void* rPointer, void* sPointer, void* aPointer, bool isOwner) {
+  _reductionPointer = rPointer;
+  _scalarPointer = sPointer;
+  _allocationPointer = aPointer;
+  _allocated = isOwner;
+}
+
+ContextBuffers::ContextBuffers(const ContextBuffers& other) {
+  //
+}
+
+void ContextBuffers::initialize() {
+  // no-op
+}
+
+void* ContextBuffers::reductionBuffer() { return _reductionPointer; }
+
+void* ContextBuffers::scalarBuffer() { return _scalarPointer; }
+
+void* ContextBuffers::allocationBuffer() { return _allocationPointer; }
+
+void ContextBuffers::setReductionBuffer(void* pointer) { _reductionPointer = pointer; }
+
+void ContextBuffers::setScalarBuffer(void* pointer) { _scalarPointer = pointer; }
+
+void ContextBuffers::setAllocationBuffer(void* pointer) { _allocationPointer = pointer; }
+
+void ContextBuffers::triggerOwnership(bool isOwner) { _allocated = isOwner; }
+
+int ContextBuffers::deviceId() { return _deviceId; }
+
+void* ContextBuffers::execStream() { return _execStream; }
+
+void* ContextBuffers::specialStream() { return _specialStream; }
+
+bool ContextBuffers::isInitialized() { return true; }
+
+void ContextBuffers::release() {
+  //
+}
+
+ContextBuffers& ContextBuffers::operator=(const ContextBuffers& other) { return *this; }
+
+ContextBuffers& ContextBuffers::operator=(ContextBuffers&& other) { return *this; }
+
+sd::ErrorReference* ContextBuffers::errorReference() { return &_errorReference; }
+}  // namespace sd

libnd4j/include/execution/cpu/LaunchContext.cpp

@@ -0,0 +1,119 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// Created by raver119 on 30.11.17.
+//
+#include <exceptions/cuda_exception.h>
+#include <execution/AffinityManager.h>
+#include <execution/LaunchContext.h>
+#include <helpers/logger.h>
+#include <thread>
+
+// NOTE: Removed thread_local to fix "cannot allocate memory in static TLS block" error
+// when JavaCPP loads library via dlopen(). This error occurs because:
+// 1. dlopen() loads the library at runtime (not at program startup)
+// 2. thread_local variables require space in the static TLS block
+// 3. The static TLS block has limited size and cannot be extended after program start
+// 4. When loaded via dlopen(), the library's TLS requirements must fit in remaining space
+// 5. If sanitizers or lifecycle tracking are enabled, TLS space may already be exhausted
+//
+// iOS/Apple/Android builds already avoid thread_local for similar reasons.
+// For CPU builds, making this a non-thread-local global is acceptable because:
+// - Each LaunchContext instance maintains its own thread-safe state
+// - The global contextBuffers is used as a default/fallback buffer pool
+// - Proper synchronization is handled at the LaunchContext level
+sd::ContextBuffers contextBuffers = sd::ContextBuffers();
+
+#if HAVE_ONEDNN
+#include <dnnl.hpp>
+#endif
+
+namespace sd {
+
+LaunchContext::~LaunchContext() {
+#if HAVE_ONEDNN
+  // Intentionally NOT deleting _engine to avoid use-after-free during shutdown
+  // The LaunchContext objects are kept alive in a function-local static vector (contexts())
+  // to survive JVM shutdown. Deleting _engine here causes crashes because
+  // OneDNN's static cleanup may have already run, making the engine invalid.
+  // This is safe because LaunchContext instances are never destroyed during
+  // normal operation - only during process exit when memory cleanup doesn't matter.
+  // delete reinterpret_cast<dnnl::engine*>(_engine);
+#endif
+}
+
+// Use pointer to prevent static destructor from running.
+// The vector is intentionally leaked to avoid crashes during shutdown when:
+// 1. An exception is thrown that references LaunchContext
+// 2. JVM shuts down and runs static destructors
+// 3. The vector's destructor tries to destroy stored contexts while they're still in use
+// This is safe because LaunchContexts are only created during initialization and
+// process exit cleans up all memory anyway.
+std::vector<LaunchContext*>& LaunchContext::contexts() {
+  static std::vector<LaunchContext*>* _contexts = new std::vector<LaunchContext*>();
+  return *_contexts;
+}
+
+SD_MAP_IMPL<int, std::mutex*> LaunchContext::_deviceMutexes;
+std::mutex LaunchContext::_mutex;
+
+////////////////////////////////////////////////////////////////////////
+LaunchContext::LaunchContext() {
+  // default constructor, just to make clang/ranlib happy
+  _workspace = nullptr;
+  _deviceID = 0;
+#if HAVE_ONEDNN
+  _engine = new dnnl::engine(dnnl::engine::kind::cpu, 0);
+#endif
+}
+
+LaunchContext::LaunchContext(sd::Pointer cudaStream, sd::Pointer reductionPointer, sd::Pointer scalarPointer,
+                             sd::Pointer allocationPointer) {}
+
+static std::mutex _lock;
+
+LaunchContext* LaunchContext::defaultContext() {
+  {
+    // synchronous block goes here
+    std::lock_guard<std::mutex> lock(_lock);
+    // TODO: we need it to be device-aware, but only once we add NUMA support for cpu
+    if (LaunchContext::contexts().empty())
+      LaunchContext::contexts().emplace_back(new LaunchContext());
+  }
+
+  // return context for current device
+  return LaunchContext::contexts().at(0);
+}
+
+std::mutex* LaunchContext::deviceMutex() { return &_mutex; }
+
+void LaunchContext::swapContextBuffers(ContextBuffers& buffers) {
+  //
+}
+
+bool LaunchContext::isInitialized() { return true; }
+
+void LaunchContext::releaseBuffers() {
+  //
+}
+
+sd::ErrorReference* LaunchContext::errorReference() { return contextBuffers.errorReference(); }
+
+void* LaunchContext::engine() { return _engine; }
+}  // namespace sd

libnd4j/include/execution/cuda/AffinityManager.cu

@@ -0,0 +1,119 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <exceptions/cuda_exception.h>
+#include <execution/AffinityManager.h>
+#include <execution/LaunchContext.h>
+#include <helpers/logger.h>
+
+thread_local int globalThreadToDevice = -1;
+
+namespace sd {
+std::mutex AffinityManager::_currentMutex;
+std::mutex AffinityManager::_numberMutex;
+int AffinityManager::_numberOfDevices = -1;
+
+int AffinityManager::currentDeviceId() {
+  // if there's no affinity set - set it now
+  if (globalThreadToDevice < 0) {
+    // this block must be thread-local
+    _currentMutex.lock();
+
+    globalThreadToDevice = _lastDevice++;
+
+    // we need to check if we've got deviceId >= number of actual devices, and reset to zero otherwise
+    if (globalThreadToDevice >= numberOfDevices()) {
+      globalThreadToDevice = 0;
+      _lastDevice = numberOfDevices() > 1 ? 1 : 0;
+    }
+
+    _currentMutex.unlock();
+
+    setCurrentNativeDevice(globalThreadToDevice);
+  }
+
+  // if we already know affinity - just return it
+  if (globalThreadToDevice >= 0) return globalThreadToDevice;
+
+  int dev = 0;
+  auto res = cudaGetDevice(&dev);
+
+  if (res != 0) throw cuda_exception::build("cudaGetDevice failed", res);
+
+  return dev;
+}
+
+int AffinityManager::currentNativeDeviceId() {
+  int dev = 0;
+  auto res = cudaGetDevice(&dev);
+
+  if (res != 0) throw cuda_exception::build("cudaGetDevice failed", res);
+
+  return dev;
+}
+
+int AffinityManager::numberOfDevices() {
+  _numberMutex.lock();
+  // we want to cache number of devices
+  if (_numberOfDevices <= 0) {
+    int dev = 0;
+    auto res = cudaGetDeviceCount(&dev);
+
+    if (res != 0) throw cuda_exception::build("cudaGetDeviceCount failed", res);
+
+    _numberOfDevices = dev;
+  }
+  _numberMutex.unlock();
+
+  return _numberOfDevices;
+}
+
+void AffinityManager::setCurrentNativeDevice(int deviceId) {
+  auto res = cudaSetDevice(deviceId);
+  if (res != 0) throw cuda_exception::build("setCurrentDevice failed", res);
+}
+
+void AffinityManager::setCurrentDevice(int deviceId) {
+  auto previousDeviceId = globalThreadToDevice;
+  if (previousDeviceId >= 0 && LaunchContext::isInitialized()) {
+    auto res = cudaStreamSynchronize(*LaunchContext::defaultContext()->getCudaStream());
+    if (res != 0) throw cuda_exception::build("setCurrentDevice -> sync failed", res);
+
+    res = cudaStreamSynchronize(*LaunchContext::defaultContext()->getCudaSpecialStream());
+    if (res != 0) throw cuda_exception::build("setCurrentDevice -> specialSync failed", res);
+
+    if (deviceId != previousDeviceId) {
+      // discard existing stuff
+      LaunchContext::releaseBuffers();
+    }
+  }
+
+  if (deviceId != previousDeviceId) {
+    auto res = cudaSetDevice(deviceId);
+    if (res != 0) throw cuda_exception::build("cudaSetDevice failed", res);
+  }
+
+  // update thread-device affinity
+  globalThreadToDevice = deviceId;
+}
+
+std::atomic<int> AffinityManager::_lastDevice;  // = std::atomic<int>(initialV);
+}  // namespace sd

libnd4j/include/execution/cuda/ContextBuffers.cu

@@ -0,0 +1,202 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <cuda.h>
+#include <cuda_device_runtime_api.h>
+#include <cuda_runtime.h>
+#include <cuda_runtime_api.h>
+#include <exceptions/cuda_exception.h>
+#include <execution/AffinityManager.h>
+#include <execution/ContextBuffers.h>
+#include <helpers/logger.h>
+
+namespace sd {
+ContextBuffers::ContextBuffers() {
+  _deviceId = AffinityManager::currentDeviceId();
+}
+
+ContextBuffers::ContextBuffers(const ContextBuffers& other) {
+  release();
+
+  this->_initialized = other._initialized;
+  this->_allocated = other._allocated;
+  this->_deviceId = other._deviceId;
+
+  this->_specialStream = other._specialStream;
+  this->_execStream = other._execStream;
+  this->_allocationPointer = other._allocationPointer;
+  this->_reductionPointer = other._reductionPointer;
+  this->_scalarPointer = other._scalarPointer;
+}
+
+ContextBuffers& ContextBuffers::operator=(const ContextBuffers& other) {
+  release();
+
+  this->_initialized = other._initialized;
+  this->_allocated = other._allocated;
+  this->_deviceId = other._deviceId;
+
+  this->_specialStream = other._specialStream;
+  this->_execStream = other._execStream;
+  this->_allocationPointer = other._allocationPointer;
+  this->_reductionPointer = other._reductionPointer;
+  this->_scalarPointer = other._scalarPointer;
+
+  return *this;
+}
+
+ContextBuffers& ContextBuffers::operator=(ContextBuffers&& other) {
+  release();
+
+  this->_initialized = other._initialized;
+  this->_allocated = other._allocated;
+  this->_deviceId = other._deviceId;
+
+  this->_specialStream = other._specialStream;
+  this->_execStream = other._execStream;
+  this->_allocationPointer = other._allocationPointer;
+  this->_reductionPointer = other._reductionPointer;
+  this->_scalarPointer = other._scalarPointer;
+
+  return *this;
+}
+
+void ContextBuffers::release() {
+  if (_allocated) {
+
+    if (_allocationPointer != nullptr) cudaFree(_allocationPointer);
+
+    if (_scalarPointer != nullptr) cudaFreeHost(_scalarPointer);
+
+    if (_allocationPointer != nullptr) cudaFree(_reductionPointer);
+
+    auto _cudaStream = reinterpret_cast<cudaStream_t*>(_execStream);
+    auto _cudaSpecialStream = reinterpret_cast<cudaStream_t*>(_specialStream);
+
+    cudaStreamSynchronize(*_cudaStream);
+    cudaStreamSynchronize(*_cudaSpecialStream);
+
+    cudaStreamDestroy(*_cudaStream);
+    cudaStreamDestroy(*_cudaSpecialStream);
+
+    delete _cudaStream;
+    delete _cudaSpecialStream;
+
+    //////
+    _allocated = false;
+    _deviceId = -1;
+
+    this->_specialStream = nullptr;
+    this->_execStream = nullptr;
+    this->_allocationPointer = nullptr;
+    this->_reductionPointer = nullptr;
+    this->_scalarPointer = nullptr;
+  }
+
+  _initialized = false;
+}
+
+ContextBuffers::~ContextBuffers() { release(); }
+
+ContextBuffers::ContextBuffers(void* rPointer, void* sPointer, void* aPointer, bool isOwner) {
+  _reductionPointer = rPointer;
+  _scalarPointer = sPointer;
+  _allocationPointer = aPointer;
+  _allocated = isOwner;
+}
+
+void ContextBuffers::initialize() {
+  _deviceId = AffinityManager::currentNativeDeviceId();
+
+  auto res = cudaMalloc(reinterpret_cast<void**>(&_reductionPointer), 1024 * 1024 * 8);
+  if (res != 0) throw cuda_exception::build("_reductionPointer allocation failed", res);
+
+  res = cudaHostAlloc(reinterpret_cast<void**>(&_scalarPointer), 16, cudaHostAllocDefault);
+  if (res != 0) throw cuda_exception::build("_scalarPointer allocation failed", res);
+
+  res = cudaMalloc(reinterpret_cast<void**>(&_allocationPointer), 1024 * 1024 * 8);
+  if (res != 0) throw cuda_exception::build("_allocationPointer allocation failed", res);
+
+  _execStream = new cudaStream_t();
+  _specialStream = new cudaStream_t();
+  if (nullptr == _execStream || nullptr == _specialStream)
+    THROW_EXCEPTION("Failed to allocate memory for new CUDA stream");
+
+  res = cudaStreamCreate(reinterpret_cast<cudaStream_t*>(_execStream));
+  if (res != 0) throw cuda_exception::build("Failed to create default CUDA stream with launch context", res);
+
+  res = cudaStreamCreate(reinterpret_cast<cudaStream_t*>(_specialStream));
+  if (res != 0) throw cuda_exception::build("Failed to create special CUDA stream with launch context", res);
+
+  _allocated = true;
+  _initialized = true;
+}
+
+void* ContextBuffers::reductionBuffer() {
+  if (!_initialized) initialize();
+
+  return _reductionPointer;
+}
+
+void* ContextBuffers::scalarBuffer() {
+  if (!_initialized) initialize();
+
+  return _scalarPointer;
+}
+
+void* ContextBuffers::allocationBuffer() {
+  if (!_initialized) initialize();
+
+  return _allocationPointer;
+}
+
+void ContextBuffers::setReductionBuffer(void* pointer) { _reductionPointer = pointer; }
+
+void ContextBuffers::setScalarBuffer(void* pointer) { _scalarPointer = pointer; }
+
+void ContextBuffers::setAllocationBuffer(void* pointer) { _allocationPointer = pointer; }
+
+void ContextBuffers::triggerOwnership(bool isOwner) { _allocated = isOwner; }
+
+int ContextBuffers::deviceId() { return _deviceId; }
+
+void* ContextBuffers::execStream() {
+  if (!_initialized) {
+    initialize();
+  } else {
+  }
+
+  return _execStream;
+}
+
+void* ContextBuffers::specialStream() {
+  if (!_initialized) {
+    initialize();
+  } else {
+  }
+
+  return _specialStream;
+}
+
+bool ContextBuffers::isInitialized() { return _initialized; }
+
+ErrorReference* ContextBuffers::errorReference() { return &_errorReference; }
+}  // namespace sd

libnd4j/include/execution/cuda/DeviceValidator.cu

@@ -0,0 +1,473 @@
+#include "DeviceValidator.h"
+
+ValidationResult::ValidationResult()
+    : isComputeCapabilitySufficient(true),
+      isManagedMemorySupported(true),
+      isComputePreemptionSupported(true),
+      isThreadsPerBlockWithinLimit(true),
+      isBlocksWithinGridSizeLimit(true),
+      isSharedMemoryUsageWithinLimit(true),
+      isRegisterUsageWithinLimit(true),
+      isTotalThreadsWithinLimit(true),
+      isGlobalMemoryUsageWithinLimit(true),
+      isMemoryUsageWithinLimit(true),
+      isLocalMemoryUsageWithinLimit(true),
+      isConcurrentKernelsSupported(true),
+      isL2CacheSizeSufficient(true) {}
+
+
+
+DeviceValidator* DeviceValidator::instance = nullptr;
+std::mutex DeviceValidator::mtx;
+DeviceValidator* DeviceValidator::getInstance(const std::string& directory, int device) {
+  std::lock_guard<std::mutex> lock(mtx);
+  if (instance == nullptr) {
+    instance = new DeviceValidator(directory, device);
+  }
+  return instance;
+}
+
+DeviceValidator::DeviceValidator(const std::string& directoryPath, int device)
+    : directoryPath(directoryPath) {
+  cudaGetDeviceProperties(&prop, device);
+  init();
+}
+
+DeviceValidator::~DeviceValidator() {
+  for (auto& pair : moduleMap) {
+    cuModuleUnload(pair.second);
+  }
+  moduleMap.clear();
+  functionMap.clear();
+}
+
+
+
+std::vector<std::string> DeviceValidator::parsePTXFile(const std::string& filePath) {
+  std::ifstream file(filePath);
+  std::string line;
+  std::vector<std::string> functionNames;
+  std::regex functionPattern("\\.entry\\s+([a-zA-Z_][a-zA-Z0-9_]*)");
+
+  while (std::getline(file, line)) {
+    std::smatch match;
+    if (std::regex_search(line, match, functionPattern) && match.size() > 1) {
+      functionNames.push_back(match.str(1));
+    }
+  }
+
+  return functionNames;
+}
+
+std::vector<std::string> DeviceValidator::parseCUBINFile(const std::string& filePath) {
+  std::string command = "cuobjdump -sass " + filePath + " | grep -oP '(?<=FUNC ).*(?=\\()'";
+  std::array<char, 128> buffer;
+  std::string result;
+  std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(command.c_str(), "r"), pclose);
+  if (!pipe) {
+    throw std::runtime_error("popen() failed!");
+  }
+  while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
+    result += buffer.data();
+  }
+  std::stringstream ss(result);
+  std::string functionName;
+  std::vector<std::string> functionNames;
+  while (std::getline(ss, functionName, '\n')) {
+    functionNames.push_back(functionName);
+  }
+  return functionNames;
+}
+
+void DeviceValidator::init() {
+// Check if cuobjdump is available
+#ifdef _WIN32
+  // Windows
+  if (system("cuobjdump --version > nul 2>&1") != 0) {
+    throw std::runtime_error("cuobjdump is not available on the system's PATH. Please install it and try again.");
+  }
+#else
+  // Linux
+  if (system("cuobjdump --version > /dev/null 2>&1") != 0) {
+    throw std::runtime_error("cuobjdump is not available on the system's PATH. Please install it and try again.");
+  }
+#endif
+
+  printf("Initializing DeviceValidator at path %s\n", directoryPath.c_str());
+  for (const auto &entry : std::filesystem::directory_iterator(directoryPath)) {
+    if (entry.path().extension() == ".ptx" || entry.path().extension() == ".cubin") {
+      CUmodule cuModule;
+      printf("Adding path %s\n",entry.path().filename().c_str());
+      if (cuModuleLoad(&cuModule, entry.path().c_str()) == CUDA_SUCCESS) {
+        moduleMap[entry.path().filename().string()] = cuModule;
+
+        if (entry.path().extension() == ".ptx") {
+          std::vector<std::string> functionNames = parsePTXFile(entry.path().c_str());
+          for (const auto& functionName : functionNames) {
+            CUfunction cuFunction;
+            if (cuModuleGetFunction(&cuFunction, cuModule, functionName.c_str()) == CUDA_SUCCESS) {
+              functionMap[functionName] = cuFunction;
+            }
+          }
+        } else if (entry.path().extension() == ".cubin") {
+          std::vector<std::string> functionNames = parseCUBINFile(entry.path().c_str());
+          for (const auto& functionName : functionNames) {
+            CUfunction cuFunction;
+            if (cuModuleGetFunction(&cuFunction, cuModule, functionName.c_str()) == CUDA_SUCCESS) {
+              functionMap[functionName] = cuFunction;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+void DeviceValidator::setKernelAttribute(const std::string& functionName, CUfunction_attribute attribute, int value) {
+  if (functionMap.find(functionName) != functionMap.end()) {
+    cuFuncSetAttribute(functionMap[functionName], attribute, value);
+  }
+}
+
+void DeviceValidator::setKernelMaxDynamicSharedSizeBytes(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, value);
+}
+
+void DeviceValidator::setKernelPreferredSharedMemoryCarveout(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, value);
+}
+
+void DeviceValidator::setKernelMaxRegisters(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_NUM_REGS, value);
+}
+
+void DeviceValidator::setKernelMaxThreadsPerBlock(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, value);
+}
+
+void DeviceValidator::setKernelNumRegs(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_NUM_REGS, value);
+}
+
+void DeviceValidator::setKernelSharedSizeBytes(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, value);
+}
+
+void DeviceValidator::setKernelBinaryVersion(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_BINARY_VERSION, value);
+}
+
+void DeviceValidator::setKernelCacheModeCA(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_CACHE_MODE_CA, value);
+}
+
+void DeviceValidator::setKernelMaxThreadsPerBlockOptIn(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, value);
+}
+
+void DeviceValidator::setKernelReservedSharedSizeBytes(const std::string& functionName, int value) {
+  setKernelAttribute(functionName, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, value);
+}
+
+
+void DeviceValidator::setAllKernelsAttribute(CUfunction_attribute attribute, int value) {
+  for (auto& pair : functionMap) {
+    cuFuncSetAttribute(pair.second, attribute, value);
+  }
+}
+
+void DeviceValidator::setAllKernelsMaxDynamicSharedSizeBytes(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, value);
+}
+
+void DeviceValidator::setAllKernelsPreferredSharedMemoryCarveout(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, value);
+}
+
+void DeviceValidator::setAllKernelsMaxRegisters(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_NUM_REGS, value);
+}
+
+void DeviceValidator::setAllKernelsMaxThreadsPerBlock(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, value);
+}
+
+void DeviceValidator::setAllKernelsNumRegs(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_NUM_REGS, value);
+}
+
+void DeviceValidator::setAllKernelsSharedSizeBytes(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, value);
+}
+
+void DeviceValidator::setAllKernelsBinaryVersion(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_BINARY_VERSION, value);
+}
+
+void DeviceValidator::setAllKernelsCacheModeCA(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_CACHE_MODE_CA, value);
+}
+
+void DeviceValidator::setAllKernelsMaxThreadsPerBlockOptIn(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, value);
+}
+
+void DeviceValidator::setAllKernelsReservedSharedSizeBytes(int value) {
+  setAllKernelsAttribute(CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, value);
+}
+
+void DeviceValidator::printKernelAttribute(const char* name, CUfunction_attribute attribute) {
+  CUfunction kernel = functionMap[name];
+  int value;
+  cuFuncGetAttribute(&value, attribute, kernel);
+  std::cout << "Attribute " << attribute << " for function " << name << " is " << value << std::endl;
+}
+
+ValidationResult DeviceValidator::validateKernelLaunch(const char* name, dim3 threadsPerBlock, dim3 numBlocks,
+                                                       size_t globalMemoryUsage) {
+  ValidationResult result;
+  CUfunction kernel;
+
+  // Look up the function from the function map
+  auto it = functionMap.find(name);
+  if (it != functionMap.end()) {
+    kernel = it->second;
+  } else {
+    // If the function is not found in the map, return an invalid ValidationResult
+    return result;
+  }
+
+  int sharedSizeBytes, numRegs, maxThreadsPerBlock;
+  cuFuncGetAttribute(&sharedSizeBytes, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, kernel);
+  cuFuncGetAttribute(&numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, kernel);
+  cuFuncGetAttribute(&maxThreadsPerBlock, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, kernel);
+
+  result.sharedSizeBytes = sharedSizeBytes;
+  result.numRegs = numRegs;
+  result.maxThreadsPerBlock = maxThreadsPerBlock;
+
+  result.isComputeCapabilitySufficient = true;
+  result.isECCMemorySupported = prop.ECCEnabled;
+  result.isManagedMemorySupported = prop.managedMemory;
+  result.isComputePreemptionSupported = prop.computePreemptionSupported;
+  result.isConcurrentKernelsSupported = prop.concurrentKernels;
+  result.isL2CacheSizeSufficient = prop.l2CacheSize;
+  result.isThreadsPerBlockWithinLimit = threadsPerBlock.x * threadsPerBlock.y * threadsPerBlock.z <= prop.maxThreadsPerBlock;
+  result.isThreadsPerBlockWithinLimit &= threadsPerBlock.x * threadsPerBlock.y * threadsPerBlock.z <= maxThreadsPerBlock;
+  result.numBlocks = numBlocks.x;
+  result.isBlocksWithinGridSizeLimit = numBlocks.x <= prop.maxGridSize[0] && numBlocks.y <= prop.maxGridSize[1] && numBlocks.z <= prop.maxGridSize[2];
+  result.isSharedMemoryUsageWithinLimit = sharedSizeBytes <= prop.sharedMemPerBlock;
+  result.isRegisterUsageWithinLimit = numRegs * threadsPerBlock.x * threadsPerBlock.y * threadsPerBlock.z <= prop.regsPerBlock;
+  result.numThreads = threadsPerBlock.x * threadsPerBlock.y * threadsPerBlock.z * numBlocks.x * numBlocks.y * numBlocks.z;
+  result.isTotalThreadsWithinLimit = threadsPerBlock.x * threadsPerBlock.y * threadsPerBlock.z * numBlocks.x * numBlocks.y * numBlocks.z <= prop.maxThreadsPerMultiProcessor * prop.multiProcessorCount;
+  result.isGlobalMemoryUsageWithinLimit = globalMemoryUsage <= prop.totalGlobalMem;
+  result.globalMemory = globalMemoryUsage;
+  size_t freeMemory, totalMemory;
+  cudaMemGetInfo(&freeMemory, &totalMemory);
+  size_t usedMemory = totalMemory - freeMemory;
+  result.memoryUsage = usedMemory + globalMemoryUsage;
+  result.isMemoryUsageWithinLimit = result.memoryUsage <= totalMemory;
+  result.isLocalMemoryUsageWithinLimit = usedMemory + globalMemoryUsage <= prop.localL1CacheSupported ? totalMemory : prop.localL1CacheSupported;
+  result.freeMemory = freeMemory;
+  result.totalMemory = totalMemory;
+
+  return result;
+}
+
+void DeviceValidator::printProblematicFunctions(dim3 threadsPerBlock, dim3 numBlocks, size_t globalMemory) {
+  printf("Attempting to print functions with size of map %d\n",functionMap.size());
+  for (const auto& pair : functionMap) {
+    const std::string functionName = pair.first;
+    CUfunction function = pair.second;
+
+    ValidationResult validationResult = validateKernelLaunch(functionName.c_str(),threadsPerBlock, numBlocks, globalMemory);
+
+    if (!validationResult.isValid()) {
+      std::cout << "Function " << functionName << " has the following problems:\n";
+      printValidationResult(functionName.c_str(),validationResult);
+      std::cout << std::endl;
+    }
+  }
+}
+
+bool ValidationResult::isValid()  {
+  return isComputeCapabilitySufficient &&
+         isManagedMemorySupported &&
+         isComputePreemptionSupported &&
+         isThreadsPerBlockWithinLimit &&
+         isBlocksWithinGridSizeLimit &&
+         isSharedMemoryUsageWithinLimit &&
+         isRegisterUsageWithinLimit &&
+         isTotalThreadsWithinLimit &&
+         isGlobalMemoryUsageWithinLimit &&
+         isMemoryUsageWithinLimit &&
+         isLocalMemoryUsageWithinLimit &&
+         isConcurrentKernelsSupported &&
+         isL2CacheSizeSufficient;
+}
+
+void DeviceValidator::printValidationResult(const char* name, ValidationResult& result) {
+  printf("Validating: %s\n",name);
+  if (!result.isValid()) {
+    std::cout << "Function " << name << " has an issue:\n";
+    if (!result.isComputeCapabilitySufficient) {
+      std::cout << " - Compute capability is not sufficient. Required: " << result.isComputeCapabilitySufficient << ", Actual: " << prop.major * 10 + prop.minor << "\n";
+    }
+
+    if (!result.isManagedMemorySupported) {
+      std::cout << " - Managed memory is not supported.\n";
+    }
+    if (!result.isComputePreemptionSupported) {
+      std::cout << " - Compute preemption is not supported.\n";
+    }
+    if (!result.isThreadsPerBlockWithinLimit) {
+      std::cout << " - Threads per block is not within limit. Max: " << prop.maxThreadsPerBlock << "\n";
+      std::cout << " - Value is: " <<  result.maxThreadsPerBlock << "\n";
+
+    }
+    if (!result.isBlocksWithinGridSizeLimit) {
+      //  result.isRegisterUsageWithinLimit = numRegs * threadsPerBlock.x * threadsPerBlock.y * threadsPerBlock.z <= prop.regsPerBlock;
+      std::cout << " - Blocks within grid size is not within limit. Max: (" << prop.maxGridSize[0] << ", " << prop.maxGridSize[1] << ", " << prop.maxGridSize[2] << ")\n";
+      std::cout << " - Value is: (" << result.numBlocks << ", " << result.numBlocks << ", " << result.numBlocks << ")\n";
+    }
+    if (!result.isSharedMemoryUsageWithinLimit) {
+      std::cout << " - Shared memory usage is not within limit. Max: " << prop.sharedMemPerBlock << "\n";
+    }
+    if (!result.isRegisterUsageWithinLimit) {
+      std::cout << " - Register usage is not within limit. Max: " << prop.regsPerBlock << "\n";
+      std::cout << " - Value is: (" << result.numRegs << ", " << result.numBlocks << "\n";
+    }
+    if (!result.isTotalThreadsWithinLimit) {
+      std::cout << " - Total threads is not within limit. Max: " << prop.maxThreadsPerMultiProcessor *  prop.multiProcessorCount << "\n";
+      std::cout << " - Value is: " << result.numThreads << "\n";
+    }
+    if (!result.isGlobalMemoryUsageWithinLimit) {
+      std::cout << " - Global memory usage is not within limit. Max: " << prop.totalGlobalMem << "\n";
+      std::cout << " - Value is: " << result.globalMemory << "\n";
+    }
+    if (!result.isMemoryUsageWithinLimit) {
+      std::cout << " - Memory usage is not within limit.\n";
+      std::cout << " - Value is: " << result.memoryUsage << "\n";
+    }
+    if (!result.isLocalMemoryUsageWithinLimit) {
+      std::cout << " - Local memory usage is not within limit. Max: " << (prop.localL1CacheSupported ? prop.localL1CacheSupported : prop.totalGlobalMem) << "\n";
+    }
+    if (!result.isConcurrentKernelsSupported) {
+      std::cout << " - Concurrent kernels is not supported.\n";
+    }
+    if (!result.isL2CacheSizeSufficient) {
+      std::cout << " - L2 cache size is not sufficient. Max: " << prop.l2CacheSize << "\n";
+    }
+  }
+}
+
+void DeviceValidator::printMaxKernelAttributes() {
+  std::cout << "Device name: " << prop.name << std::endl;
+  std::cout << "Total global memory: " << prop.totalGlobalMem << std::endl;
+  std::cout << "Shared memory per block: " << prop.sharedMemPerBlock << std::endl;
+  std::cout << "Registers per block: " << prop.regsPerBlock << std::endl;
+  std::cout << "Warp size: " << prop.warpSize << std::endl;
+  std::cout << "Max pitch: " << prop.memPitch << std::endl;
+  std::cout << "Max threads per block: " << prop.maxThreadsPerBlock << std::endl;
+  std::cout << "Max thread dimensions: (" << prop.maxThreadsDim[0] << ", " << prop.maxThreadsDim[1] << ", " << prop.maxThreadsDim[2] << ")" << std::endl;
+  std::cout << "Max grid dimensions: (" << prop.maxGridSize[0] << ", " << prop.maxGridSize[1] << ", " << prop.maxGridSize[2] << ")" << std::endl;
+  std::cout << "Clock rate: " << prop.clockRate << std::endl;
+  std::cout << "Total constant memory: " << prop.totalConstMem << std::endl;
+  std::cout << "Compute capability: " << prop.major << "." << prop.minor << std::endl;
+  std::cout << "Texture alignment: " << prop.textureAlignment << std::endl;
+  std::cout << "Concurrent copy and execution: " << (prop.deviceOverlap ? "Yes" : "No") << std::endl;
+  std::cout << "Number of multiprocessors: " << prop.multiProcessorCount << std::endl;
+  std::cout << "Kernel execution timeout: " << (prop.kernelExecTimeoutEnabled ? "Yes" : "No") << std::endl;
+  std::cout << "Integrated: " << (prop.integrated ? "Yes" : "No") << std::endl;
+  std::cout << "Can map host memory: " << (prop.canMapHostMemory ? "Yes" : "No") << std::endl;
+  std::cout << "Compute mode: " << (prop.computeMode == cudaComputeModeDefault ? "Default" : (prop.computeMode == cudaComputeModeExclusive ? "Exclusive" : (prop.computeMode == cudaComputeModeProhibited ? "Prohibited" : "Exclusive Process"))) << std::endl;
+  std::cout << "Max texture 1D: " << prop.maxTexture1D << std::endl;
+  std::cout << "Max texture 2D: (" << prop.maxTexture2D[0] << ", " << prop.maxTexture2D[1] << ")" << std::endl;
+  std::cout << "Max texture 3D: (" << prop.maxTexture3D[0] << ", " << prop.maxTexture3D[1] << ", " << prop.maxTexture3D[2] << ")" << std::endl;
+  std::cout << "Concurrent kernels: " << (prop.concurrentKernels ? "Yes" : "No") << std::endl;
+  std::cout << "ECC enabled: " << (prop.ECCEnabled ? "Yes" : "No") << std::endl;
+  std::cout << "PCI bus ID: " << prop.pciBusID << std::endl;
+  std::cout << "PCI device ID: " << prop.pciDeviceID << std::endl;
+  std::cout << "TCC driver: " << (prop.tccDriver ? "Yes" : "No") << std::endl;
+  std::cout << "Memory clock rate: " << prop.memoryClockRate << std::endl;
+  std::cout << "Global memory bus width: " << prop.memoryBusWidth << std::endl;
+  std::cout << "L2 cache size: " << prop.l2CacheSize << std::endl;
+  std::cout << "Max threads per multiprocessor: " << prop.maxThreadsPerMultiProcessor << std::endl;
+  std::cout << "Stream priorities supported: " << (prop.streamPrioritiesSupported ? "Yes" : "No") << std::endl;
+  std::cout << "Global L1 cache supported: " << (prop.globalL1CacheSupported ? "Yes" : "No") << std::endl;
+  std::cout << "Local L1 cache supported: " << (prop.localL1CacheSupported ? "Yes" : "No") << std::endl;
+  std::cout << "Shared memory per multiprocessor: " << prop.sharedMemPerMultiprocessor << std::endl;
+  std::cout << "Registers per multiprocessor: " << prop.regsPerMultiprocessor << std::endl;
+  std::cout << "Managed memory concurrent: " << (prop.concurrentManagedAccess ? "Yes" : "No") << std::endl;
+  std::cout << "Is multi GPU board: " << (prop.isMultiGpuBoard ? "Yes" : "No") << std::endl;
+  std::cout << "Multi GPU board group ID: " << prop.multiGpuBoardGroupID << std::endl;
+  std::cout << "Host native atomic supported: " << (prop.hostNativeAtomicSupported ? "Yes" : "No") << std::endl;
+  std::cout << "Single to double precision perf ratio: " << prop.singleToDoublePrecisionPerfRatio << std::endl;
+  std::cout << "Pageable memory access: " << (prop.pageableMemoryAccess ? "Yes" : "No") << std::endl;
+  std::cout << "Concurrent managed access: " << (prop.concurrentManagedAccess ? "Yes" : "No") << std::endl;
+}
+
+
+void DeviceValidator::printKernelAttributes(const char* name) {
+  auto it = functionMap.find(name);
+  if (it == functionMap.end()) {
+    std::cerr << "Error: Function " << name << " not found." << std::endl;
+    return;
+  }
+  CUfunction kernel = it->second;
+
+  int sharedSizeBytes, numRegs, maxThreadsPerBlock, binaryVersion, cacheModeCA, maxDynamicSharedSizeBytes, preferredSharedMemoryCarveout;
+  cuFuncGetAttribute(&sharedSizeBytes, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, kernel);
+  cuFuncGetAttribute(&numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, kernel);
+  cuFuncGetAttribute(&maxThreadsPerBlock, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, kernel);
+  cuFuncGetAttribute(&binaryVersion, CU_FUNC_ATTRIBUTE_BINARY_VERSION, kernel);
+  cuFuncGetAttribute(&cacheModeCA, CU_FUNC_ATTRIBUTE_CACHE_MODE_CA, kernel);
+  cuFuncGetAttribute(&maxDynamicSharedSizeBytes, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, kernel);
+  cuFuncGetAttribute(&preferredSharedMemoryCarveout, CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, kernel);
+
+  std::cout << "Kernel name: " << name << std::endl;
+  std::cout << "Shared memory per block: " << sharedSizeBytes << std::endl;
+  std::cout << "Registers per block: " << numRegs << std::endl;
+  std::cout << "Max threads per block: " << maxThreadsPerBlock << std::endl;
+  std::cout << "Binary version: " << binaryVersion << std::endl;
+  std::cout << "Cache mode CA: " << cacheModeCA << std::endl;
+  std::cout << "Max dynamic shared size bytes: " << maxDynamicSharedSizeBytes << std::endl;
+  std::cout << "Preferred shared memory carveout: " << preferredSharedMemoryCarveout << std::endl;
+
+  if (sharedSizeBytes > prop.sharedMemPerBlock) {
+    std::cout << "WARNING: The kernel uses more shared memory per block than the device supports." << std::endl;
+  }
+  if (numRegs > prop.regsPerBlock) {
+    std::cout << "WARNING: The kernel uses more registers per block than the device supports." << std::endl;
+  }
+  if (maxThreadsPerBlock > prop.maxThreadsPerBlock) {
+    std::cout << "WARNING: The kernel uses more threads per block than the device supports." << std::endl;
+  }
+  // Note: There are no device properties to compare with binaryVersion, cacheModeCA, maxDynamicSharedSizeBytes, and preferredSharedMemoryCarveout.
+}
+
+
+std::map<std::string, ValidationResult> DeviceValidator::collectResourceProblems() {
+  std::map<std::string, ValidationResult> problematicFunctions;
+  for (const auto& pair : functionMap) {
+    const char* name = pair.first.c_str();
+    ValidationResult result = validateKernelLaunch(name, dim3(1, 1, 1), dim3(1, 1, 1), 0);
+    if (!result.isComputeCapabilitySufficient ||
+        !result.isECCMemorySupported ||
+        !result.isManagedMemorySupported ||
+        !result.isComputePreemptionSupported ||
+        !result.isThreadsPerBlockWithinLimit ||
+        !result.isBlocksWithinGridSizeLimit ||
+        !result.isSharedMemoryUsageWithinLimit ||
+        !result.isRegisterUsageWithinLimit ||
+        !result.isTotalThreadsWithinLimit ||
+        !result.isGlobalMemoryUsageWithinLimit ||
+        !result.isMemoryUsageWithinLimit ||
+        !result.isLocalMemoryUsageWithinLimit ||
+        !result.isConcurrentKernelsSupported ||
+        !result.isL2CacheSizeSufficient) {
+      problematicFunctions[name] = result;
+    }
+  }
+  return problematicFunctions;
+}

libnd4j/include/execution/cuda/DeviceValidator.h

@@ -0,0 +1,107 @@
+#include <cuda.h>
+
+#include <algorithm>
+#include <filesystem>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <unordered_map>
+#include <fstream>
+#include <regex>
+#include <string>
+#include <vector>
+
+class ValidationResult {
+ public:
+  bool isComputeCapabilitySufficient;
+  bool isECCMemorySupported;
+  bool isManagedMemorySupported;
+  bool isComputePreemptionSupported;
+  bool isThreadsPerBlockWithinLimit;
+  bool isBlocksWithinGridSizeLimit;
+  bool isSharedMemoryUsageWithinLimit;
+  bool isRegisterUsageWithinLimit;
+  bool isTotalThreadsWithinLimit;
+  bool isGlobalMemoryUsageWithinLimit;
+  bool isMemoryUsageWithinLimit;
+  bool isLocalMemoryUsageWithinLimit;
+  bool isConcurrentKernelsSupported;
+  bool isL2CacheSizeSufficient;
+  int sharedSizeBytes, numRegs, maxThreadsPerBlock;
+  size_t freeMemory, totalMemory;
+  int numBlocks;
+  int globalMemory;
+  int numThreads;
+  int memoryUsage;
+  bool isValid();
+
+  ValidationResult();
+};
+
+class DeviceValidator {
+ private:
+  cudaDeviceProp prop;
+  static std::mutex mtx;
+  std::unordered_map<std::string, CUmodule> moduleMap;
+  std::unordered_map<std::string, CUfunction> functionMap;
+  std::string directoryPath;
+  static DeviceValidator* instance;
+
+  void init();
+
+ public:
+  DeviceValidator(const std::string& directoryPath, int device = 0);
+  ~DeviceValidator();
+  static DeviceValidator* getInstance(const std::string& directory, int device = 0);
+
+  // Set kernel attribute
+  void setKernelAttribute(const std::string& functionName, CUfunction_attribute attribute, int value);
+
+  void setKernelMaxDynamicSharedSizeBytes(const std::string& functionName, int value);
+
+  void setKernelPreferredSharedMemoryCarveout(const std::string& functionName, int value);
+
+  void setKernelMaxRegisters(const std::string& functionName, int value);
+
+  void setKernelMaxThreadsPerBlock(const std::string& functionName, int value);
+
+  void setKernelNumRegs(const std::string& functionName, int value);
+
+  void setKernelSharedSizeBytes(const std::string& functionName, int value);
+
+  void setKernelBinaryVersion(const std::string& functionName, int value);
+
+  void setKernelCacheModeCA(const std::string& functionName, int value);
+
+  void setKernelMaxThreadsPerBlockOptIn(const std::string& functionName, int value);
+
+  void setKernelReservedSharedSizeBytes(const std::string& functionName, int value);
+
+  void setAllKernelsAttribute(CUfunction_attribute attribute, int value);
+
+
+  void setAllKernelsMaxDynamicSharedSizeBytes(int value);
+  void setAllKernelsPreferredSharedMemoryCarveout(int value);
+  void setAllKernelsMaxRegisters(int value);
+  void setAllKernelsMaxThreadsPerBlock(int value);
+  void setAllKernelsNumRegs(int value);
+  void setAllKernelsSharedSizeBytes(int value);
+  void setAllKernelsBinaryVersion(int value);
+  void setAllKernelsCacheModeCA(int value);
+  void setAllKernelsMaxThreadsPerBlockOptIn(int value);
+  void setAllKernelsReservedSharedSizeBytes(int value);
+
+  void printKernelAttribute(const char* name, CUfunction_attribute attribute);
+  void printMaxKernelAttributes();
+  void printKernelAttributes(const char* name);
+  std::map<std::string, ValidationResult> collectResourceProblems();
+  void printValidationResult(const char* name, ValidationResult& result);
+  ValidationResult validateKernelLaunch(const char* name, dim3 threadsPerBlock, dim3 numBlocks,
+                                        size_t globalMemoryUsage);
+
+  void printProblematicFunctions(dim3 threadsPerBlock, dim3 numBlocks, size_t globalMemory);
+
+  std::vector<std::string> parseCUBINFile(const std::string& filePath);
+  std::vector<std::string> parsePTXFile(const std::string& filePath);
+};
+

libnd4j/include/execution/cuda/LaunchContext.cu

@@ -0,0 +1,160 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <exceptions/cuda_exception.h>
+#include <execution/AffinityManager.h>
+#include <execution/LaunchContext.h>
+#include <helpers/cublasHelper.h>
+#include <helpers/logger.h>
+
+#include <thread>
+
+thread_local sd::ContextBuffers contextBuffers = sd::ContextBuffers();
+
+namespace sd {
+
+// This avoids static destruction order crashes during JVM shutdown
+std::vector<LaunchContext*>& LaunchContext::contexts() {
+  static std::vector<LaunchContext*>* _contexts = new std::vector<LaunchContext*>();
+  return *_contexts;
+}
+
+std::mutex LaunchContext::_mutex;
+SD_MAP_IMPL<int, std::mutex*> LaunchContext::_deviceMutexes;
+
+////////////////////////////////////////////////////////////////////////
+LaunchContext::LaunchContext(cudaStream_t* cudaStream, cudaStream_t& specialCudaStream, void* reductionPointer,
+                             void* scalarPointer, int* allocationPointer) {
+
+  _workspace = nullptr;
+  _isAllocated = false;
+}
+
+std::mutex* LaunchContext::deviceMutex() {
+  auto deviceId = AffinityManager::currentDeviceId();
+  return _deviceMutexes[deviceId];
+}
+
+LaunchContext::~LaunchContext() {
+  if (_isAllocated) {
+  }
+}
+
+////////////////////////////////////////////////////////////////////////
+LaunchContext::LaunchContext() {
+  // default constructor, just to make clang/ranlib happy
+  _workspace = nullptr;
+  _deviceID = 0;
+
+  _isAllocated = true;
+}
+
+LaunchContext::LaunchContext(Pointer cudaStream, Pointer reductionPointer, Pointer scalarPointer,
+                             Pointer allocationPointer) {
+  _isAllocated = false;
+
+}
+
+LaunchContext* LaunchContext::defaultContext() {
+  /**
+   * This method returns LaunchContext, that has multiple entities within:
+   * 1) temporary buffers. they must be per-thread
+   * 2) CUDA stream. it must be either per-thread or per-device
+   * 3) cuBLAS handle. it must be per-device
+   */
+  auto deviceId = AffinityManager::currentDeviceId();
+
+  {
+    // we need this block synchronous, to avoid double initialization etc
+    std::lock_guard<std::mutex> lock(_mutex);
+    if (contexts().empty()) {
+      // create one context per device
+      auto numDevices = AffinityManager::numberOfDevices();
+
+      contexts().resize(numDevices);
+      for (int e = 0; e < numDevices; e++) {
+        _deviceMutexes[e] = new std::mutex();
+
+        AffinityManager::setCurrentNativeDevice(e);
+
+        contexts().at(e) = new LaunchContext();
+      }
+
+      // don't forget to restore device back again
+      AffinityManager::setCurrentNativeDevice(deviceId);
+    }
+  }
+
+  // return context for current device
+  return contexts().at(deviceId);
+}
+
+void* LaunchContext::getReductionPointer() const { return contextBuffers.reductionBuffer(); };
+
+void* LaunchContext::getScalarPointer() const { return contextBuffers.scalarBuffer(); };
+
+LongType* LaunchContext::getAllocationPointer() const { return reinterpret_cast<LongType*>(contextBuffers.allocationBuffer()); };
+
+void* LaunchContext::getCublasHandle() const { return CublasHelper::getInstance().handle(); };
+
+void* LaunchContext::getCusolverHandle() const { return CublasHelper::getInstance().solver(); };
+
+cudaStream_t* LaunchContext::getCudaStream() const {
+  return reinterpret_cast<cudaStream_t*>(contextBuffers.execStream());
+};
+
+cudaStream_t* LaunchContext::getCudaSpecialStream() const {
+  return reinterpret_cast<cudaStream_t*>(contextBuffers.specialStream());
+  ;
+};
+
+void LaunchContext::setReductionPointer(void* reductionPointer) {
+  contextBuffers.setReductionBuffer(reductionPointer);
+};
+
+void LaunchContext::setScalarPointer(void* scalarPointer) { contextBuffers.setScalarBuffer(scalarPointer); };
+
+void LaunchContext::setAllocationPointer(int* allocationPointer) {
+  contextBuffers.setAllocationBuffer(allocationPointer);
+};
+
+void LaunchContext::setCudaStream(cudaStream_t* cudaStream){
+};
+
+void LaunchContext::setCudaSpecialStream(cudaStream_t* cudaStream){
+};
+
+void LaunchContext::setCublasHandle(void* handle) { _cublasHandle = handle; };
+
+void LaunchContext::swapContextBuffers(ContextBuffers& buffers) { contextBuffers = buffers; };
+
+void LaunchContext::releaseBuffers() {
+  contextBuffers.release();
+}
+
+bool LaunchContext::isInitialized() { return contextBuffers.isInitialized(); }
+
+void* LaunchContext::getCuDnnHandle() const { return CublasHelper::getInstance().cudnn(); }
+
+ErrorReference* LaunchContext::errorReference() { return contextBuffers.errorReference(); }
+
+void* LaunchContext::engine() { return _engine; }
+}  // namespace sd

libnd4j/include/execution/cuda/LaunchDims.h

@@ -0,0 +1,867 @@
+#if !defined(LAUNCH_DIMS_H)
+#pragma  once
+#include <cstdlib>
+#include <unordered_map>
+#include <string>
+#include <cuda_runtime.h>
+#include <map>
+#include <helpers/CudaLaunchHelper.h>
+#include <array/NDArray.h>
+// Retrieve the environment variable value for the given variable name
+int getEnvVariable(const std::string& varName, int defaultValue);
+
+// Grid, block, and shared memory size definitions using environment variables
+#define GRID_SIZE_ADJUST_WEIGHTS getEnvVariable("GRID_SIZE_ADJUST_WEIGHTS", 256)
+#define BLOCK_SIZE_ADJUST_WEIGHTS getEnvVariable("BLOCK_SIZE_ADJUST_WEIGHTS", 512)
+#define SHARED_MEM_SIZE_ADJUST_WEIGHTS getEnvVariable("SHARED_MEM_SIZE_ADJUST_WEIGHTS", 8192)
+
+//note we only use the shared memory for this op the rest are array specific
+#define GRID_SIZE_SEQUENCE_MASK getEnvVariable("GRID_SIZE_SEQUENCE_MASK", 256)
+#define BLOCK_SIZE_SEQUENCE_MASK getEnvVariable("BLOCK_SIZE_SEQUENCE_MASK", 256)
+#define SHARED_MEM_SIZE_SEQUENCE_MASK getEnvVariable("SHARED_MEM_SIZE_SEQUENCE_MASK", 128)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_SEGMENT_SUM getEnvVariable("GRID_SIZE_SEGMENT_SUM", 256)
+#define BLOCK_SIZE_SEGMENT_SUM getEnvVariable("BLOCK_SIZE_SEGMENT_SUM", 256)
+#define SHARED_MEM_SIZE_SEGMENT_SUM getEnvVariable("SHARED_MEM_SIZE_SEGMENT_SUM", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_UNSORTED_SEGMENT_SUM getEnvVariable("GRID_SIZE_UNSORTED_SEGMENT_SUM", 256)
+#define BLOCK_SIZE_UNSORTED_SEGMENT_SUM getEnvVariable("BLOCK_SIZE_UNSORTED_SEGMENT_SUM", 256)
+#define SHARED_MEM_SIZE_UNSORTED_SEGMENT_SUM getEnvVariable("SHARED_MEM_SIZE_UNSORTED_SEGMENT_SUM", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_SEGMENT_SQRTN getEnvVariable("GRID_SIZE_SEGMENT_SQRTN", 128)
+#define BLOCK_SIZE_SEGMENT_SQRTN getEnvVariable("BLOCK_SIZE_SEGMENT_SQRTN", 256)
+#define SHARED_MEM_SIZE_SEGMENT_SQRTN getEnvVariable("SHARED_MEM_SIZE_SEGMENT_SQRTN", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_SEGMENT_PROD getEnvVariable("GRID_SIZE_SEGMENT_PROD", 256)
+#define BLOCK_SIZE_SEGMENT_PROD getEnvVariable("BLOCK_SIZE_SEGMENT_PROD", 256)
+#define SHARED_MEM_SIZE_SEGMENT_PROD getEnvVariable("SHARED_MEM_SIZE_SEGMENT_PROD", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_UNSORTED_SEGMENT_PROD getEnvVariable("GRID_SIZE_UNSORTED_SEGMENT_PROD", 256)
+#define BLOCK_SIZE_UNSORTED_SEGMENT_PROD getEnvVariable("BLOCK_SIZE_UNSORTED_SEGMENT_PROD", 256)
+#define SHARED_MEM_SIZE_UNSORTED_SEGMENT_PROD getEnvVariable("SHARED_MEM_SIZE_UNSORTED_SEGMENT_PROD", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_SEGMENT_MIN getEnvVariable("GRID_SIZE_SEGMENT_MIN", 256)
+#define BLOCK_SIZE_SEGMENT_MIN getEnvVariable("BLOCK_SIZE_SEGMENT_MIN", 256)
+#define SHARED_MEM_SIZE_SEGMENT_MIN getEnvVariable("SHARED_MEM_SIZE_SEGMENT_MIN", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_UNSORTED_SEGMENT_MIN getEnvVariable("GRID_SIZE_UNSORTED_SEGMENT_MIN", 256)
+#define BLOCK_SIZE_UNSORTED_SEGMENT_MIN getEnvVariable("BLOCK_SIZE_UNSORTED_SEGMENT_MIN", 256)
+#define SHARED_MEM_SIZE_UNSORTED_SEGMENT_MIN getEnvVariable("SHARED_MEM_SIZE_UNSORTED_SEGMENT_MIN", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_SEGMENT_MEAN getEnvVariable("GRID_SIZE_SEGMENT_MEAN", 256)
+#define BLOCK_SIZE_SEGMENT_MEAN getEnvVariable("BLOCK_SIZE_SEGMENT_MEAN", 256)
+#define SHARED_MEM_SIZE_SEGMENT_MEAN getEnvVariable("SHARED_MEM_SIZE_SEGMENT_MEAN", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_UNSORTED_SEGMENT_MEAN getEnvVariable("GRID_SIZE_UNSORTED_SEGMENT_MEAN", 256)
+#define BLOCK_SIZE_UNSORTED_SEGMENT_MEAN getEnvVariable("BLOCK_SIZE_UNSORTED_SEGMENT_MEAN", 256)
+#define SHARED_MEM_SIZE_UNSORTED_SEGMENT_MEAN getEnvVariable("SHARED_MEM_SIZE_UNSORTED_SEGMENT_MEAN", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_SEGMENT_MAX getEnvVariable("GRID_SIZE_SEGMENT_MAX", 256)
+#define BLOCK_SIZE_SEGMENT_MAX getEnvVariable("BLOCK_SIZE_SEGMENT_MAX", 512)
+#define SHARED_MEM_SIZE_SEGMENT_MAX getEnvVariable("SHARED_MEM_SIZE_SEGMENT_MAX", 256)
+
+//not we don't really use this due to defaults being array specific for this op
+#define GRID_SIZE_UNSORTED_SEGMENT_MAX getEnvVariable("GRID_SIZE_UNSORTED_SEGMENT_MAX", 256)
+#define BLOCK_SIZE_UNSORTED_SEGMENT_MAX getEnvVariable("BLOCK_SIZE_UNSORTED_SEGMENT_MAX", 256)
+#define SHARED_MEM_SIZE_UNSORTED_SEGMENT_MAX getEnvVariable("SHARED_MEM_SIZE_UNSORTED_SEGMENT_MAX", 256)
+
+#define GRID_SIZE_MATRIX_DIAG getEnvVariable("GRID_SIZE_MATRIX_DIAG", 256)
+#define BLOCK_SIZE_MATRIX_DIAG getEnvVariable("BLOCK_SIZE_MATRIX_DIAG", 512)
+#define SHARED_MEM_SIZE_MATRIX_DIAG getEnvVariable("SHARED_MEM_SIZE_MATRIX_DIAG", 8192)
+
+//not we don't really use this due to defaults being array specific for this op (see fillUpSegments_)
+#define GRID_SIZE_SEGMENT_FILL_UP_SEGMENTS getEnvVariable("GRID_SIZE_SEGMENT_FILL_UP_SEGMENTS", 256)
+#define BLOCK_SIZE_SEGMENT_FILL_UP_SEGMENTS getEnvVariable("BLOCK_SIZE_SEGMENT_FILL_UP_SEGMENTS", 256)
+#define SHARED_MEM_SIZE_SEGMENT_FILL_UP_SEGMENTS getEnvVariable("SHARED_MEM_SIZE_SEGMENT_FILL_UP_SEGMENTS", 256)
+
+#define GRID_SIZE_MATRIX_BAND getEnvVariable("GRID_SIZE_MATRIX_BAND", 256)
+#define BLOCK_SIZE_MATRIX_BAND getEnvVariable("BLOCK_SIZE_MATRIX_BAND", 512)
+#define SHARED_MEM_SIZE_MATRIX_BAND getEnvVariable("SHARED_MEM_SIZE_MATRIX_BAND", 8192)
+
+#define GRID_SIZE_LUP getEnvVariable("GRID_SIZE_LUP", 256)
+#define BLOCK_SIZE_LUP getEnvVariable("BLOCK_SIZE_LUP", 256)
+#define SHARED_MEM_SIZE_LUP getEnvVariable("SHARED_MEM_SIZE_LUP", 256)
+
+#define GRID_SIZE_ISMAX getEnvVariable("GRID_SIZE_ISMAX", 256)
+#define BLOCK_SIZE_ISMAX getEnvVariable("BLOCK_SIZE_ISMAX", 256)
+#define SHARED_MEM_SIZE_ISMAX getEnvVariable("SHARED_MEM_SIZE_ISMAX", 16384)
+
+
+#define GRID_SIZE_ISMAX_FILL getEnvVariable("GRID_SIZE_ISMAXGRID_SIZE_ISMAX_FILL", 128)
+#define BLOCK_SIZE_ISMAX_FILL getEnvVariable("BLOCK_SIZE_ISMAXBLOCK_SIZE_ISMAX_FILL", 512)
+#define SHARED_MEM_SIZE_ISMAX_FILL getEnvVariable("SHARED_MEM_SIZE_ISMAXSHARED_MEM_SIZE_ISMAX_FILL", 1024)
+
+#define GRID_SIZE_IMAGE_RESIZE getEnvVariable("GRID_SIZE_IMAGE_RESIZE", 256)
+#define BLOCK_SIZE_IMAGE_RESIZE getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE", 256)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE", 256)
+
+#define GRID_SIZE_DIAG getEnvVariable("GRID_SIZE_DIAG", 256)
+#define BLOCK_SIZE_DIAG getEnvVariable("BLOCK_SIZE_DIAG", 256)
+#define SHARED_MEM_SIZE_DIAG getEnvVariable("SHARED_MEM_SIZE_DIAG", 256)
+
+#define GRID_SIZE_CONFUSION_MATRIX getEnvVariable("GRID_SIZE_CONFUSION_MATRIX", 32)
+#define BLOCK_SIZE_CONFUSION_MATRIX getEnvVariable("BLOCK_SIZE_CONFUSION_MATRIX", 32)
+#define SHARED_MEM_SIZE_CONFUSION_MATRIX getEnvVariable("SHARED_MEM_SIZE_CONFUSION_MATRIX", 1024)
+
+#define GRID_SIZE_TILE getEnvVariable("GRID_SIZE_TILE", 256)
+#define BLOCK_SIZE_TILE getEnvVariable("BLOCK_SIZE_TILE", 512)
+#define SHARED_MEM_SIZE_TILE getEnvVariable("SHARED_MEM_SIZE_TILE", 8192)
+
+#define GRID_SIZE_DIAGONAL getEnvVariable("GRID_SIZE_DIAGONAL", 256)
+#define BLOCK_SIZE_DIAGONAL getEnvVariable("BLOCK_SIZE_DIAGONAL", 512)
+#define SHARED_MEM_SIZE_DIAGONAL getEnvVariable("SHARED_MEM_SIZE_DIAGONAL", 8192)
+
+#define GRID_SIZE_TEAR getEnvVariable("GRID_SIZE_TEAR", 512)
+#define BLOCK_SIZE_TEAR getEnvVariable("BLOCK_SIZE_TEAR", 512)
+#define SHARED_MEM_SIZE_TEAR getEnvVariable("SHARED_MEM_SIZE_TEAR", 512)
+
+#define GRID_SIZE_SORT_TENSOR_BY_DIM_KEY getEnvVariable("GRID_SIZE_SORT_TENSOR_BY_DIM_KEY", 256)
+#define BLOCK_SIZE_SORT_TENSOR_BY_DIM_KEY getEnvVariable("BLOCK_SIZE_SORT_TENSOR_BY_DIM_KEY", 256)
+#define SHARED_MEM_SIZE_SORT_TENSOR_BY_DIM_KEY getEnvVariable("SHARED_MEM_SIZE_SORT_TENSOR_BY_DIM_KEY", 256)
+
+#define GRID_SIZE_SORT_TENSOR_ALONG_DIM_KEY getEnvVariable("GRID_SIZE_SORT_TENSOR_ALONG_DIM_KEY", 256)
+#define BLOCK_SIZE_SORT_TENSOR_ALONG_DIM_KEY getEnvVariable("BLOCK_SIZE_SORT_TENSOR_ALONG_DIM_KEY", 256)
+#define SHARED_MEM_SIZE_SORT_TENSOR_ALONG_DIM_KEY getEnvVariable("SHARED_MEM_SIZE_SORT_TENSOR_ALONG_DIM_KEY", 256)
+
+#define GRID_SIZE_SORT_TENSOR_ALONG_DIM_VALUE getEnvVariable("GRID_SIZE_SORT_TENSOR_ALONG_DIM_VALUE", 256)
+#define BLOCK_SIZE_SORT_TENSOR_ALONG_DIM_VALUE getEnvVariable("BLOCK_SIZE_SORT_TENSOR_ALONG_DIM_VALUE", 256)
+#define SHARED_MEM_SIZE_SORT_TENSOR_ALONG_DIM_VALUE getEnvVariable("SHARED_MEM_SIZE_SORT_TENSOR_ALONG_DIM_VALUE", 256)
+
+#define GRID_SIZE_SHUFFLE getEnvVariable("GRID_SIZE_SHUFFLE", 256)
+#define BLOCK_SIZE_SHUFFLE getEnvVariable("BLOCK_SIZE_SHUFFLE", 512)
+#define SHARED_MEM_SIZE_SHUFFLE getEnvVariable("SHARED_MEM_SIZE_SHUFFLE", 8192)
+
+#define GRID_SIZE_PULLROWS getEnvVariable("GRID_SIZE_PULLROWS", 64)
+#define BLOCK_SIZE_PULLROWS getEnvVariable("BLOCK_SIZE_PULLROWS", 256)
+#define SHARED_MEM_SIZE_PULLROWS getEnvVariable("SHARED_MEM_SIZE_PULLROWS", 1024)
+
+#define GRID_SIZE_PRESCAN_ARRAY_RECURSIVE getEnvVariable("GRID_SIZE_PRESCAN_ARRAY_RECURSIVE", 256)
+#define BLOCK_SIZE_PRESCAN_ARRAY_RECURSIVE getEnvVariable("BLOCK_SIZE_PRESCAN_ARRAY_RECURSIVE", 256)
+#define SHARED_MEM_SIZE_PRESCAN_ARRAY_RECURSIVE getEnvVariable("SHARED_MEM_SIZE_PRESCAN_ARRAY_RECURSIVE", 256)
+
+#define GRID_SIZE_SCALAR_SCAN getEnvVariable("GRID_SIZE_SCALAR_SCAN", 256)
+#define BLOCK_SIZE_SCALAR_SCAN getEnvVariable("BLOCK_SIZE_SCALAR_SCAN", 512)
+#define SHARED_MEM_SIZE_SCALAR_SCAN getEnvVariable("SHARED_MEM_SIZE_SCALAR_SCAN", 8192)
+
+
+#define GRID_SIZE_SCALAR_TAD getEnvVariable("GRID_SIZE_SCALAR_TAD", 256)
+#define BLOCK_SIZE_SCALAR_TAD getEnvVariable("BLOCK_SIZE_SCALAR_TAD", 256)
+#define SHARED_MEM_SIZE_SCALAR_TAD getEnvVariable("SHARED_MEM_SIZE_SCALAR_TAD", 16384)
+
+#define GRID_SIZE_REDUCE_LONG getEnvVariable("GRID_SIZE_REDUCE_LONG", 256)
+#define BLOCK_SIZE_REDUCE_LONG getEnvVariable("BLOCK_SIZE_REDUCE_LONG", 256)
+#define SHARED_MEM_SIZE_REDUCE_LONG getEnvVariable("SHARED_MEM_SIZE_REDUCE_LONG", 1024)
+
+#define GRID_SIZE_REDUCE_BOOL getEnvVariable("GRID_SIZE_REDUCE_BOOL", 256)
+#define BLOCK_SIZE_REDUCE_BOOL getEnvVariable("BLOCK_SIZE_REDUCE_BOOL", 256)
+#define SHARED_MEM_SIZE_REDUCE_BOOL getEnvVariable("SHARED_MEM_SIZE_REDUCE_BOOL", 1024)
+
+#define GRID_SIZE_AVERAGE getEnvVariable("GRID_SIZE_AVERAGE", 256)
+#define BLOCK_SIZE_AVERAGE getEnvVariable("BLOCK_SIZE_AVERAGE", 256)
+#define SHARED_MEM_SIZE_AVERAGE getEnvVariable("SHARED_MEM_SIZE_AVERAGE", 4096)
+
+#define GRID_SIZE_ACCUMULATE getEnvVariable("GRID_SIZE_ACCUMULATE", 256)
+#define BLOCK_SIZE_ACCUMULATE getEnvVariable("BLOCK_SIZE_ACCUMULATE", 256)
+#define SHARED_MEM_SIZE_ACCUMULATE getEnvVariable("SHARED_MEM_SIZE_ACCUMULATE", 8192)
+
+#define GRID_SIZE_TRANSFORM_SCAN getEnvVariable("GRID_SIZE_TRANSFORM_SCAN", 256)
+#define BLOCK_SIZE_TRANSFORM_SCAN getEnvVariable("BLOCK_SIZE_TRANSFORM_SCAN", 256)
+#define SHARED_MEM_SIZE_TRANSFORM_SCAN getEnvVariable("SHARED_MEM_SIZE_TRANSFORM_SCAN", 1024)
+
+#define GRID_SIZE_SUMMARY_STATS getEnvVariable("GRID_SIZE_SUMMARY_STATS", 256)
+#define BLOCK_SIZE_SUMMARY_STATS getEnvVariable("BLOCK_SIZE_SUMMARY_STATS", SD_CUDA_BLOCK_SIZE)
+#define SHARED_MEM_SIZE_SUMMARY_STATS getEnvVariable("SHARED_MEM_SIZE_SUMMARY_STATS", 1024)
+
+#define GRID_SIZE_REDUCE_FLOAT getEnvVariable("GRID_SIZE_REDUCE_FLOAT", 256)
+#define BLOCK_SIZE_REDUCE_FLOAT getEnvVariable("BLOCK_SIZE_REDUCE_FLOAT", 256)
+#define SHARED_MEM_SIZE_REDUCE_FLOAT getEnvVariable("SHARED_MEM_SIZE_REDUCE_FLOAT", 256)
+
+#define GRID_SIZE_SCALAR_BOOL getEnvVariable("GRID_SIZE_SCALAR_BOOL", 256)
+#define BLOCK_SIZE_SCALAR_BOOL getEnvVariable("BLOCK_SIZE_SCALAR_BOOL", 256)
+#define SHARED_MEM_SIZE_SCALAR_BOOL getEnvVariable("SHARED_MEM_SIZE_SCALAR_BOOL", 256)
+
+#define GRID_SIZE_SCALAR_SAME getEnvVariable("GRID_SIZE_SCALAR_SAME", 256)
+#define BLOCK_SIZE_SCALAR_SAME getEnvVariable("BLOCK_SIZE_SCALAR_SAME", 256)
+#define SHARED_MEM_SIZE_SCALAR_SAME getEnvVariable("SHARED_MEM_SIZE_SCALAR_SAME", 256)
+
+#define GRID_SIZE_SCALAR_LONG getEnvVariable("GRID_SIZE_SCALAR_LONG", 256)
+#define BLOCK_SIZE_SCALAR_LONG getEnvVariable("BLOCK_SIZE_SCALAR_LONG", 256)
+#define SHARED_MEM_SIZE_SCALAR_LONG getEnvVariable("SHARED_MEM_SIZE_SCALAR_LONG", 256)
+
+#define GRID_SIZE_REDUCE_3 getEnvVariable("GRID_SIZE_REDUCE_3", 256)
+#define BLOCK_SIZE_REDUCE_3 getEnvVariable("BLOCK_SIZE_REDUCE_3", 256)
+#define SHARED_MEM_SIZE_REDUCE_3 getEnvVariable("SHARED_MEM_SIZE_REDUCE_3", 256)
+
+#define GRID_SIZE_PAIRWISE_TRANSFORMS getEnvVariable("GRID_SIZE_PAIRWISE_TRANSFORMS", 256)
+#define BLOCK_SIZE_PAIRWISE_TRANSFORMS getEnvVariable("BLOCK_SIZE_PAIRWISE_TRANSFORMS", 256)
+#define SHARED_MEM_SIZE_PAIRWISE_TRANSFORMS getEnvVariable("SHARED_MEM_SIZE_PAIRWISE_TRANSFORMS", 256)
+
+#define GRID_SIZE_BROADCAST getEnvVariable("GRID_SIZE_BROADCAST", 256)
+#define BLOCK_SIZE_BROADCAST getEnvVariable("BLOCK_SIZE_BROADCAST", 256)
+#define SHARED_MEM_SIZE_BROADCAST getEnvVariable("SHARED_MEM_SIZE_BROADCAST", 1024)
+
+#define GRID_SIZE_BROADCAST_INT getEnvVariable("GRID_SIZE_BROADCAST_INT", 256)
+#define BLOCK_SIZE_BROADCAST_INT getEnvVariable("BLOCK_SIZE_BROADCAST_INT", 256)
+#define SHARED_MEM_SIZE_BROADCAST_INT getEnvVariable("SHARED_MEM_SIZE_BROADCAST_INT", 1024)
+
+#define GRID_SIZE_BROADCAST_BOOL getEnvVariable("GRID_SIZE_BROADCAST_BOOL", 256)
+#define BLOCK_SIZE_BROADCAST_BOOL getEnvVariable("BLOCK_SIZE_BROADCAST_BOOL", 256)
+#define SHARED_MEM_SIZE_BROADCAST_BOOL getEnvVariable("SHARED_MEM_SIZE_BROADCAST_BOOL", 1024)
+
+#define GRID_SIZE_MATRIX_MULTIPLY getEnvVariable("GRID_SIZE_MATRIX_MULTIPLY", 256)
+#define BLOCK_SIZE_MATRIX_MULTIPLY getEnvVariable("BLOCK_SIZE_MATRIX_MULTIPLY", 256)
+#define SHARED_MEM_SIZE_MATRIX_MULTIPLY getEnvVariable("SHARED_MEM_SIZE_MATRIX_MULTIPLY", 256)
+
+
+
+#define GRID_SIZE_LOG_ABS_DETERMINANT getEnvVariable("GRID_SIZE_LOG_ABS_DETERMINANT", 256)
+#define BLOCK_SIZE_LOG_ABS_DETERMINANT getEnvVariable("BLOCK_SIZE_LOG_ABS_DETERMINANT", 256)
+#define SHARED_MEM_SIZE_LOG_ABS_DETERMINANT getEnvVariable("SHARED_MEM_SIZE_LOG_ABS_DETERMINANT", 1024)
+
+
+#define GRID_SIZE_DIAG_PART getEnvVariable("GRID_SIZE_DIAG_PART ", 256)
+#define BLOCK_SIZE_DIAG_PART  getEnvVariable("BLOCK_SIZE_DIAG_PART ", 512)
+#define SHARED_MEM_SIZE_DIAG_PART  getEnvVariable("SHARED_MEM_SIZE_DIAG_PART ", 8192)
+
+
+#define GRID_SIZE_RANDOM getEnvVariable("GRID_SIZE_RANDOM ", 512)
+#define BLOCK_SIZE_RANDOM  getEnvVariable("BLOCK_SIZE_RANDOM ", 512)
+#define SHARED_MEM_SIZE_RANDOM  getEnvVariable("SHARED_MEM_SIZE_RANDOM ", 32768)
+
+#define GRID_SIZE_LAMBDA getEnvVariable("GRID_SIZE_LAMBDA", 256)
+#define BLOCK_SIZE_LAMBDA getEnvVariable("BLOCK_SIZE_LAMBDA", 512)
+#define SHARED_MEM_SIZE_LAMBDA getEnvVariable("SHARED_MEM_SIZE_LAMBDA", 8192)
+
+#define GRID_SIZE_IM2COL getEnvVariable("GRID_SIZE_IM2COL", 256)
+#define BLOCK_SIZE_IM2COL getEnvVariable("BLOCK_SIZE_IM2COL", 512)
+#define SHARED_MEM_SIZE_IM2COL getEnvVariable("SHARED_MEM_SIZE_IM2COL", 8192)
+
+#define GRID_SIZE_COL2IM getEnvVariable("GRID_SIZE_COL2IM", 256)
+#define BLOCK_SIZE_COL2IM getEnvVariable("BLOCK_SIZE_COL2IM", 512)
+#define SHARED_MEM_SIZE_COL2IM getEnvVariable("SHARED_MEM_SIZE_COL2IM", 8192)
+
+#define GRID_SIZE_GEMV getEnvVariable("GRID_SIZE_GEMV", 256)
+#define BLOCK_SIZE_GEMV getEnvVariable("BLOCK_SIZE_GEMV", 512)
+#define SHARED_MEM_SIZE_GEMV getEnvVariable("SHARED_MEM_SIZE_GEMV", 8192)
+
+
+
+
+#define GRID_SIZE_ADDBIAS getEnvVariable("GRID_SIZE_ADDBIAS", 256)
+#define BLOCK_SIZE_ADDBIAS getEnvVariable("BLOCK_SIZE_ADDBIAS", 512)
+#define SHARED_MEM_SIZE_ADDBIAS getEnvVariable("SHARED_MEM_SIZE_ADDBIAS", 8192)
+
+#define GRID_SIZE_POOLING getEnvVariable("GRID_SIZE_POOLING", 256)
+#define BLOCK_SIZE_POOLING getEnvVariable("BLOCK_SIZE_POOLING", 512)
+#define SHARED_MEM_SIZE_POOLING getEnvVariable("SHARED_MEM_SIZE_POOLING", 8192)
+
+#define GRID_SIZE_COL2VOL getEnvVariable("GRID_SIZE_COL2VOL", 256)
+#define BLOCK_SIZE_COL2VOL getEnvVariable("BLOCK_SIZE_COL2VOL", 512)
+#define SHARED_MEM_SIZE_COL2VOL getEnvVariable("SHARED_MEM_SIZE_COL2VOL", 8192)
+
+#define GRID_SIZE_VOL2COL getEnvVariable("GRID_SIZE_VOL2COL", 256)
+#define BLOCK_SIZE_VOL2COL getEnvVariable("BLOCK_SIZE_VOL2COL", 512)
+#define SHARED_MEM_SIZE_VOL2COL getEnvVariable("SHARED_MEM_SIZE_VOL2COL", 8192)
+
+#define GRID_SIZE_UPSAMPLING getEnvVariable("GRID_SIZE_UPSAMPLING", 256)
+#define BLOCK_SIZE_UPSAMPLING getEnvVariable("BLOCK_SIZE_UPSAMPLING", 512)
+#define SHARED_MEM_SIZE_UPSAMPLING getEnvVariable("SHARED_MEM_SIZE_UPSAMPLING", 8192)
+
+
+#define GRID_SIZE_UPSAMPLING getEnvVariable("GRID_SIZE_PRELU", 256)
+#define BLOCK_SIZE_UPSAMPLING getEnvVariable("BLOCK_SIZE_PRELU", 512)
+#define SHARED_MEM_SIZE_UPSAMPLING getEnvVariable("SHARED_MEM_SIZE_PRELU", 512)
+
+#define GRID_SIZE_UPSAMPLING getEnvVariable("GRID_SIZE_ADJUST", 256)
+#define BLOCK_SIZE_UPSAMPLING getEnvVariable("BLOCK_SIZE_ADJUST", 512)
+#define SHARED_MEM_SIZE_UPSAMPLING getEnvVariable("SHARED_MEM_SIZE_ADJUST", 512)
+
+#define GRID_SIZE_BATCHNORM getEnvVariable("GRID_SIZE_BATCHNORM", 256)
+#define BLOCK_SIZE_BATCHNORM getEnvVariable("BLOCK_SIZE_BATCHNORM", 512)
+#define SHARED_MEM_SIZE_BATCHNORM getEnvVariable("SHARED_MEM_SIZE_BATCHNORM", 512)
+
+
+#define GRID_SIZE_COMPARE_AND_BITPACK getEnvVariable("GRID_SIZE_COMPARE_AND_BITPACK", 256)
+#define BLOCK_SIZE_COMPARE_AND_BITPACK getEnvVariable("BLOCK_SIZE_COMPARE_AND_BITPACK", 512)
+#define SHARED_MEM_SIZE_COMPARE_AND_BITPACK getEnvVariable("SHARED_MEM_SIZE_COMPARE_AND_BITPACK", 512)
+
+
+#define GRID_SIZE_CONFUSION_MATRIX getEnvVariable("GRID_SIZE_CONFUSION_MATRIX", 256)
+#define BLOCK_SIZE_CONFUSION_MATRIX getEnvVariable("BLOCK_SIZE_CONFUSION_MATRIX", 512)
+#define SHARED_MEM_SIZE_CONFUSION_MATRIX getEnvVariable("SHARED_MEM_SIZE_CONFUSION_MATRIX", 1024)
+
+#define GRID_SIZE_CLIP getEnvVariable("GRID_SIZE_CLIP", 256)
+#define BLOCK_SIZE_CLIP getEnvVariable("BLOCK_SIZE_CLIP", 512)
+#define SHARED_MEM_SIZE_CLIP getEnvVariable("SHARED_MEM_SIZE_CLIP", 8192)
+
+#define GRID_SIZE_CROSS getEnvVariable("GRID_SIZE_CROSS", 256)
+#define BLOCK_SIZE_CROSS getEnvVariable("BLOCK_SIZE_CROSS", 512)
+#define SHARED_MEM_SIZE_CROSS getEnvVariable("SHARED_MEM_SIZE_CROSS", 8192)
+
+
+#define GRID_SIZE_BETA_INC getEnvVariable("GRID_SIZE_BETA_INC", 256)
+#define BLOCK_SIZE_BETA_INC getEnvVariable("BLOCK_SIZE_BETA_INC", 512)
+#define SHARED_MEM_SIZE_BETA_INC getEnvVariable("SHARED_MEM_SIZE_BETA_INC", 8192)
+
+
+#define GRID_SIZE_ADJUST getEnvVariable("GRID_SIZE_ADJUST", 256)
+#define BLOCK_SIZE_ADJUST getEnvVariable("BLOCK_SIZE_ADJUST", 512)
+#define SHARED_MEM_SIZE_ADJUST getEnvVariable("SHARED_MEM_SIZE_ADJUST", 8192)
+
+#define GRID_SIZE_CONCAT getEnvVariable("GRID_SIZE_CONCAT", 256)
+#define BLOCK_SIZE_CONCAT getEnvVariable("BLOCK_SIZE_CONCAT", 512)
+#define SHARED_MEM_SIZE_CONCAT getEnvVariable("SHARED_MEM_SIZE_CONCAT", 8192)
+
+#define GRID_SIZE_DEPTH_TO_SPACE getEnvVariable("GRID_SIZE_DEPTH_TO_SPACE", 512)
+#define BLOCK_SIZE_DEPTH_TO_SPACE getEnvVariable("BLOCK_SIZE_DEPTH_TO_SPACE", 512)
+#define SHARED_MEM_SIZE_DEPTH_TO_SPACE getEnvVariable("SHARED_MEM_SIZE_DEPTH_TO_SPACE", 1024)
+
+
+#define GRID_SIZE_PRELU getEnvVariable("GRID_SIZE_PRELU", 512)
+#define BLOCK_SIZE_PRELU getEnvVariable("BLOCK_SIZE_PRELU", 512)
+#define SHARED_MEM_SIZE_PRELU getEnvVariable("SHARED_MEM_SIZE_PRELU", 1024)
+
+
+#define GRID_SIZE_HISTOGRAM getEnvVariable("GRID_SIZE_HISTOGRAM", 512)
+#define BLOCK_SIZE_HISTOGRAM getEnvVariable("BLOCK_SIZE_HISTOGRAM", 512)
+#define SHARED_MEM_SIZE_HISTOGRAM getEnvVariable("SHARED_MEM_SIZE_HISTOGRAM", 1024)
+
+#define GRID_SIZE_DILATION getEnvVariable("GRID_SIZE_DILATION", 512)
+#define BLOCK_SIZE_DILATION getEnvVariable("BLOCK_SIZE_DILATION", 512)
+#define SHARED_MEM_SIZE_DILATION getEnvVariable("SHARED_MEM_SIZE_DILATION", 1024)
+
+
+#define GRID_SIZE_DROPOUT getEnvVariable("GRID_SIZE_DROPOUT", 256)
+#define BLOCK_SIZE_DROPOUT getEnvVariable("BLOCK_SIZE_DROPOUT", 128)
+#define SHARED_MEM_SIZE_DROPOUT getEnvVariable("SHARED_MEM_SIZE_DROPOUT", 1024)
+
+#define GRID_SIZE_EXTRACT_PATCHES getEnvVariable("GRID_SIZE_EXTRACT_PATCHES", 128)
+#define BLOCK_SIZE_EXTRACT_PATCHES getEnvVariable("BLOCK_SIZE_EXTRACT_PATCHES", 128)
+#define SHARED_MEM_SIZE_EXTRACT_PATCHES getEnvVariable("SHARED_MEM_SIZE_EXTRACT_PATCHES", 1024)
+
+
+#define GRID_SIZE_FAKE_QUANTIZATION getEnvVariable("GRID_SIZE_FAKE_QUANTIZATION", 128)
+#define BLOCK_SIZE_FAKE_QUANTIZATION getEnvVariable("BLOCK_SIZE_FAKE_QUANTIZATION", 256)
+#define SHARED_MEM_SIZE_FAKE_QUANTIZATION getEnvVariable("SHARED_MEM_SIZE_FAKE_QUANTIZATION", 256)
+
+#define GRID_SIZE_FLATTEN getEnvVariable("GRID_SIZE_FLATTEN", 256)
+#define BLOCK_SIZE_FLATTEN getEnvVariable("BLOCK_SIZE_FLATTEN", 512)
+#define SHARED_MEM_SIZE_FLATTEN getEnvVariable("SHARED_MEM_SIZE_FLATTEN", 8192)
+
+#define GRID_SIZE_GATHER_LINEAR getEnvVariable("GRID_SIZE_GATHER_LINEAR", 128)
+#define BLOCK_SIZE_GATHER_LINEAR getEnvVariable("BLOCK_SIZE_GATHER_LINEAR", 256)
+#define SHARED_MEM_SIZE_GATHER_LINEAR getEnvVariable("SHARED_MEM_SIZE_GATHER_LINEAR", 1024)
+
+#define GRID_SIZE_GATHER getEnvVariable("GRID_SIZE_GATHER", 128)
+#define BLOCK_SIZE_GATHER getEnvVariable("BLOCK_SIZE_GATHER", 256)
+#define SHARED_MEM_SIZE_GATHER getEnvVariable("SHARED_MEM_SIZE_GATHER", 1024)
+
+
+#define GRID_SIZE_GATHERND getEnvVariable("GRID_SIZE_GATHERND", 128)
+#define BLOCK_SIZE_GATHERND getEnvVariable("BLOCK_SIZE_GATHERND", 256)
+#define SHARED_MEM_SIZE_GATHERND getEnvVariable("SHARED_MEM_SIZE_GATHERND", 1024)
+
+
+#define GRID_SIZE_HAMMING getEnvVariable("GRID_SIZE_HAMMING", 128)
+#define BLOCK_SIZE_HAMMING getEnvVariable("BLOCK_SIZE_HAMMING",SD_CUDA_BLOCK_SIZE)
+#define SHARED_MEM_SIZE_HAMMING getEnvVariable("SHARED_MEM_SIZE_HAMMING", 1024)
+
+
+#define GRID_SIZE_HASHCODE_SPLIT getEnvVariable("GRID_SIZE_HASHCODE_SPLIT", 128)
+#define BLOCK_SIZE_HASHCODE_SPLIT getEnvVariable("BLOCK_SIZE_HASHCODE_SPLIT",SD_CUDA_BLOCK_SIZE)
+#define SHARED_MEM_SIZE_HASHCODE_SPLIT getEnvVariable("SHARED_MEM_SIZE_HASHCODE_SPLIT", 1024)
+
+#define GRID_SIZE_HASHCODE_INTERNAL getEnvVariable("GRID_SIZE_HASHCODE_INTERNAL", 128)
+#define BLOCK_SIZE_HASHCODE_INTERNAL getEnvVariable("BLOCK_SIZE_HASHCODE_INTERNAL",SD_CUDA_BLOCK_SIZE)
+#define SHARED_MEM_SIZE_HASHCODE_INTERNAL getEnvVariable("SHARED_MEM_SIZE_HASHCODE_INTERNAL", 1024)
+
+
+#define GRID_SIZE_HASHCODE_LAST getEnvVariable("GRID_SIZE_HASHCODE_LAST", 1)
+#define BLOCK_SIZE_HASHCODE_LAST getEnvVariable("BLOCK_SIZE_HASHCODE_LAST",1)
+#define SHARED_MEM_SIZE_HASHCODE_LAST getEnvVariable("SHARED_MEM_SIZE_HASHCODE_LAST", 128)
+
+#define GRID_SIZE_HISTOGRAM_FIXED_WIDTH getEnvVariable("GRID_SIZE_HISTOGRAM_FIXED_WIDTH", 256)
+#define BLOCK_SIZE_HISTOGRAM_FIXED_WIDTH getEnvVariable("BLOCK_SIZE_HISTOGRAM_FIXED_WIDTH",256)
+#define SHARED_MEM_SIZE_HISTOGRAM_FIXED_WIDTH getEnvVariable("SHARED_MEM_SIZE_HISTOGRAM_FIXED_WIDTH", 1024)
+
+
+#define GRID_SIZE_DRAW_BOUNDING_BOXES getEnvVariable("GRID_SIZE_DRAW_BOUNDING_BOXES", 128)
+#define BLOCK_SIZE_DRAW_BOUNDING_BOXES getEnvVariable("BLOCK_SIZE_DRAW_BOUNDING_BOXES",128)
+#define SHARED_MEM_SIZE_DRAW_BOUNDING_BOXES getEnvVariable("SHARED_MEM_SIZE_DRAW_BOUNDING_BOXES", 1024)
+
+#define GRID_SIZE_IMAGE_RESIZE getEnvVariable("GRID_SIZE_IMAGE_RESIZE", 256)
+#define BLOCK_SIZE_IMAGE_RESIZE getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE",256)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE", 256)
+
+
+#define GRID_SIZE_IMAGE_RESIZE_INTERP_WEIGHTS getEnvVariable("GRID_SIZE_IMAGE_RESIZE_INTERP_WEIGHTS", 256)
+#define BLOCK_SIZE_IMAGE_RESIZE_INTERP_WEIGHTS getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE_INTERP_WEIGHTS",512)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE_INTERP_WEIGHTS getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE_INTERP_WEIGHTS", 512)
+
+#define GRID_SIZE_IMAGE_RESIZE_INIT_COEFFS getEnvVariable("GRID_SIZE_IMAGE_RESIZE_INIT_COEFFS", 128)
+#define BLOCK_SIZE_IMAGE_RESIZE_INIT_COEFFS getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE_INIT_COEFFS",128)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE_INIT_COEFFS getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE_INIT_COEFFS", 128)
+
+#define GRID_SIZE_IMAGE_RESIZE_COEFFS_ACCUM getEnvVariable("GRID_SIZE_IMAGE_RESIZE_COEFFS_ACCUM", 128)
+#define BLOCK_SIZE_IMAGE_RESIZE_COEFFS_ACCUM getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE_COEFFS_ACCUM",128)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE_COEFFS_ACCUM getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE_COEFFS_ACCUM", 512)
+
+#define GRID_SIZE_IMAGE_RESIZE_BICUBIC getEnvVariable("GRID_SIZE_IMAGE_RESIZE_BICUBIC", 128)
+#define BLOCK_SIZE_IMAGE_RESIZE_BICUBIC getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE_BICUBIC",1)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE_BICUBIC getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE_BICUBIC", 512)
+
+#define GRID_SIZE_IMAGE_RESIZE_FILL_INTERP getEnvVariable("GRID_SIZE_IMAGE_RESIZE_FILL_INTERP", 128)
+#define BLOCK_SIZE_IMAGE_RESIZE_FILL_INTERP getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE_FILL_INTERP",128)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE_FILL_INTERP getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE_FILL_INTERP", 256)
+
+#define GRID_SIZE_IMAGE_RESIZE_CROP_AND_RESIZE getEnvVariable("GRID_SIZE_IMAGE_RESIZE_CROP_AND_RESIZE", 128)
+#define BLOCK_SIZE_IMAGE_RESIZE_CROP_AND_RESIZE getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE_CROP_AND_RESIZE",128)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE_CROP_AND_RESIZE getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE_CROP_AND_RESIZE", 256)
+
+
+#define GRID_SIZE_IMAGE_RESIZE_V2_GATHER getEnvVariable("GRID_SIZE_IMAGE_RESIZE_V2_GATHER", 128)
+#define BLOCK_SIZE_IMAGE_RESIZE_V2_GATHER getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE_V2_GATHER",128)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE_V2_GATHER getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE_V2_GATHER", 256)
+
+#define GRID_SIZE_IMAGE_SUPPRESS_SCORES getEnvVariable("GRID_SIZE_IMAGE_SUPPRESS_SCORES", 128)
+#define BLOCK_SIZE_IMAGE_SUPPRESS_SCORES getEnvVariable("BLOCK_SIZE_IMAGE_SUPPRESS_SCORES",128)
+#define SHARED_MEM_SIZE_IMAGE_SUPPRESS_SCORES getEnvVariable("SHARED_MEM_SIZE_IMAGE_SUPPRESS_SCORES", 128)
+
+#define GRID_SIZE_IMAGE_SUPPRESS_SELECT getEnvVariable("GRID_SIZE_IMAGE_SUPPRESS_SELECT", 128)
+#define BLOCK_SIZE_IMAGE_SUPPRESS_SELECT getEnvVariable("BLOCK_SIZE_IMAGE_SUPPRESS_SELECT",256)
+#define SHARED_MEM_SIZE_IMAGE_SUPPRESS_SELECT getEnvVariable("SHARED_MEM_SIZE_IMAGE_SUPPRESS_SELECT", 1024)
+
+#define GRID_SIZE_IMAGE_SUPPRESS_NONMAX_OVERLAP getEnvVariable("GRID_SIZE_IMAGE_SUPPRESS_NONMAX_OVERLAP", 1)
+#define BLOCK_SIZE_IMAGE_SUPPRESS_NONMAX_OVERLAP getEnvVariable("BLOCK_SIZE_IMAGE_SUPPRESS_NONMAX_OVERLAP",1)
+#define SHARED_MEM_SIZE_IMAGE_SUPPRESS_NONMAX_OVERLAP getEnvVariable("SHARED_MEM_SIZE_IMAGE_SUPPRESS_NONMAX_OVERLAP", 1024)
+
+
+#define GRID_SIZE_IMAGE_HELPERS_TRIPLE getEnvVariable("GRID_SIZE_IMAGE_HELPERS_TRIPLE", 256)
+#define BLOCK_SIZE_IMAGE_HELPERS_TRIPLE getEnvVariable("BLOCK_SIZE_IMAGE_HELPERS_TRIPLE",256)
+#define SHARED_MEM_SIZE_IMAGE_HELPERS_TRIPLE getEnvVariable("SHARED_MEM_SIZE_IMAGE_HELPERS_TRIPLE", 8192)
+
+#define GRID_SIZE_IMAGE_HELPERS getEnvVariable("GRID_SIZE_IMAGE_HELPERS", 256)
+#define BLOCK_SIZE_IMAGE_HELPERS getEnvVariable("BLOCK_SIZE_IMAGE_HELPERS",256)
+#define SHARED_MEM_SIZE_GRID_SIZE_IMAGE_HELPERS getEnvVariable("SHARED_MEM_SIZE_GRID_SIZE_IMAGE_HELPERS", 8192)
+
+#define GRID_SIZE_LRN getEnvVariable("GRID_SIZE_LRN", 256)
+#define BLOCK_SIZE_LRN getEnvVariable("BLOCK_SIZE_LRN",256)
+#define SHARED_MEM_SIZE_LRN getEnvVariable("SHARED_MEM_SIZE_LRN", 8192)
+
+#define GRID_SIZE_LSTSQ_REG getEnvVariable("GRID_SIZE_LSTSQ_REG", 256)
+#define BLOCK_SIZE_LSTSQ_REG getEnvVariable("BLOCK_SIZE_LSTSQ_REG",256)
+#define SHARED_MEM_SIZE_LSTSQ_REG getEnvVariable("SHARED_MEM_SIZE_LSTSQ_REG", 128)
+
+#define GRID_SIZE_LUP getEnvVariable("GRID_SIZE_LUP", 256)
+#define GRID_SIZE_LUP getEnvVariable("GRID_SIZE_LUP",256)
+#define SHARED_MEM_SIZE_LUP getEnvVariable("SHARED_MEM_SIZE_LUP", 512)
+
+
+#define GRID_SIZE_LUP_LOW getEnvVariable("GRID_SIZE_LUP_LOW", 256)
+#define BLOCK_SIZE_LUP_LOW getEnvVariable("BLOCK_SIZE_LUP_LOW",256)
+#define SHARED_MEM_SIZE_LUP_LOW getEnvVariable("SHARED_MEM_SIZE_LUP_LOW", 512)
+
+#define GRID_SIZE_MATRIX_SET_DIAG getEnvVariable("GRID_SIZE_MATRIX_SET_DIAG", 256)
+#define BLOCK_SIZE_MATRIX_SET_DIAG getEnvVariable("BLOCK_SIZE_MATRIX_SET_DIAG",256)
+#define SHARED_MEM_SIZE_MATRIX_SET_DIAG getEnvVariable("SHARED_MEM_SIZE_MATRIX_SET_DIAG", 512)
+
+#define GRID_SIZE_MERGE getEnvVariable("GRID_SIZE_MERGE", 256)
+#define BLOCK_SIZE_MERGE getEnvVariable("BLOCK_SIZE_MERGE",256)
+#define SHARED_MEM_SIZE_MERGE getEnvVariable("SHARED_MEM_SIZE_MERGE", 512)
+
+#define GRID_SIZE_MESHGRID getEnvVariable("GRID_SIZE_MESHGRID", 256)
+#define BLOCK_SIZE_MESHGRID getEnvVariable("BLOCK_SIZE_MESHGRID",256)
+#define SHARED_MEM_SIZE_MESHGRID getEnvVariable("SHARED_MEM_SIZE_MESHGRID", 1024)
+
+#define GRID_SIZE_NTH_ELEMENT_FILL getEnvVariable("GRID_SIZE_NTH_ELEMENT_FILL", 32)
+#define BLOCK_SIZE_NTH_ELEMENT_FILL getEnvVariable("BLOCK_SIZE_NTH_ELEMENT_FILL",64)
+#define SHARED_MEM_SIZE_NTH_ELEMENT_FILL getEnvVariable("SHARED_MEM_SIZE_NTH_ELEMENT_FILL", 1024)
+
+
+#define GRID_SIZE_ONE_HOT getEnvVariable("GRID_SIZE_ONE_HOT", 32)
+#define BLOCK_SIZE_ONE_HOT getEnvVariable("BLOCK_SIZE_ONE_HOT",64)
+#define SHARED_MEM_SIZE_ONE_HOT getEnvVariable("SHARED_MEM_SIZE_ONE_HOT", 1024)
+
+#define GRID_SIZE_PAD getEnvVariable("GRID_SIZE_PAD", 32)
+#define BLOCK_SIZE_PAD getEnvVariable("BLOCK_SIZE_PAD",64)
+#define SHARED_MEM_SIZE_PAD getEnvVariable("SHARED_MEM_SIZE_PAD", 1024)
+
+#define GRID_SIZE_MIRROR_PAD_LINEAR getEnvVariable("GRID_SIZE_MIRROR_PAD_LINEAR", 256)
+#define BLOCK_SIZE_MIRROR_PAD_LINEAR getEnvVariable("BLOCK_SIZE_MIRROR_PAD_LINEAR",512)
+#define SHARED_MEM_SIZE_MIRROR_PAD_LINEAR getEnvVariable("SHARED_MEM_SIZE_MIRROR_PAD_LINEAR", 256)
+
+#define GRID_SIZE_MIRROR_PAD_TAD getEnvVariable("GRID_SIZE_MIRROR_PAD_TAD", 256)
+#define BLOCK_SIZE_MIRROR_PAD_TAD getEnvVariable("BLOCK_SIZE_MIRROR_PAD_TAD",512)
+#define SHARED_MEM_SIZE_MIRROR_PAD_TAD getEnvVariable("SHARED_MEM_SIZE_MIRROR_PAD_TAD", 256)
+
+#define GRID_SIZE_PERCENTILE getEnvVariable("GRID_SIZE_PERCENTILE", 256)
+#define BLOCK_SIZE_PERCENTILE getEnvVariable("BLOCK_SIZE_PERCENTILE",512)
+#define SHARED_MEM_SIZE_PERCENTILE getEnvVariable("SHARED_MEM_SIZE_PERCENTILE", 1024)
+
+#define GRID_SIZE_POLYGAMMA getEnvVariable("GRID_SIZE_POLYGAMMA", 256)
+#define BLOCK_SIZE_POLYGAMMA getEnvVariable("BLOCK_SIZE_POLYGAMMA",512)
+#define SHARED_MEM_SIZE_POLYGAMMA getEnvVariable("SHARED_MEM_SIZE_POLYGAMMA", 1024)
+
+
+#define GRID_SIZE_PREFIX getEnvVariable("GRID_SIZE_PREFIX", 256)
+#define BLOCK_SIZE_PREFIX getEnvVariable("BLOCK_SIZE_PREFIX",512)
+#define SHARED_MEM_SIZE_PREFIX getEnvVariable("SHARED_MEM_SIZE_PREFIX", 1024)
+
+
+#define GRID_SIZE_PRINT getEnvVariable("GRID_SIZE_PRINT", 1)
+#define BLOCK_SIZE_PRINT getEnvVariable("BLOCK_SIZE_PRINT",1)
+#define SHARED_MEM_SIZE_PRINT getEnvVariable("SHARED_MEM_SIZE_PRINT", 1024)
+
+#define GRID_SIZE_QR getEnvVariable("GRID_SIZE_QR", 28)
+#define BLOCK_SIZE_QR getEnvVariable("BLOCK_SIZE_QR",28)
+#define SHARED_MEM_SIZE_QR getEnvVariable("SHARED_MEM_SIZE_QR", 28)
+
+#define GRID_SIZE_RANDOM_GAMMA getEnvVariable("GRID_SIZE_RANDOM_GAMMA", 128)
+#define BLOCK_SIZE_RANDOM_GAMMA getEnvVariable("BLOCK_SIZE_RANDOM_GAMMA",128)
+#define SHARED_MEM_SIZE_RANDOM_GAMMA getEnvVariable("SHARED_MEM_SIZE_RANDOM_GAMMA", 256)
+
+#define GRID_SIZE_RANDOM_POISSON getEnvVariable("GRID_SIZE_RANDOM_POISSON", 128)
+#define BLOCK_SIZE_RANDOM_POISSON getEnvVariable("BLOCK_SIZE_RANDOM_POISSON",256)
+#define SHARED_MEM_SIZE_RANDOM_POISSON getEnvVariable("SHARED_MEM_SIZE_RANDOM_POISSON", 256)
+
+#define GRID_SIZE_RANDOM_UNIFORM getEnvVariable("GRID_SIZE_RANDOM_UNIFORM", 128)
+#define BLOCK_SIZE_RANDOM_UNIFORM getEnvVariable("BLOCK_SIZE_RANDOM_UNIFORM",128)
+#define SHARED_MEM_SIZE_RANDOM_UNIFORM getEnvVariable("SHARED_MEM_SIZE_RANDOM_UNIFORM", 128)
+
+#define GRID_SIZE_RANDOM_SHUFFLE_FISHER getEnvVariable("GRID_SIZE_RANDOM_SHUFFLE_FISHER", 128)
+#define BLOCK_SIZE_RANDOM_SHUFFLE_FISHER getEnvVariable("BLOCK_SIZE_RANDOM_SHUFFLE_FISHER",128)
+#define SHARED_MEM_SIZE_RANDOM_SHUFFLE_FISHER getEnvVariable("SHARED_MEM_SIZE_RANDOM_SHUFFLE_FISHER", 128)
+
+
+#define GRID_SIZE_RANDOM_SHUFFLE_MERGE getEnvVariable("GRID_SIZE_RANDOM_SHUFFLE_MERGE", 128)
+#define BLOCK_SIZE_RANDOM_SHUFFLE_MERGE getEnvVariable("BLOCK_SIZE_RANDOM_SHUFFLE_MERGE",128)
+#define SHARED_MEM_SIZE_RANDOM_SHUFFLE_MERGE getEnvVariable("SHARED_MEM_SIZE_RANDOM_SHUFFLE_MERGE", 128)
+
+#define GRID_SIZE_RANGE getEnvVariable("GRID_SIZE_RANGE", 512)
+#define BLOCK_SIZE_RANGE getEnvVariable("BLOCK_SIZE_RANGE",512)
+#define SHARED_MEM_SIZE_RANGE getEnvVariable("SHARED_MEM_SIZE_RANGE", 2048)
+
+#define GRID_SIZE_REVERSE getEnvVariable("GRID_SIZE_REVERSE", 256)
+#define BLOCK_SIZE_REVERSE getEnvVariable("BLOCK_SIZE_REVERSE",512)
+#define SHARED_MEM_SIZE_REVERSE getEnvVariable("SHARED_MEM_SIZE_REVERSEE", 8192)
+
+#define GRID_SIZE_ROLL getEnvVariable("GRID_SIZE_ROLL", 1)
+#define BLOCK_SIZE_ROLL getEnvVariable("BLOCK_SIZE_ROLL",1)
+#define SHARED_MEM_SIZE_ROLL getEnvVariable("SHARED_MEM_SIZE_ROLL", 1024)
+
+#define GRID_SIZE_BATCH_TO_SPACE_ND getEnvVariable("GRID_SIZE_BATCH_TO_SPACE_ND", 1)
+#define BLOCK_SIZE_BATCH_TO_SPACE_ND getEnvVariable("BLOCK_SIZE_BATCH_TO_SPACE_ND",1)
+#define SHARED_MEM_SIZE_BATCH_TO_SPACE_ND getEnvVariable("SHARED_MEM_SIZE_BATCH_TO_SPACE_ND", 1024)
+
+#define GRID_SIZE_SPACE_TO_BATCH getEnvVariable("GRID_SIZE_SPACE_TO_BATCH", 1)
+#define BLOCK_SIZE_SPACE_TO_BATCH getEnvVariable("BLOCK_SIZE_SPACE_TO_BATCH",1)
+#define SHARED_MEM_SIZE_SPACE_TO_BATCH getEnvVariable("SHARED_MEM_SIZE_SPACE_TO_BATCH", 1024)
+
+
+#define GRID_SIZE_SPACE_TO_BATCH_ND getEnvVariable("GRID_SIZE_SPACE_TO_BATCH_ND", 1)
+#define BLOCK_SIZE_SPACE_TO_BATCH_ND getEnvVariable("BLOCK_SIZE_SPACE_TO_BATCH_ND",1)
+#define SHARED_MEM_SIZE_SPACE_TO_BATCH_ND getEnvVariable("SHARED_MEM_SIZE_SPACE_TO_BATCH_ND", 1024)
+
+#define GRID_SIZE_SPACE_TO_DEPTH getEnvVariable("GRID_SIZE_SPACE_TO_DEPTH", 512)
+#define BLOCK_SIZE_SPACE_TO_DEPTH getEnvVariable("BLOCK_SIZE_SPACE_TO_DEPTH",512)
+#define SHARED_MEM_SIZE_SPACE_TO_DEPTH getEnvVariable("SHARED_MEM_SIZE_SPACE_TO_DEPTH", 1024)
+
+
+#define GRID_SIZE_SCATTER getEnvVariable("GRID_SIZE_SCATTER", 512)
+#define BLOCK_SIZE_SCATTER getEnvVariable("BLOCK_SIZE_SCATTER",512)
+#define SHARED_MEM_SIZE_SCATTER getEnvVariable("SHARED_MEM_SIZE_SCATTER", 1024)
+
+#define GRID_SIZE_SCATTER_CHECK_INDICES getEnvVariable("GRID_SIZE_SCATTER_CHECK_INDICES", 512)
+#define BLOCK_SIZE_SCATTER_CHECK_INDICES getEnvVariable("BLOCK_SIZE_SCATTER_CHECK_INDICES",512)
+#define SHARED_MEM_SIZE_SCATTER_CHECK_INDICES getEnvVariable("SHARED_MEM_SIZE_SCATTER_CHECK_INDICES", 1024)
+
+#define GRID_SIZE_SCATTER_ND getEnvVariable("GRID_SIZE_SCATTER_ND", 512)
+#define BLOCK_SIZE_SCATTER_ND getEnvVariable("BLOCK_SIZE_SCATTER_ND",512)
+#define SHARED_MEM_SIZE_SCATTER_ND getEnvVariable("SHARED_MEM_SIZE_SCATTER_ND", 1024)
+
+#define GRID_SIZE_SCATTER_SIMPLE getEnvVariable("GRID_SIZE_SCATTER_SIMPLE", 512)
+#define BLOCK_SIZE_SCATTER_SIMPLE getEnvVariable("BLOCK_SIZE_SCATTER_SIMPLE",256)
+#define SHARED_MEM_SIZE_SCATTER_SIMPLE getEnvVariable("SHARED_MEM_SIZE_SCATTER_SIMPLE", SD_MAX_NUM_THREADS)
+
+
+#define GRID_SIZE_SCATTER_UPDATE getEnvVariable("GRID_SIZE_SCATTER_UPDATE", 256)
+#define BLOCK_SIZE_SCATTER_UPDATE getEnvVariable("BLOCK_SIZE_SCATTER_UPDATE",256)
+#define SHARED_MEM_SIZE_SCATTER_UPDATE getEnvVariable("SHARED_MEM_SIZE_SCATTER_UPDATE", 1024)
+
+#define GRID_SIZE_SEGMENT_INDICES_VALIDATE getEnvVariable("GRID_SIZE_SEGMENT_INDICES_VALIDATE", 1)
+#define BLOCK_SIZE_SEGMENT_INDICES_VALIDATE getEnvVariable("BLOCK_SIZE_SEGMENT_INDICES_VALIDATE",1)
+#define SHARED_MEM_SIZE_SEGMENT_INDICES_VALIDATE getEnvVariable("SHARED_MEM_SIZE_SEGMENT_INDICES_VALIDATE", 128)
+
+
+#define GRID_SIZE_SEGMENT getEnvVariable("GRID_SIZE_SEGMENT", 1)
+#define BLOCK_SIZE_SEGMENT getEnvVariable("BLOCK_SIZE_SEGMENT",1)
+#define SHARED_MEM_SIZE_SEGMENT getEnvVariable("SHARED_MEM_SIZE_SEGMENT", 128)
+
+
+#define GRID_SIZE_SEGMENT_TAD getEnvVariable("GRID_SIZE_SEGMENT_TAD", 1)
+#define BLOCK_SIZE_SEGMENT_TAD getEnvVariable("BLOCK_SIZE_SEGMENT_TAD",1)
+#define SHARED_MEM_SIZE_SEGMENT_TAD getEnvVariable("SHARED_MEM_SIZE_SEGMENT_TAD", 128)
+
+#define GRID_SIZE_SEGMENT_BP getEnvVariable("GRID_SIZE_SEGMENT_BP", 1)
+#define BLOCK_SIZE_SEGMENT_BP getEnvVariable("BLOCK_SIZE_SEGMENT_BP",1)
+#define SHARED_MEM_SIZE_SEGMENT_BP getEnvVariable("SHARED_MEM_SIZE_SEGMENT_BP", 128)
+
+#define GRID_SIZE_SEGMENT_BP_TAD getEnvVariable("GRID_SIZE_SEGMENT_BP_TAD", 1)
+#define BLOCK_SIZE_SEGMENT_BP_TAD getEnvVariable("BLOCK_SIZE_SEGMENT_BP_TAD",1)
+#define SHARED_MEM_SIZE_SEGMENT_BP_TAD getEnvVariable("SHARED_MEM_SIZE_SEGMENT_BP_TAD", 128)
+
+#define GRID_SIZE_SEGMENT_PROD_2_LINEAR getEnvVariable("GRID_SIZE_SEGMENT_PROD_2_LINEAR", 256)
+#define BLOCK_SIZE_SEGMENT_PROD_2_LINEAR getEnvVariable("BLOCK_SIZE_SEGMENT_PROD_2_LINEAR",128)
+#define SHARED_MEM_SIZE_SEGMENT_PROD_2_LINEAR getEnvVariable("SHARED_MEM_SIZE_SEGMENT_PROD_2_LINEAR", 128)
+
+#define GRID_SIZE_SEGMENT_PROD_2_TAD getEnvVariable("GRID_SIZE_SEGMENT_PROD_2_TAD", 512)
+#define BLOCK_SIZE_SEGMENT_PROD_2_TAD getEnvVariable("BLOCK_SIZE_SEGMENT_PROD_TAD",128)
+#define SHARED_MEM_SIZE_SEGMENT_PROD_2_TAD getEnvVariable("SHARED_MEM_SIZE_SEGMENT_PROD_2_TAD", 2048)
+
+#define GRID_SIZE_UNSORTEDSEGMENT_PROD_2 getEnvVariable("GRID_SIZE_UNSORTEDSEGMENT_PROD_2", 128)
+#define BLOCK_SIZE_UNSORTEDSEGMENT_PROD_2 getEnvVariable("BLOCK_SIZE_UNSORTEDSEGMENT_PROD_2",256)
+#define SHARED_MEM_SIZE_UNSORTEDSEGMENT_PROD_2 getEnvVariable("SHARED_MEM_SIZE_UNSORTEDSEGMENT_PROD_2", 256)
+
+#define GRID_SIZE_SOLVE getEnvVariable("GRID_SIZE_SOLVE", 128)
+#define BLOCK_SIZE_SOLVE getEnvVariable("BLOCK_SIZE_SOLVE",256)
+#define SHARED_MEM_SIZE_SOLVE getEnvVariable("SHARED_MEM_SIZE_SOLVE", 256)
+
+#define GRID_SIZE_SRU_BI getEnvVariable("GRID_SIZE_SRU_BI", 128)
+#define BLOCK_SIZE_SRU_BI getEnvVariable("BLOCK_SIZE_SRU_BI",256)
+#define SHARED_MEM_SIZE_SRU_BI getEnvVariable("SHARED_MEM_SIZE_SRU_BI", 256)
+
+#define GRID_SIZE_STACK getEnvVariable("GRID_SIZE_STACK", 128)
+#define BLOCK_SIZE_STACK getEnvVariable("BLOCK_SIZE_STACK",256)
+#define SHARED_MEM_SIZE_STACK getEnvVariable("SHARED_MEM_SIZE_STACK", 256)
+
+
+
+#define GRID_SIZE_TOP_K_MOVER getEnvVariable("GRID_SIZE_TOP_K_MOVER", 256)
+#define BLOCK_SIZE_TOP_K_MOVER getEnvVariable("BLOCK_SIZE_TOP_K_MOVER",256)
+#define SHARED_MEM_SIZE_TOP_K_MOVER getEnvVariable("SHARED_MEM_SIZE_TOP_K_MOVER", 1024)
+
+
+#define GRID_SIZE_TOP_K_INDICES getEnvVariable("GRID_SIZE_TOP_K_INDICES", 256)
+#define BLOCK_SIZE_TOP_K_INDICES getEnvVariable("BLOCK_SIZE_TOP_K_INDICES",256)
+#define SHARED_MEM_SIZE_TOP_K_INDICES getEnvVariable("SHARED_MEM_SIZE_TOP_K_INDICES", 1024)
+
+
+#define GRID_SIZE_INVERT_PERMUTE getEnvVariable("GRID_SIZE_INVERT_PERMUTE", 256)
+#define BLOCK_SIZE_INVERT_PERMUTE getEnvVariable("BLOCK_SIZE_INVERT_PERMUTE",256)
+#define SHARED_MEM_SIZE_INVERT_PERMUTE getEnvVariable("SHARED_MEM_SIZE_INVERT_PERMUTE", 1024)
+
+#define GRID_SIZE_TRACE getEnvVariable("GRID_SIZE_TRACE", 256)
+#define BLOCK_SIZE_TRACE getEnvVariable("BLOCK_SIZE_TRACE",256)
+#define SHARED_MEM_SIZE_TRACE getEnvVariable("SHARED_MEM_SIZE_TRACE", 1024)
+
+#define GRID_SIZE_TRIU getEnvVariable("GRID_SIZE_TRIU", 256)
+#define BLOCK_SIZE_TRIU getEnvVariable("BLOCK_SIZE_TRIU",256)
+#define SHARED_MEM_SIZE_TRIU getEnvVariable("SHARED_MEM_SIZE_TRIU", 1024)
+
+#define GRID_SIZE_TILE getEnvVariable("GRID_SIZE_TILE", 256)
+#define BLOCK_SIZE_TILE getEnvVariable("BLOCK_SIZE_TILE",256)
+#define SHARED_MEM_SIZE_TILE getEnvVariable("SHARED_MEM_SIZE_TILE", 1024)
+
+#define GRID_SIZE_TRIANGULAR_SOLVE getEnvVariable("GRID_SIZE_TRIANGULAR_SOLVE", 128)
+#define BLOCK_SIZE_TRIANGULAR_SOLVE getEnvVariable("BLOCK_SIZE_TRIANGULAR_SOLVE",256)
+#define SHARED_MEM_SIZE_TRIANGULAR_SOLVE getEnvVariable("SHARED_MEM_SIZE_TRIANGULAR_SOLVE", 256)
+
+#define GRID_SIZE_UPDATER getEnvVariable("GRID_SIZE_UPDATER", 128)
+#define BLOCK_SIZE_UPDATER getEnvVariable("BLOCK_SIZE_UPDATER",256)
+#define SHARED_MEM_SIZE_UPDATER getEnvVariable("SHARED_MEM_SIZE_UPDATER", 256)
+
+
+#define GRID_SIZE_ZETA getEnvVariable("GRID_SIZE_ZETA", 128)
+#define BLOCK_SIZE_ZETA getEnvVariable("BLOCK_SIZE_ZETA",256)
+#define SHARED_MEM_SIZE_ZETA getEnvVariable("SHARED_MEM_SIZE_ZETA", 1024)
+
+#define GRID_SIZE_AVG_POOLING getEnvVariable("GRID_SIZE_AVG_POOLING", 512)
+#define BLOCK_SIZE_AVG_POOLING getEnvVariable("BLOCK_SIZE_AVG_POOLING",512)
+#define SHARED_MEM_SIZE_POOLING getEnvVariable("SHARED_MEM_SIZE_POOLING", 4192)
+
+// Grid, block, and shared memory size definitions using environment variables
+#define GRID_SIZE_MMUL getEnvVariable("GRID_SIZE_MMUL", 256)
+#define BLOCK_SIZE_MMUL getEnvVariable("BLOCK_SIZE_MMUL", 512)
+#define SHARED_MEM_SIZE_MMUL getEnvVariable("SHARED_MEM_SIZE_MMUL", 8192)
+
+
+#define GRID_SIZE_IMAGE_RESIZE_NEIGHBOR getEnvVariable("GRID_SIZE_IMAGE_RESIZE_NEIGHBOR", 256)
+#define BLOCK_SIZE_IMAGE_RESIZE_NEIGHBOR getEnvVariable("BLOCK_SIZE_IMAGE_RESIZE_NEIGHBOR", 512)
+#define SHARED_MEM_SIZE_IMAGE_RESIZE_NEIGHBOR getEnvVariable("SHARED_MEM_SIZE_IMAGE_RESIZE_NEIGHBOR", 8192)
+
+#define GRID_SIZE_SWAP_UNSAFE getEnvVariable("GRID_SIZE_SWAP_UNSAFE", 256)
+#define BLOCK_SIZE_SWAP_UNSAFE getEnvVariable("BLOCK_SIZE_SWAP_UNSAFE", 512)
+#define SHARED_MEM_SIZE_SWAP_UNSAFE getEnvVariable("SHARED_MEM_SIZE_SWAP_UNSAFE", 8192)
+
+
+#define GRID_SIZE_DIGAMMA getEnvVariable("GRID_SIZE_DIGAMMA", 256)
+#define BLOCK_SIZE_DIGAMMA getEnvVariable("BLOCK_SIZE_DIGAMMA", 512)
+#define SHARED_MEM_SIZE_DIGAMMA getEnvVariable("SHARED_MEM_SIZE_DIGAMMA", 1024)
+
+
+#define GRID_SIZE_FILL_TRI getEnvVariable("GRID_SIZE_FILL_TRI", 256)
+#define BLOCK_SIZE_FILL_TRI getEnvVariable("BLOCK_SIZE_FILL_TRI", 512)
+#define SHARED_MEM_SIZE_FILL_TRI getEnvVariable("SHARED_MEM_SIZE_FILL_TRI", 1024)
+
+#define GRID_SIZE_IDENTITY getEnvVariable("GRID_SIZE_IDENTITY", 256)
+#define BLOCK_SIZE_IDENTITY getEnvVariable("GRID_SIZE_IDENTITY", 512)
+#define SHARED_MEM_SIZE_IDENTITY getEnvVariable("SHARED_MEM_SIZE_IDENTITY", 1024)
+
+
+#define GRID_SIZE_DYNAMIC_STITCH_TAD getEnvVariable("GRID_SIZE_DYNAMIC_STITCH_TAD", 512)
+#define BLOCK_SIZE_DYNAMIC_STITCH_TAD getEnvVariable("BLOCK_SIZE_DYNAMIC_STITCH_TAD", 512)
+#define SHARED_MEM_SIZE_DYNAMIC_STITCH_TAD getEnvVariable("SHARED_MEM_SIZE_DYNAMIC_STITCH_TAD", 1024)
+
+#define GRID_SIZE_DYNAMIC_PARTITION_TAD getEnvVariable("GRID_SIZE_DYNAMIC_PARTITION_TAD", 256)
+#define BLOCK_SIZE_DYNAMIC_PARTITION_TAD getEnvVariable("BLOCK_SIZE_DYNAMIC_PARTITION_TAD", 256)
+#define SHARED_MEM_SIZE_DYNAMIC_PARTITION_TAD getEnvVariable("SHARED_MEM_SIZE_DYNAMIC_PARTITION_TAD", 1024)
+
+
+#define GRID_SIZE_SOLVE getEnvVariable("GRID_SIZE_SOLVE", 100)
+#define BLOCK_SIZE_SOLVE getEnvVariable("BLOCK_SIZE_SOLVE", 1)
+#define SHARED_MEM_SIZE_SOLVE getEnvVariable("SHARED_MEM_SIZE_SOLVE", 256)
+
+#define GRID_SIZE_LUP getEnvVariable("GRID_SIZE_LUP", 128)
+#define BLOCK_SIZE_LUP getEnvVariable("BLOCK_SIZE_LUP", 256)
+#define SHARED_MEM_SIZE_LUP getEnvVariable("SHARED_MEM_SIZE_LUP", 1024)
+
+#define GRID_SIZE_SOFTMAX getEnvVariable("GRID_SIZE_SOFTMAX", 128)
+#define BLOCK_SIZE_SOFTMAX getEnvVariable("BLOCK_SIZE_SOFTMAX", 256)
+#define SHARED_MEM_SIZE_SOFTMAX getEnvVariable("SHARED_MEM_SIZE_SOFTMAX", 1024)
+
+
+dim3 getSoftmaxDims(int numTads);
+
+dim3 getLupDims(int batchSize);
+
+dim3 getDynamicPartitionDims(int numThreads,int yDTypeSize);
+
+dim3 getIdentityLaunchDims(int len,int rank);
+dim3 getRepeatLaunchDims(int len,int rank);
+
+dim3 getFillTriLaunchDims(int len,int rank);
+
+dim3 getGemVDims(int m);
+dim3 getAddBiasDims(int len,int rank) ;
+
+dim3 getLaunchDims(const std::string& key);
+
+dim3 getCol2imLaunchParams(sd::NDArray im,sd::NDArray col);
+dim3 getim2ColLaunchParams(sd::NDArray col);
+
+dim3 getMMulDims(int length,int sizeofDataType);
+dim3 getAccumDims(int xLength);
+
+dim3 getReduceDims(int xLength);
+dim3 getReduceAllDims(int xLength);
+dim3 getSortFullDims(int xLength);
+
+dim3 getSortTadLarge(int numberTads);
+dim3 getSortTadDims(int numberTads);
+
+dim3 getFillUpSegmentsDims(int numClasses,int length);
+
+dim3 getSegmentSumDims(int numClasses,int length);
+
+dim3 getSequenceMaskLaunchDims(int maxIndex,sd::NDArray input);
+
+dim3 getPoolingDims(int length,int rank);
+
+
+dim3 getCol2VolDims(int length,int rank);
+dim3 getVol2ColDims(int length,int rank);
+
+dim3 getUpsamplingDims(int length,int rank);
+dim3 getHistogramDims(int length,int numBins);
+dim3 getAdjustDims(int numTads);
+
+dim3 getBatchNormDims(int length);
+dim3 getCompareAndBitpackDims(int length);
+dim3 getCompareElem(int length);
+
+dim3 getConcat(int length);
+
+dim3 getBetaInc(int maxIter,int length,int dataTypeSize);
+dim3 getCross(int length,int rank,int lastSize);
+dim3 getDilation(int outputLength,int weightRank,int outputRank);
+
+dim3 getGatherLinear(int numSubArrs);
+dim3 getGatherNd(int outputLength,int maxRank);
+
+dim3 getHashCodeSplit(int length,int blockSize);
+dim3 getHashCodeInternal( int numBlocks);
+
+dim3 cropAndResize(int batchSize,int imageHeight,int imageWidth,int cropHeight,int cropWidth);
+dim3 imageHelper(int numTads);
+
+dim3 lrnDims(int tadLength,int numTads, int xDTypeSize,int zDTypeSize);
+
+dim3 lupDims(int n);
+dim3 lupDimsLow(int n);
+
+dim3 matrixSetDiagDims(int length,int rank);
+
+dim3 mergeDims(int length);
+
+dim3 oneHotDims(int length,int rank,int shapeSize);
+
+dim3 padDims(int length,int rank);
+
+dim3 polygammaDims(int length);
+
+dim3 prefixDims(int numTads,int sizeOfDataType);
+
+dim3 randomShuffleFisherDims(int power,int inputDataTypeSize);
+dim3 randomShuffleMergeDims(int j,int length);
+
+dim3 batchToSpaceNdLaunch(int length,int rank);
+dim3 spaceToBatchLaunch(int length,int rank);
+dim3 spaceToBatchNdLaunch(int length,int rank);
+dim3 scatterDims(int length,int rank);
+dim3 scatterDimsCheckIndices(int length,int rank);
+dim3 scatterNdDims(int length,int rank);
+
+dim3 segmentValidateIndices(int length);
+
+dim3 segmentDims(int numClasses,int length);
+dim3 segmentTad(int size);
+
+dim3 segmentBpDims(int gradOutLen,int inputLen);
+dim3 segmentBpTad(int indicesLen,int inputLen);
+dim3 sruBiDims(int len,int rank);
+
+dim3 stackDims(int length);
+
+dim3 topkDims(int numTads);
+
+dim3 topKIndices(int scanWidth,int xDTypeSize,int yDTypeSize);
+
+dim3 invertPermutationDims(int length);
+dim3 traceDims(int length);
+dim3 triuDims(int length,int rank);
+dim3 tileDims(int length,int rank);
+
+dim3 updaterDims(int length);
+dim3 zetaDims(int length);
+
+dim3 resizeNeighborDims(int batchSize,int height,int width);
+
+dim3 clipDims(int length);
+
+dim3 mirrorPadLinearDims(int length);
+dim3 mirrorPadTad(int length,int rank);
+
+
+dim3 digammaDims(int length);
+
+#endif //LIBND4J_LAUNCHCONTEXT_H

libnd4j/include/execution/impl/BlockingQueue.cpp

@@ -0,0 +1,75 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <execution/BlockingQueue.h>
+#include <execution/CallableWithArguments.h>
+
+#include <thread>
+
+namespace samediff {
+template <typename T>
+BlockingQueue<T>::BlockingQueue(int queueSize) {
+  _size = 0;
+  _available = true;
+}
+
+template <typename T>
+T BlockingQueue<T>::poll() {
+  // locking untill there's something within queue
+  std::unique_lock<std::mutex> lock(_lock);
+  _condition.wait(lock, [&] { return this->_size.load() != 0; });
+
+  T t(std::move(_queue.front()));
+  _queue.pop();
+  _size--;
+  return t;
+}
+
+template <typename T>
+void BlockingQueue<T>::put(const T &t) {
+  {
+    // locking before push, unlocking after
+    std::unique_lock<std::mutex> lock(_lock);
+    _queue.push(t);
+    _size++;
+  }
+
+  // notifying condition
+  _condition.notify_one();
+}
+
+template <typename T>
+bool BlockingQueue<T>::available() {
+  return _available.load();
+}
+
+template <typename T>
+void BlockingQueue<T>::markAvailable() {
+  _available = true;
+}
+
+template <typename T>
+void BlockingQueue<T>::markUnavailable() {
+  _available = false;
+}
+
+template class BlockingQueue<CallableWithArguments *>;
+}  // namespace samediff

libnd4j/include/execution/impl/CallableInterface.cpp

@@ -0,0 +1,215 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <execution/CallableInterface.h>
+#include <helpers/logger.h>
+
+namespace samediff {
+CallableInterface::CallableInterface() {
+  // initial state is available
+  _available = true;
+  _filled = false;
+  _finished = false;
+}
+
+bool CallableInterface::available() { return _available.load(); }
+
+void CallableInterface::markUnavailable() { _available = false; }
+
+void CallableInterface::markAvailable() { _available = true; }
+
+void CallableInterface::fill(int threadID, int numThreads, FUNC_DO func) {
+  _function_do = std::move(func);
+
+  _branch = 0;
+  _num_threads = numThreads;
+  _thread_id = threadID;
+  _finished = false;
+  {
+    std::unique_lock<std::mutex> l(_ms);
+    _filled = true;
+  }
+  _starter.notify_one();
+}
+
+void CallableInterface::fill(int threadID, int numThreads, FUNC_1D func, int64_t startX, int64_t stopX, int64_t incX) {
+  _function_1d = std::move(func);
+  _arguments[0] = startX;
+  _arguments[1] = stopX;
+  _arguments[2] = incX;
+
+  _branch = 1;
+  _num_threads = numThreads;
+  _thread_id = threadID;
+  _finished = false;
+
+  {
+    std::unique_lock<std::mutex> l(_ms);
+    _filled = true;
+  }
+  _starter.notify_one();
+}
+
+void CallableInterface::fill(int threadID, int numThreads, FUNC_2D func, int64_t startX, int64_t stopX, int64_t incX,
+                             int64_t start_y, int64_t stop_y, int64_t inc_y) {
+  _function_2d = std::move(func);
+  _arguments[0] = startX;
+  _arguments[1] = stopX;
+  _arguments[2] = incX;
+  _arguments[3] = start_y;
+  _arguments[4] = stop_y;
+  _arguments[5] = inc_y;
+
+  _branch = 2;
+  _num_threads = numThreads;
+  _thread_id = threadID;
+  _finished = false;
+
+  {
+    std::unique_lock<std::mutex> l(_ms);
+    _filled = true;
+  }
+  _starter.notify_one();
+}
+
+void CallableInterface::fill(int threadID, int numThreads, FUNC_3D func, int64_t startX, int64_t stopX, int64_t incX,
+                             int64_t start_y, int64_t stop_y, int64_t inc_y, int64_t start_z, int64_t stop_z,
+                             int64_t inc_z) {
+  _function_3d = std::move(func);
+  _arguments[0] = startX;
+  _arguments[1] = stopX;
+  _arguments[2] = incX;
+  _arguments[3] = start_y;
+  _arguments[4] = stop_y;
+  _arguments[5] = inc_y;
+  _arguments[6] = start_z;
+  _arguments[7] = stop_z;
+  _arguments[8] = inc_z;
+
+  _branch = 3;
+  _num_threads = numThreads;
+  _thread_id = threadID;
+  _finished = false;
+
+  {
+    std::unique_lock<std::mutex> l(_ms);
+    _filled = true;
+  }
+  _starter.notify_one();
+}
+
+void CallableInterface::fill(int threadID, int numThreads, int64_t *lptr, FUNC_RL func, int64_t startX, int64_t stopX,
+                             int64_t incX) {
+  _function_rl = std::move(func);
+  _arguments[0] = startX;
+  _arguments[1] = stopX;
+  _arguments[2] = incX;
+
+  _lptr = lptr;
+
+  _branch = 4;
+  _num_threads = numThreads;
+  _thread_id = threadID;
+  _finished = false;
+
+  {
+    std::unique_lock<std::mutex> l(_ms);
+    _filled = true;
+  }
+  _starter.notify_one();
+}
+
+void CallableInterface::fill(int threadID, int numThreads, double *dptr, FUNC_RD func, int64_t startX, int64_t stopX,
+                             int64_t incX) {
+  _function_rd = std::move(func);
+  _arguments[0] = startX;
+  _arguments[1] = stopX;
+  _arguments[2] = incX;
+
+  _dptr = dptr;
+
+  _branch = 5;
+  _num_threads = numThreads;
+  _thread_id = threadID;
+  _finished = false;
+
+  {
+    std::unique_lock<std::mutex> l(_ms);
+    _filled = true;
+  }
+  _starter.notify_one();
+}
+
+void CallableInterface::waitForTask() {
+  // block until task is available
+  std::unique_lock<std::mutex> lock(_ms);
+  _starter.wait(lock, [&] { return _filled.load(); });
+}
+
+void CallableInterface::waitForCompletion() {
+  // while (!_finished.load());
+
+  // block until finished
+  std::unique_lock<std::mutex> lock(_mf);
+  _finisher.wait(lock, [&] { return _finished.load(); });
+}
+
+void CallableInterface::finish() {
+  // mark as finished
+  {
+    std::unique_lock<std::mutex> l(_mf);
+    _finished.store(true);
+  }
+  _finisher.notify_one();
+}
+
+void CallableInterface::execute() {
+  // mark it as consumed
+  _filled = false;
+
+  // actually executing op
+  switch (_branch) {
+    case 0:
+      _function_do(_thread_id, _num_threads);
+      break;
+    case 1:
+      _function_1d(_thread_id, _arguments[0], _arguments[1], _arguments[2]);
+      break;
+    case 2:
+      _function_2d(_thread_id, _arguments[0], _arguments[1], _arguments[2], _arguments[3], _arguments[4],
+                   _arguments[5]);
+      break;
+    case 3:
+      _function_3d(_thread_id, _arguments[0], _arguments[1], _arguments[2], _arguments[3], _arguments[4], _arguments[5],
+                   _arguments[6], _arguments[7], _arguments[8]);
+      break;
+    case 4:
+      _lptr[0] = _function_rl(_thread_id, _arguments[0], _arguments[1], _arguments[2]);
+      break;
+    case 5:
+      _dptr[0] = _function_rd(_thread_id, _arguments[0], _arguments[1], _arguments[2]);
+      break;
+  }
+
+  // notify that thread finished the job
+  this->finish();
+}
+}  // namespace samediff

libnd4j/include/execution/impl/CallableWithArguments.cpp

@@ -0,0 +1,90 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <execution/CallableWithArguments.h>
+
+namespace samediff {
+CallableWithArguments::CallableWithArguments(FUNC_DO func, uint64_t thread_id, uint64_t numThreads) {
+  _function_do = func;
+  _finished = false;
+  _threadId = thread_id;
+  _numThreads = numThreads;
+  _dimensions = 0;
+}
+
+CallableWithArguments::CallableWithArguments(FUNC_3D func, uint64_t thread_id, int64_t start_x, int64_t stop_x,
+                                             int64_t increment_x, int64_t start_y, int64_t stop_y, int64_t increment_y,
+                                             int64_t start_z, int64_t stop_z, int64_t increment_z) {
+  _function_3d = func;
+  _arguments = {start_x, stop_x, increment_x, start_y, stop_y, increment_y, start_z, stop_z, increment_z};
+  _finished = false;
+  _threadId = thread_id;
+  _dimensions = 3;
+}
+
+CallableWithArguments::CallableWithArguments(FUNC_1D func, uint64_t thread_id, int64_t start_x, int64_t stop_x,
+                                             int64_t increment_x) {
+  _function_1d = func;
+  _arguments = {start_x, stop_x, increment_x};
+  _finished = false;
+  _threadId = thread_id;
+  _dimensions = 1;
+}
+
+CallableWithArguments::CallableWithArguments(FUNC_2D func, uint64_t thread_id, int64_t start_x, int64_t stop_x,
+                                             int64_t increment_x, int64_t start_y, int64_t stop_y,
+                                             int64_t increment_y) {
+  _function_2d = func;
+  _arguments = {start_x, stop_x, increment_x, start_y, stop_y, increment_y};
+  _finished = false;
+  _threadId = thread_id;
+  _dimensions = 2;
+}
+
+int CallableWithArguments::dimensions() { return _dimensions; }
+
+std::vector<int64_t>& CallableWithArguments::arguments() { return _arguments; }
+
+bool CallableWithArguments::finished() { return _finished.load(); }
+
+void CallableWithArguments::finish() {
+  std::lock_guard<std::mutex> lock(_lock);
+  _finished = true;
+  _condition.notify_one();
+}
+
+void CallableWithArguments::waitUntilFinished() {
+  std::unique_lock<std::mutex> lock(_lock);
+  _condition.wait(lock, [&] { return _finished.load(); });
+}
+
+FUNC_1D CallableWithArguments::function_1d() { return _function_1d; }
+
+FUNC_2D CallableWithArguments::function_2d() { return _function_2d; }
+
+FUNC_DO CallableWithArguments::function_do() { return _function_do; }
+
+FUNC_3D CallableWithArguments::function_3d() { return _function_3d; }
+
+uint64_t CallableWithArguments::threadId() { return _threadId; }
+
+uint64_t CallableWithArguments::numThreads() { return _numThreads; }
+}  // namespace samediff

libnd4j/include/execution/impl/ErrorReference.cpp

@@ -0,0 +1,49 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#include <execution/ErrorReference.h>
+
+namespace sd {
+int ErrorReference::errorCode() { return _errorCode; }
+
+const char* ErrorReference::errorMessage() {
+  // since we're fetching error message - error code will be assumed consumed & nullified
+  _errorCode = 0;
+  if(_errorMessage != nullptr)
+    return _errorMessage->c_str();
+  return "";
+}
+
+void ErrorReference::setErrorCode(int errorCode) { _errorCode = errorCode; }
+
+void ErrorReference::setErrorMessage(std::string message) {
+  if(_errorMessage != nullptr)
+    delete _errorMessage;
+  _errorMessage = new std::string(message);
+}
+
+void ErrorReference::setErrorMessage(const char* message) {
+  if(_errorMessage != nullptr)
+    delete _errorMessage;
+  _errorMessage = new std::string(message);
+}
+}  // namespace sd

libnd4j/include/execution/impl/ThreadPool.cpp

@@ -0,0 +1,175 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <execution/ThreadPool.h>
+#include <helpers/logger.h>
+
+#include <stdexcept>
+
+
+namespace samediff {
+
+// this function executed once per thread, it polls functions from queue, and executes them via wrapper
+static void executionLoop_(int thread_id, BlockingQueue<CallableWithArguments *> *queue) {
+  while (true) {
+    // this method blocks until there's something within queue
+    auto c = queue->poll();
+    switch (c->dimensions()) {
+      case 0: {
+        c->function_do()(c->threadId(), c->numThreads());
+        c->finish();
+      } break;
+      case 1: {
+        auto args = c->arguments();
+        c->function_1d()(c->threadId(), args[0], args[1], args[2]);
+        c->finish();
+      } break;
+      case 2: {
+        auto args = c->arguments();
+        c->function_2d()(c->threadId(), args[0], args[1], args[2], args[3], args[4], args[5]);
+        c->finish();
+      } break;
+      case 3: {
+        auto args = c->arguments();
+        c->function_3d()(c->threadId(), args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7],
+                         args[8]);
+        c->finish();
+      } break;
+      default:
+        THROW_EXCEPTION("Don't know what to do with provided Callable");
+    }
+  }
+}
+
+static void executionLoopWithInterface_(int thread_id, CallableInterface *c) {
+  while (true) {
+    // blocking here until there's something to do
+    c->waitForTask();
+
+    // execute whatever we have
+    c->execute();
+  }
+}
+
+ThreadPool::ThreadPool() {
+  // TODO: number of threads must reflect number of cores for UMA system. In case of NUMA it should be per-device pool
+  // FIXME: on mobile phones this feature must NOT be used
+  _available = sd::Environment::getInstance().maxThreads();
+
+  _queues.resize(_available.load());
+  _threads.resize(_available.load());
+  _interfaces.resize(_available.load());
+
+  // we're not creating threadpool on aurora
+  // creating threads here
+  for (int e = 0; e < _available.load(); e++) {
+    _queues[e] = new BlockingQueue<CallableWithArguments *>(2);
+    _interfaces[e] = new CallableInterface();
+    _threads[e] = std::thread(executionLoopWithInterface_, e, _interfaces[e]);
+    _tickets.push(new Ticket());
+    // _threads[e] = new std::thread(executionLoop_, e, _queues[e]);
+
+    // TODO: add other platforms here as well
+    // now we must set affinity, and it's going to be platform-specific thing
+#ifdef LINUX_BUILD
+    cpu_set_t cpuset;
+    CPU_ZERO(&cpuset);
+    CPU_SET(e, &cpuset);
+    int rc = pthread_setaffinity_np(_threads[e]->native_handle(), sizeof(cpu_set_t), &cpuset);
+    if (rc != 0) THROW_EXCEPTION("Failed to set pthread affinity");
+#endif
+
+  }
+  //add an extra ticket to minimize the risk of running out of tickets due to race conditions
+  _tickets.push(new Ticket());
+}
+
+ThreadPool::~ThreadPool() {
+  // TODO: implement this one properly
+  for (size_t e = 0; e < _queues.size(); e++) {
+    // stop each and every thread
+
+    // release queue and thread
+    delete _queues[e];
+    _threads[e].detach();
+    delete _interfaces[e];
+  }
+
+  while (!_tickets.empty()) {
+    auto t = _tickets.front();
+    _tickets.pop();
+    delete t;
+  }
+}
+
+ThreadPool &ThreadPool::getInstance() {
+  static ThreadPool instance;
+  return instance;
+}
+
+void ThreadPool::release(int numThreads) { _available += numThreads; }
+
+Ticket *ThreadPool::tryAcquire(int numThreads) {
+  if (numThreads <= 0) return nullptr;
+  Ticket *t = nullptr;
+  // we check for threads availability first
+  bool threaded = false;
+  {
+    // we lock before checking availability
+    std::unique_lock<std::mutex> lock(_lock);
+    //test for both _available and _tickets in order to deal with race conditions caused by the
+    //fact that marking threads as available AND releasing tickets does not happen atomically
+    if (_available >= numThreads && !_tickets.empty()) {
+      threaded = true;
+      _available -= numThreads;
+
+      // getting a ticket from the queue
+      t = _tickets.front();
+      _tickets.pop();
+
+      // ticket must contain information about number of threads for the current session
+      t->acquiredThreads(numThreads);
+
+      // filling ticket with executable interfaces
+      for (size_t e = 0, i = 0; e < _queues.size() && i < static_cast<size_t>(numThreads); e++) {
+        if (_interfaces[e]->available()) {
+          t->attach(i++, _interfaces[e]);
+          _interfaces[e]->markUnavailable();
+        }
+      }
+    }
+  }
+
+  // we either dispatch tasks to threads, or run single-threaded
+  if (threaded) {
+    return t;
+  } else {
+    // if there's no threads available - return nullptr
+    return nullptr;
+  }
+}
+
+void ThreadPool::release(samediff::Ticket *ticket) {
+  // returning ticket back to the queue
+  std::unique_lock<std::mutex> lock(_lock);
+  _tickets.push(ticket);
+}
+}  // namespace samediff

libnd4j/include/execution/impl/Threads.cpp

@@ -0,0 +1,953 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <execution/Threads.h>
+#include <execution/ThreadPool.h>
+#include <vector>
+#include <thread>
+#include <helpers/logger.h>
+#include <math/templatemath.h>
+#include <helpers/shape.h>
+
+#ifdef _OPENMP
+
+#include <omp.h>
+
+#endif
+
+
+namespace samediff {
+
+int ThreadsHelper::numberOfThreads(int maxThreads, uint64_t numberOfElements) {
+  // let's see how many threads we actually need first
+  auto optimalThreads = sd::math::sd_max<uint64_t>(1, numberOfElements / 1024);
+
+  // now return the smallest value
+  return sd::math::sd_min<int>(optimalThreads, maxThreads);
+}
+
+Span3::Span3(int64_t startX, int64_t stopX, int64_t incX, int64_t startY, int64_t stopY, int64_t incY, int64_t startZ, int64_t stopZ, int64_t incZ) {
+  _startX = startX;
+  _startY = startY;
+  _startZ = startZ;
+  _stopX = stopX;
+  _stopY = stopY;
+  _stopZ = stopZ;
+  _incX = incX;
+  _incY = incY;
+  _incZ = incZ;
+}
+
+Span3 Span3::build(int loop, uint64_t threadID, uint64_t numThreads, int64_t startX, int64_t stopX, int64_t incX, int64_t startY, int64_t stopY, int64_t incY, int64_t startZ, int64_t stopZ, int64_t incZ) {
+  switch (loop) {
+    case 1: {
+      auto span = (stopX - startX) / numThreads;
+      auto s = span * threadID;
+      auto e = s + span;
+      if (threadID == numThreads - 1)
+        e = stopX;
+
+      return Span3(s, e, incX, startY, stopY, incY, startZ, stopZ, incZ);
+    }
+      break;
+    case 2: {
+      auto span = (stopY - startY) / numThreads;
+      auto s = span * threadID;
+      auto e = s + span;
+      if (threadID == numThreads - 1)
+        e = stopY;
+
+      return Span3(startX, stopX, incX, s, e, incY, startZ, stopZ, incZ);
+    }
+      break;
+    case 3: {
+      auto span = (stopZ - startZ) / numThreads;
+      auto s = span * threadID;
+      auto e = s + span;
+      if (threadID == numThreads - 1)
+        e = stopZ;
+
+      return Span3(startX, stopX, incX, startY, stopY, incY, s, e, incZ);
+    }
+      break;
+    default:
+      THROW_EXCEPTION("");
+  }
+  return Span3(startX, stopX, incX, startY, stopY, incY, startZ, stopZ, incZ);
+}
+
+Span::Span(int64_t startX, int64_t stopX, int64_t incX) {
+  _startX = startX;
+  _stopX = stopX;
+  _incX = incX;
+}
+
+Span Span::build(uint64_t threadID, uint64_t numThreads, int64_t startX, int64_t stopX, int64_t incX) {
+  auto span = (stopX - startX) / numThreads;
+  auto s = span * threadID;
+  auto e = s + span;
+  if (threadID == numThreads - 1)
+    e = stopX;
+
+  return Span(s, e, incX);
+}
+
+Span2::Span2(int64_t startX, int64_t stopX, int64_t incX, int64_t startY, int64_t stopY, int64_t incY) {
+  _startX = startX;
+  _startY = startY;
+  _stopX = stopX;
+  _stopY = stopY;
+  _incX = incX;
+  _incY = incY;
+}
+
+
+Span2 Span2::build(int loop, uint64_t threadID, uint64_t numThreads, int64_t startX, int64_t stopX, int64_t incX, int64_t startY, int64_t stopY, int64_t incY) {
+
+  switch (loop) {
+    case 1: {
+      auto span = (stopX - startX) / numThreads;
+      auto s = span * threadID;
+      auto e = s + span;
+      if (threadID == numThreads - 1)
+        e = stopX;
+
+      return Span2(s, e, incX, startY, stopY, incY);
+    }
+      break;
+    case 2: {
+      auto span = (stopY - startY) / numThreads;
+      auto s = span * threadID;
+      auto e = s + span;
+      if (threadID == numThreads - 1)
+        e = stopY;
+
+      return Span2(startX, stopX, incX, s, e, incY);
+    }
+      break;
+    default:
+      THROW_EXCEPTION("");
+  }
+
+  return Span2(startX, stopX, incX, 0, 0, incY);
+
+}
+
+int64_t Span::startX() const {
+  return _startX;
+}
+
+int64_t Span::stopX() const {
+  return _stopX;
+}
+
+int64_t Span::incX() const {
+  return _incX;
+}
+
+int64_t Span2::startX() const {
+  return _startX;
+}
+
+int64_t Span2::startY() const {
+  return _startY;
+}
+
+int64_t Span2::stopX() const {
+  return _stopX;
+}
+
+int64_t Span2::stopY() const {
+  return _stopY;
+}
+
+int64_t Span2::incX() const {
+  return _incX;
+}
+
+int64_t Span2::incY() const {
+  return _incY;
+}
+
+int64_t Span3::startX() const {
+  return _startX;
+}
+
+int64_t Span3::startY() const {
+  return _startY;
+}
+
+int64_t Span3::startZ() const {
+  return _startZ;
+}
+
+int64_t Span3::stopX() const {
+  return _stopX;
+}
+
+int64_t Span3::stopY() const {
+  return _stopY;
+}
+
+int64_t Span3::stopZ() const {
+  return _stopZ;
+}
+
+int64_t Span3::incX() const {
+  return _incX;
+}
+
+int64_t Span3::incY() const {
+  return _incY;
+}
+
+int64_t Span3::incZ() const {
+  return _incZ;
+}
+
+int ThreadsHelper::pickLoop2d(int numThreads, uint64_t itersX, uint64_t itersY) {
+  // if one of dimensions is definitely too small - we just pick the other one
+  if (itersX < static_cast<uint64_t>(numThreads) && itersY >= static_cast<uint64_t>(numThreads))
+    return 2;
+  if (itersY <  static_cast<uint64_t>(numThreads) && itersX >= static_cast<uint64_t>(numThreads))
+    return 1;
+
+  // next step - we pick the most balanced dimension
+  auto remX = itersX % numThreads;
+  auto remY = itersY % numThreads;
+  auto splitY = itersY / numThreads;
+
+  // if there's no remainder left in some dimension - we're picking that dimension, because it'll be the most balanced work distribution
+  if (remX == 0)
+    return 1;
+  if (remY == 0)
+    return 2;
+
+  // if there's no loop without a remainder - we're picking one with smaller remainder
+  if (remX < remY)
+    return 1;
+  if (remY < remX && splitY >= 64) // we don't want too small splits over last dimension, or vectorization will fail
+    return 2;
+  // if loops are equally sized - give the preference to the first thread
+  return 1;
+}
+
+
+static int threads_(int maxThreads, uint64_t elements) {
+  uint64_t  typeCastedMaxThreads =  static_cast<uint64_t>(maxThreads);
+  if ( static_cast<uint64_t>(elements) == typeCastedMaxThreads) {
+    return maxThreads;
+  }
+  else if (elements > typeCastedMaxThreads) {
+    // if we have full load across thread, or at least half of threads can be utilized
+    auto rem = elements % typeCastedMaxThreads;
+    if (rem == 0 || rem >= typeCastedMaxThreads / 3)
+      return maxThreads;
+    else
+      return threads_(maxThreads - 1, elements);
+
+  }
+  else if (elements < typeCastedMaxThreads) {
+    return elements;
+  }
+
+  return 1;
+}
+
+int ThreadsHelper::numberOfThreads2d(int maxThreads, uint64_t iters_x, uint64_t iters_y) {
+  uint64_t  typeCastedMaxThreads =  static_cast<uint64_t>(maxThreads);
+  // in some cases there's nothing to think about, part 1
+  if (iters_x < typeCastedMaxThreads && iters_y < typeCastedMaxThreads)
+    return sd::math::sd_max<int>(iters_x, iters_y);
+
+  auto remX = iters_x % maxThreads;
+  auto remY = iters_y % maxThreads;
+
+  // in some cases there's nothing to think about, part 2
+  if ((iters_x >= typeCastedMaxThreads && remX == 0) || (iters_y >= typeCastedMaxThreads && remY == 0))
+    return maxThreads;
+
+  // at this point we suppose that there's no loop perfectly matches number of our threads
+  // so let's pick something as equal as possible
+  if (iters_x > typeCastedMaxThreads || iters_y > typeCastedMaxThreads)
+    return maxThreads;
+  else
+    return numberOfThreads2d(maxThreads - 1, iters_x, iters_y);
+}
+
+int ThreadsHelper::numberOfThreads3d(int maxThreads, uint64_t itersX, uint64_t itersY, uint64_t itersZ) {
+  // we don't want to run underloaded threads
+  if (itersX * itersY * itersZ <= 32)
+    return 1;
+
+  uint64_t typeCastedMaxThreads =  static_cast<uint64_t>(maxThreads);
+  auto remX = itersX % maxThreads;
+  auto remY = itersY % maxThreads;
+  auto remZ = itersZ % maxThreads;
+
+  // if we have perfect balance across one of dimensions - just go for it
+  if ((itersX >= typeCastedMaxThreads && remX == 0) || (itersY >= typeCastedMaxThreads && remY == 0) || (itersZ >= typeCastedMaxThreads && remZ == 0))
+    return maxThreads;
+
+  int threadsX = 0, threadsY = 0, threadsZ = 0;
+
+  // now we look into possible number of
+  threadsX = threads_(maxThreads, itersX);
+  threadsY = threads_(maxThreads, itersY);
+  threadsZ = threads_(maxThreads, itersZ);
+
+  // we want to split as close to outer loop as possible, so checking it out first
+  if (threadsX >= threadsY && threadsX >= threadsZ)
+    return threadsX;
+  else if (threadsY >= threadsX && threadsY >= threadsZ)
+    return threadsY;
+  else if (threadsZ >= threadsX && threadsZ >= threadsY)
+    return threadsZ;
+
+  return 1;
+}
+
+int ThreadsHelper::pickLoop3d(int numThreads, uint64_t itersX, uint64_t itersY, uint64_t itersZ) {
+  auto remX = itersX % numThreads;
+  auto remY = itersY % numThreads;
+  auto remZ = itersZ % numThreads;
+
+  auto splitX = itersX / numThreads;
+  auto splitY = itersY / numThreads;
+  auto splitZ = itersZ / numThreads;
+
+  uint64_t typeCastedNumThreads =  static_cast<uint64_t>(numThreads);
+  // if there's no remainder left in some dimension - we're picking that dimension, because it'll be the most balanced work distribution
+  if (remX == 0)
+    return 1;
+  else if (remY == 0)
+    return 2;
+  else if (remZ == 0) // TODO: we don't want too smal splits over last dimension? or we do?
+    return 3;
+
+  if (itersX > typeCastedNumThreads)
+    return 1;
+  else if (itersY > typeCastedNumThreads)
+    return 2;
+  else if (itersZ > typeCastedNumThreads)
+    return 3;
+
+  return 1;
+}
+
+#ifdef _OPENMP
+
+std::mutex Threads::gThreadmutex;
+	uint64_t Threads::_nFreeThreads = sd::Environment::getInstance().maxThreads();
+
+	bool   Threads::tryAcquire(int numThreads) {
+		std::lock_guard<std::mutex> lock(gThreadmutex);
+		auto nThreads = _nFreeThreads - numThreads;
+		//error: comparison of unsigned expression in ‘>= 0’ is always true [-Werror=type-limits]
+
+		_nFreeThreads = nThreads;
+     	return true;
+
+	}
+
+	bool  Threads::freeThreads(int numThreads) {
+		std::lock_guard<std::mutex> lock(gThreadmutex);
+		_nFreeThreads += numThreads;
+		// check if correct number of threads
+		return _nFreeThreads > static_cast<uint64_t >(sd::Environment::getInstance().maxThreads());
+	}
+#endif
+
+int Threads::parallel_tad(FUNC_1D function, sd::LongType start, sd::LongType stop, sd::LongType increment,
+                          sd::LongType numThreads) {
+  if (start > stop)
+    THROW_EXCEPTION("Threads::parallel_for got start > stop");
+
+  auto delta = (stop - start);
+
+  if (numThreads > delta)
+    numThreads = delta;
+
+  if (numThreads == 0)
+    return 0;
+
+  // shortcut
+  if (numThreads == 1) {
+    function(0, start, stop, increment);
+    return 1;
+  }
+
+#ifdef _OPENMP
+  if (tryAcquire(numThreads)) {
+
+			auto span = delta / numThreads;
+#pragma omp parallel for  schedule(guided) proc_bind(close) default(shared)
+			for (sd::LongType e = 0; e < numThreads; e++) {
+				auto start_ = span * e + start;
+				auto stop_ = start_ + span;
+				if (e == numThreads - 1)
+					stop_ = stop;
+				function(e, start_, stop_, increment);
+			}
+			freeThreads(numThreads);
+			return numThreads;
+		}
+		else {
+                  // if there were no threads available - we'll execute function right within current thread
+			function(0, start, stop, increment);
+
+			// we tell that parallelism request declined
+			return 1;
+		}
+#else
+
+  sd::Environment::getInstance().maxThreads();
+  auto ticket = ThreadPool::getInstance().tryAcquire(numThreads);
+  if (ticket != nullptr) {
+
+    // if we got our threads - we'll run our jobs here
+    auto span = delta / numThreads;
+
+    for (uint32_t e = 0; e < numThreads; e++) {
+      auto start_ = span * e + start;
+      auto stop_ = start_ + span;
+
+      // last thread will process tail
+      if (e == numThreads - 1)
+        stop_ = stop;
+
+      // putting the task into the queue for a given thread
+      ticket->enqueue(e, numThreads, function, start_, stop_, increment);
+    }
+
+    // block and wait till all threads finished the job
+    ticket->waitAndRelease();
+
+    // we tell that parallelism request succeeded
+    return numThreads;
+  }
+  else {
+
+    // if there were no threads available - we'll execute function right within current thread
+    function(0, start, stop, increment);
+
+    // we tell that parallelism request declined
+    return 1;
+  }
+#endif
+}
+
+int Threads::parallel_for(FUNC_1D function, sd::LongType start, sd::LongType stop, sd::LongType increment,
+                          sd::LongType numThreads) {
+  if (start > stop)
+    THROW_EXCEPTION("Threads::parallel_for got start > stop");
+
+  auto delta = (stop - start);
+
+  // in some cases we just fire func as is
+  if (delta == 0 || numThreads == 1) {
+    function(0, start, stop, increment);
+    return 1;
+  }
+
+  auto numElements = delta / increment;
+
+  // we decide what's optimal number of threads we need here, and execute it in parallel_tad.
+  numThreads = ThreadsHelper::numberOfThreads(numThreads, numElements);
+  return parallel_tad(function, start, stop, increment, numThreads);
+}
+
+int Threads::parallel_for(FUNC_2D function, int64_t startX, int64_t stopX, int64_t incX, int64_t startY, int64_t stopY, int64_t incY, uint64_t numThreads, bool debug) {
+  if (startX > stopX)
+    THROW_EXCEPTION("Threads::parallel_for got startX > stopX");
+
+  if (startY > stopY)
+    THROW_EXCEPTION("Threads::parallel_for got startY > stopY");
+
+  // number of elements per loop
+  auto delta_x = (stopX - startX);
+  auto delta_y = (stopY - startY);
+
+  // number of iterations per loop
+  auto itersX = delta_x / incX;
+  auto itersY = delta_y / incY;
+
+  // total number of iterations
+  auto iters_t = itersX * itersY;
+
+  // we are checking the case of number of requested threads was smaller
+  numThreads = ThreadsHelper::numberOfThreads2d(numThreads, itersX, itersY);
+
+  // basic shortcut for no-threading cases
+  if (numThreads == 1) {
+    function(0, startX, stopX, incX, startY, stopY, incY);
+    return 1;
+  }
+
+  // We have couple of scenarios:
+  // either we split workload along 1st loop, or 2nd
+  auto splitLoop = ThreadsHelper::pickLoop2d(numThreads, itersX, itersY);
+
+  // for debug mode we execute things inplace, without any threads
+  if (debug) {
+    for (uint64_t e = 0; e < numThreads; e++) {
+      auto span = Span2::build(splitLoop, e, numThreads, startX, stopX, incX, startY, stopY, incY);
+
+      function(e, span.startX(), span.stopX(), span.incX(), span.startY(), span.stopY(), span.incY());
+    }
+
+    // but we still mimic multithreaded execution
+    return numThreads;
+  }
+  else {
+#ifdef _OPENMP
+
+    if (tryAcquire(numThreads)) {
+#pragma omp parallel for
+				for (uint64_t e = 0; e < numThreads; e++) {
+					auto span = Span2::build(splitLoop, e, numThreads, startX, stopX, incX, startY, stopY, incY);
+					function(e, span.startX(), span.stopX(), span.incX(), span.startY(), span.stopY(), span.incY());
+				}
+				freeThreads(numThreads);
+				return numThreads;
+			}
+			else {
+				// if there were no threads available - we'll execute function right within current thread
+				function(0, startX, stopX, incX, startY, stopY, incY);
+
+				// we tell that parallelism request declined
+				return 1;
+			}
+
+#else
+
+    auto ticket = ThreadPool::getInstance().tryAcquire(numThreads);
+    if (ticket != nullptr) {
+      for (uint64_t e = 0; e < numThreads; e++) {
+        auto threadId = numThreads - e - 1;
+        auto span = Span2::build(splitLoop, threadId, numThreads, startX, stopX, incX, startY, stopY, incY);
+
+        ticket->enqueue(e, numThreads, function, span.startX(), span.stopX(), span.incX(), span.startY(), span.stopY(), span.incY());
+      }
+
+      // block until all threads finish their job
+      ticket->waitAndRelease();
+
+      return numThreads;
+    }
+    else {
+      // if there were no threads available - we'll execute function right within current thread
+      function(0, startX, stopX, incX, startY, stopY, incY);
+
+      // we tell that parallelism request declined
+      return 1;
+    }
+#endif
+  };
+}
+
+
+int Threads::parallel_for(FUNC_3D function, int64_t startX, int64_t stopX, int64_t incX, int64_t startY, int64_t stopY, int64_t incY, int64_t startZ, int64_t stopZ, int64_t incZ, uint64_t numThreads) {
+  if (startX > stopX)
+    THROW_EXCEPTION("Threads::parallel_for got startX > stopX");
+
+  if (startY > stopY)
+    THROW_EXCEPTION("Threads::parallel_for got startY > stopY");
+
+  if (startZ > stopZ)
+    THROW_EXCEPTION("Threads::parallel_for got startZ > stopZ");
+
+  auto delta_x = stopX - startX;
+  auto delta_y = stopY - startY;
+  auto delta_z = stopZ - startZ;
+
+  auto itersX = delta_x / incX;
+  auto itersY = delta_y / incY;
+  auto itersZ = delta_z / incZ;
+
+  numThreads = ThreadsHelper::numberOfThreads3d(numThreads, itersX, itersY, itersZ);
+  if (numThreads == 1) {
+    // loop is too small - executing function as is
+    function(0, startX, stopX, incX, startY, stopY, incY, startZ, stopZ, incZ);
+    return 1;
+  }
+
+#ifdef _OPENMP
+
+  if (tryAcquire(numThreads)) {
+
+			auto splitLoop = ThreadsHelper::pickLoop3d(numThreads, itersX, itersY, itersZ);
+#pragma omp parallel for
+			for (uint64_t e = 0; e < numThreads; e++) {
+				auto thread_id = numThreads - e - 1;
+				auto span = Span3::build(splitLoop, thread_id, numThreads, startX, stopX, incX, startY, stopY, incY, startZ, stopZ, incZ);
+				function(e, span.startX(), span.stopX(), span.incX(), span.startY(), span.stopY(), span.incY(), span.startZ(), span.stopZ(), span.incZ());
+			}
+
+			freeThreads(numThreads);
+			return numThreads;
+		}
+		else {
+			// if there were no threads available - we'll execute function right within current thread
+			function(0, startX, stopX, incX, startY, stopY, incY, startZ, stopZ, incZ);
+
+			// we tell that parallelism request declined
+			return 1;
+		}
+#else
+
+  auto ticket = ThreadPool::getInstance().tryAcquire(numThreads);
+  if (ticket != nullptr) {
+    auto splitLoop = ThreadsHelper::pickLoop3d(numThreads, itersX, itersY, itersZ);
+
+    for (uint64_t e = 0; e < numThreads; e++) {
+      auto thread_id = numThreads - e - 1;
+      auto span = Span3::build(splitLoop, thread_id, numThreads, startX, stopX, incX, startY, stopY, incY, startZ, stopZ, incZ);
+
+      ticket->enqueue(e, numThreads, function, span.startX(), span.stopX(), span.incX(), span.startY(), span.stopY(), span.incY(), span.startZ(), span.stopZ(), span.incZ());
+    }
+
+    // block until we're done
+    ticket->waitAndRelease();
+
+    // we tell that parallelism request succeeded
+    return numThreads;
+  }
+  else {
+    // if there were no threads available - we'll execute function right within current thread
+    function(0, startX, stopX, incX, startY, stopY, incY, startZ, stopZ, incZ);
+
+    // we tell that parallelism request declined
+    return 1;
+  }
+#endif
+}
+
+int Threads::parallel_do(FUNC_DO function, sd::LongType numThreads) {
+
+  if (numThreads == 1) {
+    function(0, numThreads);
+    return 1;
+  }
+
+#ifdef _OPENMP
+
+  if (tryAcquire(numThreads)) {
+#pragma omp parallel for
+			for (int e = 0; e < numThreads; e++) {
+				function(e, numThreads);
+			}
+
+			freeThreads(numThreads);
+			return numThreads;
+		}
+		else {
+			// if there's no threads available - we'll execute function sequentially one by one
+			for (uint64_t e = 0; e < static_cast<uint64_t >(numThreads); e++)
+				function(e, numThreads);
+
+			return numThreads;
+		}
+#else
+
+  auto ticket = ThreadPool::getInstance().tryAcquire(numThreads - 1);
+  if (ticket != nullptr) {
+
+    // submit tasks one by one
+    for (sd::LongType e = 0; e < numThreads - 1; e++)
+      ticket->enqueue(e, numThreads, function);
+
+    function(numThreads - 1, numThreads);
+
+    ticket->waitAndRelease();
+
+    return numThreads;
+  }
+  else {
+    // if there's no threads available - we'll execute function sequentially one by one
+    for (uint64_t e = 0; e < static_cast<uint64_t>(numThreads); e++)
+      function(e, numThreads);
+
+    return numThreads;
+  }
+#endif
+
+  return numThreads;
+}
+
+int64_t Threads::parallel_long(FUNC_RL function, FUNC_AL aggregator, sd::LongType start, sd::LongType stop,
+                               sd::LongType increment, sd::LongType numThreads) {
+  if (start > stop)
+    THROW_EXCEPTION("Threads::parallel_long got start > stop");
+
+  auto delta = (stop - start);
+  if (delta == 0 || numThreads == 1)
+    return function(0, start, stop, increment);
+
+  auto numElements = delta / increment;
+
+  // we decide what's optimal number of threads we need here, and execute it
+  numThreads = ThreadsHelper::numberOfThreads(numThreads, numElements);
+  if (numThreads == 1)
+    return function(0, start, stop, increment);
+
+  // create temporary array
+  int64_t intermediatery[256];
+  auto span = delta / numThreads;
+
+#ifdef _OPENMP
+  if (tryAcquire(numThreads)) {
+#pragma omp parallel for
+			for (int e = 0; e < numThreads; e++) {
+				auto start_ = span * e + start;
+				auto stop_ = span * (e + 1) + start;
+
+				intermediatery[e] = function(e, start_, e == numThreads - 1 ? stop : stop_, increment);
+			}
+			freeThreads(numThreads);
+		}
+		else {
+			// if there were no threads available - we'll execute function right within current thread
+			return	function(0, start, stop, increment);
+		}
+#else
+  auto ticket = ThreadPool::getInstance().tryAcquire(numThreads - 1);
+  if (ticket == nullptr)
+    return function(0, start, stop, increment);
+
+  // execute threads in parallel
+  for (uint32_t e = 0; e < numThreads; e++) {
+    auto start_ = span * e + start;
+    auto stop_ = span * (e + 1) + start;
+
+    if (e == numThreads - 1)
+      intermediatery[e] = function(e, start_, stop, increment);
+    else
+      ticket->enqueue(e, numThreads, &intermediatery[e], function, start_, stop_, increment);
+  }
+
+  ticket->waitAndRelease();
+
+#endif
+
+  // aggregate results in single thread
+  for (uint64_t e = 1; e < static_cast<uint64_t>(numThreads); e++)
+    intermediatery[0] = aggregator(intermediatery[0], intermediatery[e]);
+
+  // return accumulated result
+  return intermediatery[0];
+}
+
+double Threads::parallel_double(FUNC_RD function, FUNC_AD aggregator, int64_t start, int64_t stop, int64_t increment, uint64_t numThreads) {
+  if (start > stop)
+    THROW_EXCEPTION("Threads::parallel_long got start > stop");
+
+  auto delta = (stop - start);
+  if (delta == 0 || numThreads == 1)
+    return function(0, start, stop, increment);
+
+  auto numElements = delta / increment;
+
+  // we decide what's optimal number of threads we need here, and execute it
+  numThreads = ThreadsHelper::numberOfThreads(numThreads, numElements);
+  if (numThreads == 1)
+    return function(0, start, stop, increment);
+
+  // create temporary array
+  double intermediatery[256];
+  auto span = delta / numThreads;
+
+#ifdef _OPENMP
+
+  if (tryAcquire(numThreads)) {
+#pragma omp parallel for
+			for (uint64_t e = 0; e < numThreads; e++) {
+				auto start_ = span * e + start;
+				auto stop_ = span * (e + 1) + start;
+
+				intermediatery[e] = function(e, start_, e == numThreads - 1 ? stop : stop_, increment);
+			}
+			freeThreads(numThreads);
+		}
+		else {
+			// if there were no thre ads available - we'll execute function right within current thread
+			return	function(0, start, stop, increment);
+		}
+
+#else
+
+  auto ticket = ThreadPool::getInstance().tryAcquire(numThreads - 1);
+  if (ticket == nullptr)
+    return function(0, start, stop, increment);
+
+  // execute threads in parallel
+  for (uint32_t e = 0; e < numThreads; e++) {
+    auto start_ = span * e + start;
+    auto stop_ = span * (e + 1) + start;
+
+    if (e == numThreads - 1)
+      intermediatery[e] = function(e, start_, stop, increment);
+    else
+      ticket->enqueue(e, numThreads, &intermediatery[e], function, start_, stop_, increment);
+  }
+
+  ticket->waitAndRelease();
+
+#endif
+
+  // aggregate results in single thread
+  for (uint64_t e = 1; e < numThreads; e++)
+    intermediatery[0] = aggregator(intermediatery[0], intermediatery[e]);
+
+  // return accumulated result
+  return intermediatery[0];
+}
+
+
+int  Threads::parallel_aligned_increment(FUNC_1D function, int64_t start, int64_t stop, int64_t increment,
+                                         size_t type_size,
+                                         uint32_t req_numThreads) {
+  if (start > stop)
+    THROW_EXCEPTION("Threads::parallel_for got start > stop");
+  auto num_elements = (stop - start);
+  //this way we preserve increment starts offset
+  //so we will partition considering delta but not total elements
+  auto delta = (stop - start) / increment;
+
+
+  // in some cases we just fire func as is
+  if (delta == 0 || req_numThreads == 1) {
+    function(0, start, stop, increment);
+    return 1;
+  }
+  int numThreads = 0;
+
+  struct th_span {
+    sd::LongType start;
+    sd::LongType end;
+  };
+#ifdef _OPENMP
+  constexpr int max_thread_count = 8;
+#else
+  constexpr int max_thread_count = 1024;
+#endif
+  th_span thread_spans[max_thread_count];
+
+  req_numThreads = req_numThreads > max_thread_count ? max_thread_count : req_numThreads;
+
+#ifdef _OPENMP
+  int adjusted_numThreads = max_thread_count;
+#else
+  int adjusted_numThreads =
+      ThreadsHelper::numberOfThreads(req_numThreads, (num_elements * sizeof(double)) / (200 * type_size));
+#endif
+
+  if (adjusted_numThreads > delta)
+    adjusted_numThreads = delta;
+  // shortcut
+  if (adjusted_numThreads <= 1) {
+    function(0, start, stop, increment);
+    return 1;
+  }
+
+
+
+  //take span as ceil
+  auto spand = std::ceil((double)delta / (double)adjusted_numThreads);
+  numThreads = static_cast<sd::LongType>(std::ceil((double)delta / spand));
+  auto span = static_cast<sd::LongType>(spand);
+
+
+  //tail_add is additional value of the last part
+  //it could be negative or positive
+  //we will spread that value across
+  auto tail_add = delta - numThreads * span;
+  sd::LongType begin = 0;
+  sd::LongType end = 0;
+
+  //we will try enqueue bigger parts first
+  decltype(span) span1 = 0, span2 = 0;
+  int last = 0;
+  if (tail_add >= 0) {
+    //for span == 1  , tail_add is  0
+    last = tail_add;
+    span1 = span + 1;
+    span2 = span;
+  }
+  else {
+    last = numThreads + tail_add;// -std::abs(tail_add);
+    span1 = span;
+    span2 = span - 1;
+  }
+  for (int i = 0; i < last; i++) {
+    end = begin + span1 * increment;
+    // putting the task into the queue for a given thread
+    thread_spans[i].start = begin;
+    thread_spans[i].end = end;
+    begin = end;
+  }
+  for (int i = last; i < numThreads - 1; i++) {
+    end = begin + span2 * increment;
+    // putting the task into the queue for a given thread
+    thread_spans[i].start = begin;
+    thread_spans[i].end = end;
+    begin = end;
+  }
+  //for last one enqueue last offset as stop
+  //we need it in case our ((stop-start) % increment ) > 0
+  thread_spans[numThreads - 1].start = begin;
+  thread_spans[numThreads - 1].end = stop;
+
+#ifdef _OPENMP
+  if (tryAcquire(numThreads)) {
+#pragma omp parallel for
+			for (int j = 0; j < numThreads; j++) {
+				function(j, thread_spans[j].start, thread_spans[j].end, increment);
+			}
+			freeThreads(numThreads);
+			return numThreads;
+		}
+		else {
+			function(0, start, stop, increment);
+			// we tell that parallelism request declined
+			return 1;
+		}
+#else
+  auto ticket = ThreadPool::getInstance().tryAcquire(numThreads);
+  if (ticket != nullptr) {
+
+    for (size_t j = 0; j < static_cast<size_t>(numThreads); j++) {
+      ticket->enqueue(j, numThreads, function, thread_spans[j].start, thread_spans[j].end, increment);
+    }
+    // block and wait till all threads finished the job
+    ticket->waitAndRelease();
+    // we tell that parallelism request succeeded
+    return numThreads;
+  }
+  else {
+    // if there were no threads available - we'll execute function right within current thread
+    function(0, start, stop, increment);
+    // we tell that parallelism request declined
+    return 1;
+  }
+#endif
+}
+}

libnd4j/include/execution/impl/Ticket.cpp

@@ -0,0 +1,95 @@
+/* ******************************************************************************
+ *
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ *  See the NOTICE file distributed with this work for additional
+ *  information regarding copyright ownership.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+#include <execution/ThreadPool.h>
+#include <execution/Ticket.h>
+#include <helpers/logger.h>
+
+
+namespace samediff {
+Ticket::Ticket(const std::vector<BlockingQueue<CallableWithArguments *> *> &queues) {
+  _acquired = true;
+  _queues = queues;
+}
+
+Ticket::Ticket() {
+  _acquired = true;
+  _interfaces.resize(sd::Environment::getInstance().maxThreads());
+}
+
+bool Ticket::acquired() { return _acquired; }
+
+void Ticket::enqueue(int thread_id, CallableWithArguments *callable) {
+  _queues[thread_id]->put(callable);
+  _callables.emplace_back(callable);
+}
+
+void Ticket::enqueue(uint32_t thread_id, uint32_t num_threads, FUNC_DO func) {
+  _interfaces[thread_id]->fill(thread_id, num_threads, func);
+}
+
+void Ticket::enqueue(uint32_t thread_id, uint32_t num_threads, FUNC_1D func, int64_t start_x, int64_t stop_x,
+                     int64_t inc_x) {
+  _interfaces[thread_id]->fill(thread_id, num_threads, func, start_x, stop_x, inc_x);
+}
+
+void Ticket::enqueue(uint32_t thread_id, uint32_t num_threads, int64_t *lpt, FUNC_RL func, int64_t start_x,
+                     int64_t stop_x, int64_t inc_x) {
+  _interfaces[thread_id]->fill(thread_id, num_threads, lpt, func, start_x, stop_x, inc_x);
+}
+
+void Ticket::enqueue(uint32_t thread_id, uint32_t num_threads, double *dpt, FUNC_RD func, int64_t start_x,
+                     int64_t stop_x, int64_t inc_x) {
+  _interfaces[thread_id]->fill(thread_id, num_threads, dpt, func, start_x, stop_x, inc_x);
+}
+
+void Ticket::enqueue(uint32_t thread_id, uint32_t num_threads, FUNC_2D func, int64_t start_x, int64_t stop_x,
+                     int64_t inc_x, int64_t start_y, int64_t stop_y, int64_t inc_y) {
+  _interfaces[thread_id]->fill(thread_id, num_threads, std::move(func), start_x, stop_x, inc_x, start_y, stop_y, inc_y);
+}
+
+void Ticket::enqueue(uint32_t thread_id, uint32_t num_threads, FUNC_3D func, int64_t start_x, int64_t stop_x,
+                     int64_t inc_x, int64_t start_y, int64_t stop_y, int64_t inc_y, int64_t start_z, int64_t stop_z,
+                     int64_t inc_z) {
+  _interfaces[thread_id]->fill(thread_id, num_threads, func, start_x, stop_x, inc_x, start_y, stop_y, inc_y, start_z,
+                               stop_z, inc_z);
+}
+
+void Ticket::acquiredThreads(uint32_t threads) { _acquiredThreads = threads; }
+
+void Ticket::waitAndRelease() {
+  for (uint32_t e = 0; e < this->_acquiredThreads; e++) {
+    // block until finished
+    _interfaces[e]->waitForCompletion();
+
+    // mark available
+    _interfaces[e]->markAvailable();
+
+    // increment availability counter
+    ThreadPool::getInstance().release();
+  }
+
+  // return this ticket back to the pool
+  ThreadPool::getInstance().release(this);
+}
+
+void Ticket::attach(uint32_t thread_id, CallableInterface *call_interface) { _interfaces[thread_id] = call_interface; }
+}  // namespace samediff