chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,924 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <loops/broadcasting.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
#include <cstdio>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace broadcast {
|
||||
|
||||
template <typename X, typename Y, typename Z>
|
||||
void Broadcast<X, Y, Z>::execInverse(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffset, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffset, sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TTT(execInverse,
|
||||
PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, dimension, dimensionLength, xTadShapeInfo,
|
||||
xTadOffset, zTadShapeInfo, zTadOffset, start, stop),
|
||||
BROADCAST_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Y, typename Z>
|
||||
void Broadcast<X, Y, Z>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffset,
|
||||
const sd::LongType *zTadShapeInfo, const sd::LongType *zTadOffset,
|
||||
sd::LoopKind::Kind loopKind, sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TTT(exec,
|
||||
PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, dimension, dimensionLength, xTadShapeInfo,
|
||||
xTadOffset, zTadShapeInfo, zTadOffset, loopKind, start, stop),
|
||||
BROADCAST_OPS);
|
||||
}
|
||||
template <typename X, typename Y, typename Z>
|
||||
template <typename OpType>
|
||||
void Broadcast<X, Y, Z>::exec(const void* vx, const sd::LongType* xShapeInfo,
|
||||
const void* vy, const sd::LongType* yShapeInfo,
|
||||
void* vz, const sd::LongType* zShapeInfo,
|
||||
sd::LongType* dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType* xTadShapeInfo, const sd::LongType* xTadOffset,
|
||||
const sd::LongType* zTadShapeInfo, const sd::LongType* zTadOffset,
|
||||
sd::LoopKind::Kind loopKind, sd::LongType start, sd::LongType stop) {
|
||||
|
||||
auto x = reinterpret_cast<const X*>(vx);
|
||||
auto y = reinterpret_cast<const Y*>(vy);
|
||||
auto z = reinterpret_cast<Z*>(vz);
|
||||
|
||||
// Get rank information
|
||||
const int xRank = shape::rank(xShapeInfo);
|
||||
const int yRank = shape::rank(yShapeInfo);
|
||||
const int zRank = shape::rank(zShapeInfo);
|
||||
const int xTadRank = xTadShapeInfo ? shape::rank(xTadShapeInfo) : xRank;
|
||||
const int zTadRank = zTadShapeInfo ? shape::rank(zTadShapeInfo) : zRank;
|
||||
|
||||
// Get shape information
|
||||
const sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
const sd::LongType* yShape = shape::shapeOf(yShapeInfo);
|
||||
const sd::LongType* zShape = shape::shapeOf(zShapeInfo);
|
||||
const sd::LongType* xTadShape = shape::shapeOf(xTadShapeInfo);
|
||||
const sd::LongType* zTadShape = shape::shapeOf(zTadShapeInfo);
|
||||
|
||||
// Get stride information
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
const sd::LongType* xTadStrides = shape::stride(xTadShapeInfo);
|
||||
const sd::LongType* zTadStrides = shape::stride(zTadShapeInfo);
|
||||
|
||||
// Classify array types
|
||||
// For X array or X TAD
|
||||
bool isXScalar = xTadRank == 0 || (xTadRank == 1 && xTadShape[0] == 1);
|
||||
bool isXVector = (xTadRank == 1 && xTadShape[0] > 1) ||
|
||||
(xTadRank == 2 && (xTadShape[0] == 1 || xTadShape[1] == 1));
|
||||
bool isXRowVector = (xTadRank == 1 && xTadShape[0] > 1) ||
|
||||
(xTadRank == 2 && xTadShape[0] == 1 && xTadShape[1] > 1);
|
||||
bool isXColumnVector = (xTadRank == 2 && xTadShape[0] > 1 && xTadShape[1] == 1);
|
||||
|
||||
// For Y array
|
||||
bool isYScalar = yRank == 0 || (yRank == 1 && yShape[0] == 1);
|
||||
bool isYVector = (yRank == 1 && yShape[0] > 1) ||
|
||||
(yRank == 2 && (yShape[0] == 1 || yShape[1] == 1));
|
||||
bool isYRowVector = (yRank == 1 && yShape[0] > 1) ||
|
||||
(yRank == 2 && yShape[0] == 1 && yShape[1] > 1);
|
||||
bool isYColumnVector = (yRank == 2 && yShape[0] > 1 && yShape[1] == 1);
|
||||
|
||||
// For Z array or Z TAD
|
||||
bool isZScalar = zTadRank == 0 || (zTadRank == 1 && zTadShape[0] == 1);
|
||||
bool isZVector = (zTadRank == 1 && zTadShape[0] > 1) ||
|
||||
(zTadRank == 2 && (zTadShape[0] == 1 || zTadShape[1] == 1));
|
||||
bool isZRowVector = (zTadRank == 1 && zTadShape[0] > 1) ||
|
||||
(zTadRank == 2 && zTadShape[0] == 1 && zTadShape[1] > 1);
|
||||
bool isZColumnVector = (zTadRank == 2 && zTadShape[0] > 1 && zTadShape[1] == 1);
|
||||
|
||||
// Handle scalar broadcasting as a special case first
|
||||
if (isYScalar) {
|
||||
// Scalar broadcast - apply same value to every element
|
||||
const Y scalarY = y[0];
|
||||
sd::LongType length = shape::length(xTadShapeInfo ? xTadShapeInfo : xShapeInfo);
|
||||
|
||||
if (xTadShapeInfo && zTadShapeInfo) {
|
||||
// TAD case
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + xTadOffset[i];
|
||||
auto oZ = z + zTadOffset[i];
|
||||
|
||||
// Handle different X and Z shapes
|
||||
if (isXVector && isZVector) {
|
||||
sd::LongType len = shape::length(xTadShapeInfo);
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < len; f++) {
|
||||
sd::LongType xOffset = f * xTadStrides[xTadRank-1];
|
||||
sd::LongType zOffset = f * zTadStrides[zTadRank-1];
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], scalarY);
|
||||
}
|
||||
} else {
|
||||
// General case
|
||||
for (sd::LongType f = 0; f < length; f++) {
|
||||
// Calculate proper offsets for current position
|
||||
sd::LongType xCoord[SD_MAX_RANK], zCoord[SD_MAX_RANK];
|
||||
sd::LongType xOffset, zOffset;
|
||||
|
||||
INDEX2COORDS(f, xTadRank, xTadShape, xCoord);
|
||||
INDEX2COORDS(f, zTadRank, zTadShape, zCoord);
|
||||
|
||||
COORDS2INDEX(xTadRank, xTadStrides, xCoord, xOffset);
|
||||
COORDS2INDEX(zTadRank, zTadStrides, zCoord, zOffset);
|
||||
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], scalarY);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-TAD case
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < length; f++)
|
||||
z[f] = OpType::op(x[f], scalarY);
|
||||
}
|
||||
}
|
||||
// Handle 2D broadcasting
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_2D) {
|
||||
// Determine shapes for broadcasting
|
||||
sd::LongType nRows = zTadShape[0];
|
||||
sd::LongType nCols = zTadRank > 1 ? zTadShape[1] : shape::length(zTadShapeInfo);
|
||||
|
||||
// Special vector broadcasting cases
|
||||
if (isYVector && (isXRowVector || isXColumnVector || isXVector)) {
|
||||
// Vector to vector broadcasting
|
||||
if (isYRowVector && (isXRowVector || isXVector)) {
|
||||
// Row vector to row vector
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto baseX = x + xTadOffset[i];
|
||||
auto baseZ = z + zTadOffset[i];
|
||||
|
||||
sd::LongType xStride = xTadRank > 1 ? xTadStrides[xTadRank - 1] : xTadStrides[0];
|
||||
sd::LongType yStride = yRank == 1 ? yStrides[0] : yStrides[1];
|
||||
sd::LongType zStride = zTadRank ? zTadStrides[zTadRank - 1] : zTadStrides[0];
|
||||
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < nCols; i1++) {
|
||||
auto rX = baseX + i1 * xStride;
|
||||
auto rY = y + i1 * yStride;
|
||||
auto rZ = baseZ + i1 * zStride;
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (isYColumnVector && (isXColumnVector || isXVector)) {
|
||||
// Column vector to column vector
|
||||
// Row vector to row vector
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto baseX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto baseZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
// Calculate correct strides based on shape and rank
|
||||
sd::LongType xStride;
|
||||
if (xTadRank == 1) {
|
||||
xStride = xTadStrides[0];
|
||||
} else { // xTadRank == 2
|
||||
xStride = xTadStrides[0]; // For 2D column vector, use row stride
|
||||
}
|
||||
|
||||
sd::LongType yStride;
|
||||
if (yRank == 1) {
|
||||
yStride = yStrides[0];
|
||||
} else { // yRank == 2
|
||||
yStride = yStrides[0]; // For 2D column vector, use row stride
|
||||
}
|
||||
|
||||
sd::LongType zStride;
|
||||
if (zTadRank == 1) {
|
||||
zStride = zTadStrides[0];
|
||||
} else { // zTadRank == 2
|
||||
zStride = zTadStrides[0]; // For 2D column vector, use row stride
|
||||
}
|
||||
|
||||
// Verify dimensions match
|
||||
sd::LongType xLen = isXColumnVector ? (xTadRank == 2 ? xTadShape[0] : xTadShape[0]) : xTadShape[0];
|
||||
sd::LongType yLen = yRank == 2 ? yShape[0] : yShape[0];
|
||||
printf("xLen: %lld; yLen: %lld nRows %lld,xStride %lld,yStride %lld, zStride %lld\n", xLen, yLen,nRows,xStride,yStride,zStride);
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < nRows; i1++) {
|
||||
auto rX = baseX + i1 * xStride;
|
||||
auto rY = y + i1 * yStride;
|
||||
auto rZ = baseZ + i1 * zStride;
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (isYColumnVector && isXRowVector) {
|
||||
// Column vector to row vector (outer product)
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto baseX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto baseZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType i0 = 0; i0 < nRows; i0++) {
|
||||
auto colValue = y[i0 * yStrides[0]];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < nCols; i1++) {
|
||||
auto rX = baseX + i1 * xTadStrides[xTadRank-1];
|
||||
auto rZ = baseZ + i0 * zTadStrides[0] + i1 * zTadStrides[1];
|
||||
|
||||
*rZ = OpType::op(*rX, colValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (isYRowVector && isXColumnVector) {
|
||||
// Row vector to column vector (outer product)
|
||||
for (auto i = start; i < stop; i++) {
|
||||
printf("4 2d tad: %lld\n", i);
|
||||
fflush(stdout);
|
||||
auto baseX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto baseZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType i0 = 0; i0 < nRows; i0++) {
|
||||
auto xValue = baseX[i0 * xTadStrides[0]];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < nCols; i1++) {
|
||||
auto rY = y + i1 * (yRank == 1 ? yStrides[0] : yStrides[1]);
|
||||
auto rZ = baseZ + i0 * zTadStrides[0] + i1 * zTadStrides[1];
|
||||
|
||||
*rZ = OpType::op(xValue, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Matrix with vector broadcasting
|
||||
else if ((isXRowVector && isYRowVector) || (isXColumnVector && isYColumnVector)) {
|
||||
// Matching vectors - element-wise operation
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto baseX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto baseZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
sd::LongType vecLength = isXRowVector ? nCols : nRows;
|
||||
sd::LongType xStride = isXRowVector ? xTadStrides[xTadRank-1] : xTadStrides[0];
|
||||
sd::LongType yStride = isYRowVector ? (yRank == 1 ? yStrides[0] : yStrides[1]) : yStrides[0];
|
||||
sd::LongType zStride = isZRowVector ? zTadStrides[zTadRank-1] : zTadStrides[0];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < vecLength; i1++) {
|
||||
auto rX = baseX + i1 * xStride;
|
||||
auto rY = y + i1 * yStride;
|
||||
auto rZ = baseZ + i1 * zStride;
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Matrix with row vector broadcasting
|
||||
else if (isYRowVector && xTadRank == 2 && zTadRank == 2 &&
|
||||
xTadShape[1] == (yRank == 1 ? yShape[0] : yShape[1])) {
|
||||
// Broadcasting row vector (each element applied to a column)
|
||||
for (auto i0 = start; i0 < stop; i0++) {
|
||||
auto baseX = x + (xTadOffset ? xTadOffset[i0] : 0);
|
||||
auto baseZ = z + (zTadOffset ? zTadOffset[i0] : 0);
|
||||
|
||||
for (sd::LongType i1 = 0; i1 < nRows; i1++) {
|
||||
for (sd::LongType i2 = 0; i2 < nCols; i2++) {
|
||||
// Get element from X at current position
|
||||
auto xOffset = i1 * xTadStrides[0] + i2 * xTadStrides[1];
|
||||
// Get element from Y row vector based on column index only
|
||||
auto yOffset = i2 * (yRank == 1 ? yStrides[0] : yStrides[1]);
|
||||
// Get destination element in Z at current position
|
||||
auto zOffset = i1 * zTadStrides[0] + i2 * zTadStrides[1];
|
||||
|
||||
// Apply operation
|
||||
baseZ[zOffset] = OpType::op(baseX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Matrix with column vector broadcasting
|
||||
else if (isYColumnVector) {
|
||||
// Broadcasting column vector (each element applied to a row)
|
||||
for (auto i0 = start; i0 < stop; i0++) {
|
||||
auto baseX = x + (xTadOffset ? xTadOffset[i0] : 0);
|
||||
auto baseZ = z + (zTadOffset ? zTadOffset[i0] : 0);
|
||||
|
||||
for (sd::LongType i1 = 0; i1 < nRows; i1++) {
|
||||
// Get element from column vector based on row index
|
||||
auto rY = y + i1 * yStrides[0];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i2 = 0; i2 < nCols; i2++) {
|
||||
auto rX = baseX + i1 * xTadStrides[0] + i2 * xTadStrides[1];
|
||||
auto rZ = baseZ + i1 * zTadStrides[0] + i2 * zTadStrides[1];
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Standard 2D broadcasting
|
||||
else {
|
||||
for (auto i0 = start; i0 < stop; i0++) {
|
||||
auto baseX = x + (xTadOffset ? xTadOffset[i0] : 0);
|
||||
auto baseZ = z + (zTadOffset ? zTadOffset[i0] : 0);
|
||||
|
||||
for (sd::LongType i1 = 0; i1 < nRows; i1++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i2 = 0; i2 < nCols; i2++) {
|
||||
auto rX = baseX + i1 * xTadStrides[0] + i2 * xTadStrides[1];
|
||||
auto rY = y + i1 * yStrides[0] + i2 * yStrides[1];
|
||||
auto rZ = baseZ + i1 * zTadStrides[0] + i2 * zTadStrides[1];
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Handle remaining loop kinds
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_SCALAR_X) {
|
||||
sd::LongType tadLength = shape::length(xTadShapeInfo);
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oY = y + (i * tadLength);
|
||||
auto oZ = z + (i * tadLength);
|
||||
const auto oX = x[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++)
|
||||
oZ[f] = OpType::op(oX, oY[f]);
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_SCALAR_Y) {
|
||||
sd::LongType tadLength = shape::length(xTadShapeInfo);
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (i * tadLength);
|
||||
auto oZ = z + (i * tadLength);
|
||||
const auto oY = y[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++)
|
||||
oZ[f] = OpType::op(oX[f], oY);
|
||||
}
|
||||
}
|
||||
// Handle 3D broadcasting (generalized like 2D case)
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_3D) {
|
||||
// Get TAD info
|
||||
const sd::LongType tadRank = xTadShapeInfo ? shape::rank(xTadShapeInfo) : 3;
|
||||
const sd::LongType* tadShape = xTadShapeInfo ? shape::shapeOf(xTadShapeInfo) : xShape;
|
||||
const sd::LongType* tadStride = xTadShapeInfo ? shape::stride(xTadShapeInfo) : xStrides;
|
||||
const sd::LongType tadLength = xTadShapeInfo ? shape::length(xTadShapeInfo) : shape::length(xShapeInfo);
|
||||
if (isYVector) {
|
||||
// Vector broadcasting
|
||||
const sd::LongType yLength = yRank == 1 ? yShape[0] : (yShape[0] == 1 ? yShape[1] : yShape[0]);
|
||||
const sd::LongType yStride = yRank == 1 ? yStrides[0] : (yShape[0] == 1 ? yStrides[1] : yStrides[0]);
|
||||
|
||||
// Determine which dimension this vector should be broadcast along
|
||||
// For a 3D TAD, check if vector length matches any dimension
|
||||
if (tadRank == 3) {
|
||||
if (yLength == tadShape[2]) {
|
||||
// Broadcast along last dimension
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto oZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
// Calculate TAD coords
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(j, tadRank, tadShape, coords);
|
||||
|
||||
// Get offsets
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
COORDS2INDEX(tadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
// Get Y index - use the last dimension (coords[2])
|
||||
sd::LongType yOffset = coords[2] * yStride;
|
||||
|
||||
// Apply operation
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (yLength == tadShape[1]) {
|
||||
// Broadcast along middle dimension
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto oZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
// Calculate TAD coords
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(j, tadRank, tadShape, coords);
|
||||
|
||||
// Get offsets
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
COORDS2INDEX(tadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
// Get Y index - use the middle dimension (coords[1])
|
||||
sd::LongType yOffset = coords[1] * yStride;
|
||||
|
||||
// Apply operation
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (yLength == tadShape[0]) {
|
||||
// Broadcast along first dimension
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto oZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
// Calculate TAD coords
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(j, tadRank, tadShape, coords);
|
||||
|
||||
// Get offsets
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
COORDS2INDEX(tadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
// Get Y index - use the first dimension (coords[0])
|
||||
sd::LongType yOffset = coords[0] * yStride;
|
||||
|
||||
// Apply operation
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Default broadcasting behavior - broadcast along the last dimension
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto oZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
// Calculate TAD coords
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(j, tadRank, tadShape, coords);
|
||||
|
||||
// Get offsets
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
COORDS2INDEX(tadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
// Get Y index with wrapping/broadcasting
|
||||
sd::LongType yOffset = (coords[2] % yLength) * yStride;
|
||||
|
||||
// Apply operation
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Handle lower rank TADs (1D or 2D)
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto oZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
// Calculate TAD coords
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(j, tadRank, tadShape, coords);
|
||||
|
||||
// Get offsets
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
COORDS2INDEX(tadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
// Get Y index - for lower ranks, broadcast along the last available dimension
|
||||
sd::LongType lastCoord = tadRank > 0 ? coords[tadRank - 1] : 0;
|
||||
sd::LongType yOffset = (lastCoord % yLength) * yStride;
|
||||
|
||||
// Apply operation
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (yRank == 2) {
|
||||
// Y is a 2D matrix - determine which dimensions it aligns with
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto oZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
// Calculate TAD coords
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(j, tadRank, tadShape, coords);
|
||||
|
||||
// Get offsets
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
COORDS2INDEX(tadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
// Calculate Y offset based on dimension matching
|
||||
sd::LongType yOffset;
|
||||
|
||||
// Default behavior for different 2D matrix broadcasting patterns
|
||||
if (tadRank == 3) {
|
||||
if (yShape[0] == tadShape[0] && yShape[1] == tadShape[2]) {
|
||||
// Y is aligned with dimensions 0 and 2
|
||||
yOffset = coords[0] * yStrides[0] + coords[2] * yStrides[1];
|
||||
}
|
||||
else if (yShape[0] == tadShape[0] && yShape[1] == tadShape[1]) {
|
||||
// Y is aligned with dimensions 0 and 1
|
||||
yOffset = coords[0] * yStrides[0] + coords[1] * yStrides[1];
|
||||
}
|
||||
else if (yShape[0] == tadShape[1] && yShape[1] == tadShape[2]) {
|
||||
// Y is aligned with dimensions 1 and 2
|
||||
yOffset = coords[1] * yStrides[0] + coords[2] * yStrides[1];
|
||||
}
|
||||
else {
|
||||
// Default: broadcast Y to match the last two dimensions with modulo
|
||||
yOffset = (coords[1] % yShape[0]) * yStrides[0] + (coords[2] % yShape[1]) * yStrides[1];
|
||||
}
|
||||
}
|
||||
else if (tadRank == 2) {
|
||||
// Direct mapping for 2D TAD and 2D Y
|
||||
yOffset = (coords[0] % yShape[0]) * yStrides[0] + (coords[1] % yShape[1]) * yStrides[1];
|
||||
}
|
||||
else {
|
||||
// For 1D TAD, map to the first dimension of Y
|
||||
yOffset = (coords[0] % yShape[0]) * yStrides[0];
|
||||
}
|
||||
|
||||
// Apply operation
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (yRank == 3) {
|
||||
// Y is a 3D tensor
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto oZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
// Calculate TAD coords
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(j, tadRank, tadShape, coords);
|
||||
|
||||
// Get offsets
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
COORDS2INDEX(tadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
// Calculate Y offset with modulo for broadcasting if needed
|
||||
sd::LongType yCoords[3] = {0, 0, 0};
|
||||
|
||||
// Map coordinates appropriately based on ranks
|
||||
if (tadRank == 3) {
|
||||
yCoords[0] = coords[0] % yShape[0];
|
||||
yCoords[1] = coords[1] % yShape[1];
|
||||
yCoords[2] = coords[2] % yShape[2];
|
||||
}
|
||||
else if (tadRank == 2) {
|
||||
// Map 2D to last 2 dimensions of 3D
|
||||
yCoords[0] = 0; // First dimension is broadcasted
|
||||
yCoords[1] = coords[0] % yShape[1];
|
||||
yCoords[2] = coords[1] % yShape[2];
|
||||
}
|
||||
else {
|
||||
// Map 1D to last dimension of 3D
|
||||
yCoords[0] = 0;
|
||||
yCoords[1] = 0;
|
||||
yCoords[2] = coords[0] % yShape[2];
|
||||
}
|
||||
|
||||
sd::LongType yOffset = yCoords[0] * yStrides[0] + yCoords[1] * yStrides[1] + yCoords[2] * yStrides[2];
|
||||
|
||||
// Apply operation
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// General case for other ranks of Y
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (xTadOffset ? xTadOffset[i] : 0);
|
||||
auto oZ = z + (zTadOffset ? zTadOffset[i] : 0);
|
||||
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
// Calculate TAD coords
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(j, tadRank, tadShape, coords);
|
||||
|
||||
// Get offsets
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
COORDS2INDEX(tadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
// Calculate Y offset based on rank
|
||||
sd::LongType yOffset = 0;
|
||||
|
||||
// Map coordinates to Y based on rank
|
||||
for (int d = 0; d < tadRank && d < yRank; d++) {
|
||||
yOffset += (coords[d] % yShape[d]) * yStrides[d];
|
||||
}
|
||||
|
||||
// Apply operation
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_4D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType nSize2 = shape::sizeAt(zShapeInfo, 2);
|
||||
const sd::LongType nSize3 = shape::sizeAt(zShapeInfo, 3);
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
uint64_t i0 = i / nSize1;
|
||||
uint64_t i1 = i % nSize1;
|
||||
|
||||
for (sd::LongType i2 = 0; i2 < nSize2; i2++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i3 = 0; i3 < nSize3; i3++) {
|
||||
auto rX = x + (xStrides[0] * i0 + xStrides[1] * i1 + xStrides[2] * i2 + xStrides[3] * i3);
|
||||
auto rY = y + (yStrides[0] * i0 + yStrides[1] * i1 + yStrides[2] * i2 + yStrides[3] * i3);
|
||||
auto rZ = z + (zStrides[0] * i0 + zStrides[1] * i1 + zStrides[2] * i2 + zStrides[3] * i3);
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_5D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType nSize2 = shape::sizeAt(zShapeInfo, 2);
|
||||
const sd::LongType nSize3 = shape::sizeAt(zShapeInfo, 3);
|
||||
const sd::LongType nSize4 = shape::sizeAt(zShapeInfo, 4);
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
uint32_t i0 = i / nSize1;
|
||||
uint32_t i1 = i % nSize1;
|
||||
|
||||
for (sd::LongType i2 = 0; i2 < nSize2; i2++) {
|
||||
for (sd::LongType i3 = 0; i3 < nSize3; i3++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i4 = 0; i4 < nSize4; i4++) {
|
||||
auto rX = x + (xStrides[0] * i0 + xStrides[1] * i1 + xStrides[2] * i2 + xStrides[3] * i3 + xStrides[4] * i4);
|
||||
auto rY = y + (yStrides[0] * i0 + yStrides[1] * i1 + yStrides[2] * i2 + yStrides[3] * i3 + yStrides[4] * i4);
|
||||
auto rZ = z + (zStrides[0] * i0 + zStrides[1] * i1 + zStrides[2] * i2 + zStrides[3] * i3 + zStrides[4] * i4);
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Default case for other ranks - general purpose implementation
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
sd::LongType yCoords[SD_MAX_RANK];
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
// Calculate independent coordinates for each array
|
||||
INDEX2COORDS(i, xRank, xShape, xCoords);
|
||||
INDEX2COORDS(i, yRank, yShape, yCoords);
|
||||
INDEX2COORDS(i, zRank, zShape, zCoords);
|
||||
|
||||
// Calculate offsets based on each array's coordinates and strides
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStrides, xCoords, xOffset);
|
||||
COORDS2INDEX(yRank, yStrides, yCoords, yOffset);
|
||||
COORDS2INDEX(zRank, zStrides, zCoords, zOffset);
|
||||
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
template <typename X, typename Y, typename Z>
|
||||
template <typename OpType>
|
||||
void Broadcast<X, Y, Z>::execInverse(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *yTadShapeInfo, const sd::LongType *yTadOffset,
|
||||
const sd::LongType *zTadShapeInfo, const sd::LongType *zTadOffset,
|
||||
sd::LongType start, sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const Y *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
|
||||
// Handle TAD setup
|
||||
auto yTadShapeShapeInfo = yTadShapeInfo;
|
||||
auto tadOffsets = yTadOffset;
|
||||
|
||||
// When shared_ptr goes out of scope, it deletes the TadPack and invalidates pointers!
|
||||
std::shared_ptr<sd::TadPack> tadPack = nullptr;
|
||||
|
||||
if (yTadShapeInfo == nullptr || tadOffsets == nullptr) {
|
||||
tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(const_cast<sd::LongType*>(yShapeInfo), dimension,
|
||||
dimensionLength);
|
||||
yTadShapeShapeInfo = tadPack->primaryShapeInfo();
|
||||
tadOffsets = tadPack->primaryOffsets();
|
||||
}
|
||||
|
||||
if (zTadShapeInfo == nullptr) {
|
||||
zTadShapeInfo = yTadShapeShapeInfo;
|
||||
zTadOffset = tadOffsets;
|
||||
}
|
||||
|
||||
// Get shape information
|
||||
const auto xRank = shape::rank(xShapeInfo);
|
||||
const auto yTadRank = shape::rank(yTadShapeShapeInfo);
|
||||
const auto zTadRank = shape::rank(zTadShapeInfo);
|
||||
|
||||
const auto xStrides = shape::stride(xShapeInfo);
|
||||
const auto yTadStrides = shape::stride(yTadShapeShapeInfo);
|
||||
const auto zTadStrides = shape::stride(zTadShapeInfo);
|
||||
|
||||
const auto xShape = shape::shapeOf(xShapeInfo);
|
||||
const auto yTadShape = shape::shapeOf(yTadShapeShapeInfo);
|
||||
const auto zTadShape = shape::shapeOf(zTadShapeInfo);
|
||||
|
||||
const sd::LongType tadLength = shape::length(yTadShapeShapeInfo);
|
||||
|
||||
if (yTadRank <= 3) {
|
||||
// Optimized path for lower ranks
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oZ = z + zTadOffset[i];
|
||||
auto oY = y + tadOffsets[i];
|
||||
|
||||
if (yTadRank == 1) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
oZ[j * zTadStrides[0]] = OpType::op(x[j * xStrides[0]], oY[j * yTadStrides[0]]);
|
||||
}
|
||||
}
|
||||
else if (yTadRank == 2) {
|
||||
const sd::LongType dim0 = yTadShape[0];
|
||||
const sd::LongType dim1 = yTadShape[1];
|
||||
|
||||
for (sd::LongType j0 = 0; j0 < dim0; j0++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j1 = 0; j1 < dim1; j1++) {
|
||||
const auto xOffset = j0 * xStrides[0] + j1 * xStrides[1];
|
||||
const auto yOffset = j0 * yTadStrides[0] + j1 * yTadStrides[1];
|
||||
const auto zOffset = j0 * zTadStrides[0] + j1 * zTadStrides[1];
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else { // rank 3
|
||||
const sd::LongType dim0 = yTadShape[0];
|
||||
const sd::LongType dim1 = yTadShape[1];
|
||||
const sd::LongType dim2 = yTadShape[2];
|
||||
|
||||
for (sd::LongType j0 = 0; j0 < dim0; j0++) {
|
||||
for (sd::LongType j1 = 0; j1 < dim1; j1++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j2 = 0; j2 < dim2; j2++) {
|
||||
const auto xOffset = j0 * xStrides[0] + j1 * xStrides[1] + j2 * xStrides[2];
|
||||
const auto yOffset = j0 * yTadStrides[0] + j1 * yTadStrides[1] + j2 * yTadStrides[2];
|
||||
const auto zOffset = j0 * zTadStrides[0] + j1 * zTadStrides[1] + j2 * zTadStrides[2];
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Use macros for higher ranks
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oZ = z + zTadOffset[i];
|
||||
auto oY = y + tadOffsets[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(f, yTadRank, yTadShape, coords);
|
||||
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStrides, coords, xOffset);
|
||||
COORDS2INDEX(yTadRank, yTadStrides, coords, yOffset);
|
||||
COORDS2INDEX(zTadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Y, typename Z>
|
||||
void Broadcast<X, Y, Z>::exec(const int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo) {
|
||||
DISPATCH_BY_OPNUM_TTT(exec, PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo), BROADCAST_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Y, typename Z, typename OpType>
|
||||
static void execDefault(const X *x, const sd::LongType *xShapeInfo, const Y *y, const sd::LongType *yShapeInfo, Z *z,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
|
||||
// C-style arrays CANNOT be captured by value in lambdas - they decay to pointers
|
||||
// that point to stack memory. std::array CAN be captured by value, ensuring each
|
||||
// parallel thread gets its own copy of the data with guaranteed lifetime.
|
||||
std::array<sd::LongType, SD_MAX_RANK> xShapeLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> yShapeLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> zShapeLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> xStrideLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> yStrideLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> zStrideLocal;
|
||||
|
||||
// Copy actual data from shapeInfo into std::arrays
|
||||
std::memcpy(xShapeLocal.data(), shape::shapeOf(xShapeInfo), xRank * sizeof(sd::LongType));
|
||||
std::memcpy(yShapeLocal.data(), shape::shapeOf(yShapeInfo), yRank * sizeof(sd::LongType));
|
||||
std::memcpy(zShapeLocal.data(), shape::shapeOf(zShapeInfo), zRank * sizeof(sd::LongType));
|
||||
std::memcpy(xStrideLocal.data(), shape::stride(xShapeInfo), xRank * sizeof(sd::LongType));
|
||||
std::memcpy(yStrideLocal.data(), shape::stride(yShapeInfo), yRank * sizeof(sd::LongType));
|
||||
std::memcpy(zStrideLocal.data(), shape::stride(zShapeInfo), zRank * sizeof(sd::LongType));
|
||||
|
||||
// Capture std::arrays by value - C++ will copy the entire array contents into the lambda's closure.
|
||||
// This ensures each parallel thread has its own copy of the data with no dangling pointers.
|
||||
auto func = [x, y, z, xRank, yRank, zRank, xShapeLocal, yShapeLocal, zShapeLocal, xStrideLocal, yStrideLocal, zStrideLocal](
|
||||
sd::LongType thread_id, sd::LongType start, sd::LongType stop, sd::LongType increment) -> void {
|
||||
for (auto i = start; i < stop; ++i) {
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
sd::LongType yCoords[SD_MAX_RANK];
|
||||
|
||||
// Convert linear index to coordinates based on Z (output) shape
|
||||
INDEX2COORDS(i, zRank, zShapeLocal.data(), zCoords);
|
||||
|
||||
// Broadcast Z coordinates to X and Y shapes
|
||||
// For broadcasting, we map Z coords to X and Y coords using modulo for smaller dimensions
|
||||
// When a dimension is 1 in X or Y but larger in Z, we use index 0 (broadcast)
|
||||
for (sd::LongType d = 0; d < xRank; d++) {
|
||||
xCoords[d] = xShapeLocal[d] == 1 ? 0 : (zCoords[d] % xShapeLocal[d]);
|
||||
}
|
||||
for (sd::LongType d = 0; d < yRank; d++) {
|
||||
yCoords[d] = yShapeLocal[d] == 1 ? 0 : (zCoords[d] % yShapeLocal[d]);
|
||||
}
|
||||
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStrideLocal.data(), xCoords, xOffset);
|
||||
COORDS2INDEX(yRank, yStrideLocal.data(), yCoords, yOffset);
|
||||
COORDS2INDEX(zRank, zStrideLocal.data(), zCoords, zOffset);
|
||||
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, static_cast<sd::LongType>(0), shape::length(zShapeInfo));
|
||||
}
|
||||
|
||||
template <typename X, typename Y, typename Z>
|
||||
template <typename OpType>
|
||||
void Broadcast<X, Y, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo) {
|
||||
const X *x = reinterpret_cast<const X *>(vx);
|
||||
const Y *y = reinterpret_cast<const Y *>(vy);
|
||||
Z *z = reinterpret_cast<Z *>(vz);
|
||||
|
||||
const int rank = shape::rank(zShapeInfo); // xRank = yRank = zRank
|
||||
|
||||
switch (rank) {
|
||||
default:
|
||||
execDefault<X, Y, Z, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace broadcast
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,672 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <loops/broadcasting_bool.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace broadcast {
|
||||
|
||||
template <typename X, typename Z>
|
||||
void BroadcastBool<X, Z>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *xTadShapeInfo, const sd::LongType *xTadOffset,
|
||||
const sd::LongType *zTadShapeInfo, const sd::LongType *zTadOffset, sd::LongType start,
|
||||
sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TT(exec,
|
||||
PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams, dimension, dimensionLength,
|
||||
xTadShapeInfo, xTadOffset, zTadShapeInfo, zTadOffset, start, stop),
|
||||
BROADCAST_BOOL_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void BroadcastBool<X, Z>::exec(const int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams) {
|
||||
DISPATCH_BY_OPNUM_TT(exec, PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams), BROADCAST_BOOL_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void BroadcastBool<X, Z>::execInverse(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *xTadShapeInfo, const sd::LongType *xTadOffset,
|
||||
const sd::LongType *zTadShapeInfo, const sd::LongType *zTadOffset, sd::LongType start,
|
||||
sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TT(execInverse,
|
||||
PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams, dimension, dimensionLength,
|
||||
xTadShapeInfo, xTadOffset, zTadShapeInfo, zTadOffset, start, stop),
|
||||
BROADCAST_BOOL_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void BroadcastBool<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
void *vextraParams, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *xTadShapeInfo, const sd::LongType *xTadOffset,
|
||||
const sd::LongType *zTadShapeInfo, const sd::LongType *zTadOffset, uint64_t start,
|
||||
uint64_t stop) {
|
||||
auto x = reinterpret_cast<const X*>(vx);
|
||||
auto y = reinterpret_cast<const X*>(vy);
|
||||
auto z = reinterpret_cast<Z*>(vz);
|
||||
auto extraParams = reinterpret_cast<X*>(vextraParams);
|
||||
sd::LoopKind::Kind loopKind= sd::LoopKind::deduceKindOfLoopXYZ(xTadShapeInfo, yShapeInfo, zTadShapeInfo);
|
||||
|
||||
if (loopKind == sd::LoopKind::BROADCAST_SCALAR_X) {
|
||||
sd::LongType tadLength = shape::length(xTadShapeInfo);
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oY = y + (i * tadLength);
|
||||
auto oZ = z + (i * tadLength);
|
||||
const auto oX = x[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++)
|
||||
oZ[f] = OpType::op(oX, oY[f],extraParams);
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_SCALAR_Y) {
|
||||
sd::LongType tadLength = shape::length(xTadShapeInfo);
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (i * tadLength);
|
||||
auto oZ = z + (i * tadLength);
|
||||
const auto oY = y[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++)
|
||||
oZ[f] = OpType::op(oX[f], oY,extraParams);
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_2D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType* xStrides = shape::stride(xTadShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zTadShapeInfo);
|
||||
|
||||
for (auto i0 = start; i0 < stop; i0++) {
|
||||
auto baseX = x + xTadOffset[i0];
|
||||
auto baseZ = z + zTadOffset[i0];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < nSize1; i1++) {
|
||||
auto rX = baseX + xStrides[1] * i1;
|
||||
auto rY = y + yStrides[1] * i1;
|
||||
auto rZ = baseZ + zStrides[1] * i1;
|
||||
|
||||
*rZ = OpType::op(*rX, *rY,extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_3D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType nSize2 = shape::sizeAt(zShapeInfo, 2);
|
||||
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
|
||||
for (auto i0 = start; i0 < stop; i0++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < nSize1; i1++) {
|
||||
for (sd::LongType i2 = 0; i2 < nSize2; i2++) {
|
||||
auto rX = x + (xStrides[0] * i0 + xStrides[1] * i1 + xStrides[2] * i2);
|
||||
auto rY = y + (yStrides[0] * i0 + yStrides[1] * i1 + yStrides[2] * i2);
|
||||
auto rZ = z + (zStrides[0] * i0 + zStrides[1] * i1 + zStrides[2] * i2);
|
||||
|
||||
*rZ = OpType::op(*rX, *rY,extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_4D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType nSize2 = shape::sizeAt(zShapeInfo, 2);
|
||||
const sd::LongType nSize3 = shape::sizeAt(zShapeInfo, 3);
|
||||
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
uint64_t i0 = i / nSize1;
|
||||
uint64_t i1 = i % nSize1;
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i2 = 0; i2 < nSize2; i2++) {
|
||||
for (sd::LongType i3 = 0; i3 < nSize3; i3++) {
|
||||
auto rX = x + (xStrides[0] * i0 + xStrides[1] * i1 +
|
||||
xStrides[2] * i2 + xStrides[3] * i3);
|
||||
auto rY = y + (yStrides[0] * i0 + yStrides[1] * i1 +
|
||||
yStrides[2] * i2 + yStrides[3] * i3);
|
||||
auto rZ = z + (zStrides[0] * i0 + zStrides[1] * i1 +
|
||||
zStrides[2] * i2 + zStrides[3] * i3);
|
||||
|
||||
*rZ = OpType::op(*rX, *rY,extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_5D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType nSize2 = shape::sizeAt(zShapeInfo, 2);
|
||||
const sd::LongType nSize3 = shape::sizeAt(zShapeInfo, 3);
|
||||
const sd::LongType nSize4 = shape::sizeAt(zShapeInfo, 4);
|
||||
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
uint32_t i0 = i / nSize1;
|
||||
uint32_t i1 = i % nSize1;
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i2 = 0; i2 < nSize2; i2++) {
|
||||
for (sd::LongType i3 = 0; i3 < nSize3; i3++) {
|
||||
for (sd::LongType i4 = 0; i4 < nSize4; i4++) {
|
||||
auto rX = x + (xStrides[0] * i0 + xStrides[1] * i1 +
|
||||
xStrides[2] * i2 + xStrides[3] * i3 + xStrides[4] * i4);
|
||||
auto rY = y + (yStrides[0] * i0 + yStrides[1] * i1 +
|
||||
yStrides[2] * i2 + yStrides[3] * i3 + yStrides[4] * i4);
|
||||
auto rZ = z + (zStrides[0] * i0 + zStrides[1] * i1 +
|
||||
zStrides[2] * i2 + zStrides[3] * i3 + zStrides[4] * i4);
|
||||
|
||||
*rZ = OpType::op(*rX, *rY,extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Default case for other ranks
|
||||
const int xRank = shape::rank(xShapeInfo);
|
||||
const int yRank = shape::rank(yShapeInfo);
|
||||
const int zRank = shape::rank(zShapeInfo);
|
||||
|
||||
const sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
const sd::LongType* yShape = shape::shapeOf(yShapeInfo);
|
||||
const sd::LongType* zShape = shape::shapeOf(zShapeInfo);
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
sd::LongType yCoords[SD_MAX_RANK];
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
// Calculate independent coordinates for each array
|
||||
INDEX2COORDS(i, xRank, xShape, xCoords);
|
||||
INDEX2COORDS(i, yRank, yShape, yCoords);
|
||||
INDEX2COORDS(i, zRank, zShape, zCoords);
|
||||
|
||||
// Calculate offsets based on each array's coordinates and strides
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStrides, xCoords, xOffset);
|
||||
COORDS2INDEX(yRank, yStrides, yCoords, yOffset);
|
||||
COORDS2INDEX(zRank, zStrides, zCoords, zOffset);
|
||||
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset],extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void BroadcastBool<X, Z>::execInverse(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
void *vextraParams, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *yTadShapeInfo, const sd::LongType *yTadOffset,
|
||||
const sd::LongType *zTadShapeInfo, const sd::LongType *zTadOffset, uint64_t start,
|
||||
uint64_t stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
// Handle TAD setup
|
||||
auto yTadShapeShapeInfo = yTadShapeInfo;
|
||||
auto tadOffsets = yTadOffset;
|
||||
|
||||
// When shared_ptr goes out of scope, it deletes the TadPack and invalidates pointers!
|
||||
std::shared_ptr<sd::TadPack> tadPack = nullptr;
|
||||
|
||||
if (yTadShapeInfo == nullptr || tadOffsets == nullptr) {
|
||||
tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(const_cast<sd::LongType*>(yShapeInfo), dimension,
|
||||
dimensionLength);
|
||||
yTadShapeShapeInfo = tadPack->primaryShapeInfo();
|
||||
tadOffsets = tadPack->primaryOffsets();
|
||||
}
|
||||
|
||||
if (zTadShapeInfo == nullptr) {
|
||||
zTadShapeInfo = yTadShapeShapeInfo;
|
||||
zTadOffset = tadOffsets;
|
||||
}
|
||||
|
||||
// Get shape information
|
||||
const auto xRank = shape::rank(xShapeInfo);
|
||||
const auto yTadRank = shape::rank(yTadShapeShapeInfo);
|
||||
const auto zTadRank = shape::rank(zTadShapeInfo);
|
||||
|
||||
const auto xStrides = shape::stride(xShapeInfo);
|
||||
const auto yTadStrides = shape::stride(yTadShapeShapeInfo);
|
||||
const auto zTadStrides = shape::stride(zTadShapeInfo);
|
||||
|
||||
const auto xShape = shape::shapeOf(xShapeInfo);
|
||||
const auto yTadShape = shape::shapeOf(yTadShapeShapeInfo);
|
||||
const auto zTadShape = shape::shapeOf(zTadShapeInfo);
|
||||
|
||||
const sd::LongType tadLength = shape::length(yTadShapeShapeInfo);
|
||||
|
||||
if (yTadRank <= 3) {
|
||||
// Optimized path for lower ranks
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oZ = z + zTadOffset[i];
|
||||
auto oY = y + tadOffsets[i];
|
||||
|
||||
if (yTadRank == 1) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
oZ[j * zTadStrides[0]] = OpType::op(x[j * xStrides[0]], oY[j * yTadStrides[0]],extraParams);
|
||||
}
|
||||
}
|
||||
else if (yTadRank == 2) {
|
||||
const sd::LongType dim0 = yTadShape[0];
|
||||
const sd::LongType dim1 = yTadShape[1];
|
||||
|
||||
for (sd::LongType j0 = 0; j0 < dim0; j0++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j1 = 0; j1 < dim1; j1++) {
|
||||
const auto xOffset = j0 * xStrides[0] + j1 * xStrides[1];
|
||||
const auto yOffset = j0 * yTadStrides[0] + j1 * yTadStrides[1];
|
||||
const auto zOffset = j0 * zTadStrides[0] + j1 * zTadStrides[1];
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset],extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
else { // rank 3
|
||||
const sd::LongType dim0 = yTadShape[0];
|
||||
const sd::LongType dim1 = yTadShape[1];
|
||||
const sd::LongType dim2 = yTadShape[2];
|
||||
|
||||
for (sd::LongType j0 = 0; j0 < dim0; j0++) {
|
||||
for (sd::LongType j1 = 0; j1 < dim1; j1++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j2 = 0; j2 < dim2; j2++) {
|
||||
const auto xOffset = j0 * xStrides[0] + j1 * xStrides[1] + j2 * xStrides[2];
|
||||
const auto yOffset = j0 * yTadStrides[0] + j1 * yTadStrides[1] + j2 * yTadStrides[2];
|
||||
const auto zOffset = j0 * zTadStrides[0] + j1 * zTadStrides[1] + j2 * zTadStrides[2];
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset],extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Use macros for higher ranks
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oZ = z + zTadOffset[i];
|
||||
auto oY = y + tadOffsets[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(f, yTadRank, yTadShape, coords);
|
||||
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStrides, coords, xOffset);
|
||||
COORDS2INDEX(yTadRank, yTadStrides, coords, yOffset);
|
||||
COORDS2INDEX(zTadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset],extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z, typename OpType>
|
||||
static void execRank1(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, Z *z,
|
||||
const sd::LongType *zShapeInfo, X *extraParams) {
|
||||
// Cache shape-related values
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, static_cast<sd::LongType>(0));
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, static_cast<sd::LongType>(0));
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, static_cast<sd::LongType>(0));
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, static_cast<sd::LongType>(0));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
if (zStrd0 == 1 && xStrd0 == 1 && yStrd0 == 0) {
|
||||
for (auto i0 = start; i0 < stop; ++i0) z[i0] = OpType::op(x[i0], *y, extraParams);
|
||||
} else if (zStrd0 == 1 && xStrd0 == 0 && yStrd0 == 1) {
|
||||
for (auto i0 = start; i0 < stop; ++i0) z[i0] = OpType::op(*x, y[i0], extraParams);
|
||||
} else if (zStrd0 == 1 && xStrd0 == 1 && yStrd0 == 1) {
|
||||
for (auto i0 = start; i0 < stop; ++i0) z[i0] = OpType::op(x[i0], y[i0], extraParams);
|
||||
} else {
|
||||
for (auto i0 = start; i0 < stop; ++i0) z[i0 * zStrd0] = OpType::op(x[i0 * xStrd0], y[i0 * yStrd0], extraParams);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, static_cast<sd::LongType>(0), zAxis0);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z, typename OpType>
|
||||
static void execRank2(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, Z *z,
|
||||
const sd::LongType *zShapeInfo, X *extraParams) {
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(1));
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(1));
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(1));
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(1));
|
||||
|
||||
sd::LongType zAxis1 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(0));
|
||||
sd::LongType xStrd1 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(0));
|
||||
sd::LongType yStrd1 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(0));
|
||||
sd::LongType zStrd1 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(0));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i0 = start; i0 < stop; ++i0) {
|
||||
auto x0 = x + i0 * xStrd0;
|
||||
auto y0 = y + i0 * yStrd0;
|
||||
auto z0 = z + i0 * zStrd0;
|
||||
|
||||
if (zStrd1 == 1 && xStrd1 == 1 && yStrd1 == 0)
|
||||
for (sd::LongType i1 = 0; i1 < zAxis1; ++i1) z0[i1] = OpType::op(x0[i1], *y0, extraParams);
|
||||
else if (zStrd1 == 1 && xStrd1 == 0 && yStrd1 == 1)
|
||||
for (sd::LongType i1 = 0; i1 < zAxis1; ++i1) z0[i1] = OpType::op(*x0, y0[i1], extraParams);
|
||||
else if (zStrd1 == 1 && xStrd1 == 1 && yStrd1 == 1)
|
||||
for (sd::LongType i1 = 0; i1 < zAxis1; ++i1) z0[i1] = OpType::op(x0[i1], y0[i1], extraParams);
|
||||
else
|
||||
for (sd::LongType i1 = 0; i1 < zAxis1; ++i1)
|
||||
z0[i1 * zStrd1] = OpType::op(x0[i1 * xStrd1], y0[i1 * yStrd1], extraParams);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, static_cast<sd::LongType>(0), zAxis0);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z, typename OpType>
|
||||
static void execRank3(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, Z *z,
|
||||
const sd::LongType *zShapeInfo, X *extraParams) {
|
||||
// Cache shape-related values
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(2));
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(2));
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(2));
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(2));
|
||||
|
||||
sd::LongType zAxis1 = shape::sizeAt(zShapeInfo, static_cast<sd::LongType>(1));
|
||||
sd::LongType xStrd1 = shape::strideAt(xShapeInfo, static_cast<sd::LongType>(1));
|
||||
sd::LongType yStrd1 = shape::strideAt(yShapeInfo, static_cast<sd::LongType>(1));
|
||||
sd::LongType zStrd1 = shape::strideAt(zShapeInfo, static_cast<sd::LongType>(1));
|
||||
|
||||
sd::LongType zAxis2 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(0));
|
||||
sd::LongType xStrd2 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(0));
|
||||
sd::LongType yStrd2 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(0));
|
||||
sd::LongType zStrd2 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(0));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
for (auto i0 = start_x; i0 < stop_x; ++i0) {
|
||||
for (auto i1 = start_y; i1 < stop_y; ++i1) {
|
||||
auto x1 = x + i0 * xStrd0 + i1 * xStrd1;
|
||||
auto y1 = y + i0 * yStrd0 + i1 * yStrd1;
|
||||
auto z1 = z + i0 * zStrd0 + i1 * zStrd1;
|
||||
|
||||
if (zStrd2 == 1 && xStrd2 == 1 && yStrd2 == 0)
|
||||
for (sd::LongType i2 = 0; i2 < zAxis2; ++i2) z1[i2] = OpType::op(x1[i2], *y1, extraParams);
|
||||
else if (zStrd2 == 1 && xStrd2 == 0 && yStrd2 == 1)
|
||||
for (sd::LongType i2 = 0; i2 < zAxis2; ++i2) z1[i2] = OpType::op(*x1, y1[i2], extraParams);
|
||||
else if (zStrd2 == 1 && xStrd2 == 1 && yStrd2 == 1)
|
||||
for (sd::LongType i2 = 0; i2 < zAxis2; ++i2) z1[i2] = OpType::op(x1[i2], y1[i2], extraParams);
|
||||
else
|
||||
for (sd::LongType i2 = 0; i2 < zAxis2; ++i2)
|
||||
z1[i2 * zStrd2] = OpType::op(x1[i2 * xStrd2], y1[i2 * yStrd2], extraParams);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, static_cast<sd::LongType>(0), zAxis0, static_cast<sd::LongType>(1), static_cast<sd::LongType>(0), zAxis1, static_cast<sd::LongType>(1));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z, typename OpType>
|
||||
static void execRank4(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, Z *z,
|
||||
const sd::LongType *zShapeInfo, X *extraParams) {
|
||||
// Cache shape-related values
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(3));
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(3));
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(3));
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(3));
|
||||
|
||||
sd::LongType zAxis1 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(2));
|
||||
sd::LongType xStrd1 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(2));
|
||||
sd::LongType yStrd1 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(2));
|
||||
sd::LongType zStrd1 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(2));
|
||||
|
||||
sd::LongType zAxis2 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(1));
|
||||
sd::LongType xStrd2 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(1));
|
||||
sd::LongType yStrd2 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(1));
|
||||
sd::LongType zStrd2 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(1));
|
||||
|
||||
sd::LongType zAxis3 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(0));
|
||||
sd::LongType xStrd3 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(0));
|
||||
sd::LongType yStrd3 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(0));
|
||||
sd::LongType zStrd3 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(0));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
for (auto i0 = start_x; i0 < stop_x; ++i0) {
|
||||
for (auto i1 = start_y; i1 < stop_y; ++i1) {
|
||||
for (auto i2 = start_z; i2 < stop_z; ++i2) {
|
||||
auto x2 = x + i0 * xStrd0 + i1 * xStrd1 + i2 * xStrd2;
|
||||
auto y2 = y + i0 * yStrd0 + i1 * yStrd1 + i2 * yStrd2;
|
||||
auto z2 = z + i0 * zStrd0 + i1 * zStrd1 + i2 * zStrd2;
|
||||
|
||||
if (zStrd3 == 1 && xStrd3 == 1 && yStrd3 == 0)
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) z2[i3] = OpType::op(x2[i3], *y2, extraParams);
|
||||
else if (zStrd3 == 1 && xStrd3 == 0 && yStrd3 == 1)
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) z2[i3] = OpType::op(*x2, y2[i3], extraParams);
|
||||
else if (zStrd3 == 1 && xStrd3 == 1 && yStrd3 == 1)
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) z2[i3] = OpType::op(x2[i3], y2[i3], extraParams);
|
||||
else
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3)
|
||||
z2[i3 * zStrd3] = OpType::op(x2[i3 * xStrd3], y2[i3 * yStrd3], extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, static_cast<sd::LongType>(0), zAxis0, static_cast<sd::LongType>(1),
|
||||
static_cast<sd::LongType>(0), zAxis1, static_cast<sd::LongType>(1),
|
||||
static_cast<sd::LongType>(0), zAxis2, static_cast<sd::LongType>(1));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z, typename OpType>
|
||||
static void execRank5(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, Z *z,
|
||||
const sd::LongType *zShapeInfo, X *extraParams) {
|
||||
// Cache shape-related values
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(4));
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(4));
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(4));
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(4));
|
||||
|
||||
sd::LongType zAxis1 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(3));
|
||||
sd::LongType xStrd1 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(3));
|
||||
sd::LongType yStrd1 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(3));
|
||||
sd::LongType zStrd1 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(3));
|
||||
|
||||
sd::LongType zAxis2 = shape::sizeAt(zShapeInfo, static_cast<sd::LongType>(2));
|
||||
sd::LongType xStrd2 = shape::strideAt(xShapeInfo, static_cast<sd::LongType>(2));
|
||||
sd::LongType yStrd2 = shape::strideAt(yShapeInfo, static_cast<sd::LongType>(2));
|
||||
sd::LongType zStrd2 = shape::strideAt(zShapeInfo, static_cast<sd::LongType>(2));
|
||||
|
||||
sd::LongType zAxis3 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(1));
|
||||
sd::LongType xStrd3 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(1));
|
||||
sd::LongType yStrd3 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(1));
|
||||
sd::LongType zStrd3 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(1));
|
||||
|
||||
sd::LongType zAxis4 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(4) : static_cast<sd::LongType>(0));
|
||||
sd::LongType xStrd4 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(4) : static_cast<sd::LongType>(0));
|
||||
sd::LongType yStrd4 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(4) : static_cast<sd::LongType>(0));
|
||||
sd::LongType zStrd4 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(4) : static_cast<sd::LongType>(0));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
for (auto i0 = start_x; i0 < stop_x; ++i0) {
|
||||
for (auto i1 = start_y; i1 < stop_y; ++i1) {
|
||||
for (auto i2 = start_z; i2 < stop_z; ++i2) {
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) {
|
||||
auto x3 = x + i0 * xStrd0 + i1 * xStrd1 + i2 * xStrd2 + i3 * xStrd3;
|
||||
auto y3 = y + i0 * yStrd0 + i1 * yStrd1 + i2 * yStrd2 + i3 * yStrd3;
|
||||
auto z3 = z + i0 * zStrd0 + i1 * zStrd1 + i2 * zStrd2 + i3 * zStrd3;
|
||||
|
||||
if (zStrd4 == 1 && xStrd4 == 1 && yStrd4 == 0)
|
||||
for (sd::LongType i4 = 0; i4 < zAxis4; ++i4) z3[i4] = OpType::op(x3[i4], *y3, extraParams);
|
||||
else if (zStrd4 == 1 && xStrd4 == 0 && yStrd4 == 1)
|
||||
for (sd::LongType i4 = 0; i4 < zAxis4; ++i4) z3[i4] = OpType::op(*x3, y3[i4], extraParams);
|
||||
else if (zStrd4 == 1 && xStrd4 == 1 && yStrd4 == 1)
|
||||
for (sd::LongType i4 = 0; i4 < zAxis4; ++i4) z3[i4] = OpType::op(x3[i4], y3[i4], extraParams);
|
||||
else
|
||||
for (sd::LongType i4 = 0; i4 < zAxis4; ++i4)
|
||||
z3[i4 * zStrd4] = OpType::op(x3[i4 * xStrd4], y3[i4 * yStrd4], extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, static_cast<sd::LongType>(0), zAxis0, static_cast<sd::LongType>(1),
|
||||
static_cast<sd::LongType>(0), zAxis1, static_cast<sd::LongType>(1),
|
||||
static_cast<sd::LongType>(0), zAxis2, static_cast<sd::LongType>(1));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z, typename OpType>
|
||||
static void execDefault(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, Z *z,
|
||||
const sd::LongType *zShapeInfo, X *extraParams) {
|
||||
const bool xzSameOffsets = shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo);
|
||||
const bool yzSameOffsets = shape::haveSameShapeAndStrides(yShapeInfo, zShapeInfo);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
|
||||
// C-style arrays CANNOT be captured by value in lambdas - they decay to pointers
|
||||
// that point to stack memory. std::array CAN be captured by value, ensuring each
|
||||
// parallel thread gets its own copy of the data with guaranteed lifetime.
|
||||
std::array<sd::LongType, SD_MAX_RANK> zShapeLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> zStrideLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> xShapeLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> xStrideLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> yShapeLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> yStrideLocal;
|
||||
|
||||
std::memcpy(zShapeLocal.data(), shape::shapeOf(zShapeInfo), zRank * sizeof(sd::LongType));
|
||||
std::memcpy(zStrideLocal.data(), shape::stride(zShapeInfo), zRank * sizeof(sd::LongType));
|
||||
std::memcpy(xShapeLocal.data(), shape::shapeOf(xShapeInfo), xRank * sizeof(sd::LongType));
|
||||
std::memcpy(xStrideLocal.data(), shape::stride(xShapeInfo), xRank * sizeof(sd::LongType));
|
||||
std::memcpy(yShapeLocal.data(), shape::shapeOf(yShapeInfo), yRank * sizeof(sd::LongType));
|
||||
std::memcpy(yStrideLocal.data(), shape::stride(yShapeInfo), yRank * sizeof(sd::LongType));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
sd::LongType yCoords[SD_MAX_RANK];
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
|
||||
for (auto i = start; i < stop; ++i) {
|
||||
// Convert linear index to coordinates based on Z (output) shape
|
||||
INDEX2COORDS(i, zRank, zShapeLocal.data(), zCoords);
|
||||
COORDS2INDEX(zRank, zStrideLocal.data(), zCoords, zOffset);
|
||||
|
||||
if (xzSameOffsets) {
|
||||
xOffset = zOffset;
|
||||
} else {
|
||||
// Broadcast Z coordinates to X shape
|
||||
for (sd::LongType d = 0; d < xRank; d++) {
|
||||
xCoords[d] = xShapeLocal[d] == 1 ? 0 : (zCoords[d] % xShapeLocal[d]);
|
||||
}
|
||||
COORDS2INDEX(xRank, xStrideLocal.data(), xCoords, xOffset);
|
||||
}
|
||||
if (yzSameOffsets) {
|
||||
yOffset = zOffset;
|
||||
} else {
|
||||
// Broadcast Z coordinates to Y shape
|
||||
for (sd::LongType d = 0; d < yRank; d++) {
|
||||
yCoords[d] = yShapeLocal[d] == 1 ? 0 : (zCoords[d] % yShapeLocal[d]);
|
||||
}
|
||||
COORDS2INDEX(yRank, yStrideLocal.data(), yCoords, yOffset);
|
||||
}
|
||||
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset], extraParams);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, static_cast<sd::LongType>(0), shape::length(zShapeInfo));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void BroadcastBool<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
void *vextraParams) {
|
||||
const X *x = reinterpret_cast<const X *>(vx);
|
||||
const X *y = reinterpret_cast<const X *>(vy);
|
||||
Z *z = reinterpret_cast<Z *>(vz);
|
||||
|
||||
X *extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
const int rank = shape::rank(zShapeInfo); // xRank = yRank = zRank
|
||||
|
||||
switch (rank) {
|
||||
case 1:
|
||||
execRank1<X, Z, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams);
|
||||
break;
|
||||
case 2:
|
||||
execRank2<X, Z, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams);
|
||||
break;
|
||||
case 3:
|
||||
execRank3<X, Z, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams);
|
||||
break;
|
||||
case 4:
|
||||
execRank4<X, Z, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams);
|
||||
break;
|
||||
case 5:
|
||||
execRank5<X, Z, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams);
|
||||
break;
|
||||
default:
|
||||
execDefault<X, Z, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace broadcast
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,650 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <loops/broadcasting_int.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace broadcast {
|
||||
|
||||
template <typename X>
|
||||
void BroadcastInt<X>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
sd::LongType*dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo, const sd::LongType *xTadOffset,
|
||||
const sd::LongType *zTadShapeInfo, const sd::LongType *zTadOffset, sd::LongType start,
|
||||
sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_T(exec,
|
||||
PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, dimension, dimensionLength, xTadShapeInfo,
|
||||
xTadOffset, zTadShapeInfo, zTadOffset, start, stop),
|
||||
BROADCAST_INT_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void BroadcastInt<X>::exec(const int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo) {
|
||||
DISPATCH_BY_OPNUM_T(exec, PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo), BROADCAST_INT_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void BroadcastInt<X>::execInverse(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffset, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffset, sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_T(execInverse,
|
||||
PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, dimension, dimensionLength, xTadShapeInfo,
|
||||
xTadOffset, zTadShapeInfo, zTadOffset, start, stop),
|
||||
BROADCAST_INT_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void BroadcastInt<X>::exec(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo, const sd::LongType *xTadOffset,
|
||||
const sd::LongType *zTadShapeInfo, const sd::LongType *zTadOffset, sd::LongType start,
|
||||
sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X*>(vx);
|
||||
auto y = reinterpret_cast<const X*>(vy);
|
||||
auto z = reinterpret_cast<X*>(vz);
|
||||
sd::LoopKind::Kind loopKind= sd::LoopKind::deduceKindOfLoopXYZ(xTadShapeInfo, yShapeInfo, zTadShapeInfo);
|
||||
|
||||
if (loopKind == sd::LoopKind::BROADCAST_SCALAR_X) {
|
||||
sd::LongType tadLength = shape::length(xTadShapeInfo);
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oY = y + (i * tadLength);
|
||||
auto oZ = z + (i * tadLength);
|
||||
const auto oX = x[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++)
|
||||
oZ[f] = OpType::op(oX, oY[f]);
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_SCALAR_Y) {
|
||||
sd::LongType tadLength = shape::length(xTadShapeInfo);
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oX = x + (i * tadLength);
|
||||
auto oZ = z + (i * tadLength);
|
||||
const auto oY = y[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++)
|
||||
oZ[f] = OpType::op(oX[f], oY);
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_2D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType* xStrides = shape::stride(xTadShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zTadShapeInfo);
|
||||
|
||||
for (auto i0 = start; i0 < stop; i0++) {
|
||||
auto baseX = x + xTadOffset[i0];
|
||||
auto baseZ = z + zTadOffset[i0];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < nSize1; i1++) {
|
||||
auto rX = baseX + xStrides[1] * i1;
|
||||
auto rY = y + yStrides[1] * i1;
|
||||
auto rZ = baseZ + zStrides[1] * i1;
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_3D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType nSize2 = shape::sizeAt(zShapeInfo, 2);
|
||||
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
|
||||
for (auto i0 = start; i0 < stop; i0++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i1 = 0; i1 < nSize1; i1++) {
|
||||
for (sd::LongType i2 = 0; i2 < nSize2; i2++) {
|
||||
auto rX = x + (xStrides[0] * i0 + xStrides[1] * i1 + xStrides[2] * i2);
|
||||
auto rY = y + (yStrides[0] * i0 + yStrides[1] * i1 + yStrides[2] * i2);
|
||||
auto rZ = z + (zStrides[0] * i0 + zStrides[1] * i1 + zStrides[2] * i2);
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_4D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType nSize2 = shape::sizeAt(zShapeInfo, 2);
|
||||
const sd::LongType nSize3 = shape::sizeAt(zShapeInfo, 3);
|
||||
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
uint64_t i0 = i / nSize1;
|
||||
uint64_t i1 = i % nSize1;
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i2 = 0; i2 < nSize2; i2++) {
|
||||
for (sd::LongType i3 = 0; i3 < nSize3; i3++) {
|
||||
auto rX = x + (xStrides[0] * i0 + xStrides[1] * i1 +
|
||||
xStrides[2] * i2 + xStrides[3] * i3);
|
||||
auto rY = y + (yStrides[0] * i0 + yStrides[1] * i1 +
|
||||
yStrides[2] * i2 + yStrides[3] * i3);
|
||||
auto rZ = z + (zStrides[0] * i0 + zStrides[1] * i1 +
|
||||
zStrides[2] * i2 + zStrides[3] * i3);
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (loopKind == sd::LoopKind::BROADCAST_5D) {
|
||||
const sd::LongType nSize1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType nSize2 = shape::sizeAt(zShapeInfo, 2);
|
||||
const sd::LongType nSize3 = shape::sizeAt(zShapeInfo, 3);
|
||||
const sd::LongType nSize4 = shape::sizeAt(zShapeInfo, 4);
|
||||
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
uint32_t i0 = i / nSize1;
|
||||
uint32_t i1 = i % nSize1;
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i2 = 0; i2 < nSize2; i2++) {
|
||||
for (sd::LongType i3 = 0; i3 < nSize3; i3++) {
|
||||
for (sd::LongType i4 = 0; i4 < nSize4; i4++) {
|
||||
auto rX = x + (xStrides[0] * i0 + xStrides[1] * i1 +
|
||||
xStrides[2] * i2 + xStrides[3] * i3 + xStrides[4] * i4);
|
||||
auto rY = y + (yStrides[0] * i0 + yStrides[1] * i1 +
|
||||
yStrides[2] * i2 + yStrides[3] * i3 + yStrides[4] * i4);
|
||||
auto rZ = z + (zStrides[0] * i0 + zStrides[1] * i1 +
|
||||
zStrides[2] * i2 + zStrides[3] * i3 + zStrides[4] * i4);
|
||||
|
||||
*rZ = OpType::op(*rX, *rY);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Default case for other ranks
|
||||
const int xRank = shape::rank(xShapeInfo);
|
||||
const int yRank = shape::rank(yShapeInfo);
|
||||
const int zRank = shape::rank(zShapeInfo);
|
||||
|
||||
const sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
const sd::LongType* yShape = shape::shapeOf(yShapeInfo);
|
||||
const sd::LongType* zShape = shape::shapeOf(zShapeInfo);
|
||||
const sd::LongType* xStrides = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStrides = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStrides = shape::stride(zShapeInfo);
|
||||
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
sd::LongType yCoords[SD_MAX_RANK];
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
// Calculate independent coordinates for each array
|
||||
INDEX2COORDS(i, xRank, xShape, xCoords);
|
||||
INDEX2COORDS(i, yRank, yShape, yCoords);
|
||||
INDEX2COORDS(i, zRank, zShape, zCoords);
|
||||
|
||||
// Calculate offsets based on each array's coordinates and strides
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStrides, xCoords, xOffset);
|
||||
COORDS2INDEX(yRank, yStrides, yCoords, yOffset);
|
||||
COORDS2INDEX(zRank, zStrides, zCoords, zOffset);
|
||||
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void BroadcastInt<X>::execInverse(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, const sd::LongType *yTadShapeInfo,
|
||||
const sd::LongType *yTadOffset, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffset, sd::LongType start, sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
|
||||
// Handle TAD setup
|
||||
auto yTadShapeShapeInfo = yTadShapeInfo;
|
||||
auto tadOffsets = yTadOffset;
|
||||
|
||||
// When shared_ptr goes out of scope, it deletes the TadPack and invalidates pointers!
|
||||
std::shared_ptr<sd::TadPack> tadPack = nullptr;
|
||||
|
||||
if (yTadShapeInfo == nullptr || tadOffsets == nullptr) {
|
||||
tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(const_cast<sd::LongType*>(yShapeInfo), dimension,
|
||||
dimensionLength);
|
||||
yTadShapeShapeInfo = tadPack->primaryShapeInfo();
|
||||
tadOffsets = tadPack->primaryOffsets();
|
||||
}
|
||||
|
||||
if (zTadShapeInfo == nullptr) {
|
||||
zTadShapeInfo = yTadShapeShapeInfo;
|
||||
zTadOffset = tadOffsets;
|
||||
}
|
||||
|
||||
// Get shape information
|
||||
const auto xRank = shape::rank(xShapeInfo);
|
||||
const auto yTadRank = shape::rank(yTadShapeShapeInfo);
|
||||
const auto zTadRank = shape::rank(zTadShapeInfo);
|
||||
|
||||
const auto xStrides = shape::stride(xShapeInfo);
|
||||
const auto yTadStrides = shape::stride(yTadShapeShapeInfo);
|
||||
const auto zTadStrides = shape::stride(zTadShapeInfo);
|
||||
|
||||
const auto xShape = shape::shapeOf(xShapeInfo);
|
||||
const auto yTadShape = shape::shapeOf(yTadShapeShapeInfo);
|
||||
const auto zTadShape = shape::shapeOf(zTadShapeInfo);
|
||||
|
||||
const sd::LongType tadLength = shape::length(yTadShapeShapeInfo);
|
||||
|
||||
if (yTadRank <= 3) {
|
||||
// Optimized path for lower ranks
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oZ = z + zTadOffset[i];
|
||||
auto oY = y + tadOffsets[i];
|
||||
|
||||
if (yTadRank == 1) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j = 0; j < tadLength; j++) {
|
||||
oZ[j * zTadStrides[0]] = OpType::op(x[j * xStrides[0]], oY[j * yTadStrides[0]]);
|
||||
}
|
||||
}
|
||||
else if (yTadRank == 2) {
|
||||
const sd::LongType dim0 = yTadShape[0];
|
||||
const sd::LongType dim1 = yTadShape[1];
|
||||
|
||||
for (sd::LongType j0 = 0; j0 < dim0; j0++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j1 = 0; j1 < dim1; j1++) {
|
||||
const auto xOffset = j0 * xStrides[0] + j1 * xStrides[1];
|
||||
const auto yOffset = j0 * yTadStrides[0] + j1 * yTadStrides[1];
|
||||
const auto zOffset = j0 * zTadStrides[0] + j1 * zTadStrides[1];
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else { // rank 3
|
||||
const sd::LongType dim0 = yTadShape[0];
|
||||
const sd::LongType dim1 = yTadShape[1];
|
||||
const sd::LongType dim2 = yTadShape[2];
|
||||
|
||||
for (sd::LongType j0 = 0; j0 < dim0; j0++) {
|
||||
for (sd::LongType j1 = 0; j1 < dim1; j1++) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType j2 = 0; j2 < dim2; j2++) {
|
||||
const auto xOffset = j0 * xStrides[0] + j1 * xStrides[1] + j2 * xStrides[2];
|
||||
const auto yOffset = j0 * yTadStrides[0] + j1 * yTadStrides[1] + j2 * yTadStrides[2];
|
||||
const auto zOffset = j0 * zTadStrides[0] + j1 * zTadStrides[1] + j2 * zTadStrides[2];
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Use macros for higher ranks
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto oZ = z + zTadOffset[i];
|
||||
auto oY = y + tadOffsets[i];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType f = 0; f < tadLength; f++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(f, yTadRank, yTadShape, coords);
|
||||
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStrides, coords, xOffset);
|
||||
COORDS2INDEX(yTadRank, yTadStrides, coords, yOffset);
|
||||
COORDS2INDEX(zTadRank, zTadStrides, coords, zOffset);
|
||||
|
||||
oZ[zOffset] = OpType::op(x[xOffset], oY[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename OpType>
|
||||
static void execRank1(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, X *z,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, static_cast<sd::LongType>(0));
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, static_cast<sd::LongType>(0));
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, static_cast<sd::LongType>(0));
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, static_cast<sd::LongType>(0));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
if (zStrd0 == 1 && xStrd0 == 1 && yStrd0 == 0) {
|
||||
for (auto i0 = start; i0 < stop; ++i0) z[i0] = OpType::op(x[i0], *y);
|
||||
} else if (zStrd0 == 1 && xStrd0 == 0 && yStrd0 == 1) {
|
||||
for (auto i0 = start; i0 < stop; ++i0) z[i0] = OpType::op(*x, y[i0]);
|
||||
} else if (zStrd0 == 1 && xStrd0 == 1 && yStrd0 == 1) {
|
||||
for (auto i0 = start; i0 < stop; ++i0) z[i0] = OpType::op(x[i0], y[i0]);
|
||||
} else {
|
||||
for (auto i0 = start; i0 < stop; ++i0) z[i0 * zStrd0] = OpType::op(x[i0 * xStrd0], y[i0 * yStrd0]);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, static_cast<sd::LongType>(0), zAxis0);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename OpType>
|
||||
static void execRank2(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, X *z,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(1) );
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(1) );
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(1) );
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(1) );
|
||||
|
||||
sd::LongType zAxis1 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(0) );
|
||||
sd::LongType xStrd1 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(0) );
|
||||
sd::LongType yStrd1 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(0) );
|
||||
sd::LongType zStrd1 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(0) );
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i0 = start; i0 < stop; ++i0) {
|
||||
auto x0 = x + i0 * xStrd0;
|
||||
auto y0 = y + i0 * yStrd0;
|
||||
auto z0 = z + i0 * zStrd0;
|
||||
|
||||
if (zStrd1 == 1 && xStrd1 == 1 && yStrd1 == 0)
|
||||
for (sd::LongType i1 = 0; i1 < zAxis1; ++i1) z0[i1] = OpType::op(x0[i1], *y0);
|
||||
else if (zStrd1 == 1 && xStrd1 == 0 && yStrd1 == 1)
|
||||
for (sd::LongType i1 = 0; i1 < zAxis1; ++i1) z0[i1] = OpType::op(*x0, y0[i1]);
|
||||
else if (zStrd1 == 1 && xStrd1 == 1 && yStrd1 == 1)
|
||||
for (sd::LongType i1 = 0; i1 < zAxis1; ++i1) z0[i1] = OpType::op(x0[i1], y0[i1]);
|
||||
else
|
||||
for (sd::LongType i1 = 0; i1 < zAxis1; ++i1) z0[i1 * zStrd1] = OpType::op(x0[i1 * xStrd1], y0[i1 * yStrd1]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, static_cast<sd::LongType>(0), zAxis0);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename OpType>
|
||||
static void execRank3(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, X *z,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
// Cache all shape-related values
|
||||
const sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
const sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
const sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
|
||||
auto xStride = shape::stride(xShapeInfo);
|
||||
auto yStride = shape::stride(yShapeInfo);
|
||||
auto zStride = shape::stride(zShapeInfo);
|
||||
|
||||
const sd::LongType zLen0 = shape::sizeAt(zShapeInfo, 0);
|
||||
const sd::LongType zLen1 = shape::sizeAt(zShapeInfo, 1);
|
||||
const sd::LongType zLen2 = shape::sizeAt(zShapeInfo, 2);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType coords[3];
|
||||
for (auto i0 = start_x; i0 < stop_x; ++i0) {
|
||||
for (auto i1 = start_y; i1 < stop_y; ++i1) {
|
||||
for (sd::LongType i2 = 0; i2 < zLen2; ++i2) {
|
||||
coords[0] = i0;
|
||||
coords[1] = i1;
|
||||
coords[2] = i2;
|
||||
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, coords, yOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset]);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, zLen0, 1, 0, zLen1, 1);
|
||||
}
|
||||
|
||||
template <typename X, typename OpType>
|
||||
static void execRank4(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, X *z,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(3));
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(3));
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(3));
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(3));
|
||||
|
||||
sd::LongType zAxis1 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(2));
|
||||
sd::LongType xStrd1 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(2));
|
||||
sd::LongType yStrd1 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(2));
|
||||
sd::LongType zStrd1 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(2));
|
||||
|
||||
sd::LongType zAxis2 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(1));
|
||||
sd::LongType xStrd2 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(1));
|
||||
sd::LongType yStrd2 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(1));
|
||||
sd::LongType zStrd2 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(2) : static_cast<sd::LongType>(1));
|
||||
|
||||
sd::LongType zAxis3 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(0));
|
||||
sd::LongType xStrd3 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(0));
|
||||
sd::LongType yStrd3 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(0));
|
||||
sd::LongType zStrd3 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(0));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
for (auto i0 = start_x; i0 < stop_x; ++i0) {
|
||||
for (auto i1 = start_y; i1 < stop_y; ++i1) {
|
||||
for (auto i2 = start_z; i2 < stop_z; ++i2) {
|
||||
auto x2 = x + i0 * xStrd0 + i1 * xStrd1 + i2 * xStrd2;
|
||||
auto y2 = y + i0 * yStrd0 + i1 * yStrd1 + i2 * yStrd2;
|
||||
auto z2 = z + i0 * zStrd0 + i1 * zStrd1 + i2 * zStrd2;
|
||||
|
||||
if (zStrd3 == 1 && xStrd3 == 1 && yStrd3 == 0)
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) z2[i3] = OpType::op(x2[i3], *y2);
|
||||
else if (zStrd3 == 1 && xStrd3 == 0 && yStrd3 == 1)
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) z2[i3] = OpType::op(*x2, y2[i3]);
|
||||
else if (zStrd3 == 1 && xStrd3 == 1 && yStrd3 == 1)
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) z2[i3] = OpType::op(x2[i3], y2[i3]);
|
||||
else
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) z2[i3 * zStrd3] = OpType::op(x2[i3 * xStrd3], y2[i3 * yStrd3]);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, static_cast<sd::LongType>(0), zAxis0, static_cast<sd::LongType>(1), static_cast<sd::LongType>(0), zAxis1, static_cast<sd::LongType>(1), static_cast<sd::LongType>(0), zAxis2, static_cast<sd::LongType>(1));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename OpType>
|
||||
static void execRank5(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, X *z,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
sd::LongType zAxis0 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(4));
|
||||
sd::LongType xStrd0 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(4));
|
||||
sd::LongType yStrd0 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(4));
|
||||
sd::LongType zStrd0 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(0) : static_cast<sd::LongType>(4));
|
||||
|
||||
sd::LongType zAxis1 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(3));
|
||||
sd::LongType xStrd1 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(3));
|
||||
sd::LongType yStrd1 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(3));
|
||||
sd::LongType zStrd1 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(1) : static_cast<sd::LongType>(3));
|
||||
|
||||
sd::LongType zAxis2 = shape::sizeAt(zShapeInfo, static_cast<sd::LongType>(2));
|
||||
sd::LongType xStrd2 = shape::strideAt(xShapeInfo, static_cast<sd::LongType>(2));
|
||||
sd::LongType yStrd2 = shape::strideAt(yShapeInfo, static_cast<sd::LongType>(2));
|
||||
sd::LongType zStrd2 = shape::strideAt(zShapeInfo, static_cast<sd::LongType>(2));
|
||||
|
||||
sd::LongType zAxis3 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(1));
|
||||
sd::LongType xStrd3 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(1));
|
||||
sd::LongType yStrd3 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(1));
|
||||
sd::LongType zStrd3 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(3) : static_cast<sd::LongType>(1));
|
||||
|
||||
sd::LongType zAxis4 = shape::sizeAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(4) : static_cast<sd::LongType>(0));
|
||||
sd::LongType xStrd4 = shape::strideAt(xShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(4) : static_cast<sd::LongType>(0));
|
||||
sd::LongType yStrd4 = shape::strideAt(yShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(4) : static_cast<sd::LongType>(0));
|
||||
sd::LongType zStrd4 = shape::strideAt(zShapeInfo, shape::order(zShapeInfo) == 'c' ? static_cast<sd::LongType>(4) : static_cast<sd::LongType>(0));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
for (auto i0 = start_x; i0 < stop_x; ++i0) {
|
||||
for (auto i1 = start_y; i1 < stop_y; ++i1) {
|
||||
for (auto i2 = start_z; i2 < stop_z; ++i2) {
|
||||
for (sd::LongType i3 = 0; i3 < zAxis3; ++i3) {
|
||||
auto x3 = x + i0 * xStrd0 + i1 * xStrd1 + i2 * xStrd2 + i3 * xStrd3;
|
||||
auto y3 = y + i0 * yStrd0 + i1 * yStrd1 + i2 * yStrd2 + i3 * yStrd3;
|
||||
auto z3 = z + i0 * zStrd0 + i1 * zStrd1 + i2 * zStrd2 + i3 * zStrd3;
|
||||
|
||||
if (zStrd4 == 1 && xStrd4 == 1 && yStrd4 == 0)
|
||||
for (sd::LongType i4 = 0; i4 < zAxis4; ++i4) z3[i4] = OpType::op(x3[i4], *y3);
|
||||
else if (zStrd4 == 1 && xStrd4 == 0 && yStrd4 == 1)
|
||||
for (sd::LongType i4 = 0; i4 < zAxis4; ++i4) z3[i4] = OpType::op(*x3, y3[i4]);
|
||||
else if (zStrd4 == 1 && xStrd4 == 1 && yStrd4 == 1)
|
||||
for (sd::LongType i4 = 0; i4 < zAxis4; ++i4) z3[i4] = OpType::op(x3[i4], y3[i4]);
|
||||
else
|
||||
for (sd::LongType i4 = 0; i4 < zAxis4; ++i4)
|
||||
z3[i4 * zStrd4] = OpType::op(x3[i4 * xStrd4], y3[i4 * yStrd4]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, static_cast<sd::LongType>(0), zAxis0, static_cast<sd::LongType>(1), static_cast<sd::LongType>(0), zAxis1, static_cast<sd::LongType>(1), static_cast<sd::LongType>(0), zAxis2, static_cast<sd::LongType>(1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
template <typename X, typename OpType>
|
||||
static void execDefault(const X *x, const sd::LongType *xShapeInfo, const X *y, const sd::LongType *yShapeInfo, X *z,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
// Cache all shape-related values
|
||||
const sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
const sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
const sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
const sd::LongType zLen = shape::length(zShapeInfo);
|
||||
|
||||
const bool xzSameOffsets = shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo);
|
||||
const bool yzSameOffsets = shape::haveSameShapeAndStrides(yShapeInfo, zShapeInfo);
|
||||
|
||||
// C-style arrays CANNOT be captured by value in lambdas - they decay to pointers
|
||||
// that point to stack memory. std::array CAN be captured by value, ensuring each
|
||||
// parallel thread gets its own copy of the data with guaranteed lifetime.
|
||||
std::array<sd::LongType, SD_MAX_RANK> xShapeLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> xStrideLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> yShapeLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> yStrideLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> zStrideLocal;
|
||||
std::array<sd::LongType, SD_MAX_RANK> zShapeLocal;
|
||||
|
||||
std::memcpy(xShapeLocal.data(), shape::shapeOf(xShapeInfo), xRank * sizeof(sd::LongType));
|
||||
std::memcpy(xStrideLocal.data(), shape::stride(xShapeInfo), xRank * sizeof(sd::LongType));
|
||||
std::memcpy(yShapeLocal.data(), shape::shapeOf(yShapeInfo), yRank * sizeof(sd::LongType));
|
||||
std::memcpy(yStrideLocal.data(), shape::stride(yShapeInfo), yRank * sizeof(sd::LongType));
|
||||
std::memcpy(zStrideLocal.data(), shape::stride(zShapeInfo), zRank * sizeof(sd::LongType));
|
||||
std::memcpy(zShapeLocal.data(), shape::shapeOf(zShapeInfo), zRank * sizeof(sd::LongType));
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
sd::LongType yCoords[SD_MAX_RANK];
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
|
||||
for (auto i = start; i < stop; ++i) {
|
||||
// Convert linear index to coordinates based on Z (output) shape
|
||||
INDEX2COORDS(i, zRank, zShapeLocal.data(), zCoords);
|
||||
COORDS2INDEX(zRank, zStrideLocal.data(), zCoords, zOffset);
|
||||
|
||||
if (xzSameOffsets) {
|
||||
xOffset = zOffset;
|
||||
} else {
|
||||
// Broadcast Z coordinates to X shape
|
||||
for (sd::LongType d = 0; d < xRank; d++) {
|
||||
xCoords[d] = xShapeLocal[d] == 1 ? 0 : (zCoords[d] % xShapeLocal[d]);
|
||||
}
|
||||
COORDS2INDEX(xRank, xStrideLocal.data(), xCoords, xOffset);
|
||||
}
|
||||
|
||||
if (yzSameOffsets) {
|
||||
yOffset = zOffset;
|
||||
} else {
|
||||
// Broadcast Z coordinates to Y shape
|
||||
for (sd::LongType d = 0; d < yRank; d++) {
|
||||
yCoords[d] = yShapeLocal[d] == 1 ? 0 : (zCoords[d] % yShapeLocal[d]);
|
||||
}
|
||||
COORDS2INDEX(yRank, yStrideLocal.data(), yCoords, yOffset);
|
||||
}
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, static_cast<sd::LongType>(0), zLen);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void BroadcastInt<X>::exec(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo) {
|
||||
const X *x = reinterpret_cast<const X *>(vx);
|
||||
const X *y = reinterpret_cast<const X *>(vy);
|
||||
X *z = reinterpret_cast<X *>(vz);
|
||||
|
||||
const int rank = shape::rank(zShapeInfo); // xRank = yRank = zRank
|
||||
|
||||
switch (rank) {
|
||||
case 1:
|
||||
execRank1<X, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo);
|
||||
break;
|
||||
case 2:
|
||||
execRank2<X, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo);
|
||||
break;
|
||||
case 3:
|
||||
execRank3<X, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo);
|
||||
break;
|
||||
case 4:
|
||||
execRank4<X, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo);
|
||||
break;
|
||||
case 5:
|
||||
execRank5<X, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo);
|
||||
break;
|
||||
default:
|
||||
execDefault<X, OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace broadcast
|
||||
} // namespace functions
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// BroadcastBool template instantiations
|
||||
//
|
||||
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <types/type_promote.h>
|
||||
#include <loops/broadcasting_bool.h>
|
||||
#include <loops/cpu/broadcasting_bool.hpp>
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* BroadcastBool::exec instantiated for types in @COMB1@, @COMB2@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION,
|
||||
functions::broadcast::BroadcastBool,
|
||||
::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *tadOnlyShapeInfo, const sd::LongType *tadOffsets,
|
||||
const sd::LongType *tadOnlyShapeInfoZ, const sd::LongType *tadOffsetsZ,
|
||||
sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* BroadcastBool::exec (simple version) instantiated for types in @COMB1@, @COMB2@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION,
|
||||
functions::broadcast::BroadcastBool,
|
||||
::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams)
|
||||
)
|
||||
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* BroadcastBool::execInverse instantiated for types in @COMB1@, @COMB2@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION,
|
||||
functions::broadcast::BroadcastBool,
|
||||
::execInverse(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *tadOnlyShapeInfo, const sd::LongType *tadOffsets,
|
||||
const sd::LongType *tadOnlyShapeInfoZ, const sd::LongType *tadOffsetsZ,
|
||||
sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
|
||||
/*
|
||||
*
|
||||
* Explicit instantiation for bool types (since BroadcastBool typically uses bool output)
|
||||
*/
|
||||
|
||||
template class functions::broadcast::BroadcastBool<bool, bool>;
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// BroadcastBool template instantiations - Direct instantiation version
|
||||
// This file is designed for 2-type combinations
|
||||
//
|
||||
|
||||
#include <loops/cpu/broadcasting_bool.hpp>
|
||||
#include <types/types.h>
|
||||
|
||||
// Direct instantiations will be inserted here by the CMake template system
|
||||
// The handler will generate instantiations like:
|
||||
// template void functions::broadcast::BroadcastBool<float, bool>::exec(...);
|
||||
// template void functions::broadcast::BroadcastBool<double, bool>::exec(...);
|
||||
// etc.
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Broadcast template instantiations
|
||||
//
|
||||
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <types/type_promote.h>
|
||||
#include <loops/cpu/broadcasting.hpp>
|
||||
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* Broadcast::exec instantiated for types in @COMB1@, @COMB2@, @COMB3@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS_3(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
SD_COMMON_TYPES_PART_@COMB3@,
|
||||
INSTANT_PROCESS_COMBINATION_3,
|
||||
functions::broadcast::Broadcast,
|
||||
::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *tadOnlyShapeInfo, const sd::LongType *tadOffsets,
|
||||
const sd::LongType *tadOnlyShapeInfoZ, const sd::LongType *tadOffsetsZ,
|
||||
sd::LoopKind::Kind loopKind, sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* Broadcast::exec (simple version) instantiated for types in @COMB1@, @COMB2@, @COMB3@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS_3(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
SD_COMMON_TYPES_PART_@COMB3@,
|
||||
INSTANT_PROCESS_COMBINATION_3,
|
||||
functions::broadcast::Broadcast,
|
||||
::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo)
|
||||
)
|
||||
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* Broadcast::execInverse instantiated for types in @COMB1@, @COMB2@, @COMB3@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS_3(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
SD_COMMON_TYPES_PART_@COMB3@,
|
||||
INSTANT_PROCESS_COMBINATION_3,
|
||||
functions::broadcast::Broadcast,
|
||||
::execInverse(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *tadOnlyShapeInfo, const sd::LongType *tadOffsets,
|
||||
const sd::LongType *tadOnlyShapeInfoZ, const sd::LongType *tadOffsetsZ,
|
||||
sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* BroadcastBool::exec instantiated for bool types
|
||||
*/
|
||||
|
||||
template void functions::broadcast::BroadcastBool<bool, bool>::exec(
|
||||
int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *tadOnlyShapeInfo, const sd::LongType *tadOffsets,
|
||||
const sd::LongType *tadOnlyShapeInfoZ, const sd::LongType *tadOffsetsZ,
|
||||
sd::LongType start, sd::LongType stop);
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// BroadcastInt template instantiations
|
||||
//
|
||||
|
||||
#include <loops/cpu/broadcasting_int.hpp>
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <system/type_boilerplate.h>
|
||||
|
||||
/*
|
||||
* Macro for instantiating BroadcastInt for a single type
|
||||
* This follows the pattern where BroadcastInt is specialized for integer types only
|
||||
*/
|
||||
|
||||
#define INSTANTIATE_BROADCAST_INT(TYPE_TUPLE) \
|
||||
template void functions::broadcast::BroadcastInt<GET_SECOND(TYPE_TUPLE)>::exec( \
|
||||
int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y, \
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo, \
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, \
|
||||
const sd::LongType *tadOnlyShapeInfo, const sd::LongType *tadOffsets, \
|
||||
const sd::LongType *tadOnlyShapeInfoZ, const sd::LongType *tadOffsetsZ, \
|
||||
sd::LongType start, sd::LongType stop); \
|
||||
\
|
||||
template void functions::broadcast::BroadcastInt<GET_SECOND(TYPE_TUPLE)>::exec( \
|
||||
int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y, \
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo); \
|
||||
\
|
||||
template void functions::broadcast::BroadcastInt<GET_SECOND(TYPE_TUPLE)>::execInverse( \
|
||||
int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y, \
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo, \
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, \
|
||||
const sd::LongType *tadOnlyShapeInfo, const sd::LongType *tadOffsets, \
|
||||
const sd::LongType *tadOnlyShapeInfoZ, const sd::LongType *tadOffsetsZ, \
|
||||
sd::LongType start, sd::LongType stop);
|
||||
|
||||
/*
|
||||
* Function Instantiation:
|
||||
* BroadcastInt instantiated for all integer types in SD_INTEGER_TYPES_PART_@COMB1@
|
||||
*/
|
||||
ITERATE_LIST((SD_INTEGER_TYPES_PART_@COMB1@), INSTANTIATE_BROADCAST_INT)
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 12/26/24.
|
||||
//
|
||||
|
||||
#include <helpers/cpu/loops/IndexReductionLoops.hpp>
|
||||
#include <system/type_boilerplate.h>
|
||||
// Note: Instantiations are generated to prevent compiler memory issues
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
template class sd::IndexReductionLoops,
|
||||
IndexReductionLoops,
|
||||
;
|
||||
)
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
// pairwise_instantiation_template_3.cpp.in
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 11/22/24.
|
||||
//
|
||||
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <types/type_promote.h>
|
||||
#include <loops/cpu/pairwise.hpp>
|
||||
// Note: Instantiations are generated to prevent compiler memory issues
|
||||
|
||||
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
CALLBACK_INSTANTIATE_PROMOTE,
|
||||
promote,
|
||||
;
|
||||
)
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 11/22/24.
|
||||
//
|
||||
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <types/type_promote.h>
|
||||
#include <loops/cpu/pairwise.hpp>
|
||||
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* PairWiseTransform::exec instantiated for types in @COMB1@, @COMB2@, @COMB3@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS_3(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
SD_COMMON_TYPES_PART_@COMB3@,
|
||||
INSTANT_PROCESS_COMBINATION_3,
|
||||
functions::pairwise_transforms::PairWiseTransform,
|
||||
::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 1/3/25.
|
||||
//
|
||||
|
||||
#include <loops/cpu/random.hpp>
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <system/type_boilerplate.h>
|
||||
|
||||
/*
|
||||
* Function Instantiation:
|
||||
* RandomFunction instantiated for float types in @COMB1@
|
||||
*/
|
||||
|
||||
#define LIST_CALLBACK_EXEC_TRANSFORM_1(INPUT) \
|
||||
template void functions::random::RandomFunction<GET_SECOND(INPUT)>::execTransform( \
|
||||
int opNum, \
|
||||
sd::Pointer state, \
|
||||
const void *x, \
|
||||
const sd::LongType *xShapeInfo, \
|
||||
void *z, \
|
||||
const sd::LongType *zShapeInfo, \
|
||||
void *extraArguments);
|
||||
|
||||
ITERATE_LIST((SD_COMMON_TYPES_PART_@COMB1@), LIST_CALLBACK_EXEC_TRANSFORM_1)
|
||||
|
||||
#define LIST_CALLBACK_EXEC_TRANSFORM_2(INPUT) \
|
||||
template void functions::random::RandomFunction<GET_SECOND(INPUT)>::execTransform( \
|
||||
int opNum, \
|
||||
sd::Pointer state, \
|
||||
const void *x, \
|
||||
const sd::LongType *xShapeInfo, \
|
||||
const void *y, \
|
||||
const sd::LongType *yShapeInfo, \
|
||||
void *z, \
|
||||
const sd::LongType *zShapeInfo, \
|
||||
void *extraArguments);
|
||||
|
||||
ITERATE_LIST((SD_COMMON_TYPES_PART_@COMB1@), LIST_CALLBACK_EXEC_TRANSFORM_2)
|
||||
|
||||
#define LIST_CALLBACK_EXEC_TRANSFORM_3(INPUT) \
|
||||
template void functions::random::RandomFunction<GET_SECOND(INPUT)>::execTransform( \
|
||||
int opNum, \
|
||||
sd::Pointer state, \
|
||||
void *z, \
|
||||
const sd::LongType *zShapeInfo, \
|
||||
void *extraArguments);
|
||||
|
||||
ITERATE_LIST((SD_COMMON_TYPES_PART_@COMB1@), LIST_CALLBACK_EXEC_TRANSFORM_3)
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 1/3/25.
|
||||
//
|
||||
|
||||
#include <loops/cpu/reduce3.hpp>
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <system/type_boilerplate.h>
|
||||
|
||||
/*
|
||||
* Function Instantiation:
|
||||
* Reduce3 instantiated for types @COMB1@, @COMB2@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_FLOAT_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce3::Reduce3,
|
||||
::execScalar(const int opNum, const void *vx, const sd::LongType *xShapeInfo, void *extraParamsVals, const void *vy, const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_FLOAT_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce3::Reduce3,
|
||||
::exec(int opNum, const void *vx, const sd::LongType *xShapeInfo, void *extraParamsVals, const void *vy, const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo, sd::LongType *dimension, sd::LongType dimensionLength, sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_FLOAT_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce3::Reduce3,
|
||||
::exec(int opNum, const void *vx, const sd::LongType *xShapeInfo, void *extraParamsVals, const void *vy, const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo, sd::LongType *dimension, sd::LongType dimensionLength, const sd::LongType *tadShapeInfo, const sd::LongType *tadOffsets, sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_FLOAT_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce3::Reduce3,
|
||||
::execAll(int opNum, const void *vx, const sd::LongType *xShapeInfo, void *extraParamsVals, const void *vy, const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo, sd::LongType *dimension, sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo, const sd::LongType *xOffsets, const sd::LongType *yTadShapeInfo, const sd::LongType *yOffsets, sd::LongType start, sd::LongType stop)
|
||||
)
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 1/3/25.
|
||||
//
|
||||
|
||||
#include <loops/cpu/reduce/reduce_bool.hpp>
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <system/type_boilerplate.h>
|
||||
|
||||
/*
|
||||
* Function Instantiation:
|
||||
* ReduceBoolFunction instantiated for types @COMB1@, @COMB2@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_BOOL_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceBoolFunction,
|
||||
::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_BOOL_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceBoolFunction,
|
||||
::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams, void *z, const sd::LongType *zShapeInfo)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_BOOL_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceBoolFunction,
|
||||
::exec(int opNum, sd::memory::Workspace *workspace, const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, void *vz, const sd::LongType *zShapeInfo, const sd::LongType *dims)
|
||||
)
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 1/3/25.
|
||||
//
|
||||
|
||||
#include <loops/cpu/reduce/reduce_float.hpp>
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <system/type_boilerplate.h>
|
||||
|
||||
/*
|
||||
* Function Instantiation:
|
||||
* ReduceFloatFunction instantiated for types @COMB1@, @COMB2@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_NUMERIC_TYPES_PART_@COMB1@,
|
||||
SD_FLOAT_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceFloatFunction,
|
||||
::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_NUMERIC_TYPES_PART_@COMB1@,
|
||||
SD_FLOAT_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceFloatFunction,
|
||||
::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams, void *z, const sd::LongType *zShapeInfo)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_NUMERIC_TYPES_PART_@COMB1@,
|
||||
SD_FLOAT_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceFloatFunction,
|
||||
::exec(int opNum, sd::memory::Workspace *workspace, const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, void *vz, const sd::LongType *zShapeInfo, const sd::LongType *dims)
|
||||
)
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 1/3/25.
|
||||
//
|
||||
|
||||
#include <loops/cpu/reduce/reduce_long.hpp>
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <system/type_boilerplate.h>
|
||||
|
||||
/*
|
||||
* Function Instantiation:
|
||||
* ReduceLongFunction instantiated for types @COMB1@, @COMB2@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_LONG_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceLongFunction,
|
||||
::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_LONG_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceLongFunction,
|
||||
::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams, void *z, const sd::LongType *zShapeInfo)
|
||||
)
|
||||
|
||||
ITERATE_COMBINATIONS(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_LONG_TYPES_PART_@COMB2@,
|
||||
INSTANT_PROCESS_COMBINATION_2,
|
||||
functions::reduce::ReduceLongFunction,
|
||||
::exec(const int opNum, sd::memory::Workspace *workspace, const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, void *vz, const sd::LongType *zShapeInfo, sd::LongType *dims)
|
||||
)
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 1/3/25.
|
||||
//
|
||||
|
||||
#include <loops/cpu/reduce/reduce_same.hpp>
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <system/type_boilerplate.h>
|
||||
|
||||
/*
|
||||
* Function Instantiation:
|
||||
* ReduceSameFunction instantiated for common types in @COMB1@
|
||||
*/
|
||||
|
||||
#define LIST_CALLBACK_EXEC_SCALAR_1(INPUT) \
|
||||
template void* functions::reduce::ReduceSameFunction<GET_SECOND(INPUT)>::execScalar( \
|
||||
const int opNum, \
|
||||
const void *x, \
|
||||
const sd::LongType *xShapeInfo, \
|
||||
void *extraParams);
|
||||
|
||||
ITERATE_LIST((SD_COMMON_TYPES_PART_@COMB1@), LIST_CALLBACK_EXEC_SCALAR_1)
|
||||
|
||||
#define LIST_CALLBACK_EXEC_SCALAR_2(INPUT) \
|
||||
template void functions::reduce::ReduceSameFunction<GET_SECOND(INPUT)>::execScalar( \
|
||||
const int opNum, \
|
||||
const void *x, \
|
||||
const sd::LongType *xShapeInfo, \
|
||||
void *extraParams, \
|
||||
void *z, \
|
||||
const sd::LongType *zShapeInfo);
|
||||
|
||||
ITERATE_LIST((SD_COMMON_TYPES_PART_@COMB1@), LIST_CALLBACK_EXEC_SCALAR_2)
|
||||
|
||||
#define LIST_CALLBACK_EXEC(INPUT) \
|
||||
template void functions::reduce::ReduceSameFunction<GET_SECOND(INPUT)>::exec( \
|
||||
int opNum, \
|
||||
sd::memory::Workspace *workspace, \
|
||||
const void *vx, \
|
||||
const sd::LongType *xShapeInfo, \
|
||||
void *vextraParams, \
|
||||
void *vz, \
|
||||
const sd::LongType *zShapeInfo, \
|
||||
const sd::LongType *dims);
|
||||
|
||||
ITERATE_LIST((SD_COMMON_TYPES_PART_@COMB1@), LIST_CALLBACK_EXEC)
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by agibsonccc on 12/30/24.
|
||||
//
|
||||
|
||||
#include <loops/pairwise_instantiations.h>
|
||||
#include <types/type_promote.h>
|
||||
#include <loops/cpu/scalar.hpp>
|
||||
|
||||
/*
|
||||
*
|
||||
* Function Instantiation:
|
||||
* ScalarTransform::transform instantiated for types in @COMB1@, @COMB2@, @COMB3@
|
||||
*/
|
||||
|
||||
ITERATE_COMBINATIONS_3(
|
||||
SD_COMMON_TYPES_PART_@COMB1@,
|
||||
SD_COMMON_TYPES_PART_@COMB2@,
|
||||
SD_COMMON_TYPES_PART_@COMB3@,
|
||||
INSTANT_PROCESS_COMBINATION_3,
|
||||
functions::scalar::ScalarTransform,
|
||||
::transform(int opNum, const void *x, const sd::LongType *xShapeInfo, void *result,
|
||||
const sd::LongType *resultShapeInfo, const void *scalar, void *extraParams, sd::LongType start,
|
||||
sd::LongType stop)
|
||||
)
|
||||
@@ -0,0 +1,30 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 23/09/18.
|
||||
//
|
||||
#include <loops/cpu/broadcasting_bool.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace functions {
|
||||
namespace broadcast {
|
||||
BUILD_DOUBLE_TEMPLATE(template class SD_LIB_HIDDEN BroadcastBool, , SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_BOOL_TYPES);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,30 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 23/09/18.
|
||||
//
|
||||
#include <loops/cpu/broadcasting_int.hpp>
|
||||
#cmakedefine SD_INTEGER_TYPES_GEN
|
||||
#if defined(SD_INTEGER_TYPES_GEN) && defined(SD_INTEGER_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace functions {
|
||||
namespace broadcast {
|
||||
BUILD_SINGLE_TEMPLATE(template class SD_LIB_HIDDEN BroadcastInt, , SD_INTEGER_TYPES_@FL_TYPE_INDEX@);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,30 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 23/09/18.
|
||||
//
|
||||
#include <loops/cpu/broadcasting.hpp>
|
||||
#cmakedefine SD_PAIRWISE_TYPES_GEN
|
||||
#if defined(SD_PAIRWISE_TYPES_GEN) && defined(SD_PAIRWISE_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace functions {
|
||||
namespace broadcast {
|
||||
BUILD_PAIRWISE_TEMPLATE(template class SD_LIB_HIDDEN Broadcast, , SD_PAIRWISE_TYPES_@FL_TYPE_INDEX@);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/indexreduce.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@) && defined(SD_INDEXING_TYPES_0)
|
||||
namespace functions {
|
||||
namespace indexreduce {
|
||||
BUILD_DOUBLE_TEMPLATE(template class SD_LIB_HIDDEN IndexReduce, , SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_INDEXING_TYPES_0);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/indexreduce.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@) && defined(SD_INDEXING_TYPES_1)
|
||||
namespace functions {
|
||||
namespace indexreduce {
|
||||
BUILD_DOUBLE_TEMPLATE(template class SD_LIB_HIDDEN IndexReduce, , SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_INDEXING_TYPES_1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,30 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 23/09/18.
|
||||
//
|
||||
#include <loops/cpu/pairwise.hpp>
|
||||
#cmakedefine SD_PAIRWISE_TYPES_GEN
|
||||
#if defined(SD_PAIRWISE_TYPES_GEN) && defined(SD_PAIRWISE_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace functions {
|
||||
namespace pairwise_transforms {
|
||||
BUILD_PAIRWISE_TEMPLATE(template class SD_LIB_HIDDEN PairWiseTransform, , SD_PAIRWISE_TYPES_@FL_TYPE_INDEX@);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,37 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/random.hpp>
|
||||
#cmakedefine SD_FLOAT_TYPES_GEN
|
||||
#if defined(SD_FLOAT_TYPES_GEN) && defined(SD_FLOAT_TYPES_@FL_TYPE_INDEX@)
|
||||
|
||||
// Helper macros for stringification
|
||||
#define STR_HELPER(x) #x
|
||||
#define STR(x) STR_HELPER(x)
|
||||
|
||||
// Emit the value of SD_PAIRWISE_TYPES_0 during compilation
|
||||
#pragma message("SD_PAIRWISE_TYPES_0: " STR((SD_PAIRWISE_TYPES_0)))
|
||||
namespace functions {
|
||||
namespace random {
|
||||
BUILD_SINGLE_TEMPLATE(template class SD_LIB_HIDDEN RandomFunction, , SD_FLOAT_TYPES_@FL_TYPE_INDEX@);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/reduce3.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@) && defined(SD_FLOAT_TYPES_0)
|
||||
namespace functions {
|
||||
namespace reduce3 {
|
||||
BUILD_DOUBLE_TEMPLATE(template class SD_LIB_HIDDEN Reduce3, , SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_FLOAT_TYPES_0);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/reduce3.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@) && defined(SD_FLOAT_TYPES_1)
|
||||
namespace functions {
|
||||
namespace reduce3 {
|
||||
BUILD_DOUBLE_TEMPLATE(template class SD_LIB_HIDDEN Reduce3, , SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_FLOAT_TYPES_1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/reduce3.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@) && defined(SD_FLOAT_TYPES_3)
|
||||
namespace functions {
|
||||
namespace reduce3 {
|
||||
BUILD_DOUBLE_TEMPLATE(template class SD_LIB_HIDDEN Reduce3, , SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_FLOAT_TYPES_3);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/reduce3.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@) && defined(SD_FLOAT_TYPES_2)
|
||||
namespace functions {
|
||||
namespace reduce3 {
|
||||
BUILD_DOUBLE_TEMPLATE(template class SD_LIB_HIDDEN Reduce3, , SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_FLOAT_TYPES_2);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/reduce/reduce_bool.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace functions {
|
||||
namespace reduce {
|
||||
BUILD_DOUBLE_TEMPLATE(template class ReduceBoolFunction, , SD_COMMON_TYPES_@FL_TYPE_INDEX@ , SD_BOOL_TYPES);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/reduce/reduce_float.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@) && COUNT_NARG(SD_FLOAT_TYPES) > 0
|
||||
namespace functions {
|
||||
namespace reduce {
|
||||
BUILD_DOUBLE_TEMPLATE(template class SD_LIB_HIDDEN ReduceFloatFunction, , SD_COMMON_TYPES_@FL_TYPE_INDEX@ , SD_FLOAT_TYPES);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/reduce/reduce_long.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@) && COUNT_NARG(SD_LONG_TYPES) > 0
|
||||
namespace functions {
|
||||
namespace reduce {
|
||||
BUILD_DOUBLE_TEMPLATE(template class ReduceLongFunction, , SD_COMMON_TYPES_@FL_TYPE_INDEX@ , SD_LONG_TYPES);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <loops/cpu/reduce/reduce_same.hpp>
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace functions {
|
||||
namespace reduce {
|
||||
BUILD_SINGLE_TEMPLATE(template class ReduceSameFunction, , SD_COMMON_TYPES_@FL_TYPE_INDEX@ );
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver on 9/28/2018.
|
||||
//
|
||||
#include <loops/cpu/scalar.hpp>
|
||||
#cmakedefine SD_PAIRWISE_TYPES_GEN
|
||||
#if defined(SD_PAIRWISE_TYPES_GEN) && defined(SD_PAIRWISE_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace functions {
|
||||
namespace scalar {
|
||||
BUILD_PAIRWISE_TEMPLATE(template class SD_LIB_HIDDEN ScalarTransform, , SD_PAIRWISE_TYPES_@FL_TYPE_INDEX@);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,141 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver on 4/9/2018.
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/Loops.h>
|
||||
#include <loops/indexreduce.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace indexreduce {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
sd::LongType IndexReduce<X, Z>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *extraParams) {
|
||||
RETURNING_DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(x, xShapeInfo, extraParams), INDEX_REDUCE_OPS);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
void IndexReduce<X, Z>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams, void *z,
|
||||
const sd::LongType *zShapeInfo, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *tadShapeInfo, const sd::LongType *tadOffset) {
|
||||
DISPATCH_BY_OPNUM_TT(
|
||||
exec, PARAMS(x, xShapeInfo, extraParams, z, zShapeInfo, dimension, dimensionLength, tadShapeInfo, tadOffset),
|
||||
INDEX_REDUCE_OPS);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
sd::LongType IndexReduce<X, Z>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
auto startingIndex = OpType::startingIndexValue(x);
|
||||
auto len = shape::length(xShapeInfo);
|
||||
sd::OmpLaunchHelper info(len);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
|
||||
sd::LongType xShapeInfoCast[SD_MAX_RANK];
|
||||
bool canCastX = sd::DataTypeUtils::castShapeInfo(xShapeInfo, xShapeInfoCast);
|
||||
int maxThreads = sd::math::sd_min<int>(64, sd::Environment::getInstance().maxThreads());
|
||||
IndexValue<X> intermediatery[64];
|
||||
for (int e = 0; e < maxThreads; e++) intermediatery[e].index = -1;
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
intermediatery[thread_id] = OpType::startingIndexValue(x);
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
IndexValue<X> curr(x[offset], i);
|
||||
intermediatery[thread_id] = OpType::update(intermediatery[thread_id], curr, extraParams);
|
||||
}
|
||||
};
|
||||
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, len, 1, maxThreads);
|
||||
|
||||
for (int e = 0; e < maxThreads; e++) startingIndex = OpType::update(startingIndex, intermediatery[e], extraParams);
|
||||
|
||||
return startingIndex.index;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void IndexReduce<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, sd::LongType *dimension, sd::LongType dimensionLength,
|
||||
const sd::LongType *tadShapeInfo, const sd::LongType *tadOffset) {
|
||||
auto x = reinterpret_cast<X *>(const_cast<void *>(vx));
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
const sd::LongType zLen = shape::length(zShapeInfo);
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
if (sd::ArrayOptions::arrayType(zShapeInfo) == sd::ArrayType::EMPTY) return;
|
||||
const auto indexValue = OpType::startingIndexValue(x);
|
||||
|
||||
for (sd::LongType i = 0; i < zLen; i++) z[i] = (Z)indexValue.index;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (shape::isScalar(zShapeInfo)) {
|
||||
z[0] = (Z)execScalar<OpType>(x, xShapeInfo, extraParams);
|
||||
return;
|
||||
}
|
||||
|
||||
auto tadOnlyShapeInfo = tadShapeInfo;
|
||||
auto tadOffsets = tadOffset;
|
||||
|
||||
// When shared_ptr goes out of scope, it deletes the TadPack and invalidates pointers!
|
||||
std::shared_ptr<sd::TadPack> tadPack = nullptr;
|
||||
|
||||
if (tadOnlyShapeInfo == nullptr || tadOffsets == nullptr) {
|
||||
if (dimensionLength < 1) return;
|
||||
|
||||
tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(const_cast<sd::LongType*>(xShapeInfo), dimension,
|
||||
dimensionLength);
|
||||
tadOnlyShapeInfo = tadPack->primaryShapeInfo();
|
||||
tadOffsets = tadPack->primaryOffsets();
|
||||
}
|
||||
|
||||
// Let IndexReductionLoops handle the shape caching internally since it's a separate component
|
||||
sd::IndexReductionLoops<X, Z>::template loopIndexReduce<OpType>(x, xShapeInfo, z, zShapeInfo, tadOnlyShapeInfo,
|
||||
tadOffsets, vextraParams);
|
||||
}
|
||||
|
||||
} // namespace indexreduce
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,599 @@
|
||||
#include <helpers/cpu/loops/IndexReductionLoops.hpp>
|
||||
#include <types/types.h>
|
||||
#include <system/type_boilerplate.h>
|
||||
#include "indexreduce.hpp"
|
||||
|
||||
// Macro to conditionally instantiate based on HAS_* flags
|
||||
#define CONDITIONAL_INSTANTIATE(TYPE_TUPLE, MACRO_CALL) \
|
||||
CONDITIONAL_INSTANTIATE_IMPL(GET_SECOND(TYPE_TUPLE), MACRO_CALL)
|
||||
|
||||
#define CONDITIONAL_INSTANTIATE_IMPL(TYPE, MACRO_CALL) \
|
||||
CONDITIONAL_INSTANTIATE_##TYPE(MACRO_CALL)
|
||||
|
||||
// Define conditional instantiation for each type
|
||||
#ifdef HAS_FLOAT32
|
||||
#define CONDITIONAL_INSTANTIATE_float(MACRO_CALL) MACRO_CALL((FLOAT32, float))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_float(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_DOUBLE
|
||||
#define CONDITIONAL_INSTANTIATE_double(MACRO_CALL) MACRO_CALL((DOUBLE, double))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_double(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_FLOAT16
|
||||
#define CONDITIONAL_INSTANTIATE_float16(MACRO_CALL) MACRO_CALL((HALF, float16))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_float16(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_BFLOAT16
|
||||
#define CONDITIONAL_INSTANTIATE_bfloat16(MACRO_CALL) MACRO_CALL((BFLOAT16, bfloat16))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_bfloat16(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_INT8
|
||||
#define CONDITIONAL_INSTANTIATE_int8_t(MACRO_CALL) MACRO_CALL((INT8, int8_t))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_int8_t(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_INT16
|
||||
#define CONDITIONAL_INSTANTIATE_int16_t(MACRO_CALL) MACRO_CALL((INT16, int16_t))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_int16_t(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_INT32
|
||||
#define CONDITIONAL_INSTANTIATE_Int32Type(MACRO_CALL) MACRO_CALL((INT32, Int32Type))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_Int32Type(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_LONG
|
||||
#define CONDITIONAL_INSTANTIATE_LongType(MACRO_CALL) MACRO_CALL((INT64, LongType))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_LongType(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_UINT8
|
||||
#define CONDITIONAL_INSTANTIATE_uint8_t(MACRO_CALL) MACRO_CALL((UINT8, uint8_t))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_uint8_t(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_UINT16
|
||||
#define CONDITIONAL_INSTANTIATE_uint16_t(MACRO_CALL) MACRO_CALL((UINT16, uint16_t))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_uint16_t(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_UINT32
|
||||
#define CONDITIONAL_INSTANTIATE_uint32_t(MACRO_CALL) MACRO_CALL((UINT32, uint32_t))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_uint32_t(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
#define CONDITIONAL_INSTANTIATE_unsignedlong(MACRO_CALL) MACRO_CALL((UINT64, unsigned long long))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_unsignedlong(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
#ifdef HAS_BOOL
|
||||
#define CONDITIONAL_INSTANTIATE_bool(MACRO_CALL) MACRO_CALL((BOOL, bool))
|
||||
#else
|
||||
#define CONDITIONAL_INSTANTIATE_bool(MACRO_CALL)
|
||||
#endif
|
||||
|
||||
// Now instantiate the operations with int32_t index type (only if HAS_INT32 is defined)
|
||||
#ifdef HAS_INT32
|
||||
|
||||
#define LIST_CALLBACK_OPS_int32_t(INPUT) \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), int32_t>::loopIndexReduce<simdOps::IndexMin<GET_SECOND(INPUT), int32_t>>(GET_SECOND(INPUT)*, const sd::LongType*, int32_t*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), int32_t>::loopIndexReduce<simdOps::IndexAbsoluteMin<GET_SECOND(INPUT), int32_t>>(GET_SECOND(INPUT)*, const sd::LongType*, int32_t*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), int32_t>::loopIndexReduce<simdOps::LastIndex<GET_SECOND(INPUT), int32_t>>(GET_SECOND(INPUT)*, const sd::LongType*, int32_t*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), int32_t>::loopIndexReduce<simdOps::FirstIndex<GET_SECOND(INPUT), int32_t>>(GET_SECOND(INPUT)*, const sd::LongType*, int32_t*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), int32_t>::loopIndexReduce<simdOps::IndexAbsoluteMax<GET_SECOND(INPUT), int32_t>>(GET_SECOND(INPUT)*, const sd::LongType*, int32_t*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), int32_t>::loopIndexReduce<simdOps::IndexMax<GET_SECOND(INPUT), int32_t>>(GET_SECOND(INPUT)*, const sd::LongType*, int32_t*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*);
|
||||
|
||||
// Conditionally instantiate for each type
|
||||
#ifdef HAS_FLOAT32
|
||||
LIST_CALLBACK_OPS_int32_t((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
LIST_CALLBACK_OPS_int32_t((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
LIST_CALLBACK_OPS_int32_t((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
LIST_CALLBACK_OPS_int32_t((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
LIST_CALLBACK_OPS_int32_t((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
LIST_CALLBACK_OPS_int32_t((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
LIST_CALLBACK_OPS_int32_t((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
LIST_CALLBACK_OPS_int32_t((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
LIST_CALLBACK_OPS_int32_t((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
LIST_CALLBACK_OPS_int32_t((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
LIST_CALLBACK_OPS_int32_t((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
LIST_CALLBACK_OPS_int32_t((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
LIST_CALLBACK_OPS_int32_t((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_INT32
|
||||
|
||||
// And with long long index type (only if HAS_LONG is defined)
|
||||
#ifdef HAS_LONG
|
||||
|
||||
#define LIST_CALLBACK_OPS_LONG(INPUT) \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), sd::LongType>::loopIndexReduce<simdOps::IndexMin<GET_SECOND(INPUT), sd::LongType>>(GET_SECOND(INPUT)*, const sd::LongType*, sd::LongType*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), sd::LongType>::loopIndexReduce<simdOps::IndexAbsoluteMin<GET_SECOND(INPUT), sd::LongType>>(GET_SECOND(INPUT)*, const sd::LongType*, sd::LongType*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), sd::LongType>::loopIndexReduce<simdOps::LastIndex<GET_SECOND(INPUT), sd::LongType>>(GET_SECOND(INPUT)*, const sd::LongType*, sd::LongType*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), sd::LongType>::loopIndexReduce<simdOps::FirstIndex<GET_SECOND(INPUT), sd::LongType>>(GET_SECOND(INPUT)*, const sd::LongType*, sd::LongType*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), sd::LongType>::loopIndexReduce<simdOps::IndexAbsoluteMax<GET_SECOND(INPUT), sd::LongType>>(GET_SECOND(INPUT)*, const sd::LongType*, sd::LongType*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*); \
|
||||
template void sd::IndexReductionLoops<GET_SECOND(INPUT), sd::LongType>::loopIndexReduce<simdOps::IndexMax<GET_SECOND(INPUT), sd::LongType>>(GET_SECOND(INPUT)*, const sd::LongType*, sd::LongType*, const sd::LongType*, const sd::LongType*, const sd::LongType*, void*);
|
||||
|
||||
// Conditionally instantiate for each type
|
||||
#ifdef HAS_FLOAT32
|
||||
LIST_CALLBACK_OPS_LONG((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
LIST_CALLBACK_OPS_LONG((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
LIST_CALLBACK_OPS_LONG((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
LIST_CALLBACK_OPS_LONG((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
LIST_CALLBACK_OPS_LONG((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
LIST_CALLBACK_OPS_LONG((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
LIST_CALLBACK_OPS_LONG((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
LIST_CALLBACK_OPS_LONG((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
LIST_CALLBACK_OPS_LONG((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
LIST_CALLBACK_OPS_LONG((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
LIST_CALLBACK_OPS_LONG((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
LIST_CALLBACK_OPS_LONG((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
LIST_CALLBACK_OPS_LONG((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_LONG
|
||||
|
||||
// Instantiate execScalar methods with int32_t
|
||||
#ifdef HAS_INT32
|
||||
|
||||
#define INSTANTIATE_EXEC_SCALAR_INT32(INPUT) \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::execScalar<simdOps::IndexMin<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::execScalar<simdOps::IndexAbsoluteMin<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::execScalar<simdOps::LastIndex<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::execScalar<simdOps::FirstIndex<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::execScalar<simdOps::IndexAbsoluteMax<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::execScalar<simdOps::IndexMax<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*);
|
||||
|
||||
#ifdef HAS_FLOAT32
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
INSTANTIATE_EXEC_SCALAR_INT32((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_INT32
|
||||
|
||||
// Instantiate execScalar methods with sd::LongType
|
||||
#ifdef HAS_LONG
|
||||
|
||||
#define INSTANTIATE_EXEC_SCALAR_LONG(INPUT) \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::execScalar<simdOps::IndexMin<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::execScalar<simdOps::IndexAbsoluteMin<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::execScalar<simdOps::LastIndex<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::execScalar<simdOps::FirstIndex<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::execScalar<simdOps::IndexAbsoluteMax<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*); \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::execScalar<simdOps::IndexMax<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*);
|
||||
|
||||
#ifdef HAS_FLOAT32
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
INSTANTIATE_EXEC_SCALAR_LONG((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_LONG
|
||||
|
||||
// Instantiate exec methods with int32_t
|
||||
#ifdef HAS_INT32
|
||||
|
||||
#define INSTANTIATE_EXEC_INT32(INPUT) \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::exec<simdOps::IndexMin<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::exec<simdOps::IndexAbsoluteMin<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::exec<simdOps::LastIndex<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::exec<simdOps::FirstIndex<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::exec<simdOps::IndexAbsoluteMax<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::exec<simdOps::IndexMax<GET_SECOND(INPUT), int32_t>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*);
|
||||
|
||||
#ifdef HAS_FLOAT32
|
||||
INSTANTIATE_EXEC_INT32((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
INSTANTIATE_EXEC_INT32((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
INSTANTIATE_EXEC_INT32((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
INSTANTIATE_EXEC_INT32((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
INSTANTIATE_EXEC_INT32((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
INSTANTIATE_EXEC_INT32((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
INSTANTIATE_EXEC_INT32((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
INSTANTIATE_EXEC_INT32((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
INSTANTIATE_EXEC_INT32((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
INSTANTIATE_EXEC_INT32((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
INSTANTIATE_EXEC_INT32((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
INSTANTIATE_EXEC_INT32((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
INSTANTIATE_EXEC_INT32((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_INT32
|
||||
|
||||
// Instantiate exec methods with sd::LongType
|
||||
#ifdef HAS_LONG
|
||||
|
||||
#define INSTANTIATE_EXEC_LONG(INPUT) \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::exec<simdOps::IndexMin<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::exec<simdOps::IndexAbsoluteMin<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::exec<simdOps::LastIndex<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::exec<simdOps::FirstIndex<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::exec<simdOps::IndexAbsoluteMax<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*); \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::exec<simdOps::IndexMax<GET_SECOND(INPUT), sd::LongType>>(const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*);
|
||||
|
||||
#ifdef HAS_FLOAT32
|
||||
INSTANTIATE_EXEC_LONG((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
INSTANTIATE_EXEC_LONG((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
INSTANTIATE_EXEC_LONG((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
INSTANTIATE_EXEC_LONG((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
INSTANTIATE_EXEC_LONG((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
INSTANTIATE_EXEC_LONG((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
INSTANTIATE_EXEC_LONG((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
INSTANTIATE_EXEC_LONG((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
INSTANTIATE_EXEC_LONG((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
INSTANTIATE_EXEC_LONG((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
INSTANTIATE_EXEC_LONG((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
INSTANTIATE_EXEC_LONG((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
INSTANTIATE_EXEC_LONG((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_LONG
|
||||
|
||||
// Add these instantiations for the non-templated methods (the ones with int opNum parameter)
|
||||
#ifdef HAS_INT32
|
||||
|
||||
#define INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32(INPUT) \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::execScalar(int, const void*, const sd::LongType*, void*);
|
||||
|
||||
#ifdef HAS_FLOAT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_INT32((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_INT32
|
||||
|
||||
#ifdef HAS_LONG
|
||||
|
||||
#define INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG(INPUT) \
|
||||
template sd::LongType functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::execScalar(int, const void*, const sd::LongType*, void*);
|
||||
|
||||
#ifdef HAS_FLOAT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_SCALAR_LONG((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_LONG
|
||||
|
||||
#ifdef HAS_INT32
|
||||
|
||||
#define INSTANTIATE_NON_TEMPLATED_EXEC_INT32(INPUT) \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), int32_t>::exec(int, const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*);
|
||||
|
||||
#ifdef HAS_FLOAT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_INT32((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_INT32
|
||||
|
||||
#ifdef HAS_LONG
|
||||
|
||||
#define INSTANTIATE_NON_TEMPLATED_EXEC_LONG(INPUT) \
|
||||
template void functions::indexreduce::IndexReduce<GET_SECOND(INPUT), sd::LongType>::exec(int, const void*, const sd::LongType*, void*, void*, const sd::LongType*, sd::LongType*, sd::LongType, const sd::LongType*, const sd::LongType*);
|
||||
|
||||
#ifdef HAS_FLOAT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((FLOAT32, float))
|
||||
#endif
|
||||
#ifdef HAS_DOUBLE
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((DOUBLE, double))
|
||||
#endif
|
||||
#ifdef HAS_FLOAT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((HALF, float16))
|
||||
#endif
|
||||
#ifdef HAS_BFLOAT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((BFLOAT16, bfloat16))
|
||||
#endif
|
||||
#ifdef HAS_INT8
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((INT8, int8_t))
|
||||
#endif
|
||||
#ifdef HAS_INT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((INT16, int16_t))
|
||||
#endif
|
||||
#ifdef HAS_INT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((INT32, Int32Type))
|
||||
#endif
|
||||
#ifdef HAS_LONG
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((INT64, LongType))
|
||||
#endif
|
||||
#ifdef HAS_UINT8
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((UINT8, uint8_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT16
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((UINT16, uint16_t))
|
||||
#endif
|
||||
#ifdef HAS_UINT32
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((UINT32, uint32_t))
|
||||
#endif
|
||||
#ifdef HAS_UNSIGNEDLONG
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((UINT64, uint64_t))
|
||||
#endif
|
||||
#ifdef HAS_BOOL
|
||||
INSTANTIATE_NON_TEMPLATED_EXEC_LONG((BOOL, bool))
|
||||
#endif
|
||||
|
||||
#endif // HAS_LONG
|
||||
@@ -0,0 +1,172 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by remote on 2018-09-20.
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <helpers/shape.h>
|
||||
#include <loops/pairwise_transform.h>
|
||||
#include <math/templatemath.h>
|
||||
#include <ops/ops.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
#include <array/ArrayOptions.hXX>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace pairwise_transforms {
|
||||
|
||||
template <typename X, typename Y, typename Z>
|
||||
SD_INLINE void PairWiseTransform<X, Y, Z>::exec(int opNum,
|
||||
const void *x,
|
||||
sd::LongType xEws,
|
||||
const void *y,
|
||||
sd::LongType yEws,
|
||||
void *z,
|
||||
sd::LongType zEws,
|
||||
void *extraParams,
|
||||
sd::LongType n,
|
||||
sd::LongType start,
|
||||
sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TTT(exec, PARAMS(x, xEws, y, yEws, z, zEws, extraParams, n, start, stop), PAIRWISE_TRANSFORM_OPS);
|
||||
};
|
||||
|
||||
template <typename X, typename Y, typename Z>
|
||||
template <typename OpType>
|
||||
SD_INLINE void PairWiseTransform<X, Y, Z>::exec(const void *vx, sd::LongType xEws, const void *vy, sd::LongType yEws, void *vz,
|
||||
sd::LongType zEws, void *vextraParams, sd::LongType n, sd::LongType start,
|
||||
sd::LongType stop) {
|
||||
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const Y *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
|
||||
|
||||
if (xEws == 1 && yEws == 1 && zEws == 1) {
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
z[i] = OpType::op(x[i], y[i], extraParams);
|
||||
}
|
||||
|
||||
} else {
|
||||
for (sd::LongType i = start; i < stop; i++) z[i * zEws] = OpType::op(x[i * xEws], y[i * yEws], extraParams);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
template <typename X, typename Y, typename Z>
|
||||
SD_INLINE void PairWiseTransform<X, Y, Z>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TTT(exec, PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams, start, stop),
|
||||
PAIRWISE_TRANSFORM_OPS);
|
||||
};
|
||||
|
||||
|
||||
template <typename X, typename Y, typename Z>
|
||||
template <typename OpType>
|
||||
SD_INLINE void PairWiseTransform<X, Y, Z>::exec(const void *vx,
|
||||
const sd::LongType *xShapeInfo,
|
||||
const void *vy,
|
||||
const sd::LongType *yShapeInfo,
|
||||
void *vz,
|
||||
const sd::LongType *zShapeInfo,
|
||||
void *vextraParams,
|
||||
sd::LongType start,
|
||||
sd::LongType stop) {
|
||||
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const Y *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *yShape = shape::shapeOf(yShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *yStride = shape::stride(yShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
bool allSameOrder = shape::order(xShapeInfo) == shape::order(yShapeInfo) && shape::order(xShapeInfo) == shape::order(zShapeInfo);
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, yShapeInfo)
|
||||
&& shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)
|
||||
&& !shape::isViewConst(xShapeInfo)
|
||||
&& !shape::isViewConst(yShapeInfo) && !shape::isViewConst(zShapeInfo)
|
||||
&& allSameOrder) {
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, zRank,zShape, zCoords);
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(zRank, zStride, zCoords, zOffset);
|
||||
z[zOffset] = OpType::op(x[zOffset], y[zOffset], extraParams);
|
||||
}
|
||||
} else if ((shape::haveSameShapeAndStrides(xShapeInfo, yShapeInfo)
|
||||
|| shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo))
|
||||
&& allSameOrder) {
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, zRank, zShape, zCoords);
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, zCoords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, zCoords, yOffset);
|
||||
COORDS2INDEX(zRank, zStride, zCoords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset], extraParams);
|
||||
}
|
||||
} else if (shape::haveSameShapeAndStrides(yShapeInfo, zShapeInfo)
|
||||
&& !shape::isViewConst(xShapeInfo)
|
||||
&& !shape::isViewConst(yShapeInfo) && !shape::isViewConst(zShapeInfo)
|
||||
&& allSameOrder) {
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, zRank, zShape, zCoords);
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, zCoords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, zCoords, yOffset);
|
||||
COORDS2INDEX(zRank, zStride, zCoords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset], extraParams);
|
||||
}
|
||||
} else {
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, zRank,xShape, zCoords);
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, zCoords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, zCoords, yOffset);
|
||||
COORDS2INDEX(zRank, zStride, zCoords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset], extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace pairwise_transforms
|
||||
} // namespace functions
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by remote on 2018-09-20.
|
||||
//
|
||||
#include <loops/pairwise_bool.h>
|
||||
#include <types/types.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <execution/Threads.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace pairwise_transforms {
|
||||
|
||||
|
||||
template <typename X, typename Z>
|
||||
void PairWiseBoolTransform<X, Z>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TT(exec, PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams, start, stop),
|
||||
PAIRWISE_BOOL_OPS);
|
||||
};
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void PairWiseBoolTransform<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
void *vextraParams, sd::LongType start, sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *yShape = shape::shapeOf(yShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *yStride = shape::stride(yShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
sd::LongType n = shape::length(xShapeInfo);
|
||||
|
||||
if (shape::isScalar(yShapeInfo)) {
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, zShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
z[offset] = OpType::op(x[offset], static_cast<X>(y[0]), static_cast<X*>(extraParams));
|
||||
};
|
||||
} else {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], y[0], extraParams);
|
||||
};
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const sd::LoopKind::Kind kindOfLoop = sd::LoopKind::deduceKindOfLoopXYZ(xShapeInfo, yShapeInfo, zShapeInfo);
|
||||
const bool sameShapesXY = shape::shapeEquals(xShapeInfo, yShapeInfo);
|
||||
const bool isSameLength = shape::length(xShapeInfo) == shape::length(yShapeInfo);
|
||||
|
||||
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, yShapeInfo) &&
|
||||
shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
z[offset] = OpType::op(x[offset], y[offset], extraParams);
|
||||
};
|
||||
} else if (shape::haveSameShapeAndStrides(xShapeInfo, yShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpType::op(x[offset], y[offset], extraParams);
|
||||
};
|
||||
} else if (shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset, yOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
COORDS2INDEX(yRank, yStride, coords, yOffset);
|
||||
z[offset] = OpType::op(x[offset], y[yOffset], extraParams);
|
||||
};
|
||||
} else if (shape::haveSameShapeAndStrides(yShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, yRank, yShape, coords);
|
||||
sd::LongType xOffset, offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, coords, offset);
|
||||
z[offset] = OpType::op(x[xOffset], y[offset], extraParams);
|
||||
};
|
||||
} else {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, zRank, zShape, coords);
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, coords, yOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset], extraParams);
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( class PairWiseBoolTransform, , SD_COMMON_TYPES, SD_BOOL_TYPES);
|
||||
} // namespace pairwise_transforms
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,153 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <loops/pairwise_int.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace pairwise_transforms {
|
||||
|
||||
|
||||
|
||||
template <typename X>
|
||||
void PairWiseIntTransform<X>::exec(const int opNum, const void *x, const sd::LongType *xShapeInfo, const void *y,
|
||||
const sd::LongType *yShapeInfo, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraParams, const uint64_t start, const uint64_t stop) {
|
||||
DISPATCH_BY_OPNUM_T(exec, PARAMS(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraParams, start, stop),
|
||||
PAIRWISE_INT_OPS);
|
||||
};
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void PairWiseIntTransform<X>::exec(const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
void *vextraParams, const uint64_t start, const uint64_t stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *yShape = shape::shapeOf(yShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *yStride = shape::stride(yShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
auto n = shape::length(xShapeInfo);
|
||||
auto xEws = shape::elementWiseStride(xShapeInfo);
|
||||
auto yEws = shape::elementWiseStride(yShapeInfo);
|
||||
auto zEws = shape::elementWiseStride(zShapeInfo);
|
||||
|
||||
if (shape::isScalar(yShapeInfo)) {
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
z[offset] = OpType::op(x[offset], y[0], extraParams);
|
||||
};
|
||||
} else {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], y[0], extraParams);
|
||||
};
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const sd::LoopKind::Kind kindOfLoop = sd::LoopKind::deduceKindOfLoopXYZ(xShapeInfo, yShapeInfo, zShapeInfo);
|
||||
const bool sameShapesXY = shape::shapeEquals(xShapeInfo, yShapeInfo);
|
||||
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, yShapeInfo) &&
|
||||
shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
z[offset] = OpType::op(x[offset], y[offset], extraParams);
|
||||
}
|
||||
} else if (shape::haveSameShapeAndStrides(xShapeInfo, yShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpType::op(x[offset], y[offset], extraParams);
|
||||
}
|
||||
} else if (shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset, yOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
COORDS2INDEX(yRank, yStride, coords, yOffset);
|
||||
z[offset] = OpType::op(x[offset], y[yOffset], extraParams);
|
||||
}
|
||||
} else if (shape::haveSameShapeAndStrides(yShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, yRank, yShape, coords);
|
||||
sd::LongType xOffset, offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, coords, offset);
|
||||
z[offset] = OpType::op(x[xOffset], y[offset], extraParams);
|
||||
}
|
||||
} else {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, zRank, zShape, coords);
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, coords, yOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], y[yOffset], extraParams);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( class PairWiseIntTransform, , SD_COMMON_TYPES);
|
||||
} // namespace pairwise_transforms
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,186 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com, created on 15.12.17.
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <loops/random.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace randomOps;
|
||||
|
||||
namespace functions {
|
||||
namespace random {
|
||||
|
||||
template <typename X>
|
||||
template <typename OpClass>
|
||||
void RandomFunction<X>::execTransform(sd::Pointer state, const void *vx, const sd::LongType *xShapeInfo, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
void *vextraArguments) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto extraArguments = reinterpret_cast<X *>(vextraArguments);
|
||||
|
||||
if (OpClass::requiresSpecial) {
|
||||
OpClass::specialOp(state, x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraArguments);
|
||||
return;
|
||||
}
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *yShape = shape::shapeOf(yShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *yStride = shape::stride(yShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
auto length = shape::length(zShapeInfo);
|
||||
sd::graph::RandomGenerator *rng = reinterpret_cast<sd::graph::RandomGenerator *>(state);
|
||||
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, yShapeInfo) &&
|
||||
shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
z[i] = OpClass::op(x[i], y[i], i, length, rng, extraArguments);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, length, 1);
|
||||
} else {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType xOffset, yOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, coords, yOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpClass::op(x[xOffset], y[yOffset], i, length, rng, extraArguments);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, length, 1);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpClass>
|
||||
void RandomFunction<X>::execTransform(sd::Pointer state, const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, void *vextraArguments) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto extraArguments = reinterpret_cast<X *>(vextraArguments);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
auto length = shape::length(zShapeInfo);
|
||||
sd::graph::RandomGenerator *rng = reinterpret_cast<sd::graph::RandomGenerator *>(state);
|
||||
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
z[offset] = OpClass::op(x[offset], i, length, rng, extraArguments);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, length, 1);
|
||||
} else {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpClass::op(x[xOffset], i, length, rng, extraArguments);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, length, 1);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpClass>
|
||||
void RandomFunction<X>::execTransform(sd::Pointer state, void *vz, const sd::LongType *zShapeInfo,
|
||||
void *vextraArguments) {
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto extraArguments = reinterpret_cast<X *>(vextraArguments);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
auto length = shape::length(zShapeInfo);
|
||||
sd::graph::RandomGenerator *rng = reinterpret_cast<sd::graph::RandomGenerator *>(state);
|
||||
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, zRank, zShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(zRank, zStride, coords, offset);
|
||||
z[offset] = OpClass::op(i, length, rng, extraArguments);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, length, 1);
|
||||
}
|
||||
|
||||
// Dispatch functions remain unchanged as they just route to the optimized implementations
|
||||
template <typename X>
|
||||
void RandomFunction<X>::execTransform(int opNum, sd::Pointer state, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *z, const sd::LongType *zShapeInfo, void *extraArguments) {
|
||||
DISPATCH_BY_OPNUM_T(execTransform, PARAMS(state, x, xShapeInfo, z, zShapeInfo, extraArguments), RANDOM_OPS)
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void RandomFunction<X>::execTransform(int opNum, sd::Pointer state, const void *x, const sd::LongType *xShapeInfo,
|
||||
const void *y, const sd::LongType *yShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, void *extraArguments) {
|
||||
DISPATCH_BY_OPNUM_T(execTransform, PARAMS(state, x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, extraArguments),
|
||||
RANDOM_OPS)
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void RandomFunction<X>::execTransform(int opNum, sd::Pointer state, void *z, const sd::LongType *zShapeInfo,
|
||||
void *extraArguments) {
|
||||
DISPATCH_BY_OPNUM_T(execTransform, PARAMS(state, z, zShapeInfo, extraArguments), RANDOM_OPS)
|
||||
}
|
||||
|
||||
} // namespace random
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,198 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
||||
* the License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/Loops.h>
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/reduce_bool.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace reduce {
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceBoolFunction<X, Z>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams,
|
||||
void *vz, const sd::LongType *zShapeInfo) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
const auto length = shape::length(xShapeInfo);
|
||||
|
||||
if (shape::isEmptyConst(xShapeInfo)) {
|
||||
z[0] = OpType::startingValue(x);
|
||||
return;
|
||||
}
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
if (sd::ArrayOptions::arrayType(zShapeInfo) == sd::ArrayType::EMPTY) return;
|
||||
const auto startingVal = OpType::startingValue(x);
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
z[i] = startingVal;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
auto startingValue = OpType::startingValue(x);
|
||||
int maxThreads = sd::math::sd_min<int>(64, sd::Environment::getInstance().maxThreads());
|
||||
X intermediate[64];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto e = 0; e < maxThreads; e++) {
|
||||
intermediate[e] = startingValue;
|
||||
}
|
||||
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
if(shape::isViewConst(xShapeInfo)) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType indexOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, indexOffset);
|
||||
intermediate[thread_id] = OpType::update(intermediate[thread_id], OpType::op(x[indexOffset], extraParams), extraParams);
|
||||
}
|
||||
};
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int e = 1; e < maxThreads; e++) {
|
||||
intermediate[0] = OpType::update(intermediate[0], intermediate[e], extraParams);
|
||||
}
|
||||
|
||||
z[0] = OpType::postProcess(intermediate[0], length, extraParams);
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
intermediate[thread_id] = OpType::update(intermediate[thread_id], OpType::op(x[i], extraParams), extraParams);
|
||||
}
|
||||
};
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int e = 1; e < maxThreads; e++) {
|
||||
intermediate[0] = OpType::update(intermediate[0], intermediate[e], extraParams);
|
||||
}
|
||||
|
||||
z[0] = OpType::postProcess(intermediate[0], length, extraParams);
|
||||
}
|
||||
}
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
Z SD_HOST ReduceBoolFunction<X, Z>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
const sd::LongType length = shape::length(xShapeInfo);
|
||||
auto startingValue = OpType::startingValue(x);
|
||||
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
startingValue = OpType::update(startingValue, OpType::op(x[offset], extraParams), extraParams);
|
||||
}
|
||||
return OpType::postProcess(startingValue, length, extraParams);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
Z ReduceBoolFunction<X, Z>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *extraParams) {
|
||||
RETURNING_DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(x, xShapeInfo, extraParams), REDUCE_BOOL_OPS);
|
||||
|
||||
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void ReduceBoolFunction<X, Z>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *extraParams, void *z, const sd::LongType *zShapeInfo) {
|
||||
DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(x, xShapeInfo, extraParams, z, zShapeInfo), REDUCE_BOOL_OPS);
|
||||
|
||||
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceBoolFunction<X, Z>::exec(const void *x, const sd::LongType *xShapeInfo, void *extraParams,
|
||||
void *vresult, const sd::LongType *resultShapeInfo) {
|
||||
auto z = reinterpret_cast<Z *>(vresult);
|
||||
z[0] = execScalar<OpType>(x, xShapeInfo, extraParams);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceBoolFunction<X, Z>::exec(sd::memory::Workspace *workspace, const void *vx,
|
||||
const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const sd::LongType *dims) {
|
||||
const X *x = reinterpret_cast<const X *>(vx);
|
||||
Z *z = reinterpret_cast<Z *>(vz);
|
||||
X *extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
const sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
const sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
const auto startingVal = OpType::startingValue(x);
|
||||
const auto zLen = shape::length(zShapeInfo);
|
||||
if(z != nullptr)
|
||||
for (sd::LongType i = 0; i < zLen; i++) {
|
||||
#if defined(PRINT_INDICES)
|
||||
shape::printShapeInfo(xShapeInfo);
|
||||
shape::printShapeInfo(zShapeInfo);
|
||||
printf("i: %lld\n ReduceBoolFunction<X, Z>::exec", i);
|
||||
#endif
|
||||
z[i] = startingVal;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (shape::length(zShapeInfo) == 1) {
|
||||
z[0] = execScalar<OpType>(x, xShapeInfo, extraParams);
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef SD_LOOPS_INLINED
|
||||
sd::ReductionLoops<X, Z, X>::template loopReduce<OpType>(workspace, x, xShapeInfo, z, zShapeInfo, dims, extraParams);
|
||||
#else
|
||||
sd::ReductionBoolLoops<X, Z>::template innerloopReduce<OpType>(workspace, x, xShapeInfo, z, zShapeInfo, dims,
|
||||
extraParams);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void ReduceBoolFunction<X, Z>::exec(int opNum, sd::memory::Workspace *workspace, const void *vx,
|
||||
const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const sd::LongType *dims) {
|
||||
DISPATCH_BY_OPNUM_TT(exec, PARAMS(workspace, vx, xShapeInfo, vextraParams, vz, zShapeInfo, dims), REDUCE_BOOL_OPS);
|
||||
}
|
||||
} // namespace reduce
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,380 @@
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/Loops.h>
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <helpers/ShapeBuilders.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/reduce_float.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
#include <algorithm>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace reduce {
|
||||
|
||||
// =============================================================================
|
||||
// TYPE-SAFE UTILITIES FOR ALL NUMERIC TYPES INCLUDING FLOAT16
|
||||
// =============================================================================
|
||||
|
||||
namespace SafeTypeUtils {
|
||||
|
||||
/**
|
||||
* @brief Type-safe array initialization that works with float16 and all other types
|
||||
*/
|
||||
template<typename T>
|
||||
SD_INLINE void initializeArray(T* array, size_t count) {
|
||||
if constexpr (std::is_arithmetic_v<T>) {
|
||||
// For arithmetic types including float16, use loop initialization
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
array[i] = static_cast<T>(0);
|
||||
}
|
||||
} else {
|
||||
// For non-arithmetic types, use default initialization
|
||||
std::fill_n(array, count, T{});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Safe type conversion for mixed-type operations
|
||||
*/
|
||||
template<typename From, typename To>
|
||||
SD_INLINE constexpr To safeCast(const From& value) {
|
||||
return static_cast<To>(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Convert parameter arrays between types safely
|
||||
*/
|
||||
template<typename SourceType, typename TargetType>
|
||||
SD_INLINE void convertParams(const SourceType* source, TargetType* target, size_t count = 8) {
|
||||
if (source && target) {
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
target[i] = safeCast<SourceType, TargetType>(source[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Determine appropriate parameter type for mixed operations
|
||||
* For float16, use float for better compatibility; otherwise use Z type
|
||||
*/
|
||||
template<typename X, typename Z>
|
||||
struct CompatibleParamType {
|
||||
using type = typename std::conditional_t<
|
||||
std::is_same_v<Z, float16> || std::is_same_v<Z, bfloat16>,
|
||||
float, // Use float for half-precision types
|
||||
Z // Use Z for all other types
|
||||
>;
|
||||
};
|
||||
|
||||
} // namespace SafeTypeUtils
|
||||
|
||||
// =============================================================================
|
||||
// REDUCE FLOAT FUNCTION IMPLEMENTATION WITH FLOAT16 SUPPORT
|
||||
// =============================================================================
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceFloatFunction<X, Z>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams,
|
||||
void *vz, const sd::LongType *zShapeInfo) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
|
||||
// Convert to Z* for consistency with macro expectations
|
||||
Z *extraParams = nullptr;
|
||||
Z convertedParams[8];
|
||||
|
||||
if (vextraParams != nullptr) {
|
||||
if constexpr (std::is_same_v<Z, X>) {
|
||||
extraParams = reinterpret_cast<Z*>(vextraParams);
|
||||
} else {
|
||||
// Convert parameters to Z type
|
||||
auto originalParams = reinterpret_cast<X*>(vextraParams);
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
convertedParams[i] = static_cast<Z>(originalParams[i]);
|
||||
}
|
||||
extraParams = convertedParams;
|
||||
}
|
||||
}
|
||||
|
||||
const auto length = shape::length(xShapeInfo);
|
||||
|
||||
if (shape::isEmptyConst(xShapeInfo)) {
|
||||
z[0] = static_cast<Z>(OpType::startingValue(x));
|
||||
return;
|
||||
}
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
if (sd::ArrayOptions::arrayType(zShapeInfo) == sd::ArrayType::EMPTY) return;
|
||||
const auto startingVal = static_cast<Z>(OpType::startingValue(x));
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
z[i] = startingVal;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
auto startingValue = static_cast<typename OpType::InterType>(OpType::startingValue(x));
|
||||
int maxThreads = sd::math::sd_min<int>(64, sd::Environment::getInstance().maxThreads());
|
||||
typename OpType::InterType intermediate[64];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto e = 0; e < maxThreads; e++) {
|
||||
intermediate[e] = startingValue;
|
||||
}
|
||||
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
|
||||
if(shape::isViewConst(xShapeInfo)) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType indexOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, indexOffset);
|
||||
|
||||
auto opResult = OpType::op(x[indexOffset], extraParams);
|
||||
intermediate[thread_id] = OpType::update(
|
||||
intermediate[thread_id],
|
||||
opResult,
|
||||
extraParams
|
||||
);
|
||||
}
|
||||
};
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int e = 1; e < maxThreads; e++) {
|
||||
intermediate[0] = OpType::merge(intermediate[0], intermediate[e], extraParams);
|
||||
}
|
||||
|
||||
z[0] = OpType::postProcess(intermediate[0], length, extraParams);
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto opResult = OpType::op(x[i], extraParams);
|
||||
intermediate[thread_id] = OpType::update(
|
||||
intermediate[thread_id],
|
||||
opResult,
|
||||
extraParams
|
||||
);
|
||||
}
|
||||
};
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int e = 1; e < maxThreads; e++) {
|
||||
intermediate[0] = OpType::merge(intermediate[0], intermediate[e], extraParams);
|
||||
}
|
||||
|
||||
z[0] = OpType::postProcess(intermediate[0], length, extraParams);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
Z SD_HOST ReduceFloatFunction<X, Z>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
|
||||
// Convert to Z* for compatibility with OpType::op
|
||||
Z *extraParams = nullptr;
|
||||
Z convertedParams[8];
|
||||
|
||||
if (vextraParams != nullptr) {
|
||||
if constexpr (std::is_same_v<Z, X>) {
|
||||
extraParams = reinterpret_cast<Z*>(vextraParams);
|
||||
} else {
|
||||
// Convert the parameters to Z type
|
||||
auto originalParams = reinterpret_cast<X*>(vextraParams);
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
convertedParams[i] = static_cast<Z>(originalParams[i]);
|
||||
}
|
||||
extraParams = convertedParams;
|
||||
}
|
||||
}
|
||||
|
||||
const sd::LongType length = shape::length(xShapeInfo);
|
||||
auto startingValue = static_cast<typename OpType::InterType>(OpType::startingValue(x));
|
||||
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
|
||||
auto opResult = OpType::op(x[offset], extraParams);
|
||||
startingValue = OpType::update(startingValue, opResult, extraParams);
|
||||
}
|
||||
|
||||
return OpType::postProcess(startingValue, length, extraParams);
|
||||
}
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceFloatFunction<X, Z>::exec(sd::memory::Workspace *workspace, const void *vx,
|
||||
const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const sd::LongType *dims) {
|
||||
const X *x = reinterpret_cast<const X *>(vx);
|
||||
Z *z = reinterpret_cast<Z *>(vz);
|
||||
|
||||
using CompatibleParamType = typename SafeTypeUtils::CompatibleParamType<X, Z>::type;
|
||||
CompatibleParamType *compatibleExtraParams = nullptr;
|
||||
CompatibleParamType convertedParams[8];
|
||||
SafeTypeUtils::initializeArray(convertedParams, 8);
|
||||
|
||||
if (vextraParams != nullptr) {
|
||||
if constexpr (std::is_same_v<X, CompatibleParamType>) {
|
||||
compatibleExtraParams = reinterpret_cast<CompatibleParamType*>(vextraParams);
|
||||
} else {
|
||||
SafeTypeUtils::convertParams(reinterpret_cast<X*>(vextraParams), convertedParams, 8);
|
||||
compatibleExtraParams = convertedParams;
|
||||
}
|
||||
}
|
||||
|
||||
const int xRank = shape::rank(xShapeInfo);
|
||||
const int zRank = shape::rank(zShapeInfo);
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
const auto startingVal = std::is_same<OpType, simdOps::Mean<X, Z>>::value
|
||||
? sd::DataTypeUtils::nanOrZero<Z>()
|
||||
: SafeTypeUtils::safeCast<X, Z>(OpType::startingValue(x));
|
||||
const auto zLen = shape::length(zShapeInfo);
|
||||
if (z != nullptr)
|
||||
for (sd::LongType i = 0; i < zLen; i++) z[i] = startingVal;
|
||||
return;
|
||||
}
|
||||
|
||||
if (shape::length(zShapeInfo) == 1) {
|
||||
z[0] = execScalar<OpType>(x, xShapeInfo, compatibleExtraParams);
|
||||
return;
|
||||
}
|
||||
|
||||
if (OpType::requiresSpecialAccumulation) {
|
||||
// FIXED: Handle execSpecial with flexible parameter types
|
||||
// The enhanced macro provides template overloads that accept any arithmetic type
|
||||
|
||||
if constexpr (std::is_same_v<CompatibleParamType, sd::LongType>) {
|
||||
// Direct call for sd::LongType parameters
|
||||
OpType::execSpecial(x, xShapeInfo, compatibleExtraParams, z, zShapeInfo,
|
||||
const_cast<sd::LongType *>(dims) + zRank, xRank - zRank,
|
||||
nullptr, nullptr);
|
||||
} else {
|
||||
// Convert to sd::LongType for operations that specifically need it
|
||||
sd::LongType longExtraParams[8];
|
||||
SafeTypeUtils::initializeArray(longExtraParams, 8);
|
||||
|
||||
if (compatibleExtraParams != nullptr) {
|
||||
SafeTypeUtils::convertParams(compatibleExtraParams, longExtraParams, 8);
|
||||
}
|
||||
|
||||
// Use template overload that accepts sd::LongType*
|
||||
OpType::execSpecial(x, xShapeInfo, longExtraParams, z, zShapeInfo,
|
||||
const_cast<sd::LongType *>(dims) + zRank, xRank - zRank,
|
||||
nullptr, nullptr);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef SD_LOOPS_INLINED
|
||||
sd::ReductionLoops<X, Z, CompatibleParamType>::template loopReduce<OpType>(workspace, x, xShapeInfo, z, zShapeInfo, dims, compatibleExtraParams);
|
||||
#else
|
||||
sd::ReductionFloatLoops<X, Z>::template innerloopReduce<OpType>(workspace, x, xShapeInfo, z, zShapeInfo, dims, compatibleExtraParams);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
template <typename X, typename Z>
|
||||
Z ReduceFloatFunction<X, Z>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *extraParams) {
|
||||
RETURNING_DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(x, xShapeInfo, extraParams), REDUCE_FLOAT_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void ReduceFloatFunction<X, Z>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *extraParams, void *z, const sd::LongType *zShapeInfo) {
|
||||
DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(x, xShapeInfo, extraParams, z, zShapeInfo), REDUCE_FLOAT_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceFloatFunction<X, Z>::exec(const void *x, const sd::LongType *xShapeInfo, void *extraParams,
|
||||
void *vresult, const sd::LongType *resultShapeInfo) {
|
||||
auto z = reinterpret_cast<Z *>(vresult);
|
||||
z[0] = execScalar<OpType>(x, xShapeInfo, extraParams);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
Z SD_HOST ReduceFloatFunction<X, Z>::execScalar(const void *vx, sd::LongType xEws, sd::LongType length,
|
||||
void *vextraParams) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
|
||||
using CompatibleParamType = typename SafeTypeUtils::CompatibleParamType<X, Z>::type;
|
||||
CompatibleParamType *compatibleExtraParams = nullptr;
|
||||
CompatibleParamType convertedParams[8];
|
||||
SafeTypeUtils::initializeArray(convertedParams, 8);
|
||||
|
||||
if (vextraParams != nullptr) {
|
||||
if constexpr (std::is_same_v<X, CompatibleParamType>) {
|
||||
compatibleExtraParams = reinterpret_cast<CompatibleParamType*>(vextraParams);
|
||||
} else {
|
||||
SafeTypeUtils::convertParams(reinterpret_cast<X*>(vextraParams), convertedParams, 8);
|
||||
compatibleExtraParams = convertedParams;
|
||||
}
|
||||
}
|
||||
|
||||
int maxThreads = sd::math::sd_min<int>(64, sd::Environment::getInstance().maxThreads());
|
||||
using InterType = typename OpType::InterType;
|
||||
InterType intermediate[64];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto e = 0; e < maxThreads; e++) {
|
||||
intermediate[e] = SafeTypeUtils::safeCast<X, InterType>(OpType::startingValue(x));
|
||||
}
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
if (xEws == 1) {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto opResult = OpType::op(x[i], compatibleExtraParams);
|
||||
intermediate[thread_id] = OpType::update(
|
||||
intermediate[thread_id],
|
||||
SafeTypeUtils::safeCast<decltype(opResult), InterType>(opResult),
|
||||
compatibleExtraParams
|
||||
);
|
||||
}
|
||||
} else {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto opResult = OpType::op(x[i * xEws], compatibleExtraParams);
|
||||
intermediate[thread_id] = OpType::update(
|
||||
intermediate[thread_id],
|
||||
SafeTypeUtils::safeCast<decltype(opResult), InterType>(opResult),
|
||||
compatibleExtraParams
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
|
||||
for (int e = 1; e < maxThreads; e++)
|
||||
intermediate[0] = OpType::update(intermediate[0], intermediate[e], compatibleExtraParams);
|
||||
|
||||
return SafeTypeUtils::safeCast<InterType, Z>(OpType::postProcess(intermediate[0], length, compatibleExtraParams));
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void ReduceFloatFunction<X, Z>::exec(int opNum, sd::memory::Workspace *workspace, const void *vx,
|
||||
const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const sd::LongType *dims) {
|
||||
DISPATCH_BY_OPNUM_TT(exec, PARAMS(workspace, vx, xShapeInfo, vextraParams, vz, zShapeInfo, dims), REDUCE_FLOAT_OPS);
|
||||
}
|
||||
|
||||
|
||||
} // namespace reduce
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,246 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/Loops.h>
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/reduce_long.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace reduce {
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceLongFunction<X, Z>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams,
|
||||
void *vz, const sd::LongType *zShapeInfo) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
|
||||
// For reduce_long operations like MatchCondition, extraParams contain comparison values
|
||||
// (e.g., compare value, epsilon) that must remain in input type X, not be converted to Z.
|
||||
// Converting double→LongType would truncate 0.5→0, breaking comparisons.
|
||||
auto compatibleExtraParams = reinterpret_cast<X*>(vextraParams);
|
||||
|
||||
const sd::LongType length = shape::length(xShapeInfo);
|
||||
|
||||
if (shape::isEmptyConst(xShapeInfo)) {
|
||||
z[0] = static_cast<Z>(OpType::startingValue(x));
|
||||
return;
|
||||
}
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
if (sd::ArrayOptions::arrayType(zShapeInfo) == sd::ArrayType::EMPTY) return;
|
||||
const auto startingVal = static_cast<Z>(OpType::startingValue(x));
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) z[i] = startingVal;
|
||||
return;
|
||||
}
|
||||
|
||||
int maxThreads = sd::math::sd_min<int>(64, sd::Environment::getInstance().maxThreads());
|
||||
typename OpType::InterType intermediate[64];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto e = 0; e < maxThreads; e++) {
|
||||
intermediate[e] = static_cast<typename OpType::InterType>(OpType::startingValue(x));
|
||||
}
|
||||
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType indexOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, indexOffset);
|
||||
|
||||
intermediate[thread_id] = OpType::update(
|
||||
intermediate[thread_id],
|
||||
OpType::op(x[indexOffset], compatibleExtraParams),
|
||||
compatibleExtraParams);
|
||||
}
|
||||
};
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
|
||||
for (int e = 1; e < maxThreads; e++)
|
||||
intermediate[0] = OpType::update(intermediate[0], intermediate[e], compatibleExtraParams);
|
||||
|
||||
z[0] = OpType::postProcess(intermediate[0], length, compatibleExtraParams);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
Z SD_HOST ReduceLongFunction<X, Z>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
|
||||
|
||||
// For reduce_long operations like MatchCondition, extraParams contain comparison values
|
||||
// (e.g., compare value, epsilon) that must remain in input type X, not be converted to Z.
|
||||
// Converting double→LongType would truncate 0.5→0, breaking comparisons.
|
||||
auto compatibleExtraParams = reinterpret_cast<X*>(vextraParams);
|
||||
|
||||
const sd::LongType length = shape::length(xShapeInfo);
|
||||
|
||||
auto startingValue = static_cast<typename OpType::InterType>(OpType::startingValue(x));
|
||||
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
|
||||
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType indexOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, indexOffset);
|
||||
|
||||
|
||||
startingValue = OpType::update(startingValue, OpType::op(x[indexOffset], compatibleExtraParams), compatibleExtraParams);
|
||||
}
|
||||
|
||||
Z result = OpType::postProcess(startingValue, length, compatibleExtraParams);
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
Z ReduceLongFunction<X, Z>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *extraParams) {
|
||||
RETURNING_DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(x, xShapeInfo, extraParams), REDUCE_LONG_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void ReduceLongFunction<X, Z>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *extraParams, void *z, const sd::LongType *zShapeInfo) {
|
||||
DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(x, xShapeInfo, extraParams, z, zShapeInfo), REDUCE_LONG_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceLongFunction<X, Z>::exec(const void *x, const sd::LongType *xShapeInfo, void *extraParams,
|
||||
void *vresult, const sd::LongType *resultShapeInfo) {
|
||||
|
||||
auto z = reinterpret_cast<Z *>(vresult);
|
||||
z[0] = execScalar<OpType>(x, xShapeInfo, extraParams);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
Z SD_HOST ReduceLongFunction<X, Z>::execScalar(const void *vx, sd::LongType xEws, sd::LongType length,
|
||||
void *vextraParams) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
|
||||
|
||||
// For reduce_long operations like MatchCondition, extraParams contain comparison values
|
||||
// (e.g., compare value, epsilon) that must remain in input type X, not be converted to Z.
|
||||
// Converting double→LongType would truncate 0.5→0, breaking comparisons.
|
||||
auto compatibleExtraParams = reinterpret_cast<X*>(vextraParams);
|
||||
|
||||
int maxThreads = sd::math::sd_min<int>(64, sd::Environment::getInstance().maxThreads());
|
||||
typename OpType::InterType intermediate[64];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto e = 0; e < maxThreads; e++) {
|
||||
intermediate[e] = static_cast<typename OpType::InterType>(OpType::startingValue(x));
|
||||
}
|
||||
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
if (xEws == 1) {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
intermediate[thread_id] = OpType::update(intermediate[thread_id], OpType::op(x[i], compatibleExtraParams), compatibleExtraParams);
|
||||
}
|
||||
} else {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
intermediate[thread_id] =
|
||||
OpType::update(intermediate[thread_id], OpType::op(x[i * xEws], compatibleExtraParams), compatibleExtraParams);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
|
||||
for (int e = 1; e < maxThreads; e++)
|
||||
intermediate[0] = OpType::update(intermediate[0], intermediate[e], compatibleExtraParams);
|
||||
|
||||
Z result = OpType::postProcess(intermediate[0], length, compatibleExtraParams);
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceLongFunction<X, Z>::exec(sd::memory::Workspace *workspace, const void *vx,
|
||||
const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const sd::LongType *dims) {
|
||||
const X *x = reinterpret_cast<const X *>(vx);
|
||||
Z *z = reinterpret_cast<Z *>(vz);
|
||||
|
||||
// For reduce_long operations like MatchCondition, extraParams contain comparison values
|
||||
// (e.g., compare value, epsilon) that must remain in input type X, not be converted to Z.
|
||||
// Converting double→LongType would corrupt the comparison values.
|
||||
auto compatibleExtraParams = reinterpret_cast<X*>(vextraParams);
|
||||
|
||||
const int xRank = shape::rank(xShapeInfo);
|
||||
const int zRank = shape::rank(zShapeInfo);
|
||||
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
const auto startingVal = static_cast<Z>(OpType::startingValue(x));
|
||||
const auto zLen = shape::length(zShapeInfo);
|
||||
|
||||
for (sd::LongType i = 0; i < zLen; i++) z[i] = startingVal;
|
||||
return;
|
||||
}
|
||||
|
||||
if (shape::length(zShapeInfo) == 1) {
|
||||
z[0] = execScalar<OpType>(x, xShapeInfo, vextraParams);
|
||||
return;
|
||||
}
|
||||
|
||||
if (OpType::requiresSpecialAccumulation) {
|
||||
// For execSpecial, use original LongType* extraParams as expected by that API
|
||||
auto originalParams = reinterpret_cast<sd::LongType *>(vextraParams);
|
||||
OpType::execSpecial(x, xShapeInfo, originalParams, z, zShapeInfo, const_cast<sd::LongType *>(dims) + zRank,
|
||||
xRank - zRank, nullptr, nullptr);
|
||||
return;
|
||||
}
|
||||
// Call ReductionLongLoops with properly typed X* extraParams
|
||||
sd::ReductionLongLoops<X, Z>::template innerloopReduce<OpType>(workspace, x, xShapeInfo, z, zShapeInfo, dims, compatibleExtraParams);
|
||||
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void ReduceLongFunction<X, Z>::exec(const int opNum, sd::memory::Workspace *workspace, const void *vx,
|
||||
const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, sd::LongType *dims) {
|
||||
|
||||
DISPATCH_BY_OPNUM_TT(exec, PARAMS(workspace, vx, xShapeInfo, vextraParams, vz, zShapeInfo, dims), REDUCE_LONG_OPS);
|
||||
}
|
||||
|
||||
} // namespace reduce
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,204 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
||||
* the License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/Loops.h>
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/reduce_same.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
#include <chrono>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace reduce {
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceSameFunction<X>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams,
|
||||
void *vz, const sd::LongType *zShapeInfo) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
const auto length = shape::length(xShapeInfo);
|
||||
|
||||
if (shape::isEmptyConst(xShapeInfo)) {
|
||||
z[0] = OpType::startingValue(x);
|
||||
return;
|
||||
}
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
if (sd::ArrayOptions::arrayType(zShapeInfo) == sd::ArrayType::EMPTY) return;
|
||||
const auto startingVal = OpType::startingValue(x);
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
z[i] = startingVal;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
auto startingValue = OpType::startingValue(x);
|
||||
int maxThreads = sd::math::sd_min<int>(64, sd::Environment::getInstance().maxThreads());
|
||||
X intermediate[64];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto e = 0; e < maxThreads; e++) {
|
||||
intermediate[e] = startingValue;
|
||||
}
|
||||
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
if(shape::isViewConst(xShapeInfo)) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType indexOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, indexOffset);
|
||||
intermediate[thread_id] = OpType::update(intermediate[thread_id], OpType::op(x[indexOffset], extraParams), extraParams);
|
||||
}
|
||||
};
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int e = 1; e < maxThreads; e++) {
|
||||
intermediate[0] = OpType::update(intermediate[0], intermediate[e], extraParams);
|
||||
}
|
||||
|
||||
z[0] = OpType::postProcess(intermediate[0], length, extraParams);
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
intermediate[thread_id] = OpType::update(intermediate[thread_id], OpType::op(x[i], extraParams), extraParams);
|
||||
}
|
||||
};
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int e = 1; e < maxThreads; e++) {
|
||||
intermediate[0] = OpType::update(intermediate[0], intermediate[e], extraParams);
|
||||
}
|
||||
|
||||
z[0] = OpType::postProcess(intermediate[0], length, extraParams);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
X SD_HOST ReduceSameFunction<X>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
const sd::LongType length = shape::length(xShapeInfo);
|
||||
|
||||
if (shape::isEmptyConst(xShapeInfo)) {
|
||||
return OpType::startingValue(x);
|
||||
}
|
||||
|
||||
auto startingValue = OpType::startingValue(x);
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType indexOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, indexOffset);
|
||||
startingValue = OpType::update(startingValue, OpType::op(x[indexOffset], extraParams), extraParams);
|
||||
}
|
||||
return OpType::postProcess(startingValue, length, extraParams);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
X ReduceSameFunction<X>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams) {
|
||||
RETURNING_DISPATCH_BY_OPNUM_T(execScalar, PARAMS(x, xShapeInfo, extraParams), REDUCE_SAME_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void ReduceSameFunction<X>::execScalar(const int opNum, const void *x, const sd::LongType *xShapeInfo,
|
||||
void *extraParams, void *z, const sd::LongType *zShapeInfo) {
|
||||
DISPATCH_BY_OPNUM_T(execScalar, PARAMS(x, xShapeInfo, extraParams, z, zShapeInfo), REDUCE_SAME_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceSameFunction<X>::exec(const void *x, const sd::LongType *xShapeInfo, void *extraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
z[0] = execScalar<OpType>(x, xShapeInfo, extraParams);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void SD_HOST ReduceSameFunction<X>::exec(sd::memory::Workspace *workspace, const void *vx,
|
||||
const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const sd::LongType *dims) {
|
||||
const X *x = reinterpret_cast<const X *>(vx);
|
||||
X *z = reinterpret_cast<X *>(vz);
|
||||
X *extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
const sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
const sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
const auto startingVal = OpType::startingValue(x);
|
||||
const auto zLen = shape::length(zShapeInfo);
|
||||
if (z != nullptr) {
|
||||
for (sd::LongType i = 0; i < zLen; i++) {
|
||||
z[i] = startingVal;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (shape::length(zShapeInfo) == 1) {
|
||||
z[0] = execScalar<OpType>(x, xShapeInfo, extraParams);
|
||||
return;
|
||||
}
|
||||
|
||||
if (OpType::requiresSpecialAccumulation) {
|
||||
OpType::execSpecial(x, xShapeInfo, extraParams, z, zShapeInfo, const_cast<sd::LongType *>(dims) + zRank,
|
||||
xRank - zRank, nullptr, nullptr);
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef SD_LOOPS_INLINED
|
||||
sd::ReductionLoops<X, X, X>::template loopReduce<OpType>(workspace, x, xShapeInfo, z, zShapeInfo, dims, extraParams);
|
||||
#else
|
||||
sd::ReductionSameLoops<X>::template innerloopReduce<OpType>(workspace, x, xShapeInfo, z, zShapeInfo, dims, extraParams);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void ReduceSameFunction<X>::exec(int opNum, sd::memory::Workspace *workspace, const void *vx,
|
||||
const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const sd::LongType *dims) {
|
||||
DISPATCH_BY_OPNUM_T(exec, PARAMS(workspace, vx, xShapeInfo, vextraParams, vz, zShapeInfo, dims), REDUCE_SAME_OPS);
|
||||
}
|
||||
|
||||
|
||||
} // namespace reduce
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,235 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
// @author raver119@gmail.com
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 19.11.2018
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/Loops.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/reduce3.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace reduce3 {
|
||||
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void Reduce3<X, Z>::execScalar(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
|
||||
auto length = shape::length(xShapeInfo);
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY ||
|
||||
sd::ArrayOptions::arrayType(yShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
if (sd::ArrayOptions::arrayType(zShapeInfo) == sd::ArrayType::EMPTY) return;
|
||||
const auto startingVal = OpType::startingValue(x);
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) z[i] = startingVal;
|
||||
return;
|
||||
}
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType yRank = shape::rank(yShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *yShape = shape::shapeOf(yShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *yStride = shape::stride(yShapeInfo);
|
||||
|
||||
Z extraParamsVals[3] = {(Z)0.0f, (Z)0.0f, (Z)0.0f};
|
||||
Z startingVal = OpType::startingValue(x);
|
||||
int maxThreads = sd::math::sd_min<int>(64, sd::Environment::getInstance().maxThreads());
|
||||
Z intermediate[64];
|
||||
Z extraParamsLocal[3 * 64];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int e = 0; e < maxThreads; e++) intermediate[e] = startingVal;
|
||||
|
||||
sd::ops::safe_zero(extraParamsLocal, 3);
|
||||
if (extraParams != nullptr) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int e = 0; e < maxThreads; e++) {
|
||||
extraParamsLocal[3 * e] = extraParams[0];
|
||||
extraParamsLocal[3 * e + 1] = extraParams[1];
|
||||
extraParamsLocal[3 * e + 2] = extraParams[2];
|
||||
}
|
||||
}
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
sd::LongType yCoords[SD_MAX_RANK];
|
||||
|
||||
INDEX2COORDS(i, xRank, xShape, xCoords);
|
||||
INDEX2COORDS(i, yRank, yShape, yCoords);
|
||||
sd::LongType xOffset = 0, yOffset = 0;
|
||||
COORDS2INDEX(xRank, xStride, xCoords, xOffset);
|
||||
COORDS2INDEX(yRank, yStride, yCoords, yOffset);
|
||||
|
||||
intermediate[thread_id] = OpType::update(intermediate[thread_id],
|
||||
OpType::op(x[xOffset], y[yOffset], extraParamsLocal + 3 * thread_id),
|
||||
extraParamsLocal + 3 * thread_id);
|
||||
}
|
||||
};
|
||||
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, length, 1, maxThreads);
|
||||
|
||||
|
||||
// merge step
|
||||
for (int e = 0; e < maxThreads; e++) OpType::aggregateExtraParams(extraParamsVals, extraParamsLocal + 3 * e);
|
||||
|
||||
for (int e = 0; e < maxThreads; e++) startingVal = OpType::update(startingVal, intermediate[e], extraParamsVals);
|
||||
|
||||
// writing out result
|
||||
z[0] = OpType::postProcess(startingVal, length, extraParamsVals);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
void Reduce3<X, Z>::execScalar(const int opNum, const void *vx, const sd::LongType *xShapeInfo, void *extraParamsVals,
|
||||
const void *vy, const sd::LongType *yShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo) {
|
||||
DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(vx, xShapeInfo, extraParamsVals, vy, yShapeInfo, vz, zShapeInfo),
|
||||
REDUCE3_OPS);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void Reduce3<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, sd::LongType start, sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
|
||||
if (shape::isScalar(zShapeInfo)) {
|
||||
execScalar<OpType>(vx, xShapeInfo, vextraParams, vy, yShapeInfo, vz, zShapeInfo);
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef SD_LOOPS_INLINED
|
||||
sd::Reduction3Loops<X, Z>::template loopReduce3<OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, dimension,
|
||||
dimensionLength, extraParams, start, stop);
|
||||
#else
|
||||
sd::Reduction3Loops<X, Z>::template innerloopReduce3<OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, dimension,
|
||||
dimensionLength, extraParams, start, stop);
|
||||
#endif
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Rest of the functions remain the same as they primarily dispatch to other functions
|
||||
// or use the Reduction3Loops class which handles its own shape caching internally
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void Reduce3<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *tadShapeInfo, const sd::LongType *tadOffsets,
|
||||
sd::LongType start, sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
#ifdef SD_LOOPS_INLINED
|
||||
sd::Reduction3Loops<X, Z>::template loopReduce3<OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, dimension,
|
||||
dimensionLength, extraParams, start, stop);
|
||||
#else
|
||||
sd::Reduction3Loops<X, Z>::template innerloopReduce3<OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, dimension,
|
||||
dimensionLength, extraParams, start, stop);
|
||||
#endif
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void Reduce3<X, Z>::execAll(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, const void *vy,
|
||||
const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo, const sd::LongType *xOffsets,
|
||||
const sd::LongType *yTadShapeInfo, const sd::LongType *yOffsets, sd::LongType start,
|
||||
sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto y = reinterpret_cast<const X *>(vy);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
|
||||
#ifdef SD_LOOPS_INLINED
|
||||
sd::Reduction3Loops<X, Z>::template loopReduce3All<OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo, xTadShapeInfo,
|
||||
xOffsets, yTadShapeInfo, yOffsets, extraParams, start,
|
||||
stop);
|
||||
#else
|
||||
sd::Reduction3Loops<X, Z>::template innerloopReduce3All<OpType>(x, xShapeInfo, y, yShapeInfo, z, zShapeInfo,
|
||||
xTadShapeInfo, xOffsets, yTadShapeInfo, yOffsets,
|
||||
extraParams, start, stop);
|
||||
#endif
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
void Reduce3<X, Z>::exec(int opNum, const void *vx, const sd::LongType *xShapeInfo, void *extraParamsVals,
|
||||
const void *vy, const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TT(
|
||||
exec,
|
||||
PARAMS(vx, xShapeInfo, extraParamsVals, vy, yShapeInfo, vz, zShapeInfo, dimension, dimensionLength, start, stop),
|
||||
REDUCE3_OPS);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
void Reduce3<X, Z>::exec(int opNum, const void *vx, const sd::LongType *xShapeInfo, void *extraParamsVals,
|
||||
const void *vy, const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, const sd::LongType *tadShapeInfo,
|
||||
const sd::LongType *tadOffsets, sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TT(exec,
|
||||
PARAMS(vx, xShapeInfo, extraParamsVals, vy, yShapeInfo, vz, zShapeInfo, dimension,
|
||||
dimensionLength, tadShapeInfo, tadOffsets, start, stop),
|
||||
REDUCE3_OPS);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
void Reduce3<X, Z>::execAll(int opNum, const void *vx, const sd::LongType *xShapeInfo, void *extraParamsVals,
|
||||
const void *vy, const sd::LongType *yShapeInfo, void *vz, const sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xOffsets, const sd::LongType *yTadShapeInfo,
|
||||
const sd::LongType *yOffsets, sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TT(execAll,
|
||||
PARAMS(vx, xShapeInfo, extraParamsVals, vy, yShapeInfo, vz, zShapeInfo, dimension,
|
||||
dimensionLength, xTadShapeInfo, xOffsets, yTadShapeInfo, yOffsets, start, stop),
|
||||
REDUCE3_OPS);
|
||||
}
|
||||
|
||||
} // namespace reduce3
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,143 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 08.10.2017.
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "../legacy_ops.h"
|
||||
#include "../scalar.h"
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace scalar {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y, typename Z>
|
||||
template <typename OpType>
|
||||
void ScalarTransform<X, Y, Z>::transform(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const void *vscalars, sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffsets, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffsets, sd::LongType start, sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto scalars = reinterpret_cast<const Y *>(vscalars);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
|
||||
if (zTadShapeInfo == nullptr) {
|
||||
zTadShapeInfo = xTadShapeInfo;
|
||||
zTadOffsets = xTadOffsets;
|
||||
}
|
||||
|
||||
// Cache shape-related values for TAD operations
|
||||
sd::LongType xTadRank = shape::rank(xTadShapeInfo);
|
||||
sd::LongType zTadRank = shape::rank(zTadShapeInfo);
|
||||
sd::LongType *xTadShape = shape::shapeOf(xTadShapeInfo);
|
||||
sd::LongType *zTadShape = shape::shapeOf(zTadShapeInfo);
|
||||
sd::LongType *xTadStride = shape::stride(xTadShapeInfo);
|
||||
sd::LongType *zTadStride = shape::stride(zTadShapeInfo);
|
||||
|
||||
const int tadLength = shape::tadLength(xShapeInfo, dimension, dimensionLength);
|
||||
for (auto r = start; r < stop; r++) {
|
||||
auto oZ = z + zTadOffsets[r];
|
||||
auto oX = x + xTadOffsets[r];
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int f = 0; f < tadLength; f++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
sd::LongType xOffset, zOffset;
|
||||
INDEX2COORDS(f, xTadRank, xTadShape, coords);
|
||||
INDEX2COORDS(f, zTadRank, zTadShape, coords);
|
||||
COORDS2INDEX(xTadRank, xTadStride, coords, xOffset);
|
||||
COORDS2INDEX(zTadRank, zTadStride, coords, zOffset);
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], scalars[0], extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y, typename Z>
|
||||
void ScalarTransform<X, Y, Z>::transform(int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams,
|
||||
void *z, const sd::LongType *zShapeInfo, const void *scalars,
|
||||
sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffsets, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffsets,
|
||||
sd::LongType start, sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TTT(transform,
|
||||
PARAMS(x, xShapeInfo, extraParams, z, zShapeInfo, scalars, dimension, dimensionLength,
|
||||
xTadShapeInfo, xTadOffsets, zTadShapeInfo, zTadOffsets, start, stop),
|
||||
SCALAR_OPS);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y, typename Z>
|
||||
void ScalarTransform<X, Y, Z>::transform(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, const void *scalar, void *extraParams,
|
||||
const sd::LongType start, const sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TTT(transform, PARAMS(x, xShapeInfo, z, zShapeInfo, scalar, extraParams, start, stop), SCALAR_OPS);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y, typename Z>
|
||||
template <typename OpType>
|
||||
void ScalarTransform<X, Y, Z>::transform(const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, const void *vscalar,
|
||||
void *vextraParams,
|
||||
const sd::LongType start, const sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto scalar = reinterpret_cast<const Y *>(vscalar);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
//need special handling for scalars as strides may not be set for scalars
|
||||
if(shape::length(xShapeInfo) <= 1 && shape::length(zShapeInfo) <= 1) {
|
||||
z[0] = OpType::op(x[0], scalar[0], extraParams);
|
||||
return;
|
||||
|
||||
}
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
sd::LongType zCoords[SD_MAX_RANK];
|
||||
sd::LongType xOffset, zOffset;
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
INDEX2COORDS(i, zRank, zShape, zCoords);
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(zRank, zStride, zCoords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], scalar[0], extraParams);
|
||||
};
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace scalar
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,146 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 08.10.2017.
|
||||
//
|
||||
#include "../scalar_bool.h"
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
#include "../legacy_ops.h"
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace scalar {
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void ScalarBoolTransform<X, Z>::transform(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const void *vscalars,
|
||||
sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffsets, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffsets, const sd::LongType start, const sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto scalars = reinterpret_cast<const X *>(vscalars);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
if (zTadShapeInfo == nullptr) {
|
||||
zTadShapeInfo = xTadShapeInfo;
|
||||
zTadOffsets = xTadOffsets;
|
||||
}
|
||||
|
||||
// Cache shape-related values for TAD operations
|
||||
sd::LongType xTadRank = shape::rank(xTadShapeInfo);
|
||||
sd::LongType zTadRank = shape::rank(zTadShapeInfo);
|
||||
sd::LongType *xTadShape = shape::shapeOf(xTadShapeInfo);
|
||||
sd::LongType *zTadShape = shape::shapeOf(zTadShapeInfo);
|
||||
sd::LongType *xTadStride = shape::stride(xTadShapeInfo);
|
||||
sd::LongType *zTadStride = shape::stride(zTadShapeInfo);
|
||||
|
||||
const int tadLength = shape::tadLength(xShapeInfo, dimension, dimensionLength);
|
||||
const int numTads = shape::length(xShapeInfo) / tadLength;
|
||||
int num_threads = sd::math::sd_min<int>(numTads, sd::Environment::getInstance().maxThreads());
|
||||
|
||||
for (auto r = start; r < stop; r++) {
|
||||
auto oZ = z + zTadOffsets[r];
|
||||
auto oX = x + xTadOffsets[r];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int f = 0; f < tadLength; f++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
sd::LongType xOffset, zOffset;
|
||||
INDEX2COORDS(f, xTadRank, xTadShape, coords);
|
||||
COORDS2INDEX(xTadRank, xTadStride, coords, xOffset);
|
||||
COORDS2INDEX(zTadRank, zTadStride, coords, zOffset);
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], scalars[0], extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void ScalarBoolTransform<X, Z>::transform(int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams,
|
||||
void *z, const sd::LongType *zShapeInfo, const void *scalars,
|
||||
sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffsets, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffsets, const sd::LongType start, const sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TT(transform,
|
||||
PARAMS(x, xShapeInfo, extraParams, z, zShapeInfo, scalars, dimension, dimensionLength,
|
||||
xTadShapeInfo, xTadOffsets, zTadShapeInfo, zTadOffsets, start, stop),
|
||||
SCALAR_BOOL_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void ScalarBoolTransform<X, Z>::transform(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, const void *scalar, void *extraParams,
|
||||
const sd::LongType start, const sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_TT(transform, PARAMS(x, xShapeInfo, z, zShapeInfo, scalar, extraParams, start, stop),
|
||||
SCALAR_BOOL_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void ScalarBoolTransform<X, Z>::transform(const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, const void *vscalar, void *vextraParams,
|
||||
const sd::LongType start, const sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto scalar = reinterpret_cast<const X *>(vscalar)[0];
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i2 = start; i2 < stop; i2++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i2, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
z[offset] = OpType::op(x[offset], scalar, extraParams);
|
||||
};
|
||||
} else {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i2 = start; i2 < stop; i2++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i2, xRank, xShape, coords);
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], scalar, extraParams);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( class ScalarBoolTransform, , SD_COMMON_TYPES, SD_BOOL_TYPES);
|
||||
|
||||
} // namespace scalar
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,174 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include "../scalar_int.h"
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopKind.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
#include "../legacy_ops.h"
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace scalar {
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void ScalarIntTransform<X>::transform(const void *vx, const sd::LongType *xShapeInfo, void *vextraParams, void *vz,
|
||||
const sd::LongType *zShapeInfo, const void *vscalars, sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffsets, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffsets, const sd::LongType start, const sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto scalars = reinterpret_cast<const X *>(vscalars);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
if (zTadShapeInfo == nullptr) {
|
||||
zTadShapeInfo = xTadShapeInfo;
|
||||
zTadOffsets = xTadOffsets;
|
||||
}
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xTadRank = shape::rank(xTadShapeInfo);
|
||||
sd::LongType zTadRank = shape::rank(zTadShapeInfo);
|
||||
sd::LongType *xTadShape = shape::shapeOf(xTadShapeInfo);
|
||||
sd::LongType *zTadShape = shape::shapeOf(zTadShapeInfo);
|
||||
sd::LongType *xTadStride = shape::stride(xTadShapeInfo);
|
||||
sd::LongType *zTadStride = shape::stride(zTadShapeInfo);
|
||||
|
||||
const int tadLength = shape::tadLength(xShapeInfo, dimension, dimensionLength);
|
||||
|
||||
for (auto r = start; r < stop; r++) {
|
||||
auto oZ = z + zTadOffsets[r];
|
||||
auto oX = x + xTadOffsets[r];
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int f = 0; f < tadLength; f++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
sd::LongType xOffset, zOffset;
|
||||
INDEX2COORDS(f, xTadRank, xTadShape, coords);
|
||||
COORDS2INDEX(xTadRank, xTadStride, coords, xOffset);
|
||||
COORDS2INDEX(zTadRank, zTadStride, coords, zOffset);
|
||||
oZ[zOffset] = OpType::op(oX[xOffset], scalars[0], extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void ScalarIntTransform<X>::transform(int opNum, const void *x, const sd::LongType *xShapeInfo, void *extraParams,
|
||||
void *z, const sd::LongType *zShapeInfo, const void *scalars,
|
||||
sd::LongType *dimension,
|
||||
sd::LongType dimensionLength, const sd::LongType *xTadShapeInfo,
|
||||
const sd::LongType *xTadOffsets, const sd::LongType *zTadShapeInfo,
|
||||
const sd::LongType *zTadOffsets, const sd::LongType start, const sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_T(transform,
|
||||
PARAMS(x, xShapeInfo, extraParams, z, zShapeInfo, scalars, dimension, dimensionLength,
|
||||
xTadShapeInfo, xTadOffsets, zTadShapeInfo, zTadOffsets, start, stop),
|
||||
SCALAR_INT_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void ScalarIntTransform<X>::transform(const int opNum, const void *x, sd::LongType xEws, void *z, sd::LongType zEws,
|
||||
const void *scalar, void *extraParams, const sd::LongType n, const sd::LongType start,
|
||||
const sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_T(transform, PARAMS(x, xEws, z, zEws, scalar, extraParams, n, start, stop), SCALAR_INT_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
void ScalarIntTransform<X>::transform(const int opNum, const void *x, const sd::LongType *xShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, const void *scalar, void *extraParams,
|
||||
const sd::LongType start, const sd::LongType stop) {
|
||||
DISPATCH_BY_OPNUM_T(transform, PARAMS(x, xShapeInfo, z, zShapeInfo, scalar, extraParams, start, stop),
|
||||
SCALAR_INT_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void ScalarIntTransform<X>::transform(const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, const void *vscalar, void *vextraParams,
|
||||
const sd::LongType start, const sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto scalar = reinterpret_cast<const X *>(vscalar)[0];
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
if (shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo)) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i3 = start; i3 < stop; i3++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i3, xRank, xShape, coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(xRank, xStride, coords, offset);
|
||||
z[offset] = OpType::op(x[offset], scalar, extraParams);
|
||||
};
|
||||
} else {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i2 = start; i2 < stop; i2++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i2, xRank, xShape, coords);
|
||||
sd::LongType xOffset, zOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
COORDS2INDEX(zRank, zStride, coords, zOffset);
|
||||
z[zOffset] = OpType::op(x[xOffset], scalar, extraParams);
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void ScalarIntTransform<X>::transform(const void *vx, sd::LongType xEws, void *vz, sd::LongType zEws,
|
||||
const void *vscalar, void *vextraParams, const sd::LongType len, const sd::LongType start,
|
||||
const sd::LongType stop) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto scalar = reinterpret_cast<const X *>(vscalar)[0];
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
if (scalar < static_cast<X>((sizeof(X) * 8))) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto xi = i * xEws;
|
||||
auto zi = i * zEws;
|
||||
z[zi] = OpType::op(x[xi], scalar, extraParams);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE(class ScalarIntTransform, , SD_INTEGER_TYPES);
|
||||
|
||||
// Note: Platform-specific instantiations for "unsigned long" and "unsigned int" on Linux
|
||||
// are now generated by cmake/TemplateProcessing.cmake which preserves these raw types
|
||||
// instead of canonicalizing them to typedefs. This fixes JavaCPP symbol resolution issues.
|
||||
|
||||
} // namespace scalar
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,165 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 18.12.17.
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
|
||||
#include <helpers/shape.h>
|
||||
#include <loops/summarystatsreduce.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace summarystats {
|
||||
|
||||
template <typename X, typename Z>
|
||||
Z SummaryStatsReduce<X, Z>::execScalar(const int opNum, const bool biasCorrected, void *x,
|
||||
sd::LongType *xShapeInfo, void *extraParams) {
|
||||
RETURNING_DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(biasCorrected, x, xShapeInfo, extraParams), SUMMARY_STATS_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void SummaryStatsReduce<X, Z>::execScalar(const int opNum, const bool biasCorrected, void *x,
|
||||
sd::LongType *xShapeInfo, void *extraParams, void *z,
|
||||
sd::LongType *zShapeInfo) {
|
||||
DISPATCH_BY_OPNUM_TT(execScalar, PARAMS(biasCorrected, x, xShapeInfo, extraParams, z, zShapeInfo), SUMMARY_STATS_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
void SummaryStatsReduce<X, Z>::exec(int opNum, bool biasCorrected, void *x,
|
||||
sd::LongType *xShapeInfo, void *extraParams, void *z,
|
||||
sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength) {
|
||||
DISPATCH_BY_OPNUM_TT(exec,
|
||||
PARAMS(biasCorrected, x, xShapeInfo, extraParams, z, zShapeInfo, dimension, dimensionLength),
|
||||
SUMMARY_STATS_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SummaryStatsReduce<X, Z>::execScalar(const bool biasCorrected, void *vx, sd::LongType *xShapeInfo,
|
||||
void *vextraParams, void *vz, sd::LongType *zShapeInfo) {
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
z[0] = execScalar<OpType>(biasCorrected, vx, xShapeInfo, vextraParams);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
Z SummaryStatsReduce<X, Z>::execScalar(const bool biasCorrected, void *vx, sd::LongType *xShapeInfo,
|
||||
void *vextraParams) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
|
||||
// Cache shape-related values
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
|
||||
SummaryStatsData<X> startingIndex;
|
||||
startingIndex.initialize();
|
||||
auto length = shape::length(xShapeInfo);
|
||||
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
|
||||
SummaryStatsData<X> curr;
|
||||
curr.initWithValue(x[xOffset]);
|
||||
startingIndex = update(startingIndex, curr, extraParams);
|
||||
}
|
||||
|
||||
return OpType::getValue(biasCorrected, startingIndex);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SummaryStatsReduce<X, Z>::exec(bool biasCorrected, void *vx, sd::LongType *xShapeInfo,
|
||||
void *vextraParams, void *vz, sd::LongType *zShapeInfo,
|
||||
sd::LongType *dimension, sd::LongType dimensionLength) {
|
||||
auto x = reinterpret_cast< X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
auto resultLength = shape::length(zShapeInfo);
|
||||
|
||||
if (sd::ArrayOptions::arrayType(xShapeInfo) == sd::ArrayType::EMPTY) {
|
||||
if (sd::ArrayOptions::arrayType(zShapeInfo) == sd::ArrayType::EMPTY) return;
|
||||
SummaryStatsData<X> comp;
|
||||
comp.initWithValue(x[0]);
|
||||
|
||||
for (sd::LongType i = 0; i < resultLength; i++) z[i] = OpType::getValue(biasCorrected, comp);
|
||||
return;
|
||||
}
|
||||
|
||||
if (shape::isScalar(zShapeInfo)) {
|
||||
z[0] = execScalar<OpType>(biasCorrected, (void *)x, xShapeInfo, extraParams);
|
||||
return;
|
||||
}
|
||||
|
||||
if (dimensionLength < 1) return;
|
||||
|
||||
// When shared_ptr goes out of scope, it deletes the TadPack and invalidates pointers!
|
||||
auto tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(const_cast<sd::LongType*>(xShapeInfo), dimension, dimensionLength);
|
||||
if (resultLength == 1 || dimensionLength == shape::rank(xShapeInfo) || tadPack->numberOfTads() == 1) {
|
||||
z[0] = execScalar<OpType>(biasCorrected, x, xShapeInfo, extraParams);
|
||||
return;
|
||||
}
|
||||
|
||||
auto tadShapeShapeInfo = tadPack->primaryShapeInfo();
|
||||
auto tadLength = shape::length(tadPack->primaryShapeInfo());
|
||||
|
||||
// Cache TAD shape-related values
|
||||
sd::LongType tadRank = shape::rank(tadShapeShapeInfo);
|
||||
sd::LongType *tadShape = shape::shapeOf(tadShapeShapeInfo);
|
||||
sd::LongType *tadStride = shape::stride(tadShapeShapeInfo);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto r = start; r < stop; r++) {
|
||||
auto tadOffsetForBlock = tadPack->primaryOffsets()[r];
|
||||
auto tx = x + tadOffsetForBlock;
|
||||
SummaryStatsData<X> comp;
|
||||
comp.initWithValue(tx[0]);
|
||||
|
||||
for (sd::LongType i = 1; i < tadLength; i++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
INDEX2COORDS(i, tadRank, tadShape, coords);
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(tadRank, tadStride, coords, xOffset);
|
||||
|
||||
SummaryStatsData<X> indexVal2;
|
||||
indexVal2.initWithValue(tx[xOffset]);
|
||||
|
||||
comp = update(comp, OpType::op(indexVal2, extraParams), extraParams);
|
||||
}
|
||||
|
||||
z[r] = OpType::getValue(biasCorrected, comp);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, resultLength, 1);
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( class SummaryStatsReduce, , SD_COMMON_TYPES, SD_FLOAT_TYPES);
|
||||
} // namespace summarystats
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,73 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
// Selective rendering - MUST be included before types.h to define HAS_* flags
|
||||
#include <system/selective_rendering/core.h>
|
||||
#include <system/selective_rendering/bool_types.h>
|
||||
#include <system/selective_rendering/float_types.h>
|
||||
#include <system/selective_rendering/bfloat_types.h>
|
||||
#include <system/selective_rendering/int_types.h>
|
||||
#include <system/selective_rendering/uint_types.h>
|
||||
|
||||
#include <helpers/Loops.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/transform_any.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace transform {
|
||||
|
||||
template <typename X, typename Z>
|
||||
void TransformAny<X, Z>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, void *extraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
DISPATCH_BY_OPNUM_TT(exec, PARAMS(x, xShapeInfo, z, zShapeInfo, extraParams, threadId, numThreads),
|
||||
TRANSFORM_ANY_OPS);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST TransformAny<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, void *vextraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
sd::TransformLoops<X, Z, X>::template loopTransform<OpType>(x, xShapeInfo, z, zShapeInfo, extraParams, threadId,
|
||||
numThreads);
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE(class TransformAny, , SD_NUMERIC_TYPES, SD_NUMERIC_TYPES);
|
||||
// Also instantiate numeric types -> bool for comparison operations
|
||||
BUILD_DOUBLE_TEMPLATE(class TransformAny, , SD_NUMERIC_TYPES, SD_BOOL_TYPES);
|
||||
// And bool -> numeric types for casting operations
|
||||
BUILD_DOUBLE_TEMPLATE(class TransformAny, , SD_BOOL_TYPES, SD_NUMERIC_TYPES);
|
||||
// And bool -> bool for boolean operations
|
||||
BUILD_DOUBLE_TEMPLATE(class TransformAny, , SD_BOOL_TYPES, SD_BOOL_TYPES);
|
||||
|
||||
} // namespace transform
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,56 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <helpers/Loops.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/transform_bool.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace transform {
|
||||
|
||||
template <typename X, typename Z>
|
||||
void TransformBool<X, Z>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, void *extraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
DISPATCH_BY_OPNUM_TT(exec, PARAMS(x, xShapeInfo, z, zShapeInfo, extraParams, threadId, numThreads),
|
||||
TRANSFORM_BOOL_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST TransformBool<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, void *vextraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
sd::TransformLoops<X, Z, X>::template loopTransform<OpType>(x, xShapeInfo, z, zShapeInfo, extraParams, threadId,
|
||||
numThreads);
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( class TransformBool, , SD_COMMON_TYPES, SD_BOOL_TYPES);
|
||||
} // namespace transform
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,57 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <helpers/Loops.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/transform_float.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace transform {
|
||||
template <typename X, typename Z>
|
||||
void TransformFloat<X, Z>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, void *extraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
DISPATCH_BY_OPNUM_TT(exec, PARAMS(x, xShapeInfo, z, zShapeInfo, extraParams, threadId, numThreads),
|
||||
TRANSFORM_FLOAT_OPS);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
template <typename OpType>
|
||||
void SD_HOST TransformFloat<X, Z>::exec(const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, void *vextraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<Z *>(vz);
|
||||
auto extraParams = reinterpret_cast<Z *>(vextraParams);
|
||||
|
||||
sd::TransformLoops<X, Z, Z>::template loopTransform<OpType>(x, xShapeInfo, z, zShapeInfo, extraParams, threadId,
|
||||
numThreads);
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( class TransformFloat, , SD_COMMON_TYPES, SD_FLOAT_TYPES);
|
||||
|
||||
|
||||
} // namespace transform
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,58 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <helpers/Loops.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/transform_same.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace transform {
|
||||
|
||||
template <typename X>
|
||||
void TransformSame<X>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, void *extraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
DISPATCH_BY_OPNUM_T(exec, PARAMS(x, xShapeInfo, z, zShapeInfo, extraParams, threadId, numThreads),
|
||||
TRANSFORM_SAME_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void SD_HOST TransformSame<X>::exec(const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, void *vextraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
|
||||
sd::TransformLoops<X, X, X>::template loopTransform<OpType>(x, xShapeInfo, z, zShapeInfo, extraParams, threadId,
|
||||
numThreads);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( class TransformSame, , SD_NUMERIC_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( class TransformSame, , SD_BOOL_TYPES);
|
||||
|
||||
} // namespace transform
|
||||
} // namespace functions
|
||||
@@ -0,0 +1,55 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <helpers/Loops.h>
|
||||
#include <loops/legacy_ops.h>
|
||||
#include <loops/transform_strict.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/types.h>
|
||||
|
||||
using namespace simdOps;
|
||||
|
||||
namespace functions {
|
||||
namespace transform {
|
||||
|
||||
template <typename X>
|
||||
void TransformStrict<X>::exec(int opNum, const void *x, const sd::LongType *xShapeInfo, void *z,
|
||||
const sd::LongType *zShapeInfo, void *extraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
DISPATCH_BY_OPNUM_T(exec, PARAMS(x, xShapeInfo, z, zShapeInfo, extraParams, threadId, numThreads),
|
||||
TRANSFORM_STRICT_OPS);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
template <typename OpType>
|
||||
void SD_HOST TransformStrict<X>::exec(const void *vx, const sd::LongType *xShapeInfo, void *vz,
|
||||
const sd::LongType *zShapeInfo, void *vextraParams, sd::LongType threadId,
|
||||
sd::LongType numThreads) {
|
||||
auto x = reinterpret_cast<const X *>(vx);
|
||||
auto z = reinterpret_cast<X *>(vz);
|
||||
auto extraParams = reinterpret_cast<X *>(vextraParams);
|
||||
sd::TransformLoops<X, X, X>::template loopTransform<OpType>(x, xShapeInfo, z, zShapeInfo, extraParams, threadId,
|
||||
numThreads);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( class TransformStrict, , SD_FLOAT_TYPES);
|
||||
} // namespace transform
|
||||
} // namespace functions
|
||||
Reference in New Issue
Block a user