chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,197 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author George A. Shulinok <sgazeos@gmail.com>, created on 4/18/2019
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/BarnesHutTsne.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
sd::LongType barnes_row_count(NDArray* rowP, NDArray* colP, sd::LongType N, NDArray& rowCounts) {
|
||||
int* pRowCounts = reinterpret_cast<int*>(rowCounts.buffer());
|
||||
int const* pRows = reinterpret_cast<int const*>(rowP->buffer());
|
||||
int const* pCols = reinterpret_cast<int const*>(colP->buffer());
|
||||
for (sd::LongType n = 0; n < N; n++) {
|
||||
int begin = pRows[n]; //->e<int>(n);
|
||||
int end = pRows[n + 1]; // rowP->e<int>(n + 1);
|
||||
for (int i = begin; i < end; i++) {
|
||||
bool present = false;
|
||||
for (int m = pRows[pCols[i]]; m < pRows[pCols[i] + 1]; m++)
|
||||
if (pCols[m] == n) {
|
||||
present = true;
|
||||
break;
|
||||
}
|
||||
|
||||
++pRowCounts[n];
|
||||
|
||||
if (!present) ++pRowCounts[pCols[i]];
|
||||
}
|
||||
}
|
||||
NDArray numElementsArr = rowCounts.sumNumber();
|
||||
auto numElements = numElementsArr.e<sd::LongType>(0);
|
||||
return numElements;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void barnes_symmetrize_(NDArray* rowP, NDArray* colP, NDArray* valP, sd::LongType N,
|
||||
NDArray* outputRows, NDArray* outputCols, NDArray* outputVals, NDArray* rowCounts) {
|
||||
|
||||
int const* pRows = reinterpret_cast<int const*>(rowP->buffer());
|
||||
int* symRowP = reinterpret_cast<int*>(outputRows->buffer());
|
||||
symRowP[0] = 0;
|
||||
for (sd::LongType n = 0; n < N; n++) symRowP[n + 1] = symRowP[n] + rowCounts->e<int>(n);
|
||||
|
||||
int* symColP = reinterpret_cast<int*>(outputCols->buffer());
|
||||
int const* pCols = reinterpret_cast<int const*>(colP->buffer());
|
||||
T const* pVals = reinterpret_cast<T const*>(valP->buffer());
|
||||
T* pOutput = reinterpret_cast<T*>(outputVals->buffer());
|
||||
std::vector<int> offset(N);
|
||||
|
||||
for (sd::LongType n = 0; n < N; n++) {
|
||||
int begin = pRows[n];
|
||||
int bound = pRows[n + 1];
|
||||
|
||||
for (int i = begin; i < bound; i++) {
|
||||
bool present = false;
|
||||
int colPI = pCols[i];
|
||||
int start = pRows[colPI];
|
||||
int end = pRows[colPI + 1];
|
||||
|
||||
for (int m = start; m < end; m++) {
|
||||
if (pCols[m] == n) {
|
||||
present = true;
|
||||
if (n <= colPI) {
|
||||
symColP[symRowP[n] + offset[n]] = colPI;
|
||||
symColP[symRowP[colPI] + offset[colPI]] = n;
|
||||
pOutput[symRowP[n] + offset[n]] = pVals[i] + pVals[m];
|
||||
pOutput[symRowP[colPI] + offset[colPI]] = pVals[i] + pVals[m];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!present) {
|
||||
symColP[symRowP[n] + offset[n]] = colPI;
|
||||
symColP[symRowP[pCols[i]] + offset[colPI]] = n;
|
||||
pOutput[symRowP[n] + offset[n]] = pVals[i];
|
||||
pOutput[symRowP[colPI] + offset[colPI]] = pVals[i];
|
||||
//}
|
||||
}
|
||||
// Update offsets
|
||||
if (!present || (present && n <= colPI)) {
|
||||
++offset[n];
|
||||
|
||||
if (colPI != n) ++offset[colPI];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
void barnes_symmetrize(NDArray* rowP, NDArray* colP, NDArray* valP, sd::LongType N,
|
||||
NDArray* outputRows, NDArray* outputCols, NDArray* outputVals, NDArray* rowCounts) {
|
||||
// Divide the result by two
|
||||
BUILD_SINGLE_SELECTOR(valP->dataType(), barnes_symmetrize_,
|
||||
(rowP, colP, valP, N, outputRows, outputCols, outputVals, rowCounts), SD_NUMERIC_TYPES);
|
||||
|
||||
*outputVals /= 2.0;
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void barnes_symmetrize_,
|
||||
(NDArray* rowP, NDArray* colP, NDArray* valP, sd::LongType N,
|
||||
NDArray* outputRows, NDArray* outputCols, NDArray* outputVals, NDArray* rowCounts),
|
||||
SD_NUMERIC_TYPES);
|
||||
|
||||
template <typename T>
|
||||
static void barnes_edge_forces_(NDArray* rowP, NDArray * colP, NDArray * valP, int N,
|
||||
NDArray * data, NDArray* output) {
|
||||
T const* dataP = reinterpret_cast<T const*>(data->buffer());
|
||||
T const* vals = reinterpret_cast<T const*>(valP->buffer());
|
||||
T* outputP = reinterpret_cast<T*>(output->buffer());
|
||||
int colCount = data->columns();
|
||||
|
||||
auto rowSize = sizeof(T) * colCount;
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto n = start; n < stop; n++) {
|
||||
int s = rowP->e<int>(n);
|
||||
int end = rowP->e<int>(n + 1);
|
||||
int shift = n * colCount;
|
||||
for (int i = s; i < end; i++) {
|
||||
T const* thisSlice = dataP + colP->e<int>(i) * colCount;
|
||||
T res = static_cast<T>(1);
|
||||
|
||||
for (int k = 0; k < colCount; k++) {
|
||||
auto tempVal = dataP[shift + k] - thisSlice[k]; // thisSlice[k];
|
||||
res += tempVal * tempVal;
|
||||
}
|
||||
|
||||
res = vals[i] / res;
|
||||
for (int k = 0; k < colCount; k++) outputP[shift + k] += ((dataP[shift + k] - thisSlice[k]) * res);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, N);
|
||||
}
|
||||
|
||||
void barnes_edge_forces(NDArray* rowP, NDArray * colP, NDArray * valP, int N, NDArray* output,
|
||||
NDArray& data) {
|
||||
// Loop over all edges in the graph
|
||||
BUILD_SINGLE_SELECTOR(output->dataType(), barnes_edge_forces_, (rowP, colP, valP, N, &data, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void barnes_edge_forces_,
|
||||
(NDArray* rowP, NDArray * colP, NDArray * valP, int N, NDArray * data,
|
||||
NDArray* output),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
template <typename T>
|
||||
static void barnes_gains_(NDArray* input, NDArray* gradX, NDArray* epsilon, NDArray* output) {
|
||||
auto gainsInternal = LAMBDA_TTT(x, grad, eps) {
|
||||
T res = sd::math::sd_sign<T, T>(grad) != sd::math::sd_sign<T, T>(eps) ? x + T(.2) : x * T(.8);
|
||||
if (res < .01) res = static_cast<T>(.01);
|
||||
return res;
|
||||
});
|
||||
|
||||
input->applyTriplewiseLambda<T>(gradX, epsilon, gainsInternal, output);
|
||||
}
|
||||
|
||||
void barnes_gains(NDArray* input, NDArray* gradX, NDArray* epsilon, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), barnes_gains_, (input, gradX, epsilon, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void barnes_gains_, (NDArray * input, NDArray* gradX, NDArray* epsilon, NDArray* output),
|
||||
SD_NUMERIC_TYPES);
|
||||
|
||||
bool cell_contains(NDArray* corner, NDArray* width, NDArray* point, sd::LongType dimension) {
|
||||
auto cornerMinusWidth = *corner - *width;
|
||||
auto cornerPlusWidth = *corner + *width;
|
||||
|
||||
bool result = true;
|
||||
for (sd::LongType i = 0; i < dimension && result; i++) {
|
||||
if (cornerMinusWidth->e<double>(i) > point->e<double>(i)) result = false;
|
||||
else if (cornerPlusWidth->e<double>(i) < point->e<double>(i)) result = false;
|
||||
}
|
||||
|
||||
delete cornerPlusWidth;
|
||||
delete cornerMinusWidth;
|
||||
|
||||
return result;
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1 @@
|
||||
This folder contains OpenMP implementations for operations helpers. Basically suited for homogenous x86-like platforms.
|
||||
@@ -0,0 +1,272 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 19.04.2018
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/activations.h>
|
||||
|
||||
#include <numeric>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
void static _softMaxDerivForVector(sd::LaunchContext* context, const void* input, const sd::LongType* inShapeInfo,
|
||||
void* output) {
|
||||
const T* inBuff = reinterpret_cast<const T*>(input);
|
||||
T* outBuff = reinterpret_cast<T*>(output);
|
||||
|
||||
T max = -DataTypeUtils::max<T>();
|
||||
T sum = static_cast<T>(0.);
|
||||
const sd::LongType length = shape::length(inShapeInfo);
|
||||
|
||||
const sd::LongType rank = shape::rank(inShapeInfo);
|
||||
const sd::LongType* shape = shape::shapeOf(inShapeInfo);
|
||||
const sd::LongType* stride = shape::stride(inShapeInfo);
|
||||
|
||||
LongType coords[SD_MAX_RANK];
|
||||
LongType offset;
|
||||
|
||||
// Find the maximum value in the vector
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
INDEX2COORDS(i, rank, shape, coords);
|
||||
COORDS2INDEX(rank, stride, coords, offset);
|
||||
max = sd::math::sd_max<T>(max, inBuff[offset]);
|
||||
}
|
||||
|
||||
// Calculate exponentials and sum
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
INDEX2COORDS(i, rank, shape, coords);
|
||||
COORDS2INDEX(rank, stride, coords, offset);
|
||||
outBuff[offset] = sd::math::sd_exp<T, T>(inBuff[offset] - max);
|
||||
sum += outBuff[offset];
|
||||
}
|
||||
|
||||
// Compute softmax derivatives
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
INDEX2COORDS(i, rank, shape, coords);
|
||||
COORDS2INDEX(rank, stride, coords, offset);
|
||||
outBuff[offset] /= sum;
|
||||
outBuff[offset] *= (1.f - outBuff[offset]); // derivative
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
void softmaxDerivative(sd::LaunchContext* context, NDArray& input, NDArray& output, const int dimension) {
|
||||
const int rank = input.rankOf();
|
||||
sd::LongType temp;
|
||||
|
||||
if (shape::isCommonVector(input.shapeInfo(), temp)) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), _softMaxDerivForVector,
|
||||
(context, input.buffer(), input.shapeInfo(), output.buffer()), SD_FLOAT_TYPES);
|
||||
} else {
|
||||
std::vector<sd::LongType> dimVec = {dimension};
|
||||
auto maxAlongDim = const_cast<NDArray&>(input).reduceAlongDimension(reduce::Max, &dimVec, true);
|
||||
auto minus = (input - *maxAlongDim);
|
||||
minus->applyTransform(transform::Exp, &output); // output contains exponents temporarily
|
||||
auto sumAlongDim = output.reduceAlongDimension(reduce::Sum, &dimVec, true);
|
||||
output /= *sumAlongDim;
|
||||
auto oneMinus = (1.f - output);
|
||||
output *= *oneMinus; // derivative
|
||||
delete sumAlongDim;
|
||||
delete minus;
|
||||
delete oneMinus;
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
void logSoftMaxForVector_(void const* input, sd::LongType const* inShapeInfo, void* output,
|
||||
sd::LongType const* outShapeInfo) {
|
||||
auto inBuff = reinterpret_cast<T const*>(input);
|
||||
auto outBuff = reinterpret_cast<T*>(output);
|
||||
|
||||
T max = -DataTypeUtils::max<T>();
|
||||
T sum = static_cast<T>(0);
|
||||
|
||||
auto length = shape::length(inShapeInfo);
|
||||
sd::LongType inRank = shape::rank(inShapeInfo);
|
||||
sd::LongType *inShape = shape::shapeOf(inShapeInfo);
|
||||
sd::LongType *inStrides = shape::stride(inShapeInfo);
|
||||
|
||||
sd::LongType *outShape = shape::shapeOf(outShapeInfo);
|
||||
sd::LongType *outStrides = shape::stride(outShapeInfo);
|
||||
sd::LongType outRank = shape::rank(outShapeInfo);
|
||||
sd::LongType inIndices[length];
|
||||
sd::LongType outIndices[length];
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i2 = 0; i2 < length; i2++) {
|
||||
LongType coords[SD_MAX_RANK];
|
||||
sd::LongType idx2;
|
||||
INDEX2COORDS(i2,inRank, inShape, coords);
|
||||
COORDS2INDEX(inRank, inStrides, coords, idx2);
|
||||
max = sd::math::sd_max<T,T>(max, inBuff[idx2]);
|
||||
inIndices[i2] = idx2;
|
||||
}
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i2 = 0; i2 < length; i2++) {
|
||||
LongType coords[SD_MAX_RANK];
|
||||
sd::LongType idx2;
|
||||
INDEX2COORDS(i2,outRank, outShape, coords);
|
||||
COORDS2INDEX(outRank, outStrides, coords, idx2);
|
||||
outBuff[idx2] = sd::math::sd_exp<T, T>(inBuff[inIndices[i2]] - max);
|
||||
sum += outBuff[idx2];
|
||||
}
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = 0; i < length; i++) {
|
||||
outBuff[outIndices[i]] /= sum;
|
||||
outBuff[outIndices[i]] = sd::math::sd_log<T, T>(outBuff[outIndices[i]]);
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
void logSoftMaxForVector(sd::LaunchContext* context, NDArray& input, NDArray& output) {
|
||||
if (!input.isVector() || !output.isVector())
|
||||
THROW_EXCEPTION("ops::helpers::logSoftMaxForVector function input and output arrays must be vectors !");
|
||||
|
||||
auto xType = input.dataType();
|
||||
BUILD_SINGLE_SELECTOR(xType, logSoftMaxForVector_,
|
||||
(input.buffer(), input.shapeInfo(), output.buffer(), output.shapeInfo()), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void prelu(LaunchContext* context, NDArray* input, NDArray* alpha, NDArray* output) {
|
||||
const sd::LongType inputLen = input->lengthOf();
|
||||
const sd::LongType* inputShapeInfo = input->shapeInfo();
|
||||
const sd::LongType* alphaShapeInfo = alpha->shapeInfo();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
// FIXME: double!
|
||||
double x = input->e<double>(i);
|
||||
if (x < 0.0) {
|
||||
// FIXME: double
|
||||
output->p(i, (x * alpha->e<double>(shape::subArrayIndex(i, inputShapeInfo, alphaShapeInfo))));
|
||||
} else
|
||||
output->p(i, x);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, inputLen);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void preluBP(LaunchContext* context, NDArray* input, NDArray* alpha, NDArray* dLdO, NDArray* dLdI,
|
||||
NDArray* dLdA) {
|
||||
const sd::LongType inputLen = input->lengthOf();
|
||||
const sd::LongType* inputShapeInfo = input->shapeInfo();
|
||||
const sd::LongType* alphaShapeInfo = alpha->shapeInfo();
|
||||
float zero = 0.f;
|
||||
dLdA->assign(zero);
|
||||
|
||||
for (sd::LongType i = 0; i < inputLen; ++i) {
|
||||
// FIXME: double
|
||||
double x = input->e<double>(i);
|
||||
double grO = dLdO->isScalar() ? dLdO->e<double>(0) : dLdO->e<double>(i);
|
||||
if (x < 0.0) {
|
||||
sd::LongType alphaInd = shape::subArrayIndex(i, inputShapeInfo, alphaShapeInfo);
|
||||
dLdI->p(i, grO * alpha->e<double>(alphaInd));
|
||||
double prevVal = dLdA->e<double>(alphaInd);
|
||||
prevVal += (grO * x);
|
||||
dLdA->p(alphaInd, prevVal);
|
||||
} else
|
||||
dLdI->p(i, grO);
|
||||
}
|
||||
}
|
||||
|
||||
bool checkAlphaShapeLen(std::vector<sd::LongType> const& expectedShape, sd::LongType shapeLen) {
|
||||
sd::LongType expectedAlphaLen =
|
||||
std::accumulate(expectedShape.cbegin(), expectedShape.cend(), 1, std::multiplies<sd::LongType>());
|
||||
return expectedAlphaLen == shapeLen;
|
||||
}
|
||||
template <typename T>
|
||||
static void thresholdRelu_(NDArray *input, double threshold, NDArray* output) {
|
||||
auto routine = LAMBDA_T(_x, threshold) { return _x > (T)threshold ? _x : (T)0.f; });
|
||||
input->applyLambda<T>(routine, output);
|
||||
}
|
||||
|
||||
void thresholdRelu(LaunchContext* context, NDArray* input, double threshold, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), thresholdRelu_, (input, threshold, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void thresholdReluDerivative_(sd::LaunchContext* context, NDArray* input, double theta, NDArray* dLdO,
|
||||
NDArray* output) {
|
||||
auto derivative = LAMBDA_TT(_x, grO, theta) {
|
||||
if (_x > theta)
|
||||
return grO;
|
||||
else
|
||||
return static_cast<T>(0);
|
||||
});
|
||||
|
||||
input->applyPairwiseLambda<T>(dLdO, derivative, output);
|
||||
}
|
||||
|
||||
void thresholdReluDerivative(sd::LaunchContext* context, NDArray* input, double threshold, NDArray* dLdO,
|
||||
NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), thresholdReluDerivative_, (context, input, threshold, dLdO, output),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
void logSoftmax(LaunchContext* context, NDArray* input, NDArray* output, const int dimension) {
|
||||
const int rank = input->rankOf();
|
||||
|
||||
if (input->isVector()) {
|
||||
if (rank == 1 || input->sizeAt(dimension) != 1) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), logSoftMaxForVector_,
|
||||
(input->buffer(), input->shapeInfo(), output->buffer(), output->shapeInfo()), SD_FLOAT_TYPES);
|
||||
} else
|
||||
*output = 0.;
|
||||
} else {
|
||||
std::vector<sd::LongType> dimVector = {dimension};
|
||||
auto maxAlongDim = input->reduceAlongDimension(reduce::Max, &dimVector, true);
|
||||
auto maxMinusDim = *input - *maxAlongDim;
|
||||
maxMinusDim->applyTransform(transform::Exp, output); // output contains exponents temporarily
|
||||
auto sumAlongDim = output->reduceAlongDimension(reduce::Sum, &dimVector, true);
|
||||
*output /= *sumAlongDim;
|
||||
output->applyTransform(transform::Log, output);
|
||||
delete maxAlongDim;
|
||||
delete maxMinusDim;
|
||||
delete sumAlongDim;
|
||||
}
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void thresholdReluDerivative_,
|
||||
(sd::LaunchContext * context, NDArray* input, double threshold, NDArray* dLdO, NDArray* output),
|
||||
SD_FLOAT_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( void logSoftMaxForVector_,
|
||||
(void const* input, sd::LongType const* inShapeInfo, void* output,
|
||||
sd::LongType const* outShapeInfo),
|
||||
SD_FLOAT_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( void _softMaxDerivForVector,
|
||||
(sd::LaunchContext * context, const void* input, const sd::LongType* inShapeInfo, void* output),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,99 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <ops/declarable/helpers/adjust_hue.h>
|
||||
#if NOT_EXCLUDED(OP_adjust_hue)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void adjustHue_(NDArray *input, NDArray *deltaScalarArr, NDArray *output, const sd::LongType dimC) {
|
||||
const T delta = deltaScalarArr->e<T>(0);
|
||||
const int rank = input->rankOf();
|
||||
|
||||
const T *x = input->bufferAsT<T>();
|
||||
T *z = output->bufferAsT<T>();
|
||||
|
||||
if (dimC == rank - 1 && input->ews() == 1 && output->ews() == 1 && input->ordering() == 'c' &&
|
||||
output->ordering() == 'c') {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
T h, s, v;
|
||||
|
||||
rgbToHsv<T>(x[i], x[i + 1], x[i + 2], h, s, v);
|
||||
|
||||
h += delta;
|
||||
if (h > (T)1)
|
||||
h -= (T)1;
|
||||
else if (h < 0)
|
||||
h += (T)1;
|
||||
|
||||
hsvToRgb<T>(h, s, v, z[i], z[i + 1], z[i + 2]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input->lengthOf(), 3);
|
||||
} else {
|
||||
auto packX = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(), dimC);
|
||||
auto packZ = sd::ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(), dimC);
|
||||
|
||||
const sd::LongType numOfTads = packX->numberOfTads();
|
||||
const sd::LongType xDimCstride = input->stridesOf()[dimC];
|
||||
const sd::LongType zDimCstride = output->stridesOf()[dimC];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
const T *xTad = x + packX->platformOffsets()[i];
|
||||
T *zTad = z + packZ->platformOffsets()[i];
|
||||
|
||||
T h, s, v;
|
||||
|
||||
rgbToHsv<T>(xTad[0], xTad[xDimCstride], xTad[2 * xDimCstride], h, s, v);
|
||||
|
||||
h += delta;
|
||||
if (h > (T)1)
|
||||
h -= (T)1;
|
||||
else if (h < 0)
|
||||
h += (T)1;
|
||||
|
||||
hsvToRgb<T>(h, s, v, zTad[0], zTad[zDimCstride], zTad[2 * zDimCstride]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
}
|
||||
}
|
||||
|
||||
void adjustHue(sd::LaunchContext *context, NDArray *input, NDArray *deltaScalarArr, NDArray *output,
|
||||
const sd::LongType dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), adjustHue_, (input, deltaScalarArr, output, dimC), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,98 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <ops/declarable/helpers/adjust_hue.h>
|
||||
#include <ops/declarable/helpers/adjust_saturation.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void adjustSaturation_(NDArray *input, NDArray *factorScalarArr, NDArray *output, const sd::LongType dimC) {
|
||||
const T factor = factorScalarArr->e<T>(0);
|
||||
const int rank = input->rankOf();
|
||||
|
||||
const T *x = input->bufferAsT<T>();
|
||||
T *z = output->bufferAsT<T>();
|
||||
|
||||
if (dimC == rank - 1 && input->ews() == 1 && output->ews() == 1 && input->ordering() == 'c' &&
|
||||
output->ordering() == 'c') {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
T h, s, v;
|
||||
|
||||
rgbToHsv<T>(x[i], x[i + 1], x[i + 2], h, s, v);
|
||||
|
||||
s *= factor;
|
||||
if (s > 1.f)
|
||||
s = 1.f;
|
||||
else if (s < 0.f)
|
||||
s = 0.f;
|
||||
|
||||
hsvToRgb<T>(h, s, v, z[i], z[i + 1], z[i + 2]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input->lengthOf(), 3);
|
||||
} else {
|
||||
auto packX = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(), dimC);
|
||||
auto packZ = sd::ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(), dimC);
|
||||
|
||||
const sd::LongType numOfTads = packX->numberOfTads();
|
||||
const sd::LongType xDimCstride = input->stridesOf()[dimC];
|
||||
const sd::LongType zDimCstride = output->stridesOf()[dimC];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
const T *xTad = x + packX->platformOffsets()[i];
|
||||
T *zTad = z + packZ->platformOffsets()[i];
|
||||
|
||||
T h, s, v;
|
||||
|
||||
rgbToHsv<T>(xTad[0], xTad[xDimCstride], xTad[2 * xDimCstride], h, s, v);
|
||||
|
||||
s *= factor;
|
||||
if (s > 1.f)
|
||||
s = 1.f;
|
||||
else if (s < 0.f)
|
||||
s = 0.f;
|
||||
|
||||
hsvToRgb<T>(h, s, v, zTad[0], zTad[zDimCstride], zTad[2 * zDimCstride]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
}
|
||||
}
|
||||
|
||||
void adjustSaturation(sd::LaunchContext *context, NDArray *input, NDArray *factorScalarArr, NDArray *output,
|
||||
const sd::LongType dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), adjustSaturation_, (input, factorScalarArr, output, dimC), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,87 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// CPU implementation of assign helper
|
||||
//
|
||||
|
||||
#include <ops/declarable/helpers/assign.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename X, typename Z>
|
||||
static void assignImpl_(NDArray* source, NDArray* target) {
|
||||
auto xBuffer = source->bufferAsT<X>();
|
||||
auto zBuffer = target->bufferAsT<Z>();
|
||||
|
||||
auto xShapeInfo = source->shapeInfo();
|
||||
auto zShapeInfo = target->shapeInfo();
|
||||
|
||||
const int xRank = shape::rank(xShapeInfo);
|
||||
const int zRank = shape::rank(zShapeInfo);
|
||||
const sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
const sd::LongType* zShape = shape::shapeOf(zShapeInfo);
|
||||
const sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
const sd::LongType* zStride = shape::stride(zShapeInfo);
|
||||
const sd::LongType len = target->lengthOf();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType xCoords[SD_MAX_RANK], zCoords[SD_MAX_RANK];
|
||||
sd::LongType xOffset, zOffset;
|
||||
|
||||
INDEX2COORDS(i, zRank, zShape, zCoords);
|
||||
INDEX2COORDS(i, xRank, xShape, xCoords);
|
||||
COORDS2INDEX(xRank, xStride, xCoords, xOffset);
|
||||
COORDS2INDEX(zRank, zStride, zCoords, zOffset);
|
||||
|
||||
zBuffer[zOffset] = static_cast<Z>(xBuffer[xOffset]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, len);
|
||||
}
|
||||
|
||||
void assign(sd::LaunchContext* context, sd::NDArray* target, sd::NDArray* source) {
|
||||
if (target->lengthOf() != source->lengthOf()) {
|
||||
std::string errorMsg = "assign helper: Source and target arrays must have the same length. ";
|
||||
errorMsg += "Source shape: " + ShapeUtils::shapeAsString(source) + ", ";
|
||||
errorMsg += "Target shape: " + ShapeUtils::shapeAsString(target) + ", ";
|
||||
errorMsg += "Source datatype: " + DataTypeUtils::asString(source->dataType()) + ", ";
|
||||
errorMsg += "Target datatype: " + DataTypeUtils::asString(target->dataType());
|
||||
THROW_EXCEPTION(errorMsg.c_str());
|
||||
}
|
||||
|
||||
NDArray::prepareSpecialUse({target}, {source});
|
||||
|
||||
auto xType = source->dataType();
|
||||
auto zType = target->dataType();
|
||||
|
||||
BUILD_DOUBLE_SELECTOR(xType, zType, assignImpl_, (source, target), SD_COMMON_TYPES, SD_COMMON_TYPES);
|
||||
|
||||
NDArray::registerSpecialUse({target}, {source});
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,57 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/axis.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
void adjustAxis(sd::LongType rank, NDArray* axisVector, std::vector<LongType>& output) {
|
||||
if(axisVector->isScalar()) {
|
||||
output.resize(1);
|
||||
auto ca = axisVector->e<sd::LongType>(0);
|
||||
if (ca < 0) // shift values on rank for negative vals
|
||||
ca += rank;
|
||||
output[0] = ca;
|
||||
return;
|
||||
}
|
||||
output.resize(axisVector->lengthOf());
|
||||
axisVector->tickReadDevice(); // mark input as read on device
|
||||
axisVector->syncToHost(); // sync to host
|
||||
for (int e = 0; e < axisVector->lengthOf(); e++) {
|
||||
auto ca = axisVector->e<sd::LongType>(e);
|
||||
if (ca < 0) // shift values on rank for negative vals
|
||||
ca += rank;
|
||||
|
||||
output[e] = ca;
|
||||
}
|
||||
}
|
||||
|
||||
void adjustAxis(sd::LongType rank, std::vector<LongType>& axisVector) {
|
||||
for (size_t e = 0; e < axisVector.size(); e++) {
|
||||
auto a = axisVector[e];
|
||||
if (a < 0) axisVector[e] = a + rank;
|
||||
}
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,202 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/BlasHelper.h>
|
||||
#include <ops/declarable/helpers/batched_gemm.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <types/float16.h>
|
||||
#include <indexing/NDIndexUtils.h>
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_batched_gemm)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
|
||||
void bgemm(NDArray *a, NDArray *b, NDArray *c, NDArray *alphas, NDArray *betas,
|
||||
int transA, int transB, int M, int N, int K, int lda, int ldb, int ldc,
|
||||
NDArray *all) {
|
||||
NDArray *allIndex = nullptr;
|
||||
if(all != nullptr)
|
||||
allIndex = all;
|
||||
else {
|
||||
NDArray *allLocal = NDIndexUtils::createAll();
|
||||
allIndex = allLocal;
|
||||
}
|
||||
|
||||
int batchSize = a->sizeAt(0);
|
||||
std::vector<NDArray *>inputs;
|
||||
std::vector<NDArray *> bInputs;
|
||||
std::vector<NDArray *> outputs;
|
||||
|
||||
ops::create_view createView;
|
||||
|
||||
//divide by 2: queries and keys
|
||||
for(int i = 0; i < batchSize; i++) {
|
||||
auto point = NDIndexUtils::createPoint(i);
|
||||
auto aSlice = createView.evaluate({a,point,allIndex,allIndex},{},{});
|
||||
auto bSlice = createView.evaluate({b,point,allIndex,allIndex},{},{});
|
||||
auto outSlice = createView.evaluate({c,point,allIndex,allIndex},{},{});
|
||||
inputs.push_back(aSlice.at(0));
|
||||
bInputs.push_back(bSlice.at(0));
|
||||
outputs.push_back(outSlice.at(0));
|
||||
delete point;
|
||||
}
|
||||
|
||||
delete allIndex;
|
||||
|
||||
|
||||
|
||||
bgemm(inputs, bInputs,outputs,alphas,betas,transA,transB,M,N,K,lda,ldb,ldc);
|
||||
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void bgemm_( std::vector<NDArray *> &vA, std::vector<NDArray *> &vB, std::vector<NDArray *> &vC,
|
||||
NDArray *alphas, NDArray *betas, int transA, int transB, int M, int N, int K,
|
||||
int lda, int ldb, int ldc) {
|
||||
int batchSize = vA.size();
|
||||
|
||||
|
||||
|
||||
// Use batched BLAS only when: 1) batched GEMM is available AND 2) BLAS is enabled
|
||||
// Previously used || which incorrectly entered BLAS path when BLAS was disabled
|
||||
if (BlasHelper::getInstance().hasBatchedGEMM<T>() && Environment::getInstance().isEnableBlas()) {
|
||||
auto arr = vA.at(0);
|
||||
CBLAS_TRANSPOSE *tA, *tB;
|
||||
int *tM, *tN, *tK, *tldA, *tldB, *tldC, *tsize;
|
||||
// mkl requires mnk etc as arrays, cuda doesn't
|
||||
ALLOCATE(tA, arr->getContext()->getWorkspace(), batchSize, CBLAS_TRANSPOSE);
|
||||
ALLOCATE(tB, arr->getContext()->getWorkspace(), batchSize, CBLAS_TRANSPOSE);
|
||||
ALLOCATE(tM, arr->getContext()->getWorkspace(), batchSize, int);
|
||||
ALLOCATE(tN, arr->getContext()->getWorkspace(), batchSize, int);
|
||||
ALLOCATE(tK, arr->getContext()->getWorkspace(), batchSize, int);
|
||||
ALLOCATE(tldA, arr->getContext()->getWorkspace(), batchSize, int);
|
||||
ALLOCATE(tldB, arr->getContext()->getWorkspace(), batchSize, int);
|
||||
ALLOCATE(tldC, arr->getContext()->getWorkspace(), batchSize, int);
|
||||
ALLOCATE(tsize, arr->getContext()->getWorkspace(), batchSize, int);
|
||||
|
||||
shape::fill(tA, (CBLAS_TRANSPOSE)transA, batchSize);
|
||||
shape::fill(tB, (CBLAS_TRANSPOSE)transB, batchSize);
|
||||
|
||||
shape::fill(tM, M, batchSize);
|
||||
shape::fill(tN, N, batchSize);
|
||||
shape::fill(tK, K, batchSize);
|
||||
shape::fill(tldA, lda, batchSize);
|
||||
shape::fill(tldB, ldb, batchSize);
|
||||
shape::fill(tldC, ldc, batchSize);
|
||||
shape::fill(tsize, 1, batchSize);
|
||||
|
||||
std::vector<T *> buffersA;
|
||||
std::vector<T *> buffersB;
|
||||
std::vector<T *> buffersC;
|
||||
|
||||
|
||||
|
||||
for (int e = 0; e < batchSize; e++) {
|
||||
buffersA.push_back(reinterpret_cast<T *>(vA[e]->buffer()));
|
||||
buffersB.push_back(reinterpret_cast<T *>(vB[e]->buffer()));
|
||||
buffersC.push_back(reinterpret_cast<T *>(vC[e]->buffer()));
|
||||
}
|
||||
|
||||
// Acquire BLAS lock to prevent OpenBLAS TLS corruption and race conditions
|
||||
auto blasLock = BlasHelper::getInstance().lockBlas();
|
||||
|
||||
// Inside BLAS block, only check type - BLAS enablement was already verified in outer condition
|
||||
if (std::is_same<T, double>::value) {
|
||||
BlasHelper::getInstance().dgemmBatched()(CblasColMajor, tA, tB, tM, tN, tK, (double *)alphas->buffer(),
|
||||
(double **)buffersA.data(), tldA, (double **)buffersB.data(), tldB,
|
||||
(double *)betas->buffer(), (double **)buffersC.data(), tldC, vA.size(),
|
||||
tsize);
|
||||
} else if (std::is_same<T, float>::value) {
|
||||
BlasHelper::getInstance().sgemmBatched()(
|
||||
CblasColMajor, tA, tB, tM, tN, tK, (float *)alphas->buffer(), (float **)buffersA.data(), tldA,
|
||||
(float **)buffersB.data(), tldB, (float *)betas->buffer(), (float **)buffersC.data(), tldC, vA.size(), tsize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
// release temporary arrays
|
||||
RELEASE(tA, arr->getContext()->getWorkspace());
|
||||
RELEASE(tB, arr->getContext()->getWorkspace());
|
||||
RELEASE(tM, arr->getContext()->getWorkspace());
|
||||
RELEASE(tN, arr->getContext()->getWorkspace());
|
||||
RELEASE(tK, arr->getContext()->getWorkspace());
|
||||
RELEASE(tldA, arr->getContext()->getWorkspace());
|
||||
RELEASE(tldB, arr->getContext()->getWorkspace());
|
||||
RELEASE(tldC, arr->getContext()->getWorkspace());
|
||||
RELEASE(tsize, arr->getContext()->getWorkspace());
|
||||
} else {
|
||||
|
||||
CBLAS_TRANSPOSE tA = (CBLAS_TRANSPOSE)transA;
|
||||
CBLAS_TRANSPOSE tB = (CBLAS_TRANSPOSE)transB;
|
||||
int vaSize = vA.size();
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto p = start; p < stop; p++) {
|
||||
auto A = reinterpret_cast<T *>(vA.at(p)->buffer());
|
||||
auto B = reinterpret_cast<T *>(vB.at(p)->buffer());
|
||||
auto C = reinterpret_cast<T *>(vC.at(p)->buffer());
|
||||
// Handle scalar, single-element, or empty arrays (use defaults for empty)
|
||||
auto alpha = (alphas->isScalar() || alphas->lengthOf() <= 1)
|
||||
? (alphas->lengthOf() > 0 ? alphas->e<T>(0) : static_cast<T>(1))
|
||||
: alphas->e<T>(p);
|
||||
auto beta = (betas->isScalar() || betas->lengthOf() <= 1)
|
||||
? (betas->lengthOf() > 0 ? betas->e<T>(0) : static_cast<T>(0))
|
||||
: betas->e<T>(p);
|
||||
for (int m = 0; m < M; m++) {
|
||||
for (int n = 0; n < N; n++) {
|
||||
T c_mnp = static_cast<T>(0);
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int k = 0; k < K; k++) {
|
||||
c_mnp += A[tA == CblasNoTrans ? (m + k * lda) : (m * lda + k)] *
|
||||
B[tB == CblasNoTrans ? (k + n * ldb) : (k * ldb + n)];
|
||||
}
|
||||
C[m + n * ldc] = alpha * c_mnp + beta * C[m + n * ldc];
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, vaSize);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void bgemm( std::vector<NDArray *> &vA, std::vector<NDArray *> &vB, std::vector<NDArray *> &vC,
|
||||
NDArray *alphas, NDArray *betas, int transA, int transB, int M, int N, int K, int lda,
|
||||
int ldb, int ldc) {
|
||||
auto xType = vA.at(0)->dataType();
|
||||
BUILD_SINGLE_SELECTOR(xType, bgemm_, (vA, vB, vC, alphas, betas, transA, transB, M, N, K, lda, ldb, ldc),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void bgemm_,
|
||||
( std::vector<NDArray *> &vA, std::vector<NDArray *> &vB, std::vector<NDArray *> &vC,
|
||||
NDArray *alphas, NDArray *betas, int transA, int transB, int M, int N, int K,
|
||||
int lda, int ldb, int ldc),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,256 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/OmpLaunchHelper.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/batchnorm.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_batchnorm)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void batchnorm_(NDArray* input, NDArray* mean, NDArray* variance, NDArray* gamma,
|
||||
NDArray* beta, NDArray* output, const std::vector<LongType>& axes, const double epsilon) {
|
||||
// formula: output = gamma * ((input - mean) / sqrt(variance + epsilon)) + beta
|
||||
|
||||
const T* x = input->bufferAsT<T>();
|
||||
T* z = output->bufferAsT<T>();
|
||||
const T* m = mean->bufferAsT<T>();
|
||||
const T* v = variance->bufferAsT<T>();
|
||||
const T* g = gamma == nullptr ? nullptr : gamma->bufferAsT<T>();
|
||||
const T* b = beta == nullptr ? nullptr : beta->bufferAsT<T>();
|
||||
|
||||
const bool xzSameOffset = shape::haveSameShapeAndStrides(input->shapeInfo(), output->shapeInfo());
|
||||
|
||||
bool paramSameOffset = shape::haveSameShapeAndStrides(mean->shapeInfo(), variance->shapeInfo());
|
||||
if (paramSameOffset && gamma != nullptr)
|
||||
paramSameOffset &= shape::haveSameShapeAndStrides(mean->shapeInfo(), gamma->shapeInfo());
|
||||
if (paramSameOffset && beta != nullptr)
|
||||
paramSameOffset &= shape::haveSameShapeAndStrides(mean->shapeInfo(), beta->shapeInfo());
|
||||
|
||||
const sd::LongType lenBig = input->lengthOf();
|
||||
const sd::LongType lenSmall = mean->lengthOf();
|
||||
|
||||
const sd::LongType steps = lenBig / lenSmall;
|
||||
std::vector<sd::LongType> *dimsToExclude = ShapeUtils::evalDimsToExclude(input->rankOf(), axes.size(),axes.data());
|
||||
|
||||
OmpLaunchHelper info(lenBig, lenSmall);
|
||||
|
||||
auto func = PRAGMA_THREADS_DO {
|
||||
sd::LongType* xOffsets = new sd::LongType[steps];
|
||||
sd::LongType* zOffsets = xzSameOffset ? xOffsets : new sd::LongType[steps];
|
||||
sd::LongType * auxBuff = new sd::LongType [2 * input->rankOf()];
|
||||
|
||||
sd::LongType meanRank = shape::rank(mean->shapeInfo());
|
||||
sd::LongType varianceRank = shape::rank(variance->shapeInfo());
|
||||
sd::LongType gammaRank = gamma == nullptr ? 0 : shape::rank(gamma->shapeInfo());
|
||||
sd::LongType betaRank = beta == nullptr ? 0 : shape::rank(beta->shapeInfo());
|
||||
sd::LongType *meanShape = shape::shapeOf(mean->shapeInfo());
|
||||
sd::LongType *varianceShape = shape::shapeOf(variance->shapeInfo());
|
||||
sd::LongType *gammaShape = gamma == nullptr ? nullptr : shape::shapeOf(gamma->shapeInfo());
|
||||
sd::LongType *betaShape = beta == nullptr ? nullptr : shape::shapeOf(beta->shapeInfo());
|
||||
sd::LongType *meanStride = shape::stride(mean->shapeInfo());
|
||||
sd::LongType *varianceStride = shape::stride(variance->shapeInfo());
|
||||
sd::LongType *gammaStride = gamma == nullptr ? nullptr : shape::stride(gamma->shapeInfo());
|
||||
sd::LongType *betaStride = beta == nullptr ? nullptr : shape::stride(beta->shapeInfo());
|
||||
|
||||
|
||||
for (sd::LongType j = 0; j < lenSmall; ++j) {
|
||||
const bool isOwner = (j < info._numThreads) ? thread_id == j : thread_id == (j % info._numThreads);
|
||||
|
||||
if (!isOwner) continue;
|
||||
|
||||
LongType meanCoords[SD_MAX_RANK];
|
||||
LongType varCoords[SD_MAX_RANK];
|
||||
LongType gammaCoords[SD_MAX_RANK];
|
||||
LongType betaCoords[SD_MAX_RANK];
|
||||
LongType meanOffset;
|
||||
LongType varOffset;
|
||||
LongType gammaOffset;
|
||||
LongType betaOffset;
|
||||
|
||||
INDEX2COORDS(j, meanRank, meanShape, meanCoords);
|
||||
COORDS2INDEX(meanRank, meanStride, meanCoords, meanOffset);
|
||||
varOffset = paramSameOffset ? meanOffset : 0;
|
||||
if (!paramSameOffset) {
|
||||
INDEX2COORDS(j, varianceRank, varianceShape, varCoords);
|
||||
COORDS2INDEX(varianceRank, varianceStride, varCoords, varOffset);
|
||||
}
|
||||
|
||||
const auto meanVal = m[meanOffset];
|
||||
auto sigmaInvGam = static_cast<T>(1) / sd::math::sd_sqrt<T, T>(v[varOffset] + epsilon);
|
||||
|
||||
if (g != nullptr) {
|
||||
gammaOffset = paramSameOffset ? meanOffset : 0;
|
||||
if (!paramSameOffset) {
|
||||
INDEX2COORDS(j, gammaRank, gammaShape, gammaCoords);
|
||||
COORDS2INDEX(gammaRank, gammaStride, gammaCoords, gammaOffset);
|
||||
}
|
||||
sigmaInvGam *= g[gammaOffset];
|
||||
}
|
||||
|
||||
T betaVal = static_cast<T>(0);
|
||||
if (b != nullptr) {
|
||||
betaOffset = paramSameOffset ? meanOffset : 0;
|
||||
if (!paramSameOffset) {
|
||||
INDEX2COORDS(j, betaRank, betaShape, betaCoords);
|
||||
COORDS2INDEX(betaRank, betaStride, betaCoords, betaOffset);
|
||||
}
|
||||
betaVal = b[betaOffset];
|
||||
}
|
||||
|
||||
// calculate offsets for input and output
|
||||
shape::outerArrayOffsets(xOffsets, j, input->shapeInfo(), mean->shapeInfo(), auxBuff, dimsToExclude->data());
|
||||
if (!xzSameOffset)
|
||||
shape::outerArrayOffsets(zOffsets, j, output->shapeInfo(), mean->shapeInfo(), auxBuff, dimsToExclude->data());
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType i = 0; i < steps; ++i) z[zOffsets[i]] = (x[xOffsets[i]] - meanVal) * sigmaInvGam + betaVal;
|
||||
}
|
||||
|
||||
delete[] auxBuff;
|
||||
delete[] xOffsets;
|
||||
if (!xzSameOffset) delete[] zOffsets;
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_do(func, info._numThreads);
|
||||
|
||||
delete dimsToExclude;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void batchnorm2_(NDArray* input, NDArray* mean, NDArray* variance, NDArray* gamma,
|
||||
NDArray* beta, NDArray* output, const std::vector<int>& axes, const double epsilon) {
|
||||
// formula: output = gamma * ((input - mean) / sqrt(variance + epsilon)) + beta
|
||||
|
||||
const auto x = input->bufferAsT<T>();
|
||||
auto z = output->bufferAsT<T>();
|
||||
const auto m = mean->bufferAsT<T>();
|
||||
const auto v = variance->bufferAsT<T>();
|
||||
const auto g = gamma == nullptr ? nullptr : gamma->bufferAsT<T>();
|
||||
const auto b = beta == nullptr ? nullptr : beta->bufferAsT<T>();
|
||||
|
||||
// xRank == zRank, minRank = meanRank = varianceRank = gammaRank = betaRank
|
||||
const sd::LongType xRank = input->rankOf();
|
||||
const sd::LongType minRank = mean->rankOf();
|
||||
const sd::LongType numAxes = axes.size();
|
||||
|
||||
const bool xzSameOffset = shape::haveSameShapeAndStrides(input->shapeInfo(), output->shapeInfo());
|
||||
|
||||
bool paramSameOffset = shape::haveSameShapeAndStrides(mean->shapeInfo(), variance->shapeInfo());
|
||||
if (paramSameOffset && gamma != nullptr)
|
||||
paramSameOffset &= shape::haveSameShapeAndStrides(mean->shapeInfo(), gamma->shapeInfo());
|
||||
if (paramSameOffset && beta != nullptr)
|
||||
paramSameOffset &= shape::haveSameShapeAndStrides(mean->shapeInfo(), beta->shapeInfo());
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType xzCoords[SD_MAX_RANK], minCoords[SD_MAX_RANK];
|
||||
|
||||
for (sd::LongType i = 0, j = 0; i < xRank; ++i)
|
||||
if (j < numAxes && i != axes[j])
|
||||
minCoords[i] = 0;
|
||||
else
|
||||
++j;
|
||||
|
||||
sd::LongType *inputShape = input->shapeOf();
|
||||
sd::LongType *inputStride = input->stridesOf();
|
||||
|
||||
sd::LongType *outputShape = output->shapeOf();
|
||||
sd::LongType *outputStride = output->stridesOf();
|
||||
|
||||
sd::LongType *meanShape = mean->shapeOf();
|
||||
sd::LongType *varianceShape = variance->shapeOf();
|
||||
sd::LongType *gammaShape = gamma == nullptr ? nullptr : gamma->shapeOf();
|
||||
sd::LongType *betaShape = beta == nullptr ? nullptr : beta->shapeOf();
|
||||
|
||||
sd::LongType *meanStride = mean->stridesOf();
|
||||
sd::LongType *varianceStride = variance->stridesOf();
|
||||
|
||||
sd::LongType *gammaStride = gamma == nullptr ? nullptr : gamma->stridesOf();
|
||||
sd::LongType *betaStride = beta == nullptr ? nullptr : beta->stridesOf();
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, xRank, inputShape, xzCoords);
|
||||
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(xRank, inputStride, xzCoords, xOffset);
|
||||
sd::LongType zOffset = xzSameOffset ? xOffset : 0;
|
||||
if (!xzSameOffset) {
|
||||
COORDS2INDEX(xRank,outputStride, xzCoords, zOffset);
|
||||
}
|
||||
|
||||
if (minRank == xRank) {
|
||||
for (sd::LongType j = 0; j < numAxes; ++j) minCoords[axes[j]] = xzCoords[axes[j]];
|
||||
} else // minRank = numAxes = 1 in this case
|
||||
minCoords[0] = xzCoords[axes[0]];
|
||||
|
||||
sd::LongType meanOffset, varianceOffset;
|
||||
COORDS2INDEX(minRank, meanStride, minCoords, meanOffset);
|
||||
varianceOffset = paramSameOffset ? meanOffset : 0;
|
||||
if (!paramSameOffset) {
|
||||
COORDS2INDEX(minRank, varianceStride, minCoords, varianceOffset);
|
||||
}
|
||||
|
||||
T sigmaInvGam = 1. / sd::math::sd_sqrt<T, T>(v[varianceOffset] + epsilon);
|
||||
|
||||
if (g != nullptr) {
|
||||
sd::LongType gammaOffset = paramSameOffset ? meanOffset : 0;
|
||||
if (!paramSameOffset) {
|
||||
COORDS2INDEX(minRank,gammaStride, minCoords, gammaOffset);
|
||||
}
|
||||
sigmaInvGam *= g[gammaOffset];
|
||||
}
|
||||
|
||||
z[zOffset] = (x[xOffset] - m[meanOffset]) * sigmaInvGam;
|
||||
|
||||
if (b != nullptr) {
|
||||
sd::LongType betaOffset = paramSameOffset ? meanOffset : 0;
|
||||
if (!paramSameOffset) {
|
||||
COORDS2INDEX(minRank,betaStride, minCoords, betaOffset);
|
||||
}
|
||||
z[zOffset] += b[betaOffset];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input->lengthOf());
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void batchnorm(NDArray* input, NDArray* mean, NDArray* variance, NDArray* gamma,
|
||||
NDArray* beta, NDArray* output, const std::vector<LongType>& axes, const double epsilon) {
|
||||
// batchnorm2_ is still slower ?
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), batchnorm_, (input, mean, variance, gamma, beta, output, axes, epsilon),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void batchnorm_,
|
||||
(NDArray* input, NDArray* mean, NDArray* variance, NDArray* gamma,
|
||||
NDArray* beta, NDArray* output, const std::vector<sd::LongType>& axes, const double epsilon),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,126 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by Yurii Shyrma on 11.12.2017
|
||||
//
|
||||
#include <array/DataTypeUtils.h>
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/betaInc.h>
|
||||
|
||||
#include <cmath>
|
||||
#if NOT_EXCLUDED(OP_betainc)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// modified Lentz’s algorithm for continued fractions,
|
||||
// reference: Lentz, W.J. 1976, “Generating Bessel Functions in Mie Scattering Calculations Using Continued Fractions”
|
||||
|
||||
template <typename T>
|
||||
static T continuedFraction(const T a, const T b, const T x) {
|
||||
const T min = DataTypeUtils::min_positive<T>() / static_cast<T>(DataTypeUtils::eps<T>());
|
||||
const T aPlusb = a + b;
|
||||
T val, aPlus2i;
|
||||
|
||||
T t2 = static_cast<T>(1);
|
||||
T t1 = static_cast<T>(1) - aPlusb * x / (a + static_cast<T>(1));
|
||||
if (math::sd_abs<T,T>(t1) < min) t1 = min;
|
||||
t1 = static_cast<T>(1) / t1;
|
||||
T result = t1;
|
||||
|
||||
for (sd::LongType i = 1; i <= maxIter; ++i) {
|
||||
aPlus2i = a + static_cast<T>(2 * i);
|
||||
val = i * (b - i) * x / ((aPlus2i - static_cast<T>(1)) * aPlus2i);
|
||||
// t1
|
||||
t1 = static_cast<T>(1) + val * t1;
|
||||
if (math::sd_abs<T,T>(t1) < min) t1 = min;
|
||||
t1 = static_cast<T>(1) / t1;
|
||||
// t2
|
||||
t2 = static_cast<T>(1) + val / t2;
|
||||
if (math::sd_abs<T,T>(t2) < min) t2 = min;
|
||||
// result
|
||||
result *= t2 * t1;
|
||||
val = -(a + i) * (aPlusb + i) * x / ((aPlus2i + static_cast<T>(1)) * aPlus2i);
|
||||
// t1
|
||||
t1 = static_cast<T>(1) + val * t1;
|
||||
if (math::sd_abs<T,T>(t1) < min) t1 = min;
|
||||
t1 = static_cast<T>(1) / t1;
|
||||
// t2
|
||||
t2 = static_cast<T>(1) + val / t2;
|
||||
if (math::sd_abs<T,T>(t2) < min) t2 = min;
|
||||
// result
|
||||
val = t2 * t1;
|
||||
result *= val;
|
||||
|
||||
// condition to stop loop
|
||||
if (math::sd_abs<T,T>(val - static_cast<T>(1)) <= DataTypeUtils::eps<T>()) return result;
|
||||
}
|
||||
|
||||
return DataTypeUtils::infOrMax<T>(); // no convergence, more iterations is required, return infinity
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// evaluates incomplete beta function for positive a and b, and x between 0 and 1.
|
||||
template <typename T>
|
||||
static T betaIncCore(T a, T b, T x) {
|
||||
// t^{n-1} * (1 - t)^{n-1} is symmetric function with respect to x = 0.5
|
||||
if (a == b && x == static_cast<T>(0.5)) return static_cast<T>(0.5);
|
||||
|
||||
if (x == static_cast<T>(0) || x == static_cast<T>(1)) return x;
|
||||
|
||||
const T gammaPart = static_cast<T>(lgamma(a) + lgamma(b) - lgamma(a + b));
|
||||
const T front = math::sd_exp<T, T>(math::sd_log<T, T>(x) * a + math::sd_log<T, T>(1.f - x) * b - gammaPart);
|
||||
|
||||
if (x <= (a + static_cast<T>(1)) / (a + b + static_cast<T>(2)))
|
||||
return front * continuedFraction<T>(a, b, x) / a;
|
||||
else // symmetry relation
|
||||
return static_cast<T>(1) - front * continuedFraction<T>(b, a, static_cast<T>(1) - x) / b;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void betaIncForArray(sd::LaunchContext* context, NDArray& a, NDArray& b, NDArray& x,
|
||||
NDArray& output) {
|
||||
int xLen = x.lengthOf();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) output.r<T>(i) = betaIncCore<T>(a.t<T>(i), b.t<T>(i), x.t<T>(i));
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, xLen);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// overload betaInc for arrays, shapes of a, b and x must be the same !!!
|
||||
void betaInc(sd::LaunchContext* context, NDArray& a, NDArray& b, NDArray& x, NDArray& output) {
|
||||
auto xType = a.dataType();
|
||||
BUILD_SINGLE_SELECTOR(xType, betaIncForArray, (context, a, b, x, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void betaIncForArray,
|
||||
(sd::LaunchContext * context, NDArray& a, NDArray& b, NDArray& x,
|
||||
NDArray& output),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,234 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
// @author sgazeos@gmail.com
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_clip)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void clipByNorm(LaunchContext* context, NDArray* input, NDArray* output, const std::vector<LongType>& dimensions,
|
||||
NDArray* clipNorm, const bool isInplace, const bool useAverage) {
|
||||
NDArray* z = nullptr;
|
||||
|
||||
if (isInplace) {
|
||||
z = input;
|
||||
} else {
|
||||
output->assign(input);
|
||||
z = output;
|
||||
}
|
||||
|
||||
if (dimensions.empty()) {
|
||||
std::vector<sd::LongType> emptyVec = {};
|
||||
|
||||
NDArray *norm2Result = z->reduceAlongDimension(reduce::Norm2, &emptyVec);
|
||||
if (useAverage) {
|
||||
NDArray *divResult = (*norm2Result) / z->lengthOf();
|
||||
if (divResult->e<float>(0) > clipNorm->e<float>(0)) {
|
||||
NDArray *clipDivResult = (*clipNorm) / (*divResult);
|
||||
*z *= (*clipDivResult);
|
||||
delete clipDivResult;
|
||||
}
|
||||
delete divResult;
|
||||
} else {
|
||||
if (norm2Result->e<float>(0) > clipNorm->e<float>(0)) {
|
||||
NDArray *clipDivResult = (*clipNorm) / (*norm2Result);
|
||||
*z *= (*clipDivResult);
|
||||
delete clipDivResult;
|
||||
}
|
||||
}
|
||||
delete norm2Result;
|
||||
} else {
|
||||
auto listOfSubArrs = z->allTensorsAlongDimension(dimensions);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
std::vector<sd::LongType> emptyVec = {};
|
||||
NDArray *norm2Result = listOfSubArrs.at(i)->reduceAlongDimension(reduce::Norm2, &emptyVec);
|
||||
if (useAverage) {
|
||||
NDArray *divResult = (*norm2Result) / listOfSubArrs.at(i)->lengthOf();
|
||||
if (divResult->e<float>(0) > clipNorm->e<float>(0)) {
|
||||
NDArray *clipDivResult = (*clipNorm) / (*divResult);
|
||||
*listOfSubArrs.at(i) *= (*clipDivResult);
|
||||
delete clipDivResult;
|
||||
}
|
||||
delete divResult;
|
||||
} else {
|
||||
if (norm2Result->e<float>(0) > clipNorm->e<float>(0)) {
|
||||
NDArray *clipDivResult = (*clipNorm) / (*norm2Result);
|
||||
*listOfSubArrs.at(i) *= (*clipDivResult);
|
||||
delete clipDivResult;
|
||||
}
|
||||
}
|
||||
delete norm2Result;
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, 0, listOfSubArrs.size());
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void clipByNormBp_(NDArray *input, NDArray *gradO, NDArray *gradI,
|
||||
const std::vector<LongType>& dimensions, NDArray *clipNorm, const bool useAverage) {
|
||||
const int rank = input->rankOf();
|
||||
|
||||
auto *norm2Ptr = input->reduceAlongDimension(reduce::Norm2, &dimensions);
|
||||
auto norm2 = *norm2Ptr;
|
||||
auto *sumsPtr = input->reduceAlongDimension(reduce::Sum, &dimensions);
|
||||
auto sums = *sumsPtr;
|
||||
|
||||
if (norm2.lengthOf() == 1) {
|
||||
const T norm = useAverage ? norm2.e<T>(0) / input->lengthOf() : norm2.e<T>(0);
|
||||
|
||||
auto clipVal = clipNorm->e<T>(0);
|
||||
|
||||
if (norm > clipVal) {
|
||||
const T sum = sums.e<T>(0); // reduce to scalar
|
||||
const T factor1 = clipVal / norm;
|
||||
const T factor2 = static_cast<T>(1.f) / (norm * norm); // 1 / (norm*norm*norm)
|
||||
|
||||
auto lambda = LAMBDA_TT(x, y, sum, factor1, factor2) {
|
||||
return factor1 * y * (static_cast<T>(1.f) - factor2 * x * sum);
|
||||
});
|
||||
|
||||
input->applyPairwiseLambda<T>(gradO, lambda, gradI);
|
||||
} else
|
||||
gradI->assign(gradO);
|
||||
} else {
|
||||
auto gradISubArrs = gradI->allTensorsAlongDimension({dimensions});
|
||||
auto gradOSubArrs = gradO->allTensorsAlongDimension({dimensions});
|
||||
auto inputSubArrs = input->allTensorsAlongDimension({dimensions});
|
||||
|
||||
auto clipVal = clipNorm->e<T>(0);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto gradOSubArr = gradOSubArrs.at(i);
|
||||
auto gradISubArr = gradISubArrs.at(i);
|
||||
|
||||
const T norm = useAverage ? norm2.e<T>(i) / gradISubArr->lengthOf() : norm2.e<T>(i);
|
||||
|
||||
if (norm > clipVal) {
|
||||
auto inputSubArr = inputSubArrs.at(i);
|
||||
|
||||
const T sum = sums.e<T>(i); // reduce to scalar
|
||||
const T factor1 = clipVal / norm;
|
||||
const T factor2 = static_cast<T>(1.f) / (norm * norm); // 1 / (norm*norm*norm)
|
||||
|
||||
auto lambda = LAMBDA_TT(x, y, sum, factor1, factor2) {
|
||||
return factor1 * y * (static_cast<T>(1.f) - factor2 * x * sum);
|
||||
});
|
||||
|
||||
inputSubArr->applyPairwiseLambda<T>(gradOSubArr, lambda, gradISubArr);
|
||||
} else
|
||||
gradISubArr->assign(gradOSubArr);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, 0, gradISubArrs.size());
|
||||
}
|
||||
|
||||
delete norm2Ptr;
|
||||
delete sumsPtr;
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE(void clipByNormBp_,
|
||||
(NDArray *input, NDArray *gradO, NDArray *gradI, const std::vector<sd::LongType>& dimensions,
|
||||
NDArray *clipNorm, const bool useAverage),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void clipByNormBp(sd::LaunchContext* context, NDArray *input, NDArray *gradO, NDArray *gradI,
|
||||
const std::vector<LongType>& dimensions, NDArray* clipNorm, const bool useAverage) {
|
||||
BUILD_SINGLE_SELECTOR(gradI->dataType(), clipByNormBp_, (input, gradO, gradI, dimensions, clipNorm, useAverage),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void clipByGlobalNorm_(std::vector<NDArray*>& inputs, double clipNorm, sd::memory::Workspace* workspace,
|
||||
std::vector<NDArray*>& outputs, bool isInplace) {
|
||||
T globalNorm = static_cast<T>(0);
|
||||
for (size_t i = 0; i < inputs.size(); i++) {
|
||||
auto input = inputs[i];
|
||||
auto* l2norm = input->reduceNumber(reduce::Norm2);
|
||||
T normVal = l2norm->t<T>(0);
|
||||
globalNorm += normVal * normVal;
|
||||
delete l2norm;
|
||||
}
|
||||
|
||||
auto normS = sd::math::sd_sqrt<T, T>(globalNorm);
|
||||
outputs[inputs.size()]->p(0, normS);
|
||||
|
||||
const T factor = clipNorm / normS;
|
||||
|
||||
for (size_t e = 0; e < inputs.size(); e++) {
|
||||
// all-reduce
|
||||
auto input = inputs[e];
|
||||
auto output = outputs[e];
|
||||
|
||||
if (normS <= clipNorm) {
|
||||
output->assign(input);
|
||||
} else {
|
||||
auto lambda = LAMBDA_T(_x, factor) { return _x * factor; });
|
||||
input->applyLambda<T>(lambda, output);
|
||||
}
|
||||
}
|
||||
}
|
||||
void clipByGlobalNorm(LaunchContext* context, std::vector<NDArray*>& inputs, double clipNorm,
|
||||
memory::Workspace* workspace, std::vector<NDArray*>& outputs, bool isInplace) {
|
||||
BUILD_SINGLE_SELECTOR(outputs[0]->dataType(), clipByGlobalNorm_, (inputs, clipNorm, workspace, outputs, isInplace),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void clipByGlobalNorm_,
|
||||
(std::vector<NDArray*> & inputs, double clipNorm, sd::memory::Workspace* workspace,
|
||||
std::vector<NDArray*>& outputs, bool isInplace),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
template <typename T>
|
||||
static void clipByValue_(NDArray* input, double leftBound, double rightBound, NDArray* output) {
|
||||
auto routine = LAMBDA_T(_x, leftBound, rightBound) {
|
||||
if (_x > rightBound) return static_cast<T>(rightBound);
|
||||
if (_x < leftBound) return static_cast<T>(leftBound);
|
||||
return _x;
|
||||
});
|
||||
|
||||
input->applyLambda<T>(routine, output);
|
||||
}
|
||||
|
||||
void clipByValue(LaunchContext* context, NDArray* input, double leftBound, double rightBound, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), clipByValue_, (input, leftBound, rightBound, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void clipByValue_,
|
||||
(NDArray * input, double leftBound, double rightBound, NDArray* output);
|
||||
, SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,115 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 30.11.17.
|
||||
//
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/col2im.h>
|
||||
#if NOT_EXCLUDED(OP_col2im)
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
// [bS, iC, kH, kW, oH, oW] is de-convoluted to [bS, iC, iH, iW]
|
||||
template <typename T>
|
||||
static void col2im_(sd::LaunchContext& context, NDArray* input, NDArray* output, const LongType sH, const LongType sW,
|
||||
const LongType pH, const LongType pW, const LongType iH, const LongType iW, const LongType dH, const LongType dW) {
|
||||
if(input->rankOf() != 6) {
|
||||
THROW_EXCEPTION("ops::helpers::col2im: input array must have rank = 6");
|
||||
}
|
||||
|
||||
if(output->rankOf() != 4) {
|
||||
THROW_EXCEPTION("ops::helpers::col2im: output array must have rank = 4");
|
||||
}
|
||||
|
||||
auto colBuff = input->bufferAsT<T>();
|
||||
auto imBuff = output->bufferAsT<T>();
|
||||
auto colShapeBuffer = input->shapeInfo();
|
||||
auto imShapeBuffer = output->shapeInfo();
|
||||
auto colShape = shape::shapeOf(colShapeBuffer);
|
||||
auto colStride = shape::stride(colShapeBuffer);
|
||||
auto imShape = shape::shapeOf(imShapeBuffer);
|
||||
auto imStride = shape::stride(imShapeBuffer);
|
||||
|
||||
const LongType bS = imShape[0];
|
||||
const LongType iC = imShape[1];
|
||||
const LongType kH = colShape[2];
|
||||
const LongType kW = colShape[3];
|
||||
const LongType oH = colShape[4];
|
||||
const LongType oW = colShape[5];
|
||||
const sd::LongType colStride0 = colStride[0];
|
||||
const sd::LongType colStride1 = colStride[1];
|
||||
const sd::LongType colStride2 = colStride[2];
|
||||
const sd::LongType colStride3 = colStride[3];
|
||||
const sd::LongType colStride4 = colStride[4];
|
||||
const sd::LongType colStride5 = colStride[5];
|
||||
const sd::LongType imStride0 = imStride[0];
|
||||
const sd::LongType imStride1 = imStride[1];
|
||||
const sd::LongType imStride2 = imStride[2];
|
||||
const sd::LongType imStride3 = imStride[3];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto b = start; b < stop; b++) {
|
||||
LongType im0Offset = b * imStride0;
|
||||
LongType col4Offset = b * colStride0;
|
||||
for (int colH = 0; colH < oH; ++colH) {
|
||||
LongType col5Offset = col4Offset + colH * colStride4;
|
||||
for (int colW = 0; colW < oW; ++colW) {
|
||||
LongType col1Offset = col5Offset + colW * colStride5;
|
||||
LongType im1Offset = im0Offset;
|
||||
for (int c = 0; c < iC; ++c) {
|
||||
int imRow = (-pH + colH * sH);
|
||||
LongType col2Offset = col1Offset + c * colStride1;
|
||||
LongType im2Offset = im1Offset + c * imStride1 + imRow * imStride2;
|
||||
for (int kRow = 0; kRow < kH; ++kRow) {
|
||||
int imCol = -pW + colW * sW;
|
||||
LongType col3Offset = col2Offset + kRow * colStride2;
|
||||
LongType im3Offset = im2Offset + kRow * dH * imStride2 + imCol * imStride3;
|
||||
for (int kCol = 0; kCol < kW; ++kCol) {
|
||||
if (static_cast<unsigned>(imRow) < static_cast<unsigned>(iH) &&
|
||||
static_cast<unsigned>(imCol) < static_cast<unsigned>(iW)) {
|
||||
imBuff[im3Offset] += colBuff[col3Offset];
|
||||
}
|
||||
col3Offset += colStride3;
|
||||
imCol += dW;
|
||||
im3Offset += dW * imStride3;
|
||||
}
|
||||
imRow += dH;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, bS);
|
||||
}
|
||||
void col2im(LaunchContext& context, NDArray* input, NDArray* output, const LongType sH, const LongType sW, const LongType pH,
|
||||
const LongType pW, const LongType iH, const LongType iW, const LongType dH, const LongType dW) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), col2im_, (context, input, output, sH, sW, pH, pW, iH, iW, dH, dW),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#include <execution/ThreadPool.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopsCoordsHelper.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <type_traits>
|
||||
|
||||
#if NOT_EXCLUDED(OP_compare_and_bitpack)
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename X>
|
||||
uint8_t pack(const X* buff, const X& threshold) {
|
||||
uint8_t res;
|
||||
res = (buff[0] > threshold) << 7;
|
||||
res = res | ((buff[1] > threshold) << 6);
|
||||
res = res | ((buff[2] > threshold) << 5);
|
||||
res = res | ((buff[3] > threshold) << 4);
|
||||
res = res | ((buff[4] > threshold) << 3);
|
||||
res = res | ((buff[5] > threshold) << 2);
|
||||
res = res | ((buff[6] > threshold) << 1);
|
||||
res = res | (buff[7] > threshold);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <>
|
||||
uint8_t pack<bool>(const bool* buff, const bool& threshold) {
|
||||
// ignore threshold
|
||||
uint8_t res;
|
||||
res = buff[0] << 7;
|
||||
res = res | (buff[1] << 6);
|
||||
res = res | (buff[2] << 5);
|
||||
res = res | (buff[3] << 4);
|
||||
res = res | (buff[4] << 3);
|
||||
res = res | (buff[5] << 2);
|
||||
res = res | (buff[6] << 1);
|
||||
res = res | buff[7];
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
uint8_t pack(const X* buff, int stride, const X& threshold) {
|
||||
uint8_t res;
|
||||
res = (buff[0] > threshold) << 7;
|
||||
res = res | ((buff[1 * stride] > threshold) << 6);
|
||||
res = res | ((buff[2 * stride] > threshold) << 5);
|
||||
res = res | ((buff[3 * stride] > threshold) << 4);
|
||||
res = res | ((buff[4 * stride] > threshold) << 3);
|
||||
res = res | ((buff[5 * stride] > threshold) << 2);
|
||||
res = res | ((buff[6 * stride] > threshold) << 1);
|
||||
res = res | (buff[7 * stride] > threshold);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <>
|
||||
uint8_t pack<bool>(const bool* buff, int stride, const bool& threshold) {
|
||||
// ignore threshold
|
||||
uint8_t res;
|
||||
res = buff[0] << 7;
|
||||
res = res | (buff[1 * stride] << 6);
|
||||
res = res | (buff[2 * stride] << 5);
|
||||
res = res | (buff[3 * stride] << 4);
|
||||
res = res | (buff[4 * stride] << 3);
|
||||
res = res | (buff[5 * stride] << 2);
|
||||
res = res | (buff[6 * stride] << 1);
|
||||
res = res | buff[7 * stride];
|
||||
return res;
|
||||
}
|
||||
template <typename X>
|
||||
void compareAndBitpack_(NDArray& input, NDArray& thresholdScalar, NDArray& output) {
|
||||
auto rank = input.rankOf();
|
||||
X threshold = thresholdScalar.e<X>(0);
|
||||
auto buff = input.bufferAsT<X>();
|
||||
uint8_t* outBuff = output.bufferAsT<uint8_t>();
|
||||
if (input.ordering() == 'c' && output.ordering() == 'c' && input.ews() == 1 && output.ews() == 1) {
|
||||
FUNC_1D func = [buff, outBuff, threshold](uint64_t thread_id, int64_t start, int64_t stop,
|
||||
int64_t increment) -> void {
|
||||
auto outBuffPart = outBuff + start;
|
||||
auto buffPart = buff + start * 8;
|
||||
auto len = stop - start;
|
||||
// run
|
||||
for (auto i = 0; i < len; i++) {
|
||||
outBuffPart[i] = pack<X>(&(buffPart[8 * i]), threshold);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, output.lengthOf(), 1);
|
||||
|
||||
} else {
|
||||
auto inShapes = input.shapeOf();
|
||||
auto outShapes = output.shapeOf();
|
||||
auto inStrides = input.stridesOf();
|
||||
auto outStrides = output.stridesOf();
|
||||
|
||||
if (rank == 1) {
|
||||
auto inLastStride = inStrides[rank - 1];
|
||||
auto outLastStride = outStrides[rank - 1];
|
||||
FUNC_1D func = [buff, outBuff, inLastStride, outLastStride, threshold](uint64_t thread_id, int64_t start,
|
||||
int64_t stop, int64_t increment) -> void {
|
||||
auto buffPart = buff + start * 8 * inLastStride;
|
||||
auto outBuffPart = outBuff + start * outLastStride;
|
||||
auto len = stop - start;
|
||||
// run
|
||||
for (auto i = 0; i < len; i++) {
|
||||
*outBuffPart = pack<X>(buffPart, inLastStride, threshold);
|
||||
buffPart += 8 * inLastStride;
|
||||
outBuffPart += outLastStride;
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, output.lengthOf(), 1);
|
||||
} else {
|
||||
// if output shape is {n1, n2, n3} then input shape is { n1. n2, n3 * 8}
|
||||
// therefore we can split input shape {n1, n2, n3 , 8} and correct its stride
|
||||
// as we do not need last shape info. lets just extend and correct its stride
|
||||
sd::LongType extendedStrides[SD_MAX_RANK];
|
||||
for (int i = 0; i < rank; i++) {
|
||||
extendedStrides[i] = inStrides[i];
|
||||
}
|
||||
// lets correct new stride
|
||||
extendedStrides[rank - 1] = 8 * inStrides[rank - 1];
|
||||
extendedStrides[rank] = inStrides[rank - 1];
|
||||
// general case. its slow. we can improve it for special case later
|
||||
// generic case that could be further improved. for now its slow
|
||||
FUNC_1D func = [rank, buff, outBuff, outShapes, extendedStrides, outStrides, threshold](
|
||||
uint64_t thread_id, int64_t start, int64_t stop, int64_t increment) -> void {
|
||||
sd::LongType coords[SD_MAX_RANK] = {};
|
||||
sd::LongType* ptr_coords = (sd::LongType*)&coords;
|
||||
sd::LongType len = (stop - start);
|
||||
// its extended as {rank+1} so extendedStrides[rank] is valid
|
||||
auto innermostStride = extendedStrides[rank];
|
||||
INDEX2COORDS(start, rank, outShapes, ptr_coords);
|
||||
// here last dimension will not be in coords. this way output shape and input shapes are equal
|
||||
sd::LongType inOffset, outOffset;
|
||||
COORDS2INDEX(rank + 1, extendedStrides, ptr_coords, inOffset);
|
||||
COORDS2INDEX(rank, outStrides, ptr_coords, outOffset);
|
||||
for (sd::LongType k = 0; k < len; k++) {
|
||||
auto buffPart = &(buff[inOffset]);
|
||||
auto outBuffPart = &(outBuff[outOffset]);
|
||||
*outBuffPart = pack<X>(buffPart, innermostStride, threshold);
|
||||
inOffset += extendedStrides[rank];
|
||||
outOffset += outStrides[rank - 1];
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, output.lengthOf(), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
void compareAndBitpack(sd::graph::Context& block, NDArray& input, NDArray& threshold, NDArray& output) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), compareAndBitpack_, (input, threshold, output), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void compareAndBitpack_,
|
||||
(NDArray& input, NDArray& threshold, NDArray& output), SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,71 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/compare_elem.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static void _compare_elem(NDArray* input, bool isStrictlyIncreasing, bool& output) {
|
||||
auto length = shape::length(input->shapeInfo());
|
||||
|
||||
int elementsPerThread = length / ELEMENT_THRESHOLD;
|
||||
int num_threads = sd::math::sd_max<int>(1, elementsPerThread);
|
||||
num_threads = sd::math::sd_min<int>(num_threads, omp_get_max_threads());
|
||||
sd::LongType sumt = 0;
|
||||
|
||||
if (isStrictlyIncreasing) {
|
||||
auto func = PRAGMA_REDUCE_LONG {
|
||||
sd::LongType sum = 0;
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto val0 = input->t<T>(i);
|
||||
auto val1 = input->t<T>(i + 1);
|
||||
sum += val0 >= val1 ? -1 : 0;
|
||||
}
|
||||
return sum;
|
||||
};
|
||||
sumt = samediff::Threads::parallel_long(func, LAMBDA_SUML, 0, length - 1);
|
||||
} else {
|
||||
auto func = PRAGMA_REDUCE_LONG {
|
||||
sd::LongType sum = 0;
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto val0 = input->t<T>(i);
|
||||
auto val1 = input->t<T>(i + 1);
|
||||
sum += val0 > val1 ? -1 : 0;
|
||||
}
|
||||
|
||||
return sum;
|
||||
};
|
||||
sumt = samediff::Threads::parallel_long(func, LAMBDA_SUML, 0, length - 1);
|
||||
}
|
||||
|
||||
output = (sumt > -1);
|
||||
}
|
||||
|
||||
void compare_elem(sd::LaunchContext* context, NDArray* input, bool isStrictlyIncreasing, bool& output) {
|
||||
auto xType = input->dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, _compare_elem, (input, isStrictlyIncreasing, output), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _compare_elem, (NDArray * A, bool isStrictlyIncreasing, bool& output);
|
||||
, SD_COMMON_TYPES);
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,37 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#include <ops/declarable/helpers/cpu/indexReductions.hpp>
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_argamax)
|
||||
|
||||
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
BUILD_DOUBLE_TEMPLATE( SD_LIB_HIDDEN void argAbsMax_, (NDArray& input, NDArray& output, const std::vector<sd::LongType>& dimensions), SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_INDEXING_TYPES);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,35 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#include <ops/declarable/helpers/cpu/indexReductions.hpp>
|
||||
#include <system/op_boilerplate.h>
|
||||
#include <system/selective_rendering.h>
|
||||
#if NOT_EXCLUDED(OP_argamin)
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
BUILD_DOUBLE_TEMPLATE( SD_LIB_HIDDEN void argAbsMin_, (NDArray& input, NDArray& output, const std::vector<sd::LongType>& dimensions), SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_INDEXING_TYPES);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,34 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#include <ops/declarable/helpers/cpu/indexReductions.hpp>
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_argmax)
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
BUILD_DOUBLE_TEMPLATE( SD_LIB_HIDDEN void argMax_, (NDArray& input, NDArray& output, const std::vector<sd::LongType>& dimensions), SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_INDEXING_TYPES);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,35 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#include <ops/declarable/helpers/cpu/indexReductions.hpp>
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_argmin)
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
BUILD_DOUBLE_TEMPLATE( SD_LIB_HIDDEN void argMin_, (NDArray& input, NDArray& output, const std::vector<sd::LongType>& dimensions), SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_INDEXING_TYPES);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
|
||||
#include <ops/declarable/helpers/crop_and_resize.h>
|
||||
#include <ops/declarable/helpers/cpu/crop_and_resize.hpp>
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_crop_and_resize)
|
||||
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
BUILD_TRIPLE_TEMPLATE(template SD_LIB_HIDDEN void cropAndResizeFunctor_, (NDArray *images, NDArray *boxes, NDArray *indices, NDArray *cropSize, int method, double extrapolationVal, NDArray *crops), SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_FLOAT_TYPES, SD_INTEGER_TYPES);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,35 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_standard_deviation)
|
||||
//inform Cmake that each of LIBND4J_TYPE will be generated and used in separate cpp files.
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#include <ops/declarable/helpers/cpu/summaryReductions.hpp>
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
BUILD_DOUBLE_TEMPLATE( SD_LIB_HIDDEN void standardDeviation_, (NDArray& input, NDArray& output, const std::vector<sd::LongType>& dimensions, bool biasCorrected), SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_FLOAT_TYPES);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,37 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
|
||||
//inform Cmake that each of LIBND4J_TYPE will be generated and used in separate cpp files.
|
||||
#cmakedefine SD_COMMON_TYPES_GEN
|
||||
#include <ops/declarable/helpers/cpu/summaryReductions.hpp>
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_variance)
|
||||
|
||||
#if defined(SD_COMMON_TYPES_GEN) && defined(SD_COMMON_TYPES_@FL_TYPE_INDEX@)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
BUILD_DOUBLE_TEMPLATE( SD_LIB_HIDDEN void variance_, (NDArray& input, NDArray& output, const std::vector<sd::LongType>& dimensions, bool biasCorrected), SD_COMMON_TYPES_@FL_TYPE_INDEX@, SD_FLOAT_TYPES);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,47 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 20.04.2018
|
||||
//
|
||||
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
#include <ops/specials.h>
|
||||
#include <system/selective_rendering.h>
|
||||
#if NOT_EXCLUDED(OP_concat)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void concat_(const std::vector<NDArray*>& inArrs, NDArray& output, const int axis) {
|
||||
sd::SpecialMethods<T>::concatCpuGeneric(inArrs, output, axis);
|
||||
}
|
||||
|
||||
void concat(sd::LaunchContext* context, const std::vector<NDArray*>& inArrs, NDArray& output, const int axis) {
|
||||
auto outputTYpe = output.dataType();
|
||||
BUILD_SINGLE_SELECTOR(output.dataType(), concat_, (inArrs, output, axis), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void concat_,
|
||||
(const std::vector<NDArray*>& inArrs, NDArray& output, const int axis), SD_COMMON_TYPES);
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,61 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author GS <sgazeos@gmail.com>
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/confusion.h>
|
||||
#if NOT_EXCLUDED(OP_confusion_matrix)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void _confusionFunctor(NDArray* labels, NDArray* predictions, NDArray* weights, NDArray* output) {
|
||||
ResultSet arrs = output->allTensorsAlongDimension({1});
|
||||
int lLen = labels->lengthOf();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType j = start; j < stop; j++) {
|
||||
auto label = labels->e<sd::LongType>(j);
|
||||
auto pred = predictions->e<sd::LongType>(j);
|
||||
T value = (weights == nullptr ? (T)1.0f : weights->e<T>(j));
|
||||
T curr = arrs.at(label)->e<T>(pred);
|
||||
arrs.at(label)->p<T>(pred, curr + value);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, lLen);
|
||||
}
|
||||
|
||||
void confusionFunctor(sd::LaunchContext* context, NDArray* labels, NDArray* predictions, NDArray* weights,
|
||||
NDArray* output) {
|
||||
auto xType = output->dataType(); // weights can be null
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, _confusionFunctor, (labels, predictions, weights, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _confusionFunctor,
|
||||
(NDArray * labels, NDArray* predictions, NDArray* weights, NDArray* output);
|
||||
, SD_NUMERIC_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,161 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// [bS, iC, kD, kH, kW, oD, oH, oW] is de-convoluted to [bS, iC, iD, iH, iW]
|
||||
template <typename T>
|
||||
static void col2vol_(NDArray& columns, NDArray& volume, const LongType sD, const LongType sH, const LongType sW, const LongType pD,
|
||||
const LongType pH, const LongType pW, const LongType dD, const LongType dH, const LongType dW) {
|
||||
// initial zeroing of volume content
|
||||
volume.nullify();
|
||||
|
||||
const LongType bS = volume.sizeAt(0);
|
||||
const LongType iC = volume.sizeAt(1);
|
||||
const LongType iD = volume.sizeAt(2);
|
||||
const LongType iH = volume.sizeAt(3);
|
||||
const LongType iW = volume.sizeAt(4);
|
||||
const LongType kD = columns.sizeAt(2);
|
||||
const LongType kH = columns.sizeAt(3);
|
||||
const LongType kW = columns.sizeAt(4);
|
||||
const LongType oD = columns.sizeAt(5);
|
||||
const LongType oH = columns.sizeAt(6);
|
||||
const LongType oW = columns.sizeAt(7);
|
||||
const sd::LongType colStride0 = columns.stridesOf()[0];
|
||||
const sd::LongType colStride1 = columns.stridesOf()[1];
|
||||
const sd::LongType colStride2 = columns.stridesOf()[2];
|
||||
const sd::LongType colStride3 = columns.stridesOf()[3];
|
||||
const sd::LongType colStride4 = columns.stridesOf()[4];
|
||||
const sd::LongType colStride5 = columns.stridesOf()[5];
|
||||
const sd::LongType colStride6 = columns.stridesOf()[6];
|
||||
const sd::LongType colStride7 = columns.stridesOf()[7];
|
||||
const sd::LongType volStride0 = volume.stridesOf()[0];
|
||||
const sd::LongType volStride1 = volume.stridesOf()[1];
|
||||
const sd::LongType volStride2 = volume.stridesOf()[2];
|
||||
const sd::LongType volStride3 = volume.stridesOf()[3];
|
||||
const sd::LongType volStride4 = volume.stridesOf()[4];
|
||||
|
||||
T* volBuff = volume.bufferAsT<T>();
|
||||
T* colBuff = const_cast<NDArray&>(columns).bufferAsT<T>();
|
||||
|
||||
if (volume.ordering() == 'c' && columns.ordering() == 'c' && shape::strideDescendingCAscendingF(volume.shapeInfo()) &&
|
||||
shape::strideDescendingCAscendingF(columns.shapeInfo())) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
T *col, *vol;
|
||||
sd::LongType volDep, volRow, volCol;
|
||||
|
||||
for (sd::LongType b = start; b < stop; b++) {
|
||||
for (sd::LongType c = 0; c < iC; c++) {
|
||||
for (sd::LongType kDep = 0; kDep < kD; ++kDep) {
|
||||
for (sd::LongType kRow = 0; kRow < kH; ++kRow) {
|
||||
for (sd::LongType kCol = 0; kCol < kW; ++kCol) {
|
||||
for (sd::LongType colD = 0; colD < oD; ++colD) {
|
||||
for (sd::LongType colH = 0; colH < oH; ++colH) {
|
||||
for (sd::LongType colW = 0; colW < oW; ++colW) {
|
||||
volDep = (-pD + kDep * dD) + colD * sD;
|
||||
volRow = (-pH + kRow * dH) + colH * sH;
|
||||
volCol = (-pW + kCol * dW) + colW * sW;
|
||||
|
||||
if (volDep >= 0 && volDep < iD &&
|
||||
volRow >= 0 && volRow < iH &&
|
||||
volCol >= 0 && volCol < iW) {
|
||||
|
||||
auto colIndex = b * colStride0 + c * colStride1 + kDep * colStride2 + kRow * colStride3 +
|
||||
kCol * colStride4 + colD * colStride5 + colH * colStride6 + colW * colStride7;
|
||||
auto volIndex = b * volStride0 + c * volStride1 + volDep * volStride2 + volRow * volStride3 +
|
||||
volCol * volStride4;
|
||||
|
||||
|
||||
|
||||
col = colBuff + colIndex;
|
||||
vol = volBuff + volIndex;
|
||||
*vol += *col;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, bS);
|
||||
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
T *col, *vol;
|
||||
sd::LongType volDep, volRow, volCol;
|
||||
|
||||
for (sd::LongType b = start; b < stop; b++) {
|
||||
for (sd::LongType colD = 0; colD < oD; colD++) {
|
||||
for (sd::LongType colH = 0; colH < oH; ++colH) {
|
||||
for (sd::LongType colW = 0; colW < oW; ++colW) {
|
||||
for (sd::LongType c = 0; c < iC; ++c) {
|
||||
for (sd::LongType kDep = 0; kDep < kD; ++kDep) {
|
||||
for (sd::LongType kRow = 0; kRow < kH; ++kRow) {
|
||||
for (sd::LongType kCol = 0; kCol < kW; ++kCol) {
|
||||
volDep = (-pD + kDep * dD) + colD * sD;
|
||||
volRow = (-pH + kRow * dH) + colH * sH;
|
||||
volCol = (-pW + kCol * dW) + colW * sW;
|
||||
|
||||
if (volDep >= 0 && volDep < iD &&
|
||||
volRow >= 0 && volRow < iH &&
|
||||
volCol >= 0 && volCol < iW) {
|
||||
|
||||
auto colIndex = b * colStride0 + c * colStride1 + kDep * colStride2 + kRow * colStride3 +
|
||||
kCol * colStride4 + colD * colStride5 + colH * colStride6 + colW * colStride7;
|
||||
auto volIndex = b * volStride0 + c * volStride1 + volDep * volStride2 + volRow * volStride3 +
|
||||
volCol * volStride4;
|
||||
|
||||
col = colBuff + colIndex;
|
||||
vol = volBuff + volIndex;
|
||||
*vol += *col;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, bS);
|
||||
}
|
||||
}
|
||||
|
||||
void ConvolutionUtils::col2vol(sd::graph::Context& block, NDArray& columns, NDArray& volume, const LongType sD,
|
||||
const LongType sH, const LongType sW, const LongType pD, const LongType pH, const LongType pW, const LongType dD,
|
||||
const LongType dH, const LongType dW) {
|
||||
BUILD_SINGLE_SELECTOR(volume.dataType(), col2vol_, (columns, volume, sD, sH, sW, pD, pH, pW, dD, dH, dW),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,153 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/helpers/addBias.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
#include <ops/declarable/helpers/im2col.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_col2im) && NOT_EXCLUDED(OP_im2col)
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y>
|
||||
static void conv2d_(sd::graph::Context& block, NDArray* input, NDArray* weights, NDArray* bias,
|
||||
NDArray* output, const LongType kH, const LongType kW, const LongType sH, const LongType sW, LongType pH, LongType pW,
|
||||
const LongType dH, const LongType dW, const int paddingMode, const int isNCHW, const int wFormat) {
|
||||
|
||||
// input [bS, iH, iW, iC] (NHWC) or [bS, iC, iH, iW] (NCHW)
|
||||
// weights [kH, kW, iC, oC], [oC, iC, kH, kW], [oC, kH, kW, iC]
|
||||
// bias [oC]
|
||||
// output [bS, oH, oW, oC] (NHWC) or [bS, oC, oH, oW] (NCHW)
|
||||
|
||||
LongType bS = input->sizeAt(0);
|
||||
LongType iC = ConvolutionUtils::inChannels(weights->shapeInfo(), wFormat);
|
||||
LongType oC = ConvolutionUtils::outChannels(weights->shapeInfo(), wFormat);
|
||||
LongType iH = ConvolutionUtils::inputHeight(input->shapeInfo(), isNCHW);
|
||||
LongType iW = ConvolutionUtils::inputWidth(input->shapeInfo(), isNCHW);
|
||||
LongType oH = ConvolutionUtils::calcOutDimConv(iH, kH, sH, pH, dH, paddingMode);
|
||||
LongType oW = ConvolutionUtils::calcOutDimConv(iW, kW, sW, pW, dW, paddingMode);
|
||||
std::vector<LongType> wAxes;
|
||||
if (0 == wFormat)
|
||||
wAxes = {0, 1, 2};
|
||||
else if (1 == wFormat)
|
||||
wAxes = {2, 3, 1};
|
||||
else
|
||||
wAxes = {1, 2, 3};
|
||||
|
||||
|
||||
std::vector<sd::LongType> colShape = {bS, oH, oW, kH, kW, iC};
|
||||
std::vector<sd::LongType> perm = {0, 3, 4, 5, 1, 2};
|
||||
NDArray *col = new NDArray('c', colShape, input->dataType(), input->getContext());
|
||||
NDArray *colPFrom = col->permute(perm, false, false);
|
||||
NDArray *colP = new NDArray(colPFrom); // {bS, iC, kH, kW, oH, oW}
|
||||
std::vector<sd::LongType> mmulResultShape = {bS * oH * oW, oC};
|
||||
NDArray mmulResult('f', mmulResultShape, output->dataType(), output->getContext());
|
||||
std::vector<LongType> permuteForOutput = {0, 3, 1, 2};
|
||||
|
||||
//----- calculation of output -----//
|
||||
auto ctx = block.launchContext();
|
||||
|
||||
NDArray *inputNchw = nullptr; // Track NHWC permutation for cleanup
|
||||
NDArray *zeroVal = NDArrayFactory::create(0.f, input->getContext());
|
||||
if (isNCHW) {
|
||||
helpers::im2col(*ctx, *input, *colP, kH, kW, sH, sW, pH, pW, dH, dW,
|
||||
*zeroVal);
|
||||
} else {
|
||||
std::vector<sd::LongType> permute = {0, 3, 1, 2};
|
||||
// For NHWC, we need to permute the input to NCHW before im2col
|
||||
inputNchw = input->permute(permute, false,false);
|
||||
helpers::im2col(*ctx, *inputNchw, *colP, kH, kW, sH, sW, pH, pW, dH, dW,
|
||||
*zeroVal);
|
||||
}
|
||||
|
||||
delete zeroVal;
|
||||
delete colPFrom; // View wrapper from permute - no longer needed
|
||||
delete col; // Original col array - no longer needed
|
||||
block.pushIntermediateResult(colP);
|
||||
|
||||
std::vector<sd::LongType> shape = {bS * oH * oW, kH * kW * iC};
|
||||
NDArray *colReshaped = colP->reshape('c', shape, false);
|
||||
std::vector<sd::LongType> perm2 = {3,2,1,0};
|
||||
|
||||
NDArray *weightsPermuted = weights->permute(perm2, false, false);
|
||||
|
||||
std::vector<sd::LongType> wShape = {iC * kH * kW, oC};
|
||||
NDArray *reshapedW = weightsPermuted->reshape('f',wShape, false);
|
||||
NDArray *colpPReshapedAddr = colReshaped;
|
||||
|
||||
NDArray *reshapedWAddr = reshapedW;
|
||||
MmulHelper::matmul(colpPReshapedAddr, reshapedWAddr, &mmulResult, false, false, 1.0, 0.0);
|
||||
|
||||
// Clean up after matmul
|
||||
delete colReshaped;
|
||||
delete weightsPermuted;
|
||||
delete reshapedW;
|
||||
|
||||
std::vector<sd::LongType>lastShape = {oH,oW,bS,oC};
|
||||
NDArray *reshaped = mmulResult.reshape('f', lastShape, false);
|
||||
std::vector<sd::LongType> permute2 = {2,3,1,0};
|
||||
NDArray *permuted = reshaped->permute(permute2, false, false);
|
||||
|
||||
// Clean up reshaped after permute
|
||||
delete reshaped;
|
||||
|
||||
// Reshape and copy result to output
|
||||
if (isNCHW) {
|
||||
output->assign(permuted);
|
||||
delete permuted;
|
||||
} else {
|
||||
std::vector<sd::LongType> perm3 = {0,2,3,1};
|
||||
NDArray *oldPermuted = permuted; // Save old pointer before reassignment
|
||||
permuted = permuted->permute(perm3, false, false);
|
||||
output->assign(permuted);
|
||||
delete oldPermuted; // Delete the first permutation
|
||||
delete permuted; // Delete the second permutation
|
||||
}
|
||||
|
||||
// Clean up NHWC permutation if it was created
|
||||
if (inputNchw != nullptr) {
|
||||
delete inputNchw;
|
||||
}
|
||||
|
||||
//----- add biases if required -----//
|
||||
if (bias) {
|
||||
helpers::addBias(block, *output, *bias, *output, isNCHW);
|
||||
}
|
||||
|
||||
}
|
||||
void ConvolutionUtils::conv2d(sd::graph::Context& block, NDArray* input, NDArray* weights,
|
||||
NDArray* bias, NDArray* output, const LongType kH, const LongType kW, const LongType sH,
|
||||
const LongType sW, LongType pH, LongType pW, const LongType dH, const LongType dW, const int paddingMode,
|
||||
const int isNCHW, const int wFormat) {
|
||||
BUILD_SINGLE_SELECTOR_TWICE(
|
||||
input->dataType(), conv2d_,
|
||||
(block, input, weights, bias, output, kH, kW, sH, sW, pH, pW, dH, dW, paddingMode, isNCHW, wFormat),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,176 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/helpers/addBias.h>
|
||||
#include <ops/declarable/helpers/col2im.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
#include <ops/declarable/helpers/im2col.h>
|
||||
|
||||
#include "helpers/ShapeUtils.h"
|
||||
#if NOT_EXCLUDED(OP_col2im) && NOT_EXCLUDED(OP_im2col)
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
template <typename X, typename Y>
|
||||
static void conv2dBP_(sd::graph::Context& block, NDArray* input, NDArray* weights, NDArray* bias,
|
||||
NDArray* gradO, NDArray* gradI, NDArray* gradW, NDArray* gradB, const LongType kH, const LongType kW,
|
||||
const LongType sH, const LongType sW, LongType pH, LongType pW, const LongType dH, const LongType dW,
|
||||
const int paddingMode, const int isNCHW, const int wFormat) {
|
||||
|
||||
// input [bS, iH, iW, iC] (NHWC) or [bS, iC, iH, iW] (NCHW)
|
||||
// weights [kH, kW, iC, oC], [oC, iC, kH, kW], [oC, kH, kW, iC]
|
||||
// bias [oC]
|
||||
// gradO [bS, oH, oW, oC] (NHWC) or [bS, oC, oH, oW] (NCHW), epsilon_next
|
||||
// gradI [bS, iH, iW, iC] (NHWC) or [bS, iC, iH, iW] (NCHW), epsilon
|
||||
// gradW [kH, kW, iC, oC], [oC, iC, kH, kW], [oC, kH, kW, iC]
|
||||
// gradB [oC]
|
||||
|
||||
const LongType bS = input->sizeAt(0); // batch size
|
||||
const LongType iC = isNCHW ? input->sizeAt(1) : input->sizeAt(3); // input channels
|
||||
const LongType iH = isNCHW ? input->sizeAt(2) : input->sizeAt(1); // input height
|
||||
const LongType iW = isNCHW ? input->sizeAt(3) : input->sizeAt(2); // input width
|
||||
|
||||
const LongType oC = isNCHW ? gradO->sizeAt(1) : gradO->sizeAt(3); // output channels
|
||||
const LongType oH = isNCHW ? gradO->sizeAt(2) : gradO->sizeAt(1); // output height
|
||||
const LongType oW = isNCHW ? gradO->sizeAt(3) : gradO->sizeAt(2); // output width
|
||||
NDArray *inputPermuted, *gradOPermuted, *gradIPermuted;
|
||||
if (!isNCHW) {
|
||||
std::vector<sd::LongType> permute = {0, 3, 1, 2};
|
||||
inputPermuted = input->permute(permute, false, false); // [bS, iH, iW, iC] -> [bS, iC, iH, iW]
|
||||
gradOPermuted = gradO->permute(permute, false, false); // [bS, oH, oW, oC] -> [bS, oC, oH, oW]
|
||||
gradIPermuted = gradI->permute(permute, false, false); // [bS, iH, iW, iC] -> [bS, iC, iH, iW]
|
||||
} else {
|
||||
inputPermuted = input;
|
||||
gradOPermuted = gradO;
|
||||
gradIPermuted = gradI;
|
||||
}
|
||||
|
||||
std::vector<sd::LongType> gradOShape = {oC, bS * oH * oW};
|
||||
// Reshape gradO to 2D: [oC, bS * oH * oW]
|
||||
NDArray *gradO2d = gradOPermuted->reshape(gradOPermuted->ordering(), gradOShape,false);
|
||||
|
||||
// Perform im2col
|
||||
NDArray* columns;
|
||||
if (block.hasIntermediateResults()) {
|
||||
columns = block.intermediateResult(0);
|
||||
if (columns->rankOf() < 6) {
|
||||
columns->reshapei({bS, iC, kH, kW, oH, oW});
|
||||
}
|
||||
} else {
|
||||
std::vector<sd::LongType> colShape = {bS, iC, kH, kW, oH, oW};
|
||||
columns = new NDArray(inputPermuted->ordering(), colShape, inputPermuted->dataType(), inputPermuted->getContext());
|
||||
auto ctx = block.launchContext();
|
||||
NDArray *zeroVal = NDArrayFactory::create<double>(0., inputPermuted->getContext());
|
||||
helpers::im2col(*ctx, *inputPermuted, *columns, kH, kW, sH, sW, pH, pW, dH, dW,
|
||||
*zeroVal);
|
||||
delete zeroVal;
|
||||
}
|
||||
|
||||
// Calculate gradW
|
||||
if (gradW) {
|
||||
std::vector<sd::LongType> colShape = {bS * oH * oW, iC * kH * kW};
|
||||
std::vector<sd::LongType> wShape = {oC, iC * kH * kW};
|
||||
NDArray *columns2d = columns->reshape('c',colShape,false);
|
||||
std::vector<sd::LongType> permute = {1,0};
|
||||
NDArray *gradW2d = gradW->reshape('f', wShape, false)->permute(permute, false, false);
|
||||
|
||||
MmulHelper::matmul( columns2d,gradO2d, gradW2d, true, true, 1.0, 0.0, gradW2d);
|
||||
gradW->assign(gradW2d);
|
||||
delete columns2d;
|
||||
|
||||
}
|
||||
|
||||
// Calculate gradB
|
||||
if (gradB) {
|
||||
std::vector<LongType> axes = {1}; // Sum over bS, oH, oW
|
||||
gradO2d->reduceAlongDimension(reduce::Sum, gradB, &axes);
|
||||
}
|
||||
|
||||
// Calculate gradI
|
||||
NDArray *weights2d;
|
||||
if (wFormat == 0) {
|
||||
std::vector<sd::LongType> perm = {3,2,1,0};
|
||||
std::vector<sd::LongType> wShape = {iC * kH * kW,oC};
|
||||
weights2d = weights->permute(perm, false, false)->reshape('f', wShape);
|
||||
} else if (wFormat == 1) {
|
||||
std::vector<sd::LongType> wShape2 = {iC * kH * kW,oC};
|
||||
weights2d = weights->reshape('f', wShape2);
|
||||
} else {
|
||||
std::vector<sd::LongType> wPermute = {0,2,3,1};
|
||||
std::vector<sd::LongType> weights2dShape = {iC * kH * kW,oC};
|
||||
weights2d = weights->permute(wPermute, false, false)->reshape('f', weights2dShape);
|
||||
}
|
||||
|
||||
std::vector<sd::LongType> columns2dShape = {iC * kH * kW, bS * oH * oW};
|
||||
NDArray columns2d('c', columns2dShape, columns->dataType(), columns->getContext());
|
||||
|
||||
|
||||
MmulHelper::matmul(weights2d, gradO2d, &columns2d, false, false, 1.0, 0.0);
|
||||
delete weights2d;
|
||||
//Calculate epsilonNext by doing im2col reduction.
|
||||
//Current col2im implementation expects input with order: [miniBatch,channels,kH,kW,outH,outW]
|
||||
//currently have [kH,kW,inDepth,outW,outH,miniBatch] -> permute first
|
||||
auto eps6d = columns2d.newShapeNoCopy({kH, kW,iC, oW, oH, bS }, 'f');
|
||||
std::vector<sd::LongType> epsPermute = {5,2,1,0,4,3};
|
||||
auto permuted = eps6d->permute(epsPermute, false, false);
|
||||
|
||||
// Perform col2im
|
||||
auto ctx = block.launchContext();
|
||||
helpers::col2im(*ctx, permuted, gradIPermuted, sH, sW, pH, pW, iH, iW, dH, dW);
|
||||
// Handle NHWC format if necessary
|
||||
if (!isNCHW) {
|
||||
std::vector<sd::LongType> perm = {0,2,3,1};
|
||||
gradI->assign(gradIPermuted->permute(perm, false, false)); // [bS, iC, iH, iW] -> [bS, iH, iW, iC]
|
||||
}
|
||||
|
||||
delete gradO2d;
|
||||
// Clean up
|
||||
if (!isNCHW) {
|
||||
delete inputPermuted;
|
||||
delete gradOPermuted;
|
||||
delete gradIPermuted;
|
||||
}
|
||||
if (!block.hasIntermediateResults()) {
|
||||
delete columns;
|
||||
}
|
||||
}
|
||||
|
||||
void ConvolutionUtils::conv2dBP(sd::graph::Context& block, NDArray* input, NDArray* weights,
|
||||
NDArray* bias, NDArray* gradO, NDArray* gradI, NDArray* gradW,
|
||||
NDArray* gradB, const LongType kH, const LongType kW, const LongType sH, const LongType sW, LongType pH, LongType pW,
|
||||
const LongType dH, const LongType dW, const int paddingMode, const int isNCHW,
|
||||
const int wFormat) {
|
||||
BUILD_SINGLE_SELECTOR_TWICE(input->dataType(), conv2dBP_,
|
||||
(block, input, weights, bias, gradO, gradI, gradW, gradB, kH, kW, sH, sW, pH, pW, dH, dW,
|
||||
paddingMode, isNCHW, wFormat),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,120 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/helpers/addBias.h>
|
||||
#include <ops/declarable/helpers/col2im.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
#include <ops/declarable/helpers/im2col.h>
|
||||
#if NOT_EXCLUDED(OP_col2im) && NOT_EXCLUDED(OP_im2col)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y>
|
||||
static void depthwiseConv2d_(sd::graph::Context& block, NDArray* input, NDArray* weights,
|
||||
NDArray* bias, NDArray* output, const LongType kH, const LongType kW, const LongType sH,
|
||||
const LongType sW, LongType pH, LongType pW, const LongType dH, const LongType dW, const int paddingMode,
|
||||
const int isNCHW, const int wFormat) {
|
||||
// input [bS, iH, iW, iC] (NHWC) or [bS, iC, iH, iW] (NCHW)
|
||||
// weights [kH, kW, iC, mC], [mC, iC, kH, kW], [mC, kH, kW, iC]
|
||||
// bias [oC] = iC*mC
|
||||
// output [bS, oH, oW, iC*mC] (NHWC) or [bS, iC*mC, oH, oW] (NCHW)
|
||||
|
||||
// kH filter(kernel) height
|
||||
// kW filter(kernel) width
|
||||
// sH strides height
|
||||
// sW strides width
|
||||
// pH paddings height
|
||||
// pW paddings width
|
||||
// dH dilations height
|
||||
// dW dilations width
|
||||
// paddingMode 0-VALID, 1-SAME
|
||||
// isNCHW 0-NCHW, 1-NHWC
|
||||
|
||||
LongType bS, iC, iH, iW, mC, oC, oH, oW; // batch size, input channels, input height/width, channels multiplier(oC =
|
||||
// iC*mC), output channels, output height/width
|
||||
LongType indIOioC, indIiH, indWmC, indWiC, indWkH, indOoH; // corresponding indexes
|
||||
ConvolutionUtils::getSizesAndIndexesConv2d(isNCHW, wFormat, *input, *output, bS, iC, iH, iW, oC, oH, oW, indIOioC,
|
||||
indIiH, indWiC, indWmC, indWkH, indOoH);
|
||||
mC = weights->sizeAt(indWmC); // channels multiplier
|
||||
|
||||
std::vector<std::vector<sd::LongType>> modifColumns = {
|
||||
{1, 0, 4, 5, 2, 3},
|
||||
{iC, bS * oH * oW, kH * kW}}; // [bS,iC,kH,kW,oH,oW] -> [iC,bS,oH,oW,kH,kW] -> [iC,bS*oH*oW,kH*kW]
|
||||
std::vector<std::vector<sd::LongType>> modifOutput, modifWeights;
|
||||
std::vector<sd::LongType> outReShape;
|
||||
|
||||
if (!isNCHW) {
|
||||
outReShape = {bS, oH, oW, iC, mC}; // [bS,oH,oW,iC*mC] -> [bS,oH,oW,iC,mC]
|
||||
modifOutput = {{3, 0, 1, 2, 4},
|
||||
{iC, bS * oH * oW, mC}}; // [bS,oH,oW,iC,mC] -> [iC,bS,oH,oW,mC] -> [iC,bS*oH*oW,mC]
|
||||
std::vector<sd::LongType> perm = {0, 3, 1, 2}; // [bS,iH,iW,iC] -> [bS,iC,iH,iW]
|
||||
input = input->permute(perm, false, false); // [bS,iH,iW,iC] -> [bS,iC,iH,iW]
|
||||
} else {
|
||||
outReShape = {bS, iC, mC, oH, oW}; // [bS,iC*mC,oH,oW] -> [bS,iC,mC,oH,oW]
|
||||
modifOutput = {{1, 0, 3, 4, 2},
|
||||
{iC, bS * oH * oW, mC}}; // [bS,iC,mC,oH,oW] -> [iC,bS,oH,oW,mC] -> [iC,bS*oH*oW,mC]
|
||||
}
|
||||
|
||||
if (0 == wFormat)
|
||||
modifWeights = {{2, 0, 1, 3}, {iC, kH * kW, mC}};
|
||||
else if (1 == wFormat)
|
||||
modifWeights = {{1, 2, 3, 0}, {iC, kH * kW, mC}};
|
||||
else
|
||||
modifWeights = {{3, 1, 2, 0}, {iC, kH * kW, mC}};
|
||||
|
||||
if (paddingMode == 1) // SAME
|
||||
ConvolutionUtils::calcPadding2D(pH, pW, oH, oW, iH, iW, kH, kW, sH, sW, dH, dW);
|
||||
|
||||
std::vector<sd::LongType> colShape = {bS, iC, kH, kW, oH, oW};
|
||||
NDArray columns(input->ordering(),colShape, input->dataType(), input->getContext());
|
||||
NDArray *outputReshaped = output->reshape(output->ordering(), outReShape, false);
|
||||
NDArray *zero = NDArrayFactory::create(0.f, input->getContext());
|
||||
helpers::im2col(
|
||||
*output->getContext(), *input, columns, kH, kW, sH, sW, pH, pW, dH, dW,
|
||||
*zero); // [bS, iC, iH, iW] is convoluted to [bS, iC, kH, kW, oH, oW]
|
||||
MmulHelper::tensorDot(&columns, weights, outputReshaped, modifColumns, modifWeights,
|
||||
modifOutput); // [iC, bS*oH*oW, kW*kH] x [iC, kH*kW, mC] = [iC, bS*oH*oW, mC]
|
||||
delete zero;
|
||||
|
||||
if (bias)
|
||||
helpers::addBias(block, *output, *bias, *output, isNCHW);
|
||||
|
||||
delete outputReshaped;
|
||||
if (!isNCHW) delete input;
|
||||
}
|
||||
|
||||
void ConvolutionUtils::depthwiseConv2d(sd::graph::Context& block, NDArray* input, NDArray* weights,
|
||||
NDArray* bias, NDArray* output, const LongType kH, const LongType kW, const LongType sH,
|
||||
const LongType sW, LongType pH, LongType pW, const LongType dH, const LongType dW, const int paddingMode,
|
||||
const int isNCHW, const int wFormat) {
|
||||
BUILD_SINGLE_SELECTOR_TWICE(
|
||||
input->dataType(), depthwiseConv2d_,
|
||||
(block, input, weights, bias, output, kH, kW, sH, sW, pH, pW, dH, dW, paddingMode, isNCHW, wFormat),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,145 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/helpers/col2im.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
#include <ops/declarable/helpers/im2col.h>
|
||||
#if NOT_EXCLUDED(OP_col2im) && NOT_EXCLUDED(OP_im2col)
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y>
|
||||
static void depthwiseConv2dBP_(NDArray* input, NDArray* weights, NDArray* bias, NDArray* gradO,
|
||||
NDArray* gradI, NDArray* gradW, NDArray* gradB, const LongType kH, const LongType kW, const LongType sH,
|
||||
const LongType sW, LongType pH, LongType pW, const LongType dH, const LongType dW, const int paddingMode,
|
||||
const int isNCHW, const int wFormat) {
|
||||
// input [bS, iH, iW, iC] (NDHWC) or [bS, iC, iH, iW] (NCDHW)
|
||||
// weights [kH, kW, iC, mC], [mC, iC, kH, kW], [mC, kH, kW, iC]
|
||||
// bias [oC] = [iC*mC]
|
||||
// gradO [bS, oH, oW, oC] (NDHWC) or [bS, oC, oH, oW] (NCDHW), epsilon_next
|
||||
// gradI [bS, iH, iW, iC] (NDHWC) or [bS, iC, iH, iW] (NCDHW), epsilon
|
||||
// gradW [kH, kW, iC, mC], [mC, iC, kH, kW], [mC, kH, kW, iC]
|
||||
// gradB [oC]
|
||||
|
||||
// kH filter(kernel) height
|
||||
// kW filter(kernel) width
|
||||
// sH strides height
|
||||
// sW strides width
|
||||
// pH paddings height
|
||||
// pW paddings width
|
||||
// dH dilations height
|
||||
// dW dilations width
|
||||
// paddingMode 0-VALID, 1-SAME
|
||||
// isNCHW 0-NHWC, 1-NCHW
|
||||
|
||||
LongType bS, iC, iH, iW, mC, oC, oH, oW; // batch size, input channels, input height/width, channels multiplier(oC =
|
||||
// iC*mC), output channels, output height/width
|
||||
LongType indIOioC, indIiH, indWmC, indWiC, indWkH, indOoH; // corresponding indexes
|
||||
ConvolutionUtils::getSizesAndIndexesConv2d(isNCHW, wFormat, *input, *gradO, bS, iC, iH, iW, oC, oH, oW, indIOioC,
|
||||
indIiH, indWiC, indWmC, indWkH, indOoH);
|
||||
mC = weights->sizeAt(indWmC); // channels multiplier
|
||||
|
||||
std::vector<std::vector<sd::LongType>> modifColumns = {
|
||||
{1, 2, 3, 0, 4, 5}, {iC, kH * kW, bS * oH * oW}}; // [bS,iC,kH,kW,oH,oW] -> [iC, kH*kW, bS*oH*oW]
|
||||
std::vector<std::vector<sd::LongType>> modifGradO1, modifGradO2, modifWeights;
|
||||
std::vector<sd::LongType> gradOreShape;
|
||||
|
||||
if (!isNCHW) {
|
||||
gradOreShape = {bS, oH, oW, iC, mC}; // [bS,oH,oW,iC*mC] -> [bS,oH,oW,iC,mC]
|
||||
modifGradO1 = {{3, 0, 1, 2, 4},
|
||||
{iC, bS * oH * oW, mC}}; // [bS,oH,oW,iC,mC] -> [iC,bS,oH,oW,mC] -> [iC,bS*oH*oW,mC]
|
||||
modifGradO2 = {{3, 0, 1, 2}, {iC, mC, bS * oH * oW}}; // [bS,oH,oW,iC*mC] -> [iC*mC,bS,oH,oW] -> [iC,mC,bS*oH*oW]
|
||||
std::vector<sd::LongType> perm = {0,3,1,2};
|
||||
input = input->permute(perm, false, false); // [bS,iH,iW,iC] -> [bS,iC,iH,iW]
|
||||
gradI = gradI->permute(perm, false, false); // [bS,iH,iW,iC] -> [bS,iC,iH,iW]
|
||||
} else {
|
||||
gradOreShape = {bS, iC, mC, oH, oW}; // [bS,iC*mC,oH,oW] -> [bS,iC,mC,oH,oW]
|
||||
modifGradO1 = {{1, 0, 3, 4, 2},
|
||||
{iC, bS * oH * oW, mC}}; // [bS,iC,mC,oH,oW] -> [iC,bS,oH,oW,mC] -> [iC,bS*oH*oW,mC]
|
||||
modifGradO2 = {{1, 0, 2, 3}, {iC, mC, bS * oH * oW}}; // [bS,iC*mC,oH,oW] -> [iC*mC,bS,oH,oW] -> [iC,mC,bS*oH*oW]
|
||||
}
|
||||
|
||||
if (0 == wFormat)
|
||||
modifWeights = {{2, 0, 1, 3}, {iC, kH * kW, mC}};
|
||||
else if (1 == wFormat)
|
||||
modifWeights = {{1, 2, 3, 0}, {iC, kH * kW, mC}};
|
||||
else
|
||||
modifWeights = {{3, 1, 2, 0}, {iC, kH * kW, mC}};
|
||||
|
||||
if (paddingMode == 1) // SAME
|
||||
ConvolutionUtils::calcPadding2D(pH, pW, oH, oW, iH, iW, kH, kW, sH, sW, dH, dW);
|
||||
|
||||
std::vector<LongType> colShape = {bS, iC, kH, kW, oH, oW};
|
||||
NDArray columns(input->ordering(), colShape, input->dataType(), input->getContext());
|
||||
NDArray *gradOreshaped = gradO->reshape(gradO->ordering(), gradOreShape);
|
||||
|
||||
// ----- calculation of gradW and gradB ----- //
|
||||
NDArray *zero = NDArrayFactory::create(0.f, input->getContext());
|
||||
helpers::im2col(
|
||||
*input->getContext(), *input, columns, kH, kW, sH, sW, pH, pW, dH, dW,
|
||||
*zero); // [bS, iC, iH, iW] is convoluted to [bS, iC, kH, kW, oH, oW]
|
||||
sd::MmulHelper::tensorDot(&columns, gradOreshaped, gradW, modifColumns, modifGradO1,
|
||||
modifWeights); // [iC, kW*kH, bS*oH*oW] x [iC, bS*oH*oW, mC] = [iC, kH*kW, mC]
|
||||
|
||||
delete zero;
|
||||
// ----- calculation of gradB ----- //
|
||||
if (gradB) {
|
||||
NDArray* gradBR = gradB;
|
||||
std::vector<LongType> shape = {gradB->lengthOf()};
|
||||
if (gradB->rankOf() == 2) gradBR =gradB->reshape(gradB->ordering(), shape, false);
|
||||
std::vector<sd::LongType> axes = {0, indOoH, indOoH + 1};
|
||||
gradO->reduceAlongDimension(reduce::Sum, gradBR, &axes); // sum over bS, oH, oW
|
||||
|
||||
if (gradBR != gradB) delete gradBR;
|
||||
}
|
||||
|
||||
//----- calculation of gradI -----//
|
||||
sd::MmulHelper::tensorDot(weights, gradO, &columns, modifWeights, modifGradO2,
|
||||
modifColumns); // [iC, kH*kW, mC] x [iC, mC, bS*oH*oW] = [iC, kW*kH, bS*oH*oW]
|
||||
helpers::col2im(*input->getContext(), &columns, gradI, sH, sW, pH, pW, iH, iW, dH,
|
||||
dW); // [bS, iC, kH, kW, oH, oW] is de-convoluted to [bS, iC, iH, iW]
|
||||
|
||||
if (!isNCHW) {
|
||||
delete input;
|
||||
delete gradI;
|
||||
}
|
||||
|
||||
delete gradOreshaped;
|
||||
}
|
||||
|
||||
void ConvolutionUtils::depthwiseConv2dBP(graph::Context& block, NDArray* input, NDArray* weights,
|
||||
NDArray* bias, NDArray* gradO, NDArray* gradI, NDArray* gradW,
|
||||
NDArray* gradB, const LongType kH, const LongType kW, const LongType sH, const LongType sW, LongType pH,
|
||||
LongType pW, const LongType dH, const LongType dW, const int paddingMode, const int isNCHW,
|
||||
const int wFormat) {
|
||||
BUILD_SINGLE_SELECTOR_TWICE(
|
||||
input->dataType(), depthwiseConv2dBP_,
|
||||
(input, weights, bias, gradO, gradI, gradW, gradB, kH, kW, sH, sW, pH, pW, dH, dW, paddingMode, isNCHW, wFormat),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,236 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
#include <stdexcept>
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void pooling2d_(sd::graph::Context& block, NDArray& input, NDArray& output, const LongType kH, const LongType kW,
|
||||
const LongType sH, const LongType sW, const LongType pH, const LongType pW, const LongType dH, const LongType dW,
|
||||
const int poolingMode, const int extraParam0) {
|
||||
// Cache shape information
|
||||
const auto inShapeInfo = input.shapeInfo();
|
||||
const auto outShapeInfo = output.shapeInfo();
|
||||
|
||||
// Cache input dimensions
|
||||
const auto* inShape = shape::shapeOf(inShapeInfo);
|
||||
const LongType bS = inShape[0];
|
||||
const LongType iC = inShape[1];
|
||||
const LongType iH = inShape[2];
|
||||
const LongType iW = inShape[3];
|
||||
|
||||
// Cache output dimensions
|
||||
const auto* outShape = shape::shapeOf(outShapeInfo);
|
||||
const LongType oH = outShape[2];
|
||||
const LongType oW = outShape[3];
|
||||
|
||||
// Cache strides
|
||||
const auto* inStride = shape::stride(inShapeInfo);
|
||||
const auto* outStride = shape::stride(outShapeInfo);
|
||||
|
||||
const sd::LongType iStride0 = inStride[0];
|
||||
const sd::LongType iStride1 = inStride[1];
|
||||
const sd::LongType iStride2 = inStride[2];
|
||||
const sd::LongType iStride3 = inStride[3];
|
||||
const sd::LongType oStride0 = outStride[0];
|
||||
const sd::LongType oStride1 = outStride[1];
|
||||
const sd::LongType oStride2 = outStride[2];
|
||||
const sd::LongType oStride3 = outStride[3];
|
||||
|
||||
T* out = output.bufferAsT<T>();
|
||||
T* in = const_cast<NDArray&>(input).bufferAsT<T>();
|
||||
|
||||
const int kHEff = kH + (kH - 1) * (dH - 1);
|
||||
const int kWEff = kW + (kW - 1) * (dW - 1);
|
||||
|
||||
const sd::LongType iStep2 = dH * iStride2;
|
||||
const sd::LongType iStep3 = dW * iStride3;
|
||||
const int kProd = kH * kW;
|
||||
|
||||
if (poolingMode == 0) { // max
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType hstart, wstart, hend, wend;
|
||||
T* pIn;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (hstart < 0)
|
||||
hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0)
|
||||
wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (hend > iH)
|
||||
hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW)
|
||||
wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
hstart *= iStride2;
|
||||
hend *= iStride2;
|
||||
wstart *= iStride3;
|
||||
wend *= iStride3;
|
||||
|
||||
T max = -DataTypeUtils::max<T>();
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep2)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep3) {
|
||||
T val = pIn[kh + kw];
|
||||
if (val > max) max = val;
|
||||
}
|
||||
out[b * oStride0 + c * oStride1 + oh * oStride2 + ow * oStride3] = max;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
}
|
||||
/*************************************************************************/
|
||||
else if (poolingMode == 1) { // avg
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType hstart, wstart, hend, wend;
|
||||
T* pIn;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (hstart < 0)
|
||||
hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0)
|
||||
wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (hend > iH)
|
||||
hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW)
|
||||
wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
hstart *= iStride2;
|
||||
hend *= iStride2;
|
||||
wstart *= iStride3;
|
||||
wend *= iStride3;
|
||||
|
||||
T sum = static_cast<T>(0.f);
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep2)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep3)
|
||||
sum += pIn[kh + kw];
|
||||
|
||||
if (extraParam0 == 0) { // Exclude padding
|
||||
int a = (hend - hstart) / iStep2 + ((hend - hstart) % iStep2 == 0 ? 0 : 1);
|
||||
int r = (wend - wstart) / iStep3 + ((wend - wstart) % iStep3 == 0 ? 0 : 1);
|
||||
sum /= static_cast<T>(a * r); // Accounts for dilation
|
||||
} else if (extraParam0 == 1) // Include padding
|
||||
sum /= kProd;
|
||||
|
||||
out[b * oStride0 + c * oStride1 + oh * oStride2 + ow * oStride3] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
}
|
||||
/*************************************************************************/
|
||||
else if (poolingMode == 2) { // pnorm
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType hstart, wstart, hend, wend;
|
||||
T* pIn;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (hstart < 0)
|
||||
hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0)
|
||||
wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (hend > iH)
|
||||
hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW)
|
||||
wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
hstart *= iStride2;
|
||||
hend *= iStride2;
|
||||
wstart *= iStride3;
|
||||
wend *= iStride3;
|
||||
|
||||
T sum = static_cast<T>(0.f);
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep2)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep3)
|
||||
sum += sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(pIn[kh + kw]), static_cast<T>(extraParam0));
|
||||
|
||||
sum = sd::math::sd_pow<T, T, T>(sum, static_cast<T>((T)1.f) / extraParam0);
|
||||
|
||||
out[b * oStride0 + c * oStride1 + oh * oStride2 + ow * oStride3] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
} else {
|
||||
char errorMsg[512];
|
||||
snprintf(errorMsg, sizeof(errorMsg),
|
||||
"ConvolutionUtils::pooling2d: pooling mode argument can take three values only: 0, 1, 2, but got %i instead!",
|
||||
poolingMode);
|
||||
THROW_EXCEPTION(errorMsg);
|
||||
}
|
||||
}
|
||||
|
||||
void ConvolutionUtils::pooling2d(sd::graph::Context& block, NDArray& input, NDArray& output, const LongType kH,
|
||||
const LongType kW, const LongType sH, const LongType sW, const LongType pH, const LongType pW, const LongType dH,
|
||||
const LongType dW, const PoolingType poolingMode, const int extraParam0) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), pooling2d_,
|
||||
(block, input, output, kH, kW, sH, sW, pH, pW, dH, dW, poolingMode, extraParam0),
|
||||
SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,317 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void pooling2dBP_(sd::graph::Context& block, NDArray& input, NDArray& gradO, NDArray& gradI,
|
||||
const LongType kH, const LongType kW, const LongType sH, const LongType sW, const LongType pH, const LongType pW,
|
||||
const LongType dH, const LongType dW, const int poolingMode, const int extraParam0) {
|
||||
// input [bS, iC, iH, iW]
|
||||
// gradI [bS, iC, iH, iW] -> gradI is output in this function
|
||||
// gradO [bS, iC, oH, oW]
|
||||
|
||||
// initial zeroing of gradI
|
||||
gradI.nullify();
|
||||
|
||||
T* in = const_cast<NDArray&>(input).bufferAsT<T>();
|
||||
T* gO = const_cast<NDArray&>(gradO).bufferAsT<T>();
|
||||
T* gI = gradI.bufferAsT<T>();
|
||||
|
||||
const int kHEff = kH + (kH - 1) * (dH - 1);
|
||||
const int kWEff = kW + (kW - 1) * (dW - 1);
|
||||
|
||||
const int bS = gradI.sizeAt(0);
|
||||
const int iC = gradI.sizeAt(1);
|
||||
const int iH = gradI.sizeAt(2);
|
||||
const int iW = gradI.sizeAt(3);
|
||||
const int oC = gradO.sizeAt(1);
|
||||
const int oH = gradO.sizeAt(2);
|
||||
const int oW = gradO.sizeAt(3);
|
||||
|
||||
// sd_debug("MKL-DNN is not used for pooling2d_bp!\n", 0);
|
||||
|
||||
const sd::LongType iStride0 = input.stridesOf()[0];
|
||||
const sd::LongType iStride1 = input.stridesOf()[1];
|
||||
const sd::LongType iStride2 = input.stridesOf()[2];
|
||||
const sd::LongType iStride3 = input.stridesOf()[3];
|
||||
const sd::LongType gIStride0 = gradI.stridesOf()[0];
|
||||
const sd::LongType gIStride1 = gradI.stridesOf()[1];
|
||||
const sd::LongType gIStride2 = gradI.stridesOf()[2];
|
||||
const sd::LongType gIStride3 = gradI.stridesOf()[3];
|
||||
const sd::LongType oStride0 = gradO.stridesOf()[0];
|
||||
const sd::LongType oStride1 = gradO.stridesOf()[1];
|
||||
const sd::LongType oStride2 = gradO.stridesOf()[2];
|
||||
const sd::LongType oStride3 = gradO.stridesOf()[3];
|
||||
const sd::LongType iStep2 = dH * iStride2;
|
||||
const sd::LongType iStep3 = dW * iStride3;
|
||||
const sd::LongType gIStep2 = dH * gIStride2;
|
||||
const sd::LongType gIStep3 = dW * gIStride3;
|
||||
const int kProd = kH * kW;
|
||||
|
||||
const bool sameStrides =
|
||||
iStride0 == gIStride0 && iStride1 == gIStride1 && iStride2 == gIStride2 && iStride3 == gIStride3;
|
||||
|
||||
if (poolingMode == 0) { // max
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType hstart, wstart, hend, wend, maxKH, maxKW;
|
||||
T sum, valO, *pIn, *pgI;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (hstart < 0)
|
||||
hstart +=
|
||||
dH * ((-hstart + dH - 1) /
|
||||
dH);
|
||||
if (wstart < 0)
|
||||
wstart +=
|
||||
dW * ((-wstart + dW - 1) /
|
||||
dW);
|
||||
if (hend > iH)
|
||||
hend -=
|
||||
dH * ((hend - iH + dH - 1) /
|
||||
dH);
|
||||
if (wend > iW)
|
||||
wend -=
|
||||
dW * ((wend - iW + dW - 1) /
|
||||
dW);
|
||||
|
||||
sum = -DataTypeUtils::max<T>();
|
||||
valO = gO[b * oStride0 + c * oStride1 + oh * oStride2 + ow * oStride3];
|
||||
|
||||
if (sameStrides) {
|
||||
hstart *= iStride2;
|
||||
hend *= iStride2;
|
||||
wstart *= iStride3;
|
||||
wend *= iStride3;
|
||||
|
||||
// we set these to default values
|
||||
maxKH = hstart;
|
||||
maxKW = wstart;
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep2)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep3) {
|
||||
T valIn = pIn[kh + kw];
|
||||
if (valIn > sum) {
|
||||
sum = valIn;
|
||||
maxKH = kh;
|
||||
maxKW = kw;
|
||||
}
|
||||
}
|
||||
gI[pIn - in + maxKH + maxKW] += valO;
|
||||
} else {
|
||||
// we set these to default values
|
||||
maxKH = hstart;
|
||||
maxKW = wstart;
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += dH)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += dW) {
|
||||
T valIn = pIn[kh * iStride2 + kw * iStride3];
|
||||
if (valIn > sum) {
|
||||
sum = valIn;
|
||||
maxKH = kh;
|
||||
maxKW = kw;
|
||||
}
|
||||
}
|
||||
|
||||
gI[b * gIStride0 + c * gIStride1 + maxKH * gIStride2 + maxKW * gIStride3] += valO;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
}
|
||||
/*************************************************************************/
|
||||
else if (poolingMode == 1) { // avg
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType hstart, wstart, hend, wend, maxKH, maxKW;
|
||||
T sum, valO, *pIn, *pgI;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pgI = gI + b * gIStride0 + c * gIStride1;
|
||||
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (hstart < 0)
|
||||
hstart +=
|
||||
dH * ((-hstart + dH - 1) /
|
||||
dH); // (sd::LongType)sd::math::sd_ceil<T,T>(static_cast<T>(-hstart) / static_cast<T>(dH));
|
||||
if (wstart < 0)
|
||||
wstart +=
|
||||
dW * ((-wstart + dW - 1) /
|
||||
dW); //(sd::LongType)sd::math::sd_ceil<T,T>(static_cast<T>(-wstart) / static_cast<T>(dW));
|
||||
if (hend > iH)
|
||||
hend -=
|
||||
dH * ((hend - iH + dH - 1) /
|
||||
dH); //(sd::LongType)sd::math::sd_ceil<T,T>(static_cast<T>(hend-iH) / static_cast<T>(dH));
|
||||
if (wend > iW)
|
||||
wend -=
|
||||
dW * ((wend - iW + dW - 1) /
|
||||
dW); //(sd::LongType)sd::math::sd_ceil<T,T>(static_cast<T>(wend-iW) / static_cast<T>(dW));
|
||||
|
||||
hstart *= gIStride2;
|
||||
hend *= gIStride2;
|
||||
wstart *= gIStride3;
|
||||
wend *= gIStride3;
|
||||
|
||||
valO = gO[b * oStride0 + c * oStride1 + oh * oStride2 + ow * oStride3];
|
||||
|
||||
if ((int)extraParam0 == 0) // Exclude padding
|
||||
valO /=
|
||||
static_cast<T>(sd::math::sd_ceil<double, T>(static_cast<double>(hend - hstart) /
|
||||
static_cast<double>(gIStep2))) *
|
||||
static_cast<T>(sd::math::sd_ceil<double, T>(
|
||||
static_cast<double>(wend - wstart) / static_cast<double>(gIStep3))); // Accounts for dilation
|
||||
else if ((int)extraParam0 == 1) // Include padding
|
||||
valO /= kProd;
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += gIStep2)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += gIStep3) pgI[kh + kw] += valO;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
}
|
||||
/*************************************************************************/
|
||||
else if (poolingMode == 2) { // pnorm
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType hstart, wstart, hend, wend, maxKH, maxKW;
|
||||
T sum, valO, *pIn, *pgI;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
pgI = sameStrides ? gI + (pIn - in) : gI + b * gIStride0 + c * gIStride1;
|
||||
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (hstart < 0)
|
||||
hstart +=
|
||||
dH * ((-hstart + dH - 1) /
|
||||
dH); // (sd::LongType)sd::math::sd_ceil<T,T>(static_cast<T>(-hstart) / static_cast<T>(dH));
|
||||
if (wstart < 0)
|
||||
wstart +=
|
||||
dW * ((-wstart + dW - 1) /
|
||||
dW); //(sd::LongType)sd::math::sd_ceil<T,T>(static_cast<T>(-wstart) / static_cast<T>(dW));
|
||||
if (hend > iH)
|
||||
hend -=
|
||||
dH * ((hend - iH + dH - 1) /
|
||||
dH); //(sd::LongType)sd::math::sd_ceil<T,T>(static_cast<T>(hend-iH) / static_cast<T>(dH));
|
||||
if (wend > iW)
|
||||
wend -=
|
||||
dW * ((wend - iW + dW - 1) /
|
||||
dW); //(sd::LongType)sd::math::sd_ceil<T,T>(static_cast<T>(wend-iW) / static_cast<T>(dW));
|
||||
|
||||
sum = static_cast<T>(0.f);
|
||||
valO = gO[b * oStride0 + c * oStride1 + oh * oStride2 + ow * oStride3];
|
||||
|
||||
if (sameStrides) {
|
||||
hstart *= iStride2;
|
||||
hend *= iStride2;
|
||||
wstart *= iStride3;
|
||||
wend *= iStride3;
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep2)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep3)
|
||||
sum += sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(pIn[kh + kw]), static_cast<T>(extraParam0));
|
||||
|
||||
valO *= sd::math::sd_pow<T, T, T>(sum, ((T)1. - extraParam0) / extraParam0);
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep2)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep3)
|
||||
pgI[kh + kw] += valO *
|
||||
sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(pIn[kh + kw]), static_cast<T>(extraParam0) - 1.f) *
|
||||
sd::math::sd_sgn<T, T>(pIn[kh + kw]);
|
||||
} else {
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += dH)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += dW)
|
||||
sum +=
|
||||
sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(pIn[kh * iStride2 + kw * iStride3]), static_cast<T>(extraParam0));
|
||||
|
||||
valO *= sd::math::sd_pow<T, T, T>(sum, ((T)1. - extraParam0) / extraParam0);
|
||||
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += dH) {
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += dW) {
|
||||
const auto inVal = pIn[kh * iStride2 + kw * iStride3];
|
||||
pgI[kh * gIStride2 + kw * gIStride3] +=
|
||||
valO * sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(inVal), static_cast<T>(extraParam0) - static_cast<T>(1.f)) *
|
||||
sd::math::sd_sgn<T, T>(inVal);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
} else {
|
||||
sd_printf(
|
||||
"ConvolutionUtils::pooling2dBP: pooling mode argument can take three values only: 0, 1, 2, but got %i instead "
|
||||
"!\n",
|
||||
poolingMode);
|
||||
THROW_EXCEPTION("Incorrect pooling2dBP mode");
|
||||
}
|
||||
}
|
||||
|
||||
void ConvolutionUtils::pooling2dBP(sd::graph::Context& block, NDArray& input, NDArray& gradO,
|
||||
NDArray& gradI, const LongType kH, const LongType kW, const LongType sH, const LongType sW, const LongType pH,
|
||||
const LongType pW, const LongType dH, const LongType dW, const int poolingMode,
|
||||
const int extraParam0) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), pooling2dBP_,
|
||||
(block, input, gradO, gradI, kH, kW, sH, sW, pH, pW, dH, dW, poolingMode, extraParam0),
|
||||
SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,251 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void pooling3d_(sd::graph::Context& block, NDArray& input, NDArray& output, const LongType kD, const LongType kH,
|
||||
const LongType kW, const LongType sD, const LongType sH, const LongType sW, const LongType pD, const LongType pH, const LongType pW,
|
||||
const LongType dD, const LongType dH, const LongType dW, const LongType poolingMode, const int extraParam0) {
|
||||
// input is [bS, iC, iD, iH, iW]
|
||||
// output is [bS, iC, oD, oH, oW]
|
||||
T* out = output.bufferAsT<T>();
|
||||
T* in = const_cast<NDArray&>(input).bufferAsT<T>();
|
||||
|
||||
const int kDEff = kD + (kD - 1) * (dD - 1);
|
||||
const int kHEff = kH + (kH - 1) * (dH - 1);
|
||||
const int kWEff = kW + (kW - 1) * (dW - 1);
|
||||
|
||||
const int bS = input.sizeAt(0);
|
||||
const int iC = input.sizeAt(1);
|
||||
const int iD = input.sizeAt(2);
|
||||
const int iH = input.sizeAt(3);
|
||||
const int iW = input.sizeAt(4);
|
||||
const int oC = output.sizeAt(1);
|
||||
const int oD = output.sizeAt(2);
|
||||
const int oH = output.sizeAt(3);
|
||||
const int oW = output.sizeAt(4);
|
||||
|
||||
sd_debug("MKL-DNN is not used for pooling3d!\n", 0);
|
||||
|
||||
const sd::LongType iStride0 = input.stridesOf()[0];
|
||||
const sd::LongType iStride1 = input.stridesOf()[1];
|
||||
const sd::LongType iStride2 = input.stridesOf()[2];
|
||||
const sd::LongType iStride3 = input.stridesOf()[3];
|
||||
const sd::LongType iStride4 = input.stridesOf()[4];
|
||||
const sd::LongType oStride0 = output.stridesOf()[0];
|
||||
const sd::LongType oStride1 = output.stridesOf()[1];
|
||||
const sd::LongType oStride2 = output.stridesOf()[2];
|
||||
const sd::LongType oStride3 = output.stridesOf()[3];
|
||||
const sd::LongType oStride4 = output.stridesOf()[4];
|
||||
const sd::LongType iStep2 = dD * iStride2;
|
||||
const sd::LongType iStep3 = dH * iStride3;
|
||||
const sd::LongType iStep4 = dW * iStride4;
|
||||
const int kProd = kD * kH * kW;
|
||||
|
||||
if (poolingMode == 0) { // max
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
sd::LongType dstart, hstart, wstart, dend, hend, wend;
|
||||
T sum, *pIn;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int od = start_z; od < stop_z; od += inc_z) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
|
||||
dstart = od * sD - pD;
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
dend = dstart + kDEff;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (dstart < 0) dstart += dD * ((-dstart + dD - 1) / dD);
|
||||
if (hstart < 0) hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0) wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (dend > iD) dend -= dD * ((dend - iD + dD - 1) / dD);
|
||||
if (hend > iH) hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW) wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
dstart *= iStride2;
|
||||
dend *= iStride2;
|
||||
hstart *= iStride3;
|
||||
hend *= iStride3;
|
||||
wstart *= iStride4;
|
||||
wend *= iStride4;
|
||||
|
||||
sum = -DataTypeUtils::max<T>();
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += iStep2)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep3)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep4) {
|
||||
T val = pIn[kd + kh + kw];
|
||||
if (val > sum) sum = val;
|
||||
}
|
||||
|
||||
out[b * oStride0 + c * oStride1 + od * oStride2 + oh * oStride3 + ow * oStride4] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1, 0, oD, 1);
|
||||
}
|
||||
/*************************************************************************/
|
||||
else if (poolingMode == 1) { // avg
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
sd::LongType dstart, hstart, wstart, dend, hend, wend;
|
||||
T sum, *pIn;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int od = start_z; od < stop_z; od += inc_z) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
|
||||
dstart = od * sD - pD;
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
dend = dstart + kDEff;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (dstart < 0) dstart += dD * ((-dstart + dD - 1) / dD);
|
||||
if (hstart < 0) hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0) wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (dend > iD) dend -= dD * ((dend - iD + dD - 1) / dD);
|
||||
if (hend > iH) hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW) wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
dstart *= iStride2;
|
||||
dend *= iStride2;
|
||||
hstart *= iStride3;
|
||||
hend *= iStride3;
|
||||
wstart *= iStride4;
|
||||
wend *= iStride4;
|
||||
|
||||
sum = static_cast<T>(0.);
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += iStep2)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep3)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep4) sum += pIn[kd + kh + kw];
|
||||
|
||||
if (extraParam0 == 0) // Exclude padding
|
||||
sum /=
|
||||
sd::math::sd_ceil<double, T>(static_cast<double>(dend - dstart) / static_cast<double>(iStep2)) *
|
||||
sd::math::sd_ceil<double, T>(static_cast<double>(hend - hstart) / static_cast<double>(iStep3)) *
|
||||
sd::math::sd_ceil<double, T>(static_cast<double>(wend - wstart) /
|
||||
static_cast<double>(iStep4)); // Accounts for dilation
|
||||
else if (extraParam0 == 1) // Include padding
|
||||
sum /= kProd;
|
||||
|
||||
out[b * oStride0 + c * oStride1 + od * oStride2 + oh * oStride3 + ow * oStride4] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1, 0, oD, 1);
|
||||
}
|
||||
/*************************************************************************/
|
||||
else if (poolingMode == 2) { // pnorm
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
sd::LongType dstart, hstart, wstart, dend, hend, wend;
|
||||
T sum, *pIn;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int od = start_z; od < stop_z; od += inc_z) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
|
||||
dstart = od * sD - pD;
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
dend = dstart + kDEff;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (dstart < 0) dstart += dD * ((-dstart + dD - 1) / dD);
|
||||
if (hstart < 0) hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0) wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (dend > iD) dend -= dD * ((dend - iD + dD - 1) / dD);
|
||||
if (hend > iH) hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW) wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
dstart *= iStride2;
|
||||
dend *= iStride2;
|
||||
hstart *= iStride3;
|
||||
hend *= iStride3;
|
||||
wstart *= iStride4;
|
||||
wend *= iStride4;
|
||||
|
||||
sum = static_cast<T>(0.);
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += iStep2)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep3)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep4)
|
||||
sum += sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(pIn[kd + kh + kw]), static_cast<T>(extraParam0));
|
||||
|
||||
sum = sd::math::sd_pow<T, T, T>(sum, (T)1.f / extraParam0);
|
||||
|
||||
out[b * oStride0 + c * oStride1 + od * oStride2 + oh * oStride3 + ow * oStride4] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1, 0, oD, 1);
|
||||
} else {
|
||||
sd_printf(
|
||||
"ConvolutionUtils::pooling3d: pooling mode argument can take three values only: 0, 1, 2, but got %i instead "
|
||||
"!\n",
|
||||
poolingMode);
|
||||
THROW_EXCEPTION("Incorrect poooling3d mode");
|
||||
}
|
||||
}
|
||||
|
||||
void ConvolutionUtils::pooling3d(sd::graph::Context& block, NDArray& input, NDArray& output, const LongType kD,
|
||||
const LongType kH, const LongType kW, const LongType sD, const LongType sH, const LongType sW, const LongType pD,
|
||||
const LongType pH, const LongType pW, const LongType dD, const LongType dH, const LongType dW,
|
||||
const int poolingMode, const int extraParam0) {
|
||||
BUILD_SINGLE_SELECTOR(
|
||||
input.dataType(), pooling3d_,
|
||||
(block, input, output, kD, kH, kW, sD, sH, sW, pD, pH, pW, dD, dH, dW, poolingMode, extraParam0), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,325 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void pooling3dBP_(sd::graph::Context& block, NDArray& input, NDArray& gradO, NDArray& gradI,
|
||||
const LongType kD, const LongType kH, const LongType kW, const LongType sD, const LongType sH, const LongType sW,
|
||||
const LongType pD, const LongType pH, const LongType pW, const LongType dD, const LongType dH, const LongType dW,
|
||||
const int poolingMode, const int extraParam0) {
|
||||
// input [bS, iC, iD, iH, iW]
|
||||
// gradI [bS, iC, iD, iH, iW] -> gradI is output in this function
|
||||
// gradO [bS, iC, oD, oH, oW]
|
||||
|
||||
// initial zeroing of gradI
|
||||
gradI.nullify();
|
||||
|
||||
T* in = const_cast<NDArray&>(input).bufferAsT<T>();
|
||||
T* gO = const_cast<NDArray&>(gradO).bufferAsT<T>();
|
||||
T* gI = gradI.bufferAsT<T>();
|
||||
|
||||
const int kDEff = kD + (kD - 1) * (dD - 1);
|
||||
const int kHEff = kH + (kH - 1) * (dH - 1);
|
||||
const int kWEff = kW + (kW - 1) * (dW - 1);
|
||||
|
||||
const int bS = gradI.sizeAt(0);
|
||||
const int iC = gradI.sizeAt(1);
|
||||
const int iD = gradI.sizeAt(2);
|
||||
const int iH = gradI.sizeAt(3);
|
||||
const int iW = gradI.sizeAt(4);
|
||||
const int oC = gradO.sizeAt(1);
|
||||
const int oD = gradO.sizeAt(2);
|
||||
const int oH = gradO.sizeAt(3);
|
||||
const int oW = gradO.sizeAt(4);
|
||||
|
||||
sd_debug("MKL-DNN is not used for pooling3d_bp!\n", 0);
|
||||
|
||||
const sd::LongType iStride0 = input.stridesOf()[0];
|
||||
const sd::LongType iStride1 = input.stridesOf()[1];
|
||||
const sd::LongType iStride2 = input.stridesOf()[2];
|
||||
const sd::LongType iStride3 = input.stridesOf()[3];
|
||||
const sd::LongType iStride4 = input.stridesOf()[4];
|
||||
const sd::LongType gIStride0 = gradI.stridesOf()[0];
|
||||
const sd::LongType gIStride1 = gradI.stridesOf()[1];
|
||||
const sd::LongType gIStride2 = gradI.stridesOf()[2];
|
||||
const sd::LongType gIStride3 = gradI.stridesOf()[3];
|
||||
const sd::LongType gIStride4 = gradI.stridesOf()[4];
|
||||
const sd::LongType oStride0 = gradO.stridesOf()[0];
|
||||
const sd::LongType oStride1 = gradO.stridesOf()[1];
|
||||
const sd::LongType oStride2 = gradO.stridesOf()[2];
|
||||
const sd::LongType oStride3 = gradO.stridesOf()[3];
|
||||
const sd::LongType oStride4 = gradO.stridesOf()[4];
|
||||
const sd::LongType iStep2 = dD * iStride2;
|
||||
const sd::LongType iStep3 = dH * iStride3;
|
||||
const sd::LongType iStep4 = dW * iStride4;
|
||||
const sd::LongType gIStep2 = dD * gIStride2;
|
||||
const sd::LongType gIStep3 = dH * gIStride3;
|
||||
const sd::LongType gIStep4 = dW * gIStride4;
|
||||
const int kProd = kD * kH * kW;
|
||||
|
||||
const bool sameStrides = iStride0 == gIStride0 && iStride1 == gIStride1 && iStride2 == gIStride2 &&
|
||||
iStride3 == gIStride3 && iStride4 == gIStride4;
|
||||
|
||||
if (poolingMode == 0) { // max
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType dstart, hstart, wstart, dend, hend, wend, maxKD, maxKH, maxKW;
|
||||
T sum, valO, *pIn, *pgI;
|
||||
|
||||
for (int b = start_x; b < stop_x; b++) {
|
||||
for (int c = start_y; c < stop_y; c++) {
|
||||
for (int od = 0; od < oD; od++) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
|
||||
dstart = od * sD - pD;
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
dend = dstart + kDEff;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (dstart < 0) dstart += dD * ((-dstart + dD - 1) / dD);
|
||||
if (hstart < 0) hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0) wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (dend > iD) dend -= dD * ((dend - iD + dD - 1) / dD);
|
||||
if (hend > iH) hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW) wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
sum = -DataTypeUtils::max<T>();
|
||||
valO = gO[b * oStride0 + c * oStride1 + od * oStride2 + oh * oStride3 + ow * oStride4];
|
||||
|
||||
if (sameStrides) {
|
||||
dstart *= iStride2;
|
||||
dend *= iStride2;
|
||||
hstart *= iStride3;
|
||||
hend *= iStride3;
|
||||
wstart *= iStride4;
|
||||
wend *= iStride4;
|
||||
|
||||
maxKD = dstart;
|
||||
maxKH = hstart;
|
||||
maxKW = wstart;
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += iStep2)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep3)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep4) {
|
||||
T valIn = pIn[kd + kh + kw];
|
||||
if (valIn > sum) {
|
||||
sum = valIn;
|
||||
maxKD = kd;
|
||||
maxKH = kh;
|
||||
maxKW = kw;
|
||||
}
|
||||
}
|
||||
gI[pIn - in + maxKD + maxKH + maxKW] += valO;
|
||||
} else {
|
||||
// we set these to default values
|
||||
maxKH = hstart;
|
||||
maxKW = wstart;
|
||||
maxKD = dstart;
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += dD)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += dH)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += dW) {
|
||||
T valIn = pIn[kd * iStride2 + kh * iStride3 + kw * iStride4];
|
||||
if (valIn > sum) {
|
||||
sum = valIn;
|
||||
maxKD = kd;
|
||||
maxKH = kh;
|
||||
maxKW = kw;
|
||||
}
|
||||
}
|
||||
|
||||
gI[b * gIStride0 + c * gIStride1 + maxKD * gIStride2 + maxKH * gIStride3 + maxKW * gIStride4] += valO;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
}
|
||||
/*************************************************************************/
|
||||
else if (poolingMode == 1) { // avg
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType dstart, hstart, wstart, dend, hend, wend, maxKD, maxKH, maxKW;
|
||||
T sum, valO, *pIn, *pgI;
|
||||
|
||||
for (int b = start_x; b < stop_x; b++) {
|
||||
for (int c = start_y; c < stop_y; c++) {
|
||||
for (int od = 0; od < oD; od++) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pgI = gI + b * gIStride0 + c * gIStride1;
|
||||
|
||||
dstart = od * sD - pD;
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
dend = dstart + kDEff;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (dstart < 0) dstart += dD * ((-dstart + dD - 1) / dD);
|
||||
if (hstart < 0) hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0) wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (dend > iD) dend -= dD * ((dend - iD + dD - 1) / dD);
|
||||
if (hend > iH) hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW) wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
dstart *= gIStride2;
|
||||
dend *= gIStride2;
|
||||
hstart *= gIStride3;
|
||||
hend *= gIStride3;
|
||||
wstart *= gIStride4;
|
||||
wend *= gIStride4;
|
||||
|
||||
valO = gO[b * oStride0 + c * oStride1 + od * oStride2 + oh * oStride3 + ow * oStride4];
|
||||
|
||||
if (extraParam0 == 0) // Exclude padding
|
||||
valO /=
|
||||
sd::math::sd_ceil<double, T>(static_cast<double>(dend - dstart) / static_cast<double>(gIStep2)) *
|
||||
sd::math::sd_ceil<double, T>(static_cast<double>(hend - hstart) / static_cast<double>(gIStep3)) *
|
||||
sd::math::sd_ceil<double, T>(static_cast<double>(wend - wstart) /
|
||||
static_cast<double>(gIStep4)); // Accounts for dilation
|
||||
else if (extraParam0 == 1) // Include padding
|
||||
valO /= kProd;
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += gIStep2)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += gIStep3)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += gIStep4) pgI[kd + kh + kw] += valO;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
}
|
||||
/*************************************************************************/
|
||||
else if (poolingMode == 2) { // pnorm
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType dstart, hstart, wstart, dend, hend, wend, maxKD, maxKH, maxKW;
|
||||
T sum, valO, *pIn, *pgI;
|
||||
|
||||
for (int b = start_x; b < stop_x; b++) {
|
||||
for (int c = start_y; c < stop_y; c++) {
|
||||
for (int od = 0; od < oD; od++) {
|
||||
for (int oh = 0; oh < oH; ++oh) {
|
||||
for (int ow = 0; ow < oW; ++ow) {
|
||||
pIn = in + b * iStride0 + c * iStride1;
|
||||
pgI = gI + (pIn - in);
|
||||
|
||||
dstart = od * sD - pD;
|
||||
hstart = oh * sH - pH;
|
||||
wstart = ow * sW - pW;
|
||||
dend = dstart + kDEff;
|
||||
hend = hstart + kHEff;
|
||||
wend = wstart + kWEff;
|
||||
|
||||
if (dstart < 0) dstart += dD * ((-dstart + dD - 1) / dD);
|
||||
if (hstart < 0) hstart += dH * ((-hstart + dH - 1) / dH);
|
||||
if (wstart < 0) wstart += dW * ((-wstart + dW - 1) / dW);
|
||||
if (dend > iD) dend -= dD * ((dend - iD + dD - 1) / dD);
|
||||
if (hend > iH) hend -= dH * ((hend - iH + dH - 1) / dH);
|
||||
if (wend > iW) wend -= dW * ((wend - iW + dW - 1) / dW);
|
||||
|
||||
sum = static_cast<T>(0.);
|
||||
valO = gO[b * oStride0 + c * oStride1 + od * oStride2 + oh * oStride3 + ow * oStride4];
|
||||
|
||||
if (sameStrides) {
|
||||
dstart *= iStride2;
|
||||
dend *= iStride2;
|
||||
hstart *= iStride3;
|
||||
hend *= iStride3;
|
||||
wstart *= iStride4;
|
||||
wend *= iStride4;
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += iStep2)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep3)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep4)
|
||||
sum += sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(pIn[kd + kh + kw]), static_cast<T>(extraParam0));
|
||||
|
||||
valO *= sd::math::sd_pow<T, T, T>(sum, ((T)1.f - extraParam0) / extraParam0);
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += iStep2)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += iStep3)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += iStep4)
|
||||
pgI[kd + kh + kw] +=
|
||||
valO *
|
||||
sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(pIn[kd + kh + kw]), extraParam0 - (T)1.f) *
|
||||
sd::math::sd_sgn<T, T>(pIn[kd + kh + kw]);
|
||||
} else {
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += dD)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += dH)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += dW)
|
||||
sum += sd::math::sd_pow<T, T, T>(
|
||||
sd::math::sd_abs<T,T>(pIn[kd * iStride2 + kh * iStride3 + kw * iStride4]), static_cast<T>(extraParam0));
|
||||
|
||||
valO *= sd::math::sd_pow<T, T, T>(sum, ((T)1.f - extraParam0) / extraParam0);
|
||||
|
||||
for (sd::LongType kd = dstart; kd < dend; kd += dD)
|
||||
for (sd::LongType kh = hstart; kh < hend; kh += dH)
|
||||
for (sd::LongType kw = wstart; kw < wend; kw += dW) {
|
||||
const auto inVal = pIn[kD * iStride2 + kh * iStride3 + kw * iStride4];
|
||||
pgI[kd * gIStride2 + kh * gIStride3 + kw * gIStride4] +=
|
||||
valO * sd::math::sd_pow<T, T, T>(sd::math::sd_abs<T,T>(inVal), static_cast<T>(extraParam0) - 1.f) *
|
||||
sd::math::sd_sgn<T, T>(inVal);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1);
|
||||
} else {
|
||||
char errorMsg[512];
|
||||
snprintf(errorMsg, sizeof(errorMsg),
|
||||
"ConvolutionUtils::pooling3dBP: pooling mode argument can take three values only: 0, 1, 2, but got %i instead!",
|
||||
poolingMode);
|
||||
THROW_EXCEPTION(errorMsg);
|
||||
}
|
||||
}
|
||||
|
||||
void ConvolutionUtils::pooling3dBP(sd::graph::Context& block, NDArray& input, NDArray& gradO,
|
||||
NDArray& gradI, const LongType kD, const LongType kH, const LongType kW, const LongType sD, const LongType sH,
|
||||
const LongType sW, const LongType pD, const LongType pH, const LongType pW, const LongType dD, const LongType dH,
|
||||
const LongType dW, const int poolingMode, const int extraParam0) {
|
||||
BUILD_SINGLE_SELECTOR(
|
||||
input.dataType(), pooling3dBP_,
|
||||
(block, input, gradO, gradI, kD, kH, kW, sD, sH, sW, pD, pH, pW, dD, dH, dW, poolingMode, extraParam0),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,88 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
#if NOT_EXCLUDED(OP_col2im) && NOT_EXCLUDED(OP_im2col)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y>
|
||||
static void sconv2d_(graph::Context& block, NDArray* input, NDArray* weightsDepth,
|
||||
NDArray* weightsPoint, NDArray* bias, NDArray* output, const LongType kH, const LongType kW,
|
||||
const LongType sH, const LongType sW, LongType pH, LongType pW, const LongType dH, const LongType dW, const int paddingMode,
|
||||
const int isNCHW, const int wFormat) {
|
||||
// input [bS, iH, iW, iC] (NHWC) or [bS, iC, iH, iW] (NCHW)
|
||||
// weightsDepth [kH, kW, iC, mC], [mC, iC, kH, kW], [mC, kH, kW, iC]
|
||||
// weightsPoint [1, 1, iC*mC, oC], [oC, iC*mC, 1, 1], [oC, 1, 1, iC*mC]
|
||||
// bias [oC], oC = iC*mC if weightsPoint=nullptr
|
||||
// output is [bS, oH, oW, oC] (NHWC) or [bS, oC, oH, oW] (NCHW)
|
||||
|
||||
// kH filter(kernel) height
|
||||
// kW filter(kernel) width
|
||||
// sH strides height
|
||||
// sW strides width
|
||||
// pH paddings height
|
||||
// pW paddings width
|
||||
// dH dilations height
|
||||
// dW dilations width
|
||||
// paddingMode 0-VALID, 1-SAME
|
||||
// isNCHW 1-NCHW, 0-NHWC
|
||||
|
||||
LongType bS, iC, iH, iW, mC, oC, oH,
|
||||
oW; // batch size, input channels, input height/width, channels multiplier, output channels, output height/width
|
||||
LongType indIOioC, indIiH, indWmC, indWiC, indWkH, indOoH; // corresponding indexes
|
||||
ConvolutionUtils::getSizesAndIndexesConv2d(isNCHW, wFormat, *input, *output, bS, iC, iH, iW, oC, oH, oW, indIOioC,
|
||||
indIiH, indWiC, indWmC, indWkH, indOoH);
|
||||
mC = weightsDepth->sizeAt(indWmC); // channels multiplier
|
||||
|
||||
NDArray* outputDepth = output;
|
||||
if (weightsPoint) { // if pointwise convolution is expected
|
||||
std::vector<sd::LongType> shape3 =
|
||||
!isNCHW ? std::vector<sd::LongType>({bS, oH, oW, iC * mC}) : std::vector<sd::LongType>({bS, iC * mC, oH, oW});
|
||||
outputDepth = new NDArray(output->ordering(), shape3, input->dataType(), input->getContext());
|
||||
}
|
||||
// ----- perform depthwise convolution (if weightsPoint is absent then oC = iC*mC) ----- //
|
||||
ConvolutionUtils::depthwiseConv2d(block, input, weightsDepth, weightsPoint ? nullptr : bias, outputDepth, kH, kW, sH,
|
||||
sW, pH, pW, dH, dW, paddingMode, isNCHW, wFormat);
|
||||
|
||||
// ----- perform pointwise convolution (oH = iH, oW = iW) ----- //
|
||||
if (weightsPoint) {
|
||||
ConvolutionUtils::conv2d(block, outputDepth, weightsPoint, bias, output, 1, 1, 1, 1, 0, 0, 1, 1, paddingMode,
|
||||
isNCHW, wFormat); // in this case oH=iH, oW=iW
|
||||
delete outputDepth;
|
||||
}
|
||||
}
|
||||
|
||||
void ConvolutionUtils::sconv2d(graph::Context& block, NDArray* input, NDArray* weightsDepth,
|
||||
NDArray* weightsPoint, NDArray* bias, NDArray* output, const LongType kH,
|
||||
const LongType kW, const LongType sH, const LongType sW, LongType pH, LongType pW, const LongType dH, const LongType dW,
|
||||
const int paddingMode, const int isNCHW, const int wFormat) {
|
||||
BUILD_SINGLE_SELECTOR_TWICE(input->dataType(), sconv2d_,
|
||||
(block, input, weightsDepth, weightsPoint, bias, output, kH, kW, sH, sW, pH, pW, dH, dW,
|
||||
paddingMode, isNCHW, wFormat),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,83 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void upsampling2d_(NDArray& input, NDArray& output, const LongType factorH, const LongType factorW,
|
||||
const bool isNCHW) {
|
||||
// input has shape [bS, iC, iH, iW] (NCHW) or [bS, iH, iW, iC] (NHWC)
|
||||
// output has shape [bS, iC, factorH*iH, factorW*iW ] (NCHW) or [bS, factorH*iH, factorW*iW, iC] (NHWC)
|
||||
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
|
||||
const sd::LongType dimIH = isNCHW ? 2 : 1;
|
||||
const sd::LongType dimIC = isNCHW ? 1 : 3;
|
||||
|
||||
const sd::LongType bS = input.sizeAt(0);
|
||||
const sd::LongType iC = input.sizeAt(dimIC);
|
||||
const sd::LongType oH = output.sizeAt(dimIH);
|
||||
const sd::LongType oW = output.sizeAt(dimIH + 1);
|
||||
|
||||
const sd::LongType xStride0 = input.stridesOf()[0];
|
||||
const sd::LongType xStride1 = input.stridesOf()[dimIC];
|
||||
const sd::LongType xStride2 = input.stridesOf()[dimIH];
|
||||
const sd::LongType xStride3 = input.stridesOf()[dimIH + 1];
|
||||
|
||||
const sd::LongType zStride0 = output.stridesOf()[0];
|
||||
const sd::LongType zStride1 = output.stridesOf()[dimIC];
|
||||
const sd::LongType zStride2 = output.stridesOf()[dimIH];
|
||||
const sd::LongType zStride3 = output.stridesOf()[dimIH + 1];
|
||||
|
||||
// loop through output array
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
sd::LongType xCoord2, xCoord3;
|
||||
for (sd::LongType b = start_x; b < stop_x; b += inc_x) {
|
||||
for (sd::LongType c = start_y; c < stop_y; c += inc_y) {
|
||||
for (sd::LongType h = start_z; h < stop_z; h += inc_z) {
|
||||
for (sd::LongType w = 0; w < oW; ++w) {
|
||||
xCoord2 = h / factorH;
|
||||
xCoord3 = w / factorW;
|
||||
|
||||
z[b * zStride0 + c * zStride1 + h * zStride2 + w * zStride3] =
|
||||
x[b * xStride0 + c * xStride1 + xCoord2 * xStride2 + xCoord3 * xStride3];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1, 0, oH, 1);
|
||||
}
|
||||
|
||||
void ConvolutionUtils::upsampling2d(sd::graph::Context& block, NDArray& input, NDArray& output, const LongType factorH,
|
||||
const LongType factorW, const bool isNCHW) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), upsampling2d_, (input, output, factorH, factorW, isNCHW), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,86 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void upsampling2dBP_(NDArray& gradO, NDArray& gradI, const bool isNCHW) {
|
||||
// gradO has shape [bS, iC, factorH*iH, factorW*iW ] (NCHW) or [bS, factorH*iH, factorW*iW, iC] (NHWC)
|
||||
// gradI has shape [bS, iC, iH, iW] (NCHW) or [bS, iH, iW, iC] (NHWC)
|
||||
|
||||
const T* x = gradO.bufferAsT<T>();
|
||||
T* z = gradI.bufferAsT<T>();
|
||||
|
||||
const sd::LongType dimIH = isNCHW ? 2 : 1;
|
||||
const sd::LongType dimIC = isNCHW ? 1 : 3;
|
||||
|
||||
const sd::LongType bS = gradI.sizeAt(0);
|
||||
const sd::LongType iC = gradI.sizeAt(dimIC);
|
||||
const sd::LongType iH = gradI.sizeAt(dimIH);
|
||||
const sd::LongType iW = gradI.sizeAt(dimIH + 1);
|
||||
|
||||
const sd::LongType factorH = gradO.sizeAt(dimIH) / iH;
|
||||
const sd::LongType factorW = gradO.sizeAt(dimIH + 1) / iW;
|
||||
|
||||
const sd::LongType xStride0 = gradO.stridesOf()[0];
|
||||
const sd::LongType xStride1 = gradO.stridesOf()[dimIC];
|
||||
const sd::LongType xStride2 = gradO.stridesOf()[dimIH];
|
||||
const sd::LongType xStride3 = gradO.stridesOf()[dimIH + 1];
|
||||
|
||||
const sd::LongType zStride0 = gradI.stridesOf()[0];
|
||||
const sd::LongType zStride1 = gradI.stridesOf()[dimIC];
|
||||
const sd::LongType zStride2 = gradI.stridesOf()[dimIH];
|
||||
const sd::LongType zStride3 = gradI.stridesOf()[dimIH + 1];
|
||||
|
||||
// loop through output array
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
for (sd::LongType b = start_x; b < stop_x; b += inc_x) {
|
||||
for (sd::LongType c = start_y; c < stop_y; c += inc_y) {
|
||||
for (sd::LongType h = start_z; h < stop_z; h += inc_z) {
|
||||
for (sd::LongType w = 0; w < iW; ++w) {
|
||||
const auto zOffset = b * zStride0 + c * zStride1 + h * zStride2 + w * zStride3;
|
||||
|
||||
z[zOffset] = 0;
|
||||
|
||||
for (sd::LongType xh = h * factorH; xh < h * factorH + factorH; ++xh)
|
||||
for (sd::LongType xw = w * factorW; xw < w * factorW + factorW; ++xw)
|
||||
z[zOffset] += x[b * xStride0 + c * xStride1 + xh * xStride2 + xw * xStride3];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1, 0, iH, 1);
|
||||
}
|
||||
|
||||
void ConvolutionUtils::upsampling2dBP(sd::graph::Context& block, NDArray& gradO, NDArray& gradI,
|
||||
const bool isNCHW) {
|
||||
BUILD_SINGLE_SELECTOR(gradO.dataType(), upsampling2dBP_, (gradO, gradI, isNCHW), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,92 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void upsampling3d_(NDArray& input, NDArray& output, const LongType factorD, const LongType factorH,
|
||||
const LongType factorW, const bool isNCDHW) {
|
||||
// input has shape [bS, iC, iD, iH, iW] (NCDHW) or [bS, iD, iH, iW, iC] (NDHWC)
|
||||
// output has shape [bS, iC, factorD*iD, factorH*iH, factorW*iW ] (NCDHW) or [bS, factorD*iD, factorH*iH, factorW*iW,
|
||||
// iC] (NDHWC)
|
||||
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
|
||||
const sd::LongType dimID = isNCDHW ? 2 : 1;
|
||||
const sd::LongType dimIC = isNCDHW ? 1 : 4;
|
||||
|
||||
const sd::LongType bS = input.sizeAt(0);
|
||||
const sd::LongType iC = input.sizeAt(dimIC);
|
||||
const sd::LongType oD = output.sizeAt(dimID);
|
||||
const sd::LongType oH = output.sizeAt(dimID + 1);
|
||||
const sd::LongType oW = output.sizeAt(dimID + 2);
|
||||
|
||||
const sd::LongType xStride0 = input.stridesOf()[0];
|
||||
const sd::LongType xStride1 = input.stridesOf()[dimIC];
|
||||
const sd::LongType xStride2 = input.stridesOf()[dimID];
|
||||
const sd::LongType xStride3 = input.stridesOf()[dimID + 1];
|
||||
const sd::LongType xStride4 = input.stridesOf()[dimID + 2];
|
||||
|
||||
const sd::LongType zStride0 = output.stridesOf()[0];
|
||||
const sd::LongType zStride1 = output.stridesOf()[dimIC];
|
||||
const sd::LongType zStride2 = output.stridesOf()[dimID];
|
||||
const sd::LongType zStride3 = output.stridesOf()[dimID + 1];
|
||||
const sd::LongType zStride4 = output.stridesOf()[dimID + 2];
|
||||
|
||||
// loop through output array
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
sd::LongType xCoord2, xCoord3, xCoord4;
|
||||
|
||||
for (sd::LongType b = start_x; b < stop_x; b += inc_x) {
|
||||
for (sd::LongType c = start_y; c < stop_y; c += inc_y) {
|
||||
for (sd::LongType d = start_z; d < stop_z; d += inc_z) {
|
||||
for (sd::LongType h = 0; h < oH; ++h) {
|
||||
for (sd::LongType w = 0; w < oW; ++w) {
|
||||
xCoord2 = d / factorD;
|
||||
xCoord3 = h / factorH;
|
||||
xCoord4 = w / factorW;
|
||||
|
||||
z[b * zStride0 + c * zStride1 + d * zStride2 + h * zStride3 + w * zStride4] =
|
||||
x[b * xStride0 + c * xStride1 + xCoord2 * xStride2 + xCoord3 * xStride3 + xCoord4 * xStride4];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1, 0, oD, 1);
|
||||
}
|
||||
|
||||
void ConvolutionUtils::upsampling3d(sd::graph::Context& block, NDArray& input, NDArray& output, const LongType factorD,
|
||||
const LongType factorH, const LongType factorW, const bool isNCDHW) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), upsampling3d_, (input, output, factorD, factorH, factorW, isNCDHW),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,94 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void upsampling3dBP_(NDArray& gradO, NDArray& gradI, const bool isNCDHW) {
|
||||
// input has shape [bS, iC, iD, iH, iW] (NCDHW) or [bS, iD, iH, iW, iC] (NDHWC)
|
||||
// output has shape [bS, iC, factorD*iD, factorH*iH, factorW*iW ] (NCDHW) or [bS, factorD*iD, factorH*iH, factorW*iW,
|
||||
// iC] (NDHWC)
|
||||
|
||||
const T* x = gradO.bufferAsT<T>();
|
||||
T* z = gradI.bufferAsT<T>();
|
||||
|
||||
const sd::LongType dimID = isNCDHW ? 2 : 1;
|
||||
const sd::LongType dimIC = isNCDHW ? 1 : 4;
|
||||
|
||||
const sd::LongType bS = gradI.sizeAt(0);
|
||||
const sd::LongType iC = gradI.sizeAt(dimIC);
|
||||
const sd::LongType iD = gradI.sizeAt(dimID);
|
||||
const sd::LongType iH = gradI.sizeAt(dimID + 1);
|
||||
const sd::LongType iW = gradI.sizeAt(dimID + 2);
|
||||
|
||||
const sd::LongType factorD = gradO.sizeAt(dimID) / iD;
|
||||
const sd::LongType factorH = gradO.sizeAt(dimID + 1) / iH;
|
||||
const sd::LongType factorW = gradO.sizeAt(dimID + 2) / iW;
|
||||
|
||||
const sd::LongType xStride0 = gradO.stridesOf()[0];
|
||||
const sd::LongType xStride1 = gradO.stridesOf()[dimIC];
|
||||
const sd::LongType xStride2 = gradO.stridesOf()[dimID];
|
||||
const sd::LongType xStride3 = gradO.stridesOf()[dimID + 1];
|
||||
const sd::LongType xStride4 = gradO.stridesOf()[dimID + 2];
|
||||
|
||||
const sd::LongType zStride0 = gradI.stridesOf()[0];
|
||||
const sd::LongType zStride1 = gradI.stridesOf()[dimIC];
|
||||
const sd::LongType zStride2 = gradI.stridesOf()[dimID];
|
||||
const sd::LongType zStride3 = gradI.stridesOf()[dimID + 1];
|
||||
const sd::LongType zStride4 = gradI.stridesOf()[dimID + 2];
|
||||
|
||||
// loop through output array
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
for (sd::LongType b = start_x; b < stop_x; b += inc_x) {
|
||||
for (sd::LongType c = start_y; c < stop_y; c += inc_y) {
|
||||
for (sd::LongType d = start_z; d < stop_z; d += inc_z) {
|
||||
for (sd::LongType h = 0; h < iH; ++h) {
|
||||
for (sd::LongType w = 0; w < iW; ++w) {
|
||||
const auto zOffset = b * zStride0 + c * zStride1 + d * zStride2 + h * zStride3 + w * zStride4;
|
||||
|
||||
z[zOffset] = 0;
|
||||
|
||||
for (sd::LongType xd = d * factorD; xd < d * factorD + factorD; ++xd)
|
||||
for (sd::LongType xh = h * factorH; xh < h * factorH + factorH; ++xh)
|
||||
for (sd::LongType xw = w * factorW; xw < w * factorW + factorW; ++xw)
|
||||
z[zOffset] += x[b * xStride0 + c * xStride1 + xd * xStride2 + xh * xStride3 + xw * xStride4];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1, 0, iD, 1);
|
||||
}
|
||||
|
||||
void ConvolutionUtils::upsampling3dBP(sd::graph::Context& block, NDArray& gradO, NDArray& gradI,
|
||||
const bool isNCHW) {
|
||||
BUILD_SINGLE_SELECTOR(gradO.dataType(), upsampling3dBP_, (gradO, gradI, isNCHW), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,152 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// [bS, iC, iD, iH, iW] is convoluted to [bS, iC, kD, kH, kW, oD, oH, oW]
|
||||
template <typename T>
|
||||
static void vol2col_(NDArray* volume, NDArray* columns, const int sD, const int sH, const int sW, const int pD,
|
||||
const int pH, const int pW, const int dD, const int dH, const int dW) {
|
||||
const int bS = volume->sizeAt(0);
|
||||
const int iC = volume->sizeAt(1);
|
||||
const int iD = volume->sizeAt(2);
|
||||
const int iH = volume->sizeAt(3);
|
||||
const int iW = volume->sizeAt(4);
|
||||
const int kD = columns->sizeAt(2);
|
||||
const int kH = columns->sizeAt(3);
|
||||
const int kW = columns->sizeAt(4);
|
||||
const int oD = columns->sizeAt(5);
|
||||
const int oH = columns->sizeAt(6);
|
||||
const int oW = columns->sizeAt(7);
|
||||
const int colStride0 = columns->stridesOf()[0];
|
||||
const int colStride1 = columns->stridesOf()[1];
|
||||
const int colStride2 = columns->stridesOf()[2];
|
||||
const int colStride3 = columns->stridesOf()[3];
|
||||
const int colStride4 = columns->stridesOf()[4];
|
||||
const int colStride5 = columns->stridesOf()[5];
|
||||
const int colStride6 = columns->stridesOf()[6];
|
||||
const int colStride7 = columns->stridesOf()[7];
|
||||
const int volStride0 = volume->stridesOf()[0];
|
||||
const int volStride1 = volume->stridesOf()[1];
|
||||
const int volStride2 = volume->stridesOf()[2];
|
||||
const int volStride3 = volume->stridesOf()[3];
|
||||
const int volStride4 = volume->stridesOf()[4];
|
||||
T* colBuff = columns->bufferAsT<T>();
|
||||
T* volBuff = volume->bufferAsT<T>();
|
||||
|
||||
if (volume->ordering() == 'c' && columns->ordering() == 'c' && shape::strideDescendingCAscendingF(volume->shapeInfo()) &&
|
||||
shape::strideDescendingCAscendingF(columns->shapeInfo())) {
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
T *col, *vol;
|
||||
int volDep, volRow, volCol;
|
||||
|
||||
for (int b = start_x; b < stop_x; b += inc_x) {
|
||||
for (int c = start_y; c < stop_y; c += inc_y) {
|
||||
for (int kDep = start_z; kDep < stop_z; kDep += inc_z) {
|
||||
for (int kRow = 0; kRow < kH; ++kRow) {
|
||||
for (int kCol = 0; kCol < kW; ++kCol) {
|
||||
for (int colD = 0; colD < oD; ++colD) {
|
||||
for (int colH = 0; colH < oH; ++colH) {
|
||||
for (int colW = 0; colW < oW; ++colW) {
|
||||
volDep = (-pD + kDep * dD) + colD * sD;
|
||||
volRow = (-pH + kRow * dH) + colH * sH;
|
||||
volCol = (-pW + kCol * dW) + colW * sW;
|
||||
|
||||
col = colBuff + b * colStride0 + c * colStride1 + kDep * colStride2 + kRow * colStride3 +
|
||||
kCol * colStride4 + colD * colStride5 + colH * colStride6 + colW * colStride7;
|
||||
|
||||
if (static_cast<unsigned>(volDep) >= static_cast<unsigned>(iD) ||
|
||||
static_cast<unsigned>(volRow) >= static_cast<unsigned>(iH) ||
|
||||
static_cast<unsigned>(volCol) >= static_cast<unsigned>(iW))
|
||||
*col = static_cast<T>(0.);
|
||||
else {
|
||||
vol = volBuff + b * volStride0 + c * volStride1 + volDep * volStride2 + volRow * volStride3 +
|
||||
volCol * volStride4;
|
||||
*col = static_cast<T>(*vol);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, iC, 1, 0, kD, 1);
|
||||
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
T *col, *vol;
|
||||
int volDep, volRow, volCol;
|
||||
|
||||
for (int b = start_x; b < stop_x; b++) {
|
||||
for (int colD = start_y; colD < stop_y; colD++) {
|
||||
for (int colH = 0; colH < oH; ++colH) {
|
||||
for (int colW = 0; colW < oW; ++colW) {
|
||||
for (int c = 0; c < iC; ++c) {
|
||||
for (int kDep = 0; kDep < kD; ++kDep) {
|
||||
for (int kRow = 0; kRow < kH; ++kRow) {
|
||||
for (int kCol = 0; kCol < kW; ++kCol) {
|
||||
volDep = (-pD + kDep * dD) + colD * sD;
|
||||
volRow = (-pH + kRow * dH) + colH * sH;
|
||||
volCol = (-pW + kCol * dW) + colW * sW;
|
||||
|
||||
col = colBuff + b * colStride0 + c * colStride1 + kDep * colStride2 + kRow * colStride3 +
|
||||
kCol * colStride4 + colD * colStride5 + colH * colStride6 + colW * colStride7;
|
||||
if (static_cast<unsigned>(volDep) >= static_cast<unsigned>(iD) ||
|
||||
static_cast<unsigned>(volRow) >= static_cast<unsigned>(iH) ||
|
||||
static_cast<unsigned>(volCol) >= static_cast<unsigned>(iW))
|
||||
*col = static_cast<T>(0.0);
|
||||
else {
|
||||
vol = volBuff + b * volStride0 + c * volStride1 + volDep * volStride2 + volRow * volStride3 +
|
||||
volCol * volStride4;
|
||||
*col = static_cast<T>(*vol);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, oD, 1);
|
||||
}
|
||||
}
|
||||
|
||||
void ConvolutionUtils::vol2col(graph::Context& block, NDArray* vol, NDArray* col, const LongType sD, const LongType sH,
|
||||
const LongType sW, const LongType pD, const LongType pH, const LongType pW,
|
||||
const LongType dD, const LongType dH, const LongType dW) {
|
||||
BUILD_SINGLE_SELECTOR(vol->dataType(), vol2col_, (vol, col, sD, sH, sW, pD, pH, pW, dD, dH, dW),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include "crop_and_resize.hpp"
|
||||
#include <system/selective_rendering.h>
|
||||
#if NOT_EXCLUDED(OP_crop_and_resize)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
// ------------------------------------------------------------------------------------------------------------------ //
|
||||
// ------------------------------------------------------------------------------------------------------------------ //
|
||||
// crop and resize helper functor:
|
||||
// \@param context - launch context for operation
|
||||
// \@param images - batch of images (4D tensor) with shape {batch, width, height, channels} with given type
|
||||
// \@param boxes - float boxes for crop
|
||||
// \@param indices - integer boxes indices for crop
|
||||
// \@param cropSize - integer size (newWidth, newHeight)
|
||||
// \@param method - one of bilinear (0) or nearest neighbour (1) interpolation algorithm
|
||||
// \@param extrapolationVal - radix to increase/decrease image
|
||||
// \@param crops - output image batch (4D with given type)
|
||||
//
|
||||
|
||||
void cropAndResizeFunctor(sd::LaunchContext *context, NDArray *images, NDArray *boxes,
|
||||
NDArray *indices, NDArray *cropSize, int method, double extrapolationVal,
|
||||
NDArray *crops) {
|
||||
auto imagesDType = images->dataType();
|
||||
auto boxesDType = boxes->dataType();
|
||||
auto indicesDType = indices->dataType();
|
||||
BUILD_TRIPLE_SELECTOR(images->dataType(), boxes->dataType(), indices->dataType(), cropAndResizeFunctor_,
|
||||
(context,images, boxes, indices, cropSize, method, extrapolationVal, crops), SD_NUMERIC_TYPES,
|
||||
SD_FLOAT_TYPES, SD_INTEGER_TYPES);
|
||||
}
|
||||
|
||||
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,147 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/crop_and_resize.h>
|
||||
#if NOT_EXCLUDED(OP_crop_and_resize)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// cropAndResizeFunctor main algorithm
|
||||
// context - launch context
|
||||
// images - batch of images (4D tensor - [batch, width, height, pixels])
|
||||
// boxes - 2D tensor with boxes for crop
|
||||
// indices - 2D int tensor with indices of boxes to crop
|
||||
// cropSize - 2D int tensor with crop box sizes
|
||||
// method - (one of 0 - bilinear, 1 - nearest)
|
||||
// extrapolationVal - double value of extrapolation
|
||||
// crops - output (4D tensor - [batch, outWidth, outHeight, pixels])
|
||||
//
|
||||
template <typename T, typename Z, typename I>
|
||||
SD_LIB_EXPORT void cropAndResizeFunctor_(LaunchContext* context, NDArray * images, NDArray * boxes,
|
||||
NDArray * indices, NDArray * cropSize, int method, double extrapolationVal,
|
||||
NDArray* crops) {
|
||||
const int batchSize = images->sizeAt(0);
|
||||
const int imageHeight = images->sizeAt(1);
|
||||
const int imageWidth = images->sizeAt(2);
|
||||
|
||||
const int numBoxes = crops->sizeAt(0);
|
||||
const int cropHeight = crops->sizeAt(1);
|
||||
const int cropWidth = crops->sizeAt(2);
|
||||
const int depth = crops->sizeAt(3);
|
||||
|
||||
for (auto b = 0; b < numBoxes; ++b) {
|
||||
Z y1 = static_cast<Z>(boxes->t<Z>(b, 0));
|
||||
Z x1 = static_cast<Z>(boxes->t<Z>(b, 1));
|
||||
Z y2 = static_cast<Z>(boxes->t<Z>(b, 2));
|
||||
Z x2 = static_cast<Z>(boxes->t<Z>(b, 3));
|
||||
|
||||
int bIn = indices->e<I>(b);
|
||||
if (bIn >= batchSize) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Z heightScale = (cropHeight > 1)
|
||||
? Z((y2 - y1) * (imageHeight - 1) / (cropHeight - 1))
|
||||
: Z(0);
|
||||
Z widthScale = (cropWidth > 1)
|
||||
? Z((x2 - x1) * (imageWidth - 1) / (cropWidth - 1))
|
||||
: Z(0);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto y = start; y < stop; y++) {
|
||||
const float inY =
|
||||
(cropHeight > 1) ? y1 * (imageHeight - 1) + y * heightScale : 0.5 * (y1 + y2) * (imageHeight - 1);
|
||||
|
||||
if (inY < 0 || inY > imageHeight - 1) {
|
||||
for (auto x = 0; x < cropWidth; ++x) {
|
||||
for (auto d = 0; d < depth; ++d) {
|
||||
crops->p(b, y, x, d, extrapolationVal);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (method == 0 /* bilinear */) {
|
||||
const int topYIndex = sd::math::p_floor(inY);
|
||||
const int bottomYIndex = sd::math::p_ceil(inY);
|
||||
const float y_lerp = inY - topYIndex;
|
||||
|
||||
for (auto x = 0; x < cropWidth; ++x) {
|
||||
const float in_x =
|
||||
(cropWidth > 1) ? x1 * (imageWidth - 1) + x * widthScale : 0.5 * (x1 + x2) * (imageWidth - 1);
|
||||
|
||||
if (in_x < 0 || in_x > imageWidth - 1) {
|
||||
for (auto d = 0; d < depth; ++d) {
|
||||
crops->p(b, y, x, d, extrapolationVal);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int left_x_index = math::p_floor(in_x);
|
||||
int right_x_index = math::p_ceil(in_x);
|
||||
T x_lerp = static_cast<T>(in_x - left_x_index);
|
||||
|
||||
for (auto d = 0; d < depth; ++d) {
|
||||
const T topLeft(images->e<T>(bIn, topYIndex, left_x_index, d));
|
||||
const T topRight(images->e<T>(bIn, topYIndex, right_x_index, d));
|
||||
const T bottomLeft(images->e<T>(bIn, bottomYIndex, left_x_index, d));
|
||||
const T bottomRight(images->e<T>(bIn, bottomYIndex, right_x_index, d));
|
||||
const T top = topLeft + (topRight - topLeft) * x_lerp;
|
||||
const T bottom = bottomLeft + (bottomRight - bottomLeft) * x_lerp;
|
||||
crops->p(b, y, x, d, top + (bottom - top) * y_lerp);
|
||||
}
|
||||
}
|
||||
} else { // method is "nearest neighbor"
|
||||
for (auto x = 0; x < cropWidth; ++x) {
|
||||
const float inX =
|
||||
(cropWidth > 1) ? x1 * (imageWidth - 1) + x * widthScale : 0.5 * (x1 + x2) * (imageWidth - 1);
|
||||
|
||||
if (inX < 0 || inX > imageWidth - 1) {
|
||||
for (auto d = 0; d < depth; ++d) {
|
||||
crops->p(b, y, x, d, extrapolationVal);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const int closestXIndex = roundf(inX);
|
||||
const int closestYIndex = roundf(inY);
|
||||
for (auto d = 0; d < depth; ++d) {
|
||||
crops->p(b, y, x, d, images->e<T>(bIn, closestYIndex, closestXIndex, d));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, cropHeight);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
|
||||
BUILD_TRIPLE_TEMPLATE(void sd::ops::helpers::cropAndResizeFunctor_,
|
||||
(sd::LaunchContext * context, NDArray * images, NDArray * boxes, NDArray * indices,
|
||||
NDArray * cropSize, int method, double extrapolationVal, NDArray* crops),
|
||||
SD_NUMERIC_TYPES, SD_FLOAT_TYPES, SD_INTEGER_TYPES);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,59 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author GS (sgazeos@gmail.com), created on 10/1/2018
|
||||
//
|
||||
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/helpers/cross.h>
|
||||
#if NOT_EXCLUDED(OP_cross)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
void crossBatched(sd::LaunchContext *context, NDArray *a, NDArray *b, NDArray *o) {
|
||||
std::vector<sd::LongType> shape2= {-1,3};
|
||||
auto _a = a->reshape(a->ordering(), shape2);
|
||||
auto _b = b->reshape(b->ordering(), shape2);
|
||||
auto _o = o->reshape(o->ordering(), shape2, false);
|
||||
|
||||
auto tadsA = _a->allTensorsAlongDimension({1});
|
||||
auto tadsB = _b->allTensorsAlongDimension({1});
|
||||
auto tadsO = _o->allTensorsAlongDimension({1});
|
||||
|
||||
int tads = tadsA.size();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
auto a_ = tadsA.at(e);
|
||||
auto b_ = tadsB.at(e);
|
||||
auto o_ = tadsO.at(e);
|
||||
|
||||
helpers::cross(context, a_, b_, o_);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, tads);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,433 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (c) 2021 Deeplearning4j Contributors
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*******************************************************************************/
|
||||
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#include <execution/ThreadPool.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopsCoordsHelper.h>
|
||||
#include <ops/declarable/helpers/ctc.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <type_traits>
|
||||
#include <system/selective_rendering.h>
|
||||
#if NOT_EXCLUDED(OP_ctc_loss)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <bool IsLogPStrided = false, bool IsLblStrided = false, typename Type, typename IndexType>
|
||||
Type forward(Type *alphaPtr, const sd::LongType &incA, const Type *logP, const sd::LongType &incP, const IndexType *lbl,
|
||||
const sd::LongType &lenSB, const sd::LongType &lenT, const int &blankIndex, int elwiseP = 1,
|
||||
int elwiseS = 1) {
|
||||
Type negInf = negative_infinity<Type>();
|
||||
// initialize alphas at t=0
|
||||
alphaPtr[0] = element<IsLogPStrided>(logP, blankIndex, elwiseP);
|
||||
// alphaPtr[1] =logP[lbl[0]];
|
||||
alphaPtr[1] = element<IsLogPStrided>(logP, *lbl, elwiseP);
|
||||
// the rest initialization was skipped
|
||||
// as its assumed the array already were initialized with negative infinity
|
||||
// move to the next frame
|
||||
Type *alphaPrevPtr = alphaPtr;
|
||||
alphaPtr += incA;
|
||||
logP += incP;
|
||||
|
||||
auto startX = lenSB - 2 * lenT;
|
||||
// process the rest
|
||||
for (auto t = 1; t < lenT; t++) {
|
||||
// start = max(0,L-2*(T-t))
|
||||
auto s = startX + 2 * t;
|
||||
s = s > 0 ? s : 0;
|
||||
for (; s < lenSB; s++) {
|
||||
auto ind = s / 2; // our real index
|
||||
// we force blanks for even indexes
|
||||
// strided version of lbl[ind] => element<IsLblStrided>(lbl, ind, elwiseS)
|
||||
auto currentInd = (s % 2 == 0) ? blankIndex : element<IsLblStrided>(lbl, ind, elwiseS);
|
||||
// {t-1,s}
|
||||
Type alphaS = alphaPrevPtr[s];
|
||||
Type alphaS_1 = s > 0 ? alphaPrevPtr[s - 1] : negInf;
|
||||
// logP[currentInd] or logP[currentInd*elwiseP]
|
||||
auto currentProb = element<IsLogPStrided>(logP, currentInd, elwiseP);
|
||||
// if blank or the same as previous
|
||||
if (s > 1 && currentInd != blankIndex && currentInd != element<IsLblStrided>(lbl, ind - 1, elwiseS)) {
|
||||
Type alphaS_2 = alphaPrevPtr[s - 2];
|
||||
alphaPtr[s] = log_sum_exp(alphaS, alphaS_1, alphaS_2) + currentProb;
|
||||
} else {
|
||||
alphaPtr[s] = log_sum_exp(alphaS, alphaS_1) + currentProb;
|
||||
}
|
||||
}
|
||||
|
||||
// store t-1 alpha Ptr
|
||||
alphaPrevPtr = alphaPtr;
|
||||
logP += incP;
|
||||
alphaPtr += incA;
|
||||
}
|
||||
auto logP0 = alphaPrevPtr[lenSB - 1];
|
||||
auto logP1 = alphaPrevPtr[lenSB - 2];
|
||||
return -log_sum_exp(logP0, logP1);
|
||||
}
|
||||
|
||||
//#undef CALCULATE_ALL_IN_ONE_FRAME_LOOP
|
||||
|
||||
template <bool IsLogPStrided = false, bool IsLblStrided = false, bool isGradStrided = false, typename Type,
|
||||
typename IndexType = int>
|
||||
void backwardAndGrad(Type forwardLogLoss, Type *alphaPtr, Type *bettaPtr, int incA, const Type *logP, int incP,
|
||||
Type *gradPtr, int incG, const IndexType *lbl, const sd::LongType &lenS, const sd::LongType &lenT,
|
||||
const sd::LongType &lenK, const int &blankIndex, int elwiseP = 1, int elwiseS = 1,
|
||||
int elwiseG = 1) {
|
||||
Type negInf = negative_infinity<Type>();
|
||||
sd::LongType lenSB = 2 * lenS + 1;
|
||||
auto origBetta = bettaPtr;
|
||||
auto origLogP = logP;
|
||||
// move to the last frame
|
||||
bettaPtr += (lenT - 1) * incA;
|
||||
logP += (lenT - 1) * incP;
|
||||
|
||||
// initialize bettas at t=lenT
|
||||
bettaPtr[lenSB - 1] = element<IsLogPStrided>(logP, blankIndex, elwiseP);
|
||||
auto lblIndex = element<IsLblStrided>(lbl, lenS - 1, elwiseS);
|
||||
bettaPtr[lenSB - 2] = element<IsLogPStrided>(logP, lblIndex, elwiseP); // logP[lbl[lenS - 1]];
|
||||
|
||||
#if defined(CALCULATE_ALL_IN_ONE_FRAME_LOOP)
|
||||
// move to the last
|
||||
gradPtr += (lenT - 1) * incG;
|
||||
alphaPtr += (lenT - 1) * incA;
|
||||
for (auto s = lenSB - 1; s >= 0; s--) {
|
||||
auto ind = s / 2; // our real index
|
||||
// we forced blanks for even indexes
|
||||
auto currentInd = (s % 2 == 0) ? blankIndex : element<IsLblStrided>(lbl, ind, elwiseS);
|
||||
// alpha(s)*betta(s) in log scale but still store in alpha to save memory
|
||||
auto alphaBettaS = alphaPtr[s] + bettaPtr[s];
|
||||
|
||||
// sum (alpha(s)*betta(s) ) over real indexes
|
||||
auto ¤tGrad = element<isGradStrided>(gradPtr, currentInd, elwiseG); // gradPtr[currentInd];
|
||||
if (currentGrad == negInf) {
|
||||
currentGrad = alphaBettaS;
|
||||
} else {
|
||||
Type cMax = std::max(currentGrad, alphaBettaS);
|
||||
currentGrad = std::log(std::exp(currentGrad - cMax) + std::exp(alphaBettaS - cMax)) + cMax;
|
||||
}
|
||||
}
|
||||
for (int k = 0; k < lenK; k++) {
|
||||
// compute the rest grad
|
||||
|
||||
// prob(t,k) - grad(k) / ((prob(t,k)*Z) )
|
||||
|
||||
// p2= grad(k) / (prob(t,k)*Z )
|
||||
// in logscale . plus we have Z as -logLoss
|
||||
// auto p2 = std::exp(gradPtr[k] + forwardLogLoss - logP[k]);
|
||||
// gradPtr[k] = std::exp(logP[k]) - p2;
|
||||
auto currentProb = element<IsLogPStrided>(logP, k, elwiseP);
|
||||
auto ¤tGrad = element<isGradStrided>(gradPtr, k, elwiseG);
|
||||
auto p2 = std::exp(currentGrad + forwardLogLoss - currentProb);
|
||||
currentGrad = std::exp(currentProb) - p2;
|
||||
}
|
||||
gradPtr -= incG;
|
||||
alphaPtr -= incA;
|
||||
#endif
|
||||
|
||||
auto bettaPrevPtr = bettaPtr;
|
||||
bettaPtr -= incA;
|
||||
logP -= incP;
|
||||
// process the rest
|
||||
for (auto t = lenT - 2; t >= 0; t--) {
|
||||
#if defined(CALCULATE_ALL_IN_ONE_FRAME_LOOP)
|
||||
auto end = lenSB - 1;
|
||||
#else
|
||||
auto end = std::min(2 * t + 2, lenSB - 1);
|
||||
#endif
|
||||
for (auto s = end; s >= 0; s--) {
|
||||
auto ind = s / 2; // our real index
|
||||
// we forced blanks for even indexes
|
||||
auto currentInd = (s % 2 == 0) ? blankIndex : element<IsLblStrided>(lbl, ind, elwiseS); // lbl[ind];
|
||||
// {t-1,s}
|
||||
Type bettaS = bettaPrevPtr[s];
|
||||
Type bettaS_1 = s < lenSB - 1 ? bettaPrevPtr[s + 1] : negInf;
|
||||
// logP[currentInd]
|
||||
auto currentProb = element<IsLogPStrided>(logP, currentInd, elwiseP);
|
||||
// if blank or the same as previous
|
||||
if (s < lenSB - 2 && currentInd != blankIndex && currentInd != element<IsLblStrided>(lbl, ind + 1, elwiseS)) {
|
||||
Type bettaS_2 = bettaPrevPtr[s + 2];
|
||||
bettaPtr[s] = log_sum_exp(bettaS, bettaS_1, bettaS_2) + currentProb;
|
||||
} else {
|
||||
bettaPtr[s] = log_sum_exp(bettaS, bettaS_1) + currentProb;
|
||||
}
|
||||
|
||||
#if defined(CALCULATE_ALL_IN_ONE_FRAME_LOOP)
|
||||
// alpha(s)*betta(s) in log scale but still store in alpha to save memory
|
||||
auto alphaBettaS = alphaPtr[s] + bettaPtr[s];
|
||||
|
||||
// sum (alpha(s)*betta(s) ) over real indexes
|
||||
auto ¤tGrad = element<isGradStrided>(gradPtr, currentInd, elwiseG); // gradPtr[currentInd];
|
||||
if (currentGrad == negInf) {
|
||||
currentGrad = alphaBettaS;
|
||||
} else {
|
||||
Type cMax = std::max(currentGrad, alphaBettaS);
|
||||
currentGrad = std::log(std::exp(currentGrad - cMax) + std::exp(alphaBettaS - cMax)) + cMax;
|
||||
}
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
#if defined(CALCULATE_ALL_IN_ONE_FRAME_LOOP)
|
||||
for (int k = 0; k < lenK; k++) {
|
||||
// compute the rest grad
|
||||
|
||||
// prob(t,k) - grad(k) / ((prob(t,k)*Z) )
|
||||
|
||||
// p2= grad(k) / (prob(t,k)*Z )
|
||||
// in logscale . plus we have Z as -logLoss
|
||||
// auto p2 = std::exp(gradPtr[k] + forwardLogLoss - logP[k]);
|
||||
// gradPtr[k] = std::exp(logP[k]) - p2;
|
||||
auto currentProb = element<IsLogPStrided>(logP, k, elwiseP);
|
||||
auto ¤tGrad = element<isGradStrided>(gradPtr, k, elwiseG);
|
||||
auto p2 = std::exp(currentGrad + forwardLogLoss - currentProb);
|
||||
currentGrad = std::exp(currentProb) - p2;
|
||||
}
|
||||
alphaPtr -= incA;
|
||||
gradPtr -= incG;
|
||||
#endif
|
||||
|
||||
bettaPrevPtr = bettaPtr;
|
||||
bettaPtr -= incA;
|
||||
logP -= incP;
|
||||
}
|
||||
|
||||
|
||||
#if !defined(CALCULATE_ALL_IN_ONE_FRAME_LOOP)
|
||||
// alpha*betta
|
||||
bettaPtr = origBetta;
|
||||
logP = origLogP;
|
||||
|
||||
for (int t = 0; t < lenT; t++) {
|
||||
for (int s = 0; s < lenSB; s++) {
|
||||
auto ind = s / 2; // our real index
|
||||
// we forced blanks for even indexes
|
||||
auto currentInd = (s % 2 == 0) ? blankIndex : element<IsLblStrided>(lbl, ind, elwiseS); // lbl[ind];
|
||||
// alpha(s)*betta(s) in log scale but still store in alpha to save memory
|
||||
auto alphaBettaS = alphaPtr[s] + bettaPtr[s];
|
||||
|
||||
// sum (alpha(s)*betta(s) ) over real indexes
|
||||
auto ¤tGrad = element<isGradStrided>(gradPtr, currentInd, elwiseG); // gradPtr[currentInd];
|
||||
if (currentGrad == negInf) {
|
||||
currentGrad = alphaBettaS;
|
||||
} else {
|
||||
currentGrad = log_sum_exp(currentGrad, alphaBettaS);
|
||||
}
|
||||
// alphaPtr[s] = alphaBettaS;
|
||||
}
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int k = 0; k < lenK; k++) {
|
||||
// compute the rest grad
|
||||
|
||||
// prob(t,k) - grad(k) / ((prob(t,k)*Z) )
|
||||
|
||||
// p2= grad(k) / (prob(t,k)*Z )
|
||||
// in logscale . plus we have Z as -logLoss
|
||||
// auto p2 = std::exp(gradPtr[k] + forwardLogLoss - logP[k]);
|
||||
// gradPtr[k] = std::exp(logP[k]) - p2;
|
||||
auto currentProb = element<IsLogPStrided>(logP, k, elwiseP);
|
||||
auto ¤tGrad = element<isGradStrided>(gradPtr, k, elwiseG);
|
||||
auto p2 = std::exp(currentGrad + forwardLogLoss - currentProb);
|
||||
currentGrad = std::exp(currentProb) - p2;
|
||||
}
|
||||
|
||||
gradPtr += incG;
|
||||
bettaPtr += incA;
|
||||
alphaPtr += incA;
|
||||
logP += incP;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates ctc loss and fills gradients
|
||||
* @param logP logits matrix(lenT,lenK) pointer (log soft max input of rnn)
|
||||
* @param incP stride of logits for the next time frame
|
||||
* @param gradPtr gradient for output
|
||||
* @param incG stride of the gradient for the next time frame
|
||||
* @param lbl target label
|
||||
* @param lenT frame length
|
||||
* @param lenK class length
|
||||
* @param lenS target label length
|
||||
* @param blankIndex index of the blank label in logit class
|
||||
*/
|
||||
template <bool IsLogPStrided = true, bool IsLblStrided = true, bool IsGradStrided = true, typename Type,
|
||||
typename IndexType>
|
||||
Type unitLossAndGrad(const Type *logP, int incP, Type *gradPtr, int incG, const IndexType *lbl, int lenT, int lenK,
|
||||
int lenS, int blankIndex, int elwiseP = 1, int elwiseS = 1, int elwiseG = 1) {
|
||||
auto lenSB = 2 * lenS + 1;
|
||||
// create temp Array for holding bettaArr [lenT,lenSB]
|
||||
// create temp Array for holding alphaArr [lenT,lenSB]
|
||||
int bufferC = gradPtr ? 2 : 1;
|
||||
NDArray *bufferArr = NDArrayFactory::create<Type>('c', {bufferC, lenT, lenSB});
|
||||
auto bufferPtr = bufferArr->bufferAsT<Type>();
|
||||
auto incA = bufferArr->stridesOf()[1];
|
||||
auto bettaBufferPtr = bufferPtr + bufferArr->stridesOf()[0];
|
||||
Type negInf = negative_infinity<Type>();
|
||||
|
||||
if (gradPtr) {
|
||||
if (elwiseG == 1) {
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int i = 0; i < lenK * lenT; i++) {
|
||||
gradPtr[i] = negInf;
|
||||
}
|
||||
} else {
|
||||
auto tempPtr = gradPtr;
|
||||
for (int i = 0; i < lenT; i++) {
|
||||
for (int j = 0; j < lenK; j++) element<false>(tempPtr, j, elwiseG) = negInf;
|
||||
tempPtr += incG;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// set all vals to neginf
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int i = 0; i < bufferC * lenSB * lenT; i++) {
|
||||
bufferPtr[i] = negInf;
|
||||
}
|
||||
|
||||
// forward
|
||||
Type logLoss =
|
||||
forward<IsLogPStrided, IsLblStrided>(bufferPtr, incA, logP, incP, lbl, lenSB, lenT, blankIndex, elwiseP, elwiseS);
|
||||
// backward and gradient if gradptr supplied
|
||||
if (gradPtr)
|
||||
backwardAndGrad<IsLogPStrided, IsLblStrided, IsGradStrided>(logLoss, bufferPtr, bettaBufferPtr, incA, logP, incP,
|
||||
gradPtr, incG, lbl, lenS, lenT, lenK, blankIndex,
|
||||
elwiseP, elwiseS, elwiseG);
|
||||
|
||||
delete bufferArr;
|
||||
return logLoss;
|
||||
}
|
||||
|
||||
template <typename Type, typename IndexType>
|
||||
void ctc_loss_(NDArray&logits, NDArray&targetLabels, NDArray&logitsLengths,
|
||||
NDArray&targetLabelLengths, NDArray &logLosses, NDArray &gradients, int blankIndex) {
|
||||
// lenT - input length of T
|
||||
// lenS - lenght of sequence
|
||||
// lenSB - length with blanks
|
||||
auto lenBatch = logits.shapeOf()[0];
|
||||
|
||||
auto maxLenT = logits.shapeOf()[1];
|
||||
auto lenK = logits.shapeOf()[2];
|
||||
auto maxLenS = targetLabels.shapeOf()[1];
|
||||
|
||||
// get probability buffer and targetLabels buffer
|
||||
auto logP = logits.bufferAsT<Type>();
|
||||
auto lblPtr = targetLabels.bufferAsT<IndexType>();
|
||||
|
||||
auto lenTPtr = logitsLengths.bufferAsT<IndexType>();
|
||||
auto lenSPtr = targetLabelLengths.bufferAsT<IndexType>();
|
||||
|
||||
auto batchLbl = targetLabels.stridesOf()[0];
|
||||
auto batchP = logits.stridesOf()[0];
|
||||
auto incP = logits.stridesOf()[1];
|
||||
|
||||
auto elwiseSLen = targetLabelLengths.stridesOf()[0];
|
||||
auto elwiseT = logitsLengths.stridesOf()[0];
|
||||
auto elwiseS = targetLabels.stridesOf()[1];
|
||||
auto elwiseP = logits.stridesOf()[2];
|
||||
|
||||
int elwiseLL = 0;
|
||||
Type *logLossPtr = nullptr;
|
||||
if (!logLosses.isEmpty()) {
|
||||
elwiseLL = logLosses.stridesOf()[0];
|
||||
logLossPtr = logLosses.bufferAsT<Type>();
|
||||
}
|
||||
// defaulting blankIndex to the last class if its incorrect or -1
|
||||
if (blankIndex > maxLenS || blankIndex < 0) blankIndex = maxLenS - 1;
|
||||
auto func = [logP, batchP, incP, elwiseP, lenK, lenTPtr, lenSPtr, logLossPtr, lblPtr, maxLenT, maxLenS, batchLbl,
|
||||
blankIndex, elwiseT, elwiseLL, elwiseSLen, elwiseS,
|
||||
&gradients](uint64_t thread_id, int64_t start, int64_t stop, int64_t increment) -> void {
|
||||
Type *gradPtr = nullptr;
|
||||
Type resultLoss;
|
||||
int batchG, incG, elwiseG;
|
||||
if (!gradients.isEmpty()) {
|
||||
batchG = gradients.stridesOf()[0];
|
||||
incG = gradients.stridesOf()[1];
|
||||
elwiseG = gradients.stridesOf()[2];
|
||||
gradPtr = gradients.bufferAsT<Type>() + start * batchG;
|
||||
} else {
|
||||
elwiseG = 1;
|
||||
}
|
||||
auto logPtr = logP + start * batchP;
|
||||
auto tempLblPtr = lblPtr + start * batchLbl;
|
||||
|
||||
if (elwiseP == 1 && elwiseS == 1 && elwiseG == 1) {
|
||||
// choose ews one
|
||||
for (int batchIndex = start; batchIndex < stop; batchIndex += increment) {
|
||||
auto lenT = lenTPtr[batchIndex * elwiseT];
|
||||
auto lenS = lenSPtr[batchIndex * elwiseSLen];
|
||||
lenT = lenT > maxLenT ? maxLenT : lenT;
|
||||
lenS = lenS > maxLenS ? maxLenS : lenS;
|
||||
if (lenS <= 0 || lenT <= 0) {
|
||||
resultLoss = negative_infinity<Type>();
|
||||
} else {
|
||||
if (lenS > lenT) lenS = lenT;
|
||||
resultLoss = unitLossAndGrad<false, false, false, Type, IndexType>(logPtr, incP, gradPtr, incG, tempLblPtr,
|
||||
lenT, lenK, lenS, blankIndex);
|
||||
}
|
||||
if (gradPtr) gradPtr += batchG;
|
||||
if (logLossPtr) logLossPtr[batchIndex * elwiseLL] = resultLoss;
|
||||
logPtr += batchP;
|
||||
tempLblPtr += batchLbl;
|
||||
}
|
||||
} else {
|
||||
// slow strided case for all 3
|
||||
for (int batchIndex = start; batchIndex < stop; batchIndex += increment) {
|
||||
auto lenT = lenTPtr[batchIndex * elwiseT];
|
||||
auto lenS = lenSPtr[batchIndex * elwiseSLen];
|
||||
lenT = lenT > maxLenT ? maxLenT : lenT;
|
||||
lenS = lenS > maxLenS ? maxLenS : lenS;
|
||||
if (lenS <= 0 || lenT <= 0) {
|
||||
resultLoss = negative_infinity<Type>();
|
||||
} else {
|
||||
if (lenS > lenT) lenS = lenT;
|
||||
resultLoss = unitLossAndGrad<true, true, true, Type, IndexType>(
|
||||
logPtr, incP, gradPtr, incG, tempLblPtr, lenT, lenK, lenS, blankIndex, elwiseP, elwiseS, elwiseG);
|
||||
}
|
||||
if (gradPtr) gradPtr += batchG;
|
||||
if (logLossPtr) logLossPtr[batchIndex * elwiseLL] = resultLoss;
|
||||
logPtr += batchP;
|
||||
tempLblPtr += batchLbl;
|
||||
}
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, lenBatch, 1);
|
||||
}
|
||||
|
||||
void ctcLoss(graph::Context &block, NDArray&logits, NDArray&targetLabels, NDArray&logitsLengths,
|
||||
NDArray&targetLabelLengths, NDArray &logLosses, NDArray &gradients, int blankIndex) {
|
||||
auto logitsDType = logits.dataType();
|
||||
auto targetLabelsDType = targetLabels.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(logits.dataType(), targetLabels.dataType(), ctc_loss_,
|
||||
(logits, targetLabels, logitsLengths, targetLabelLengths, logLosses, gradients, blankIndex),
|
||||
SD_FLOAT_TYPES, SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( void ctc_loss_,
|
||||
(NDArray&logits, NDArray&targetLabels, NDArray&logitsLengths,
|
||||
NDArray&targetLabelLengths, NDArray &logLosses, NDArray &gradients, int blankIndex),
|
||||
SD_FLOAT_TYPES, SD_INDEXING_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,110 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/d_t_s.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void __depthToSpace(NDArray&input, NDArray *output, int block_size, bool isNHWC) {
|
||||
T const *input_ptr = reinterpret_cast<T const *>(input.buffer());
|
||||
T *output_ptr = reinterpret_cast<T *>(output->buffer());
|
||||
|
||||
const int batch_size = input.sizeAt(0);
|
||||
const int input_depth = isNHWC ? input.sizeAt(3) : input.sizeAt(1);
|
||||
const int input_height = isNHWC ? input.sizeAt(1) : input.sizeAt(2);
|
||||
const int input_width = isNHWC ? input.sizeAt(2) : input.sizeAt(3);
|
||||
|
||||
const int output_depth = isNHWC ? output->sizeAt(3) : output->sizeAt(1);
|
||||
const int output_height = isNHWC ? output->sizeAt(1) : output->sizeAt(2);
|
||||
const int output_width = isNHWC ? output->sizeAt(2) : output->sizeAt(3);
|
||||
|
||||
const int input_area = input_width * input_height;
|
||||
const int input_depth_by_input_area = input_depth * input_area;
|
||||
const int output_depth_by_input_height = output_depth * input_height;
|
||||
|
||||
if (isNHWC) {
|
||||
const int total_count = batch_size * output_height * output_width * output_depth;
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto out_idx = start; out_idx < stop; out_idx++) {
|
||||
const int d = out_idx % output_depth;
|
||||
const int out_idx2 = out_idx / output_depth;
|
||||
const int w = out_idx2 % output_width;
|
||||
const int out_idx3 = out_idx2 / output_width;
|
||||
const int h = out_idx3 % output_height;
|
||||
const int b = out_idx3 / output_height;
|
||||
|
||||
const int in_h = h / block_size;
|
||||
const int offset_h = h % block_size;
|
||||
const int in_w = w / block_size;
|
||||
const int offset_w = w % block_size;
|
||||
const int offset_d = (offset_h * block_size + offset_w) * output_depth;
|
||||
const int in_d = d + offset_d;
|
||||
const int inp_idx = in_d + input_depth * (in_w + input_width * (in_h + input_height * b));
|
||||
(output_ptr + out_idx)[0] = (input_ptr + inp_idx)[0];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, total_count);
|
||||
} else {
|
||||
const int total_count = batch_size * input_depth_by_input_area;
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (int input_idx = start; input_idx < stop; input_idx++) {
|
||||
const int n_bY_bX_oC_iY = input_idx / input_width;
|
||||
const int iX = input_idx - n_bY_bX_oC_iY * input_width;
|
||||
|
||||
const int n_bY_bX = n_bY_bX_oC_iY / output_depth_by_input_height;
|
||||
const int oC_iY = n_bY_bX_oC_iY - n_bY_bX * output_depth_by_input_height;
|
||||
|
||||
const int n_bY = n_bY_bX / block_size;
|
||||
const int bX = n_bY_bX - n_bY * block_size;
|
||||
|
||||
const int n = n_bY / block_size;
|
||||
const int bY = n_bY - n * block_size;
|
||||
|
||||
const int output_idx =
|
||||
bX + block_size * (iX + input_width * (bY + block_size * (oC_iY + n * output_depth_by_input_height)));
|
||||
|
||||
(output_ptr + output_idx)[0] = (input_ptr + input_idx)[0];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, total_count);
|
||||
}
|
||||
}
|
||||
|
||||
void _depthToSpace(sd::LaunchContext *context, NDArray&input, NDArray *output, int block_size, bool isNHWC) {
|
||||
auto xType = input.dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, __depthToSpace, (input, output, block_size, isNHWC), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void __depthToSpace,
|
||||
(NDArray&input, NDArray *output, int block_size, bool isNHWC);
|
||||
, SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/gammaMathFunc.h>
|
||||
#if NOT_EXCLUDED(OP_digamma)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// calculate digamma function for array elements
|
||||
template <typename T>
|
||||
static void diGamma_(NDArray& x, NDArray& z) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) z.p(i, diGammaScalar<T>(x.e<T>(i)));
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, x.lengthOf());
|
||||
}
|
||||
|
||||
void diGamma(sd::LaunchContext* context, NDArray& x, NDArray& z) {
|
||||
BUILD_SINGLE_SELECTOR(x.dataType(), diGamma_, (x, z), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void diGamma_, (NDArray& x, NDArray& z), SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,63 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by GS <sgazeos@gmail.com> on 4/6/2018.
|
||||
//
|
||||
#include <array/ResultSet.h>
|
||||
#include <ops/declarable/helpers/diag.h>
|
||||
#if NOT_EXCLUDED(OP_diag)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Returns a batched matrix tensor with new batched diagonal values.
|
||||
// for detailed explanations please take a look on web page:
|
||||
// https://www.tensorflow.org/api_docs/python/tf/matrix_set_diag
|
||||
template <typename T>
|
||||
static void _diagFunctor(NDArray* input, NDArray* output) {
|
||||
const int inLength = input->isScalar() ? 1 : input->lengthOf();
|
||||
|
||||
for (int i = 0; i < inLength; ++i) output->p<T>(i * (inLength + 1), (*input).e<T>(i));
|
||||
}
|
||||
|
||||
void diagFunctor(sd::LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
auto xType = input->dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, _diagFunctor, (input, output), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _diagFunctor, (NDArray* input, NDArray* output);, SD_COMMON_TYPES);
|
||||
|
||||
void diagPartFunctor(sd::LaunchContext* context, NDArray * input, NDArray* output) {
|
||||
const int outLen = output->lengthOf();
|
||||
const int inLen = input->lengthOf();
|
||||
int i(0), j(0);
|
||||
while (j < outLen) {
|
||||
auto currE = input->e(i);
|
||||
output->p(j, &currE);
|
||||
i += outLen + 1;
|
||||
++j;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @autkhor raver119@gmail.com
|
||||
//
|
||||
#include <array/DataTypeUtils.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/dilation2d.h>
|
||||
#if NOT_EXCLUDED(OP_dilation2d)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename X, typename Z>
|
||||
static void dilation2d_(NDArray* input, NDArray* weights, NDArray* output, const sd::LongType sH, const sd::LongType sW, const sd::LongType pH,
|
||||
const sd::LongType pW, const sd::LongType dH, const sd::LongType dW) {
|
||||
// input [bS, iH, iW, iC]
|
||||
// weights [kH, kW, iC]
|
||||
// output [bS, oH, oW, iC]
|
||||
|
||||
const X* x = input->bufferAsT<X>();
|
||||
const X* y = weights->bufferAsT<X>();
|
||||
Z* z = output->bufferAsT<Z>();
|
||||
|
||||
const sd::LongType* xShapeInfo = input->shapeInfo();
|
||||
const sd::LongType* yShapeInfo = weights->shapeInfo();
|
||||
const sd::LongType* zShapeInfo = output->shapeInfo();
|
||||
|
||||
const sd::LongType bS = input->sizeAt(0);
|
||||
const sd::LongType iH = input->sizeAt(1);
|
||||
const sd::LongType iW = input->sizeAt(2);
|
||||
const sd::LongType iC = input->sizeAt(3);
|
||||
|
||||
const sd::LongType kH = weights->sizeAt(0);
|
||||
const sd::LongType kW = weights->sizeAt(1);
|
||||
|
||||
const sd::LongType oH = output->sizeAt(1);
|
||||
const sd::LongType oW = output->sizeAt(2);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
for (auto b = start_x; b < stop_x; b += inc_x) {
|
||||
for (auto oh = start_y; oh < stop_y; oh += inc_y) {
|
||||
for (sd::LongType ow = 0; ow < oW; ++ow) {
|
||||
for (sd::LongType c = 0; c < iC; ++c) {
|
||||
X max = -DataTypeUtils::max<X>();
|
||||
|
||||
for (sd::LongType kh = 0; kh < kH; ++kh) {
|
||||
const int ih = oh * sH - pH + kh * dH;
|
||||
if (ih < 0 || ih >= iH) continue;
|
||||
|
||||
for (sd::LongType kw = 0; kw < kW; ++kw) {
|
||||
const int iw = ow * sW - pW + kw * dW;
|
||||
if (iw < 0 || iw >= iW) continue;
|
||||
|
||||
sd::LongType xCoords[4] = {static_cast<sd::LongType>(b), static_cast<sd::LongType>(ih),
|
||||
static_cast<sd::LongType>(iw), c};
|
||||
sd::LongType yCoords[3] = {kh, kw, c};
|
||||
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(shape::rank(xShapeInfo), shape::stride(xShapeInfo), xCoords, xOffset);
|
||||
|
||||
sd::LongType yOffset;
|
||||
COORDS2INDEX(shape::rank(yShapeInfo), shape::stride(yShapeInfo), yCoords, yOffset);
|
||||
|
||||
const X val = x[xOffset] + y[yOffset];
|
||||
if (val > max) max = val;
|
||||
}
|
||||
}
|
||||
|
||||
sd::LongType zCoords[4] = {static_cast<sd::LongType>(b), static_cast<sd::LongType>(oh), ow, c};
|
||||
sd::LongType zOffset;
|
||||
COORDS2INDEX(shape::rank(zShapeInfo), shape::stride(zShapeInfo), zCoords, zOffset);
|
||||
|
||||
z[zOffset] = static_cast<Z>(max);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, oH, 1);
|
||||
}
|
||||
|
||||
|
||||
void dilation2d(sd::LaunchContext* context, NDArray* input, NDArray* weights, NDArray* output, const sd::LongType sH,
|
||||
const sd::LongType sW, const sd::LongType pH, const sd::LongType pW, const sd::LongType dH, const sd::LongType dW) {
|
||||
BUILD_SINGLE_SELECTOR_TWICE(input->dataType(), dilation2d_, (input, weights, output, sH, sW, pH, pW, dH, dW),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,189 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <legacy/NativeOps.h>
|
||||
#include <ops/declarable/helpers/dropout.h>
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#if NOT_EXCLUDED(OP_dropout)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void dropoutSimple(NDArray* input, NDArray* output, double probValue, int seed, NDArray* mask) {
|
||||
sd::graph::RandomGenerator nodeRng(3019L, seed);
|
||||
int inLen = input->lengthOf();
|
||||
std::vector<sd::LongType> inShape = {inLen};
|
||||
std::vector<sd::LongType> outShape = {output->lengthOf()};
|
||||
auto flattenedInput = input->reshape('c',inShape,false);
|
||||
auto flattenedOutput = output->reshape('c',outShape,false);
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
float val = nodeRng.relativeT<T>(e, T(0.f), T(1.f));
|
||||
//dropout mask might not be the same length
|
||||
if (mask != nullptr && e < mask->lengthOf()) mask->p<T>(e, static_cast<T>(val));
|
||||
if (val < probValue) flattenedOutput->p<T>(e, flattenedInput->e<T>(e));
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, inLen);
|
||||
|
||||
delete flattenedInput;
|
||||
delete flattenedOutput;
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void dropoutSimple, (NDArray* input, NDArray* output, double probValue, int seed,NDArray *mask),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
template <typename T>
|
||||
sd::Status dropOutFunctor_(graph::Context& context, NDArray* input, NDArray* output, NDArray* reduceShape, int seed,
|
||||
double probValue, NDArray* mask) {
|
||||
|
||||
if (reduceShape == nullptr) {
|
||||
dropoutSimple<T>(input, output, probValue, seed, mask);
|
||||
} else {
|
||||
REQUIRE_TRUE(reduceShape->lengthOf() <= input->rankOf(), 0, "dropout: Noise shape should be fittable to input");
|
||||
|
||||
std::vector<sd::LongType> dims(reduceShape->lengthOf());
|
||||
|
||||
bool fit = true;
|
||||
for (size_t i = 0; i < dims.size(); i++) {
|
||||
if (fit) {
|
||||
dims[i] = reduceShape->e<sd::LongType>(i);
|
||||
for (int e = 0; e < input->rankOf(); ++e)
|
||||
if (fit)
|
||||
if (input->sizeAt(e) % dims[i]) {
|
||||
fit = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check dims to fit input
|
||||
REQUIRE_TRUE(fit, 0, "dropout: Noise shape should fit to input rank.");
|
||||
std::unique_ptr<NDArray> chunk(new NDArray('c', dims, output->dataType(), output->getContext()));
|
||||
float assign = 1.f;
|
||||
chunk->assign(assign);
|
||||
dropoutSimple<T>(chunk.get(), chunk.get(), probValue, seed, nullptr);
|
||||
// broadcast chunk to full matrix
|
||||
mask->assign(assign);
|
||||
|
||||
*mask += *chunk;
|
||||
NDArray *assign5 = *input * *mask;
|
||||
output->assign(assign5);
|
||||
delete assign5;
|
||||
}
|
||||
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status dropOutFunctor(graph::Context& context, NDArray* input, NDArray* output, NDArray* reduceShape, int seed,
|
||||
double probValue, NDArray* mask) {
|
||||
auto xType = input->dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, return dropOutFunctor_, (context, input, output, reduceShape, seed, probValue,mask),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status dropOutFunctor_, (graph::Context & context, NDArray* input, NDArray* output,
|
||||
NDArray* reduceShape, int seed, double probValue,NDArray *mask);
|
||||
, SD_FLOAT_TYPES);
|
||||
|
||||
/////////////////////////////////// backprpopagations ///////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static Status dropOutFunctorBP_(graph::Context& context, NDArray* input, NDArray* gradOut, NDArray* output,
|
||||
NDArray* reduceShape, int seed, double probValue, NDArray* mask) {
|
||||
auto mask2 = *gradOut * *mask;
|
||||
*output = *mask2;
|
||||
delete mask2;
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static Status alphaDropOutFunctor_(graph::Context& context, NDArray* input, NDArray* output, NDArray* reduceShape,
|
||||
int seed, double probValue, double alpha, double alpha1, double beta,
|
||||
NDArray* mask) {
|
||||
|
||||
sd::graph::RandomGenerator nodeRng(3019L, seed);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
float randVal = nodeRng.relativeT(e, T(0.f), T(1.f));
|
||||
float xVal = input->e<float>(e);
|
||||
float maskVal = randVal >= probValue ? alpha * beta + alpha1 : alpha * 1 + alpha1;
|
||||
mask->p<float>(e, maskVal);
|
||||
output->p<float>(e, randVal >= probValue ? alpha * beta + alpha1 : alpha * xVal + alpha1);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input->lengthOf());
|
||||
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
sd::Status alphaDropOutFunctorBP_(graph::Context& context, NDArray* input, NDArray* gradOut, NDArray* output,
|
||||
NDArray* reduceShape, int seed, double probValue, double alpha, double alpha1,
|
||||
double beta, NDArray* mask) {
|
||||
|
||||
auto mask2 = *gradOut * *mask;
|
||||
*output *= *mask2;
|
||||
delete mask2;
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status dropOutFunctorBP(graph::Context& context, NDArray* input, NDArray* gradOut, NDArray* output,
|
||||
NDArray* reduceShape, int seed, double probValue, NDArray* mask) {
|
||||
BUILD_SINGLE_SELECTOR(context.dataType(), return dropOutFunctorBP_,
|
||||
(context, input, gradOut, output, reduceShape, seed, probValue,mask), SD_FLOAT_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status dropOutFunctorBP_,
|
||||
(::Context & context, NDArray* input, NDArray* gradOut, NDArray* output,
|
||||
NDArray* reduceShape, int seed, double probValue,NDArray* mask),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
sd::Status alphaDropOutFunctor(graph::Context& context, NDArray* input, NDArray* output, NDArray* reduceShape, int seed,
|
||||
double probValue, double alpha, double alpha1, double beta, NDArray* mask) {
|
||||
BUILD_SINGLE_SELECTOR(context.dataType(), return alphaDropOutFunctor_,
|
||||
(context, input, output, reduceShape, seed, probValue, alpha, alpha1, beta,mask), SD_FLOAT_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status alphaDropOutFunctor_,
|
||||
(graph::Context & context, NDArray* input, NDArray* output, NDArray* reduceShape, int seed,
|
||||
double probValue, double alpha, double alpha1, double beta,NDArray* mask),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
sd::Status alphaDropOutFunctorBP(graph::Context& context, NDArray* input, NDArray* gradOut, NDArray* output,
|
||||
NDArray* reduceShape, int seed, double probValue, double alpha, double alpha1,
|
||||
double beta, NDArray* mask) {
|
||||
BUILD_SINGLE_SELECTOR(context.dataType(), return alphaDropOutFunctorBP_,
|
||||
(context, input, gradOut, output, reduceShape, seed, probValue, alpha, alpha1, beta,mask),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status alphaDropOutFunctorBP_,
|
||||
(graph::Context & context, NDArray* input, NDArray* gradOut, NDArray* output,
|
||||
NDArray* reduceShape, int seed, double probValue, double alpha, double alpha1, double beta,NDArray *mask),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,230 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by george on 05.04.18.
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/dynamic.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void _dynamicPartitionFunctor(NDArray * input, NDArray * indices, std::vector<NDArray*>& outputList) {
|
||||
std::vector<std::pair<NDArray*, sd::LongType>> outputs(outputList.size());
|
||||
int sourceDimsLen = input->rankOf() - indices->rankOf();
|
||||
if (sourceDimsLen) {
|
||||
std::vector<sd::LongType> sourceDims(sourceDimsLen);
|
||||
|
||||
for (sd::LongType i = sourceDimsLen; i > 0; i--) sourceDims[sourceDimsLen - i] = input->rankOf() - i;
|
||||
|
||||
ResultSet listOfTensors = input->allTensorsAlongDimension(sourceDims);
|
||||
|
||||
sd::LongType outSize = outputList.size();
|
||||
|
||||
for (sd::LongType i = 0; i < outSize; i++) {
|
||||
outputs[i].first = outputList[i];
|
||||
std::vector<sd::LongType > outDims(outputs[i].first->rankOf() - 1);
|
||||
|
||||
sd::LongType r = outputs[i].first->rankOf();
|
||||
|
||||
for (sd::LongType k = 1; k < r; k++) outDims[k - 1] = k;
|
||||
|
||||
ResultSet listOutForCurrent = outputs[i].first->allTensorsAlongDimension(outDims);
|
||||
|
||||
outputs[i].second = 0;
|
||||
|
||||
for (sd::LongType e = 0; e < indices->lengthOf(); ++e)
|
||||
if ((*indices).e<sd::LongType>(e) == i) {
|
||||
listOutForCurrent.at(outputs[i].second++)->assign(listOfTensors.at(e));
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
sd::LongType outSize = outputList.size();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
outputs[i].first = outputList[i];
|
||||
outputs[i].second = 0;
|
||||
for (sd::LongType e = 0; e < indices->lengthOf(); ++e)
|
||||
if (indices->e<sd::LongType>(e) == i) outputs[i].first->p(outputs[i].second++, input->e<T>(e));
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, outSize);
|
||||
}
|
||||
}
|
||||
template <typename T>
|
||||
static sd::Status _dynamicStitchFunctor(std::vector<NDArray*> const& inputs, std::vector<NDArray*> const& indices,
|
||||
NDArray* output) {
|
||||
sd::LongType numOfData = inputs.size();
|
||||
|
||||
if (output->isVector()) {
|
||||
for (sd::LongType e = 0; e < numOfData; e++) {
|
||||
auto data = inputs[e];
|
||||
auto index = indices[e];
|
||||
for (sd::LongType i = 0; i < index->lengthOf(); i++) {
|
||||
sd::LongType pos = index->e<sd::LongType>(i);
|
||||
if (pos < 0) {
|
||||
sd_printf("dynamic_stitch: Index value should be non-negative. But %i was given", pos);
|
||||
return sd::Status::VALIDATION;
|
||||
}
|
||||
if (pos >= output->lengthOf()) {
|
||||
sd_printf("dynamic_stitch: Index should be less than %i. But %i was given", output->lengthOf(), pos);
|
||||
return sd::Status::VALIDATION;
|
||||
}
|
||||
output->p<T>(pos, data->e<T>(i));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
std::vector<sd::LongType > restDims(output->rankOf() - 1);
|
||||
for (auto i = restDims.size(); i > 0; i--) restDims[restDims.size() - i] = output->rankOf() - i;
|
||||
|
||||
ResultSet listOfOutTensors = output->allTensorsAlongDimension(restDims);
|
||||
for (int e = 0; e < numOfData; e++) {
|
||||
auto data = inputs[e];
|
||||
auto index = indices[e];
|
||||
std::vector<sd::LongType > sourceDims(data->rankOf() - index->rankOf());
|
||||
for (auto i = sourceDims.size(); i > 0; i--) sourceDims[sourceDims.size() - i] = data->rankOf() - i;
|
||||
|
||||
ResultSet listOfTensors = data->allTensorsAlongDimension(sourceDims);
|
||||
|
||||
for (sd::LongType i = 0; i < index->lengthOf(); i++) {
|
||||
auto pos = index->e<sd::LongType>(i);
|
||||
if (pos < 0) {
|
||||
sd_printf("dynamic_stitch: Index value should be non-negative. But %i was given", pos);
|
||||
return sd::Status::VALIDATION;
|
||||
}
|
||||
if (pos >= output->lengthOf()) {
|
||||
sd_printf("dynamic_stitch: Index should be less than %i. But %i was given", output->lengthOf(), pos);
|
||||
return sd::Status::VALIDATION;
|
||||
}
|
||||
|
||||
listOfOutTensors.at(pos)->assign(listOfTensors.at(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void _dynamicPartitionFunctorBP(NDArray * input, NDArray * indices,
|
||||
std::vector<NDArray*> const& inputGradientList,
|
||||
std::vector<NDArray*>& outputList) {
|
||||
std::vector<std::pair<NDArray*, sd::LongType>> outputs(inputGradientList.size());
|
||||
|
||||
int sourceDimsLen = input->rankOf() - indices->rankOf();
|
||||
if (sourceDimsLen) { // multidimensional case
|
||||
std::vector<sd::LongType > sourceDims(sourceDimsLen);
|
||||
|
||||
for (sd::LongType i = sourceDimsLen; i > 0; i--) sourceDims[sourceDimsLen - i] = input->rankOf() - i;
|
||||
|
||||
ResultSet listOfTensors = outputList[0]->allTensorsAlongDimension(sourceDims);
|
||||
|
||||
for (size_t i = 0; i < inputGradientList.size(); i++) {
|
||||
outputs[i].first = inputGradientList[i];
|
||||
if (outputs[i].first->rankOf() < 1) continue; // skip empty gradient outs
|
||||
std::vector<sd::LongType > outDims(outputs[i].first->rankOf() - 1);
|
||||
|
||||
for (int k = 1; k < outputs[i].first->rankOf(); k++) outDims[k - 1] = k;
|
||||
|
||||
ResultSet listOutForCurrent = outputs[i].first->allTensorsAlongDimension(outDims);
|
||||
|
||||
outputs[i].second = 0;
|
||||
|
||||
for (sd::LongType e = 0; e < indices->lengthOf(); ++e)
|
||||
if (indices->e<sd::LongType>(e) == static_cast<sd::LongType>(i)) listOfTensors.at(e)->assign(listOutForCurrent.at(outputs[i].second++));
|
||||
}
|
||||
} else { // one-dimensional case
|
||||
auto output = outputList[0];
|
||||
unsigned int gradsSize = inputGradientList.size();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
outputs[i].first = inputGradientList[i];
|
||||
outputs[i].second = 0;
|
||||
for (sd::LongType e = 0; e < indices->lengthOf(); ++e)
|
||||
if (indices->e<sd::LongType>(e) == i) output->p<T>(e, outputs[i].first->e<T>(outputs[i].second++));
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, gradsSize);
|
||||
}
|
||||
|
||||
outputList[1]->assign(indices);
|
||||
}
|
||||
|
||||
void dynamicPartitionFunctor(sd::LaunchContext* context, NDArray * input, NDArray * indices,
|
||||
std::vector<NDArray*>& outputList) {
|
||||
auto xType = input->dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, _dynamicPartitionFunctor, (input, indices, outputList), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static sd::Status _dynamicStitchFunctorBP(std::vector<NDArray*> const& inputs, std::vector<NDArray*> const& indices,
|
||||
NDArray * gradInput, std::vector<NDArray*>& outputList) {
|
||||
THROW_EXCEPTION("Not implemented yet");
|
||||
}
|
||||
|
||||
sd::Status dynamicStitchFunctor(sd::LaunchContext* context, std::vector<NDArray*> const& inputs,
|
||||
std::vector<NDArray*> const& indices, NDArray* output) {
|
||||
auto xType = inputs.at(0)->dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, return _dynamicStitchFunctor, (inputs, indices, output), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
sd::Status dynamicStitchFunctorBP(sd::LaunchContext* context, std::vector<NDArray*> const& inputs,
|
||||
std::vector<NDArray*> const& indices, NDArray * gradInput,
|
||||
std::vector<NDArray*>& outputList) {
|
||||
auto xType = inputs.at(0)->dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, return _dynamicStitchFunctorBP, (inputs, indices, gradInput, outputList),
|
||||
SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
void dynamicPartitionFunctorBP(sd::LaunchContext* context, NDArray * input, NDArray * indices,
|
||||
std::vector<NDArray*> const& inputGradientList, std::vector<NDArray*>& outputList) {
|
||||
auto xType = input->dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, _dynamicPartitionFunctorBP, (input, indices, inputGradientList, outputList),
|
||||
SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _dynamicPartitionFunctorBP,
|
||||
(NDArray * input, NDArray * indices, std::vector<NDArray*> const& inputGradientList,
|
||||
std::vector<NDArray*>& outputList);
|
||||
, SD_COMMON_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status _dynamicStitchFunctorBP,
|
||||
(std::vector<NDArray*> const& inputs, std::vector<NDArray*> const& indices,
|
||||
NDArray * gradInput, std::vector<NDArray*>& outputList);
|
||||
, SD_COMMON_TYPES);
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _dynamicPartitionFunctor,
|
||||
(NDArray * input, NDArray * indices, std::vector<NDArray*>& outputList);
|
||||
, SD_COMMON_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status _dynamicStitchFunctor,
|
||||
(std::vector<NDArray*> const& inputs, std::vector<NDArray*> const& indices, NDArray* output);
|
||||
, SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,106 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/axis.h>
|
||||
#if NOT_EXCLUDED(OP_extract_patches)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void _extractPatches(NDArray* images, NDArray* output, int sizeRow, int sizeCol, int strideRow, int strideCol,
|
||||
int rateRow, int rateCol, bool theSame) {
|
||||
std::vector<sd::LongType> restDims({1, 2, 3}); // the first and the last dims
|
||||
ResultSet listOfMatricies = images->allTensorsAlongDimension(restDims);
|
||||
ResultSet listOfOutputs = output->allTensorsAlongDimension(restDims);
|
||||
// 3D matricies - 2D matricies of vectors (if last dim is greater than 1)
|
||||
// int e = 0;
|
||||
const int ksizeRowsEffective = sizeRow + (sizeRow - 1) * (rateRow - 1);
|
||||
const int ksizeColsEffective = sizeCol + (sizeCol - 1) * (rateCol - 1);
|
||||
const int ksize = ksizeRowsEffective * ksizeColsEffective;
|
||||
int batchCount = listOfMatricies.size(); // lengthOf() / ksize;
|
||||
sd::LongType lastDim = images->sizeAt(3);
|
||||
sd::LongType outLastDim = output->sizeAt(3);
|
||||
sd::LongType rowDim = images->sizeAt(1);
|
||||
sd::LongType colDim = images->sizeAt(2);
|
||||
sd::LongType outRowDim = output->sizeAt(1);
|
||||
sd::LongType outColDim = output->sizeAt(2);
|
||||
auto rowCast = 1; //(sizeRow - 1)*rateRow < outRowDim/sizeRow ?0:1;///(ksize * lastDim > rowDim * ksizeColsEffective
|
||||
//+ lastDim?1:0);
|
||||
auto colCast = 1; // colDim / ksizeColsEffective +2 <= sizeCol?0:1;//(ksize * lastDim > ksizeRowsEffective * colDim +
|
||||
// lastDim?1:0);
|
||||
if (sizeRow * rateRow < 3) rowCast = 0;
|
||||
if (sizeCol * rateCol < 3) colCast = 0;
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto batch = 0; batch < stop; batch++) {
|
||||
auto patch = listOfMatricies.at(batch);
|
||||
auto outMatrix = listOfOutputs.at(batch);
|
||||
|
||||
for (sd::LongType i = 0; i < outRowDim; i++) {
|
||||
for (sd::LongType j = 0; j < outColDim; j++) {
|
||||
sd::LongType pos = 0;
|
||||
// for (sd::LongType k = 0; k < outputLastDim; k++) {
|
||||
auto rowStart = i * strideRow - (theSame ? rowCast : 0);
|
||||
auto colStart = j * strideCol - (theSame ? colCast : 0);
|
||||
auto rowEnd = rowStart + sizeRow * rateRow;
|
||||
auto colEnd = colStart + sizeCol * rateCol;
|
||||
if (!theSame) {
|
||||
rowEnd = math::sd_min(rowStart + sizeRow * rateRow, rowDim);
|
||||
colEnd = math::sd_min(colStart + sizeCol * rateCol, colDim);
|
||||
}
|
||||
// auto pixel = 0LL;
|
||||
for (auto row = rowStart; row < rowEnd; row += rateRow)
|
||||
for (auto col = colStart; col < colEnd; col += rateCol)
|
||||
for (auto pixel = 0; pixel < lastDim; pixel++) {
|
||||
bool setUp = (theSame && row >= 0 && col >= 0 && row < rowDim && col < colDim) || (!theSame);
|
||||
if (setUp) {
|
||||
outMatrix->r<T>(i, j, pos) = patch->e<T>(row, col, pixel);
|
||||
}
|
||||
pos++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, batchCount);
|
||||
}
|
||||
|
||||
void extractPatches(sd::LaunchContext* context, NDArray* images, NDArray* output, int sizeRow, int sizeCol,
|
||||
int stradeRow, int stradeCol, int rateRow, int rateCol, bool theSame) {
|
||||
auto xType = images->dataType();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, _extractPatches,
|
||||
(images, output, sizeRow, sizeCol, stradeRow, stradeCol, rateRow, rateCol, theSame),
|
||||
SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _extractPatches,
|
||||
(NDArray * input, NDArray* output, int sizeRow, int sizeCol, int stradeRow, int stradeCol,
|
||||
int rateRow, int rateCol, bool theSame),
|
||||
SD_NUMERIC_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,45 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 20.04.2018
|
||||
//
|
||||
|
||||
#include <helpers/Loops.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
#if NOT_EXCLUDED(OP_eye)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void eye(sd::LaunchContext* context, NDArray& output) {
|
||||
const int rank = output.rankOf();
|
||||
auto arrs = output.allTensorsAlongDimension({rank - 2, rank - 1});
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) arrs.at(i)->setIdentity();
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, arrs.size());
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,128 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <ops/declarable/helpers/fake_quantization.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//
|
||||
// nudge - nudged min max over scale
|
||||
// scale = (Max - Min) / (quantMax - quantMin)
|
||||
// quantMin = 0 or 1, quantMax = 2^b - 1 == (1 << b) - 1
|
||||
//
|
||||
template <typename T>
|
||||
static void nudge(T min, T max, int quantMin, int quantMax, T* scale, T* nudgedMin, T* nudgedMax) {
|
||||
// floating point instead integers
|
||||
T quantMaxF = static_cast<T>(quantMax);
|
||||
T quantMinF = static_cast<T>(quantMin);
|
||||
// compute scale
|
||||
*scale = (max - min) / (quantMaxF - quantMinF);
|
||||
// compute left bound point
|
||||
auto zeroPointFromMin = quantMinF - min / *scale;
|
||||
// bound zero point to conform with range [0 or 1, 2^b - 1]
|
||||
uint16_t const nudged_zero_point = [zeroPointFromMin, quantMin, quantMax, quantMaxF, quantMinF] {
|
||||
if (zeroPointFromMin < quantMinF) {
|
||||
return static_cast<uint16_t>(quantMin);
|
||||
}
|
||||
if (zeroPointFromMin > quantMaxF) {
|
||||
return static_cast<uint16_t>(quantMax);
|
||||
}
|
||||
return (uint16_t)sd::math::sd_round<T, int>(zeroPointFromMin);
|
||||
}();
|
||||
// compute nudged min and max with computed nudged zero point
|
||||
*nudgedMin = (quantMinF - nudged_zero_point) * (*scale);
|
||||
*nudgedMax = (quantMaxF - nudged_zero_point) * (*scale);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void fakeQuantWithMinMaxVarsPerChannel_(NDArray* input, NDArray* min, NDArray* max, int numBits, bool narrowed,
|
||||
NDArray* output) {
|
||||
int lowIntBound = narrowed ? 1 : 0; // 0 or 1
|
||||
int upperIntBound = (1 << numBits) - 1; // 2^b - 1
|
||||
auto channels = input->sizeAt(-1); // last dimension
|
||||
|
||||
PRAGMA_OMP_PARALLEL_FOR
|
||||
for (auto i = 0; i < channels; i++) {
|
||||
T scale, nudged_min, nudged_max;
|
||||
// nudge min and max first, with scale computing
|
||||
nudge<T>(min->t<T>(i), max->t<T>(i), lowIntBound, upperIntBound, &scale, &nudged_min, &nudged_max);
|
||||
// slide using last dimension and process all for given channel
|
||||
for (auto e = 0; e < input->lengthOf(); e += channels) {
|
||||
T val = input->t<T>(e + i);
|
||||
if (val <= nudged_min)
|
||||
val = nudged_min;
|
||||
else if (val >= nudged_max)
|
||||
val = nudged_max;
|
||||
// quantization itself
|
||||
output->r<T>(e + i) = math::sd_floor<T, T>((val - nudged_min) / scale + T(0.5)) * scale + nudged_min;
|
||||
}
|
||||
}
|
||||
}
|
||||
//
|
||||
// const auto clamped = inputs.cwiseMin(nudged_max).cwiseMax(nudged_min);
|
||||
// const auto clamped_shifted = clamped - nudged_min;
|
||||
// outputs.device(d) = (clamped_shifted / nudged_scale_repl + 0.5f).floor() *
|
||||
// nudged_scale_repl +
|
||||
// nudged_min;
|
||||
//
|
||||
template <typename T>
|
||||
void fakeQuantWithMinMaxVars_(NDArray* input, NDArray* min, NDArray* max, int numBits, bool narrowed, NDArray* output) {
|
||||
int lowIntBound = narrowed ? 1 : 0;
|
||||
int upperIntBound = (1 << numBits) - 1;
|
||||
|
||||
T nudgedMin, nudgedMax, scale;
|
||||
// nudge with given min and max and compute scale and nudged min and max
|
||||
nudge<T>(min->t<T>(0), max->t<T>(0), lowIntBound, upperIntBound, &scale, &nudgedMin, &nudgedMax);
|
||||
// quantization as one
|
||||
auto fakeQuantizationWithMinMax = LAMBDA_T(x, nudgedMin, nudgedMax, scale) {
|
||||
T val = x; // boundign value between nudged min and max
|
||||
if (val < nudgedMin) {
|
||||
val = nudgedMin;
|
||||
} else if (val > nudgedMax)
|
||||
val = nudgedMax;
|
||||
// converse value with scale and shifted with nudged min
|
||||
val -= nudgedMin;
|
||||
return (sd::math::sd_floor<T, T>(val / scale + T(0.5f)) * scale + nudgedMin);
|
||||
});
|
||||
|
||||
input->applyLambda<T>(fakeQuantizationWithMinMax, output);
|
||||
}
|
||||
|
||||
void fakeQuantWithMinMaxVars(NDArray* input, NDArray* min, NDArray* max, int numBits, bool narrowed, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), fakeQuantWithMinMaxVars_, (input, min, max, numBits, narrowed, output),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
void fakeQuantWithMinMaxVarsPerChannel(LaunchContext* context, NDArray* input, NDArray* min, NDArray* max, int numBits,
|
||||
bool narrowed, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), fakeQuantWithMinMaxVarsPerChannel_,
|
||||
(input, min, max, numBits, narrowed, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void fakeQuantWithMinMaxVars_,
|
||||
(NDArray * input, NDArray* min, NDArray* max, int numBits, bool narrowed, NDArray* output),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,71 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/flatten.h>
|
||||
#if NOT_EXCLUDED(OP_flatten)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void flatten_(std::vector<NDArray *> &inputs, NDArray *output, const char order) {
|
||||
int numArrays = inputs.size();
|
||||
std::vector<sd::LongType> offsets(numArrays);
|
||||
sd::LongType cOffset = 0;
|
||||
|
||||
// calculating offsets in output
|
||||
for (int e = 0; e < numArrays; e++) {
|
||||
offsets[e] = cOffset;
|
||||
cOffset += inputs[e]->lengthOf();
|
||||
}
|
||||
|
||||
// actually transferring data
|
||||
for (sd::LongType e = 0; e < numArrays; e++) {
|
||||
auto z = reinterpret_cast<T *>(output->bufferWithOffset(offsets[e]));
|
||||
auto xBuffer = inputs[e]->bufferAsT<T>();
|
||||
auto xShapeInfo = inputs[e]->shapeInfo();
|
||||
|
||||
// Cache shape-related values outside the inner loop
|
||||
const int xRank = shape::rank(xShapeInfo);
|
||||
const sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
const sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
const sd::LongType xLength = inputs[e]->lengthOf();
|
||||
|
||||
for (sd::LongType i = 0; i < xLength; i++) {
|
||||
sd::LongType xOffset;
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
|
||||
// Use cached shape values for coordinate transforms
|
||||
INDEX2COORDS(i, xRank, xShape, xCoords);
|
||||
COORDS2INDEX(xRank, xStride, xCoords, xOffset);
|
||||
|
||||
z[i] = xBuffer[xOffset];
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
void flatten(sd::LaunchContext *context, std::vector<NDArray *> &inputs, NDArray *output, char order) {
|
||||
BUILD_SINGLE_SELECTOR(output->dataType(), flatten_, (inputs, output, order), SD_COMMON_TYPES);
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,355 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 07.03.2019
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/gather.h>
|
||||
#include <legacy/NativeOpExecutioner.h>
|
||||
|
||||
#include <numeric>
|
||||
#if NOT_EXCLUDED(OP_gather)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
void gather(sd::LaunchContext* context, NDArray* input, NDArray* indices, NDArray* output,
|
||||
const std::vector<LongType>& intArgs) {
|
||||
sd::LongType axis = intArgs.size() > 0 ? intArgs[0] : 0;
|
||||
const sd::LongType inputRank = input->rankOf();
|
||||
if (axis < 0) axis += inputRank;
|
||||
|
||||
const sd::LongType numOfIntArgs = intArgs.size();
|
||||
|
||||
// Special handling for 1D input with axis=0
|
||||
// This handles cases like gathering from shape arrays where we want flat indexing
|
||||
bool is1DFlatGather = (inputRank == 1 && axis == 0);
|
||||
|
||||
if (indices != nullptr) {
|
||||
// Validate indices
|
||||
for (sd::LongType i = 0; i < indices->lengthOf(); ++i) {
|
||||
auto idx = indices->e<sd::LongType>(i);
|
||||
|
||||
if (is1DFlatGather) {
|
||||
// For 1D arrays with axis=0, treat as flat array access
|
||||
if (idx >= input->lengthOf() || idx < 0) {
|
||||
std::string error = "helpers::gather function: invalid flat index ";
|
||||
error += std::to_string(idx);
|
||||
error += " at position ";
|
||||
error += std::to_string(i);
|
||||
error += ". Input is 1D with length ";
|
||||
error += std::to_string(input->lengthOf());
|
||||
error += ", valid range is [0, ";
|
||||
error += std::to_string(input->lengthOf() - 1);
|
||||
error += "]";
|
||||
THROW_EXCEPTION(error.c_str());
|
||||
}
|
||||
} else {
|
||||
// Standard axis-based validation
|
||||
if (idx >= input->sizeAt(axis) || idx < 0) {
|
||||
std::string error = "helpers::gather function: invalid index ";
|
||||
error += std::to_string(idx);
|
||||
error += " at position ";
|
||||
error += std::to_string(i);
|
||||
error += ". Input shape ";
|
||||
error += ShapeUtils::shapeAsString(input->shapeInfo());
|
||||
error += ", axis ";
|
||||
error += std::to_string(axis);
|
||||
error += ", valid range is [0, ";
|
||||
error += std::to_string(input->sizeAt(axis) - 1);
|
||||
error += "]";
|
||||
THROW_EXCEPTION(error.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (is1DFlatGather) {
|
||||
// Special case: 1D input with axis=0 - treat as flat array gather
|
||||
// This handles gathering from shape arrays like [1, 512] -> gather index 1 -> get 512
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto idx = indices->e<sd::LongType>(i);
|
||||
auto value = input->e<double>(idx); // Get value at flat index
|
||||
output->p(i, value); // Put in output at position i
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, indices->lengthOf());
|
||||
|
||||
} else {
|
||||
// Standard gather implementation
|
||||
//
|
||||
// For gather with axis=A on input shape [..., dimA, ...] and indices shape [I1, I2, ...]:
|
||||
// - Output shape is: input[0:A] + indices_shape + input[A+1:]
|
||||
// - Input TADs: iterate along axis A, each TAD has shape input[A+1:]
|
||||
// - Output TADs: iterate along indices dimensions, each TAD has same shape as input TAD
|
||||
//
|
||||
// tadForDimensions takes dimensions to KEEP in each TAD (not to exclude)
|
||||
// It then internally calls evalDimsToExclude to find which dims to iterate over
|
||||
|
||||
std::vector<sd::LongType> axesVec = {axis};
|
||||
auto dimensions = ShapeUtils::evalDimsToExclude(input->rankOf(), 1, axesVec.data());
|
||||
|
||||
// For output TADs, we want the same shape as input TADs
|
||||
// Input TAD shape = all dims except axis
|
||||
// Output shape = input[0:axis] + indices_shape + input[axis+1:]
|
||||
// Output TAD dims should be: dims 0 to axis-1, then dims axis+indicesRank to end
|
||||
// This gives TAD shape matching input's TAD shape
|
||||
std::vector<sd::LongType> outputTadDims;
|
||||
sd::LongType indicesRank = indices->rankOf();
|
||||
|
||||
// Add dimensions before the indices dimensions (0 to axis-1)
|
||||
for (sd::LongType d = 0; d < axis; d++) {
|
||||
outputTadDims.push_back(d);
|
||||
}
|
||||
// Add dimensions after the indices dimensions (axis+indicesRank to outputRank-1)
|
||||
for (sd::LongType d = axis + indicesRank; d < output->rankOf(); d++) {
|
||||
outputTadDims.push_back(d);
|
||||
}
|
||||
|
||||
// If outputTadDims is empty, it means each TAD is a scalar - handle this case
|
||||
// by using the same approach as input (which would also have empty TAD dims)
|
||||
|
||||
// Get TAD packs - these are cached and should not be deleted
|
||||
auto tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(), dimensions);
|
||||
auto tadPackOut = sd::ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(), &outputTadDims);
|
||||
|
||||
// Validate TAD packs before use
|
||||
if (tadPack == nullptr || tadPackOut == nullptr) {
|
||||
if (dimensions) delete dimensions;
|
||||
THROW_EXCEPTION("gather: Failed to create TAD packs");
|
||||
}
|
||||
|
||||
// Now safe to delete dimensions as TAD helper has made internal copy
|
||||
delete dimensions;
|
||||
|
||||
auto tadShapeInfo = tadPack->primaryShapeInfo();
|
||||
auto tadOffsets = tadPack->primaryOffsets();
|
||||
auto tadShapeInfoOut = tadPackOut->primaryShapeInfo();
|
||||
auto tadOffsetsOut = tadPackOut->primaryOffsets();
|
||||
|
||||
// Validate that input and output TAD shapes match
|
||||
auto inputTadLength = shape::length(tadShapeInfo);
|
||||
auto outputTadLength = shape::length(tadShapeInfoOut);
|
||||
if (inputTadLength != outputTadLength) {
|
||||
std::string error = "gather: TAD shape mismatch - input TAD length ";
|
||||
error += std::to_string(inputTadLength);
|
||||
error += " != output TAD length ";
|
||||
error += std::to_string(outputTadLength);
|
||||
error += ". Input shape: ";
|
||||
error += ShapeUtils::shapeAsString(input->shapeInfo());
|
||||
error += ", Output shape: ";
|
||||
error += ShapeUtils::shapeAsString(output->shapeInfo());
|
||||
error += ", Indices shape: ";
|
||||
error += ShapeUtils::shapeAsString(indices->shapeInfo());
|
||||
error += ", axis: ";
|
||||
error += std::to_string(axis);
|
||||
THROW_EXCEPTION(error.c_str());
|
||||
}
|
||||
|
||||
auto tadShapeInfoCast = const_cast<sd::LongType *>(tadShapeInfo);
|
||||
auto tadShapeInfoOutCast = const_cast<sd::LongType *>(tadShapeInfoOut);
|
||||
|
||||
// Calculate the number of gather operations (equal to indices length)
|
||||
const sd::LongType numGatherOps = indices->lengthOf();
|
||||
|
||||
// Validate bounds before parallel execution
|
||||
if (numGatherOps > tadPackOut->numberOfTads()) {
|
||||
std::string error = "gather: indices length ";
|
||||
error += std::to_string(numGatherOps);
|
||||
error += " exceeds output TAD count ";
|
||||
error += std::to_string(tadPackOut->numberOfTads());
|
||||
THROW_EXCEPTION(error.c_str());
|
||||
}
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto idx = indices->e<sd::LongType>(i);
|
||||
|
||||
// Bounds check for input TAD access
|
||||
if (idx >= tadPack->numberOfTads() || idx < 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Bounds check for output TAD access
|
||||
if (i >= tadPackOut->numberOfTads()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto offsetIn = tadOffsets[idx];
|
||||
auto offsetOut = tadOffsetsOut[i];
|
||||
|
||||
NativeOpExecutioner::execTransformAny(input->getContext(),
|
||||
transform::Assign,
|
||||
input->bufferWithOffset(offsetIn), tadShapeInfoCast,
|
||||
nullptr, nullptr,
|
||||
output->bufferWithOffset(offsetOut), tadShapeInfoOutCast,
|
||||
nullptr, nullptr,
|
||||
nullptr, false);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, 0, numGatherOps);
|
||||
}
|
||||
|
||||
} else {
|
||||
// Integer arguments case
|
||||
for (int i = 1; i < numOfIntArgs; ++i) {
|
||||
if (is1DFlatGather) {
|
||||
// For 1D arrays with axis=0, validate against total length
|
||||
if (intArgs[i] >= input->lengthOf() || intArgs[i] < 0) {
|
||||
std::string error = "helpers::gather function: invalid flat index ";
|
||||
error += std::to_string(intArgs[i]);
|
||||
error += " at position ";
|
||||
error += std::to_string(i-1);
|
||||
error += ". Input is 1D with length ";
|
||||
error += std::to_string(input->lengthOf());
|
||||
error += ", valid range is [0, ";
|
||||
error += std::to_string(input->lengthOf() - 1);
|
||||
error += "]";
|
||||
THROW_EXCEPTION(error.c_str());
|
||||
}
|
||||
} else {
|
||||
// Standard validation
|
||||
if (intArgs[i] >= input->sizeAt(axis) || intArgs[i] < 0) {
|
||||
std::string error = "helpers::gather function: invalid index ";
|
||||
error += std::to_string(intArgs[i]);
|
||||
error += " at position ";
|
||||
error += std::to_string(i-1);
|
||||
error += ". Input shape ";
|
||||
error += ShapeUtils::shapeAsString(input->shapeInfo());
|
||||
error += ", axis ";
|
||||
error += std::to_string(axis);
|
||||
error += ", valid range is [0, ";
|
||||
error += std::to_string(input->sizeAt(axis) - 1);
|
||||
error += "]";
|
||||
THROW_EXCEPTION(error.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (numOfIntArgs == 2) {
|
||||
if (is1DFlatGather) {
|
||||
// For 1D flat gather with single index
|
||||
auto value = input->e<double>(intArgs[1]);
|
||||
output->assign(value);
|
||||
} else {
|
||||
// Standard single index gather
|
||||
NDArray *copy = (*input)(intArgs[1], {axis});
|
||||
output->assign(copy);
|
||||
delete copy;
|
||||
}
|
||||
} else {
|
||||
if (is1DFlatGather) {
|
||||
// Multiple indices for 1D flat gather
|
||||
for (int i = 1; i < numOfIntArgs; ++i) {
|
||||
auto idx = intArgs[i];
|
||||
auto value = input->e<double>(idx);
|
||||
output->p(i - 1, value);
|
||||
}
|
||||
} else {
|
||||
// Standard multiple indices gather
|
||||
// Use the same dimension calculation for input and output TADs
|
||||
std::vector<sd::LongType> axesVec = {axis};
|
||||
auto dimensions = ShapeUtils::evalDimsToExclude(input->rankOf(), 1, axesVec.data());
|
||||
|
||||
// Get TAD packs - these are cached and should not be deleted
|
||||
auto tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(), dimensions);
|
||||
auto tadPackOut = sd::ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(), dimensions);
|
||||
|
||||
// Validate TAD packs before use
|
||||
if (tadPack == nullptr || tadPackOut == nullptr) {
|
||||
if (dimensions) delete dimensions;
|
||||
THROW_EXCEPTION("gather: Failed to create TAD packs");
|
||||
}
|
||||
|
||||
// Now safe to delete dimensions as TAD helper has made internal copy
|
||||
delete dimensions;
|
||||
|
||||
auto tadShapeInfo = tadPack->primaryShapeInfo();
|
||||
auto tadOffsets = tadPack->primaryOffsets();
|
||||
auto tadShapeInfoOut = tadPackOut->primaryShapeInfo();
|
||||
auto tadOffsetsOut = tadPackOut->primaryOffsets();
|
||||
|
||||
// Validate that input and output TAD shapes match
|
||||
auto inputTadLength = shape::length(tadShapeInfo);
|
||||
auto outputTadLength = shape::length(tadShapeInfoOut);
|
||||
if (inputTadLength != outputTadLength) {
|
||||
std::string error = "gather: TAD shape mismatch - input TAD length ";
|
||||
error += std::to_string(inputTadLength);
|
||||
error += " != output TAD length ";
|
||||
error += std::to_string(outputTadLength);
|
||||
error += ". Input shape: ";
|
||||
error += ShapeUtils::shapeAsString(input->shapeInfo());
|
||||
error += ", Output shape: ";
|
||||
error += ShapeUtils::shapeAsString(output->shapeInfo());
|
||||
error += ", axis: ";
|
||||
error += std::to_string(axis);
|
||||
THROW_EXCEPTION(error.c_str());
|
||||
}
|
||||
|
||||
// Number of gather operations (number of indices provided as int args)
|
||||
const sd::LongType numGatherOps = numOfIntArgs - 1;
|
||||
|
||||
// Validate bounds before parallel execution
|
||||
if (numGatherOps > tadPackOut->numberOfTads()) {
|
||||
std::string error = "gather: number of indices ";
|
||||
error += std::to_string(numGatherOps);
|
||||
error += " exceeds output TAD count ";
|
||||
error += std::to_string(tadPackOut->numberOfTads());
|
||||
THROW_EXCEPTION(error.c_str());
|
||||
}
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto idx = intArgs[i + 1];
|
||||
|
||||
// Bounds check for input TAD access
|
||||
if (idx >= tadPack->numberOfTads() || idx < 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Bounds check for output TAD access
|
||||
if (i >= tadPackOut->numberOfTads()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto offsetIn = tadOffsets[idx];
|
||||
auto offsetOut = tadOffsetsOut[i];
|
||||
|
||||
NativeOpExecutioner::execTransformAny(input->getContext(),
|
||||
transform::Assign,
|
||||
input->bufferWithOffset(offsetIn), const_cast<sd::LongType*>(tadShapeInfo),
|
||||
nullptr, nullptr,
|
||||
output->bufferWithOffset(offsetOut), const_cast<sd::LongType*>(tadShapeInfoOut),
|
||||
nullptr, nullptr,
|
||||
nullptr, false);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, 0, numGatherOps);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,203 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 20.04.2018
|
||||
//
|
||||
|
||||
#include <helpers/Loops.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
|
||||
#include <numeric>
|
||||
#include <system/selective_rendering.h>
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y>
|
||||
static void gatherND_(NDArray& input, NDArray& indices, NDArray& output) {
|
||||
const X* x = reinterpret_cast<X*>(input.buffer());
|
||||
const Y* y = reinterpret_cast<Y*>(indices.buffer());
|
||||
X* z = reinterpret_cast<X*>(output.buffer());
|
||||
|
||||
const sd::LongType xRank = input.rankOf();
|
||||
const sd::LongType yRank = indices.rankOf();
|
||||
const sd::LongType zRank = output.rankOf();
|
||||
const sd::LongType maxRank = sd::math::sd_max<sd::LongType>(yRank, sd::math::sd_max<sd::LongType>(xRank, zRank));
|
||||
|
||||
const sd::LongType zLen = output.lengthOf();
|
||||
|
||||
const sd::LongType yLastDim = indices.sizeAt(-1);
|
||||
|
||||
const int diff = zRank - xRank;
|
||||
const bool bEqual = yLastDim == xRank;
|
||||
|
||||
sd::LongType outputRank = output.rankOf();
|
||||
sd::LongType* outputShape = shape::shapeOf(output.shapeInfo());
|
||||
sd::LongType* outputStride = shape::stride(output.shapeInfo());
|
||||
sd::LongType indicesRank = indices.rankOf();
|
||||
sd::LongType* indicesShape = shape::shapeOf(indices.shapeInfo());
|
||||
sd::LongType* indicesStride = shape::stride(indices.shapeInfo());
|
||||
|
||||
sd::LongType inputRank = input.rankOf();
|
||||
sd::LongType* inputShape = shape::shapeOf(input.shapeInfo());
|
||||
sd::LongType* inputStride = shape::stride(input.shapeInfo());
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType xCoords[SD_MAX_RANK], zCoords[SD_MAX_RANK], temp;
|
||||
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, outputRank, outputShape, zCoords);
|
||||
|
||||
sd::LongType zOffset;
|
||||
COORDS2INDEX(outputRank, outputStride, zCoords, zOffset);
|
||||
|
||||
temp = zCoords[yRank - 1];
|
||||
zCoords[yRank - 1] = 0;
|
||||
|
||||
sd::LongType yOffset;
|
||||
COORDS2INDEX(indicesRank, indicesStride, zCoords, yOffset);
|
||||
|
||||
zCoords[yRank - 1] = temp;
|
||||
|
||||
if (bEqual)
|
||||
memcpy(xCoords, zCoords, zRank * sizeof(sd::LongType));
|
||||
else if (diff >= 0)
|
||||
memcpy(xCoords, zCoords + diff, xRank * sizeof(sd::LongType));
|
||||
else
|
||||
memcpy(xCoords - diff, zCoords, zRank * sizeof(sd::LongType));
|
||||
|
||||
for (sd::LongType j = 0; j < yLastDim; ++j)
|
||||
xCoords[j] = y[yOffset + j * indicesStride[yRank - 1]]; // last stride
|
||||
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(inputRank, inputStride, xCoords, xOffset);
|
||||
|
||||
z[zOffset] = x[xOffset];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, zLen);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
void gatherND(sd::LaunchContext* context, NDArray& input, NDArray& indices, NDArray& output) {
|
||||
auto inputDType = input.dataType();
|
||||
auto indicesDType = indices.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(input.dataType(), indices.dataType(), gatherND_, (input, indices, output), SD_COMMON_TYPES,
|
||||
SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void gather_(NDArray* input, NDArray* indices, NDArray* output, const std::vector<int>& intArgs) {
|
||||
int axis = intArgs.size() > 0 ? intArgs[0] : 0;
|
||||
const int inputRank = input->rankOf();
|
||||
if (axis < 0) axis += inputRank;
|
||||
|
||||
const int numOfIntArgs = intArgs.size();
|
||||
|
||||
if (indices != nullptr) {
|
||||
for (sd::LongType i = 0; i < indices->lengthOf(); ++i)
|
||||
if (indices->e<sd::LongType>(i) >= input->sizeAt(axis))
|
||||
THROW_EXCEPTION(
|
||||
"helpers::gather function: indices array contains wrong elements, each element must be smaller than "
|
||||
"corresponding dimension of input array !");
|
||||
|
||||
// first case: indices consist of only one scalar
|
||||
if (indices->isScalar()) {
|
||||
if (input->rankOf() <= 1) {
|
||||
// For scalar indices, rank 0 or 1 input: can't do tensor along dimension 0 as this is whole array... instead,
|
||||
// we want to get a scalar
|
||||
auto idx = indices->e<sd::LongType>(0);
|
||||
auto scalarNDArray = input->e(idx);
|
||||
output->assign(&scalarNDArray);
|
||||
} else {
|
||||
// tadForDimensions expects the dimensions to create TADs along,
|
||||
// NOT the dimensions to exclude
|
||||
std::vector<sd::LongType> axesVec = {axis};
|
||||
// Pass the axis directly - TadCalculator will handle the exclusion internally
|
||||
auto tadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(), &axesVec);
|
||||
|
||||
auto tadArr = NDArray(reinterpret_cast<void*>(reinterpret_cast<T*>(input->buffer()) +
|
||||
tadPack->primaryOffsets()[indices->e<sd::LongType>(0)]),
|
||||
tadPack->primaryShapeInfo(), output->getContext(), 0, 0);
|
||||
output->assign(&tadArr);
|
||||
}
|
||||
} else if (input->rankOf() == 1 && indices->isVector()) {
|
||||
// special case
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) output->p(e, input->e<T>(indices->e<sd::LongType>(e)));
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, indices->lengthOf());
|
||||
} else {
|
||||
std::vector<sd::LongType> dimsOut(indices->rankOf());
|
||||
std::iota(dimsOut.begin(), dimsOut.end(), axis); // fill with axis, axis+1, ... indices->rankOf()-1
|
||||
const sd::LongType numOfSubArrs = ShapeUtils::getNumOfSubArrs(output->shapeInfo(), dimsOut);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
NDArray *subArrOut = (*output)(i, dimsOut);
|
||||
NDArray *subArrIn = (*input)(indices->e<sd::LongType>(i), {axis});
|
||||
subArrOut->assign(subArrIn);
|
||||
delete subArrOut;
|
||||
delete subArrIn;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfSubArrs);
|
||||
}
|
||||
} else {
|
||||
for (int i = 1; i < numOfIntArgs; ++i)
|
||||
if (intArgs[i] >= input->sizeAt(axis))
|
||||
THROW_EXCEPTION(
|
||||
"helpers::gather function: some of input indexes is larger than corresponding shape of input array !");
|
||||
|
||||
// we only allow scalar/vector case here
|
||||
if (numOfIntArgs == 2) { // scalar case
|
||||
NDArray *view = (*input)(intArgs[1], {axis});
|
||||
output->assign(view);
|
||||
delete view;
|
||||
} else { // vector case
|
||||
const sd::LongType numOfSubArrs = ShapeUtils::getNumOfSubArrs(output->shapeInfo(), {axis});
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
NDArray *subArrOut = (*output)(i, {axis});
|
||||
NDArray *subArrIn = (*input)(intArgs[i + 1], {axis});
|
||||
subArrOut->assign(subArrIn);
|
||||
delete subArrIn;
|
||||
delete subArrOut;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfSubArrs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void gather(NDArray* input, NDArray* indices, NDArray* output, const std::vector<int>& intArgs) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), gather_, (input, indices, output, intArgs), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,42 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/axis.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static void applyGradientDescent_(NDArray* input, NDArray* step, double weight, NDArray* output) {
|
||||
auto lambda = LAMBDA_TT(_x, _y, weight) { return _x - (_y * weight); });
|
||||
|
||||
input->applyPairwiseLambda<T>(step, lambda, output);
|
||||
}
|
||||
|
||||
void applyGradientDescent(sd::LaunchContext* context, NDArray* input, NDArray* step, double weight, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), applyGradientDescent_, (input, step, weight, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void applyGradientDescent_,
|
||||
(NDArray * input, NDArray* step, double weight, NDArray* output), SD_FLOAT_TYPES);
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,103 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/hamming.h>
|
||||
#include <ops/declarable/helpers/helpers.h>
|
||||
#include <system/selective_rendering.h>
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
static sd::LongType hamming_distance(unsigned long long x, unsigned long long y) {
|
||||
sd::LongType dist = 0;
|
||||
|
||||
for (unsigned long long val = x ^ y; val > 0; val /= 2) {
|
||||
if (val & 1) dist++;
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
static void _hamming(LaunchContext *context, NDArray &x, NDArray &y, NDArray &z) {
|
||||
auto xEws = x.ews();
|
||||
auto yEws = y.ews();
|
||||
|
||||
auto xBuffer = x.bufferAsT<X>();
|
||||
auto yBuffer = y.bufferAsT<X>();
|
||||
|
||||
sd::LongType distance = 0;
|
||||
auto lengthOf = x.lengthOf();
|
||||
int maxThreads = sd::math::sd_min<int>(256, omp_get_max_threads());
|
||||
sd::LongType intermediate[256];
|
||||
|
||||
// nullify temp values
|
||||
for (int e = 0; e < maxThreads; e++) intermediate[e] = 0;
|
||||
|
||||
if (xEws == 1 && yEws == 1 && x.ordering() == y.ordering()) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
auto _x = static_cast<unsigned long long>(xBuffer[e]);
|
||||
auto _y = static_cast<unsigned long long>(yBuffer[e]);
|
||||
|
||||
intermediate[thread_id] += hamming_distance(_x, _y);
|
||||
}
|
||||
};
|
||||
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, lengthOf);
|
||||
} else if (xEws > 1 && yEws > 1 && x.ordering() == y.ordering()) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
auto _x = static_cast<unsigned long long>(xBuffer[e * xEws]);
|
||||
auto _y = static_cast<unsigned long long>(yBuffer[e * yEws]);
|
||||
|
||||
intermediate[thread_id] += hamming_distance(_x, _y);
|
||||
}
|
||||
};
|
||||
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, lengthOf);
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
auto _x = static_cast<unsigned long long>(x.e<sd::LongType>(e));
|
||||
auto _y = static_cast<unsigned long long>(y.e<sd::LongType>(e));
|
||||
|
||||
intermediate[thread_id] += hamming_distance(_x, _y);
|
||||
}
|
||||
};
|
||||
|
||||
maxThreads = samediff::Threads::parallel_for(func, 0, lengthOf);
|
||||
}
|
||||
|
||||
// accumulate intermediate variables into output array
|
||||
for (int e = 0; e < maxThreads; e++) distance += intermediate[e];
|
||||
|
||||
z.p(0, distance);
|
||||
}
|
||||
|
||||
void hamming(LaunchContext *context, NDArray &x, NDArray &y, NDArray &output) {
|
||||
auto xDType = x.dataType();
|
||||
auto outputDType = output.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(x.dataType(), output.dataType(), _hamming, (context, x, y, output), SD_INTEGER_TYPES, SD_INTEGER_TYPES);
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,107 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/hashcode.h>
|
||||
#if NOT_EXCLUDED(OP_hashcode)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static void hashCode_(LaunchContext *context, NDArray &array, NDArray &result) {
|
||||
sd::LongType blockSize = 32;
|
||||
auto length = array.lengthOf();
|
||||
int numBlocks = length / blockSize + ((length % blockSize == 0) ? 0 : 1);
|
||||
auto tempA = NDArrayFactory::create<sd::LongType>('c', {numBlocks}, context);
|
||||
auto tempB = NDArrayFactory::create<sd::LongType>('c', {numBlocks / blockSize + 1}, context);
|
||||
|
||||
auto buffer = array.bufferAsT<T>();
|
||||
auto tempBufferA = tempA->bufferAsT<sd::LongType>();
|
||||
auto tempBufferB = tempB->bufferAsT<sd::LongType>();
|
||||
|
||||
// default buffer is the first one, because it might be the last one in case of small arrays (< blockSize)
|
||||
auto tempBuffer = tempBufferA;
|
||||
auto tempResult = tempBufferB;
|
||||
|
||||
// we divide array into 32 element chunks, and store intermediate results once
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto b = start; b < stop; b++) {
|
||||
auto blockBuffer = buffer + b * numBlocks;
|
||||
|
||||
sd::LongType r = 1;
|
||||
for (sd::LongType e = 0; e < blockSize && e + (b * numBlocks) < length; e++) {
|
||||
auto v = longBytes<T>(blockBuffer[e]);
|
||||
r = 31 * r + v;
|
||||
}
|
||||
|
||||
tempBuffer[b] = r;
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, 0, numBlocks);
|
||||
|
||||
// we replace pointer with intermediate one, and repeat only one chunk left
|
||||
int iterationCount = 0;
|
||||
while (numBlocks > 1) {
|
||||
int lastLength = numBlocks;
|
||||
numBlocks = lastLength / blockSize + ((lastLength % blockSize == 0) ? 0 : 1);
|
||||
|
||||
auto func2 = PRAGMA_THREADS_FOR {
|
||||
for (auto b = start; b < stop; b++) {
|
||||
auto blockBuffer = tempBuffer + b * numBlocks;
|
||||
|
||||
sd::LongType r = 1;
|
||||
for (sd::LongType e = 0; e < blockSize && e + (b * numBlocks) < lastLength; e++) {
|
||||
auto v = longBytes<T>(static_cast<T>(blockBuffer[e]));
|
||||
r = 31 * r + v;
|
||||
}
|
||||
|
||||
tempResult[b] = r;
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func2, 0, numBlocks);
|
||||
|
||||
iterationCount++;
|
||||
// swapping buffers
|
||||
if (iterationCount % 2 == 0) {
|
||||
tempBuffer = tempBufferA;
|
||||
tempResult = tempBufferB;
|
||||
} else {
|
||||
tempBuffer = tempBufferB;
|
||||
tempResult = tempBufferA;
|
||||
}
|
||||
}
|
||||
|
||||
if (length <= blockSize)
|
||||
result.p(0, tempBufferA[0]);
|
||||
else
|
||||
result.p(0, tempResult[0]);
|
||||
|
||||
delete tempA;
|
||||
delete tempB;
|
||||
}
|
||||
|
||||
void hashCode(LaunchContext *context, NDArray &array, NDArray &result) {
|
||||
BUILD_SINGLE_SELECTOR(array.dataType(), hashCode_, (context, array, result), SD_COMMON_TYPES);
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,82 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/histogram.h>
|
||||
#include <system/selective_rendering.h>
|
||||
#if NOT_EXCLUDED(OP_histogram)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename X, typename Z>
|
||||
static void histogram_(void const *xBuffer, sd::LongType const *xShapeInfo, void *zBuffer,
|
||||
sd::LongType const *zShapeInfo, sd::LongType numBins, double min_val, double max_val) {
|
||||
auto dx = reinterpret_cast<X const *>(xBuffer);
|
||||
auto result = reinterpret_cast<Z *>(zBuffer);
|
||||
|
||||
int length = shape::length(xShapeInfo);
|
||||
|
||||
X binSize = static_cast<X>((max_val - min_val) / (numBins));
|
||||
|
||||
// FIXME: this op should be parallelized
|
||||
{
|
||||
int *bins = new int[numBins];
|
||||
std::memset(bins, 0, sizeof(int) * numBins);
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (int x = 0; x < length; x++) {
|
||||
int idx = (int)((dx[x] - min_val) / binSize);
|
||||
if (idx < 0)
|
||||
idx = 0;
|
||||
else if (idx >= numBins)
|
||||
idx = numBins - 1;
|
||||
|
||||
bins[idx]++;
|
||||
}
|
||||
|
||||
PRAGMA_OMP_SIMD
|
||||
for (sd::LongType x = 0; x < numBins; x++) {
|
||||
result[x] += bins[x];
|
||||
}
|
||||
|
||||
delete[] bins;
|
||||
}
|
||||
}
|
||||
|
||||
void histogramHelper(sd::LaunchContext *context, NDArray &input, NDArray &output) {
|
||||
sd::LongType numBins = output.lengthOf();
|
||||
auto min = input.reduceNumber(reduce::SameOps::Min);
|
||||
auto max = input.reduceNumber(reduce::SameOps::Max);
|
||||
double min_val = min->e<double>(0);
|
||||
double max_val = max->e<double>(0);
|
||||
auto inputDType = input.dataType();
|
||||
auto outputDType = output.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(
|
||||
input.dataType(), output.dataType(), histogram_,
|
||||
(input.buffer(), input.shapeInfo(), output.buffer(), output.shapeInfo(), numBins, min_val, max_val),
|
||||
SD_COMMON_TYPES, SD_INDEXING_TYPES);
|
||||
|
||||
delete min;
|
||||
delete max;
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,68 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 31.08.2018
|
||||
//
|
||||
#include <ops/declarable/helpers/histogramFixedWidth.h>
|
||||
#if NOT_EXCLUDED(OP_histogram_fixed_width)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
void histogramFixedWidth_(NDArray& input, NDArray& range, NDArray& output) {
|
||||
const int nbins = output.lengthOf();
|
||||
|
||||
// firstly initialize output with zeros
|
||||
output.nullify();
|
||||
|
||||
const T leftEdge = static_cast<T>(range.e<double>(0));
|
||||
const T rightEdge = static_cast<T>(range.e<double>(1));
|
||||
|
||||
const T binWidth = (rightEdge - leftEdge) / nbins;
|
||||
const T secondEdge = leftEdge + binWidth;
|
||||
const T lastButOneEdge = rightEdge - binWidth;
|
||||
|
||||
sd::LongType inputLength = input.lengthOf();
|
||||
|
||||
// FIXME: make this one parallel without CRITICAL section
|
||||
for (sd::LongType i = 0; i < inputLength; ++i) {
|
||||
const T value = input.e<T>(i);
|
||||
|
||||
if (value < secondEdge) {
|
||||
output.p<sd::LongType>(0, output.e<sd::LongType>(0) + 1);
|
||||
} else if (value >= lastButOneEdge) {
|
||||
output.p<sd::LongType>(nbins - 1, output.e<sd::LongType>(nbins - 1) + 1);
|
||||
} else {
|
||||
sd::LongType currInd = static_cast<sd::LongType>((value - leftEdge) / binWidth);
|
||||
output.p<sd::LongType>(currInd, output.e<sd::LongType>(currInd) + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void histogramFixedWidth(sd::LaunchContext* context, NDArray& input, NDArray& range, NDArray& output) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), histogramFixedWidth_, (input, range, output), SD_COMMON_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void histogramFixedWidth_, (NDArray& input, NDArray& range, NDArray& output),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,124 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 19.09.2018
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/im2col.h>
|
||||
#if NOT_EXCLUDED(OP_im2col)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void im2col_(sd::LaunchContext& context, NDArray& input, NDArray& output, const LongType kH,
|
||||
const LongType kW, const LongType sH, const LongType sW, const LongType pH, const LongType pW,
|
||||
const LongType dH, const LongType dW, NDArray& arrZeroPadVal) {
|
||||
// input [bS, iC, iH, iW] is convoluted to output [bS, iC, kH, kW, oH, oW]
|
||||
if(input.rankOf() != 4) {
|
||||
THROW_EXCEPTION("ops::helpers::im2col: input array must have rank = 4");
|
||||
}
|
||||
|
||||
if(output.rankOf() != 6) {
|
||||
THROW_EXCEPTION("ops::helpers::im2col: output array must have rank = 6");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
auto imBuff = static_cast<T const*>(input.buffer());
|
||||
auto colBuff = static_cast<T*>(output.buffer());
|
||||
auto imShapeBuffer = input.shapeInfo();
|
||||
auto colShapeBuffer = output.shapeInfo();
|
||||
auto colShape = shape::shapeOf(colShapeBuffer);
|
||||
auto colStride = shape::stride(colShapeBuffer);
|
||||
auto imShape = shape::shapeOf(imShapeBuffer);
|
||||
auto imStride = shape::stride(imShapeBuffer);
|
||||
|
||||
const T zeroPadVal = arrZeroPadVal.e<T>(0);
|
||||
|
||||
const LongType bS = imShape[0];
|
||||
const LongType iC = imShape[1];
|
||||
const LongType iH = imShape[2];
|
||||
const LongType iW = imShape[3];
|
||||
const LongType oH = colShape[4];
|
||||
const LongType oW = colShape[5];
|
||||
const sd::LongType colStride0 = colStride[0];
|
||||
const sd::LongType colStride1 = colStride[1];
|
||||
const sd::LongType colStride2 = colStride[2];
|
||||
const sd::LongType colStride3 = colStride[3];
|
||||
const sd::LongType colStride4 = colStride[4];
|
||||
const sd::LongType colStride5 = colStride[5];
|
||||
const sd::LongType imStride0 = imStride[0];
|
||||
const sd::LongType imStride1 = imStride[1];
|
||||
const sd::LongType imStride2 = imStride[2];
|
||||
const sd::LongType imStride3 = imStride[3];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
sd::LongType imRow, imCol, colIndex, imIndex;
|
||||
|
||||
for (auto b = start_x; b < stop_x; b += inc_x) {
|
||||
for (auto colH = start_y; colH < stop_y; colH += inc_y) {
|
||||
for (sd::LongType colW = 0; colW < oW; ++colW) {
|
||||
for (sd::LongType c = 0; c < iC; ++c) {
|
||||
for (sd::LongType kRow = 0; kRow < kH; ++kRow) {
|
||||
for (sd::LongType kCol = 0; kCol < kW; ++kCol) {
|
||||
imRow = (-pH + kRow * dH) + colH * sH;
|
||||
imCol = (-pW + kCol * dW) + colW * sW;
|
||||
|
||||
colIndex = b * colStride0 + c * colStride1 + kRow * colStride2 + kCol * colStride3 +
|
||||
colH * colStride4 + colW * colStride5;
|
||||
|
||||
if (static_cast<LongType>(imRow) >= static_cast<LongType>(iH) ||
|
||||
static_cast<LongType>(imRow) < 0 ||
|
||||
static_cast<LongType>(imCol) >= static_cast<LongType>(iW) ||
|
||||
static_cast<LongType>(imCol) < 0) {
|
||||
if (colIndex < output.lengthOf()) {
|
||||
colBuff[colIndex] = zeroPadVal;
|
||||
}
|
||||
} else {
|
||||
imIndex = b * imStride0 + c * imStride1 + imRow * imStride2 + imCol * imStride3;
|
||||
if (colIndex < output.lengthOf() && imIndex < input.lengthOf()) {
|
||||
colBuff[colIndex] = static_cast<T>(imBuff[imIndex]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, oH, 1);
|
||||
}
|
||||
|
||||
void im2col(sd::LaunchContext& context, NDArray& im, NDArray& col, const LongType kH, const LongType kW,
|
||||
const LongType sH, const LongType sW, const LongType pH, const LongType pW, const LongType dH,
|
||||
const LongType dW, NDArray& arrZeroPadVal) {
|
||||
|
||||
BUILD_SINGLE_SELECTOR(im.dataType(), im2col_, (context, im, col, kH, kW, sH, sW, pH, pW, dH, dW, arrZeroPadVal),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,158 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <array/NDArray.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
typedef std::vector<std::vector<float>> ColorTable_t;
|
||||
static ColorTable_t DefaultColorTable(int depth) {
|
||||
std::vector<std::vector<float>> colorTable;
|
||||
colorTable.emplace_back(std::vector<float>({1, 1, 0, 1})); // 0: yellow
|
||||
colorTable.emplace_back(std::vector<float>({0, 0, 1, 1})); // 1: blue
|
||||
colorTable.emplace_back(std::vector<float>({1, 0, 0, 1})); // 2: red
|
||||
colorTable.emplace_back(std::vector<float>({0, 1, 0, 1})); // 3: lime
|
||||
colorTable.emplace_back(std::vector<float>({0.5, 0, 0.5, 1})); // 4: purple
|
||||
colorTable.emplace_back(std::vector<float>({0.5, 0.5, 0, 1})); // 5: olive
|
||||
colorTable.emplace_back(std::vector<float>({0.5, 0, 0, 1})); // 6: maroon
|
||||
colorTable.emplace_back(std::vector<float>({0, 0, 0.5, 1})); // 7: navy blue
|
||||
colorTable.emplace_back(std::vector<float>({0, 1, 1, 1})); // 8: aqua
|
||||
colorTable.emplace_back(std::vector<float>({1, 0, 1, 1})); // 9: fuchsia
|
||||
|
||||
if (depth == 1) {
|
||||
for (size_t i = 0; i < colorTable.size(); i++) {
|
||||
colorTable[i][0] = 1;
|
||||
}
|
||||
}
|
||||
return colorTable;
|
||||
}
|
||||
|
||||
void drawBoundingBoxesFunctor(sd::LaunchContext* context, NDArray* images, NDArray* boxes, NDArray* colors,
|
||||
NDArray* output) {
|
||||
// images - batch of 3D images with BW (last dim = 1), RGB (last dim = 3) or RGBA (last dim = 4) channel set
|
||||
// boxes - batch of 2D bounds with last dim (y_start, x_start, y_end, x_end) to compute i and j as
|
||||
// floor((height - 1 ) * y_start) => rowStart, floor((height - 1) * y_end) => rowEnd
|
||||
// floor((width - 1 ) * x_start) => colStart, floor((width - 1) * x_end) => colEnd
|
||||
// height = images->sizeAt(1), width = images->sizeAt(2)
|
||||
// colors - colors for each box given
|
||||
// set up color for each box as frame
|
||||
auto batchSize = images->sizeAt(0);
|
||||
auto boxSize = boxes->sizeAt(0);
|
||||
auto height = images->sizeAt(1);
|
||||
auto width = images->sizeAt(2);
|
||||
auto channels = images->sizeAt(3);
|
||||
|
||||
output->assign(images); // fill up all output with input images, then fill up boxes
|
||||
ColorTable_t colorTable;
|
||||
if (colors) {
|
||||
for (auto i = 0; i < colors->sizeAt(0); i++) {
|
||||
std::vector<float> colorValue(4);
|
||||
for (auto j = 0; j < 4; j++) {
|
||||
colorValue[j] = j < colors->sizeAt(1) ? colors->e<float>(i, j) : 1.f;
|
||||
}
|
||||
colorTable.emplace_back(colorValue);
|
||||
}
|
||||
}
|
||||
if (colorTable.empty()) colorTable = DefaultColorTable(channels);
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto batch = start; batch < stop; ++batch) { // loop by batch
|
||||
const sd::LongType numBoxes = boxes->sizeAt(1);
|
||||
for (auto boxIndex = 0; boxIndex < numBoxes; ++boxIndex) {
|
||||
auto colorIndex = boxIndex % colorTable.size();
|
||||
auto rowStart = sd::LongType((height - 1) * boxes->t<float>(batch, boxIndex, 0));
|
||||
auto rowStartBound = sd::math::sd_max(sd::LongType(0), rowStart);
|
||||
auto rowEnd = sd::LongType((height - 1) * boxes->t<float>(batch, boxIndex, 2));
|
||||
auto rowEndBound = sd::math::sd_min(sd::LongType(height - 1), rowEnd);
|
||||
auto colStart = sd::LongType((width - 1) * boxes->t<float>(batch, boxIndex, 1));
|
||||
auto colStartBound = sd::math::sd_max(sd::LongType(0), colStart);
|
||||
auto colEnd = sd::LongType((width - 1) * boxes->t<float>(batch, boxIndex, 3));
|
||||
auto colEndBound = sd::math::sd_min(sd::LongType(width - 1), colEnd);
|
||||
|
||||
if (rowStart > rowEnd || colStart > colEnd) {
|
||||
sd_debug(
|
||||
"helpers::drawBoundingBoxesFunctor: Bounding box (%lld, %lld, %lld, %lld) is inverted "
|
||||
"and will not be drawn\n",
|
||||
rowStart, colStart, rowEnd, colEnd);
|
||||
continue;
|
||||
}
|
||||
if (rowStart >= height || rowEnd < 0 || colStart >= width || colEnd < 0) {
|
||||
sd_debug(
|
||||
"helpers::drawBoundingBoxesFunctor: Bounding box (%lld, %lld, %lld, %lld) is completely "
|
||||
"outside the image and not be drawn\n ",
|
||||
rowStart, colStart, rowEnd, colEnd);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Draw upper line
|
||||
if (rowStart >= 0) {
|
||||
for (auto j = colStartBound; j <= colEndBound; ++j)
|
||||
for (auto c = 0; c < channels; c++) {
|
||||
output->p(batch, rowStart, j, c, colorTable[colorIndex][c]);
|
||||
}
|
||||
}
|
||||
// Draw bottom line.
|
||||
if (rowEnd < height) {
|
||||
for (auto j = colStartBound; j <= colEndBound; ++j)
|
||||
for (auto c = 0; c < channels; c++) {
|
||||
output->p(batch, rowEnd, j, c, colorTable[colorIndex][c]);
|
||||
}
|
||||
}
|
||||
|
||||
// Draw left line.
|
||||
if (colStart >= 0) {
|
||||
for (auto i = rowStartBound; i <= rowEndBound; ++i)
|
||||
for (auto c = 0; c < channels; c++) {
|
||||
output->p(batch, i, colStart, c, colorTable[colorIndex][c]);
|
||||
}
|
||||
}
|
||||
// Draw right line.
|
||||
if (colEnd < width) {
|
||||
for (auto i = rowStartBound; i <= rowEndBound; ++i)
|
||||
for (auto c = 0; c < channels; c++) {
|
||||
output->p(batch, i, colEnd, c, colorTable[colorIndex][c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, 0, batchSize);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,926 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/headers/parity_ops.h>
|
||||
#include <ops/declarable/helpers/image_resize.h>
|
||||
|
||||
#include "../cross.h"
|
||||
#include <system/selective_rendering.h>
|
||||
#if NOT_EXCLUDED(OP_image_resize)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <class Scaler>
|
||||
static inline void computeInterpolationWeights(const Scaler scaler, sd::LongType outSize, sd::LongType inSize,
|
||||
double scale, BilinearInterpolationData* interpolationData) {
|
||||
interpolationData[outSize].bottomIndex = 0;
|
||||
interpolationData[outSize].topIndex = 0;
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto k = start; k < stop; k++) {
|
||||
auto i = (outSize - k - 1);
|
||||
double const in = scaler(i, scale);
|
||||
double const in_f = sd::math::sd_floor<double, double>(in);
|
||||
double const in_c = sd::math::sd_ceil<double, double>(in);
|
||||
interpolationData[i].bottomIndex =
|
||||
sd::math::sd_max(static_cast<sd::LongType>(in_f), (sd::LongType)0LL); // static_cast<sd::LongType>(in);
|
||||
interpolationData[i].topIndex = sd::math::sd_min(static_cast<sd::LongType>(in_c), inSize - 1);
|
||||
interpolationData[i].interpolarValue = in - in_f;
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, outSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the bilinear interpolation from the appropriate 4 float points
|
||||
* and the linear interpolation weights.
|
||||
*/
|
||||
// static void
|
||||
// resizeImage(NDArray *images, sd::LongType batchSize, sd::LongType inHeight, sd::LongType inWidth,
|
||||
// sd::LongType outHeight,
|
||||
// sd::LongType outWidth, sd::LongType channels,
|
||||
// std::vector<BilinearInterpolationData> const& xs,
|
||||
// std::vector<BilinearInterpolationData> const& ys,
|
||||
// NDArray *output);
|
||||
|
||||
template <typename T, typename Z>
|
||||
static void resizeImage_(T const* pInputBuf, sd::LongType batchSize, sd::LongType inHeight, sd::LongType inWidth,
|
||||
sd::LongType outHeight, sd::LongType outWidth, sd::LongType channels,
|
||||
std::vector<BilinearInterpolationData> const& xs,
|
||||
std::vector<BilinearInterpolationData> const& ys, Z* pOutputBuf) {
|
||||
sd::LongType inRowSize = inWidth * channels;
|
||||
sd::LongType inBatchNumValues = inHeight * inRowSize;
|
||||
sd::LongType outRowSize = outWidth * channels;
|
||||
|
||||
BilinearInterpolationData const* xsPtr = xs.data();
|
||||
|
||||
auto computeBilinear = [](double topLeft, double topRight, double bottomLeft, double bottomRight, double xVal,
|
||||
double yVal) {
|
||||
double top = topLeft + (topRight - topLeft) * xVal;
|
||||
double bottom = bottomLeft + (bottomRight - bottomLeft) * xVal;
|
||||
return top + (bottom - top) * yVal;
|
||||
};
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto batch = start; batch < stop; ++batch) {
|
||||
auto pInput = pInputBuf + batch * inBatchNumValues;
|
||||
for (sd::LongType y = 0; y < outHeight; ++y) {
|
||||
auto pOutput = pOutputBuf + (batch * outHeight + y) * outRowSize;
|
||||
const T* ysInputLowerPtr = pInput + ys[y].bottomIndex * inRowSize;
|
||||
const T* ysInputUpperPtr = pInput + ys[y].topIndex * inRowSize;
|
||||
double yVal = ys[y].interpolarValue;
|
||||
for (sd::LongType x = 0; x < outWidth; ++x) {
|
||||
auto xsBottom = xsPtr[x].bottomIndex;
|
||||
auto xsTop = xsPtr[x].topIndex;
|
||||
auto xVal = xsPtr[x].interpolarValue;
|
||||
for (sd::LongType c = 0; c < channels; ++c) {
|
||||
double topLeft(ysInputLowerPtr[xsBottom + c]);
|
||||
double topRight(ysInputLowerPtr[xsTop + c]);
|
||||
double bottomLeft(ysInputUpperPtr[xsBottom + c]);
|
||||
double bottomRight(ysInputUpperPtr[xsTop + c]);
|
||||
pOutput[x * channels + c] = computeBilinear(topLeft, topRight, bottomLeft, bottomRight, xVal, yVal);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, 0, batchSize);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
static sd::Status resizeBilinearFunctor_(NDArray * images, int const width, int const height,
|
||||
bool const alignCorners, bool const halfPixelCenter, NDArray* output) {
|
||||
ImageResizerState st(alignCorners, halfPixelCenter);
|
||||
st.validateAndCalculateOutputSize(images, width, height);
|
||||
|
||||
const sd::LongType batchSize = images->sizeAt(0);
|
||||
const sd::LongType inHeight = images->sizeAt(1);
|
||||
const sd::LongType inWidth = images->sizeAt(2);
|
||||
const sd::LongType channels = images->sizeAt(3);
|
||||
|
||||
const sd::LongType outHeight = output->sizeAt(1);
|
||||
const sd::LongType outWidth = output->sizeAt(2);
|
||||
|
||||
// Handle no-op resizes efficiently.
|
||||
if (outHeight == inHeight && outWidth == inWidth) {
|
||||
output->assign(images);
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
std::vector<BilinearInterpolationData> ys(outHeight + 1);
|
||||
std::vector<BilinearInterpolationData> xs(outWidth + 1);
|
||||
if (halfPixelCenter) {
|
||||
computeInterpolationWeights(HalfPixelScaler(), outHeight, inHeight, st.heightScale, ys.data());
|
||||
computeInterpolationWeights(HalfPixelScaler(), outWidth, inWidth, st.widthScale, xs.data());
|
||||
|
||||
} else {
|
||||
// Compute the cached interpolation weights on the x and y dimensions.
|
||||
computeInterpolationWeights(LegacyScaler(), outHeight, inHeight, st.heightScale, ys.data());
|
||||
computeInterpolationWeights(LegacyScaler(), outWidth, inWidth, st.widthScale, xs.data());
|
||||
}
|
||||
int xsSize = xs.size();
|
||||
// Scale x interpolation weights to avoid a multiplication during iteration.
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
xs[i].bottomIndex *= channels;
|
||||
xs[i].topIndex *= channels;
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, xsSize);
|
||||
|
||||
resizeImage_<X, Z>(images->getDataBuffer()->primaryAsT<X>(), batchSize, inHeight, inWidth, outHeight, outWidth,
|
||||
channels, xs, ys, output->dataBuffer()->primaryAsT<Z>());
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
template <class Scaler, typename T>
|
||||
void resizeNeighborImpl(ImageResizerState const& st, NDArray * images, NearestMode nearestMode, NDArray* output) {
|
||||
const sd::LongType batchSize = st.batchSize;
|
||||
const sd::LongType inHeight = st.inHeight;
|
||||
const sd::LongType inWidth = st.inWidth;
|
||||
const sd::LongType channels = st.channels;
|
||||
|
||||
const sd::LongType outHeight = st.outHeight;
|
||||
const sd::LongType outWidth = st.outWidth;
|
||||
Scaler scaler;
|
||||
constexpr bool halfPixelCenter =
|
||||
std::is_same<Scaler, HalfPixelScaler>::value || std::is_same<Scaler, HalfPixelScalerNN>::value;
|
||||
|
||||
float (*modeFunc)(float);
|
||||
switch (nearestMode) {
|
||||
case NearestMode::FLOOR:
|
||||
modeFunc = [](float x){return sd::math::p_floor<float>(x);};
|
||||
break;
|
||||
case NearestMode::ROUND_PREFER_FLOOR:
|
||||
modeFunc = [](float x){return sd::math::p_round_prefer_floor<float>(x);};
|
||||
break;
|
||||
case NearestMode::ROUND_PREFER_CEIL:
|
||||
modeFunc = [](float x){return sd::math::p_round_prefer_ceil<float>(x);};
|
||||
break;
|
||||
case NearestMode::CEIL:
|
||||
modeFunc = [](float x){return sd::math::p_ceil<float>(x);};
|
||||
break;
|
||||
default:
|
||||
modeFunc = [](float x){return sd::math::p_floor<float>(x);};
|
||||
}
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
for (auto b = start_x; b < stop_x; b += inc_x) {
|
||||
for (auto y = start_y; y < stop_y; y += inc_y) {
|
||||
auto posY = static_cast<sd::LongType>(modeFunc(scaler(y, st.heightScale)));
|
||||
sd::LongType inY = sd::math::sd_min(posY, inHeight - 1);
|
||||
if (halfPixelCenter) {
|
||||
inY = sd::math::sd_max(0LL, inY);
|
||||
}
|
||||
for (sd::LongType x = 0; x < outWidth; ++x) {
|
||||
auto posX = static_cast<sd::LongType>(modeFunc(scaler(x, st.widthScale)));
|
||||
sd::LongType inX = sd::math::sd_min(posX, inWidth - 1);
|
||||
if (halfPixelCenter) {
|
||||
inX = sd::math::sd_max(0LL, inX);
|
||||
}
|
||||
// copy pixel over all channels
|
||||
for (sd::LongType e = 0; e < channels; e++) output->r<T>(b, y, x, e) = images->t<T>(b, inY, inX, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, batchSize, 1, 0, outHeight, 1);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
sd::Status resizeNeighborFunctor_(NDArray * images, int const width, int const height,
|
||||
CoordinateTransformationMode coorMode, NearestMode nearestMode, bool alignCorner,
|
||||
NDArray* output) {
|
||||
ImageResizerState st(alignCorner, (coorMode == HALF_PIXEL_NN));
|
||||
st.validateAndCalculateOutputSize(images, width, height);
|
||||
|
||||
// Handle no-op resizes efficiently.
|
||||
if (output->sizeAt(1) == images->sizeAt(1) && output->sizeAt(2) == images->sizeAt(2)) {
|
||||
output->assign(images);
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
switch (coorMode) {
|
||||
case ASYMMETRIC:
|
||||
resizeNeighborImpl<LegacyScaler, T>(st, images, nearestMode, output);
|
||||
break;
|
||||
case HALF_PIXEL:
|
||||
resizeNeighborImpl<HalfPixelScaler, T>(st, images, nearestMode, output);
|
||||
break;
|
||||
case HALF_PIXEL_NN:
|
||||
resizeNeighborImpl<HalfPixelScalerNN, T>(st, images, nearestMode, output);
|
||||
break;
|
||||
default:
|
||||
resizeNeighborImpl<HalfPixelScaler, T>(st, images, nearestMode, output);
|
||||
break;
|
||||
};
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
|
||||
|
||||
sd::Status resizeBilinearFunctor(sd::LaunchContext* context, NDArray * images, int const width, int const height,
|
||||
bool const alignCorners, bool const halfPixelCenter, NDArray* output) {
|
||||
|
||||
auto imagesDtype = images->dataType();
|
||||
auto outputDType = output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(images->dataType(), output->dataType(), return resizeBilinearFunctor_,
|
||||
(images, width, height, alignCorners, halfPixelCenter, output), SD_NUMERIC_TYPES,
|
||||
SD_FLOAT_TYPES);
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status resizeNeighborFunctor(sd::LaunchContext* context, NDArray * images, int const width, int const height,
|
||||
CoordinateTransformationMode coorMode, NearestMode nearestMode, bool alignCorner,
|
||||
NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(images->dataType(), return resizeNeighborFunctor_,
|
||||
(images, width, height, coorMode, nearestMode, alignCorner, output), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// ------------------------------------------------------------------------------------------------------------------ //
|
||||
// Bicubic interpolation
|
||||
// ------------------------------------------------------------------------------------------------------------------ //
|
||||
|
||||
template <typename T>
|
||||
std::unique_ptr<T[]> initCoeffsTable(const double a) {
|
||||
// Allocate and initialize coefficients table using Bicubic
|
||||
// convolution algorithm.
|
||||
// https://en.wikipedia.org/wiki/Bicubic_interpolation
|
||||
KeysCubicKernelFunc<T> kernel(static_cast<T>(a));
|
||||
std::unique_ptr<T[]> coeffsTableUniq(new T[(kTableSize + 1) * 2]);
|
||||
T* coeffsTable = coeffsTableUniq.get();
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i <= stop; ++i) {
|
||||
float x = i * 1.0 / kTableSize;
|
||||
coeffsTable[i * 2] = kernel.calc_less1pt0(x);
|
||||
x += 1.0;
|
||||
coeffsTable[i * 2 + 1] = kernel.calc_less2pt0(x);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, kTableSize);
|
||||
return coeffsTableUniq;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
sd::Status resizeBicubicFunctor_(sd::LaunchContext* context, NDArray * image, int width, int height,
|
||||
bool preserveAspectRatio, bool antialias, NDArray* output) {
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status resizeBicubicFunctor(sd::LaunchContext* context, NDArray * image, int width, int height,
|
||||
bool preserveAspectRatio, bool antialias, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(image->dataType(), return resizeBicubicFunctor_,
|
||||
(context, image, width, height, preserveAspectRatio, antialias, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
// ------------------------------------------------------------------------------------------------------------------ //
|
||||
|
||||
template <typename Scaler>
|
||||
static void computeXWeightsAndIndices(const ImageResizerState& resizer_state, const float* coeffs_table,
|
||||
std::vector<WeightsAndIndices>* x_wais, bool exclude_outside) {
|
||||
CachedInterpolationCalculator calc;
|
||||
for (auto x = 0; x < resizer_state.outWidth; ++x) {
|
||||
WeightsAndIndices& x_wai = (*x_wais)[x];
|
||||
;
|
||||
getWeightsAndIndices<Scaler>(coeffs_table, resizer_state.widthScale, x, resizer_state.inWidth, &x_wai,
|
||||
exclude_outside);
|
||||
x_wai._advance = calc.Advance(x_wai._index0, x_wai._index1, x_wai._index2, x_wai._index3);
|
||||
(*x_wais)[x]._index0 *= resizer_state.wStride;
|
||||
(*x_wais)[x]._index1 *= resizer_state.wStride;
|
||||
(*x_wais)[x]._index2 *= resizer_state.wStride;
|
||||
(*x_wais)[x]._index3 *= resizer_state.wStride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename F, typename Scaler>
|
||||
static void bicubicInterpolateWithCaching(NDArray * image, ImageResizerState const& resizerState,
|
||||
const double coefficient, bool exclude_outside, NDArray* output) {
|
||||
std::vector<WeightsAndIndices> xWais(resizerState.outWidth);
|
||||
auto coeffs_table_uniq = initCoeffsTable<float>(coefficient);
|
||||
float* coeffs_table = coeffs_table_uniq.get();
|
||||
|
||||
computeXWeightsAndIndices<Scaler>(resizerState, coeffs_table, &xWais, exclude_outside);
|
||||
|
||||
const auto numChannels = resizerState.channels;
|
||||
const auto batchNum = resizerState.batchSize;
|
||||
const auto outHeight = resizerState.outHeight;
|
||||
const auto outWidth = resizerState.outWidth;
|
||||
const auto batchStride = image->strideAt(0);
|
||||
const auto hStride = image->strideAt(1);
|
||||
const auto cStride = image->strideAt(3);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
const T* inputPtr = image->getDataBuffer()->primaryAsT<T>();
|
||||
F* pOutputY = output->dataBuffer()->primaryAsT<F>(); // output is float anyway
|
||||
std::vector<float> cachedValue(numChannels == 3 ? 0 : 4 * numChannels, 0);
|
||||
for (auto b = start; b < stop; ++b) {
|
||||
auto pInput = inputPtr + b * batchStride;
|
||||
for (sd::LongType y = 0; y < outHeight; ++y) {
|
||||
auto pOutput = &pOutputY[(b * outHeight + y) * outWidth * numChannels];
|
||||
|
||||
WeightsAndIndices yWai;
|
||||
getWeightsAndIndices<Scaler>(coeffs_table, resizerState.heightScale, y, resizerState.inHeight, &yWai,
|
||||
exclude_outside);
|
||||
// Make pointers represent offsets of data in inputBPtr.
|
||||
const T* y_ptr_0 = pInput + yWai._index0 * hStride;
|
||||
const T* y_ptr_1 = pInput + yWai._index1 * hStride;
|
||||
const T* y_ptr_2 = pInput + yWai._index2 * hStride;
|
||||
const T* y_ptr_3 = pInput + yWai._index3 * hStride;
|
||||
|
||||
if (numChannels == 3) {
|
||||
// Manually unroll case of 3 channels.
|
||||
F cached_value_0[4] = {0};
|
||||
F cached_value_1[4] = {0};
|
||||
F cached_value_2[4] = {0};
|
||||
for (sd::LongType x = 0; x < resizerState.outWidth; ++x) {
|
||||
const WeightsAndIndices& xWai = xWais[x];
|
||||
// Shift values in cached_value_* to fill first '_advance' values.
|
||||
switch (xWai._advance) {
|
||||
case 3:
|
||||
cached_value_0[0] = cached_value_0[1];
|
||||
cached_value_0[1] = cached_value_0[2];
|
||||
cached_value_0[2] = cached_value_0[3];
|
||||
cached_value_1[0] = cached_value_1[1];
|
||||
cached_value_1[1] = cached_value_1[2];
|
||||
cached_value_1[2] = cached_value_1[3];
|
||||
cached_value_2[0] = cached_value_2[1];
|
||||
cached_value_2[1] = cached_value_2[2];
|
||||
cached_value_2[2] = cached_value_2[3];
|
||||
break;
|
||||
case 2:
|
||||
cached_value_0[0] = cached_value_0[2];
|
||||
cached_value_0[1] = cached_value_0[3];
|
||||
cached_value_1[0] = cached_value_1[2];
|
||||
cached_value_1[1] = cached_value_1[3];
|
||||
cached_value_2[0] = cached_value_2[2];
|
||||
cached_value_2[1] = cached_value_2[3];
|
||||
break;
|
||||
case 1: {
|
||||
cached_value_0[0] = cached_value_0[3];
|
||||
cached_value_1[0] = cached_value_1[3];
|
||||
cached_value_2[0] = cached_value_2[3];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Set the remaining '4-_advance' values by computing.
|
||||
switch (xWai._advance) {
|
||||
case 0:
|
||||
cached_value_0[0] = computeYInterpolation(0, 0, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
cached_value_1[0] = computeYInterpolation(0, cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
cached_value_2[0] =
|
||||
computeYInterpolation(0, 2 * cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
|
||||
case 1:
|
||||
cached_value_0[1] = computeYInterpolation(1, 0, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
cached_value_1[1] = computeYInterpolation(1, cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
cached_value_2[1] =
|
||||
computeYInterpolation(1, 2 * cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
|
||||
case 2:
|
||||
cached_value_0[2] = computeYInterpolation(2, 0, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
cached_value_1[2] = computeYInterpolation(2, cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
cached_value_2[2] =
|
||||
computeYInterpolation(2, 2 * cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
|
||||
case 3:
|
||||
cached_value_0[3] = computeYInterpolation(3, 0, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
cached_value_1[3] = computeYInterpolation(3, cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
cached_value_2[3] =
|
||||
computeYInterpolation(3, 2 * cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
break;
|
||||
}
|
||||
pOutput[x * numChannels + 0] =
|
||||
compute(cached_value_0, xWai._weight0, xWai._weight1, xWai._weight2, xWai._weight3);
|
||||
pOutput[x * numChannels + 1] =
|
||||
compute(cached_value_1, xWai._weight0, xWai._weight1, xWai._weight2, xWai._weight3);
|
||||
pOutput[x * numChannels + 2] =
|
||||
compute(cached_value_2, xWai._weight0, xWai._weight1, xWai._weight2, xWai._weight3);
|
||||
}
|
||||
} else {
|
||||
for (sd::LongType x = 0; x < resizerState.outWidth; ++x) {
|
||||
const WeightsAndIndices& xWai = xWais[x];
|
||||
// Shift values in cachedValue to fill first '_advance' values.
|
||||
switch (xWai._advance) {
|
||||
case 3:
|
||||
for (auto c = 0; c < numChannels; ++c) {
|
||||
cachedValue[4 * c + 0] = cachedValue[4 * c + 1];
|
||||
cachedValue[4 * c + 1] = cachedValue[4 * c + 2];
|
||||
cachedValue[4 * c + 2] = cachedValue[4 * c + 3];
|
||||
}
|
||||
break;
|
||||
case 2:
|
||||
for (auto c = 0; c < numChannels; ++c) {
|
||||
cachedValue[4 * c + 0] = cachedValue[4 * c + 2];
|
||||
cachedValue[4 * c + 1] = cachedValue[4 * c + 3];
|
||||
}
|
||||
break;
|
||||
case 1: {
|
||||
for (auto c = 0; c < numChannels; ++c) {
|
||||
cachedValue[4 * c + 0] = cachedValue[4 * c + 3];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Set the remaining '4-_advance' values by computing.
|
||||
switch (xWai._advance) {
|
||||
case 0:
|
||||
for (auto c = 0; c < numChannels; ++c) {
|
||||
cachedValue[4 * c + 0] =
|
||||
computeYInterpolation(0, c * cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
}
|
||||
case 1:
|
||||
for (auto c = 0; c < numChannels; ++c) {
|
||||
cachedValue[4 * c + 1] =
|
||||
computeYInterpolation(1, c * cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
}
|
||||
case 2:
|
||||
for (auto c = 0; c < numChannels; ++c) {
|
||||
cachedValue[4 * c + 2] =
|
||||
computeYInterpolation(2, c * cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
}
|
||||
case 3:
|
||||
for (auto c = 0; c < numChannels; ++c) {
|
||||
cachedValue[4 * c + 3] =
|
||||
computeYInterpolation(3, c * cStride, yWai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, xWai);
|
||||
}
|
||||
break;
|
||||
}
|
||||
for (auto c = 0; c < numChannels; ++c) {
|
||||
pOutput[x * numChannels + c] =
|
||||
(F)compute(&cachedValue[4 * c], xWai._weight0, xWai._weight1, xWai._weight2, xWai._weight3);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(func, 0, batchNum);
|
||||
}
|
||||
|
||||
// simplified bicubic resize without antialiasing
|
||||
//
|
||||
template <typename T>
|
||||
sd::Status resizeBicubicFunctorA_(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const alignCorners, CoordinateTransformationMode coorMode, bool exclude_outside,
|
||||
double coefficient, NDArray* output) {
|
||||
ImageResizerState st(alignCorners, coorMode == HALF_PIXEL); // align_corners, half_pixel_align
|
||||
auto res = st.validateAndCreateOutput(image, width, height);
|
||||
if (res == sd::Status::OK) {
|
||||
switch (coorMode) {
|
||||
case ASYMMETRIC:
|
||||
bicubicInterpolateWithCaching<T, float, LegacyScaler>(image, st, coefficient, exclude_outside, output);
|
||||
break;
|
||||
case HALF_PIXEL:
|
||||
bicubicInterpolateWithCaching<T, float, HalfPixelScaler>(image, st, coefficient, exclude_outside, output);
|
||||
break;
|
||||
case HALF_PIXEL_NN:
|
||||
bicubicInterpolateWithCaching<T, float, HalfPixelScalerNN>(image, st, coefficient, exclude_outside, output);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
sd::Status resizeBicubicFunctorA(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const alignCorners, CoordinateTransformationMode coorMode, bool exclude_outside,
|
||||
double coefficient, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(image->dataType(), return resizeBicubicFunctorA_,
|
||||
(context, image, width, height, alignCorners, coorMode, exclude_outside, coefficient, output),
|
||||
SD_NUMERIC_TYPES);
|
||||
}
|
||||
// ------------------------------------------------------------------------------------------------------------------ //
|
||||
|
||||
template <typename T>
|
||||
static void resizeArea(ImageResizerState const& st, std::vector<CachedInterpolation> const& caches,
|
||||
NDArray * input, NDArray* output) {
|
||||
T const* inputPtr = input->bufferAsT<T>();
|
||||
float scale = 1.f / (st.heightScale * st.widthScale);
|
||||
auto outputPtr = output->bufferAsT<float>(); // output is always float. TO DO: provide another float types also with
|
||||
// template <typename X, typename Z> declaration
|
||||
|
||||
auto batchProcess = PRAGMA_THREADS_FOR {
|
||||
for (auto batch = start; batch < stop; batch++) {
|
||||
for (auto y = 0; y < st.outHeight; ++y) {
|
||||
const float inY = y * st.heightScale;
|
||||
const float inY1 = (y + 1) * st.heightScale;
|
||||
// The start and end height indices of all the cells that could
|
||||
// contribute to the target cell.
|
||||
const sd::LongType yStart = math::sd_floor<float, sd::LongType>(inY);
|
||||
const sd::LongType yEnd = math::sd_ceil<float, sd::LongType>(inY1);
|
||||
|
||||
std::vector<ScaleCache<T>> yCaches;
|
||||
auto cacheLen = yEnd - yStart;
|
||||
if (cacheLen) {
|
||||
yCaches.resize(cacheLen);
|
||||
};
|
||||
ScaleCache<T>* yCachesPtr = yCaches.data();
|
||||
sd::LongType yCachesSize = yCaches.size();
|
||||
for (auto i = yStart, k = sd::LongType(0); i < yEnd; ++i, ++k) {
|
||||
ScaleCache<T> scaleCache;
|
||||
if (i < inY) {
|
||||
scaleCache.yScale = (i + 1 > inY1 ? st.heightScale : i + 1 - inY);
|
||||
} else {
|
||||
scaleCache.yScale = (i + 1 > inY1 ? inY1 - i : 1.0);
|
||||
}
|
||||
scaleCache.yPtr = inputPtr + (batch * st.bStride + bound(i, st.inHeight) * st.hStride);
|
||||
yCaches[k] = scaleCache;
|
||||
}
|
||||
float* output = outputPtr + (batch * st.outHeight + y) * st.channels * st.outWidth;
|
||||
|
||||
if (st.channels == 3) {
|
||||
for (sd::LongType x = 0; x < st.outWidth; ++x) {
|
||||
const CachedInterpolation& xCache = caches[x];
|
||||
computePatchSumOf3Channels<T>(scale, st, yCachesPtr, yCachesSize, xCache, output);
|
||||
output += st.channels;
|
||||
}
|
||||
} else {
|
||||
for (sd::LongType x = 0; x < st.outWidth; ++x) {
|
||||
const CachedInterpolation& xCache = caches[x];
|
||||
computePatchSum<T>(scale, st, yCachesPtr, yCachesSize, xCache, output);
|
||||
output += st.channels;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(batchProcess, 0, st.batchSize, 1);
|
||||
}
|
||||
|
||||
template <typename X>
|
||||
sd::Status resizeAreaFunctor_(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const alignCorners, NDArray* output) {
|
||||
ImageResizerState st(alignCorners, false); // Create resize info
|
||||
auto res = st.validateAndCalculateOutputSize(image, width, height);
|
||||
if (Status::OK == res) {
|
||||
std::vector<CachedInterpolation> xCached(st.outWidth);
|
||||
auto cachingProcedure = PRAGMA_THREADS_FOR {
|
||||
for (auto x = start; x < stop; x++) {
|
||||
auto& xCache = xCached[x];
|
||||
const float inX = x * st.widthScale;
|
||||
const float inX1 = (x + 1) * st.widthScale;
|
||||
|
||||
sd::LongType v = math::sd_floor<float, sd::LongType>(inX);
|
||||
xCache.start = v;
|
||||
xCache.startScale = v < inX ? (v + 1 > inX1 ? st.widthScale : v + 1 - inX) : (v + 1 > inX1 ? inX1 - v : 1.f);
|
||||
v = math::sd_ceil<float, sd::LongType>(inX1);
|
||||
xCache.end = v--;
|
||||
xCache.endMinusOneScale =
|
||||
v < inX ? (v + 1 > inX1 ? st.widthScale : v + 1 - inX) : (v + 1 > inX1 ? inX1 - v : 1.f);
|
||||
xCache.needsBounding =
|
||||
bound(xCache.start, st.inWidth) != xCache.start || bound(xCache.end - 1, st.inWidth) != (xCache.end - 1);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(cachingProcedure, 0, xCached.size(), 1);
|
||||
|
||||
resizeArea<X>(st, xCached, image, output);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
sd::Status resizeAreaFunctor(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const alignCorners, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(image->dataType(), return resizeAreaFunctor_,
|
||||
(context, image, width, height, alignCorners, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
static sd::Status computeSpans(IKernelFunc<float>* kernel, sd::LongType const outSize, sd::LongType const inSize,
|
||||
float const scale, float const translate, bool const antialias, Spans& spans) {
|
||||
// When sampling, we need the inverse scale and translation, to map from an
|
||||
// output to an input pixel.
|
||||
float const invScale = 1.f / scale;
|
||||
float const invTranslate = -invScale * translate;
|
||||
// When downsampling the kernel should be scaled since we want to low pass
|
||||
// filter and interpolate, but when upsampling it should not be since we only
|
||||
// want to interpolate.
|
||||
float const kernelScale = antialias ? math::sd_max(invScale, 1.f) : 1.f;
|
||||
spans._spanSize =
|
||||
math::sd_min(2 * static_cast<int>(std::ceil(kernel->radius() * kernelScale)) + 1, static_cast<int>(inSize));
|
||||
auto starts = NDArrayFactory::create<int>('c', {outSize});
|
||||
auto weights = NDArrayFactory::create<float>('c', {outSize, spans._spanSize});
|
||||
spans._starts = starts;
|
||||
spans._weights = weights;
|
||||
|
||||
auto startsVec = spans._starts->bufferAsT<int>();
|
||||
auto weightsVector = spans._weights->bufferAsT<float>();
|
||||
spans._weights->nullify();
|
||||
|
||||
const float invKernelScale = 1.f / kernelScale;
|
||||
int maxSpanSize = 0;
|
||||
std::vector<float> tempWeights;
|
||||
|
||||
// return value if within bounds or bounds otherwise
|
||||
auto boundsAmp = [](sd::LongType const low, sd::LongType const high, sd::LongType const value) {
|
||||
if (high < value) return high;
|
||||
if (value < low) return low;
|
||||
return value;
|
||||
};
|
||||
|
||||
for (auto x = 0LL; x < outSize; ++x) {
|
||||
const float columnFloat = x + 0.5f;
|
||||
const float sampleFloat = columnFloat * invScale + invTranslate;
|
||||
|
||||
// Don't sample when the sampling location is outside the source image.
|
||||
if (sampleFloat < 0 || sampleFloat > inSize) {
|
||||
// Add an empty span.
|
||||
startsVec[x] = 0;
|
||||
continue;
|
||||
}
|
||||
sd::LongType spanStart = math::sd_ceil<float, float>(sampleFloat - kernel->radius() * kernelScale - 0.5f);
|
||||
sd::LongType spanEnd = math::sd_floor<float, float>(sampleFloat + kernel->radius() * kernelScale - 0.5f);
|
||||
spanStart = boundsAmp(0LL, inSize - 1, spanStart);
|
||||
spanEnd = boundsAmp(0LL, inSize - 1, spanEnd) + 1;
|
||||
int const spanSize = spanEnd - spanStart;
|
||||
if (spanSize > spans._spanSize) {
|
||||
return Logger::logStatusMsg(
|
||||
Status::BAD_INPUT,
|
||||
"Span is too large: "); // + spanSize + " vs " + spans._spanSize);//, spanSize, spans._spanSize));
|
||||
}
|
||||
float totalWeightSum = 0.f;
|
||||
tempWeights.clear();
|
||||
for (int source = spanStart; source < spanEnd; ++source) {
|
||||
float kernelPos = static_cast<float>(source) + 0.5f - sampleFloat;
|
||||
float weight = (*kernel)(kernelPos * invKernelScale);
|
||||
totalWeightSum += weight;
|
||||
tempWeights.push_back(weight);
|
||||
}
|
||||
maxSpanSize = std::max(maxSpanSize, spanSize);
|
||||
if (math::sd_abs<float,float>(totalWeightSum) >= 1000.f * DataTypeUtils::min_positive<float>()) { //
|
||||
auto totalWeightSumInverted = 1.0f / totalWeightSum;
|
||||
auto outIndex = spans._spanSize * x;
|
||||
for (auto weight : tempWeights) {
|
||||
weightsVector[outIndex] = weight * totalWeightSumInverted;
|
||||
++outIndex;
|
||||
}
|
||||
}
|
||||
startsVec[x] = spanStart;
|
||||
}
|
||||
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
static void gatherSpans(int const rowSpanSize, NDArray& rowStarts, NDArray& rowWeights,
|
||||
int const colSpanSize, NDArray& columnStarts, NDArray& columnWeights,
|
||||
NDArray * images, NDArray& intermediate, NDArray* output) {
|
||||
auto batchSize = images->sizeAt(0);
|
||||
auto inputHeight = images->sizeAt(1);
|
||||
auto inputWidth = images->sizeAt(2);
|
||||
auto channels = images->sizeAt(3);
|
||||
|
||||
auto outputHeight = output->sizeAt(1);
|
||||
auto outputWidth = output->sizeAt(2);
|
||||
|
||||
auto inputPixPerBatch = images->strideAt(0);
|
||||
auto intermediatePixPerBatch = inputWidth * outputHeight * channels;
|
||||
auto outputPixPerBatch = outputWidth * outputHeight * channels;
|
||||
Z* intermediatePtr = intermediate.bufferAsT<Z>();
|
||||
bool inputEws1 = images->ews() == 1;
|
||||
auto inRowStride = images->strideAt(1);
|
||||
auto wStride = images->strideAt(2);
|
||||
auto cStride = images->strideAt(3);
|
||||
const X* imagePtr = images->bufferAsT<X>();
|
||||
Z* outPtr = output->bufferAsT<Z>();
|
||||
for (int b = 0; b < batchSize;
|
||||
++b, imagePtr += inputPixPerBatch, intermediatePtr += intermediatePixPerBatch, outPtr += outputPixPerBatch) {
|
||||
gatherRows<X, Z>(rowSpanSize, rowStarts.bufferAsT<int>(), rowWeights.bufferAsT<Z>(), imagePtr, inputHeight,
|
||||
inputWidth, outputHeight, inputWidth, channels, intermediatePtr, inputEws1, inRowStride, wStride,
|
||||
cStride);
|
||||
gatherColumns<Z>(colSpanSize, columnStarts.bufferAsT<int>(), columnWeights.bufferAsT<Z>(), intermediatePtr,
|
||||
outputHeight, inputWidth, outputHeight, outputWidth, channels, outPtr);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
static sd::Status resizeKernel(IKernelFunc<float>* transformationKernel, NDArray * input, sd::LongType outWidth,
|
||||
sd::LongType outHeight, bool antialias, NDArray* output) {
|
||||
sd::LongType const batchSize = input->sizeAt(0);
|
||||
sd::LongType const inputHeight = input->sizeAt(1);
|
||||
sd::LongType const inputWidth = input->sizeAt(2);
|
||||
sd::LongType const channels = input->sizeAt(3);
|
||||
|
||||
Z rowScale = Z(outHeight) / Z(inputHeight);
|
||||
Z columnScale = Z(outWidth) / Z(inputWidth);
|
||||
|
||||
// Return if the output is empty.
|
||||
if (output->lengthOf() == 0) return sd::Status::OK;
|
||||
|
||||
Spans colSpans;
|
||||
|
||||
auto res = computeSpans(transformationKernel, outWidth, inputWidth, columnScale, 0.f, antialias, colSpans);
|
||||
if (res != sd::Status::OK) return res;
|
||||
Spans rowSpans;
|
||||
res = computeSpans(transformationKernel, outHeight, inputHeight, rowScale, 0.f, antialias, rowSpans);
|
||||
|
||||
NDArray *intermediate = NDArrayFactory::create<Z>('c', {batchSize, outHeight, inputWidth, channels});
|
||||
|
||||
// const functor::Spans& const_row_spans = row_spans;
|
||||
// typename TTypes<int32, 1>::ConstTensor row_starts(
|
||||
// const_row_spans.starts.tensor<int32, 1>());
|
||||
auto rowStarts = *rowSpans._starts; // shape {outWidth}
|
||||
auto rowWeights = *rowSpans._weights; // shape {outWidth, numSpans}
|
||||
auto columnStarts = *colSpans._starts; // shape {outHeights}
|
||||
auto columnWeights = *colSpans._weights; // shape {outHeights, numSpans}
|
||||
|
||||
gatherSpans<X, Z>(rowSpans._spanSize, rowStarts, rowWeights, colSpans._spanSize, columnStarts, columnWeights, input,
|
||||
*intermediate, output);
|
||||
delete intermediate;
|
||||
return res;
|
||||
}
|
||||
#if defined(HAS_FLOAT32)
|
||||
static sd::Status resizeBilinear(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const antialias, NDArray* output) {
|
||||
auto kernel = std::unique_ptr<IKernelFunc<float>>(new TriangleKernelFunc());
|
||||
auto imageDType = image->dataType();
|
||||
auto outputDtype = output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(image->dataType(), output->dataType(), return resizeKernel,
|
||||
(kernel.get(), image, (sd::LongType)width, (sd::LongType)height, antialias, output),
|
||||
SD_NUMERIC_TYPES, SKIP_FIRST_COMMA(TTYPE_FLOAT32));
|
||||
return Logger::logStatusMsg(Status::VALIDATION, "helpers::resizeBilinear: Unknown error occured.");
|
||||
}
|
||||
|
||||
static sd::Status resizeBicubicA(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
CoordinateTransformationMode coorMode, bool exclude_outside, double coefficient,
|
||||
NDArray* output) {
|
||||
constexpr bool alignCorners = false;
|
||||
return resizeBicubicFunctorA(context, image, width, height, alignCorners, coorMode, exclude_outside, coefficient,
|
||||
output);
|
||||
}
|
||||
|
||||
static sd::Status resizeBicubicAntialias(sd::LaunchContext* context, NDArray * image, int const width,
|
||||
int const height, bool const antialias, double coefficient, NDArray* output) {
|
||||
// coorMode is HALF_PIXEL exlude_outside is True
|
||||
auto kernel = std::unique_ptr<IKernelFunc<float>>(new KeysCubicKernelFunc<float>(coefficient));
|
||||
auto imageDType = image->dataType();
|
||||
auto outputDtype = output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(image->dataType(), output->dataType(), return resizeKernel,
|
||||
(kernel.get(), image, (sd::LongType)width, (sd::LongType)height, antialias, output),
|
||||
SD_NUMERIC_TYPES, SKIP_FIRST_COMMA(TTYPE_FLOAT32));
|
||||
return sd::Status::OK;
|
||||
}
|
||||
#endif
|
||||
|
||||
static sd::Status resizeArea(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const antialias, NDArray* output) {
|
||||
return resizeAreaFunctor(context, image, width, height, false, output);
|
||||
}
|
||||
#if defined(HAS_FLOAT32)
|
||||
static sd::Status resizeLanczos3(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const antialias, NDArray* output) {
|
||||
auto kernel = std::unique_ptr<IKernelFunc<float>>(new LanczosKernelFunc(3.f));
|
||||
auto imageDType = image->dataType();
|
||||
auto outputDtype = output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(image->dataType(), output->dataType(), return resizeKernel,
|
||||
(kernel.get(), image, (sd::LongType)width, (sd::LongType)height, antialias, output),
|
||||
SD_NUMERIC_TYPES, SKIP_FIRST_COMMA(TTYPE_FLOAT32));
|
||||
return Logger::logStatusMsg(Status::VALIDATION, "helpers::resizeLanczos3: Unknown error occured.");
|
||||
}
|
||||
|
||||
static sd::Status resizeLanczos5(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const antialias, NDArray* output) {
|
||||
auto kernel = std::unique_ptr<IKernelFunc<float>>(new LanczosKernelFunc(5.f));
|
||||
auto imageDType = image->dataType();
|
||||
auto outputDtype = output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(image->dataType(), output->dataType(), return resizeKernel,
|
||||
(kernel.get(), image, (sd::LongType)width, (sd::LongType)height, antialias, output),
|
||||
SD_NUMERIC_TYPES, SKIP_FIRST_COMMA(TTYPE_FLOAT32));
|
||||
return Logger::logStatusMsg(Status::VALIDATION, "helpers::resizeLanczos5: Unknown error occured.");
|
||||
}
|
||||
|
||||
static sd::Status resizeGaussian(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
bool const antialias, NDArray* output) {
|
||||
auto kernel = std::unique_ptr<IKernelFunc<float>>(new GaussianKernelFunc());
|
||||
auto imageDType = image->dataType();
|
||||
auto outputDtype = output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(image->dataType(), output->dataType(), return resizeKernel,
|
||||
(kernel.get(), image, (sd::LongType)width, (sd::LongType)height, antialias, output),
|
||||
SD_NUMERIC_TYPES, SKIP_FIRST_COMMA(TTYPE_FLOAT32));
|
||||
return Logger::logStatusMsg(Status::VALIDATION, "helpers::resizeGaussian: Unknown error occured.");
|
||||
}
|
||||
|
||||
static sd::Status resizeMitchellcubic(sd::LaunchContext* context, NDArray * image, int const width,
|
||||
int const height, bool const antialias, NDArray* output) {
|
||||
auto kernel = std::unique_ptr<IKernelFunc<float>>(new MitchellCubicKernelFunc());
|
||||
auto imageDType = image->dataType();
|
||||
auto outputDtype = output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(image->dataType(), output->dataType(), return resizeKernel,
|
||||
(kernel.get(), image, (sd::LongType)width, (sd::LongType)height, antialias, output),
|
||||
SD_NUMERIC_TYPES, SKIP_FIRST_COMMA(TTYPE_FLOAT32));
|
||||
return Logger::logStatusMsg(Status::VALIDATION, "helpers::ResizeMitchellcubic: Unknown error occured.");
|
||||
}
|
||||
#endif
|
||||
// ------------------------------------------------------------------------------------------------------------------ //
|
||||
sd::Status resizeImagesFunctor(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
ImageResizeMethods method, bool alignCorners, NDArray* output) {
|
||||
switch (method) {
|
||||
case kResizeBilinear:
|
||||
return resizeBilinearFunctor(context, image, width, height, alignCorners, false, output);
|
||||
case kResizeNearest:
|
||||
return resizeNeighborFunctor(context, image, width, height, CoordinateTransformationMode::ASYMMETRIC,
|
||||
alignCorners ? NearestMode::ROUND_PREFER_CEIL : NearestMode::FLOOR, alignCorners,
|
||||
output);
|
||||
case kResizeBicubic:
|
||||
return resizeBicubicFunctor(context, image, width, height, alignCorners, false, output);
|
||||
case kResizeArea:
|
||||
return resizeAreaFunctor(context, image, width, height, alignCorners, output);
|
||||
case kResizeGaussian:
|
||||
break;
|
||||
case kResizeLanczos3:
|
||||
break;
|
||||
case kResizeLanczos5:
|
||||
break;
|
||||
case kResizeMitchellcubic:
|
||||
break;
|
||||
}
|
||||
sd_printf("helper::resizeImagesFunctor: Wrong resize method %i\n", (int)method);
|
||||
return Logger::logStatusMsg(Status::BAD_INPUT, "helper::resizeImagesFunctor: Wrong resize method");
|
||||
}
|
||||
// ------------------------------------------------------------------------------------------------------------------ //
|
||||
sd::Status resizeFunctor(sd::LaunchContext* context, NDArray * image, int const width, int const height,
|
||||
ImageResizeMethods method, CoordinateTransformationMode coorMode, bool exclude_outside,
|
||||
NearestMode nearestMode, double coefficient, bool antialias, NDArray* output) {
|
||||
switch (method) {
|
||||
case kResizeNearest:
|
||||
return resizeNeighborFunctor(context, image, width, height, coorMode, nearestMode, false, output);
|
||||
case kResizeArea:
|
||||
return resizeArea(context, image, width, height, antialias, output);
|
||||
#if defined(HAS_FLOAT32)
|
||||
case kResizeBilinear:
|
||||
return resizeBilinear(context, image, width, height, antialias, output);
|
||||
case kResizeBicubic: {
|
||||
// if antialias then coorMode is HALF_PIXEL and exlude_outside is true
|
||||
if (antialias) {
|
||||
return resizeBicubicAntialias(context, image, width, height, antialias, coefficient, output);
|
||||
} else {
|
||||
// use modified v1
|
||||
return resizeBicubicA(context, image, width, height, coorMode, exclude_outside, coefficient, output);
|
||||
}
|
||||
}
|
||||
case kResizeLanczos3:
|
||||
return resizeLanczos3(context, image, width, height, antialias, output);
|
||||
case kResizeLanczos5:
|
||||
return resizeLanczos5(context, image, width, height, antialias, output);
|
||||
case kResizeGaussian:
|
||||
return resizeGaussian(context, image, width, height, antialias, output);
|
||||
case kResizeMitchellcubic:
|
||||
return resizeMitchellcubic(context, image, width, height, antialias, output);
|
||||
#else
|
||||
case kResizeBilinear:
|
||||
case kResizeBicubic:
|
||||
case kResizeLanczos3:
|
||||
case kResizeLanczos5:
|
||||
case kResizeGaussian:
|
||||
case kResizeMitchellcubic: {
|
||||
sd_printf("helper::resizeFunctor: only float type is supported by this resize method %i\n", (int)method);
|
||||
return Logger::logStatusMsg(Status::BAD_INPUT, "helper::resizeFunctor: only float type supported");
|
||||
}
|
||||
|
||||
#endif
|
||||
}
|
||||
sd_printf("helper::resizeFunctor: Wrong resize method %i\n", (int)method);
|
||||
return Logger::logStatusMsg(Status::BAD_INPUT, "helper::resizeFunctor: Wrong resize method");
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,272 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <ops/declarable/helpers/image_suppression.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <numeric>
|
||||
#include <queue>
|
||||
#include <system/selective_rendering.h>
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void nonMaxSuppressionV2_(NDArray* boxes, NDArray* scales, int maxSize, double overlapThreshold,
|
||||
double scoreThreshold, NDArray* output) {
|
||||
std::vector<int> indices(scales->lengthOf());
|
||||
std::iota(indices.begin(), indices.end(), 0);
|
||||
auto actualIndicesCount = indices.size();
|
||||
for (auto e = 0; e < scales->lengthOf(); e++) {
|
||||
if (scales->e<float>(e) < (float)scoreThreshold) {
|
||||
indices[e] = -1;
|
||||
actualIndicesCount--;
|
||||
}
|
||||
}
|
||||
std::sort(indices.begin(), indices.end(),
|
||||
[scales](int i, int j) { return i >= 0 && j >= 0 ? scales->e<T>(i) > scales->e<T>(j) : (i > j); });
|
||||
|
||||
std::vector<int> selectedIndices(output->lengthOf(), 0);
|
||||
auto needToSuppressWithThreshold = [](NDArray& boxes, int previousIndex, int nextIndex, T threshold) -> bool {
|
||||
if (previousIndex < 0 || nextIndex < 0) return true;
|
||||
T minYPrev = sd::math::sd_min(boxes.t<T>(previousIndex, 0), boxes.t<T>(previousIndex, 2));
|
||||
T minXPrev = sd::math::sd_min(boxes.t<T>(previousIndex, 1), boxes.t<T>(previousIndex, 3));
|
||||
T maxYPrev = sd::math::sd_max(boxes.t<T>(previousIndex, 0), boxes.t<T>(previousIndex, 2));
|
||||
T maxXPrev = sd::math::sd_max(boxes.t<T>(previousIndex, 1), boxes.t<T>(previousIndex, 3));
|
||||
T minYNext = sd::math::sd_min(boxes.t<T>(nextIndex, 0), boxes.t<T>(nextIndex, 2));
|
||||
T minXNext = sd::math::sd_min(boxes.t<T>(nextIndex, 1), boxes.t<T>(nextIndex, 3));
|
||||
T maxYNext = sd::math::sd_max(boxes.t<T>(nextIndex, 0), boxes.t<T>(nextIndex, 2));
|
||||
T maxXNext = sd::math::sd_max(boxes.t<T>(nextIndex, 1), boxes.t<T>(nextIndex, 3));
|
||||
T areaPrev = (maxYPrev - minYPrev) * (maxXPrev - minXPrev);
|
||||
T areaNext = (maxYNext - minYNext) * (maxXNext - minXNext);
|
||||
|
||||
if (areaNext <= T(0.f) || areaPrev <= T(0.f)) return false;
|
||||
|
||||
T minIntersectionY = sd::math::sd_max(minYPrev, minYNext);
|
||||
T minIntersectionX = sd::math::sd_max(minXPrev, minXNext);
|
||||
T maxIntersectionY = sd::math::sd_min(maxYPrev, maxYNext);
|
||||
T maxIntersectionX = sd::math::sd_min(maxXPrev, maxXNext);
|
||||
T intersectionArea = sd::math::sd_max(T(maxIntersectionY - minIntersectionY), T(0.0f)) *
|
||||
sd::math::sd_max(T(maxIntersectionX - minIntersectionX), T(0.0f));
|
||||
T intersectionValue = intersectionArea / (areaPrev + areaNext - intersectionArea);
|
||||
return intersectionValue > threshold;
|
||||
};
|
||||
// int numSelected = 0;
|
||||
int numBoxes = actualIndicesCount; // boxes->sizeAt(0);
|
||||
int numSelected = 0;
|
||||
|
||||
for (int i = 0; i < numBoxes; ++i) {
|
||||
bool shouldSelect = numSelected < output->lengthOf();
|
||||
|
||||
// FIXME: add parallelism here
|
||||
for (int j = numSelected - 1; j >= 0; --j) {
|
||||
if (shouldSelect)
|
||||
if (needToSuppressWithThreshold(*boxes, indices[i], indices[selectedIndices[j]], T(overlapThreshold))) {
|
||||
shouldSelect = false;
|
||||
}
|
||||
}
|
||||
if (shouldSelect) {
|
||||
output->p(numSelected, indices[i]);
|
||||
selectedIndices[numSelected++] = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Return intersection-over-union overlap between boxes i and j
|
||||
template <typename T>
|
||||
static inline T similirityV3_(NDArray& boxes, sd::LongType i, sd::LongType j) {
|
||||
const T zero = static_cast<T>(0.f);
|
||||
const T yminI = math::sd_min(boxes.t<T>(i, 0), boxes.t<T>(i, 2));
|
||||
const T xminI = math::sd_min(boxes.t<T>(i, 1), boxes.t<T>(i, 3));
|
||||
const T ymaxI = math::sd_max(boxes.t<T>(i, 0), boxes.t<T>(i, 2));
|
||||
const T xmaxI = math::sd_max(boxes.t<T>(i, 1), boxes.t<T>(i, 3));
|
||||
const T yminJ = math::sd_min(boxes.t<T>(j, 0), boxes.t<T>(j, 2));
|
||||
const T xminJ = math::sd_min(boxes.t<T>(j, 1), boxes.t<T>(j, 3));
|
||||
const T ymaxJ = math::sd_max(boxes.t<T>(j, 0), boxes.t<T>(j, 2));
|
||||
const T xmaxJ = math::sd_max(boxes.t<T>(j, 1), boxes.t<T>(j, 3));
|
||||
const T areaI = (ymaxI - yminI) * (xmaxI - xminI);
|
||||
const T areaJ = (ymaxJ - yminJ) * (xmaxJ - xminJ);
|
||||
if (areaI <= zero || areaJ <= zero) {
|
||||
return zero;
|
||||
}
|
||||
const T intersectionYmin = math::sd_max(yminI, yminJ);
|
||||
const T intersectionXmin = math::sd_max(xminI, xminJ);
|
||||
const T intersectionYmax = math::sd_min(ymaxI, ymaxJ);
|
||||
const T intersectionXmax = math::sd_min(xmaxI, xmaxJ);
|
||||
const T intersectionY = intersectionYmax - intersectionYmin;
|
||||
const T intersectionX = intersectionXmax - intersectionXmin;
|
||||
const T intersectionArea = math::sd_max(intersectionY, zero) * math::sd_max(intersectionX, zero);
|
||||
return intersectionArea / (areaI + areaJ - intersectionArea);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline T similarityOverlaps_(NDArray& boxes, sd::LongType i, sd::LongType j) {
|
||||
return boxes.t<T>(i, j);
|
||||
}
|
||||
|
||||
typedef NDArray (*SimilarityFunc)(NDArray& boxes, sd::LongType i, sd::LongType j);
|
||||
|
||||
static NDArray similiratyOverlaps(NDArray& boxes, sd::LongType i, sd::LongType j) {
|
||||
NDArray res(boxes.dataType(), boxes.getContext());
|
||||
BUILD_SINGLE_SELECTOR(boxes.dataType(), res = similarityOverlaps_, (boxes, i, j), SD_FLOAT_TYPES);
|
||||
return res;
|
||||
}
|
||||
|
||||
static NDArray similarityV3(NDArray& boxes, sd::LongType i, sd::LongType j) {
|
||||
NDArray res(boxes.dataType(), boxes.getContext());
|
||||
BUILD_SINGLE_SELECTOR(boxes.dataType(), res = similirityV3_, (boxes, i, j), SD_FLOAT_TYPES);
|
||||
return res;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template <typename T, typename I>
|
||||
static sd::LongType nonMaxSuppressionGeneric_(sd::LaunchContext* context, NDArray* boxes, NDArray* scores,
|
||||
int outputSize, float overlapThreshold, float scoreThreshold,
|
||||
NDArray* output, SimilarityFunc f) {
|
||||
auto numBoxes = boxes->sizeAt(0);
|
||||
T* scoresData = scores->dataBuffer()->primaryAsT<T>();
|
||||
|
||||
// Data structure for a selection candidate in NMS.
|
||||
struct Candidate {
|
||||
int _boxIndex;
|
||||
T _score;
|
||||
int _suppressBeginIndex;
|
||||
};
|
||||
|
||||
auto cmp = [](const Candidate& bsI, const Candidate& bsJ) -> bool {
|
||||
return ((bsI._score == bsJ._score) && (bsI._boxIndex > bsJ._boxIndex)) || (bsI._score < bsJ._score);
|
||||
};
|
||||
|
||||
std::priority_queue<Candidate, std::deque<Candidate>, decltype(cmp)> candidatePriorityQueue(cmp);
|
||||
for (auto i = 0; i < scores->lengthOf(); ++i) {
|
||||
if ((float)scoresData[i] > (float)scoreThreshold) {
|
||||
candidatePriorityQueue.emplace(Candidate({i, scoresData[i], 0}));
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<I> selected;
|
||||
T similarity, originalScore;
|
||||
Candidate nextCandidate;
|
||||
|
||||
while (selected.size() < static_cast<size_t>(outputSize) && !candidatePriorityQueue.empty()) {
|
||||
nextCandidate = candidatePriorityQueue.top();
|
||||
originalScore = nextCandidate._score;
|
||||
candidatePriorityQueue.pop();
|
||||
|
||||
// Overlapping boxes are likely to have similar scores, therefore we
|
||||
// iterate through the previously selected boxes backwards in order to
|
||||
// see if `nextCandidate` should be suppressed. We also enforce a property
|
||||
// that a candidate can be suppressed by another candidate no more than
|
||||
// once via `suppress_begin_index` which tracks which previously selected
|
||||
// boxes have already been compared against next_candidate prior to a given
|
||||
// iteration. These previous selected boxes are then skipped over in the
|
||||
// following loop.
|
||||
bool shouldHardSuppress = false;
|
||||
for (int j = static_cast<int>(selected.size()) - 1; j >= nextCandidate._suppressBeginIndex; --j) {
|
||||
auto similarityA =
|
||||
f(*boxes, nextCandidate._boxIndex, selected[j]); // boxes->t<T>(nextCandidate._boxIndex, selected[j]);
|
||||
similarity = similarityA.template t<T>(0);
|
||||
nextCandidate._score *= T(similarity <= overlapThreshold ? 1.0 : 0.); // suppressWeightFunc(similarity);
|
||||
|
||||
// First decide whether to perform hard suppression
|
||||
if ((float)similarity >= static_cast<float>(overlapThreshold)) {
|
||||
shouldHardSuppress = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// If next_candidate survives hard suppression, apply soft suppression
|
||||
if ((float)nextCandidate._score <= (float)scoreThreshold) break;
|
||||
}
|
||||
// If `nextCandidate._score` has not dropped below `scoreThreshold`
|
||||
// by this point, then we know that we went through all of the previous
|
||||
// selections and can safely update `suppress_begin_index` to
|
||||
// `selected.size()`. If on the other hand `next_candidate.score`
|
||||
// *has* dropped below the score threshold, then since `suppressWeight`
|
||||
// always returns values in [0, 1], further suppression by items that were
|
||||
// not covered in the above for loop would not have caused the algorithm
|
||||
// to select this item. We thus do the same update to
|
||||
// `suppressBeginIndex`, but really, this element will not be added back
|
||||
// into the priority queue in the following.
|
||||
nextCandidate._suppressBeginIndex = selected.size();
|
||||
|
||||
if (!shouldHardSuppress) {
|
||||
if (nextCandidate._score == originalScore) {
|
||||
// Suppression has not occurred, so select next_candidate
|
||||
selected.push_back(nextCandidate._boxIndex);
|
||||
}
|
||||
if ((float)nextCandidate._score > (float)scoreThreshold) {
|
||||
// Soft suppression has occurred and current score is still greater than
|
||||
// score_threshold; add next_candidate back onto priority queue.
|
||||
candidatePriorityQueue.push(nextCandidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (output) {
|
||||
DataBuffer buf(selected.data(), selected.size() * sizeof(I), DataTypeUtils::fromT<I>());
|
||||
output->dataBuffer()->copyBufferFrom(buf, buf.getLenInBytes());
|
||||
}
|
||||
|
||||
return (sd::LongType)selected.size();
|
||||
}
|
||||
|
||||
sd::LongType nonMaxSuppressionGeneric(sd::LaunchContext* context, NDArray* boxes, NDArray* scores, int maxSize,
|
||||
double overlapThreshold, double scoreThreshold, NDArray* output) {
|
||||
auto boxesDType = boxes->dataType();
|
||||
auto outputDType = output == nullptr ? DataType::INT32 : output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(boxes->dataType(), output == nullptr ? DataType::INT32 : output->dataType(),
|
||||
return nonMaxSuppressionGeneric_,
|
||||
(context, boxes, scores, maxSize, overlapThreshold, scoreThreshold, output, similiratyOverlaps),
|
||||
SD_FLOAT_TYPES, SD_INTEGER_TYPES);
|
||||
return 0;
|
||||
}
|
||||
|
||||
sd::LongType nonMaxSuppressionV3(sd::LaunchContext* context, NDArray* boxes, NDArray* scores, int maxSize,
|
||||
double overlapThreshold, double scoreThreshold, NDArray* output) {
|
||||
auto boxesDType = boxes->dataType();
|
||||
auto outputDType = output == nullptr ? DataType::INT32 : output->dataType();
|
||||
BUILD_DOUBLE_SELECTOR(boxes->dataType(), output == nullptr ? DataType::INT32 : output->dataType(),
|
||||
return nonMaxSuppressionGeneric_,
|
||||
(context, boxes, scores, maxSize, overlapThreshold, scoreThreshold, output, similarityV3),
|
||||
SD_FLOAT_TYPES, SD_INTEGER_TYPES);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( sd::LongType nonMaxSuppressionGeneric_,
|
||||
(sd::LaunchContext * context, NDArray* boxes, NDArray* scores, int maxSize,
|
||||
float overlapThreshold, float scoreThreshold, NDArray* output, SimilarityFunc SimilarityFunc),
|
||||
SD_FLOAT_TYPES, SD_INTEGER_TYPES);
|
||||
|
||||
void nonMaxSuppression(sd::LaunchContext* context, NDArray* boxes, NDArray* scales, int maxSize,
|
||||
double overlapThreshold, double scoreThreshold, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(boxes->dataType(), nonMaxSuppressionV2_,
|
||||
(boxes, scales, maxSize, overlapThreshold, scoreThreshold, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void nonMaxSuppressionV2_,
|
||||
(NDArray * boxes, NDArray* scales, int maxSize, double overlapThreshold, double scoreThreshold,
|
||||
NDArray* output),
|
||||
SD_NUMERIC_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,326 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Oleh Semeniv (oleg.semeniv@gmail.com)
|
||||
// @author AbdelRauf (rauf@konduit.ai)
|
||||
//
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <ops/declarable/helpers/adjust_hue.h>
|
||||
#include <ops/declarable/helpers/imagesHelpers.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void rgbToGrs_(NDArray& input, NDArray& output, const int dimC) {
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
const int rank = input.rankOf();
|
||||
|
||||
if (dimC == rank - 1 && 'c' == input.ordering() && 1 == input.ews() && 'c' == output.ordering() &&
|
||||
1 == output.ews()) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
const auto xStep = i * 3;
|
||||
z[i] = 0.2989f * x[xStep] + 0.5870f * x[xStep + 1] + 0.1140f * x[xStep + 2];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, output.lengthOf(), 1);
|
||||
return;
|
||||
}
|
||||
|
||||
sd::LongType *outputShape = shape::shapeOf(output.shapeInfo());
|
||||
sd::LongType *outputStride = shape::stride(output.shapeOf());
|
||||
sd::LongType *inputStride = shape::stride(input.shapeInfo());
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, rank,outputShape, coords);
|
||||
sd::LongType zOffset, xOffset0, xOffset1, xOffset2;
|
||||
COORDS2INDEX(rank, outputStride, coords, zOffset);
|
||||
COORDS2INDEX(rank, inputStride, coords, xOffset0);
|
||||
coords[dimC]++;
|
||||
COORDS2INDEX(rank,inputStride, coords, xOffset1);
|
||||
coords[dimC]++;
|
||||
COORDS2INDEX(rank, inputStride, coords, xOffset2);
|
||||
z[zOffset] = 0.2989f * x[xOffset0] + 0.5870f * x[xOffset1] + 0.1140f * x[xOffset2];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, output.lengthOf(), 1);
|
||||
return;
|
||||
}
|
||||
|
||||
void transformRgbGrs(sd::LaunchContext* context, NDArray& input, NDArray& output, const int dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), rgbToGrs_, (input, output, dimC), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_INLINE static void tripleTransformer(NDArray* input, NDArray* output, const int dimC, T (&tr)[3][3]) {
|
||||
const int rank = input->rankOf();
|
||||
|
||||
const T* x = input->bufferAsT<T>();
|
||||
T* z = output->bufferAsT<T>();
|
||||
// TODO: Use tensordot or other optimizied helpers to see if we can get better performance.
|
||||
|
||||
if (dimC == rank - 1 && input->ews() == 1 && output->ews() == 1 && input->ordering() == 'c' &&
|
||||
output->ordering() == 'c') {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
// simple M*v //tr.T*v.T // v * tr //rule: (AB)' =B'A'
|
||||
// v.shape (1,3) row vector
|
||||
T x0, x1, x2;
|
||||
x0 = x[i]; // just additional hint
|
||||
x1 = x[i + 1];
|
||||
x2 = x[i + 2];
|
||||
z[i] = x0 * tr[0][0] + x1 * tr[1][0] + x2 * tr[2][0];
|
||||
z[i + 1] = x0 * tr[0][1] + x1 * tr[1][1] + x2 * tr[2][1];
|
||||
z[i + 2] = x0 * tr[0][2] + x1 * tr[1][2] + x2 * tr[2][2];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input->lengthOf(), 3);
|
||||
} else {
|
||||
auto packX = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(), dimC);
|
||||
auto packZ = sd::ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(), dimC);
|
||||
|
||||
const sd::LongType numOfTads = packX->numberOfTads();
|
||||
const sd::LongType xDimCstride = input->stridesOf()[dimC];
|
||||
const sd::LongType zDimCstride = output->stridesOf()[dimC];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
const T* xTad = x + packX->platformOffsets()[i];
|
||||
T* zTad = z + packZ->platformOffsets()[i];
|
||||
// simple M*v //tr.T*v
|
||||
T x0, x1, x2;
|
||||
x0 = xTad[0];
|
||||
x1 = xTad[xDimCstride];
|
||||
x2 = xTad[2 * xDimCstride];
|
||||
zTad[0] = x0 * tr[0][0] + x1 * tr[1][0] + x2 * tr[2][0];
|
||||
zTad[zDimCstride] = x0 * tr[0][1] + x1 * tr[1][1] + x2 * tr[2][1];
|
||||
zTad[2 * zDimCstride] = x0 * tr[0][2] + x1 * tr[1][2] + x2 * tr[2][2];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_INLINE static void rgbYiq(NDArray* input, NDArray* output, const int dimC) {
|
||||
T arr[3][3] = {{(T)0.299, (T)0.59590059, (T)0.2115},
|
||||
{(T)0.587, (T)-0.27455667, (T)-0.52273617},
|
||||
{(T)0.114, (T)-0.32134392, (T)0.31119955}};
|
||||
return tripleTransformer<T>(input, output, dimC, arr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_INLINE static void yiqRgb(NDArray* input, NDArray* output, const int dimC) {
|
||||
// TODO: this operation does not use the clamp operation, so there is a possibility being out of range.
|
||||
// Justify that it will not be out of range for images data
|
||||
T arr[3][3] = {{(T)1, (T)1, (T)1},
|
||||
{(T)0.95598634, (T)-0.27201283, (T)-1.10674021},
|
||||
{(T)0.6208248, (T)-0.64720424, (T)1.70423049}};
|
||||
return tripleTransformer<T>(input, output, dimC, arr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_INLINE static void hsvRgb(NDArray* input, NDArray* output, const int dimC) {
|
||||
const int rank = input->rankOf();
|
||||
|
||||
const T* x = input->bufferAsT<T>();
|
||||
T* z = output->bufferAsT<T>();
|
||||
|
||||
if (dimC == rank - 1 && input->ews() == 1 && output->ews() == 1 && input->ordering() == 'c' &&
|
||||
output->ordering() == 'c') {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
sd::ops::helpers::hsvToRgb<T>(x[i], x[i + 1], x[i + 2], z[i], z[i + 1], z[i + 2]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input->lengthOf(), 3);
|
||||
} else {
|
||||
auto packX = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(), dimC);
|
||||
auto packZ = sd::ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(), dimC);
|
||||
|
||||
const sd::LongType numOfTads = packX->numberOfTads();
|
||||
const sd::LongType xDimCstride = input->stridesOf()[dimC];
|
||||
const sd::LongType zDimCstride = output->stridesOf()[dimC];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
const T* xTad = x + packX->platformOffsets()[i];
|
||||
T* zTad = z + packZ->platformOffsets()[i];
|
||||
sd::ops::helpers::hsvToRgb<T>(xTad[0], xTad[xDimCstride], xTad[2 * xDimCstride], zTad[0], zTad[zDimCstride],
|
||||
zTad[2 * zDimCstride]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_INLINE static void rgbHsv(NDArray* input, NDArray* output, const int dimC) {
|
||||
const int rank = input->rankOf();
|
||||
|
||||
const T* x = input->bufferAsT<T>();
|
||||
T* z = output->bufferAsT<T>();
|
||||
|
||||
if (dimC == rank - 1 && input->ews() == 1 && output->ews() == 1 && input->ordering() == 'c' &&
|
||||
output->ordering() == 'c') {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
sd::ops::helpers::rgbToHsv<T>(x[i], x[i + 1], x[i + 2], z[i], z[i + 1], z[i + 2]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input->lengthOf(), 3);
|
||||
} else {
|
||||
auto packX = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(), dimC);
|
||||
auto packZ = sd::ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(), dimC);
|
||||
|
||||
const sd::LongType numOfTads = packX->numberOfTads();
|
||||
const sd::LongType xDimCstride = input->stridesOf()[dimC];
|
||||
const sd::LongType zDimCstride = output->stridesOf()[dimC];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
const T* xTad = x + packX->platformOffsets()[i];
|
||||
T* zTad = z + packZ->platformOffsets()[i];
|
||||
sd::ops::helpers::rgbToHsv<T>(xTad[0], xTad[xDimCstride], xTad[2 * xDimCstride], zTad[0], zTad[zDimCstride],
|
||||
zTad[2 * zDimCstride]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_INLINE static void rgbYuv_(NDArray& input, NDArray& output, const int dimC) {
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
const int rank = input.rankOf();
|
||||
bool bSimple = (dimC == rank - 1 && 'c' == input.ordering() && 1 == input.ews() && 'c' == output.ordering() &&
|
||||
1 == output.ews());
|
||||
|
||||
if (bSimple) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
sd::ops::helpers::rgbYuv<T>(x[i], x[i + 1], x[i + 2], z[i], z[i + 1], z[i + 2]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input.lengthOf(), 3);
|
||||
return;
|
||||
}
|
||||
|
||||
auto packX = sd::ConstantTadHelper::getInstance().tadForDimensions(input.shapeInfo(), dimC);
|
||||
auto packZ = sd::ConstantTadHelper::getInstance().tadForDimensions(output.shapeInfo(), dimC);
|
||||
|
||||
const sd::LongType numOfTads = packX->numberOfTads();
|
||||
const sd::LongType xDimCstride = input.stridesOf()[dimC];
|
||||
const sd::LongType zDimCstride = output.stridesOf()[dimC];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
const T* xTad = x + packX->platformOffsets()[i];
|
||||
T* zTad = z + packZ->platformOffsets()[i];
|
||||
sd::ops::helpers::rgbYuv<T>(xTad[0], xTad[xDimCstride], xTad[2 * xDimCstride], zTad[0], zTad[zDimCstride],
|
||||
zTad[2 * zDimCstride]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SD_INLINE static void yuvRgb_(NDArray& input, NDArray& output, const int dimC) {
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
const int rank = input.rankOf();
|
||||
bool bSimple = (dimC == rank - 1 && 'c' == input.ordering() && 1 == input.ews() && 'c' == output.ordering() &&
|
||||
1 == output.ews());
|
||||
|
||||
if (bSimple) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
sd::ops::helpers::yuvRgb<T>(x[i], x[i + 1], x[i + 2], z[i], z[i + 1], z[i + 2]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, input.lengthOf(), 3);
|
||||
return;
|
||||
}
|
||||
|
||||
auto packX = sd::ConstantTadHelper::getInstance().tadForDimensions(input.shapeInfo(), dimC);
|
||||
auto packZ = sd::ConstantTadHelper::getInstance().tadForDimensions(output.shapeInfo(), dimC);
|
||||
|
||||
const sd::LongType numOfTads = packX->numberOfTads();
|
||||
const sd::LongType xDimCstride = input.stridesOf()[dimC];
|
||||
const sd::LongType zDimCstride = output.stridesOf()[dimC];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) {
|
||||
const T* xTad = x + packX->platformOffsets()[i];
|
||||
T* zTad = z + packZ->platformOffsets()[i];
|
||||
sd::ops::helpers::yuvRgb<T>(xTad[0], xTad[xDimCstride], xTad[2 * xDimCstride], zTad[0], zTad[zDimCstride],
|
||||
zTad[2 * zDimCstride]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
return;
|
||||
}
|
||||
|
||||
void transformRgbYuv(sd::LaunchContext* context, NDArray& input, NDArray& output, const int dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), rgbYuv_, (input, output, dimC), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
void transformYuvRgb(sd::LaunchContext* context, NDArray& input, NDArray& output, const int dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), yuvRgb_, (input, output, dimC), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
void transformHsvRgb(sd::LaunchContext* context, NDArray* input, NDArray* output, const int dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), hsvRgb, (input, output, dimC), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
void transformRgbHsv(sd::LaunchContext* context, NDArray* input, NDArray* output, const int dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), rgbHsv, (input, output, dimC), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
void transformYiqRgb(sd::LaunchContext* context, NDArray* input, NDArray* output, const int dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), yiqRgb, (input, output, dimC), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
void transformRgbYiq(sd::LaunchContext* context, NDArray* input, NDArray* output, const int dimC) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), rgbYiq, (input, output, dimC), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,70 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#include <ops/declarable/helpers/reductions.h>
|
||||
#include <system/selective_rendering.h>
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
void argMax_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions);
|
||||
|
||||
template <typename X, typename Z>
|
||||
void argMin_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions);
|
||||
|
||||
template <typename X, typename Z>
|
||||
void argAbsMax_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions);
|
||||
|
||||
template <typename X, typename Z>
|
||||
void argAbsMin_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void argMax(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
auto inputDType = input.dataType();
|
||||
auto outputDType = output.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(input.dataType(), output.dataType(), argMax_, (input, output, dimensions), SD_COMMON_TYPES,
|
||||
SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
void argMin(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
auto inputDType = input.dataType();
|
||||
auto outputDType = output.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(input.dataType(), output.dataType(), argMin_, (input, output, dimensions), SD_COMMON_TYPES,
|
||||
SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
void argAbsMax(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
auto inputDType = input.dataType();
|
||||
auto outputDType = output.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(input.dataType(), output.dataType(), argAbsMax_, (input, output, dimensions), SD_COMMON_TYPES,
|
||||
SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
void argAbsMin(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
auto inputDType = input.dataType();
|
||||
auto outputDType = output.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(input.dataType(), output.dataType(), argAbsMin_, (input, output, dimensions), SD_COMMON_TYPES,
|
||||
SD_INDEXING_TYPES);
|
||||
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,894 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#include <execution/ThreadPool.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopsCoordsHelper.h>
|
||||
#include <ops/declarable/helpers/reductions.h>
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <type_traits>
|
||||
#if 1
|
||||
#define LOG_CALLS(X)
|
||||
#else
|
||||
|
||||
#define LOG_CALLS(X) sd_printf("___%s_________%d+\n", __PRETTY_FUNCTION__, X);
|
||||
#endif
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
constexpr int threadingThreshold = 4096;
|
||||
template <typename X, typename Z, typename ReductionOp>
|
||||
static SD_INLINE void indexInnerReductionRank1(const X* buffer, X& current, Z& argCurrent,
|
||||
const sd::LongType& loopCount) {
|
||||
argCurrent = 0;
|
||||
current = buffer[0];
|
||||
LOG_CALLS(0)
|
||||
sd::LongType j_offset = 0;
|
||||
for (Z j = 0; j < loopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, buffer[j], j);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp>
|
||||
static SD_INLINE void indexInnerReductionRank1(const X* buffer, X& current, Z& argCurrent,
|
||||
const sd::LongType& loopCount, const sd::LongType& inner_stride) {
|
||||
argCurrent = 0;
|
||||
current = buffer[0];
|
||||
LOG_CALLS(0)
|
||||
sd::LongType j_offset = 0;
|
||||
for (Z j = 0; j < loopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, buffer[j_offset], j);
|
||||
j_offset += inner_stride;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, size_t constRank, bool LastIndexFaster = true>
|
||||
static SD_INLINE void indexInnerReductionConstRank(const X* buffer, X& current, Z& argCurrent,
|
||||
const sd::LongType* bases, const sd::LongType* strides,
|
||||
const sd::LongType outerLoopCount,
|
||||
const sd::LongType& innerLoopCount) {
|
||||
// skip 1 from the beginning or end depending the Order
|
||||
constexpr size_t updated_index = LastIndexFaster ? 0 : 1;
|
||||
constexpr size_t updated_rank = constRank - 1;
|
||||
sd::CoordsState<updated_rank - 1> cst;
|
||||
// we skip 1
|
||||
size_t offset =
|
||||
sd::init_coords<updated_rank, 0, LastIndexFaster>(cst, 0, bases + updated_index, strides + updated_index);
|
||||
Z startIndex = 0;
|
||||
argCurrent = 0;
|
||||
current = buffer[offset];
|
||||
LOG_CALLS(0)
|
||||
for (Z i = 0; i < outerLoopCount; i++) {
|
||||
const X* inner_buffer = &(buffer[offset]);
|
||||
// typename std::make_signed<Z>::type iArgMax = -1;
|
||||
for (Z j = 0; j < innerLoopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, inner_buffer[j], j + startIndex);
|
||||
}
|
||||
// we skip 1
|
||||
offset = sd::inc_coords<updated_rank, 0, LastIndexFaster>(cst, offset);
|
||||
startIndex += innerLoopCount;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, size_t constRank, bool LastIndexFaster = true>
|
||||
static SD_INLINE void indexInnerReductionConstRank(const X* buffer, X& current, Z& argCurrent,
|
||||
const sd::LongType* bases, const sd::LongType* strides,
|
||||
const sd::LongType outerLoopCount,
|
||||
const sd::LongType& innerLoopCount,
|
||||
const sd::LongType& inner_stride) {
|
||||
// skip 1 from the beginning or end depending the Order
|
||||
constexpr size_t updated_index = LastIndexFaster ? 0 : 1;
|
||||
constexpr size_t updated_rank = constRank - 1;
|
||||
sd::CoordsState<updated_rank - 1> cst;
|
||||
// we skip 1
|
||||
size_t offset =
|
||||
sd::init_coords<updated_rank, 0, LastIndexFaster>(cst, 0, bases + updated_index, strides + updated_index);
|
||||
Z startIndex = 0;
|
||||
argCurrent = 0;
|
||||
current = buffer[offset];
|
||||
LOG_CALLS(0)
|
||||
for (Z i = 0; i < outerLoopCount; i++) {
|
||||
const X* inner_buffer = &(buffer[offset]);
|
||||
for (Z j = 0; j < innerLoopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, *inner_buffer, j + startIndex);
|
||||
inner_buffer += inner_stride;
|
||||
}
|
||||
// we alreaddy skiped
|
||||
offset = sd::inc_coords<updated_rank, 0, LastIndexFaster>(cst, offset);
|
||||
startIndex += innerLoopCount;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, bool LastIndexFaster = true>
|
||||
static SD_INLINE void indexInnerReduction(const LongType& rank, const X* buffer, X& current, Z& argCurrent,
|
||||
const sd::LongType* bases, const sd::LongType* strides,
|
||||
const sd::LongType& outerLoopStart, const sd::LongType& outerLoopStop,
|
||||
const sd::LongType& innerLoopCount) {
|
||||
size_t offset = 0;
|
||||
sd::LongType outerLoopCount = outerLoopStop - outerLoopStart;
|
||||
sd::LongType coords[SD_MAX_RANK] = {};
|
||||
sd::LongType* ptr_coords = (sd::LongType*)&coords;
|
||||
if (outerLoopStart > 0) {
|
||||
INDEX2COORDS(outerLoopStart, rank - 1, bases, ptr_coords);
|
||||
COORDS2INDEX(rank, strides, ptr_coords, offset);
|
||||
}
|
||||
Z startIndex = outerLoopStart * innerLoopCount;
|
||||
argCurrent = startIndex;
|
||||
current = buffer[offset];
|
||||
LOG_CALLS(0)
|
||||
for (Z i = 0; i < outerLoopCount; i++) {
|
||||
const X* inner_buffer = &(buffer[offset]);
|
||||
for (Z j = 0; j < innerLoopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, inner_buffer[j], j + startIndex);
|
||||
}
|
||||
offset = inc_coords<true>(bases, strides, ptr_coords, offset, rank, 1);
|
||||
startIndex += innerLoopCount;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, bool LastIndexFaster = true>
|
||||
static SD_INLINE void indexInnerReduction(const int& rank, const X* buffer, X& current, Z& argCurrent,
|
||||
const sd::LongType* bases, const sd::LongType* strides,
|
||||
const sd::LongType& outerLoopStart, const sd::LongType& outerLoopStop,
|
||||
const sd::LongType& innerLoopCount, const sd::LongType& inner_stride) {
|
||||
size_t offset = 0;
|
||||
sd::LongType outerLoopCount = outerLoopStop - outerLoopStart;
|
||||
sd::LongType coords[SD_MAX_RANK] = {};
|
||||
sd::LongType* ptr_coords = (sd::LongType*)&coords;
|
||||
if (outerLoopStart > 0) {
|
||||
INDEX2COORDS(outerLoopStart, rank - 1, bases, ptr_coords);
|
||||
COORDS2INDEX(rank, strides, ptr_coords, offset);
|
||||
}
|
||||
Z startIndex = outerLoopStart * innerLoopCount;
|
||||
argCurrent = startIndex;
|
||||
current = buffer[offset];
|
||||
LOG_CALLS(0)
|
||||
for (Z i = 0; i < outerLoopCount; i++) {
|
||||
const X* inner_buffer = &(buffer[offset]);
|
||||
for (Z j = 0; j < innerLoopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, inner_buffer[j * inner_stride], startIndex + j);
|
||||
}
|
||||
offset = inc_coords<true>(bases, strides, ptr_coords, offset, rank, 1);
|
||||
startIndex += innerLoopCount;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp>
|
||||
static SD_INLINE void indexInnerReductionRank1Block4WithMerge(const X* buffer, X& current, Z& argCurrent,
|
||||
const sd::LongType& loopCount) {
|
||||
argCurrent = 0;
|
||||
current = buffer[0];
|
||||
LOG_CALLS(0)
|
||||
sd::LongType loopCount4 = loopCount / 4;
|
||||
sd::LongType loopCountEnd = loopCount4 + (loopCount & 3);
|
||||
const X* buffer1 = buffer + 1 * loopCount4;
|
||||
const X* buffer2 = buffer1 + 1 * loopCount4;
|
||||
const X* buffer3 = buffer2 + 1 * loopCount4;
|
||||
X current1 = *buffer1;
|
||||
X current2 = *buffer2;
|
||||
X current3 = *buffer3;
|
||||
Z argCurrent1 = 0;
|
||||
Z argCurrent2 = 0;
|
||||
Z argCurrent3 = 0;
|
||||
for (Z j = 0; j < loopCount4; j++) {
|
||||
ReductionOp::update(current, argCurrent, buffer[j], j);
|
||||
ReductionOp::update(current1, argCurrent1, buffer1[j], j);
|
||||
ReductionOp::update(current2, argCurrent2, buffer2[j], j);
|
||||
ReductionOp::update(current3, argCurrent3, buffer3[j], j);
|
||||
}
|
||||
// tail
|
||||
for (Z j = loopCount4; j < loopCountEnd; j++) {
|
||||
ReductionOp::update(current3, argCurrent3, buffer3[j], j);
|
||||
}
|
||||
// merge
|
||||
argCurrent1 += loopCount4;
|
||||
argCurrent2 += 2 * loopCount4;
|
||||
argCurrent3 += 3 * loopCount4;
|
||||
ReductionOp::update(current, argCurrent, current1, argCurrent1);
|
||||
ReductionOp::update(current, argCurrent, current2, argCurrent2);
|
||||
ReductionOp::update(current, argCurrent, current3, argCurrent3);
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp>
|
||||
static SD_INLINE void indexInnerReductionRank1Block4WithMerge(const X* buffer, X& current, Z& argCurrent,
|
||||
const sd::LongType& loopCount,
|
||||
const sd::LongType& inner_stride) {
|
||||
argCurrent = 0;
|
||||
current = buffer[0];
|
||||
LOG_CALLS(0)
|
||||
sd::LongType loopCount4 = loopCount / 4;
|
||||
sd::LongType loopCountEnd = loopCount4 + (loopCount & 3);
|
||||
const X* buffer1 = buffer + inner_stride * loopCount4;
|
||||
const X* buffer2 = buffer1 + inner_stride * loopCount4;
|
||||
const X* buffer3 = buffer2 + inner_stride * loopCount4;
|
||||
X current1 = *buffer1;
|
||||
X current2 = *buffer2;
|
||||
X current3 = *buffer3;
|
||||
Z argCurrent1 = 0;
|
||||
Z argCurrent2 = 0;
|
||||
Z argCurrent3 = 0;
|
||||
sd::LongType j_offset = 0;
|
||||
for (Z j = 0; j < loopCount4; j++) {
|
||||
ReductionOp::update(current, argCurrent, buffer[j_offset], j);
|
||||
ReductionOp::update(current1, argCurrent1, buffer1[j_offset], j);
|
||||
ReductionOp::update(current2, argCurrent2, buffer2[j_offset], j);
|
||||
ReductionOp::update(current3, argCurrent3, buffer3[j_offset], j);
|
||||
j_offset += inner_stride;
|
||||
}
|
||||
// tail
|
||||
for (Z j = loopCount4; j < loopCountEnd; j++) {
|
||||
ReductionOp::update(current3, argCurrent3, buffer3[j_offset], j);
|
||||
j_offset += inner_stride;
|
||||
}
|
||||
// merge
|
||||
argCurrent1 += loopCount4;
|
||||
argCurrent2 += 2 * loopCount4;
|
||||
argCurrent3 += 3 * loopCount4;
|
||||
ReductionOp::update(current, argCurrent, current1, argCurrent1);
|
||||
ReductionOp::update(current, argCurrent, current2, argCurrent2);
|
||||
ReductionOp::update(current, argCurrent, current3, argCurrent3);
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp>
|
||||
static SD_INLINE void indexInnerReductionRank1Block4(const X* buffer, const X* buffer1, const X* buffer2,
|
||||
const X* buffer3, Z* output, Z* output1, Z* output2, Z* output3,
|
||||
const sd::LongType& loopCount) {
|
||||
LOG_CALLS(0)
|
||||
Z argCurrent = 0;
|
||||
Z argCurrent1 = 0;
|
||||
Z argCurrent2 = 0;
|
||||
Z argCurrent3 = 0;
|
||||
X current = buffer[0];
|
||||
X current1 = buffer1[0];
|
||||
X current2 = buffer2[0];
|
||||
X current3 = buffer3[0];
|
||||
for (Z j = 0; j < loopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, buffer[j], j);
|
||||
ReductionOp::update(current1, argCurrent1, buffer1[j], j);
|
||||
ReductionOp::update(current2, argCurrent2, buffer2[j], j);
|
||||
ReductionOp::update(current3, argCurrent3, buffer3[j], j);
|
||||
}
|
||||
*output = argCurrent;
|
||||
*output1 = argCurrent1;
|
||||
*output2 = argCurrent2;
|
||||
*output3 = argCurrent3;
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp>
|
||||
static SD_INLINE void indexInnerReductionRank1Block4(const X* buffer, const X* buffer1, const X* buffer2,
|
||||
const X* buffer3, Z* output, Z* output1, Z* output2, Z* output3,
|
||||
const sd::LongType& loopCount, const sd::LongType& inner_stride) {
|
||||
LOG_CALLS(0)
|
||||
Z argCurrent = 0;
|
||||
Z argCurrent1 = 0;
|
||||
Z argCurrent2 = 0;
|
||||
Z argCurrent3 = 0;
|
||||
X current = buffer[0];
|
||||
X current1 = buffer1[0];
|
||||
X current2 = buffer2[0];
|
||||
X current3 = buffer3[0];
|
||||
sd::LongType j_offset = 0;
|
||||
for (Z j = 0; j < loopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, buffer[j_offset], j);
|
||||
ReductionOp::update(current1, argCurrent1, buffer1[j_offset], j);
|
||||
ReductionOp::update(current2, argCurrent2, buffer2[j_offset], j);
|
||||
ReductionOp::update(current3, argCurrent3, buffer3[j_offset], j);
|
||||
j_offset += inner_stride;
|
||||
}
|
||||
*output = argCurrent;
|
||||
*output1 = argCurrent1;
|
||||
*output2 = argCurrent2;
|
||||
*output3 = argCurrent3;
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, size_t constRank, bool LastIndexFaster = true>
|
||||
static SD_INLINE void indexInnerReductionConstRankBlock4(const X* buffer, const X* buffer1, const X* buffer2,
|
||||
const X* buffer3, Z* output, Z* output1, Z* output2,
|
||||
Z* output3, const sd::LongType* bases,
|
||||
const sd::LongType* strides,
|
||||
const sd::LongType& outerLoopCount,
|
||||
const sd::LongType& innerLoopCount) {
|
||||
LOG_CALLS(0)
|
||||
// skip 1 from the beginning or end depending the Order
|
||||
constexpr size_t updated_index = LastIndexFaster ? 0 : 1;
|
||||
constexpr size_t updated_rank = constRank - 1;
|
||||
sd::CoordsState<updated_rank - 1> cst;
|
||||
// we skip 1
|
||||
size_t offset =
|
||||
sd::init_coords<updated_rank, 0, LastIndexFaster>(cst, 0, bases + updated_index, strides + updated_index);
|
||||
Z startIndex = 0;
|
||||
Z argCurrent = 0;
|
||||
Z argCurrent1 = 0;
|
||||
Z argCurrent2 = 0;
|
||||
Z argCurrent3 = 0;
|
||||
X current = buffer[0];
|
||||
X current1 = buffer1[0];
|
||||
X current2 = buffer2[0];
|
||||
X current3 = buffer3[0];
|
||||
// LOG_CALLS(0)
|
||||
for (Z i = 0; i < outerLoopCount; i++) {
|
||||
const X* inner_buffer = &(buffer[offset]);
|
||||
const X* inner_buffer1 = &(buffer1[offset]);
|
||||
const X* inner_buffer2 = &(buffer2[offset]);
|
||||
const X* inner_buffer3 = &(buffer3[offset]);
|
||||
// typename std::make_signed<Z>::type iArgMax = -1;
|
||||
for (Z j = 0; j < innerLoopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, inner_buffer[j], j + startIndex);
|
||||
ReductionOp::update(current1, argCurrent1, inner_buffer1[j], j + startIndex);
|
||||
ReductionOp::update(current2, argCurrent2, inner_buffer2[j], j + startIndex);
|
||||
ReductionOp::update(current3, argCurrent3, inner_buffer3[j], j + startIndex);
|
||||
}
|
||||
// we skip 1
|
||||
offset = sd::inc_coords<updated_rank, 0, LastIndexFaster>(cst, offset);
|
||||
startIndex += innerLoopCount;
|
||||
}
|
||||
*output = argCurrent;
|
||||
*output1 = argCurrent1;
|
||||
*output2 = argCurrent2;
|
||||
*output3 = argCurrent3;
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, size_t constRank, bool LastIndexFaster = true>
|
||||
static SD_INLINE void indexInnerReductionConstRankBlock4(
|
||||
const X* buffer, const X* buffer1, const X* buffer2, const X* buffer3, Z* output, Z* output1, Z* output2,
|
||||
Z* output3, const sd::LongType* bases, const sd::LongType* strides, const sd::LongType& outerLoopCount,
|
||||
const sd::LongType& innerLoopCount, const sd::LongType& inner_stride) {
|
||||
LOG_CALLS(0)
|
||||
// skip 1 from the beginning or end depending the Order
|
||||
constexpr size_t updated_index = LastIndexFaster ? 0 : 1;
|
||||
constexpr size_t updated_rank = constRank - 1;
|
||||
sd::CoordsState<updated_rank - 1> cst;
|
||||
// we skip 1
|
||||
size_t offset =
|
||||
sd::init_coords<updated_rank, 0, LastIndexFaster>(cst, 0, bases + updated_index, strides + updated_index);
|
||||
Z startIndex = 0;
|
||||
Z argCurrent = 0;
|
||||
Z argCurrent1 = 0;
|
||||
Z argCurrent2 = 0;
|
||||
Z argCurrent3 = 0;
|
||||
X current = buffer[0];
|
||||
X current1 = buffer1[0];
|
||||
X current2 = buffer2[0];
|
||||
X current3 = buffer3[0];
|
||||
// LOG_CALLS(0)
|
||||
for (Z i = 0; i < outerLoopCount; i++) {
|
||||
const X* inner_buffer = &(buffer[offset]);
|
||||
const X* inner_buffer1 = &(buffer1[offset]);
|
||||
const X* inner_buffer2 = &(buffer2[offset]);
|
||||
const X* inner_buffer3 = &(buffer3[offset]);
|
||||
// typename std::make_signed<Z>::type iArgMax = -1;
|
||||
sd::LongType inner_offset = 0;
|
||||
for (Z j = 0; j < innerLoopCount; j++) {
|
||||
ReductionOp::update(current, argCurrent, inner_buffer[inner_offset], j + startIndex);
|
||||
ReductionOp::update(current1, argCurrent1, inner_buffer1[inner_offset], j + startIndex);
|
||||
ReductionOp::update(current2, argCurrent2, inner_buffer2[inner_offset], j + startIndex);
|
||||
ReductionOp::update(current3, argCurrent3, inner_buffer3[inner_offset], j + startIndex);
|
||||
inner_offset += inner_stride;
|
||||
}
|
||||
// we skip 1
|
||||
offset = sd::inc_coords<updated_rank, 0, LastIndexFaster>(cst, offset);
|
||||
startIndex += innerLoopCount;
|
||||
}
|
||||
*output = argCurrent;
|
||||
*output1 = argCurrent1;
|
||||
*output2 = argCurrent2;
|
||||
*output3 = argCurrent3;
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, bool LastIndexFaster = true>
|
||||
static void argIndexCase1Scalar(const int& second_rank, const sd::LongType* inner_bases,
|
||||
const sd::LongType* inner_strides, const X* bufferX, Z* outputZ) {
|
||||
sd::LongType inner_total;
|
||||
sd::LongType inner_last = 0;
|
||||
int maxThreads = sd::Environment::getInstance().maxMasterThreads();
|
||||
if (second_rank == 1) {
|
||||
inner_total = inner_bases[0];
|
||||
if (inner_total < threadingThreshold) {
|
||||
maxThreads = 1;
|
||||
}
|
||||
} else {
|
||||
inner_total = getLength<LastIndexFaster>(inner_bases, second_rank, 1, inner_last);
|
||||
if (inner_total * inner_last < threadingThreshold) {
|
||||
maxThreads = 1;
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<X[]> maxValues(new X[maxThreads]);
|
||||
std::unique_ptr<Z[]> maxIndices(new Z[maxThreads]);
|
||||
X* ptrMaxValues = maxValues.get();
|
||||
Z* ptrMaxIndices = maxIndices.get();
|
||||
auto func = [ptrMaxValues, ptrMaxIndices, inner_last, second_rank, inner_bases, inner_strides, bufferX](
|
||||
uint64_t thread_id, int64_t start, int64_t stop, int64_t increment) -> void {
|
||||
// LOG_CALLS(0)
|
||||
const sd::LongType inner_stride = LastIndexFaster ? inner_strides[second_rank - 1] : inner_strides[0];
|
||||
Z argCurrent;
|
||||
X current;
|
||||
if (second_rank == 1) {
|
||||
const sd::LongType loopTotal = stop - start;
|
||||
if (inner_stride == 1) {
|
||||
indexInnerReductionRank1Block4WithMerge<X, Z, ReductionOp>(&(bufferX[start]), current, argCurrent, loopTotal);
|
||||
} else {
|
||||
indexInnerReductionRank1Block4WithMerge<X, Z, ReductionOp>(&(bufferX[start * inner_stride]), current,
|
||||
argCurrent, loopTotal, inner_stride);
|
||||
}
|
||||
ptrMaxIndices[thread_id] = argCurrent + start;
|
||||
} else {
|
||||
if (inner_stride == 1) {
|
||||
indexInnerReduction<X, Z, ReductionOp, LastIndexFaster>(second_rank, bufferX, current, argCurrent, inner_bases,
|
||||
inner_strides, start, stop, inner_last, inner_stride);
|
||||
} else {
|
||||
indexInnerReduction<X, Z, ReductionOp, LastIndexFaster>(second_rank, bufferX, current, argCurrent, inner_bases,
|
||||
inner_strides, start, stop, inner_last, inner_stride);
|
||||
}
|
||||
ptrMaxIndices[thread_id] = argCurrent;
|
||||
}
|
||||
ptrMaxValues[thread_id] = current;
|
||||
};
|
||||
#if 0
|
||||
int Count = 0;
|
||||
func(0, 0, inner_total, 1);
|
||||
#else
|
||||
int Count = samediff::Threads::parallel_tad(func, 0, inner_total, 1, maxThreads);
|
||||
#endif
|
||||
Z arg = 0;
|
||||
X current = ptrMaxValues[0];
|
||||
|
||||
for (Z i = 1; i < Count; i++) {
|
||||
ReductionOp::update(current, arg, ptrMaxValues[i], i);
|
||||
}
|
||||
|
||||
*outputZ = ptrMaxIndices[arg];
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, typename Movement, bool LastIndexFaster = true>
|
||||
static void argReductionInnerCases(Movement& movement, sd::LongType loopTotal, const int& second_rank,
|
||||
const sd::LongType* inner_bases, const sd::LongType* inner_strides, const X* bufferX,
|
||||
Z* outputZ) {
|
||||
sd::LongType inner_stride = true /*LastIndexFaster*/ ? inner_strides[second_rank - 1] : inner_strides[0];
|
||||
|
||||
sd::LongType loopTotal_K = loopTotal / 4;
|
||||
sd::LongType loopTotal_Tail = loopTotal & 3;
|
||||
if (inner_stride == 1) {
|
||||
if (second_rank == 1) {
|
||||
LOG_CALLS(0)
|
||||
sd::LongType inner_total = getLength<true>(inner_bases, second_rank);
|
||||
for (sd::LongType i = 0; i < loopTotal_K; i++) {
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
Z* output0 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer1 = &(bufferX[movement.First()]);
|
||||
Z* output1 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer2 = &(bufferX[movement.First()]);
|
||||
Z* output2 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer3 = &(bufferX[movement.First()]);
|
||||
Z* output3 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
indexInnerReductionRank1Block4<X, Z, ReductionOp>(buffer0, buffer1, buffer2, buffer3, output0, output1, output2,
|
||||
output3, inner_total);
|
||||
}
|
||||
if (inner_total >= 2048) {
|
||||
for (sd::LongType i = 0; i < loopTotal_Tail; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReductionRank1Block4WithMerge<X, Z, ReductionOp>(buffer0, current, outputZ[movement.Second()],
|
||||
inner_total);
|
||||
movement.increment();
|
||||
}
|
||||
} else {
|
||||
for (sd::LongType i = 0; i < loopTotal_Tail; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReductionRank1<X, Z, ReductionOp>(buffer0, current, outputZ[movement.Second()], inner_total);
|
||||
movement.increment();
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
sd::LongType inner_last;
|
||||
sd::LongType inner_loop = getLength<true>(inner_bases, second_rank, 1, inner_last);
|
||||
if (second_rank == 2) {
|
||||
LOG_CALLS(1)
|
||||
for (sd::LongType i = 0; i < loopTotal_K; i++) {
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
Z* output0 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer1 = &(bufferX[movement.First()]);
|
||||
Z* output1 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer2 = &(bufferX[movement.First()]);
|
||||
Z* output2 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer3 = &(bufferX[movement.First()]);
|
||||
Z* output3 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
indexInnerReductionConstRankBlock4<X, Z, ReductionOp, 2>(buffer0, buffer1, buffer2, buffer3, output0, output1,
|
||||
output2, output3, inner_bases, inner_strides,
|
||||
inner_loop, inner_last);
|
||||
}
|
||||
for (sd::LongType i = 0; i < loopTotal_Tail; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReductionConstRank<X, Z, ReductionOp, 2>(buffer0, current, outputZ[movement.Second()], inner_bases,
|
||||
inner_strides, inner_loop, inner_last);
|
||||
movement.increment();
|
||||
}
|
||||
|
||||
} else if (second_rank == 3) {
|
||||
LOG_CALLS(2)
|
||||
for (sd::LongType i = 0; i < loopTotal_K; i++) {
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
Z* output0 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer1 = &(bufferX[movement.First()]);
|
||||
Z* output1 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer2 = &(bufferX[movement.First()]);
|
||||
Z* output2 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer3 = &(bufferX[movement.First()]);
|
||||
Z* output3 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
indexInnerReductionConstRankBlock4<X, Z, ReductionOp, 3>(buffer0, buffer1, buffer2, buffer3, output0, output1,
|
||||
output2, output3, inner_bases, inner_strides,
|
||||
inner_loop, inner_last);
|
||||
}
|
||||
for (sd::LongType i = 0; i < loopTotal_Tail; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReductionConstRank<X, Z, ReductionOp, 3>(buffer0, current, outputZ[movement.Second()], inner_bases,
|
||||
inner_strides, inner_loop, inner_last);
|
||||
movement.increment();
|
||||
}
|
||||
|
||||
} else {
|
||||
LOG_CALLS(3)
|
||||
for (sd::LongType i = 0; i < loopTotal; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReduction<X, Z, ReductionOp>(second_rank, buffer0, current, outputZ[movement.Second()], inner_bases,
|
||||
inner_strides, 0, inner_loop, inner_last);
|
||||
movement.increment();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
if (second_rank == 1) {
|
||||
LOG_CALLS(10)
|
||||
sd::LongType inner_total = getLength<true>(inner_bases, second_rank);
|
||||
for (sd::LongType i = 0; i < loopTotal_K; i++) {
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
Z* output0 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer1 = &(bufferX[movement.First()]);
|
||||
Z* output1 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer2 = &(bufferX[movement.First()]);
|
||||
Z* output2 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer3 = &(bufferX[movement.First()]);
|
||||
Z* output3 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
indexInnerReductionRank1Block4<X, Z, ReductionOp>(buffer0, buffer1, buffer2, buffer3, output0, output1, output2,
|
||||
output3, inner_total, inner_stride);
|
||||
}
|
||||
if (inner_total >= 2048) {
|
||||
for (sd::LongType i = 0; i < loopTotal_Tail; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReductionRank1Block4WithMerge<X, Z, ReductionOp>(buffer0, current, outputZ[movement.Second()],
|
||||
inner_total, inner_stride);
|
||||
movement.increment();
|
||||
}
|
||||
} else {
|
||||
for (sd::LongType i = 0; i < loopTotal_Tail; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReductionRank1<X, Z, ReductionOp>(buffer0, current, outputZ[movement.Second()], inner_total,
|
||||
inner_stride);
|
||||
movement.increment();
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
sd::LongType inner_last;
|
||||
sd::LongType inner_loop = getLength<true>(inner_bases, second_rank, 1, inner_last);
|
||||
if (second_rank == 2) {
|
||||
LOG_CALLS(11)
|
||||
for (sd::LongType i = 0; i < loopTotal_K; i++) {
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
Z* output0 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer1 = &(bufferX[movement.First()]);
|
||||
Z* output1 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer2 = &(bufferX[movement.First()]);
|
||||
Z* output2 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer3 = &(bufferX[movement.First()]);
|
||||
Z* output3 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
indexInnerReductionConstRankBlock4<X, Z, ReductionOp, 2>(buffer0, buffer1, buffer2, buffer3, output0, output1,
|
||||
output2, output3, inner_bases, inner_strides,
|
||||
inner_loop, inner_last, inner_stride);
|
||||
}
|
||||
for (sd::LongType i = 0; i < loopTotal_Tail; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReductionConstRank<X, Z, ReductionOp, 2>(buffer0, current, outputZ[movement.Second()], inner_bases,
|
||||
inner_strides, inner_loop, inner_last, inner_stride);
|
||||
movement.increment();
|
||||
}
|
||||
|
||||
} else if (second_rank == 3) {
|
||||
LOG_CALLS(12)
|
||||
for (sd::LongType i = 0; i < loopTotal_K; i++) {
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
Z* output0 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer1 = &(bufferX[movement.First()]);
|
||||
Z* output1 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer2 = &(bufferX[movement.First()]);
|
||||
Z* output2 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
const X* buffer3 = &(bufferX[movement.First()]);
|
||||
Z* output3 = &(outputZ[movement.Second()]);
|
||||
movement.increment();
|
||||
indexInnerReductionConstRankBlock4<X, Z, ReductionOp, 3>(buffer0, buffer1, buffer2, buffer3, output0, output1,
|
||||
output2, output3, inner_bases, inner_strides,
|
||||
inner_loop, inner_last, inner_stride);
|
||||
}
|
||||
for (sd::LongType i = 0; i < loopTotal_Tail; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReductionConstRank<X, Z, ReductionOp, 3>(buffer0, current, outputZ[movement.Second()], inner_bases,
|
||||
inner_strides, inner_loop, inner_last, inner_stride);
|
||||
movement.increment();
|
||||
}
|
||||
|
||||
} else {
|
||||
LOG_CALLS(13)
|
||||
for (sd::LongType i = 0; i < loopTotal; i++) {
|
||||
X current;
|
||||
const X* buffer0 = &(bufferX[movement.First()]);
|
||||
indexInnerReduction<X, Z, ReductionOp>(second_rank, buffer0, current, outputZ[movement.Second()], inner_bases,
|
||||
inner_strides, 0, inner_loop, inner_last, inner_stride);
|
||||
movement.increment();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp, bool LastIndexFaster = true>
|
||||
static void argIndexCaseNonScalar(const int& first_rank, const int& output_rank, bool squashed, const int& second_rank,
|
||||
const sd::LongType*& outer_bases, const sd::LongType* outer_strides,
|
||||
const sd::LongType* output_strides, const sd::LongType& output_stride,
|
||||
const sd::LongType*& inner_bases, const sd::LongType* inner_strides, const X* bufferX,
|
||||
Z* outputZ) {
|
||||
sd::LongType total = getLength<LastIndexFaster>(outer_bases, first_rank);
|
||||
sd::LongType inner_stride = true /*LastIndexFaster*/ ? inner_strides[second_rank - 1] : inner_strides[0];
|
||||
sd::LongType outer_stride = LastIndexFaster ? outer_strides[second_rank - 1] : outer_strides[0];
|
||||
auto func = [first_rank, output_rank, squashed, outer_bases, outer_strides, output_strides, output_stride,
|
||||
second_rank, inner_bases, inner_strides, bufferX,
|
||||
outputZ](uint64_t thread_id, int64_t start, int64_t stop, int64_t increment) -> void {
|
||||
sd::LongType loopTotal = stop - start;
|
||||
sd::LongType stride = LastIndexFaster ? outer_strides[first_rank - 1] : outer_strides[0];
|
||||
if (first_rank == 1) {
|
||||
if (stride == 1) {
|
||||
ZipGenericCoordsRank1Stride1 movement;
|
||||
movement.init(nullptr, nullptr, nullptr, 0, start);
|
||||
argReductionInnerCases<X, Z, ReductionOp>(movement, loopTotal, second_rank, inner_bases, inner_strides, bufferX,
|
||||
outputZ);
|
||||
} else {
|
||||
ZipGenericCoordsRank1BothStrideN movement;
|
||||
movement.init(nullptr, &stride, &output_stride, 0, start);
|
||||
argReductionInnerCases<X, Z, ReductionOp>(movement, loopTotal, second_rank, inner_bases, inner_strides, bufferX,
|
||||
outputZ);
|
||||
}
|
||||
|
||||
} else if (squashed && first_rank <= output_rank) {
|
||||
if (first_rank == 2) {
|
||||
if (output_stride == 1) {
|
||||
ZipGenericCoordsConstMovementSecondStride1<2, LastIndexFaster> movement;
|
||||
movement.init(outer_bases, outer_strides, nullptr, first_rank, start);
|
||||
argReductionInnerCases<X, Z, ReductionOp>(movement, loopTotal, second_rank, inner_bases, inner_strides,
|
||||
bufferX, outputZ);
|
||||
|
||||
} else {
|
||||
ZipGenericCoordsConstMovementSecondStrideN<2, LastIndexFaster> movement;
|
||||
movement.init(outer_bases, outer_strides, &output_stride, first_rank, start);
|
||||
argReductionInnerCases<X, Z, ReductionOp>(movement, loopTotal, second_rank, inner_bases, inner_strides,
|
||||
bufferX, outputZ);
|
||||
}
|
||||
} else if (first_rank == 3) {
|
||||
if (output_stride == 1) {
|
||||
ZipGenericCoordsConstMovementSecondStride1<3, LastIndexFaster> movement;
|
||||
movement.init(outer_bases, outer_strides, nullptr, first_rank, start);
|
||||
argReductionInnerCases<X, Z, ReductionOp>(movement, loopTotal, second_rank, inner_bases, inner_strides,
|
||||
bufferX, outputZ);
|
||||
|
||||
} else {
|
||||
ZipGenericCoordsConstMovementSecondStrideN<3, LastIndexFaster> movement;
|
||||
movement.init(outer_bases, outer_strides, &output_stride, first_rank, start);
|
||||
argReductionInnerCases<X, Z, ReductionOp>(movement, loopTotal, second_rank, inner_bases, inner_strides,
|
||||
bufferX, outputZ);
|
||||
}
|
||||
} else {
|
||||
ZipGenericCoordsMovementSecondStrideN<LastIndexFaster> movement;
|
||||
movement.init(outer_bases, outer_strides, &output_stride, first_rank, start);
|
||||
|
||||
argReductionInnerCases<X, Z, ReductionOp>(movement, loopTotal, second_rank, inner_bases, inner_strides, bufferX,
|
||||
outputZ);
|
||||
}
|
||||
|
||||
} else {
|
||||
ZipGenericCoordsMovement<LastIndexFaster> movement;
|
||||
movement.init(outer_bases, outer_strides, output_strides, first_rank, start);
|
||||
|
||||
argReductionInnerCases<X, Z, ReductionOp>(movement, loopTotal, second_rank, inner_bases, inner_strides, bufferX,
|
||||
outputZ);
|
||||
}
|
||||
};
|
||||
#if 0
|
||||
func(0, 0, total, 1);
|
||||
#else
|
||||
//
|
||||
uint32_t numThreads = sd::Environment::getInstance().maxMasterThreads();
|
||||
sd::LongType inner_total = getLength<true>(inner_bases, second_rank);
|
||||
if (total * inner_total <= threadingThreshold) {
|
||||
numThreads = 1;
|
||||
} else {
|
||||
if (inner_stride > outer_stride && total <= 256) {
|
||||
auto desired = total > 4 ? (total / 4) : 1;
|
||||
numThreads = numThreads > desired ? desired : numThreads;
|
||||
}
|
||||
}
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, total, 1, numThreads);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename X, typename Z, typename ReductionOp>
|
||||
SD_LIB_HIDDEN void argIndex_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
char input_order = input.ordering();
|
||||
bool try_squash_outer = (input_order == output.ordering()) && output.ews() != 0;
|
||||
const sd::LongType* input_shapeInfo = input.shapeInfo();
|
||||
const sd::LongType* output_shapeInfo = output.shapeInfo();
|
||||
const sd::LongType rank = input_shapeInfo[0];
|
||||
const sd::LongType* input_bases = &(input_shapeInfo[1]);
|
||||
const sd::LongType* input_strides = &(input_shapeInfo[rank + 1]);
|
||||
const sd::LongType output_rank = output_shapeInfo[0];
|
||||
const sd::LongType* output_strides = &(output_shapeInfo[output_rank + 1]);
|
||||
sd::LongType new_bases[SD_MAX_RANK];
|
||||
sd::LongType new_strides[SD_MAX_RANK];
|
||||
sd::LongType first_begin, first_end, second_begin, second_end;
|
||||
// rePartition into two parts based on the selection
|
||||
rePartition(input_order, dimensions, rank, input_bases, input_strides, new_bases, new_strides, first_begin, first_end,
|
||||
second_begin, second_end, try_squash_outer, input_order == 'c');
|
||||
int first_rank = first_end - first_begin; // the first rank can be 0 for scalar cases
|
||||
int second_rank = second_end - second_begin;
|
||||
auto bufferX = input.bufferAsT<X>();
|
||||
auto outputZ = output.bufferAsT<Z>();
|
||||
const sd::LongType* outer_bases = &(new_bases[first_begin]);
|
||||
const sd::LongType* outer_strides = &(new_strides[first_begin]);
|
||||
const sd::LongType* inner_bases = &(new_bases[second_begin]);
|
||||
const sd::LongType* inner_strides = &(new_strides[second_begin]);
|
||||
const sd::LongType output_stride = output.ordering() == 'c' ? output_strides[output_rank - 1] : output_strides[0];
|
||||
if (input_order == 'c') {
|
||||
if (first_rank == 0) {
|
||||
argIndexCase1Scalar<X, Z, ReductionOp>(second_rank, inner_bases, inner_strides, bufferX, outputZ);
|
||||
} else {
|
||||
argIndexCaseNonScalar<X, Z, ReductionOp>(first_rank, output_rank, try_squash_outer, second_rank, outer_bases,
|
||||
outer_strides, output_strides, output_stride, inner_bases, inner_strides,
|
||||
bufferX, outputZ);
|
||||
}
|
||||
} else {
|
||||
if (first_rank == 0) {
|
||||
LOG_CALLS(0);
|
||||
if (second_rank == 1) {
|
||||
argIndexCase1Scalar<X, Z, ReductionOp, false>(second_rank, inner_bases, inner_strides, bufferX, outputZ);
|
||||
} else {
|
||||
argIndexCase1Scalar<X, Z, ReductionOp, true>(second_rank, inner_bases, inner_strides, bufferX, outputZ);
|
||||
}
|
||||
} else {
|
||||
LOG_CALLS(1);
|
||||
argIndexCaseNonScalar<X, Z, ReductionOp, false>(first_rank, output_rank, try_squash_outer, second_rank,
|
||||
outer_bases, outer_strides, output_strides, output_stride,
|
||||
inner_bases, inner_strides, bufferX, outputZ);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
struct IndexMax {
|
||||
static SD_INLINE void update(X& current, Z& currentIndex, const X& candidate, const Z& candidateIndex) {
|
||||
if (candidate > current) {
|
||||
current = candidate;
|
||||
currentIndex = candidateIndex;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename X, typename Z>
|
||||
struct IndexMin {
|
||||
static SD_INLINE void update(X& current, Z& currentIndex, const X& candidate, const Z& candidateIndex) {
|
||||
if (candidate < current) {
|
||||
current = candidate;
|
||||
currentIndex = candidateIndex;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename X, typename Z>
|
||||
struct IndexAbsMax {
|
||||
static SD_INLINE void update(X& current, Z& currentIndex, const X& candidate, const Z& candidateIndex) {
|
||||
auto absCandidate = sd::math::sd_abs<X,X>(candidate);
|
||||
if (absCandidate > current) {
|
||||
current = absCandidate;
|
||||
currentIndex = candidateIndex;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename X, typename Z>
|
||||
struct IndexAbsMin {
|
||||
static SD_INLINE void update(X& current, Z& currentIndex, const X& candidate, const Z& candidateIndex) {
|
||||
auto absCandidate = sd::math::sd_abs<X,X>(candidate);
|
||||
if (absCandidate < current) {
|
||||
current = absCandidate;
|
||||
currentIndex = candidateIndex;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
SD_LIB_HIDDEN void argMax_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
return argIndex_<X, Z, IndexMax<X, Z>>(input, output, dimensions);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
SD_LIB_HIDDEN void argMin_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
return argIndex_<X, Z, IndexMin<X, Z>>(input, output, dimensions);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
SD_LIB_HIDDEN void argAbsMax_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
return argIndex_<X, Z, IndexAbsMax<X, Z>>(input, output, dimensions);
|
||||
}
|
||||
|
||||
template <typename X, typename Z>
|
||||
SD_LIB_HIDDEN void argAbsMin_(NDArray& input, NDArray& output, const std::vector<LongType>& dimensions) {
|
||||
return argIndex_<X, Z, IndexAbsMin<X, Z>>(input, output, dimensions);
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,52 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 20.04.2018
|
||||
//
|
||||
|
||||
#include <helpers/Loops.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
#if NOT_EXCLUDED(OP_invert_permutation)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
void invertPermutation(sd::LaunchContext* context, NDArray& input, NDArray& output) {
|
||||
std::set<int> uniqueElems;
|
||||
const int length = input.lengthOf();
|
||||
|
||||
for (int i = 0; i < length; ++i) {
|
||||
int elem = input.e<int>(i);
|
||||
|
||||
if (!uniqueElems.insert(elem).second) // this operation forbids us to use #pragma omp
|
||||
THROW_EXCEPTION("helpers::invertPermutation function: input array contains duplicates !");
|
||||
|
||||
if (elem < 0 || elem > length - 1)
|
||||
THROW_EXCEPTION(
|
||||
"helpers::invertPermutation function: element of input array is out of range (0, length-1) !");
|
||||
|
||||
output.p<int>(elem, i);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,111 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma, created on 21.09.2018
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
// CPU implementation of ismax helper
|
||||
//
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <ops/declarable/helpers/ismax.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void ismax_(LaunchContext* context, NDArray* input, NDArray* output,
|
||||
const std::vector<LongType>& dimensions) {
|
||||
// Initialize output to zeros
|
||||
output->nullify();
|
||||
|
||||
if (dimensions.size() == 0 || (dimensions.size() == 1 && dimensions[0] == sd::DataTypeUtils::max<int>())) {
|
||||
// Scalar case - find the single maximum in the entire array
|
||||
auto indexMax = input->applyIndexReduce(indexreduce::IndexMax, &dimensions);
|
||||
auto targetIdx = indexMax->e<LongType>(0);
|
||||
|
||||
// Set the maximum position to 1
|
||||
output->p(targetIdx, static_cast<T>(1));
|
||||
|
||||
delete indexMax;
|
||||
} else {
|
||||
// Dimensional case - find maximum along specified dimensions
|
||||
std::vector<LongType> copy(dimensions);
|
||||
|
||||
// Get the indices of maximum values along the specified dimensions
|
||||
auto indexMaxArr = input->applyIndexReduce(indexreduce::IndexMax, &dimensions);
|
||||
|
||||
// Get TAD information for the output
|
||||
auto packZ = ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(), copy.data(), copy.size());
|
||||
auto zTadShapeInfo = packZ->primaryShapeInfo();
|
||||
auto zTadOffsets = packZ->primaryOffsets();
|
||||
|
||||
auto numTads = packZ->numberOfTads();
|
||||
auto tadLen = shape::length(zTadShapeInfo);
|
||||
|
||||
auto zBuffer = output->bufferAsT<T>();
|
||||
|
||||
// For each TAD, set the maximum index position to 1
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto t = start; t < stop; t++) {
|
||||
auto zTadOffset = zTadOffsets[t];
|
||||
auto maxIdx = indexMaxArr->e<LongType>(t);
|
||||
|
||||
// Calculate the actual offset within this TAD
|
||||
if (maxIdx >= 0 && maxIdx < tadLen) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
sd::LongType zOffset;
|
||||
|
||||
const int tadRank = shape::rank(zTadShapeInfo);
|
||||
const sd::LongType* tadShape = shape::shapeOf(zTadShapeInfo);
|
||||
const sd::LongType* tadStride = shape::stride(zTadShapeInfo);
|
||||
|
||||
INDEX2COORDS(maxIdx, tadRank, tadShape, coords);
|
||||
COORDS2INDEX(tadRank, tadStride, coords, zOffset);
|
||||
|
||||
zBuffer[zTadOffset + zOffset] = static_cast<T>(1);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, numTads);
|
||||
|
||||
delete indexMaxArr;
|
||||
}
|
||||
}
|
||||
|
||||
void ismax(LaunchContext* context, NDArray* input, NDArray* output, const std::vector<LongType>& dimensions) {
|
||||
NDArray::prepareSpecialUse({output}, {input});
|
||||
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), ismax_, (context, input, output, dimensions), SD_COMMON_TYPES);
|
||||
|
||||
NDArray::registerSpecialUse({output}, {input});
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE(void ismax_,
|
||||
(sd::LaunchContext* context, NDArray* input, NDArray* output,
|
||||
const std::vector<sd::LongType>& dimensions),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,347 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author GS <sgazeos@gmail.com>
|
||||
//
|
||||
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <ops/declarable/helpers/legacy_helpers.h>
|
||||
#include <ops/ops.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static void reluDerivative__(NDArray* theFirst, NDArray* theSecond) {
|
||||
auto functor = LAMBDA_TT(x, y) { return x > (T)0.f ? y : T(0.f); });
|
||||
|
||||
theFirst->applyPairwiseLambda<T>(theSecond, functor, theFirst);
|
||||
}
|
||||
|
||||
void reluDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), reluDerivative__, (theFirst, theSecond), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void reluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
T zero = (T)0.f;
|
||||
auto functor = LAMBDA_TT(x, y, zero) { return x > zero ? y : zero; });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
|
||||
}
|
||||
|
||||
void reluDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), reluDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void relu6Derivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) { return x > (T)0.f && x < (T)6.f ? y : T(0.f); });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void relu6Derivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), relu6Derivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void leakyReluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output, const float alpha) {
|
||||
const T alphaT = static_cast<T>(alpha);
|
||||
|
||||
auto functor = LAMBDA_TT(x, y, alphaT) { return x < 0 ? alphaT * y : y; });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void leakyReluDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput,
|
||||
const float alpha) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), leakyReluDerivative_, (theFirst, theSecond, theOutput, alpha),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void eluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output, const float alpha) {
|
||||
const T alphaT = static_cast<T>(alpha);
|
||||
|
||||
auto functor = LAMBDA_TT(x, y, alphaT) { return y * sd::math::sd_eluderivative<T, T>(x, alphaT); });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void eluDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput,
|
||||
const float alpha) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), eluDerivative_, (theFirst, theSecond, theOutput, alpha), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void seluDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) { return y * simdOps::SELUDerivative<T>::op(x, nullptr); });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void seluDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), seluDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void cubeDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) { return y * (3 * x * x); });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void cubeDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), cubeDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
// return (x >= X(0.f) ? y: -y);
|
||||
template <typename T>
|
||||
static void reduceNorm1_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) { return x > T(0.f) ? y : -y; });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void reduceNorm1(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), reduceNorm1_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void sigmCrossEntropy_(NDArray* logits, NDArray* labels, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) {
|
||||
return sd::math::sd_max<T>(x, (T)0.f) - x * y +
|
||||
sd::math::sd_log<T, T>((T)1.f + sd::math::sd_exp<T, T>(-sd::math::sd_abs<T,T>(x)));
|
||||
});
|
||||
|
||||
logits->applyPairwiseLambda<T>(labels, functor, output);
|
||||
}
|
||||
|
||||
void sigmCrossEntropy(sd::LaunchContext* context, NDArray* logits, NDArray* labels, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(logits->dataType(), sigmCrossEntropy_, (logits, labels, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void sigmCrossEntropyGrad_(NDArray* logits, NDArray* labels, NDArray* output) {
|
||||
// 1 - labels - 1 / (1 + exp(logits))
|
||||
auto functor = LAMBDA_TT(x, y) {
|
||||
if (x <= 0) return static_cast<T>(1.) - y - static_cast<T>(1.) / (static_cast<T>(1.) + sd::math::sd_exp<T, T>(x));
|
||||
auto e = sd::math::sd_exp<T, T>(-x);
|
||||
return static_cast<T>(1.) - y - e / (static_cast<T>(1.) + e);
|
||||
});
|
||||
|
||||
logits->applyPairwiseLambda<T>(labels, functor, output);
|
||||
}
|
||||
|
||||
void sigmCrossEntropyGrad(sd::LaunchContext* context, NDArray* logits, NDArray* labels, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(logits->dataType(), sigmCrossEntropyGrad_, (logits, labels, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void tanhDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) {
|
||||
T th = sd::math::sd_tanh<T, T>(x);
|
||||
return y * ((T)1.0f - (th * th));
|
||||
});
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void tanhDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), tanhDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void hardTanhDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) {
|
||||
return y * simdOps::HardTanhDerivative<T>::op(x, nullptr);
|
||||
});
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor,output);
|
||||
}
|
||||
|
||||
void hardTanhDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), hardTanhDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void rationalTanhDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) { return y * simdOps::RationalTanhDerivative<T>::op(x, nullptr); });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void rationalTanhDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), rationalTanhDerivative_, (theFirst, theSecond, theOutput),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void rectifiedTanhDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) { return x > (T)0.0f ? y * (sd::math::sd_tanhderivative<T, T>(x)) : (T)0.0f; });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void rectifiedTanhDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), rectifiedTanhDerivative_, (theFirst, theSecond, theOutput),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
|
||||
|
||||
template <typename T>
|
||||
static void softSignDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) {
|
||||
T ss = (T)1.f + sd::math::sd_abs<T,T>(x);
|
||||
return y * ((T)1.0f / (ss * ss));
|
||||
});
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void softSignDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), softSignDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void softPlusDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) {
|
||||
T p = sd::math::sd_pow<T, T, T>(static_cast<T>(M_E), x);
|
||||
return y * (p / (p + 1.));
|
||||
});
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void softPlusDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), softPlusDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
///
|
||||
/// \param theFirst
|
||||
/// \param theSecond
|
||||
/// \param theOutput
|
||||
template <typename T>
|
||||
static void sigmoidDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) {
|
||||
T s = sd::math::sd_sigmoid<T, T>(x);
|
||||
return y * (s * ((T)1.0f - s));
|
||||
});
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void sigmoidDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), sigmoidDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void hardSigmoidDerivative_(NDArray* input, NDArray* epsilon, NDArray* output) {
|
||||
auto functor = LAMBDA_TT(x, y) { return y * simdOps::HardSigmoidDerivative<T>::op(x, nullptr); });
|
||||
|
||||
input->applyPairwiseLambda<T>(epsilon, functor, output);
|
||||
}
|
||||
|
||||
void hardSigmoidDerivative(sd::LaunchContext* context, NDArray* theFirst, NDArray* theSecond, NDArray* theOutput) {
|
||||
BUILD_SINGLE_SELECTOR(theFirst->dataType(), hardSigmoidDerivative_, (theFirst, theSecond, theOutput), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void logSumExp_(NDArray* input, NDArray* axis, NDArray* output) {
|
||||
// reduce along axis with
|
||||
NDArray *tempInput = input->dup();
|
||||
input->applyTransform(transform::Exp, tempInput);
|
||||
std::vector<sd::LongType> axisVector;
|
||||
if (axis != nullptr) {
|
||||
axisVector.resize(axis->lengthOf());
|
||||
for (size_t i = 0; i < axisVector.size(); ++i) axisVector[i] = axis->e<sd::LongType>(i);
|
||||
}
|
||||
tempInput->reduceAlongDimension(reduce::Sum, output, &axisVector);
|
||||
output->applyTransform(transform::Log, output);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void logSumExp_(NDArray* input, NDArray* subtrah, NDArray* axis, NDArray* output) {
|
||||
// reduce along axis with
|
||||
NDArray *tempInput = input->dup();
|
||||
input->applyPairwiseTransform(pairwise::Subtract, subtrah, tempInput);
|
||||
tempInput->applyTransform(transform::Exp, tempInput);
|
||||
|
||||
std::vector<sd::LongType> axisVector;
|
||||
if (axis != nullptr) {
|
||||
axisVector.resize(axis->lengthOf());
|
||||
for (size_t i = 0; i < axisVector.size(); ++i) axisVector[i] = axis->e<sd::LongType>(i);
|
||||
}
|
||||
tempInput->reduceAlongDimension(reduce::Sum, output, &axisVector);
|
||||
output->applyTransform(transform::Log, output);
|
||||
}
|
||||
|
||||
void logSumExp(sd::LaunchContext* context, NDArray* input, NDArray* axis, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), logSumExp_, (input, axis, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
void logSumExp(sd::LaunchContext* context, NDArray* input, NDArray* subtrah, NDArray* axis, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), logSumExp_, (input, subtrah, axis, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void weightedCrossEntropyWithLogitsFunctor_(NDArray * targets, NDArray * input, NDArray * weights,
|
||||
NDArray* output) {
|
||||
T posWeight = weights->e<T>(0);
|
||||
|
||||
auto mainRoutineT1 = LAMBDA_TT(_x, _z, posWeight) {
|
||||
T targetWeight = (1. + (posWeight - (T)1.f) * _z);
|
||||
return (1. - _z) * _x +
|
||||
targetWeight * (sd::math::sd_log<T, T>((T)1.f + sd::math::sd_exp<T, T>(-sd::math::sd_abs<T,T>(_x))) +
|
||||
sd::math::sd_max(-_x, T(0.f)));
|
||||
});
|
||||
|
||||
auto mainRoutineT2 = LAMBDA_TTT(_x, _z, _w) {
|
||||
return (((T)1.0 - _z) * _x) + _w * (sd::math::sd_log<T, T>(T(1.) + sd::math::sd_exp<T, T>(-sd::math::sd_abs<T,T>(_x))) +
|
||||
sd::math::sd_max(-_x, T(0.f)));
|
||||
});
|
||||
|
||||
if (weights->isScalar()) {
|
||||
input->applyPairwiseLambda<T>(targets, mainRoutineT1, output);
|
||||
} else {
|
||||
weights->applyScalar(scalar::Add, -1.f, weights);
|
||||
auto add = (*targets * *targets);
|
||||
auto addOne = (*add) + T(1.f);
|
||||
*targets = *addOne;
|
||||
delete addOne;
|
||||
delete add;
|
||||
input->applyTriplewiseLambda<T>(targets, targets,mainRoutineT2, output);
|
||||
}
|
||||
}
|
||||
|
||||
void weightedCrossEntropyWithLogitsFunctor(sd::LaunchContext* context, NDArray * targets, NDArray * input,
|
||||
NDArray * weights, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(targets->dataType(), weightedCrossEntropyWithLogitsFunctor_, (targets, input, weights, output),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author George A. Shulinok <sgazeos@gmail.com>
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/lgamma.h>
|
||||
#if NOT_EXCLUDED(OP_lgamma)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// calculate digamma function for array elements
|
||||
template <typename T>
|
||||
static void lgamma_(NDArray* x, NDArray* z) {
|
||||
auto lgammaProc = LAMBDA_T(x_) {
|
||||
auto output = math::sd_lgamma<T,T>(x_);
|
||||
return output;
|
||||
});
|
||||
|
||||
x->applyLambda<T>(lgammaProc, z);
|
||||
}
|
||||
|
||||
void lgamma(sd::LaunchContext* context, NDArray* x, NDArray* z) {
|
||||
BUILD_SINGLE_SELECTOR(x->dataType(), lgamma_, (x, z), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void lgamma_, (NDArray * x, NDArray* z), SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,325 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <ops/declarable/helpers/lrn.h>
|
||||
#if NOT_EXCLUDED(OP_lrn)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static sd::Status lrnFunctor_(sd::graph::Context& block, NDArray* input, NDArray* output, int depth, float bias,
|
||||
float alpha, float beta) {
|
||||
sd_debug("MKL-DNN is not used for lrn!\n", 0);
|
||||
|
||||
const int rank = input->rankOf();
|
||||
|
||||
auto inTadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(input->shapeInfo(),
|
||||
rank - 1);
|
||||
std::shared_ptr<TadPack> outTadPack;
|
||||
|
||||
if (shape::haveSameShapeAndStrides(input->shapeInfo(), output->shapeInfo()))
|
||||
outTadPack = inTadPack;
|
||||
else
|
||||
outTadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(output->shapeInfo(),
|
||||
rank - 1);
|
||||
|
||||
const sd::LongType numOfTads = inTadPack->numberOfTads();
|
||||
const sd::LongType tadLen = input->sizeAt(-1);
|
||||
|
||||
const sd::LongType* inTadOffsets = inTadPack->primaryOffsets();
|
||||
const sd::LongType* outTadOffsets = outTadPack->primaryOffsets();
|
||||
|
||||
const sd::LongType inTadEws = shape::elementWiseStride(inTadPack->primaryShapeInfo());
|
||||
const sd::LongType outTadEws = shape::elementWiseStride(outTadPack->primaryShapeInfo());
|
||||
|
||||
const T* inBuff = reinterpret_cast<T*>(input->buffer());
|
||||
T* outBuff = reinterpret_cast<T*>(output->buffer());
|
||||
|
||||
const T tbias = static_cast<T>(bias);
|
||||
const T tbeta = static_cast<T>(beta);
|
||||
|
||||
if (inTadEws == 1 && outTadEws == 1) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
const T* x = inBuff + inTadOffsets[i];
|
||||
T* y = outBuff + outTadOffsets[i];
|
||||
|
||||
T prev = static_cast<T>(0);
|
||||
|
||||
// calculate squared sum of elements per each j-th element range [j - depth, j + depth + 1]
|
||||
// we store each squared sum in corresponding element of y array
|
||||
for (sd::LongType j = 0; j < tadLen; ++j) {
|
||||
const sd::LongType begin = sd::math::sd_max<int>(0, j - depth);
|
||||
const sd::LongType last = depth + j + 1;
|
||||
const sd::LongType end = sd::math::sd_min<int>(last, tadLen);
|
||||
|
||||
if (j == 0) {
|
||||
for (sd::LongType s = begin; s < end; ++s) prev = prev + x[s] * x[s];
|
||||
y[j] = prev;
|
||||
} else if (begin == 0 && last <= tadLen)
|
||||
y[j] = prev + x[end - 1] * x[end - 1];
|
||||
else if (begin > 0 && last <= tadLen)
|
||||
y[j] = prev + x[end - 1] * x[end - 1] - x[begin - 1] * x[begin - 1];
|
||||
else if (begin > 0 && last > tadLen)
|
||||
y[j] = prev - x[begin - 1] * x[begin - 1];
|
||||
else
|
||||
y[j] = prev;
|
||||
|
||||
if (j != 0) prev = y[j];
|
||||
|
||||
y[j] = x[j] / sd::math::sd_pow<T, T, T>(tbias + alpha * prev, tbeta);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType i = 0; i < numOfTads; ++i) {
|
||||
const T* x = inBuff + inTadOffsets[i];
|
||||
T* y = outBuff + outTadOffsets[i];
|
||||
|
||||
T prev = static_cast<T>(0);
|
||||
|
||||
// calculate squared sum of elements per each j-th element range [j - depth, j + depth + 1]
|
||||
// we store each squared sum in corresponding element of y array
|
||||
for (sd::LongType j = 0; j < tadLen; ++j) {
|
||||
const sd::LongType begin = sd::math::sd_max<int>(0, j - depth);
|
||||
const sd::LongType last = depth + j + 1;
|
||||
const sd::LongType end = sd::math::sd_min<int>(last, tadLen);
|
||||
|
||||
if (j == 0) {
|
||||
for (sd::LongType s = begin; s < end; ++s) prev = prev + x[s * inTadEws] * x[s * inTadEws];
|
||||
y[j * outTadEws] = prev;
|
||||
} else if (begin == 0 && last <= tadLen)
|
||||
y[j * outTadEws] = prev + x[(end - 1) * inTadEws] * x[(end - 1) * inTadEws];
|
||||
else if (begin > 0 && last <= tadLen)
|
||||
y[j * outTadEws] = prev + x[(end - 1) * inTadEws] * x[(end - 1) * inTadEws] -
|
||||
x[(begin - 1) * inTadEws] * x[(begin - 1) * inTadEws];
|
||||
else if (begin > 0 && last > tadLen)
|
||||
y[j * outTadEws] = prev - x[(begin - 1) * inTadEws] * x[(begin - 1) * inTadEws];
|
||||
else
|
||||
y[j * outTadEws] = prev;
|
||||
|
||||
if (j != 0) prev = y[j * outTadEws];
|
||||
|
||||
y[j * outTadEws] = x[j * inTadEws] / sd::math::sd_pow<T, T, T>(tbias + alpha * prev, tbeta);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
}
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status lrnFunctor_,
|
||||
(sd::graph::Context & block, NDArray* input, NDArray* output, int depth, float bias, float alpha,
|
||||
float beta),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
sd::Status lrnFunctor(sd::graph::Context& block, NDArray* input, NDArray* output, int depth, double bias, double alpha,
|
||||
double beta) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return lrnFunctor_, (block, input, output, depth, bias, alpha, beta),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Y>
|
||||
static void lrnBP_(NDArray& input, NDArray& gradO, NDArray& gradI, const int depth, const float bias,
|
||||
const float alpha, const float beta) {
|
||||
const int rank = input.rankOf();
|
||||
|
||||
auto inTadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(input.shapeInfo(),
|
||||
rank - 1);
|
||||
std::shared_ptr<TadPack> gradITadPack;
|
||||
|
||||
if (shape::haveSameShapeAndStrides(input.shapeInfo(), gradI.shapeInfo()))
|
||||
gradITadPack = inTadPack;
|
||||
else
|
||||
gradITadPack = sd::ConstantTadHelper::getInstance().tadForDimensions(gradI.shapeInfo(),
|
||||
rank - 1);
|
||||
|
||||
const sd::LongType numOfTads = inTadPack->numberOfTads();
|
||||
const sd::LongType tadLen = input.sizeAt(-1);
|
||||
|
||||
const sd::LongType* inTadOffsets = inTadPack->primaryOffsets();
|
||||
const sd::LongType* gradITadOffsets = gradITadPack->primaryOffsets();
|
||||
|
||||
const sd::LongType inTadEws = shape::elementWiseStride(inTadPack->primaryShapeInfo());
|
||||
const sd::LongType gradITadEws = shape::elementWiseStride(gradITadPack->primaryShapeInfo());
|
||||
|
||||
const X* inBuff = reinterpret_cast<X const*>(input.buffer());
|
||||
Y* gradIBuff = reinterpret_cast<Y*>(gradI.buffer());
|
||||
|
||||
const Y tbias = static_cast<Y>(bias);
|
||||
const Y tbeta = static_cast<Y>(beta);
|
||||
const Y talpha = static_cast<Y>(alpha);
|
||||
const Y coeff = talpha * tbeta;
|
||||
|
||||
if (inTadEws == 1 && gradITadEws == 1) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
const X* x = inBuff + inTadOffsets[i];
|
||||
Y* y = gradIBuff + gradITadOffsets[i];
|
||||
|
||||
// this loop calculates squared sum of elements per each j-th element range [j - depth, j + depth + 1]
|
||||
// we store each squared sum in corresponding element of y array
|
||||
for (sd::LongType j = 0; j < tadLen; ++j) {
|
||||
const sd::LongType begin = sd::math::sd_max<int>(0, j - depth);
|
||||
const sd::LongType last = depth + j + 1;
|
||||
const sd::LongType end = sd::math::sd_min<int>(last, tadLen);
|
||||
|
||||
if (j == 0) {
|
||||
y[0] = 0;
|
||||
for (sd::LongType s = begin; s < end; ++s) y[0] = y[0] + x[s] * x[s];
|
||||
} else if (begin == 0 && last <= tadLen)
|
||||
y[j] = y[j - 1] + x[end - 1] * x[end - 1];
|
||||
else if (begin > 0 && last <= tadLen)
|
||||
y[j] = y[j - 1] + x[end - 1] * x[end - 1] - x[begin - 1] * x[begin - 1];
|
||||
else if (begin > 0 && last > tadLen)
|
||||
y[j] = y[j - 1] - x[begin - 1] * x[begin - 1];
|
||||
else
|
||||
y[j] = y[j - 1];
|
||||
}
|
||||
|
||||
Y* factor = new Y[tadLen];
|
||||
|
||||
Y prev = static_cast<Y>(0);
|
||||
// second loop calculates derivatives using information gained in first loop above
|
||||
for (sd::LongType j = 0; j < tadLen; ++j) {
|
||||
const sd::LongType begin = sd::math::sd_max<int>(0, j - depth);
|
||||
const sd::LongType last = depth + j + 1;
|
||||
const sd::LongType end = sd::math::sd_min<int>(last, tadLen);
|
||||
|
||||
Y init = tbias + talpha * y[j];
|
||||
|
||||
if (j == 0) {
|
||||
for (sd::LongType s = begin; s < end; ++s) {
|
||||
factor[s] = sd::math::sd_pow<Y, Y, Y>(tbias + talpha * y[s], -tbeta - 1);
|
||||
prev = prev + x[s] * factor[s];
|
||||
}
|
||||
y[0] = prev;
|
||||
} else if (begin == 0 && last <= tadLen) {
|
||||
factor[end - 1] = sd::math::sd_pow<Y, Y, Y>(tbias + talpha * y[end - 1], -tbeta - 1);
|
||||
y[j] = prev + x[end - 1] * factor[end - 1];
|
||||
} else if (begin > 0 && last <= tadLen) {
|
||||
factor[end - 1] = sd::math::sd_pow<Y, Y, Y>(tbias + talpha * y[end - 1], -tbeta - 1);
|
||||
y[j] = prev + x[end - 1] * factor[end - 1] - x[begin - 1] * factor[begin - 1];
|
||||
} else if (begin > 0 && last > tadLen)
|
||||
y[j] = prev - x[begin - 1] * factor[begin - 1];
|
||||
else
|
||||
y[j] = prev;
|
||||
|
||||
if (j != 0) prev = y[j];
|
||||
|
||||
y[j] = factor[j] * init - 2 * x[j] * coeff * prev;
|
||||
}
|
||||
|
||||
delete[] factor;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
const X* x = inBuff + inTadOffsets[i];
|
||||
Y* y = gradIBuff + gradITadOffsets[i];
|
||||
|
||||
// this loop calculates squared sum of elements per each j-th element range [j - depth, j + depth + 1]
|
||||
// we store each squared sum in corresponding element of y array
|
||||
for (sd::LongType j = 0; j < tadLen; ++j) {
|
||||
const sd::LongType begin = sd::math::sd_max<int>(0, j - depth);
|
||||
const sd::LongType last = depth + j + 1;
|
||||
const sd::LongType end = sd::math::sd_min<int>(last, tadLen);
|
||||
|
||||
if (j == 0) {
|
||||
y[0] = 0;
|
||||
for (sd::LongType s = begin; s < end; ++s) y[0] = y[0] + x[s * inTadEws] * x[s * inTadEws];
|
||||
} else if (begin == 0 && last <= tadLen)
|
||||
y[j * gradITadEws] = y[(j - 1) * gradITadEws] + x[(end - 1) * inTadEws] * x[(end - 1) * inTadEws];
|
||||
else if (begin > 0 && last <= tadLen)
|
||||
y[j * gradITadEws] = y[(j - 1) * gradITadEws] + x[(end - 1) * inTadEws] * x[(end - 1) * inTadEws] -
|
||||
x[(begin - 1) * inTadEws] * x[(begin - 1) * inTadEws];
|
||||
else if (begin > 0 && last > tadLen)
|
||||
y[j * gradITadEws] = y[(j - 1) * gradITadEws] - x[(begin - 1) * inTadEws] * x[(begin - 1) * inTadEws];
|
||||
else
|
||||
y[j * gradITadEws] = y[(j - 1) * gradITadEws];
|
||||
}
|
||||
|
||||
Y* factor = new Y[tadLen];
|
||||
|
||||
Y prev = static_cast<Y>(0);
|
||||
// second loop calculates derivatives using information gained in first loop above
|
||||
for (sd::LongType j = 0; j < tadLen; ++j) {
|
||||
const sd::LongType begin = sd::math::sd_max<int>(0, j - depth);
|
||||
const sd::LongType last = depth + j + 1;
|
||||
const sd::LongType end = sd::math::sd_min<int>(last, tadLen);
|
||||
|
||||
Y init = tbias + talpha * y[j * gradITadEws];
|
||||
|
||||
if (j == 0) {
|
||||
for (sd::LongType s = begin; s < end; ++s) {
|
||||
factor[s] = sd::math::sd_pow<Y, Y, Y>(tbias + talpha * y[s * gradITadEws], -tbeta - 1);
|
||||
prev = prev + x[s * inTadEws] * factor[s];
|
||||
}
|
||||
y[0] = prev;
|
||||
} else if (begin == 0 && last <= tadLen) {
|
||||
factor[end - 1] = sd::math::sd_pow<Y, Y, Y>(tbias + talpha * y[(end - 1) * gradITadEws], -tbeta - 1);
|
||||
y[j * gradITadEws] = prev + x[(end - 1) * inTadEws] * factor[end - 1];
|
||||
} else if (begin > 0 && last <= tadLen) {
|
||||
factor[end - 1] = sd::math::sd_pow<Y, Y, Y>(tbias + talpha * y[(end - 1) * gradITadEws], -tbeta - 1);
|
||||
y[j * gradITadEws] =
|
||||
prev + x[(end - 1) * inTadEws] * factor[end - 1] - x[(begin - 1) * inTadEws] * factor[begin - 1];
|
||||
} else if (begin > 0 && last > tadLen)
|
||||
y[j * gradITadEws] = prev - x[(begin - 1) * inTadEws] * factor[begin - 1];
|
||||
else
|
||||
y[j * gradITadEws] = prev;
|
||||
|
||||
if (j != 0) prev = y[j * gradITadEws];
|
||||
|
||||
y[j * gradITadEws] = factor[j] * init - 2 * x[j * inTadEws] * coeff * prev;
|
||||
}
|
||||
|
||||
delete[] factor;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, numOfTads);
|
||||
}
|
||||
gradI *= gradO;
|
||||
}
|
||||
|
||||
void lrnBP(sd::graph::Context& block, NDArray& input, NDArray& gradO, NDArray& gradI, const int depth,
|
||||
const float bias, const float alpha, const float beta) {
|
||||
BUILD_DOUBLE_SELECTOR(input.dataType(), gradO.dataType(), lrnBP_, (input, gradO, gradI, depth, bias, alpha, beta),
|
||||
SD_FLOAT_TYPES, SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,293 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma, created on 14.02.2018
|
||||
//
|
||||
|
||||
// implementation of operation for LSTM cell with peep hole connections:
|
||||
// http://www.bioinf.jku.at/publications/older/2604.pdf
|
||||
// S. Hochreiter and J. Schmidhuber. "Long Short-Term Memory". Neural Computation, 9(8):1735-1780, 1997.
|
||||
// and
|
||||
// https://research.google.com/pubs/archive/43905.pdf
|
||||
// Hasim Sak, Andrew Senior, and Francoise Beaufays. "Long short-term memory recurrent neural network architectures for
|
||||
// large scale acoustic modeling." INTERSPEECH, 2014.
|
||||
|
||||
#include <array/NDArrayList.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <graph/VariableSpace.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/helpers/legacy_helpers.h>
|
||||
#include <ops/declarable/helpers/lstm.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
|
||||
#include <iterator>
|
||||
# if NOT_EXCLUDED(OP_concat) && NOT_EXCLUDED(OP_lstm_cell) && NOT_EXCLUDED(OP_sigmoid)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void lstmCell(sd::LaunchContext* context, NDArray* xt, NDArray* ht_1, NDArray* ct_1,
|
||||
NDArray* Wx, NDArray* Wh, NDArray* Wc, NDArray* Wp, NDArray* b, NDArray* ht,
|
||||
NDArray* ct, const std::vector<double>& params) {
|
||||
// xt input [bS x nIn]
|
||||
// ht_1 previous cell output [bS x numProj], that is at previous time step t-1, in case of projection=false ->
|
||||
// numProj=nOut!!! ct_1 previous cell state [bS x nOut], that is at previous time step t-1
|
||||
|
||||
// Wx input-to-hidden weights, [nIn x 4*nOut]
|
||||
// Wh hidden-to-hidden weights, [numProj x 4*nOut]
|
||||
// Wc diagonal weights for peephole connections [3*nOut]
|
||||
// Wp projection weights [nOut x numProj]
|
||||
// b biases, [4*nOut]
|
||||
|
||||
// ht current cell output [bS x numProj], that is at current time step t
|
||||
// ct current cell state [bS x nOut], that is at current time step t
|
||||
|
||||
const bool peephole = (bool)params[0]; // if true, provide peephole connections
|
||||
const bool projection =
|
||||
(bool)params[1]; // if true, then projection is performed, if false then numProj==nOut is mandatory!!!!
|
||||
double clippingCellValue =
|
||||
params[2]; // clipping value for ct, if it is not equal to zero, then cell state is clipped
|
||||
double clippingProjValue =
|
||||
params[3]; // clipping value for projected ht, if it is not equal to zero, then projected cell output is clipped
|
||||
const double forgetBias = params[4];
|
||||
|
||||
const int bS = xt->sizeAt(0);
|
||||
const int nIn = xt->sizeAt(1);
|
||||
const int numProj = ht_1->sizeAt(1);
|
||||
const int nOut = ct_1->sizeAt(1);
|
||||
|
||||
NDArray *mmulXt = mmul(*xt, *Wx);
|
||||
NDArray *mmulHt = mmul(*ht_1, *Wh);
|
||||
NDArray *addMmuls = (*mmulXt) + (*mmulHt);
|
||||
NDArray *z = (*addMmuls) + (*b); // [bS x 4*nOut] + [bS x 4*nOut] + [1 x 4*nOut] = [bS x 4*nOut]
|
||||
delete mmulXt;
|
||||
delete mmulHt;
|
||||
delete addMmuls;
|
||||
|
||||
NDArray *zit = (*z)({0, 0, 0, nOut}); // z for input gate, = mmul(Wxi,xt) + mmul(Whi,ht_1) + bi = [bS x nOut]
|
||||
NDArray *zft = (*z)({0, 0, nOut, 2 * nOut}); // z for forget gate, = mmul(Wxf,xt) + mmul(Whf,ht_1) + bf = [bS x nOut]
|
||||
NDArray *zct = (*z)({0, 0, 2 * nOut, 3 * nOut}); // z for cell state, = mmul(Wxc,xt) + mmul(Whc,ht_1) + bc = [bS x nOut]
|
||||
NDArray *zot = (*z)({0, 0, 3 * nOut, 4 * nOut}); // z for output gate, = mmul(Wxo,xt) + mmul(Who,ht_1) + bo = [bS x nOut]
|
||||
|
||||
if (peephole) { // add peephole connections: z + ct_1*Wc
|
||||
NDArray *wcFirst = (*Wc)({0, nOut});
|
||||
NDArray *wcSecond = (*Wc)({nOut, 2 * nOut});
|
||||
NDArray *peepholeFirst = (*ct_1) * (*wcFirst);
|
||||
NDArray *peepholeSecond = (*ct_1) * (*wcSecond);
|
||||
*zit += (*peepholeFirst); // add peephole connections to input gate
|
||||
*zft += (*peepholeSecond); // add peephole connections to forget gate
|
||||
delete peepholeFirst;
|
||||
delete peepholeSecond;
|
||||
delete wcFirst;
|
||||
delete wcSecond;
|
||||
}
|
||||
|
||||
// current sell state = ft*ct_1 + it*tanh(mmul(Wxc,xt) + mmul(Whc,ht_1) + bc
|
||||
NDArray *zftPlusBias = (*zft) + forgetBias;
|
||||
NDArray sigmoidZft = sigmoid(*zftPlusBias);
|
||||
NDArray sigmoidZit = sigmoid(*zit);
|
||||
NDArray tanhZct = tanh(*zct);
|
||||
NDArray *sigmoidZftMulCt1 = sigmoidZft * (*ct_1);
|
||||
NDArray *sigmoidZitMulTanhZct = sigmoidZit * tanhZct;
|
||||
NDArray *sigmoidOut = (*sigmoidZftMulCt1) + (*sigmoidZitMulTanhZct);
|
||||
ct->assign(sigmoidOut);
|
||||
delete zftPlusBias;
|
||||
delete sigmoidZftMulCt1;
|
||||
delete sigmoidZitMulTanhZct;
|
||||
delete sigmoidOut;
|
||||
|
||||
// if clipping value is provided then cell state is clipped by this value prior to the cell output activation
|
||||
if (clippingCellValue > 0.0) ct->applyScalar(scalar::LstmClip, clippingCellValue, ct);
|
||||
|
||||
if (peephole) {
|
||||
NDArray *wcThird = (*Wc)({{2 * nOut, 3 * nOut}});
|
||||
NDArray *peepholeThird = (*ct) * (*wcThird);
|
||||
*zot += (*peepholeThird); // add peephole connections to output gate zot + ct*Wc
|
||||
delete peepholeThird;
|
||||
delete wcThird;
|
||||
}
|
||||
|
||||
// current cell output = ot*tanh(ct)
|
||||
NDArray sigmoidZot = sigmoid(*zot);
|
||||
NDArray tanhCt = tanh(*ct);
|
||||
NDArray *htNoPeepHole = sigmoidZot * tanhCt; // = [bS x nOut]
|
||||
|
||||
// apply projection
|
||||
if (projection) {
|
||||
NDArray *assign = mmul(*htNoPeepHole, *Wp);
|
||||
ht->assign(assign); // [bS x nOut] * [ nOut x numProj] = [bS x numProj]
|
||||
delete assign;
|
||||
// if clipping projection is provided then projected cell output state is clipped by this value
|
||||
if (clippingProjValue != 0.) ht->applyScalar(scalar::LstmClip, clippingProjValue, ht);
|
||||
} else
|
||||
ht->assign(htNoPeepHole);
|
||||
|
||||
delete htNoPeepHole;
|
||||
delete z;
|
||||
delete zit;
|
||||
delete zft;
|
||||
delete zct;
|
||||
delete zot;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void fusedTanh(NDArray* z, NDArray* i, NDArray* c, NDArray* cLast, NDArray* f, NDArray* h) {
|
||||
// cell state = blockInput .* inputGate + prevCellState .* forgetGate
|
||||
auto uLen = static_cast<sd::LongType>(z->lengthOf());
|
||||
auto c_ = c->bufferAsT<T>();
|
||||
auto z_ = z->bufferAsT<T>();
|
||||
auto i_ = i->bufferAsT<T>();
|
||||
auto f_ = f->bufferAsT<T>();
|
||||
auto cLast_ = cLast->bufferAsT<T>();
|
||||
auto h_ = h->bufferAsT<T>();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
c_[e] = z_[e] * i_[e] + (f_[e] * cLast_[e]);
|
||||
h_[e] = sd::math::sd_tanh<T, T>(c_[e]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, uLen);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void lstmBlockCell(NDArray* xt, NDArray* cLast, NDArray* yLast, NDArray* W, NDArray* Wci,
|
||||
NDArray* Wcf, NDArray* Wco, NDArray* b, NDArray* i, NDArray* c, NDArray* f,
|
||||
NDArray* o, NDArray* z, NDArray* h, NDArray* y, const std::vector<double>& params) {
|
||||
/* Input arrays:
|
||||
* 0: xt - input [bS, nIn] at time t
|
||||
* 1: cLast (cs_prev) - previous cell state [bS, nOut], time t-1
|
||||
* 2: yLast (h_prev) - previous output [bS, nOut], time t-1
|
||||
* 3: W - Weights - concatenated (input-to-hidden, hidden-to-hidden weights) weights, [(nIn+nOut),
|
||||
* 4*nOut] 4: Wci - weights - cell peephole (t-1) connections to input modulation gate, [nOut] 5: Wcf -
|
||||
* weights - cell peephole (t-1) connections to forget gate, [nOut] 6: Wco - weights - cell peephole (t)
|
||||
* connections to output gate, [nOut] 7: b - biases, [4*nOut]
|
||||
*
|
||||
* Input integer arguments:
|
||||
* 0: if not zero, provide peephole connections
|
||||
*
|
||||
* Input float arguments:
|
||||
* 0: the bias added to forget gates in order to reduce the scale of forgetting in the beginning of the training
|
||||
* 1: clipping value for cell state, if it is not equal to zero, then cell state is clipped
|
||||
*
|
||||
* Output arrays:
|
||||
* 0: i - Input modulation gate activations [bS, nOut]
|
||||
* 1: c (cs) - Cell state (pre tanh) [bs, nOut] (cs)
|
||||
* 2: f - Output - forget gate activations [bs, nOut]
|
||||
* 3: o - Output - output gate activations [bs, nOut]
|
||||
* 4: z (ci) - Output - block input [bs, nOut]
|
||||
* 5: h (co) - Cell state, post tanh [bs, nOut]
|
||||
* 6: y (h) - Current cell output [bS, nOut], time t
|
||||
*/
|
||||
const bool peephole = (bool)params[0]; // if true, provide peephole connections
|
||||
const double forgetBias = params[1];
|
||||
const double clippingCellValue =
|
||||
params[2]; // clipping value for ct, if it is not equal to zero, then cell state is clipped
|
||||
|
||||
const int bS = xt->sizeAt(0);
|
||||
const int nIn = xt->sizeAt(1);
|
||||
const int nOut = cLast->sizeAt(1);
|
||||
|
||||
std::vector<sd::LongType> cOutShape = {xt->sizeAt(0),xt->sizeAt(1), xt->sizeAt(1) + yLast->sizeAt(1)};
|
||||
// Concat inputs: [xt, yt-1]: concat([bs,nIn],[bs,nOut]) -> [bs, (nIn+nOut)]
|
||||
NDArray concatOut(xt->ordering(), cOutShape, xt->dataType(),
|
||||
xt->getContext());
|
||||
helpers::concat(xt->getContext(), {const_cast<NDArray*>(xt), const_cast<NDArray*>(yLast)}, concatOut, 1);
|
||||
|
||||
NDArray *m = mmul(concatOut, *W); // mmul: [bs, (nIn+nOut)] * [(nIn+nOut), 4*nOut] = [bs, 4*nOut]
|
||||
*m += (*b); // addiRowVector
|
||||
|
||||
// Note: weights are ordered [inputGate, blockInput, forgetGate, outputGate] to match TF (TF code comments state
|
||||
// [i,f,z/ci,o] but behaviour is [i,z,f,o])
|
||||
NDArray *zi = (*m)({0, 0, 0, nOut}); // z for input modulation gate, [bS, nOut]
|
||||
NDArray *zz = (*m)({0, 0, nOut, 2 * nOut}); // z for block input, [bS, nOut]
|
||||
NDArray *zf = (*m)({0, 0, 2 * nOut, 3 * nOut}); // z for forget gate, [bS, nOut]
|
||||
NDArray *zo = (*m)({0, 0, 3 * nOut, 4 * nOut}); // z for output gate, [bS, nOut]
|
||||
|
||||
if (peephole) { // add peephole connections: z + ct_1*Wc
|
||||
NDArray *peepholeI = (*cLast) * (*Wci);
|
||||
NDArray *peepholeF = (*cLast) * (*Wcf);
|
||||
*zi += (*peepholeI); // add peephole connections to input gate
|
||||
*zf += (*peepholeF); // add peephole connections to forget gate
|
||||
delete peepholeI;
|
||||
delete peepholeF;
|
||||
}
|
||||
|
||||
// current sell state = ft*cLast + it*tanh(mmul(Wxc,xt) + mmul(Whc,ht_1) + bc
|
||||
if (forgetBias != 0.0) *zf += forgetBias;
|
||||
|
||||
PRAGMA_OMP_PARALLEL
|
||||
PRAGMA_OMP_SINGLE {
|
||||
PRAGMA_OMP_TASK
|
||||
zz->applyTransform(transform::Tanh, z); // z = tanh(zz)
|
||||
|
||||
PRAGMA_OMP_TASK
|
||||
zi->applyTransform(transform::Sigmoid, i); // i = sigmoid(zi)
|
||||
|
||||
PRAGMA_OMP_TASK
|
||||
zf->applyTransform(transform::Sigmoid, f); // f = sigmoid(zf);
|
||||
}
|
||||
|
||||
if (z->ews() == 1 && i->ews() == 1 && c->ews() == 1 && cLast->ews() == 1 && f->ews() == 1 && h->ews() == 1 &&
|
||||
z->ordering() == i->ordering() && z->ordering() == c->ordering() && z->ordering() == cLast->ordering() &&
|
||||
z->ordering() == f->ordering() && z->ordering() == h->ordering()) {
|
||||
// cell state = blockInput .* inputGate + prevCellState .* forgetGate
|
||||
BUILD_SINGLE_SELECTOR(z->dataType(), fusedTanh, (z, i, c, cLast, f, h), SD_FLOAT_TYPES);
|
||||
} else {
|
||||
// cell state = blockInput .* inputGate + prevCellState .* forgetGate
|
||||
z->applyPairwiseTransform(pairwise::Multiply, i, c); // c = z * i
|
||||
NDArray *temp = (*f) * (*cLast);
|
||||
*c += (*temp); // c = (i * z) + (zf * (*cLast))
|
||||
delete temp;
|
||||
c->applyTransform(transform::Tanh, h); // h = tanh(c)
|
||||
}
|
||||
|
||||
// if clipping value is provided then cell state is clipped by this value prior to the cell output activation
|
||||
if (clippingCellValue > 0.0) c->applyScalar(scalar::LstmClip, clippingCellValue, c);
|
||||
|
||||
// add peephole connections to output gate zot + ct*Wc
|
||||
if (peephole) {
|
||||
NDArray *prod = (*c) * (*Wco);
|
||||
*zo += (*prod);
|
||||
delete prod;
|
||||
}
|
||||
|
||||
zo->applyTransform(transform::Sigmoid, o); // o = sigmoid(zo)
|
||||
|
||||
// current cell output = ot*tanh(ct)
|
||||
c->applyTransform(transform::Tanh, h); // h = tanh(c)
|
||||
o->applyPairwiseTransform(pairwise::Multiply, h, y); // y = o * h
|
||||
|
||||
delete m;
|
||||
delete zi;
|
||||
delete zz;
|
||||
delete zf;
|
||||
delete zo;
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author GS <sgazeos@gmail.com>
|
||||
//
|
||||
#include <array/NDArray.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/lstsq.h>
|
||||
#include <ops/declarable/helpers/lup.h>
|
||||
#include <ops/declarable/helpers/qr.h>
|
||||
#include <ops/declarable/helpers/triangular_solve.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_lstsq)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void fillRegularizer(NDArray* ioMatrix, double const value) {
|
||||
auto lastDims = ioMatrix->allTensorsAlongDimension({-2, -1});
|
||||
auto rows = ioMatrix->sizeAt(-2);
|
||||
|
||||
for (auto x = 0; x < lastDims.size(); x++) {
|
||||
for (auto r = 0; r < rows; r++) {
|
||||
lastDims[x]->r<T>(r, r) = (T)value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
sd::Status leastSquaresSolveFunctor_(sd::LaunchContext* context, NDArray* leftInput, NDArray* rightInput,
|
||||
double const l2Regularizer, bool const fast, NDArray* output) {
|
||||
NDArray::preparePrimaryUse({output}, {leftInput, rightInput});
|
||||
if (fast) { // Cholesky decomposition approach
|
||||
// Equation for solve A^T * Ax = A^T * b, so
|
||||
// 1. Computing A2:
|
||||
auto tAtShape = ShapeUtils::evalShapeForMatmul(leftInput->shapeInfo(), leftInput->shapeInfo(), true, false);
|
||||
// tAtShape[tAtShape.size() - 2] = output->sizeAt(-2);
|
||||
NDArray leftOutput('c', tAtShape, output->dataType(), context);
|
||||
MmulHelper::matmul(leftInput, leftInput, &leftOutput, true, false, 0, 0, &leftOutput); // Computing A2 = A^T * A
|
||||
// 2. Computing B' = A^T * b
|
||||
auto rightOutput = output->ulike();
|
||||
|
||||
MmulHelper::matmul(leftInput, rightInput, rightOutput, true, false, 0, 0, rightOutput); // Computing B' = A^T * b
|
||||
// 3. due l2Regularizer = 0, skip regularization ( indeed A' = A2 - l2Regularizer * I)
|
||||
auto regularizer = leftOutput.ulike();
|
||||
fillRegularizer<T>(regularizer, l2Regularizer);
|
||||
leftOutput += *regularizer;
|
||||
// 4. Cholesky decomposition -- output matrix is square and lower triangular
|
||||
// auto leftOutputT = leftOutput.ulike();
|
||||
auto status = helpers::cholesky(context, &leftOutput, &leftOutput, true); // inplace decomposition
|
||||
if (status != sd::Status::OK) return status;
|
||||
// alternate moment: inverse lower triangular matrix to solve equation A'x = b' => L^Tx = L^-1 * b'
|
||||
// solve one upper triangular system (to avoid float problems)
|
||||
|
||||
// 5. Solve two triangular systems:
|
||||
auto rightB = rightOutput->ulike();
|
||||
helpers::triangularSolveFunctor(context, &leftOutput, rightOutput, true, false, rightB);
|
||||
helpers::adjointMatrix(context, &leftOutput, true, &leftOutput);
|
||||
helpers::triangularSolveFunctor(context, &leftOutput, rightB, false, false, output);
|
||||
// All done
|
||||
} else { // QR decomposition approach
|
||||
// Equation for solve Rx = Q^T * b, where A = Q * R, where Q - orthogonal matrix, and R - upper triangular
|
||||
// 1. QR decomposition
|
||||
std::vector<sd::LongType> *qShapePtr = leftInput->getShapeAsVector();
|
||||
std::vector<sd::LongType> qShape = *qShapePtr;
|
||||
delete qShapePtr;
|
||||
|
||||
std::vector<sd::LongType> *rShapePtr = leftInput->getShapeAsVector();
|
||||
std::vector<sd::LongType> rShape = *rShapePtr;
|
||||
delete rShapePtr;
|
||||
|
||||
qShape[leftInput->rankOf() - 1] = leftInput->sizeAt(-2);
|
||||
|
||||
NDArray Q(leftInput->ordering(), qShape, leftInput->dataType(), context); // = leftInput->ulike();
|
||||
NDArray R(leftInput->ordering(), rShape, leftInput->dataType(), context); // = rightInput->ulike();
|
||||
helpers::qr(context, leftInput, &Q, &R, true);
|
||||
// 2. b` = Q^t * b:
|
||||
auto rightOutput = rightInput->ulike();
|
||||
MmulHelper::matmul(&Q, rightInput, rightOutput, true, false, 0, 0, rightOutput);
|
||||
// 3. Solve triangular system
|
||||
helpers::triangularSolveFunctor(context, &R, rightOutput, false, false, output);
|
||||
}
|
||||
NDArray::registerPrimaryUse({output}, {leftInput, rightInput});
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status leastSquaresSolveFunctor(sd::LaunchContext* context, NDArray* leftInput, NDArray* rightInput,
|
||||
double const l2Regularizer, bool const fast, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(leftInput->dataType(), return leastSquaresSolveFunctor_,
|
||||
(context, leftInput, rightInput, l2Regularizer, fast, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,634 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/helpers/top_k.h>
|
||||
#if NOT_EXCLUDED(OP_lup)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void swapRows_(NDArray* matrix, sd::LongType theFirst, sd::LongType theSecond) {
|
||||
if (theFirst != theSecond)
|
||||
for (sd::LongType i = 0; i < matrix->columns(); i++) {
|
||||
math::sd_swap(matrix->r<T>(theFirst, i), matrix->r<T>(theSecond, i));
|
||||
}
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void swapRows_, (NDArray * matrix, sd::LongType theFirst, sd::LongType theSecond), SD_FLOAT_TYPES);
|
||||
|
||||
template <typename T>
|
||||
static void swapRows(T* matrixBuf, sd::LongType const* matrixShape, sd::LongType theFirst, sd::LongType theSecond) {
|
||||
if (theFirst != theSecond) {
|
||||
auto n = shape::sizeAt(matrixShape, static_cast<sd::LongType>(-1));
|
||||
|
||||
auto loop = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType theFirstPos[] = {theFirst, i};
|
||||
sd::LongType theSecondPos[] = {theSecond, i};
|
||||
|
||||
sd::LongType theFirstIndex;
|
||||
COORDS2INDEX(shape::rank(matrixShape), shape::stride(matrixShape), theFirstPos, theFirstIndex);
|
||||
|
||||
sd::LongType theSecondIndex;
|
||||
COORDS2INDEX(shape::rank(matrixShape), shape::stride(matrixShape), theSecondPos, theSecondIndex);
|
||||
|
||||
math::sd_swap(matrixBuf[theFirstIndex], matrixBuf[theSecondIndex]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(loop, 0, n, 1);
|
||||
}
|
||||
}
|
||||
|
||||
void swapRows(NDArray* matrix, sd::LongType theFirst, sd::LongType theSecond) {
|
||||
BUILD_SINGLE_SELECTOR(matrix->dataType(), swapRows_, (matrix, theFirst, theSecond), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void invertLowerMatrix_(NDArray* inputMatrix, NDArray* invertedMatrix) {
|
||||
sd::LongType n = inputMatrix->rows();
|
||||
invertedMatrix->setIdentity();
|
||||
|
||||
if (inputMatrix->isIdentityMatrix()) return;
|
||||
|
||||
auto invertDiagonals = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType i = start; i < stop; i += increment) invertedMatrix->r<T>(i, i) /= inputMatrix->t<T>(i, i);
|
||||
};
|
||||
|
||||
auto invertSubDiagonals = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType i = start; i < stop; i += increment)
|
||||
invertedMatrix->r<T>(i, i - 1) -=
|
||||
(inputMatrix->t<T>(i, i - 1) * invertedMatrix->t<T>(i - 1, i - 1) / inputMatrix->t<T>(i, i));
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(invertDiagonals, 0, n, 1);
|
||||
samediff::Threads::parallel_for(invertSubDiagonals, 1, n, 1);
|
||||
|
||||
for (sd::LongType i = 1; i < n; i++) {
|
||||
for (sd::LongType j = 0; j < i - 1; j++)
|
||||
for (sd::LongType k = 0; k < i; k++)
|
||||
invertedMatrix->r<T>(i, j) -=
|
||||
((invertedMatrix->t<T>(k, j) * inputMatrix->t<T>(i, k) / inputMatrix->t<T>(i, i)));
|
||||
}
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void invertLowerMatrix_, (NDArray * inputMatrix, NDArray* invertedMatrix);
|
||||
, SD_FLOAT_TYPES);
|
||||
|
||||
void invertLowerMatrix(NDArray* inputMatrix, NDArray* invertedMatrix) {
|
||||
BUILD_SINGLE_SELECTOR(inputMatrix->dataType(), invertLowerMatrix_, (inputMatrix, invertedMatrix), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void _invertUpperMatrix(NDArray* inputMatrix, NDArray* invertedMatrix) {
|
||||
sd::LongType n = inputMatrix->rows();
|
||||
invertedMatrix->setIdentity();
|
||||
|
||||
if (inputMatrix->isIdentityMatrix()) { // the inverse for I is I
|
||||
return;
|
||||
}
|
||||
|
||||
auto invertDiagonals = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment) invertedMatrix->r<T>(i, i) /= inputMatrix->t<T>(i, i);
|
||||
};
|
||||
|
||||
// PRAGMA_OMP_PARALLEL_FOR_IF(n > Environment::getInstance().elementwiseThreshold())
|
||||
auto invertUpDiagonals = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i += increment)
|
||||
invertedMatrix->r<T>(i, i + 1) -=
|
||||
(inputMatrix->t<T>(i, i + 1) * invertedMatrix->t<T>(i + 1, i + 1) / inputMatrix->t<T>(i, i));
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(invertDiagonals, 0, n, 1);
|
||||
samediff::Threads::parallel_for(invertUpDiagonals, 0, n - 1, 1);
|
||||
|
||||
for (auto i = n - 2; i >= 0; i--) {
|
||||
for (auto j = i + 2; j < n; j++)
|
||||
for (auto k = i; k < n; k++)
|
||||
invertedMatrix->r<T>(i, j) -=
|
||||
((invertedMatrix->t<T>(k, j) * inputMatrix->t<T>(i, k) / inputMatrix->t<T>(i, i)));
|
||||
}
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _invertUpperMatrix, (NDArray * inputMatrix, NDArray* invertedMatrix);
|
||||
, SD_FLOAT_TYPES);
|
||||
|
||||
void invertUpperMatrix(NDArray* inputMatrix, NDArray* invertedMatrix) {
|
||||
BUILD_SINGLE_SELECTOR(inputMatrix->dataType(), _invertUpperMatrix, (inputMatrix, invertedMatrix), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T, typename I>
|
||||
static NDArray lup_(LaunchContext* context, NDArray* input, NDArray* compound, NDArray* permutation) {
|
||||
const sd::LongType rowNum = input->rows();
|
||||
const sd::LongType columnNum = input->columns();
|
||||
|
||||
// FIXED: Use stack allocation instead of heap to avoid memory leak
|
||||
NDArray determinant(DataTypeUtils::fromT<T>(), context, true); // scalar initialized to 0
|
||||
determinant.p<T>(0, static_cast<T>(1.f)); // set value to 1
|
||||
NDArray compoundMatrix = *input; // copy
|
||||
NDArray permutationMatrix(input, false, context); // has same shape as input and contiguous strides
|
||||
permutationMatrix.setIdentity();
|
||||
|
||||
T pivotValue; // = T(0.0);
|
||||
sd::LongType pivot; // = -1;
|
||||
sd::LongType swapCount = 0;
|
||||
|
||||
for (sd::LongType i = 0; i < rowNum; i++) {
|
||||
pivotValue = T(0.0);
|
||||
pivot = -1;
|
||||
for (sd::LongType rowCounter = i; rowCounter < rowNum; rowCounter++) {
|
||||
if (sd::math::sd_abs<T,T>(compoundMatrix.t<T>(rowCounter, i)) > pivotValue) {
|
||||
pivotValue = sd::math::sd_abs<T,T>(compoundMatrix.t<T>(rowCounter, i));
|
||||
pivot = rowCounter;
|
||||
}
|
||||
}
|
||||
|
||||
if (pivotValue > DataTypeUtils::min_positive<T>()) {
|
||||
swapRows(&compoundMatrix, pivot, i);
|
||||
swapRows(&permutationMatrix, pivot, i);
|
||||
if (pivot != i) swapCount++;
|
||||
|
||||
for (sd::LongType j = i + 1; j < rowNum; j++) {
|
||||
compoundMatrix.r<T>(j, i) /= compoundMatrix.t<T>(i, i);
|
||||
for (sd::LongType k = i + 1; k < rowNum; k++) {
|
||||
compoundMatrix.r<T>(j, k) -= compoundMatrix.t<T>(j, i) * compoundMatrix.t<T>(i, k);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (sd::LongType e = 0; e < rowNum; e++) {
|
||||
determinant.p<T>(0, determinant.e<T>(0) * compoundMatrix.e<T>(e, e));
|
||||
}
|
||||
if (swapCount % 2) {
|
||||
determinant.p<T>(0, -determinant.e<T>(0));
|
||||
}
|
||||
if (compound != nullptr) compound->assign(&compoundMatrix);
|
||||
if (permutation != nullptr) {
|
||||
auto permutaionVector = NDArrayFactory::create('c', {rowNum}, DataTypeUtils::fromT<I>(), input->getContext());
|
||||
for (auto i = 0; i < rowNum; i++) {
|
||||
for (auto j = 0; j < columnNum; j++) {
|
||||
if (permutationMatrix.t<T>(i, j) != 0) {
|
||||
permutaionVector->template r<I>(i) = j;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (permutationMatrix.isSameShape(permutation))
|
||||
permutation->assign(&permutationMatrix);
|
||||
else if (permutation->isSameShape(permutaionVector)) {
|
||||
permutation->assign(permutaionVector);
|
||||
}
|
||||
}
|
||||
return determinant; // FIXED: Return stack-allocated object instead of dereferencing pointer
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( NDArray lup_,
|
||||
(LaunchContext * context, NDArray* input, NDArray* output, NDArray* permutation), SD_FLOAT_TYPES,
|
||||
SD_INDEXING_TYPES);
|
||||
/*
|
||||
* lu decomposition with naive algorithm with partial pivoting
|
||||
* */
|
||||
template <typename T, typename I>
|
||||
static I argmaxCol(I column, T* compoundBuffer, sd::LongType const* compoundShape) {
|
||||
auto rowNum = shape::sizeAt(compoundShape, static_cast<sd::LongType>(0));
|
||||
sd::LongType xInitial[] = {column, column};
|
||||
sd::LongType xInitialIndex;
|
||||
COORDS2INDEX(shape::rank(compoundShape), shape::stride(compoundShape), xInitial, xInitialIndex);
|
||||
auto maxValue = T(0);
|
||||
auto result = -1;
|
||||
auto start = column;
|
||||
auto stop = rowNum;
|
||||
auto increment = 1;
|
||||
for (auto rowCounter = start; rowCounter < stop; rowCounter++) {
|
||||
sd::LongType xPos[] = {rowCounter, column};
|
||||
sd::LongType xIndex;
|
||||
COORDS2INDEX(shape::rank(compoundShape), shape::stride(compoundShape), xPos, xIndex);
|
||||
if (sd::math::sd_abs<T,T>(compoundBuffer[xIndex]) > maxValue) {
|
||||
maxValue = sd::math::sd_max(maxValue, sd::math::sd_abs<T,T>(compoundBuffer[xIndex]));
|
||||
result = rowCounter;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void processColumns(sd::LongType currentRow, sd::LongType rowNum, T* compoundBuf, sd::LongType const* compoundShape) {
|
||||
sd::LongType xDiag[] = {currentRow, currentRow};
|
||||
sd::LongType diagIndex;
|
||||
COORDS2INDEX(shape::rank(compoundShape), shape::stride(compoundShape), xDiag, diagIndex);
|
||||
auto loop = PRAGMA_THREADS_FOR {
|
||||
for (auto j = start; j < stop; j++) {
|
||||
sd::LongType xRow[] = {j, currentRow};
|
||||
sd::LongType rowIndex;
|
||||
COORDS2INDEX(shape::rank(compoundShape), shape::stride(compoundShape), xRow, rowIndex);
|
||||
compoundBuf[rowIndex] /= compoundBuf[diagIndex]; // output->t<T>(i, i);
|
||||
for (sd::LongType k = currentRow + 1; k < rowNum; k++) {
|
||||
sd::LongType yRow[] = {j, k};
|
||||
sd::LongType yCol[] = {currentRow, k};
|
||||
sd::LongType rowIndexY, colIndex;
|
||||
COORDS2INDEX(shape::rank(compoundShape), shape::stride(compoundShape), yRow, rowIndexY);
|
||||
COORDS2INDEX(shape::rank(compoundShape), shape::stride(compoundShape), yCol, colIndex);
|
||||
compoundBuf[rowIndexY] -= compoundBuf[rowIndex] * compoundBuf[colIndex];
|
||||
}
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_tad(loop, currentRow + 1, rowNum, 1);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void doolitleLU(LaunchContext* context, NDArray* compound, sd::LongType rowNum) {
|
||||
auto input = compound->dup();
|
||||
compound->nullify();
|
||||
|
||||
// Decomposing matrix into Upper and Lower
|
||||
// triangular matrix
|
||||
for (auto i = 0; i < rowNum; i++) {
|
||||
// Upper Triangular
|
||||
for (auto k = i; k < rowNum; k++) {
|
||||
// Summation of L(i, j) * U(j, k)
|
||||
sd::LongType sum = 0;
|
||||
for (sd::LongType j = 0; j < i; j++) sum += compound->t<T>(i, j) * compound->t<T>(j, k);
|
||||
|
||||
// Evaluating U(i, k)
|
||||
compound->r<T>(i, k) = input->t<T>(i, k) - sum;
|
||||
}
|
||||
|
||||
// Lower Triangular
|
||||
for (sd::LongType k = i + 1; k < rowNum; k++) {
|
||||
// Summation of L(k, j) * U(j, i)
|
||||
sd::LongType sum = 0;
|
||||
for (sd::LongType j = 0; j < i; j++) sum += compound->t<T>(k, j) * compound->t<T>(j, i);
|
||||
|
||||
// Evaluating L(k, i)
|
||||
compound->r<T>(k, i) = (input->t<T>(k, i) - sum) / compound->t<T>(i, i);
|
||||
}
|
||||
}
|
||||
delete input; // Clean up duped array
|
||||
}
|
||||
|
||||
template <typename T, typename I>
|
||||
static void luNN_(LaunchContext* context, NDArray* compound, NDArray* permutation, sd::LongType rowNum) {
|
||||
if (permutation) { // LUP algorithm
|
||||
// Initialize permutation array
|
||||
permutation->linspace(0);
|
||||
|
||||
// Cache all buffers and shape data upfront
|
||||
auto permutationBuf = permutation->bufferAsT<I>();
|
||||
auto compoundBuf = compound->bufferAsT<T>();
|
||||
auto compoundShape = compound->shapeInfo();
|
||||
auto permutationShape = permutation->shapeInfo();
|
||||
|
||||
// Cache shape-related values outside the main loop
|
||||
const int permRank = shape::rank(permutationShape);
|
||||
const sd::LongType* permShape = shape::shapeOf(permutationShape);
|
||||
const sd::LongType* permStride = shape::stride(permutationShape);
|
||||
|
||||
// Main LU decomposition loop
|
||||
for (sd::LongType i = 0; i < rowNum - 1; i++) {
|
||||
auto pivotIndex = argmaxCol(i, compoundBuf, compoundShape);
|
||||
if (pivotIndex < 0) {
|
||||
THROW_EXCEPTION("helpers::luNN_: input matrix is singular.");
|
||||
}
|
||||
|
||||
// Use cached shape values for coordinate transforms
|
||||
sd::LongType firstIndexCoords[SD_MAX_RANK];
|
||||
sd::LongType secondIndexCoords[SD_MAX_RANK];
|
||||
sd::LongType firstIndex;
|
||||
sd::LongType secondIndex;
|
||||
|
||||
// Transform coordinates using cached shape data
|
||||
INDEX2COORDS(i, permRank, permShape, firstIndexCoords);
|
||||
COORDS2INDEX(permRank, permStride, firstIndexCoords, firstIndex);
|
||||
|
||||
INDEX2COORDS(pivotIndex, permRank, permShape, secondIndexCoords);
|
||||
COORDS2INDEX(permRank, permStride, secondIndexCoords, secondIndex);
|
||||
|
||||
// Perform the swaps
|
||||
math::sd_swap(permutationBuf[firstIndex], permutationBuf[secondIndex]);
|
||||
swapRows(compoundBuf, compoundShape, i, pivotIndex);
|
||||
|
||||
// Process remaining columns
|
||||
processColumns(i, rowNum, compoundBuf, compoundShape);
|
||||
}
|
||||
} else { // Doolitle algorithm with LU decomposition
|
||||
doolitleLU<T>(context, compound, rowNum);
|
||||
}
|
||||
}
|
||||
template <typename T, typename I>
|
||||
static void lu_(LaunchContext* context, NDArray* input, NDArray* output, NDArray* permutationVectors) {
|
||||
auto n = input->sizeAt(-1);
|
||||
|
||||
output->assign(input); // fill up output tensor with zeros
|
||||
ResultSet outputs = output->allTensorsAlongDimension({-2, -1});
|
||||
ResultSet permutations;
|
||||
if (permutationVectors) permutations = permutationVectors->allTensorsAlongDimension({-1});
|
||||
|
||||
auto loop = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
luNN_<T, I>(context, outputs.at(i), permutationVectors ? permutations.at(i) : nullptr, n);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(loop, 0, outputs.size(), 1);
|
||||
}
|
||||
|
||||
void lu(LaunchContext* context, NDArray* input, NDArray* output, NDArray* permutation) {
|
||||
BUILD_DOUBLE_SELECTOR(input->dataType(), permutation ? permutation->dataType() : DataType::INT32, lu_,
|
||||
(context, input, output, permutation), SD_FLOAT_TYPES, SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
|
||||
|
||||
template <typename T>
|
||||
static sd::Status determinant_(LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
sd::LongType n = input->sizeAt(-1);
|
||||
sd::LongType n2 = n * n;
|
||||
|
||||
auto matrix =
|
||||
NDArrayFactory::create(input->ordering(), {n, n}, input->dataType(), context); //, block.getWorkspace());
|
||||
|
||||
for (sd::LongType e = 0; e < output->lengthOf(); e++) {
|
||||
for (sd::LongType k = e * n2, row = 0; k < (e + 1) * n2; ++k, ++row) matrix->p(row, input->e<T>(k));
|
||||
auto ret = lup_<T, sd::LongType>(context, matrix, (NDArray*)nullptr, (NDArray*)nullptr);
|
||||
output->p(e, &ret);
|
||||
}
|
||||
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status determinant(sd::LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return determinant_, (context, input, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
sd::Status logAbsDeterminant_(LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
sd::LongType n = input->sizeAt(-1);
|
||||
sd::LongType n2 = n * n;
|
||||
|
||||
NDArray *matrix =
|
||||
NDArrayFactory::create(input->ordering(), {n, n}, input->dataType(), context); //, block.getWorkspace());
|
||||
for (sd::LongType e = 0; e < output->lengthOf(); e++) {
|
||||
for (sd::LongType k = e * n2, row = 0; k < (e + 1) * n2; ++k, ++row) {
|
||||
matrix->p(row, input->e<T>(k));
|
||||
}
|
||||
NDArray det = lup_<T, sd::LongType>(context, matrix, (NDArray*)nullptr, (NDArray*)nullptr);
|
||||
if (det.e<T>(0) != 0.f) output->p(e, sd::math::sd_log<T, T>(sd::math::sd_abs<T,T>(det.t<T>(0))));
|
||||
}
|
||||
|
||||
delete matrix;
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status logAbsDeterminant(sd::LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return logAbsDeterminant_, (context, input, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static sd::Status inverse_(LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
auto n = input->sizeAt(-1);
|
||||
auto n2 = n * n;
|
||||
auto totalCount = output->lengthOf() / n2;
|
||||
float zerof = 0.f;
|
||||
output->assign(zerof); // fill up output tensor with zeros
|
||||
auto matrix = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
auto compound = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
auto permutation = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
auto lowerMatrix = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
auto upperMatrix = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
float zero = 0.f;
|
||||
for (sd::LongType e = 0; e < totalCount; e++) {
|
||||
if (e) matrix->assign(zero);
|
||||
|
||||
for (sd::LongType k = e * n2, row = 0; k < (e + 1) * n2; k++) {
|
||||
matrix->p(row++, input->e<T>(k));
|
||||
}
|
||||
T det = lup_<T, sd::LongType>(context, matrix, compound, permutation).template e<T>(0);
|
||||
|
||||
// FIXME: and how this is going to work on float16?
|
||||
if (sd::math::sd_abs<T,T>(det) < T(0.000001)) {
|
||||
sd_printf("matrix_inverse: The matrix %i has no inverse due determinant is %lf. Quiting...\n", e, det);
|
||||
return sd::Status::VALIDATION;
|
||||
}
|
||||
lowerMatrix->setIdentity(); // set up U to identity matrix
|
||||
for (sd::LongType k = 1; k < n; k++) { // and then put all values under main diagonal on to it
|
||||
for (sd::LongType j = 0; j < k; j++) lowerMatrix->template r<T>(k, j) = compound->template t<T>(k, j);
|
||||
}
|
||||
upperMatrix->setIdentity(); // set up U to identity matrix
|
||||
for (sd::LongType k = 0; k < n; k++) { // and then put all values under main diagonal on to it
|
||||
for (sd::LongType j = k; j < n; j++) upperMatrix->template r<T>(k, j) = compound->template t<T>(k, j);
|
||||
}
|
||||
invertUpperMatrix(upperMatrix, matrix);
|
||||
|
||||
invertLowerMatrix(lowerMatrix, upperMatrix);
|
||||
|
||||
sd::MmulHelper::mmul(matrix, upperMatrix, compound, 1.0, 0.0);
|
||||
sd::MmulHelper::mmul(compound, permutation, matrix, 1.0, 0.0);
|
||||
for (sd::LongType k = e * n2, row = 0; k < (e + 1) * n2; k++) {
|
||||
output->r<T>(k) = matrix->template t<T>(row++);
|
||||
}
|
||||
}
|
||||
|
||||
delete matrix;
|
||||
delete compound;
|
||||
delete upperMatrix;
|
||||
delete lowerMatrix;
|
||||
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static sd::Status lowerInverse_(LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
auto n = input->sizeAt(-1);
|
||||
auto n2 = n * n;
|
||||
auto totalCount = output->lengthOf() / n2;
|
||||
float zero = 0.f;
|
||||
output->assign(zero); // fill up output tensor with zeros
|
||||
auto matrix = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
auto compound = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
auto permutation = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
auto lowerMatrix = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
auto upperMatrix = NDArrayFactory::create('c', {n, n}, DataTypeUtils::fromT<T>(), context);
|
||||
for (sd::LongType e = 0; e < totalCount; e++) {
|
||||
if (e) matrix->assign(zero);
|
||||
|
||||
for (sd::LongType k = e * n2, row = 0; k < (e + 1) * n2; k++) {
|
||||
matrix->p(row++, input->e<T>(k));
|
||||
}
|
||||
T det = T(1.f);
|
||||
for (auto i = 0; i < n; i++) {
|
||||
det *= matrix->template t<T>(i, i);
|
||||
}
|
||||
|
||||
// FIXME: a->d how this is going to work on float16?
|
||||
if (sd::math::sd_abs<T,T>(det) < T(0.000001)) {
|
||||
sd_printf("matrix_inverse: The matrix %i has no inverse due determinant is %lf. Quitting...\n", e, det);
|
||||
return sd::Status::VALIDATION;
|
||||
}
|
||||
lowerMatrix->nullify();
|
||||
invertLowerMatrix(matrix, lowerMatrix);
|
||||
|
||||
for (sd::LongType k = e * n2, row = 0; k < (e + 1) * n2; k++) {
|
||||
output->r<T>(k) = lowerMatrix->template t<T>(row++);
|
||||
}
|
||||
}
|
||||
|
||||
delete matrix;
|
||||
delete lowerMatrix;
|
||||
delete compound;
|
||||
delete permutation;
|
||||
delete upperMatrix;
|
||||
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static sd::Status upperInverse_(LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
auto n = input->sizeAt(-1);
|
||||
auto n2 = n * n;
|
||||
|
||||
output->nullify(); // fill up output tensor with zeros
|
||||
auto inputPart = input->allTensorsAlongDimension({-2, -1});
|
||||
auto outputPart = output->allTensorsAlongDimension({-2, -1});
|
||||
auto totalCount = outputPart.size();
|
||||
for (sd::LongType e = 0; e < totalCount; e++) {
|
||||
invertUpperMatrix(inputPart.at(e), outputPart.at(e));
|
||||
}
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status inverse(sd::LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return inverse_, (context, input, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
sd::Status lowerInverseFunctor(sd::LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return lowerInverse_, (context, input, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
sd::Status upperInverseFunctor(sd::LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return upperInverse_, (context, input, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static bool checkCholeskyInput_(sd::LaunchContext* context, NDArray * input) {
|
||||
ResultSet lastMatrixList = input->allTensorsAlongDimension({input->rankOf() - 2, input->rankOf() - 1});
|
||||
for (sd::LongType i = 0; i < lastMatrixList.size(); i++) {
|
||||
auto thisMatrix = lastMatrixList.at(i);
|
||||
// check for symmetric
|
||||
for (sd::LongType r = 0; r < thisMatrix->rows(); r++)
|
||||
for (sd::LongType c = 0; c < thisMatrix->columns(); c++)
|
||||
if (sd::math::sd_abs<T,T>(thisMatrix->e<T>(r, c) - lastMatrixList.at(i)->e<T>(c, r)) >
|
||||
DataTypeUtils::min_positive<T>())
|
||||
return false;
|
||||
|
||||
NDArray *output = NDArrayFactory::create<T>(static_cast<T>(0.), context);
|
||||
if (sd::Status::OK != determinant(context, thisMatrix, output)) return false;
|
||||
if (output->e<T>(0) <= T(0)) return 0;
|
||||
NDArray reversedMatrix(*thisMatrix);
|
||||
if (sd::Status::OK != inverse(context, thisMatrix, &reversedMatrix)) return false;
|
||||
if (sd::Status::OK != determinant(context, &reversedMatrix, output)) return false;
|
||||
if (output->e<T>(0) <= T(0)) return 0;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool checkCholeskyInput(sd::LaunchContext* context, NDArray * input) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return checkCholeskyInput_, (context, input), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
sd::Status cholesky_(LaunchContext* context, NDArray* input, NDArray* output, bool inplace) {
|
||||
auto n = input->sizeAt(-1);
|
||||
auto n2 = n * n;
|
||||
auto totalCount = output->lengthOf() / n2;
|
||||
float zero = 0.f;
|
||||
if (!inplace) output->assign(zero); // fill up output tensor with zeros only inplace=false
|
||||
|
||||
std::vector<sd::LongType> shape = {n,n};
|
||||
std::unique_ptr<NDArray> matrix(
|
||||
NDArrayFactory::create_('c', shape, input->dataType(), context)); //, block.getWorkspace());
|
||||
std::unique_ptr<NDArray> lowerMatrix(NDArrayFactory::create_('c',shape, input->dataType(), context));
|
||||
|
||||
for (sd::LongType e = 0; e < totalCount; e++) {
|
||||
// fill up matrix
|
||||
for (sd::LongType k = e * n2, l = 0; k < (e + 1) * n2; k++) {
|
||||
matrix->p(l++, input->e<T>(k));
|
||||
}
|
||||
// if (e) // from the second loop need to zero matrix
|
||||
lowerMatrix->assign(zero);
|
||||
|
||||
for (sd::LongType col = 0; col < n; col++) {
|
||||
for (sd::LongType row = 0; row < col; row++) {
|
||||
T rowSum = static_cast<T>(0);
|
||||
for (sd::LongType k = 0; k < row; ++k) rowSum += (lowerMatrix->e<T>(col, k) * lowerMatrix->e<T>(row, k));
|
||||
lowerMatrix->p(col, row, (matrix->e<T>(row, col) - rowSum) / lowerMatrix->e<T>(row, row));
|
||||
}
|
||||
T diagonalSum = static_cast<T>(0);
|
||||
for (sd::LongType k = 0; k < col; ++k) diagonalSum += lowerMatrix->e<T>(col, k) * lowerMatrix->e<T>(col, k);
|
||||
lowerMatrix->p(col, col, sd::math::sd_sqrt<T, T>(matrix->e<T>(col, col) - diagonalSum));
|
||||
}
|
||||
for (sd::LongType k = e * n2, l = 0; k < (e + 1) * n2; k++) {
|
||||
output->p(k, lowerMatrix->e<T>(l++));
|
||||
}
|
||||
}
|
||||
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status cholesky(sd::LaunchContext* context, NDArray* input, NDArray* output, bool inplace) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return cholesky_, (context, input, output, inplace), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
sd::Status logdetFunctor_(LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
auto tempOutput = input->dup();
|
||||
auto res = cholesky_<T>(context, input, tempOutput, false);
|
||||
if (res != sd::Status::OK) return res;
|
||||
auto n = input->sizeAt(-1);
|
||||
auto totalCount = output->lengthOf();
|
||||
std::vector<T> d(n);
|
||||
ResultSet matrices = tempOutput->allTensorsAlongDimension({input->rankOf() - 2, input->rankOf() - 1});
|
||||
|
||||
for (sd::LongType e = 0; e < totalCount; e++) {
|
||||
for (sd::LongType i = 0; i < n; ++i)
|
||||
output->r<T>(e) += sd::math::sd_log<T, T>(sd::math::sd_pow<T, T, T>(matrices.at(e)->t<T>(i, i), T(2)));
|
||||
}
|
||||
delete tempOutput; // Clean up duped array
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status logdetFunctor(sd::LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return logdetFunctor_, (context, input, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
sd::Status lup(sd::LaunchContext* context, NDArray* input, NDArray* compound, NDArray* permutation) {
|
||||
BUILD_DOUBLE_SELECTOR(input->dataType(), permutation->dataType(), lup_, (context, input, compound, permutation),
|
||||
SD_FLOAT_NATIVE, SD_INDEXING_TYPES);
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,99 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <array/ResultSet.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/matrixSetDiag.h>
|
||||
#if NOT_EXCLUDED(OP_matrix_set_diag)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
void matrixSetDiag_(NDArray& input, NDArray& diagonal, NDArray& output, const bool zeroPad) {
|
||||
// input and output are the same array (x == z) when zeroPad = true
|
||||
// xRank = zRank, xRank = yRank + 1
|
||||
// xLen = zLen
|
||||
|
||||
const T* x = input.bufferAsT<T>();
|
||||
const T* y = diagonal.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
|
||||
// Cache all shape information upfront
|
||||
const sd::LongType* xShapeInfo = input.shapeInfo();
|
||||
const sd::LongType* yShapeInfo = diagonal.shapeInfo();
|
||||
const sd::LongType* zShapeInfo = output.shapeInfo();
|
||||
|
||||
// Cache shape-related values
|
||||
const int xRank = input.rankOf();
|
||||
const auto xLen = input.lengthOf();
|
||||
|
||||
// Cache shape and stride pointers
|
||||
const sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
const sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
const sd::LongType* yStride = shape::stride(yShapeInfo);
|
||||
const sd::LongType* zStride = shape::stride(zShapeInfo);
|
||||
|
||||
// Check if input and output have same offsets
|
||||
const bool areSameOffsets = shape::haveSameShapeAndStrides(xShapeInfo, zShapeInfo);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
// Pre-allocate coords array outside the loop
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
|
||||
for (sd::LongType i = 0; i < xLen; ++i) {
|
||||
// Use cached shape data for coordinate transforms
|
||||
INDEX2COORDS(i, xRank, xShape, coords);
|
||||
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(xRank, xStride, coords, xOffset);
|
||||
|
||||
sd::LongType zOffset;
|
||||
if (areSameOffsets) {
|
||||
zOffset = xOffset;
|
||||
} else {
|
||||
COORDS2INDEX(xRank, zStride, coords, zOffset);
|
||||
}
|
||||
|
||||
// Check diagonal condition using cached rank
|
||||
if (coords[xRank - 2] == coords[xRank - 1]) {
|
||||
sd::LongType yOffset;
|
||||
COORDS2INDEX(xRank - 1, yStride, coords, yOffset);
|
||||
z[zOffset] = y[yOffset];
|
||||
} else {
|
||||
z[zOffset] = zeroPad ? static_cast<T>(0) : x[xOffset];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, xLen);
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void matrixSetDiag(sd::LaunchContext* context, NDArray& input, NDArray& diagonal, NDArray& output,
|
||||
const bool zeroPad) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), matrixSetDiag_, (input, diagonal, output, zeroPad), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,71 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author George A. Shulinok <sgazeos@gmail.com>
|
||||
//
|
||||
#include <ops/declarable/helpers/matrix_band.h>
|
||||
#if NOT_EXCLUDED(OP_matrix_band)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
void matrixBandPart_(NDArray* input, NDArray* output, sd::LongType lowerBand, sd::LongType upperBand) {
|
||||
// TO DO: retrieve all 2D submatrices with last dimensions and process them with given bands
|
||||
sd::LongType M = input->sizeAt(-2);
|
||||
sd::LongType N = input->sizeAt(-1);
|
||||
sd::LongType lastDim = input->rankOf() - 1;
|
||||
sd::LongType preLastDim = input->rankOf() - 2;
|
||||
ResultSet listOut = output->allTensorsAlongDimension({preLastDim, lastDim});
|
||||
ResultSet listDiag = input->allTensorsAlongDimension({preLastDim, lastDim});
|
||||
for (sd::LongType e = 0; e < static_cast<sd::LongType>(listOut.size()); ++e) {
|
||||
NDArray* inputMatrix = listDiag.at(e);
|
||||
NDArray* outputMatrix = listOut.at(e);
|
||||
if (outputMatrix != inputMatrix) // if not inplace
|
||||
outputMatrix->assign(inputMatrix);
|
||||
if (lowerBand >= 0) {
|
||||
for (sd::LongType row = 0; row < inputMatrix->rows(); ++row) {
|
||||
for (sd::LongType col = 0; col < row; ++col) {
|
||||
if ((row - col) > lowerBand) outputMatrix->p(row, col, 0.);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
if (upperBand >= 0) {
|
||||
for (sd::LongType col = 0; col < inputMatrix->columns(); ++col) {
|
||||
for (sd::LongType row = 0; row < col; ++row) {
|
||||
if ((col - row) > upperBand) outputMatrix->p(row, col, 0.);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void matrixBandPart(sd::LaunchContext* context, NDArray* input, NDArray* output, sd::LongType lowerBand,
|
||||
sd::LongType upperBand) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), matrixBandPart_, (input, output, lowerBand, upperBand), SD_FLOAT_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void matrixBandPart_,
|
||||
(NDArray * input, NDArray* output, sd::LongType lowerBand, sd::LongType upperBand),
|
||||
SD_FLOAT_TYPES);
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,66 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by GS <sgazeos@gmail.com> on 3/21/2018.
|
||||
//
|
||||
#include <array/ResultSet.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/matrix_diag_part.h>
|
||||
#if NOT_EXCLUDED(OP_matrix_diag_part)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Returns a batched matrix tensor with new batched diagonal values.
|
||||
// for detailed explanations please take a look on web page:
|
||||
// https://www.tensorflow.org/api_docs/python/tf/matrix_set_diag
|
||||
template <typename T>
|
||||
static sd::Status _matrixDiagPart(NDArray* input, NDArray* output) {
|
||||
auto listOut = output->allTensorsAlongDimension({output->rankOf() - 1});
|
||||
auto listDiag = input->allTensorsAlongDimension({input->rankOf() - 2, input->rankOf() - 1});
|
||||
|
||||
if (listOut.size() != listDiag.size()) {
|
||||
sd_printf("matrix_diag_part: Input matrix has wrong shape.", "");
|
||||
return sd::Status::VALIDATION;
|
||||
}
|
||||
sd::LongType lastDimension = sd::math::sd_min(input->sizeAt(-2), input->sizeAt(-1));
|
||||
// TODO: tune this properly
|
||||
sd::LongType lO = listOut.size();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType i = start; i < stop; i++)
|
||||
for (sd::LongType j = 0; j < lastDimension; ++j) listOut.at(i)->p(j, listDiag.at(i)->e<T>(j, j));
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, lO);
|
||||
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status matrixDiagPart(sd::LaunchContext* context, NDArray* input, NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return _matrixDiagPart, (input, output), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status _matrixDiagPart, (NDArray* input, NDArray* output), SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,84 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/convolutions.h>
|
||||
#include <ops/declarable/helpers/max_pooling.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void maxPoolingFunctor_(sd::graph::Context& block, NDArray* input, NDArray* values,
|
||||
const std::vector<LongType>& params, NDArray* indices) {
|
||||
LongType kY = params[0];
|
||||
LongType kX = params[1];
|
||||
|
||||
LongType sY = params[2];
|
||||
LongType sX = params[3];
|
||||
|
||||
sd::LongType pY = params[4];
|
||||
sd::LongType pX = params[5];
|
||||
|
||||
LongType dY = params[6];
|
||||
LongType dX = params[7];
|
||||
|
||||
LongType oY = 0;
|
||||
LongType oX = 0;
|
||||
|
||||
const LongType bSize = input->sizeAt(0);
|
||||
const LongType inD = input->sizeAt(1);
|
||||
const LongType inY = input->sizeAt(2);
|
||||
const LongType inX = input->sizeAt(3);
|
||||
|
||||
const bool isSameMode = params[8] != 0;
|
||||
|
||||
ConvolutionUtils::calcOutSizePool2D(oY, oX, kY, kX, sY, sX, pY, pX, dY, dX, inY, inX, isSameMode);
|
||||
|
||||
if (isSameMode)
|
||||
ConvolutionUtils::calcPadding2D(pY, pX, oY, oX, inY, inX, params[0], params[1], params[2], params[3], params[6],
|
||||
params[7]);
|
||||
|
||||
// 0,1 - kernel Height/Width; 2,3 - stride Height/Width; 4,5 - pad Height/Width; 6,7 - dilation Height/Width; 8 -
|
||||
// poolingMode; 9 - divisor;
|
||||
ConvolutionUtils::pooling2d(block, *input, *values, kY, kX, sY, sX, pY, pX, dY, dX, PoolingType::MAX_POOL, 1);
|
||||
|
||||
if (nullptr != indices) {
|
||||
// for max_pool_with_argmax
|
||||
int total = input->lengthOf();
|
||||
int part = total / bSize;
|
||||
|
||||
for (int k = 0; k < total;)
|
||||
for (int i = 0; i < part; i++) {
|
||||
indices->p(k++, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void maxPoolingFunctor(sd::LaunchContext* context, sd::graph::Context& block, NDArray* input, NDArray* values,
|
||||
const std::vector<LongType>& params, NDArray* indices) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), maxPoolingFunctor_, (block, input, values, params, indices),
|
||||
SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,259 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 20.04.2018
|
||||
// @author Oleh Semeniv (oleg.semeniv@gmail.com)
|
||||
//
|
||||
#include <helpers/Loops.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
#if NOT_EXCLUDED(OP_merge)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename X, typename Z>
|
||||
static void mergeMaxIndex_(const std::vector<NDArray*>& inArrs, NDArray& output) {
|
||||
const sd::LongType numArgs = inArrs.size();
|
||||
auto x = inArrs[0];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
X max = -DataTypeUtils::max<X>();
|
||||
Z idx = static_cast<Z>(0);
|
||||
|
||||
for (sd::LongType i = 0; i < numArgs; i++) {
|
||||
X v = inArrs[i]->t<X>(e);
|
||||
if (v > max) {
|
||||
max = v;
|
||||
idx = static_cast<Z>(i);
|
||||
}
|
||||
}
|
||||
|
||||
output.r<Z>(e) = static_cast<Z>(idx);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, x->lengthOf());
|
||||
}
|
||||
|
||||
void mergeMaxIndex(sd::LaunchContext* context, const std::vector<NDArray*>& inArrs, NDArray& output) {
|
||||
BUILD_DOUBLE_SELECTOR(inArrs[0]->dataType(), output.dataType(), mergeMaxIndex_, (inArrs, output), SD_NUMERIC_TYPES,
|
||||
SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void mergeMax_(const std::vector<NDArray*>& inArrs, NDArray& output) {
|
||||
const sd::LongType numArgs = inArrs.size();
|
||||
auto x = inArrs[0];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
T max = -DataTypeUtils::max<T>();
|
||||
for (sd::LongType i = 0; i < numArgs; i++) {
|
||||
T v = inArrs[i]->e<T>(e);
|
||||
if (v > max) max = v;
|
||||
}
|
||||
output.p(e, max);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, x->lengthOf());
|
||||
}
|
||||
|
||||
void mergeMax(sd::LaunchContext* context, const std::vector<NDArray*>& inArrs, NDArray& output) {
|
||||
BUILD_SINGLE_SELECTOR(output.dataType(), mergeMax_, (inArrs, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void mergeMaxBp_(const std::vector<NDArray*>& inArrs, std::vector<NDArray*>& outArrs) {
|
||||
// outArrs.size() == inArrs.size() - 1
|
||||
const sd::LongType numArgs = outArrs.size();
|
||||
// last array is gradient
|
||||
const auto gradient = inArrs[numArgs]->bufferAsT<T>();
|
||||
auto length = inArrs[numArgs]->lengthOf();
|
||||
|
||||
auto gradShape = inArrs[numArgs]->shapeInfo();
|
||||
std::vector<bool> vbSameShaepeAndStrides(numArgs);
|
||||
std::vector<sd::LongType*> vShapePtrs(numArgs);
|
||||
std::vector<sd::LongType*> vStridePtrs(numArgs);
|
||||
std::vector<sd::LongType> vRanks(numArgs);
|
||||
for (int i = 0; i < numArgs; ++i) {
|
||||
vbSameShaepeAndStrides[i] = shape::haveSameShapeAndStrides(gradShape, inArrs[i]->shapeInfo());
|
||||
vShapePtrs[i] = shape::shapeOf(inArrs[i]->shapeInfo());
|
||||
vStridePtrs[i] = shape::stride(inArrs[i]->shapeInfo());
|
||||
vRanks[i] = shape::rank(inArrs[i]->shapeInfo());
|
||||
}
|
||||
|
||||
|
||||
std::vector<sd::LongType *> outShapePtrs(numArgs);
|
||||
std::vector<sd::LongType *> outStridePtrs(numArgs);
|
||||
std::vector<sd::LongType> outRanks(numArgs);
|
||||
for (int i = 0; i < numArgs; ++i) {
|
||||
outShapePtrs[i] = shape::shapeOf(outArrs[i]->shapeInfo());
|
||||
outStridePtrs[i] = shape::stride(outArrs[i]->shapeInfo());
|
||||
outRanks[i] = shape::rank(outArrs[i]->shapeInfo());
|
||||
}
|
||||
|
||||
sd::LongType gradRank = shape::rank(gradShape);
|
||||
sd::LongType *gradShapeOf = shape::shapeOf(gradShape);
|
||||
sd::LongType *gradStride = shape::stride(gradShape);
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
for (auto e = start; e < stop; e++) {
|
||||
INDEX2COORDS(e, gradRank, gradShapeOf, coords);
|
||||
|
||||
sd::LongType gradOffset;
|
||||
COORDS2INDEX(gradRank,gradStride, coords, gradOffset);
|
||||
|
||||
T max = -DataTypeUtils::max<T>();
|
||||
sd::LongType nMaxIndex = 0;
|
||||
|
||||
for (sd::LongType i = 0; i < numArgs; i++) {
|
||||
sd::LongType xOffset;
|
||||
if (vbSameShaepeAndStrides[i]) {
|
||||
xOffset = gradOffset;
|
||||
} else {
|
||||
COORDS2INDEX(vRanks[i],vStridePtrs[i], coords, xOffset);
|
||||
}
|
||||
const T* v = inArrs[i]->bufferAsT<T>();
|
||||
if (v[xOffset] > max) {
|
||||
max = v[xOffset];
|
||||
nMaxIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
sd::LongType zOffset;
|
||||
if (vbSameShaepeAndStrides[nMaxIndex]) {
|
||||
zOffset = gradOffset;
|
||||
} else {
|
||||
COORDS2INDEX(outRanks[nMaxIndex],outStridePtrs[nMaxIndex], coords, zOffset);
|
||||
}
|
||||
|
||||
T* z = outArrs[nMaxIndex]->bufferAsT<T>();
|
||||
z[zOffset] = gradient[gradOffset];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, length);
|
||||
return;
|
||||
}
|
||||
|
||||
void mergeMaxBp(sd::LaunchContext* context, const std::vector<NDArray*>& inArrs, std::vector<NDArray*>& outArrs) {
|
||||
BUILD_SINGLE_SELECTOR(outArrs[0]->dataType(), mergeMaxBp_, (inArrs, outArrs), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void mergeAvg_(const std::vector<NDArray*>& inArrs, NDArray& output) {
|
||||
const sd::LongType numArgs = inArrs.size();
|
||||
const T factor = static_cast<T>(1.f / numArgs);
|
||||
auto x = inArrs[0];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
T sum = static_cast<T>(0);
|
||||
for (sd::LongType i = 0; i < numArgs; i++) {
|
||||
T v = inArrs[i]->e<T>(e);
|
||||
sum += v;
|
||||
}
|
||||
output.p<T>(e, sum * factor);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, x->lengthOf());
|
||||
}
|
||||
|
||||
void mergeAvg(sd::LaunchContext* context, const std::vector<NDArray*>& inArrs, NDArray& output) {
|
||||
BUILD_SINGLE_SELECTOR(output.dataType(), mergeAvg_, (inArrs, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void mergeAvgBp_(NDArray& gradient, std::vector<NDArray*>& outArrs) {
|
||||
const sd::LongType numArgs = outArrs.size();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
T v = gradient.e<T>(e) / numArgs;
|
||||
|
||||
for (sd::LongType i = 0; i < numArgs; i++) {
|
||||
outArrs[i]->p<T>(e, v);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, gradient.lengthOf());
|
||||
}
|
||||
|
||||
void mergeAvgBp(sd::LaunchContext* context, NDArray& gradient, std::vector<NDArray*>& outArrs) {
|
||||
BUILD_SINGLE_SELECTOR(gradient.dataType(), mergeAvgBp_, (gradient, outArrs), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void mergeAdd_(const std::vector<NDArray*>& inArrs, NDArray& output) {
|
||||
const sd::LongType numArgs = inArrs.size();
|
||||
auto x = inArrs[0];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
T sum = (T)0.f;
|
||||
for (sd::LongType i = 0; i < numArgs; i++) sum += inArrs[i]->e<T>(e);
|
||||
|
||||
output.p(e, sum);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, x->lengthOf());
|
||||
}
|
||||
void mergeAdd(sd::LaunchContext* context, const std::vector<NDArray*>& inArrs, NDArray& output) {
|
||||
BUILD_SINGLE_SELECTOR(output.dataType(), mergeAdd_, (inArrs, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void mergeAddBp_(NDArray& gradient, std::vector<NDArray*>& outArrs) {
|
||||
const sd::LongType numArgs = outArrs.size();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
T v = gradient.e<T>(e);
|
||||
|
||||
for (sd::LongType i = 0; i < numArgs; i++) {
|
||||
outArrs[i]->p<T>(e, v);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, gradient.lengthOf());
|
||||
}
|
||||
|
||||
void mergeAddBp(sd::LaunchContext* context, NDArray& gradient, std::vector<NDArray*>& outArrs) {
|
||||
BUILD_SINGLE_SELECTOR(gradient.dataType(), mergeAddBp_, (gradient, outArrs), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,53 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.04.2018
|
||||
//
|
||||
|
||||
#include <array/ResultSet.h>
|
||||
#include <ops/declarable/helpers/meshgrid.h>
|
||||
|
||||
#include <numeric>
|
||||
#if NOT_EXCLUDED(OP_meshgrid)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void meshgrid(sd::LaunchContext* context, const std::vector<NDArray*>& inArrs, const std::vector<NDArray*>& outArrs,
|
||||
const bool swapFirst2Dims) {
|
||||
const int rank = inArrs.size();
|
||||
int inIndices[SD_MAX_RANK];
|
||||
std::iota(inIndices, inIndices + rank, 0);
|
||||
if (swapFirst2Dims && rank > 1) {
|
||||
inIndices[0] = 1;
|
||||
inIndices[1] = 0;
|
||||
}
|
||||
|
||||
for (int i = 0; i < rank; ++i) {
|
||||
auto list = outArrs[i]->allTensorsAlongDimension({inIndices[i]});
|
||||
for (int j = 0; j < list.size(); ++j) list.at(j)->assign(inArrs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,182 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <array/NDArray.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/minimax.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void minimumBPFunctor_(LaunchContext* context, NDArray* x, NDArray* y, NDArray* epsNext, NDArray* gradX, NDArray* gradY) {
|
||||
auto lambdaX = LAMBDA_TTT(_e, _x, _y) { return _x <= _y ? _e : (T)0.; });
|
||||
|
||||
auto lambdaY = LAMBDA_TTT(_e, _x, _y) { return _x >= _y ? _e : (T)0.; });
|
||||
|
||||
if (x->isSameShape(y)) {
|
||||
// PWT case case
|
||||
|
||||
// X gradient
|
||||
epsNext->applyTriplewiseLambda<T>(x, y, lambdaX, gradX);
|
||||
|
||||
// Y gradient
|
||||
epsNext->applyTriplewiseLambda<T>(x, y, lambdaY, gradY);
|
||||
|
||||
} else if (y->isScalar()) {
|
||||
T s = y->e<T>(0);
|
||||
auto lambdaS = LAMBDA_TT(_e, _x, s) { return _x <= s ? _e : (T)0.; });
|
||||
float zero = 0.0f;
|
||||
// scalar case
|
||||
auto tmp = epsNext->reduceNumber(reduce::Sum);
|
||||
if (x <= y)
|
||||
gradY->assign(tmp);
|
||||
else
|
||||
gradY->assign(zero);
|
||||
|
||||
epsNext->applyPairwiseLambda<T>(x, lambdaS, gradX);
|
||||
delete tmp;
|
||||
} else {
|
||||
// broadcast case
|
||||
|
||||
// in this case we want to boost our X and Y shapes to the size of FF pass output (or epsNext, which has the same
|
||||
// shape)
|
||||
auto preX = x->dup();
|
||||
auto preY = y->dup();
|
||||
|
||||
auto targetShape = epsNext->getShapeAsVector();
|
||||
|
||||
preX->tileToShape(*targetShape, *preX);
|
||||
preY->tileToShape(*targetShape, *preY);
|
||||
|
||||
epsNext->applyTriplewiseLambda<T>(preX, preY, lambdaX, preX);
|
||||
epsNext->applyTriplewiseLambda<T>(preX, preY, lambdaY, preY);
|
||||
|
||||
auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
|
||||
auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
|
||||
|
||||
if (axisX.size() > 0) {
|
||||
auto sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
|
||||
gradX->assign(sum);
|
||||
} else
|
||||
gradX->assign(preX);
|
||||
|
||||
if (axisY.size() > 0) {
|
||||
auto sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
|
||||
gradY->assign(sum);
|
||||
delete sum;
|
||||
} else
|
||||
gradY->assign(preY);
|
||||
|
||||
delete targetShape;
|
||||
delete preX; // Clean up duped array
|
||||
delete preY; // Clean up duped array
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void maximumBPFunctor_(LaunchContext* context, NDArray* x, NDArray* y, NDArray* epsNext, NDArray* gradX, NDArray* gradY) {
|
||||
auto lambdaX = LAMBDA_TTT(_e, _x, _y) { return _x >= _y ? _e : (T)0.; });
|
||||
|
||||
auto lambdaY = LAMBDA_TTT(_e, _x, _y) { return _x <= _y ? _e : (T)0.; });
|
||||
|
||||
if (x->isSameShape(y)) {
|
||||
// PWT case case
|
||||
|
||||
// X gradient
|
||||
epsNext->applyTriplewiseLambda<T>(x, y, lambdaX, gradX);
|
||||
|
||||
// Y gradient
|
||||
epsNext->applyTriplewiseLambda<T>(x, y, lambdaY, gradY);
|
||||
|
||||
} else if (y->isScalar()) {
|
||||
T s = y->e<T>(0);
|
||||
auto lambdaS = LAMBDA_TT(_e, _x, s) { return _x >= s ? _e : (T)0.; });
|
||||
|
||||
// scalar case
|
||||
auto tmp = epsNext->reduceNumber(reduce::Sum);
|
||||
float zero = 0.0f;
|
||||
if (x <= y)
|
||||
gradY->assign(tmp);
|
||||
else
|
||||
gradY->assign(zero);
|
||||
|
||||
delete tmp;
|
||||
epsNext->applyPairwiseLambda<T>(x, lambdaS, gradX);
|
||||
} else {
|
||||
// broadcast case
|
||||
|
||||
// in this case we want to boost our X and Y shapes to the size of FF pass output (or epsNext, which has the same
|
||||
// shape)
|
||||
auto preX = x->dup();
|
||||
auto preY = y->dup();
|
||||
|
||||
auto targetShape = epsNext->getShapeAsVector();
|
||||
|
||||
preX->tileToShape(*targetShape, *preX);
|
||||
preY->tileToShape(*targetShape, *preY);
|
||||
|
||||
epsNext->applyTriplewiseLambda<T>(preX, preY, lambdaX, preX);
|
||||
epsNext->applyTriplewiseLambda<T>(preX, preY, lambdaY, preY);
|
||||
|
||||
auto axisX = ShapeUtils::evalBroadcastBackwardAxis(x->shapeInfo(), epsNext->shapeInfo());
|
||||
auto axisY = ShapeUtils::evalBroadcastBackwardAxis(y->shapeInfo(), epsNext->shapeInfo());
|
||||
|
||||
if (axisX.size() > 0) {
|
||||
auto sum = preX->reduceAlongDimension(reduce::Sum, &axisX);
|
||||
gradX->assign(sum);
|
||||
delete sum;
|
||||
} else
|
||||
gradX->assign(preX);
|
||||
|
||||
if (axisY.size() > 0) {
|
||||
auto sum = preY->reduceAlongDimension(reduce::Sum, &axisY);
|
||||
gradY->assign(sum);
|
||||
delete sum;
|
||||
} else
|
||||
gradY->assign(preY);
|
||||
|
||||
delete preX; // Clean up duped array
|
||||
delete preY; // Clean up duped array
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void minimumBPFunctor(LaunchContext* context, NDArray* x, NDArray* y, NDArray* epsNext, NDArray* gradX,
|
||||
NDArray* gradY) {
|
||||
BUILD_SINGLE_SELECTOR(x->dataType(), minimumBPFunctor_, (context, x, y, epsNext, gradX, gradY), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
void maximumBPFunctor(LaunchContext* context, NDArray* x, NDArray* y, NDArray* epsNext, NDArray* gradX,
|
||||
NDArray* gradY) {
|
||||
BUILD_SINGLE_SELECTOR(x->dataType(), maximumBPFunctor_, (context, x, y, epsNext, gradX, gradY), SD_NUMERIC_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void minimumBPFunctor_,
|
||||
(LaunchContext* context, NDArray* x, NDArray* y, NDArray* epsNext, NDArray* gradX, NDArray* gradY), SD_NUMERIC_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( void maximumBPFunctor_,
|
||||
(LaunchContext* context, NDArray* x, NDArray* y, NDArray* epsNext, NDArray* gradX, NDArray* gradY), SD_NUMERIC_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,72 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
|
||||
#include <ops/declarable/helpers/nth_element.h>
|
||||
#include <system/selective_rendering.h>
|
||||
#include "ops/specials.h"
|
||||
#if NOT_EXCLUDED(OP_nth_element)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
void nthElementFunctor_(NDArray* input, sd::LongType n, NDArray* output, bool reverse) {
|
||||
NDArray sortedVals(*input);
|
||||
if (input->isVector()) {
|
||||
SpecialMethods<T>::sortGeneric(input, reverse);
|
||||
output->p(0, input->e<T>(n));
|
||||
} else { // rank greater than 1
|
||||
std::vector<sd::LongType> lastDims(
|
||||
{input->rankOf() - 1});
|
||||
SpecialMethods<T>::sortTadGeneric(&sortedVals, lastDims.data(), lastDims.size(),
|
||||
reverse);
|
||||
|
||||
ResultSet rows = sortedVals.allTensorsAlongDimension(lastDims);
|
||||
sd::LongType oL = output->lengthOf();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
auto row = rows.at(e);
|
||||
output->p(e, row->e<T>(n));
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, oL);
|
||||
}
|
||||
}
|
||||
|
||||
void nthElementFunctor(sd::LaunchContext* launchContext, NDArray* input, sd::LongType n, NDArray* output,
|
||||
bool reverse) {
|
||||
|
||||
auto inputDType = input->dataType();
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), nthElementFunctor_, (input, n, output, reverse), SD_NUMERIC_TYPES);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void nthElementFunctor_,
|
||||
(NDArray * input, sd::LongType n, NDArray* output, bool reverse), SD_NUMERIC_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,97 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 30.05.2019
|
||||
//
|
||||
// CPU implementation of one_hot helper
|
||||
//
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/one_hot.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename X, typename Z>
|
||||
static void onehotImpl_(NDArray* indices, NDArray* output,
|
||||
const LongType axis, const LongType depth,
|
||||
const double on, const double off) {
|
||||
auto xBuffer = indices->bufferAsT<X>();
|
||||
auto zBuffer = output->bufferAsT<Z>();
|
||||
|
||||
auto xShapeInfo = indices->shapeInfo();
|
||||
auto zShapeInfo = output->shapeInfo();
|
||||
|
||||
const int xRank = shape::rank(xShapeInfo);
|
||||
const int zRank = shape::rank(zShapeInfo);
|
||||
const sd::LongType* xShape = shape::shapeOf(xShapeInfo);
|
||||
const sd::LongType* zShape = shape::shapeOf(zShapeInfo);
|
||||
const sd::LongType* xStride = shape::stride(xShapeInfo);
|
||||
const sd::LongType* zStride = shape::stride(zShapeInfo);
|
||||
const sd::LongType zLen = output->lengthOf();
|
||||
|
||||
const Z onVal = static_cast<Z>(on);
|
||||
const Z offVal = static_cast<Z>(off);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType coord[SD_MAX_RANK];
|
||||
|
||||
// Compute output coordinate and offset
|
||||
INDEX2COORDS(i, zRank, zShape, coord);
|
||||
sd::LongType zOffset;
|
||||
COORDS2INDEX(zRank, zStride, coord, zOffset);
|
||||
|
||||
// Extract depth coordinate and shift axis
|
||||
const auto depthCoord = coord[axis];
|
||||
for (int j = axis; j < zRank - 1; ++j) {
|
||||
coord[j] = coord[j + 1];
|
||||
}
|
||||
|
||||
// Compute input offset
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(xRank, xStride, coord, xOffset);
|
||||
|
||||
// Check if the depth matches the index
|
||||
const LongType idx = static_cast<LongType>(xBuffer[xOffset]);
|
||||
zBuffer[zOffset] = (depthCoord == idx) ? onVal : offVal;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, zLen);
|
||||
}
|
||||
|
||||
void onehot(const LaunchContext* context, NDArray* indices, NDArray* output,
|
||||
const LongType axis, const LongType depth, const double on, const double off) {
|
||||
const auto xType = indices->dataType();
|
||||
const auto zType = output->dataType();
|
||||
|
||||
NDArray::prepareSpecialUse({output}, {indices});
|
||||
|
||||
BUILD_DOUBLE_SELECTOR(xType, zType, onehotImpl_, (indices, output, axis, depth, on, off),
|
||||
SD_COMMON_TYPES, SD_COMMON_TYPES);
|
||||
|
||||
NDArray::registerSpecialUse({output}, {indices});
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,293 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 20.04.2018
|
||||
//
|
||||
|
||||
#include <helpers/Loops.h>
|
||||
#include <helpers/LoopsCoordsHelper.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
#include <system/Environment.h>
|
||||
|
||||
#include <type_traits>
|
||||
#if NOT_EXCLUDED(OP_pad)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T, size_t constRank>
|
||||
static void copy_core_rank(const T* x, T* coreZ, const sd::LongType* xShapes, const sd::LongType* xStrides,
|
||||
const sd::LongType* zStrides, int start, int stop) {
|
||||
static_assert(constRank > 1, "implement rank 1 directly");
|
||||
size_t loop_count = (stop - start);
|
||||
sd::ZipCoordsState<constRank - 1> cst;
|
||||
sd::zip_size_t offset = sd::init_coords<constRank - 1>(cst, start, xShapes, xStrides, zStrides);
|
||||
auto lastStrideX = xStrides[constRank - 1];
|
||||
auto lastStrideZ = zStrides[constRank - 1];
|
||||
auto inputLastSize = xShapes[constRank - 1];
|
||||
if (lastStrideZ == 1 && lastStrideX == 1) {
|
||||
for (auto k = 0; k < (stop - start); k++) {
|
||||
auto xPtr = &(x[offset.first]);
|
||||
auto zPtr = &(coreZ[offset.second]);
|
||||
for (int i = 0; i < inputLastSize; i++) {
|
||||
zPtr[i] = xPtr[i];
|
||||
}
|
||||
offset = sd::inc_coords<constRank - 1>(cst, offset);
|
||||
}
|
||||
} else {
|
||||
for (size_t k = 0; k < loop_count; k++) {
|
||||
auto xPtr = &(x[offset.first]);
|
||||
auto zPtr = &(coreZ[offset.second]);
|
||||
for (int i = 0; i < inputLastSize; i++) {
|
||||
zPtr[i * lastStrideZ] = xPtr[i * lastStrideX];
|
||||
}
|
||||
offset = sd::inc_coords<constRank - 1>(cst, offset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void copy_core_generic(int rank, const T* x, T* coreZ, const sd::LongType* xShapes, const sd::LongType* xStrides,
|
||||
const sd::LongType* zStrides, int start, int stop) {
|
||||
auto lastStrideX = xStrides[rank - 1];
|
||||
auto lastStrideZ = zStrides[rank - 1];
|
||||
auto inputLastSize = xShapes[rank - 1];
|
||||
sd::LongType coords[SD_MAX_RANK] = {};
|
||||
sd::LongType* ptrCoords = (sd::LongType*)&coords;
|
||||
|
||||
zip_size_t offset = {};
|
||||
if (rank > 1) {
|
||||
INDEX2COORDS(start, rank - 1, xShapes, ptrCoords);
|
||||
COORDS2INDEX(rank - 1, xStrides, ptrCoords, offset.first);
|
||||
COORDS2INDEX(rank - 1, zStrides, ptrCoords, offset.second);
|
||||
}
|
||||
if (lastStrideZ == 1 && lastStrideX == 1) {
|
||||
for (auto k = 0; k < (stop - start); k++) {
|
||||
auto xPtr = &(x[offset.first]);
|
||||
auto zPtr = &(coreZ[offset.second]);
|
||||
for (int i = 0; i < inputLastSize; i++) {
|
||||
zPtr[i] = xPtr[i];
|
||||
}
|
||||
offset = inc_coords(xShapes, xStrides, zStrides, ptrCoords, offset, rank - 1);
|
||||
}
|
||||
} else {
|
||||
for (auto k = 0; k < (stop - start); k++) {
|
||||
auto xPtr = &(x[offset.first]);
|
||||
auto zPtr = &(coreZ[offset.second]);
|
||||
for (int i = 0; i < inputLastSize; i++) {
|
||||
zPtr[i * lastStrideZ] = xPtr[i * lastStrideX];
|
||||
}
|
||||
offset = inc_coords(xShapes, xStrides, zStrides, ptrCoords, offset, rank - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
void pad_(const int mode, NDArray& input, NDArray& paddings, NDArray& output, NDArray& padValue) {
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
|
||||
const sd::LongType* xShape = input.shapeOf();
|
||||
const sd::LongType* zShape = output.shapeOf();
|
||||
|
||||
const int rank = input.rankOf(); // both input and output have the same rank
|
||||
const int rankMinusOne = rank - 1;
|
||||
const auto zLen = output.lengthOf();
|
||||
|
||||
if (mode == 0) { // CONSTANT case
|
||||
|
||||
T padVal = padValue.e<T>(0);
|
||||
|
||||
auto xShapes = input.shapeOf();
|
||||
auto outShapes = output.shapeOf();
|
||||
auto xStrides = input.stridesOf();
|
||||
auto zStrides = output.stridesOf();
|
||||
sd::LongType paddingOffsetCoords[SD_MAX_RANK] = {};
|
||||
sd::LongType* ptrPaddingCoords = (sd::LongType*)&paddingOffsetCoords;
|
||||
bool all_paddings_zero = true;
|
||||
for (int j = 0; j < rank; j++) {
|
||||
auto p0 = paddings.e<sd::LongType>(j, 0);
|
||||
auto p1 = paddings.e<sd::LongType>(j, 1);
|
||||
paddingOffsetCoords[j] = p0;
|
||||
|
||||
all_paddings_zero = all_paddings_zero && (p0 == 0) && (p1 == 0);
|
||||
}
|
||||
|
||||
sd::LongType paddingOffset;
|
||||
COORDS2INDEX(rank, zStrides, ptrPaddingCoords, paddingOffset);
|
||||
|
||||
auto inputLastSize = xShapes[rank - 1];
|
||||
|
||||
// fill everything with padding Value
|
||||
if (!all_paddings_zero) output.assign(padVal, true);
|
||||
|
||||
// fill the core from input
|
||||
auto coreZ = &(z[paddingOffset]);
|
||||
// iterate over core
|
||||
auto len = input.lengthOf() / inputLastSize;
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
if (rank == 3) {
|
||||
copy_core_rank<T, 3>(x, coreZ, xShapes, xStrides, zStrides, start, stop);
|
||||
} else if (rank == 4) {
|
||||
copy_core_rank<T, 4>(x, coreZ, xShapes, xStrides, zStrides, start, stop);
|
||||
} else if (rank == 5) {
|
||||
copy_core_rank<T, 5>(x, coreZ, xShapes, xStrides, zStrides, start, stop);
|
||||
} else {
|
||||
copy_core_generic(rank, x, coreZ, xShapes, xStrides, zStrides, start, stop);
|
||||
}
|
||||
};
|
||||
// fixed restriction for smaller inputs
|
||||
auto numThreads = (zLen > 64 || inputLastSize > 4096) ? sd::Environment::getInstance().maxMasterThreads() : 1;
|
||||
samediff::Threads::parallel_tad(func, 0, len, 1, numThreads);
|
||||
|
||||
} else { // REFLECT and SYMMETRIC cases
|
||||
|
||||
const sd::LongType shift1 = mode == 1 ? 0 : 1; // REFLECT : SYMMETRIC
|
||||
const sd::LongType shift2 = mode == 1 ? 2 : 1; // REFLECT : SYMMETRIC
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType zCoords[SD_MAX_RANK], xCoords[SD_MAX_RANK];
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, rank, shape::shapeOf(output.shapeInfo()), zCoords);
|
||||
sd::LongType zOffset;
|
||||
COORDS2INDEX(rank, shape::stride(output.shapeInfo()), zCoords, zOffset);
|
||||
|
||||
memcpy(xCoords, zCoords, rank * sizeof(sd::LongType));
|
||||
|
||||
for (int j = rankMinusOne; j >= 0; --j) {
|
||||
if (xShape[j] == zShape[j]) continue;
|
||||
|
||||
xCoords[j] =
|
||||
zCoords[j] - paddings.e<sd::LongType>(j, 0); // are ready to fill middle (within input dimension range)
|
||||
|
||||
if (xCoords[j] < 0)
|
||||
xCoords[j] = -xCoords[j] - shift1; // means fill from left
|
||||
else if (xCoords[j] >= xShape[j])
|
||||
xCoords[j] = 2 * xShape[j] - xCoords[j] - shift2; // means fill from right
|
||||
}
|
||||
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(rank, shape::stride(input.shapeInfo()), xCoords, xOffset);
|
||||
z[zOffset] = x[xOffset];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, zLen);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void pad(sd::LaunchContext* context, const int mode, NDArray& input, NDArray& paddings, NDArray& output,
|
||||
NDArray& padValue) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), pad_, (mode, input, paddings, output, padValue), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void mirrorPad_(NDArray& input, NDArray& paddings, NDArray& output, const int mode) {
|
||||
// mode: 0 - REFLECT, else - SYMMETRIC
|
||||
const int reflBorder = (bool)mode ? 1 : 0;
|
||||
const int rank = input.rankOf();
|
||||
const sd::LongType outLen = output.lengthOf();
|
||||
|
||||
// Cache shape information
|
||||
const sd::LongType* inShapeInfo = input.shapeInfo();
|
||||
const sd::LongType* outShapeInfo = output.shapeInfo();
|
||||
const sd::LongType* inShape = shape::shapeOf(inShapeInfo);
|
||||
const sd::LongType* outShape = shape::shapeOf(outShapeInfo);
|
||||
const sd::LongType* inStride = shape::stride(inShapeInfo);
|
||||
const sd::LongType* outStride = shape::stride(outShapeInfo);
|
||||
|
||||
// Cache buffers
|
||||
T* outBuf = reinterpret_cast<T*>(output.buffer());
|
||||
const T* inBuf = reinterpret_cast<T const*>(input.buffer());
|
||||
|
||||
if (input.isScalar() || input.isVector()) {
|
||||
const sd::LongType inLen = input.isScalar() ? 1 : input.lengthOf();
|
||||
const auto leftSide = paddings.e<sd::LongType>(0);
|
||||
const auto leftSideCorrected = leftSide - reflBorder;
|
||||
const sd::LongType len = 2 * (inLen - 1) + leftSide + reflBorder;
|
||||
|
||||
for (int i = 0; i < outLen; ++i) {
|
||||
if (i < leftSide) // left side
|
||||
output.p(i, input.e<T>(leftSideCorrected - i));
|
||||
else if (i >= leftSide && i < leftSide + inLen) // middle
|
||||
output.p(i, input.e<T>(i - leftSide));
|
||||
else // right side
|
||||
output.p(i, input.e<T>(len - i));
|
||||
}
|
||||
} else {
|
||||
// Cache input sizes
|
||||
std::vector<sd::LongType> inSizes(rank);
|
||||
std::vector<sd::LongType> leftSides(rank);
|
||||
std::vector<sd::LongType> leftSidesCorrected(rank);
|
||||
std::vector<sd::LongType> lens(rank);
|
||||
|
||||
// Pre-calculate size-related values for each dimension
|
||||
for (int j = 0; j < rank; ++j) {
|
||||
inSizes[j] = input.sizeAt(j);
|
||||
leftSides[j] = paddings.e<T>(j, 0);
|
||||
leftSidesCorrected[j] = leftSides[j] - reflBorder;
|
||||
lens[j] = 2 * (inSizes[j] - 1) + leftSides[j] + reflBorder;
|
||||
}
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
// Pre-allocate coordinate arrays
|
||||
sd::LongType inIdx[SD_MAX_RANK], outIdx[SD_MAX_RANK];
|
||||
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, rank, outShape, outIdx);
|
||||
|
||||
for (int j = 0; j < rank; ++j) {
|
||||
if (outIdx[j] < leftSides[j]) // left side
|
||||
inIdx[j] = leftSidesCorrected[j] - outIdx[j];
|
||||
else if (outIdx[j] >= leftSides[j] && outIdx[j] < leftSides[j] + inSizes[j]) // middle
|
||||
inIdx[j] = outIdx[j] - leftSides[j];
|
||||
else // right side
|
||||
inIdx[j] = lens[j] - outIdx[j];
|
||||
}
|
||||
|
||||
sd::LongType outOffset, inOffset;
|
||||
COORDS2INDEX(rank, outStride, outIdx, outOffset);
|
||||
COORDS2INDEX(rank, inStride, inIdx, inOffset);
|
||||
outBuf[outOffset] = inBuf[inOffset];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, outLen);
|
||||
}
|
||||
}
|
||||
void mirrorPad(sd::LaunchContext* context, NDArray& input, NDArray& paddings, NDArray& output,
|
||||
const int mode) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), mirrorPad_, (input, paddings, output, mode), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void mirrorPad_,
|
||||
(NDArray& input, NDArray& paddings, NDArray& output, const int mode),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,90 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 17.05.2018
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <array/ResultSet.h>
|
||||
#include <ops/declarable/helpers/percentile.h>
|
||||
#if NOT_EXCLUDED(OP_percentile)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void _percentile(NDArray& input, NDArray& output, std::vector<LongType>& axises, const float q,
|
||||
const int interpolation) {
|
||||
const int inputRank = input.rankOf();
|
||||
|
||||
if (axises.empty())
|
||||
for (int i = 0; i < inputRank; ++i) axises.push_back(i);
|
||||
else
|
||||
shape::checkDimensions(inputRank, &axises); // check, sort dimensions and remove duplicates if they are present
|
||||
|
||||
auto listOfSubArrs = input.allTensorsAlongDimension(axises);
|
||||
|
||||
std::vector<sd::LongType> shapeOfSubArr(listOfSubArrs.at(0)->rankOf());
|
||||
for (size_t i = 0; i < shapeOfSubArr.size(); ++i) shapeOfSubArr[i] = listOfSubArrs.at(0)->shapeOf()[i];
|
||||
|
||||
auto flattenedArr = NDArrayFactory::create('c', shapeOfSubArr, input.dataType(), input.getContext());
|
||||
const int len = flattenedArr->lengthOf();
|
||||
|
||||
const float fraction = 1.f - q / 100.;
|
||||
sd::LongType position = 0;
|
||||
|
||||
switch (interpolation) {
|
||||
case 0: // lower
|
||||
position = static_cast<sd::LongType>(math::sd_ceil<float, T>((len - 1) * fraction));
|
||||
break;
|
||||
case 1: // higher
|
||||
position = static_cast<sd::LongType>(math::sd_floor<float, T>((len - 1) * fraction));
|
||||
break;
|
||||
case 2: // nearest
|
||||
position = static_cast<sd::LongType>(math::sd_round<float, T>((len - 1) * fraction));
|
||||
break;
|
||||
}
|
||||
position = len - position - 1;
|
||||
|
||||
// FIXME: our sort impl should be used instead, so this operation might be implemented as generic
|
||||
// FIXME: parallelism !
|
||||
for (int i = 0; i < listOfSubArrs.size(); ++i) {
|
||||
auto buff = reinterpret_cast<T*>(flattenedArr->buffer());
|
||||
flattenedArr->assign(listOfSubArrs.at(i));
|
||||
std::sort(buff, buff + len);
|
||||
output.p(i, flattenedArr->e<T>(position));
|
||||
}
|
||||
|
||||
delete flattenedArr;
|
||||
}
|
||||
|
||||
void percentile(sd::LaunchContext* context, NDArray& input, NDArray& output, std::vector<LongType>& axises,
|
||||
const float q, const int interpolation) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), _percentile, (input, output, axises, q, interpolation), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _percentile,
|
||||
(NDArray& input, NDArray& output, std::vector<LongType>& axises, const float q,
|
||||
const int interpolation),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,86 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by Yurii Shyrma on 12.12.2017
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/gammaMathFunc.h>
|
||||
#include <ops/declarable/helpers/zeta.h>
|
||||
#if NOT_EXCLUDED(OP_polygamma)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// calculate factorial
|
||||
template <typename T>
|
||||
static SD_INLINE T getFactorial(const int n) {
|
||||
if (n < 0) THROW_EXCEPTION("factorial is not defined for negative number !");
|
||||
|
||||
if (n == 0 || n == 1) return (T)1.f;
|
||||
|
||||
T result = (T)1.f;
|
||||
|
||||
for (int i = 2; i <= n; ++i) result *= i;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// implementation is based on serial representation written in terms of the Hurwitz zeta function as polygamma =
|
||||
// (-1)^{n+1} * n! * zeta(n+1, x)
|
||||
template <typename T>
|
||||
static SD_INLINE T polyGammaScalar(sd::LaunchContext* context, const int n, const T x) {
|
||||
int sign = (n + 1) % 2 ? -1 : 1;
|
||||
T zeta = zetaScalar<T>(T(n + 1), x);
|
||||
return T(sign) * getFactorial<T>(n) * zeta;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// calculate polygamma function for arrays
|
||||
template <typename T>
|
||||
static void polyGamma_(sd::LaunchContext* context, NDArray& n, NDArray& x, NDArray& output) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
const T order = n.e<T>(i);
|
||||
if (order !=
|
||||
static_cast<int>(order)) // if order has fractional part then do not perform calculations and return NAN
|
||||
output.p(i, std::numeric_limits<T>::quiet_NaN());
|
||||
else if (order == 0) // polygamma function of zero order is digamma function
|
||||
output.p(i, diGammaScalar<T>(x.e<T>(i)));
|
||||
else
|
||||
output.p(i, polyGammaScalar<T>(context, order, x.e<T>(i)));
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, x.lengthOf());
|
||||
}
|
||||
|
||||
void polyGamma(sd::LaunchContext* context, NDArray& n, NDArray& x, NDArray& output) {
|
||||
BUILD_SINGLE_SELECTOR(x.dataType(), polyGamma_, (context, n, x, output), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void polyGamma_,
|
||||
(sd::LaunchContext * context, NDArray& n, NDArray& x, NDArray& output),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,127 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#include <helpers/shape.h>
|
||||
#include <ops/declarable/helpers/prefix.h>
|
||||
#include <ops/ops.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static void prefix_(scalar::Ops op, const void* vx, sd::LongType const* xShapeInfo, void* vz,
|
||||
sd::LongType const* zShapeInfo, bool exclusive, bool reverse) {
|
||||
const auto x = reinterpret_cast<const T*>(vx);
|
||||
auto z = reinterpret_cast<T*>(vz);
|
||||
auto length = shape::length(xShapeInfo);
|
||||
|
||||
T prevSum = op == scalar::Add ? (T)0 : (T)1;
|
||||
T sum = prevSum;
|
||||
|
||||
LongType xCoords[SD_MAX_RANK];
|
||||
LongType zCoords[SD_MAX_RANK];
|
||||
LongType xOffset;
|
||||
LongType zOffset;
|
||||
|
||||
sd::LongType xRank = shape::rank(xShapeInfo);
|
||||
sd::LongType zRank = shape::rank(zShapeInfo);
|
||||
sd::LongType *xShape = shape::shapeOf(xShapeInfo);
|
||||
sd::LongType *xStride = shape::stride(xShapeInfo);
|
||||
sd::LongType *zShape = shape::shapeOf(zShapeInfo);
|
||||
sd::LongType *zStride = shape::stride(zShapeInfo);
|
||||
|
||||
|
||||
if (reverse) {
|
||||
for (sd::LongType e = length - 1; e >= 0; --e) {
|
||||
INDEX2COORDS(e, xRank, xShape, xCoords);
|
||||
COORDS2INDEX(xRank, xStride, xCoords, xOffset);
|
||||
INDEX2COORDS(e, zRank, zShape, zCoords);
|
||||
COORDS2INDEX(zRank, zStride, zCoords, zOffset);
|
||||
|
||||
sum = op == scalar::Add ? simdOps::Add<T, T, T>::op(sum, x[xOffset])
|
||||
: simdOps::Multiply<T, T, T>::op(sum, x[xOffset]);
|
||||
|
||||
if (!exclusive) prevSum = sum;
|
||||
|
||||
z[zOffset] = prevSum;
|
||||
prevSum = sum;
|
||||
}
|
||||
} else {
|
||||
for (sd::LongType e = 0; e < length; e++) {
|
||||
INDEX2COORDS(e, xRank, xShape, xCoords);
|
||||
COORDS2INDEX(xRank, xStride, xCoords, xOffset);
|
||||
INDEX2COORDS(e, zRank, zShape, zCoords);
|
||||
COORDS2INDEX(zRank, zStride, zCoords, zOffset);
|
||||
|
||||
sum = op == scalar::Add ? simdOps::Add<T, T, T>::op(sum, x[xOffset])
|
||||
: simdOps::Multiply<T, T, T>::op(sum, x[xOffset]);
|
||||
|
||||
if (!exclusive) prevSum = sum;
|
||||
|
||||
z[zOffset] = prevSum;
|
||||
prevSum = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
template <typename T>
|
||||
static void prefix_(scalar::Ops op, NDArray* x, NDArray* z, const std::vector<LongType>& dims, bool exclusive,
|
||||
bool reverse) {
|
||||
auto xTads = x->allTensorsAlongDimension(dims);
|
||||
auto zTads = z->allTensorsAlongDimension(dims);
|
||||
auto t = xTads.size();
|
||||
|
||||
for (int e = 0; e < t; e++) {
|
||||
auto tx = xTads.at(e);
|
||||
auto tz = zTads.at(e);
|
||||
|
||||
prefix_<T>(op, tx->buffer(), tx->shapeInfo(), tz->buffer(), tz->shapeInfo(), exclusive, reverse);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
static void prefix_(scalar::Ops op, NDArray* x, NDArray* z, bool exclusive, bool reverse) {
|
||||
prefix_<T>(op, x->buffer(), x->shapeInfo(), z->buffer(), z->shapeInfo(), exclusive, reverse);
|
||||
};
|
||||
|
||||
void prefix(sd::LaunchContext* context, scalar::Ops op, NDArray* x, NDArray* z, bool exclusive, bool reverse) {
|
||||
BUILD_SINGLE_SELECTOR(x->dataType(), prefix_, (op, x, z, exclusive, reverse), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
void prefix(sd::LaunchContext* context, scalar::Ops op, NDArray* x, NDArray* z, const std::vector<sd::LongType>& dims,
|
||||
bool exclusive, bool reverse) {
|
||||
BUILD_SINGLE_SELECTOR(x->dataType(), prefix_, (op, x, z, dims, exclusive, reverse), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void prefix_,
|
||||
(scalar::Ops op, const void* vx, sd::LongType const* xShapeInfo, void* vz,
|
||||
sd::LongType const* zShapeInfo, bool exclusive, bool reverse),
|
||||
SD_NUMERIC_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( void prefix_,
|
||||
(scalar::Ops op, NDArray* x, NDArray* z, const std::vector<sd::LongType>& dims, bool exclusive,
|
||||
bool reverse),
|
||||
SD_NUMERIC_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( void prefix_,
|
||||
(scalar::Ops op, NDArray* x, NDArray* z, bool exclusive, bool reverse), SD_NUMERIC_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,32 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/print_variable.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
void print_special(LaunchContext &ctx, NDArray&array, const std::string &message) {
|
||||
array.printIndexedBuffer(message.c_str());
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author George A. Shulinok <sgazeos@gmail.com>
|
||||
//
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/helpers/qr.h>
|
||||
#if NOT_EXCLUDED(OP_qr)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
NDArray matrixMinor(NDArray& in, sd::LongType col) {
|
||||
NDArray* m = in.ulike();
|
||||
m->setIdentity();
|
||||
auto mRef = *m;
|
||||
auto view = mRef({col, m->rows(), col, m->columns()});
|
||||
auto inView = in({col, m->rows(), col, m->columns()});
|
||||
view->assign(inView);
|
||||
delete view;
|
||||
delete inView;
|
||||
delete m;
|
||||
|
||||
return mRef;
|
||||
}
|
||||
|
||||
/* m = I - v v^T */
|
||||
template <typename T>
|
||||
NDArray vmul(NDArray& v, int n) {
|
||||
std::vector<sd::LongType> nShape = {n,n};
|
||||
NDArray res('c', nShape, v.dataType(), v.getContext()); // x = matrix_new(n, n);
|
||||
T const* vBuf = v.getDataBuffer()->primaryAsT<T>();
|
||||
T* resBuf = res.dataBuffer()->primaryAsT<T>();
|
||||
auto interloop = PRAGMA_THREADS_FOR_2D {
|
||||
for (auto i = start_x; i < n; i += inc_x)
|
||||
for (auto j = start_y; j < n; j += inc_y) resBuf[i * n + j] = -2 * vBuf[i] * vBuf[j] + (i == j ? T(1) : T(0));
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(interloop, 0, n, 1, 0, n, 1);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void qrSingle(NDArray* matrix, NDArray* Q, NDArray* R, bool const fullMatricies) {
|
||||
sd::LongType M = matrix->sizeAt(-2);
|
||||
sd::LongType N = matrix->sizeAt(-1);
|
||||
auto resQ = fullMatricies ? Q->ulike() : new NDArray(NDArrayFactory::create<T>(matrix->ordering(), {M, M}, Q->getContext()));
|
||||
auto resR = fullMatricies ? R->ulike() : matrix->ulike();
|
||||
std::vector<NDArray*> q(M, nullptr);
|
||||
|
||||
std::vector<sd::LongType> mShape = {M};
|
||||
NDArray z = *matrix;
|
||||
NDArray e('c', mShape, DataTypeUtils::fromT<T>(), Q->getContext()); // two internal buffers and scalar for squared norm
|
||||
|
||||
for (sd::LongType k = 0; k < N && k < M - 1; k++) { // loop for columns, but not further then row number
|
||||
e.nullify();
|
||||
z = matrixMinor<T>(z, k); // minor computing for current column with given matrix z (initally is a input matrix)
|
||||
|
||||
std::vector<sd::LongType> zeroVec = {0};
|
||||
auto currentColumn = z({0, 0, k, k + 1}); // retrieve k column from z to x buffer
|
||||
auto *normPtr = currentColumn->reduceAlongDimension(reduce::Norm2,&zeroVec);
|
||||
NDArray norm = *normPtr;
|
||||
delete normPtr;
|
||||
|
||||
if (matrix->t<T>(k, k) > T(0.f)) { // negate on positive matrix diagonal element
|
||||
NDArray *negNorm = norm * T(-1.f);
|
||||
norm.assign(negNorm);
|
||||
delete negNorm;
|
||||
}
|
||||
|
||||
e.p(k, &norm);
|
||||
NDArray *ePlusColumn = e + (*currentColumn);
|
||||
e.assign(ePlusColumn);
|
||||
delete ePlusColumn;
|
||||
|
||||
auto *normEPtr = e.reduceAlongDimension(reduce::Norm2, &zeroVec);
|
||||
NDArray *eDivNormE = e / (*normEPtr);
|
||||
e.assign(eDivNormE);
|
||||
delete eDivNormE;
|
||||
delete normEPtr;
|
||||
|
||||
q[k] = new NDArray(vmul<T>(e, M));
|
||||
auto qQ = z.ulike();
|
||||
MmulHelper::matmul(q[k], &z, qQ, false, false, 0, 0, qQ);
|
||||
z = std::move(*qQ);
|
||||
|
||||
delete currentColumn;
|
||||
}
|
||||
|
||||
|
||||
resQ->assign(q[0]); //
|
||||
|
||||
for (sd::LongType i = 1; i < N && i < M - 1; i++) {
|
||||
auto tempResQ = resQ;
|
||||
MmulHelper::matmul(q[i], resQ, tempResQ, false, false, 0, 0, tempResQ); // use mmulMxM?
|
||||
resQ = std::move(tempResQ);
|
||||
}
|
||||
MmulHelper::matmul(resQ, matrix, resR, false, false, 0, 0, resR);
|
||||
// resR *= -1.f;
|
||||
resQ->transposei();
|
||||
if (fullMatricies) {
|
||||
Q->assign(resQ);
|
||||
R->assign(resR);
|
||||
} else {
|
||||
auto resQRef = *resQ;
|
||||
auto resRRef = *resR;
|
||||
auto resQView = resQRef({0, 0, 0, N});
|
||||
auto resRView = resRRef({0, N, 0, 0});
|
||||
Q->assign(resQView);
|
||||
R->assign(resRView);
|
||||
delete resQView;
|
||||
delete resRView;
|
||||
}
|
||||
|
||||
// Clean up allocated NDArrays in q vector
|
||||
for (sd::LongType i = 0; i < M; i++) {
|
||||
if (q[i] != nullptr) {
|
||||
delete q[i];
|
||||
}
|
||||
}
|
||||
|
||||
delete resQ;
|
||||
delete resR;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void qr_(NDArray * input, NDArray* outputQ, NDArray* outputR, bool const fullMatricies) {
|
||||
sd::LongType lastDim = input->rankOf() - 1;
|
||||
sd::LongType preLastDim = input->rankOf() - 2;
|
||||
ResultSet listOutQ(outputQ->allTensorsAlongDimension({(int)preLastDim, (int)lastDim}));
|
||||
ResultSet listOutR(outputR->allTensorsAlongDimension({(int)preLastDim, (int)lastDim}));
|
||||
ResultSet listInput(input->allTensorsAlongDimension({(int)preLastDim, (int)lastDim}));
|
||||
auto batching = PRAGMA_THREADS_FOR {
|
||||
for (auto batch = start; batch < stop; batch++) {
|
||||
// qr here
|
||||
qrSingle<T>(listInput.at(batch), listOutQ.at(batch), listOutR.at(batch), fullMatricies);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(batching, 0, listOutQ.size(), 1);
|
||||
}
|
||||
|
||||
void qr(sd::LaunchContext* context, NDArray * input, NDArray* outputQ, NDArray* outputR,
|
||||
bool const fullMatricies) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), qr_, (input, outputQ, outputR, fullMatricies), SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,297 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/random.h>
|
||||
#include <memory>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ConstantTadHelper.h>
|
||||
#include <helpers/RandomLauncher.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#if NOT_EXCLUDED(OP_random)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
/**
|
||||
* gammaLess - compute gamma distributed value for shapes (alpha) from 0 to 1
|
||||
* @tparam T - any float types are acceptable
|
||||
* @param rng - random generator for uniformly vals
|
||||
* @param alpha - shape of distribution
|
||||
* @param beta - scale of distributed values
|
||||
* @return gamma distributed value
|
||||
*/
|
||||
template <typename T>
|
||||
T gammaLess(graph::RandomGenerator& rng, T const alpha, T const beta) {
|
||||
auto d = T(1.0334f) - T(0.0766f) * math::p_exp(T(2.2942f) * alpha);
|
||||
auto a = math::p_pow(T(2.f), alpha) * math::p_pow<T>(T(1.f) - math::p_exp(-d * T(0.5f)), alpha);
|
||||
auto b = alpha * math::p_pow(d, alpha - T(1.f)) * exp(-d);
|
||||
auto c = a + b;
|
||||
T rawX;
|
||||
static sd::LongType index = 0;
|
||||
const T underAlpha = T(1.f) / alpha;
|
||||
const T powerAlpha = math::p_pow<T>(T(2.f), alpha - T(1.f));
|
||||
|
||||
for (;;) {
|
||||
auto u = rng.relativeT<T>(index++, T(0.f), T(1.f));
|
||||
|
||||
if (u <= a / c)
|
||||
rawX = -T(2.f) * math::p_log(T(1.f) - T(0.5f) * math::p_pow(T(c * u), underAlpha));
|
||||
else
|
||||
rawX = -math::p_log(c * (T(1.f) - u) / (alpha * math::p_pow(d, alpha - T(1.f))));
|
||||
|
||||
T v = static_cast<T>(rng.relativeT(index++, 0.f, 1.f));
|
||||
if (rawX <= d) {
|
||||
auto testVal = (math::p_pow(rawX, alpha - 1.f) * math::p_exp(-T(0.5f) * rawX)) /
|
||||
(powerAlpha * math::p_pow(T(1.f) - math::p_exp(-T(0.5f) * rawX), alpha - T(1.f)));
|
||||
if (testVal < v) continue;
|
||||
break;
|
||||
} else {
|
||||
if (v <= math::p_pow<T>(d / rawX, T(1.f) - alpha)) break;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return rawX / beta;
|
||||
}
|
||||
|
||||
/**
|
||||
* gammaGreat - generate gamma distributed value for shape (alpha) greater then 1
|
||||
* @tparam T - given type (any float type is accepted.)
|
||||
* @param rng - random generator
|
||||
* @param alpha - shape of the gamma distribution (alpha)
|
||||
* @param beta - scale of the gamma distribution (beta)
|
||||
* @return - gamma distributed value with given params
|
||||
*/
|
||||
template <typename T>
|
||||
T gammaGreat(graph::RandomGenerator& rng, T const alpha, T const beta) {
|
||||
auto decreasedAlpha = alpha - T(1.f / 3.f);
|
||||
auto c = T(1.) / math::p_sqrt(T(9.f) * decreasedAlpha);
|
||||
static sd::LongType index = 0;
|
||||
T x;
|
||||
auto normalDistributed = [](graph::RandomGenerator& rng, sd::LongType& index) {
|
||||
auto v1 = rng.relativeT(index++, T(0.f), T(1.f));
|
||||
auto v2 = rng.relativeT(index++, T(0.f), T(1.f));
|
||||
|
||||
return math::p_cos(T(2.f * 3.141592f) * v2) * math::p_sqrt(T(-2.f) * math::p_log(v1));
|
||||
};
|
||||
|
||||
float normalizedVar;
|
||||
for (;;) {
|
||||
do {
|
||||
x = normalDistributed(rng, index);
|
||||
normalizedVar = T(1.f) + c * x;
|
||||
} while (normalizedVar < T(0.f));
|
||||
normalizedVar = normalizedVar * normalizedVar * normalizedVar; // v * v * v;
|
||||
|
||||
auto u = rng.relativeT<T>(index++, T(0.f), T(1.f));
|
||||
if (u < T(1.f) - T(.0331f) * (x * x) * (x * x)) break;
|
||||
if (log(u) < 0.5f * x * x + decreasedAlpha * (1. - normalizedVar + math::p_log(normalizedVar))) break;
|
||||
}
|
||||
return (decreasedAlpha * normalizedVar / beta);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void fillRandomGamma_(LaunchContext* context, graph::RandomGenerator& rng, NDArray* alpha, NDArray* beta,
|
||||
NDArray* output) {
|
||||
auto broadcasted = alpha->shapeInfo();
|
||||
if (beta != nullptr) {
|
||||
sd::LongType* broadcastedShape = nullptr;
|
||||
ShapeUtils::evalBroadcastShapeInfo(alpha->shapeInfo(), beta->shapeInfo(), true, broadcastedShape, context->getWorkspace());
|
||||
broadcasted = broadcastedShape;
|
||||
}
|
||||
|
||||
auto step = shape::length(broadcasted);
|
||||
auto shift = output->lengthOf() / step;
|
||||
|
||||
auto copyAlpha = alpha;
|
||||
auto copyBeta = beta;
|
||||
if (beta != nullptr) {
|
||||
NDArray alphaBroadcasted(broadcasted, alpha->dataType(), false, context);
|
||||
NDArray betaBroadcasted(broadcasted, beta->dataType(), false, context);
|
||||
|
||||
copyAlpha = alphaBroadcasted.applyTrueBroadcast(BroadcastOpsTuple::Assign(), alpha);
|
||||
copyBeta = betaBroadcasted.applyTrueBroadcast(BroadcastOpsTuple::Assign(), beta);
|
||||
}
|
||||
bool directOutput = output->ews() == 1 && output->ordering() == 'c';
|
||||
T* outputBuf = output->dataBuffer()->primaryAsT<T>();
|
||||
|
||||
PRAGMA_OMP_PARALLEL_FOR
|
||||
for (sd::LongType k = 0; k < shift; k++) {
|
||||
auto pos = k * step;
|
||||
for (sd::LongType e = 0; e < step; e++)
|
||||
if (directOutput) {
|
||||
outputBuf[pos + e] = copyAlpha->t<T>(e) <= 1
|
||||
? gammaLess(rng, copyAlpha->t<T>(e), beta ? copyBeta->t<T>(e) : T(1.f))
|
||||
: gammaGreat(rng, copyAlpha->t<T>(e), beta ? copyBeta->t<T>(e) : T(1.f));
|
||||
} else {
|
||||
output->r<T>(pos + e) = copyAlpha->t<T>(e) <= 1
|
||||
? gammaLess(rng, copyAlpha->t<T>(e), beta ? copyBeta->t<T>(e) : T(1.f))
|
||||
: gammaGreat(rng, copyAlpha->t<T>(e), beta ? copyBeta->t<T>(e) : T(1.f));
|
||||
}
|
||||
}
|
||||
|
||||
if (beta != nullptr) {
|
||||
delete copyAlpha;
|
||||
delete copyBeta;
|
||||
}
|
||||
}
|
||||
|
||||
void fillRandomGamma(LaunchContext* context, graph::RandomGenerator& rng, NDArray* alpha, NDArray* beta,
|
||||
NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(output->dataType(), fillRandomGamma_, (context, rng, alpha, beta, output), SD_FLOAT_NATIVE);
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void fillRandomGamma_,
|
||||
(LaunchContext * context, graph::RandomGenerator& rng, NDArray* alpha, NDArray* beta,
|
||||
NDArray* output),
|
||||
SD_FLOAT_NATIVE);
|
||||
|
||||
/*
|
||||
* algorithm Poisson generator based upon the inversion by sequential search:[48]:505
|
||||
init:
|
||||
Let x ← 0, p ← e−λ, s ← p.
|
||||
Generate uniform random number u in [0,1].
|
||||
while u > s do:
|
||||
x ← x + 1.
|
||||
p ← p * λ / x.
|
||||
s ← s + p.
|
||||
return x.
|
||||
* */
|
||||
template <typename T, typename Z>
|
||||
void fillRandomPoisson_(LaunchContext* context, graph::RandomGenerator& rng, NDArray* lambda, NDArray* output) {
|
||||
auto shift = output->lengthOf() / lambda->lengthOf();
|
||||
auto step = lambda->lengthOf();
|
||||
T* lambdaBuf = lambda->dataBuffer()->primaryAsT<T>();
|
||||
Z* outputBuf = output->dataBuffer()->primaryAsT<Z>();
|
||||
bool directLa = lambda->ews() == 1 && lambda->ordering() == 'c';
|
||||
bool directOut = output->ews() == 1 && output->ordering() == 'c';
|
||||
PRAGMA_OMP_PARALLEL_FOR
|
||||
for (sd::LongType k = 0; k < shift; k++) {
|
||||
auto pos = k * step;
|
||||
auto u = rng.relativeT<T>(k, static_cast<T>(0.), static_cast<T>(1.));
|
||||
for (sd::LongType e = 0; e < step; e++) {
|
||||
auto p = math::sd_exp<T, T>(-lambda->t<T>(e));
|
||||
auto s = p;
|
||||
auto x = Z(0.f);
|
||||
while (u > s) {
|
||||
x += 1.f;
|
||||
p *= static_cast<T>(directLa ? lambdaBuf[e] / x : lambda->t<T>(e) / x);
|
||||
s += p;
|
||||
}
|
||||
if (directOut)
|
||||
outputBuf[pos + e] = x;
|
||||
else
|
||||
output->r<Z>(pos + e) = x;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void fillRandomPoisson(LaunchContext* context, graph::RandomGenerator& rng, NDArray* lambda, NDArray* output) {
|
||||
BUILD_DOUBLE_SELECTOR(lambda->dataType(), output->dataType(), fillRandomPoisson_, (context, rng, lambda, output),
|
||||
SD_FLOAT_TYPES, SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( void fillRandomPoisson_,
|
||||
(LaunchContext * context, graph::RandomGenerator& rng, NDArray* lambda, NDArray* output),
|
||||
SD_FLOAT_TYPES, SD_FLOAT_TYPES);
|
||||
|
||||
template <typename T>
|
||||
void fillRandomUniform_(LaunchContext* context, graph::RandomGenerator& rng, NDArray* min, NDArray* max,
|
||||
NDArray* output) {
|
||||
T minVal = T(0);
|
||||
T maxVal = DataTypeUtils::max<T>();
|
||||
if (min) minVal = min->t<T>(0);
|
||||
if (max) maxVal = max->t<T>(0);
|
||||
|
||||
if (output->isR())
|
||||
RandomLauncher::fillUniform(context, rng, output, minVal, maxVal);
|
||||
else {
|
||||
PRAGMA_OMP_PARALLEL_FOR
|
||||
for (sd::LongType i = 0; i < output->lengthOf(); i++) {
|
||||
output->r<T>(i) = rng.relativeT<T>(i, minVal, maxVal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void fillRandomUniform(LaunchContext* context, graph::RandomGenerator& rng, NDArray* min, NDArray* max,
|
||||
NDArray* output) {
|
||||
BUILD_SINGLE_SELECTOR(output->dataType(), fillRandomUniform_, (context, rng, min, max, output), SD_NUMERIC_TYPES);
|
||||
}
|
||||
|
||||
// used https://en.wikipedia.org/wiki/Categorical_distribution
|
||||
// methods: gumbel trick + softmax + argmax
|
||||
template <typename Tx, typename Tz>
|
||||
void fillRandomMultiNomial_(LaunchContext* context, graph::RandomGenerator& rng, NDArray& input, NDArray& output,
|
||||
const sd::LongType numOfSamples, const int dimC) {
|
||||
const Tx* x = input.bufferAsT<Tx>();
|
||||
Tz* z = output.bufferAsT<Tz>();
|
||||
|
||||
Tx minVal = DataTypeUtils::min_positive<Tx>();
|
||||
Tx maxVal = static_cast<Tx>(1.0);
|
||||
|
||||
auto dimA = (0 == dimC) ? 1 : 0;
|
||||
const sd::LongType batchValue = output.sizeAt(dimC);
|
||||
const sd::LongType numOfClassX = input.sizeAt(dimA);
|
||||
|
||||
const sd::LongType zDimAstride = output.stridesOf()[dimA];
|
||||
const sd::LongType xDimAstride = input.stridesOf()[dimA];
|
||||
const sd::LongType zDimCstride = output.stridesOf()[dimC];
|
||||
const sd::LongType xDimCstride = input.stridesOf()[dimC];
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
for (auto nBatchIndex = start_x; nBatchIndex < stop_x; nBatchIndex += inc_x) {
|
||||
for (auto nSampleIndexInBatch = start_y; nSampleIndexInBatch < stop_y; nSampleIndexInBatch += inc_y) {
|
||||
const Tx* xTad = x + (nBatchIndex * xDimCstride);
|
||||
Tz* zTad = z + (nBatchIndex * zDimCstride);
|
||||
Tz& arg = zTad[nSampleIndexInBatch * zDimAstride];
|
||||
Tx Max = -minVal;
|
||||
|
||||
auto nSamplesPerBatch = nBatchIndex * numOfClassX * numOfSamples;
|
||||
auto nClassesPerSample = nSampleIndexInBatch * numOfClassX;
|
||||
for (sd::LongType nClass = 0; nClass < numOfClassX; nClass += 1) {
|
||||
auto nIndex = nSamplesPerBatch + nClassesPerSample + nClass;
|
||||
auto unifornLog =
|
||||
sd::math::sd_log<Tx, Tx>(-sd::math::sd_log<Tx, Tx>(rng.relativeT<Tx>(nIndex, minVal, maxVal)));
|
||||
Tx tValue = (xTad[nClass * xDimAstride] - unifornLog);
|
||||
if (tValue > Max) {
|
||||
Max = tValue;
|
||||
arg = nClass;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, batchValue, 1, 0, numOfSamples, 1);
|
||||
rng.rewindH(output.lengthOf() * numOfClassX);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void fillRandomMultiNomial(LaunchContext* context, graph::RandomGenerator& rng, NDArray& input, NDArray& output,
|
||||
const sd::LongType numOfSamples, const int dimC) {
|
||||
BUILD_DOUBLE_SELECTOR(input.dataType(), output.dataType(), fillRandomMultiNomial_,
|
||||
(context, rng, input, output, numOfSamples, dimC), SD_FLOAT_TYPES, SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,161 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 20.04.2018
|
||||
// implementation is based on following article:
|
||||
// "MergeShuffle: A Very Fast, Parallel Random Permutation Algorithm", https://arxiv.org/abs/1508.03167
|
||||
|
||||
#include <graph/RandomGenerator.h>
|
||||
#include <helpers/Loops.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
|
||||
#include <numeric>
|
||||
#if NOT_EXCLUDED(OP_random_shuffle)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Fisher-Yates shuffle
|
||||
template <typename T>
|
||||
static void fisherYates(sd::graph::RandomGenerator& rng, T* buff, const sd::LongType& len, const sd::LongType& ews,
|
||||
sd::LongType ind) {
|
||||
for (sd::LongType i = len - 1; i > 0; --i) {
|
||||
const sd::LongType j = rng.relativeLong(ind++) % (i + 1);
|
||||
if (i != j) math::sd_swap<T>(buff[i * ews], buff[j * ews]);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// mutual shuffle of two adjacent already shuffled ranges with length len1 and (totLen - len1) correspondingly
|
||||
template <typename T>
|
||||
static void mergeShuffle(sd::graph::RandomGenerator& rng, T* buff, const sd::LongType& len1, const sd::LongType& totLen,
|
||||
const sd::LongType& ews, sd::LongType ind) {
|
||||
sd::LongType beg = 0; // beginning
|
||||
sd::LongType mid = len1; // middle
|
||||
|
||||
while (true) {
|
||||
if (rng.relativeLong(ind++) % 2) {
|
||||
if (mid == totLen) break;
|
||||
math::sd_swap<T>(buff[ews * beg], buff[ews * mid++]);
|
||||
} else {
|
||||
if (beg == mid) break;
|
||||
}
|
||||
++beg;
|
||||
}
|
||||
|
||||
// fisherYates
|
||||
while (beg < totLen) {
|
||||
const sd::LongType j = rng.relativeLong(ind++) % (beg + 1);
|
||||
if (beg != j) math::sd_swap<T>(buff[ews * beg], buff[ews * j]);
|
||||
++beg;
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void randomShuffle_(NDArray& input, NDArray& output, sd::graph::RandomGenerator& rng, const bool isInplace) {
|
||||
const int firstDim = input.sizeAt(0);
|
||||
sd::LongType temp;
|
||||
|
||||
if (input.lengthOf() == 1 || firstDim == 1) {
|
||||
if (!isInplace) output.assign(&input);
|
||||
} else if (shape::isCommonVector(input.shapeInfo(), temp)) {
|
||||
NDArray* arr = &input;
|
||||
|
||||
if (!isInplace) {
|
||||
output.assign(&input);
|
||||
arr = &output;
|
||||
}
|
||||
|
||||
const sd::LongType ews = arr->ews();
|
||||
|
||||
const sd::LongType len = arr->lengthOf();
|
||||
const sd::LongType threshold = 1 << 22; // this number was deduced from diagram in article
|
||||
|
||||
int power = 0;
|
||||
while ((len >> power) > threshold) ++power;
|
||||
|
||||
const sd::LongType numChunks = 1 << power;
|
||||
|
||||
auto funcFisherYates = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; ++i) {
|
||||
sd::LongType offset = (len * i) >> power;
|
||||
sd::LongType currLen = ((len * (i + 1)) >> power) - offset;
|
||||
fisherYates<T>(rng, arr->bufferAsT<T>() + offset * ews, currLen, ews, offset);
|
||||
}
|
||||
};
|
||||
|
||||
auto funcMerge = PRAGMA_THREADS_FOR {
|
||||
for (int64_t i = start, k = 1; i < stop; i += increment, ++k) {
|
||||
sd::LongType offset = len * i >> power;
|
||||
sd::LongType len1 = (len * (i + increment / 2) >> power) - offset;
|
||||
sd::LongType totLen = (len * (i + increment) >> power) - offset;
|
||||
mergeShuffle<T>(rng, arr->bufferAsT<T>() + offset * ews, len1, totLen, ews, len * k + offset);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(funcFisherYates, 0, numChunks);
|
||||
|
||||
for (int j = 1; j < numChunks; j += j) samediff::Threads::parallel_for(funcMerge, 0, numChunks, 2 * j);
|
||||
|
||||
rng.rewindH((len + 1) * power);
|
||||
} else {
|
||||
std::vector<sd::LongType> zeroDim = {0};
|
||||
auto dimsToExclude = ShapeUtils::evalDimsToExclude(input.rankOf(), 1,zeroDim.data());
|
||||
|
||||
if (isInplace) {
|
||||
auto subArrsList = input.allTensorsAlongDimension(*dimsToExclude);
|
||||
|
||||
// Fisher-Yates shuffle
|
||||
for (int i = firstDim - 1; i > 0; --i) {
|
||||
const int j = rng.relativeInt(i) % (i + 1);
|
||||
if (i != j) subArrsList.at(i)->swapUnsafe(*subArrsList.at(j));
|
||||
}
|
||||
} else {
|
||||
auto subArrsListIn = input.allTensorsAlongDimension(*dimsToExclude);
|
||||
auto subArrsListOut = output.allTensorsAlongDimension(*dimsToExclude);
|
||||
delete dimsToExclude;
|
||||
std::vector<int> indices(firstDim);
|
||||
std::iota(indices.begin(), indices.end(), 0); // 0,1,2,3, ... firstDim-1
|
||||
|
||||
// shuffle indices
|
||||
fisherYates<int>(rng, indices.data(), firstDim, 1, 0);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; ++i) subArrsListOut.at(i)->assign(subArrsListIn.at(indices[i]));
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, firstDim);
|
||||
}
|
||||
|
||||
rng.rewindH(firstDim - 1);
|
||||
}
|
||||
}
|
||||
|
||||
void randomShuffle(sd::LaunchContext* context, NDArray& input, NDArray& output, sd::graph::RandomGenerator& rng,
|
||||
const bool isInplace) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), randomShuffle_, (input, output, rng, isInplace), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,77 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/random_crop.h>
|
||||
#include <graph/Context.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_random_shuffle)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static sd::Status _randomCropFunctor(graph::Context& context, NDArray* input, NDArray* shape, NDArray* output,
|
||||
int seed) {
|
||||
graph::RandomGenerator rngX(context.getRng());
|
||||
// functions::random::RandomFunction<T>::template execTransform<randomOps::UniformDistribution<T>>(rng,
|
||||
// output->buffer(), output->shapeInfo(), std::vector<T>({T(0.), shape->e(last)}).data());
|
||||
// NativeOpExecutioner::execRandom(random::UniformDistribution, rng, output->buffer(), output->shapeInfo(),
|
||||
// std::vector<T>({T(0.), shape->e<T>(last)}).data());
|
||||
sd::LongType last = shape->lengthOf() - 1;
|
||||
|
||||
rngX.setSeed(seed);
|
||||
// functions::random::RandomFunction<T>::template execTransform<randomOps::UniformDistribution<T>>(rng,
|
||||
// output->buffer(), output->shapeInfo(), std::vector<T>({T(0.), shape->getScalar(last)}).data());
|
||||
for (sd::LongType e = 0; e < output->lengthOf(); ++e) {
|
||||
T put = rngX.relativeT<T>(e, static_cast<T>(0), static_cast<T>(shape->e<sd::LongType>(last)));
|
||||
output->p(e, put);
|
||||
}
|
||||
sd::LongType maxIndex = output->argMax();
|
||||
sd::LongType startPos = output->e<sd::LongType>(maxIndex);
|
||||
sd::LongType lastDim = input->sizeAt(-1);
|
||||
sd::LongType pos = 0;
|
||||
sd::LongType width = startPos + shape->e<sd::LongType>(last);
|
||||
if (width >= lastDim) {
|
||||
startPos -= (width - lastDim);
|
||||
width = lastDim;
|
||||
}
|
||||
|
||||
for (sd::LongType i = 0; i < input->lengthOf(); i += lastDim) {
|
||||
for (sd::LongType k = startPos; k < width && pos < output->lengthOf(); k++) {
|
||||
output->p(pos++, input->e<T>(i + k));
|
||||
}
|
||||
}
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status randomCropFunctor(graph::Context& context, NDArray* input, NDArray* shape, NDArray* output, int seed) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return _randomCropFunctor, (context, input, shape, output, seed),
|
||||
SD_FLOAT_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status _randomCropFunctor,
|
||||
(graph::Context & context, NDArray* input, NDArray* shape, NDArray* output, int seed),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,57 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 27.08.2018
|
||||
//
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/range.h>
|
||||
#if NOT_EXCLUDED(OP_range)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// be careful: outVector must have c-order and ews = 1 !!!
|
||||
template <typename T>
|
||||
static void _range(NDArray& start, NDArray& delta, NDArray& outVector) {
|
||||
const sd::LongType len = outVector.lengthOf();
|
||||
|
||||
auto buff = outVector.bufferAsT<T>();
|
||||
auto s = start.e<T>(0);
|
||||
auto d = delta.e<T>(0);
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
buff[i] = s + i * d;
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, len);
|
||||
}
|
||||
|
||||
void range(sd::LaunchContext* context, NDArray& start, NDArray& delta, NDArray& outVector) {
|
||||
BUILD_SINGLE_SELECTOR(outVector.dataType(), _range, (start, delta, outVector), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _range, (NDArray& start, NDArray& delta, NDArray& outVector),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,177 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma, created on 16.04.2018
|
||||
//
|
||||
#include <array/ResultSet.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/reverse.h>
|
||||
#if NOT_EXCLUDED(OP_reverse)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
inline void swap(T* arr, sd::LongType from, sd::LongType to) {
|
||||
T tmp = arr[from];
|
||||
arr[from] = arr[to];
|
||||
arr[to] = tmp;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////
|
||||
// this legacy op is written by raver119@gmail.com
|
||||
|
||||
template <typename T>
|
||||
static void reverseArray(sd::LaunchContext* context, void const* vinArr, sd::LongType const* inShapeBuffer,
|
||||
void* voutArr, sd::LongType const* outShapeBuffer, int numOfElemsToReverse = 0) {
|
||||
auto inArr = reinterpret_cast<T const*>(vinArr);
|
||||
auto outArr = reinterpret_cast<T*>(voutArr);
|
||||
|
||||
// Cache shape information
|
||||
const auto inRank = shape::rank(inShapeBuffer);
|
||||
const auto outRank = shape::rank(outShapeBuffer);
|
||||
const auto* inShape = shape::shapeOf(inShapeBuffer);
|
||||
const auto* outShape = shape::shapeOf(outShapeBuffer);
|
||||
const auto* inStride = shape::stride(inShapeBuffer);
|
||||
const auto* outStride = shape::stride(outShapeBuffer);
|
||||
|
||||
sd::LongType inLength = shape::length(inShapeBuffer);
|
||||
sd::LongType outLength = shape::length(outShapeBuffer);
|
||||
if (numOfElemsToReverse == 0) numOfElemsToReverse = inLength;
|
||||
sd::LongType sLength = numOfElemsToReverse - 1;
|
||||
|
||||
LongType inCoords[SD_MAX_RANK];
|
||||
LongType outCoords[SD_MAX_RANK];
|
||||
LongType inOffset;
|
||||
LongType outOffset;
|
||||
|
||||
// two step phase here
|
||||
if (inArr == outArr) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType e = start; e < stop; e++) {
|
||||
INDEX2COORDS(e, inRank, inShape, inCoords);
|
||||
COORDS2INDEX(inRank, inStride, inCoords, inOffset);
|
||||
INDEX2COORDS(sLength - e, inRank, inShape, outCoords);
|
||||
COORDS2INDEX(inRank, inStride, outCoords, outOffset);
|
||||
swap(const_cast<T*>(inArr), inOffset, outOffset);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, numOfElemsToReverse / 2);
|
||||
} else {
|
||||
// single step phase here
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType e = start; e < stop; e++) {
|
||||
INDEX2COORDS(e, inRank, inShape, inCoords);
|
||||
COORDS2INDEX(inRank, inStride, inCoords, inOffset);
|
||||
INDEX2COORDS(sLength - e, outRank, outShape, outCoords);
|
||||
COORDS2INDEX(outRank, outStride, outCoords, outOffset);
|
||||
outArr[outOffset] = inArr[inOffset];
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, numOfElemsToReverse);
|
||||
|
||||
if (inLength != numOfElemsToReverse) {
|
||||
auto f2 = PRAGMA_THREADS_FOR {
|
||||
for (sd::LongType e = start; e < stop; e++) {
|
||||
INDEX2COORDS(e, inRank, inShape, inCoords);
|
||||
COORDS2INDEX(inRank, inStride, inCoords, inOffset);
|
||||
INDEX2COORDS(e, outRank, outShape, outCoords);
|
||||
COORDS2INDEX(outRank, outStride, outCoords, outOffset);
|
||||
outArr[outOffset] = inArr[inOffset];
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(f2, numOfElemsToReverse, inLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void reverseSequence_(sd::LaunchContext* context, NDArray* input, NDArray* seqLengths,
|
||||
NDArray* output, int seqDim, const int batchDim) {
|
||||
int posOfNonUnityDim = -1;
|
||||
if (input->isVector() || shape::isLikeVector(input->shapeInfo(), posOfNonUnityDim)) {
|
||||
if ((seqDim == 0 && input->sizeAt(0) == 1) || (batchDim == posOfNonUnityDim))
|
||||
output->assign(input);
|
||||
else
|
||||
helpers::reverseArray<T>(context, const_cast<NDArray*>(input)->buffer(), const_cast<NDArray*>(input)->shapeInfo(),
|
||||
output->buffer(), output->shapeInfo(), seqLengths->e<int>(0));
|
||||
} else {
|
||||
if (seqDim > batchDim) --seqDim;
|
||||
|
||||
std::vector<sd::LongType> batchDimVec = {batchDim};
|
||||
std::vector<sd::LongType> *dimensions = ShapeUtils::evalDimsToExclude(input->rankOf(), 1,batchDimVec.data());
|
||||
|
||||
auto inSubArrsSet = input->allTensorsAlongDimension(*dimensions);
|
||||
auto outSubArrsSet = output->allTensorsAlongDimension(*dimensions);
|
||||
delete dimensions;
|
||||
|
||||
for (int i = 0; i < inSubArrsSet.size(); ++i) {
|
||||
sd::LongType numOfElemsToReverse = seqLengths->e<sd::LongType>(i);
|
||||
|
||||
if (numOfElemsToReverse == 0 || numOfElemsToReverse == 1) {
|
||||
outSubArrsSet.at(i)->assign(inSubArrsSet.at(i));
|
||||
} else {
|
||||
auto inInnerSet = inSubArrsSet.at(i)->allTensorsAlongDimension({seqDim});
|
||||
auto outInnerSet = outSubArrsSet.at(i)->allTensorsAlongDimension({seqDim});
|
||||
for (int j = 0; j < inInnerSet.size(); ++j)
|
||||
helpers::reverseArray<T>(context, inInnerSet.at(j)->buffer(), inInnerSet.at(j)->shapeInfo(),
|
||||
outInnerSet.at(j)->buffer(), outInnerSet.at(j)->shapeInfo(), numOfElemsToReverse);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void reverseSequence(sd::LaunchContext* context, NDArray* input, NDArray* seqLengths, NDArray* output,
|
||||
int seqDim, const int batchDim) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), reverseSequence_, (context, input, seqLengths, output, seqDim, batchDim),
|
||||
SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void reverse(sd::LaunchContext* context, NDArray* input, NDArray* output, const std::vector<LongType>* intArgs) {
|
||||
auto listOut = output->allTensorsAlongDimension(*intArgs);
|
||||
auto listIn = input->allTensorsAlongDimension(*intArgs);
|
||||
|
||||
NDArray *subArrIn, *subArrOut;
|
||||
|
||||
for (int i = 0; i < listIn.size(); ++i) { // listIn.size() = listOut.size()
|
||||
subArrIn = listIn.at(i);
|
||||
subArrOut = listOut.at(i);
|
||||
BUILD_SINGLE_SELECTOR(
|
||||
input->dataType(), helpers::reverseArray,
|
||||
(context, subArrIn->buffer(), subArrIn->shapeInfo(), subArrOut->buffer(), subArrOut->shapeInfo()),
|
||||
SD_COMMON_TYPES);
|
||||
}
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void reverseSequence_,
|
||||
(sd::LaunchContext * context, NDArray* input, NDArray* seqLengths, NDArray* output,
|
||||
int seqDim, const int batchDim),
|
||||
SD_COMMON_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( void reverseArray,
|
||||
(sd::LaunchContext * context, void const* inArr, sd::LongType const* inShapeBuffer, void* outArr,
|
||||
sd::LongType const* outShapeBuffer, int numOfElemsToReverse),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,113 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <ops/declarable/helpers/roll.h>
|
||||
#if NOT_EXCLUDED(OP_roll)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static void rollFunctorLinear_(NDArray* input, NDArray* output, int shift, bool inplace) {
|
||||
auto source = input;
|
||||
if (!inplace) output->assign(input);
|
||||
|
||||
int fullLen = source->lengthOf();
|
||||
int actualShift = shift; // % fullLen; // shift already non-negative then
|
||||
if (actualShift < 0) {
|
||||
actualShift -= fullLen * (actualShift / fullLen - 1);
|
||||
} else
|
||||
actualShift %= fullLen;
|
||||
|
||||
if (actualShift) {
|
||||
int shiftCount = fullLen / actualShift - 1;
|
||||
int remainShift = fullLen % actualShift;
|
||||
|
||||
// stage 1) swap last actualShift elements with first ones.
|
||||
for (int e = 0; e < actualShift; ++e) {
|
||||
int sourceIndex = fullLen - actualShift + e;
|
||||
|
||||
auto _e0 = output->e<T>(e);
|
||||
auto _e1 = output->e<T>(sourceIndex);
|
||||
|
||||
output->p<T>(e, _e1);
|
||||
output->p<T>(sourceIndex, _e0);
|
||||
}
|
||||
|
||||
// stage 2) swap swapped actualShift elements with rest remainShiftCount times.
|
||||
for (int count = 1; count < shiftCount; ++count) {
|
||||
for (int e = 0; e < actualShift; ++e) {
|
||||
int destinationIndex = fullLen - (count + 1) * actualShift + e;
|
||||
int sourceIndex = fullLen - count * actualShift + e;
|
||||
|
||||
auto _e0 = output->e<T>(destinationIndex);
|
||||
auto _e1 = output->e<T>(sourceIndex);
|
||||
|
||||
output->p<T>(destinationIndex, _e1);
|
||||
output->p<T>(sourceIndex, _e0);
|
||||
}
|
||||
}
|
||||
|
||||
// stage 3) swap remainder of items.
|
||||
if (remainShift && shiftCount)
|
||||
for (int i = actualShift; i < 2 * actualShift; ++i) {
|
||||
auto _e0 = output->e<T>(i);
|
||||
auto _e1 = output->e<T>(i + remainShift);
|
||||
output->p<T>(i, _e1);
|
||||
output->p<T>(i + remainShift, _e0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void rollFunctorFull(sd::LaunchContext* context, NDArray* input, NDArray* output, const std::vector<LongType>& shifts,
|
||||
const std::vector<LongType>& axes, bool inplace) {
|
||||
if (!inplace) output->assign(input);
|
||||
|
||||
auto source = output; // input;
|
||||
for (size_t i = 0; i < axes.size(); i++) {
|
||||
int axe = axes[i];
|
||||
ResultSet listOfTensors = source->allTensorsAlongDimension({axe});
|
||||
ResultSet listOfOutTensors = output->allTensorsAlongDimension({axe});
|
||||
int fullLen = listOfTensors.size();
|
||||
sd_debug("Roll: fullLen at last dimension is %d\n", fullLen);
|
||||
int theShift = shifts[i];
|
||||
if (theShift > 0) {
|
||||
theShift %= fullLen;
|
||||
} else {
|
||||
theShift -= fullLen * (theShift / fullLen - 1);
|
||||
}
|
||||
for (int k = 0; k < fullLen; k++) {
|
||||
rollFunctorLinear(context, listOfTensors.at(k), listOfOutTensors.at(k), theShift, true);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void rollFunctorLinear(sd::LaunchContext* context, NDArray* input, NDArray* output, int shift, bool inplace) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), rollFunctorLinear_, (input, output, shift, inplace), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void rollFunctorLinear_, (NDArray * input, NDArray* output, int shift, bool inplace),
|
||||
SD_COMMON_TYPES);
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,408 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/s_t_b.h>
|
||||
#if NOT_EXCLUDED(OP_space_to_batch)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void batchToSpace_(NDArray& input, NDArray& output, const sd::LongType cropBottom,
|
||||
const sd::LongType cropTop, const sd::LongType cropLeft, const sd::LongType cropRight) {
|
||||
// input [bS, H * blockSize, W * blockSize, iC]
|
||||
// output [bS, H * blockSize - cropBottom - cropTop, W * blockSize - cropLeft - cropRight, iC]
|
||||
|
||||
// if (cropTop = cropBottom = cropRight = cropLeft = 0) shapes are the same
|
||||
// else:
|
||||
// oH -> [cropBottom, iH - cropTop]
|
||||
// oW -> [cropLeft, iH - cropRight]
|
||||
// xLen > zLen
|
||||
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
|
||||
const int rank = 4;
|
||||
|
||||
const sd::LongType* xShapeInfo = input.shapeInfo();
|
||||
const sd::LongType* zShapeInfo = output.shapeInfo();
|
||||
|
||||
const sd::LongType bS = xShapeInfo[1];
|
||||
const sd::LongType iH = xShapeInfo[2];
|
||||
const sd::LongType iW = xShapeInfo[3];
|
||||
const sd::LongType iC = xShapeInfo[4];
|
||||
|
||||
// loop through output array
|
||||
auto func = PRAGMA_THREADS_FOR_3D {
|
||||
for (auto b = start_x; b < stop_x; b += inc_x) {
|
||||
for (auto h = start_y; h < stop_y; h += inc_y) {
|
||||
for (auto w = start_z; w < stop_z; w += inc_z) {
|
||||
for (sd::LongType c = 0; c < iC; ++c) {
|
||||
const sd::LongType xOffset = b * xShapeInfo[5] + h * xShapeInfo[6] + w * xShapeInfo[7] + c * xShapeInfo[8];
|
||||
const sd::LongType zOffset = b * zShapeInfo[5] + (h - cropBottom) * zShapeInfo[6] +
|
||||
(w - cropLeft) * zShapeInfo[7] + c * zShapeInfo[8];
|
||||
|
||||
z[zOffset] = x[xOffset];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, cropBottom, iH - cropTop, 1, cropLeft, iW - cropRight, 1);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void batchToSpace_,
|
||||
(NDArray& input, NDArray& output, const sd::LongType cropBottom, const sd::LongType cropTop,
|
||||
const sd::LongType cropLeft, const sd::LongType cropRight),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void batchToSpace(sd::LaunchContext* context, NDArray input, NDArray& output, const sd::LongType cropBottom,
|
||||
const sd::LongType cropTop, const sd::LongType cropLeft, const sd::LongType cropRight,
|
||||
const sd::LongType blockSize) {
|
||||
// [bS*blockSize*blockSize, H/blockSize, W/blockSize, iC] is rearranged/permuted to [bS, oH, oW, iC]
|
||||
// oH = H - cropTop - cropBottom
|
||||
// oW = W - cropLeft - cropRight
|
||||
|
||||
std::vector<sd::LongType> shape = {blockSize, blockSize, output.sizeAt(0), input.sizeAt(1), input.sizeAt(2), input.sizeAt(3)};
|
||||
NDArray *inputRearranged0 = input.reshape(
|
||||
input.ordering(),shape);
|
||||
inputRearranged0->permutei({2, 3, 0, 4, 1, 5}, false, false);
|
||||
|
||||
if (input.lengthOf() == output.lengthOf())
|
||||
output.assign(inputRearranged0);
|
||||
else {
|
||||
std::vector<sd::LongType> temp = {output.sizeAt(0), input.sizeAt(1) * blockSize, input.sizeAt(2) * blockSize, input.sizeAt(3)};
|
||||
NDArray *inputRearranged1 = inputRearranged0->reshape(
|
||||
input.ordering(),
|
||||
temp);
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), batchToSpace_,
|
||||
(*inputRearranged1, output, cropBottom, cropTop, cropLeft, cropRight), SD_COMMON_TYPES);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void batchToSpaceND_(NDArray* input, NDArray* crop, NDArray* output,
|
||||
const LongType numOfSpatialDims) {
|
||||
// input [bS, H * blockShape[0], W * blockShape[1], iC]
|
||||
// output [bS, H * blockShape[0] - cropBottom - cropTop, W * blockShape[1] - cropLeft - cropRight, iC]
|
||||
|
||||
// if (cropTop = cropBottom = cropRight = cropLeft = 0) shapes are the same
|
||||
// else:
|
||||
// oH -> [cropBottom, iH - cropTop]
|
||||
// oW -> [cropLeft, iH - cropRight]
|
||||
// xLen >= zLen
|
||||
|
||||
const T* x = input->bufferAsT<T>();
|
||||
T* z = output->bufferAsT<T>();
|
||||
|
||||
const sd::LongType rank = input->rankOf();
|
||||
const sd::LongType zLen = output->lengthOf();
|
||||
|
||||
// loop through input array
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType zCoords[SD_MAX_RANK], xCoords[SD_MAX_RANK];
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, rank, shape::shapeOf(output->shapeInfo()), zCoords);
|
||||
|
||||
memcpy(xCoords, zCoords, rank * sizeof(sd::LongType));
|
||||
|
||||
// evaluate spatial coordinates for x
|
||||
for (sd::LongType j = 1; j <= numOfSpatialDims; ++j)
|
||||
xCoords[j] += crop->e<sd::LongType>(j - 1, 0); // add crop left
|
||||
|
||||
sd::LongType zOffset, xOffset;
|
||||
COORDS2INDEX(rank, shape::stride(output->shapeInfo()), zCoords, zOffset);
|
||||
COORDS2INDEX(rank, shape::stride(input->shapeInfo()), xCoords, xOffset);
|
||||
|
||||
z[zOffset] = x[xOffset];
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, zLen);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void batchToSpaceND_,
|
||||
(NDArray* input, NDArray* crop, NDArray* output, const sd::LongType numOfSpatialDims),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void batchToSpaceND(sd::LaunchContext* context, NDArray& input, NDArray& blockShape, NDArray& crop,
|
||||
NDArray& output){
|
||||
// 4D example, numOfSpatialDims = 2 - two spatial dimensions
|
||||
// [bS*blockShape[0]*blockShape[1], iH, iW, iC] is rearranged/permuted to [bS, iH*blockShape[0] - cropTop -
|
||||
// cropBottom, iW*blockShape[1] - cropLeft - cropRight, iC]
|
||||
|
||||
const sd::LongType rank = input.rankOf();
|
||||
const sd::LongType numOfSpatialDims = blockShape.sizeAt(0);
|
||||
|
||||
//*** construct reshaping std::vector for first reshape of input array ***//
|
||||
|
||||
std::vector<sd::LongType> temp(numOfSpatialDims + rank);
|
||||
|
||||
sd::LongType i;
|
||||
for (i = 0; i < numOfSpatialDims; ++i) temp[i] = blockShape.e<sd::LongType>(i);
|
||||
temp[i++] = output.sizeAt(0);
|
||||
for (sd::LongType j = 1; j < rank; ++i, ++j) temp[i] = input.sizeAt(j);
|
||||
|
||||
NDArray *inputRearranged0 = input.reshape(input.ordering(), temp);
|
||||
|
||||
//*** construct permuting std::vector for permutation of input array ***//
|
||||
|
||||
temp[0] = numOfSpatialDims;
|
||||
|
||||
for (i = 1; i <= numOfSpatialDims; ++i) {
|
||||
temp[2 * i - 1] = numOfSpatialDims + i;
|
||||
temp[2 * i] = i - 1;
|
||||
}
|
||||
for (i = 2 * numOfSpatialDims + 1; i < static_cast<sd::LongType>(temp.size()); ++i) temp[i] = i;
|
||||
|
||||
inputRearranged0->permutei(temp, false, false);
|
||||
|
||||
if (input.lengthOf() == output.lengthOf()) {
|
||||
output.assign(inputRearranged0);
|
||||
} else {
|
||||
//*** construct reshaping std::vector for second reshape of input array ***//
|
||||
|
||||
temp.resize(rank);
|
||||
|
||||
temp[0] = output.sizeAt(0);
|
||||
|
||||
for (i = 1; i < rank; ++i)
|
||||
temp[i] = (i <= numOfSpatialDims) ? input.sizeAt(i) * blockShape.e<sd::LongType>(i - 1) : input.sizeAt(i);
|
||||
|
||||
NDArray *inputRearranged1 = inputRearranged0->reshape(input.ordering(), temp);
|
||||
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), batchToSpaceND_, (inputRearranged1, &crop, &output, numOfSpatialDims),
|
||||
SD_COMMON_TYPES);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void spaceToBatch_(NDArray& input, NDArray& output, const sd::LongType padBottom,
|
||||
const sd::LongType padTop, const sd::LongType padLeft, const sd::LongType padRight) {
|
||||
// input [bS, H * blockSize - padBottom - padTop, W * blockSize - padLeft - padRight, iC]
|
||||
// output [bS, H * blockSize, W * blockSize, iC]
|
||||
|
||||
// if (padTop = padBottom = padRight = padLeft = 0) shapes are the same
|
||||
// else:
|
||||
// iH -> [padBottom, oH - padTop]
|
||||
// iW -> [padLeft, oW - padRight]
|
||||
// zLen > xLen
|
||||
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
|
||||
const int rank = 4;
|
||||
|
||||
const sd::LongType* xShapeInfo = input.shapeInfo();
|
||||
const sd::LongType* zShapeInfo = output.shapeInfo();
|
||||
|
||||
const sd::LongType bS = zShapeInfo[1];
|
||||
const sd::LongType oH = zShapeInfo[2];
|
||||
const sd::LongType oW = zShapeInfo[3];
|
||||
const sd::LongType iC = zShapeInfo[4];
|
||||
|
||||
// loop through output array
|
||||
auto func = PRAGMA_THREADS_FOR_2D {
|
||||
for (auto b = start_x; b < stop_x; b += inc_x) {
|
||||
for (auto h = start_y; h < stop_y; h += inc_y) {
|
||||
for (sd::LongType w = 0; w < oW; ++w) {
|
||||
for (sd::LongType c = 0; c < iC; ++c) {
|
||||
const sd::LongType zOffset = b * zShapeInfo[5] + h * zShapeInfo[6] + w * zShapeInfo[7] + c * zShapeInfo[8];
|
||||
|
||||
if (h >= padBottom && h < oH - padTop && w >= padLeft && w < oW - padRight) {
|
||||
const sd::LongType xOffset = b * xShapeInfo[5] + (h - padBottom) * xShapeInfo[6] +
|
||||
(w - padLeft) * xShapeInfo[7] + c * xShapeInfo[8];
|
||||
z[zOffset] = x[xOffset];
|
||||
} else
|
||||
z[zOffset] = 0.f;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, bS, 1, 0, oH, 1);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void spaceToBatch_,
|
||||
(NDArray& input, NDArray& output, const sd::LongType padBottom, const sd::LongType padTop,
|
||||
const sd::LongType padLeft, const sd::LongType padRight),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void spaceToBatch(sd::LaunchContext* context, NDArray& input, NDArray& output, const sd::LongType padBottom,
|
||||
const sd::LongType padTop, const sd::LongType padLeft, const sd::LongType padRight,
|
||||
const sd::LongType blockSize) {
|
||||
// [bS, iH, iW, iC] is rearranged/permuted to [bS*blockSize*blockSize, (iH + padBottom + padTop)/blockSize, (iW +
|
||||
// padLeft + padRight)/blockSize, iC]
|
||||
|
||||
std::vector<sd::LongType> shape1 = {blockSize, blockSize, input.sizeAt(0), output.sizeAt(1), output.sizeAt(2), output.sizeAt(3)};
|
||||
NDArray *outputRearranged0 = output.reshape(
|
||||
output.ordering(), shape1,
|
||||
false);
|
||||
outputRearranged0->permutei({2, 3, 0, 4, 1, 5}, false, false);
|
||||
|
||||
if (input.lengthOf() == output.lengthOf()) {
|
||||
outputRearranged0->assign(&input);
|
||||
} else {
|
||||
std::vector<sd::LongType> shape2 = {input.sizeAt(0), output.sizeAt(1) * blockSize, output.sizeAt(2) * blockSize, output.sizeAt(3)};
|
||||
NDArray *outputRearranged1 = outputRearranged0->reshape(
|
||||
output.ordering(),
|
||||
shape2, false);
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), spaceToBatch_,
|
||||
(input, *outputRearranged1, padBottom, padTop, padLeft, padRight), SD_COMMON_TYPES);
|
||||
|
||||
if (output.buffer() != outputRearranged1->buffer()) outputRearranged0->assign(outputRearranged1);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void spaceToBatchND_(NDArray& input, NDArray& padding, NDArray& output,
|
||||
const LongType numOfSpatialDims) {
|
||||
// 4D example
|
||||
// input [bS, H * blockShape[0] - padBottom - padTop, W * blockShape[1] - padLeft - padRight, iC]
|
||||
// output [bS, H * blockShape[0], W * blockShape[1], iC]
|
||||
|
||||
// if (padTop = padBottom = padRight = padLeft = 0) shapes are the same
|
||||
// else:
|
||||
// iH -> [padBottom, oH - padTop]
|
||||
// iW -> [padLeft, oW - padRight]
|
||||
// zLen > xLen
|
||||
|
||||
const T* x = input.bufferAsT<T>();
|
||||
T* z = output.bufferAsT<T>();
|
||||
|
||||
const int rank = input.rankOf();
|
||||
const sd::LongType zLen = output.lengthOf();
|
||||
|
||||
// loop through output array
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType zCoords[SD_MAX_RANK], xCoords[SD_MAX_RANK];
|
||||
|
||||
for (sd::LongType i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, rank, shape::shapeOf(output.shapeInfo()), zCoords);
|
||||
|
||||
sd::LongType zOffset;
|
||||
COORDS2INDEX(rank, shape::stride(output.shapeInfo()), zCoords, zOffset);
|
||||
|
||||
memcpy(xCoords, zCoords, rank * sizeof(LongType));
|
||||
|
||||
bool within = true;
|
||||
|
||||
for (sd::LongType j = 1; j <= numOfSpatialDims; ++j) {
|
||||
const auto padLeft = padding.e<sd::LongType>(j - 1, 0);
|
||||
const auto padRight = padding.e<sd::LongType>(j - 1, 1);
|
||||
|
||||
within &= zCoords[j] >= padLeft && zCoords[j] < output.sizeAt(j) - padRight;
|
||||
|
||||
if (!within) break;
|
||||
|
||||
xCoords[j] = zCoords[j] - padLeft; // get coordinates for x
|
||||
}
|
||||
|
||||
if (within) {
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(rank, shape::stride(input.shapeInfo()), xCoords, xOffset);
|
||||
z[zOffset] = x[xOffset];
|
||||
} else {
|
||||
z[zOffset] = 0.f;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, zLen);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void spaceToBatchND_,
|
||||
(NDArray& input, NDArray& padding, NDArray& output,
|
||||
const sd::LongType numOfSpatialDims),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void spaceToBatchND(sd::LaunchContext* context, NDArray& input, NDArray& blockShape, NDArray& padding,
|
||||
NDArray& output) {
|
||||
// 4D example with two spatial dimensions
|
||||
// [bS, iH, iW, iC] is rearranged/permuted to [bS*blockShape[0]*blockShape[1], (iH + padBottom +
|
||||
// padTop)/blockShape[0], (iW + padLeft + padRight)/blockShape[1], iC]
|
||||
|
||||
const sd::LongType rank = input.rankOf();
|
||||
|
||||
const sd::LongType numOfSpatialDims = blockShape.sizeAt(0);
|
||||
|
||||
//*** construct reshaping std::vector for first reshape of output array ***//
|
||||
std::vector<sd::LongType> temp(numOfSpatialDims + rank);
|
||||
|
||||
int i;
|
||||
for (i = 0; i < numOfSpatialDims; ++i) temp[i] = blockShape.e<sd::LongType>(i);
|
||||
temp[i++] = input.sizeAt(0);
|
||||
for (int j = 1; j < rank; ++i, ++j) temp[i] = output.sizeAt(j);
|
||||
|
||||
NDArray *outputRearranged0 = output.reshape(output.ordering(), temp, false);
|
||||
|
||||
//*** construct permuting std::vector for permutation of output array ***//
|
||||
|
||||
temp[0] = numOfSpatialDims;
|
||||
|
||||
for (i = 1; i <= numOfSpatialDims; ++i) {
|
||||
temp[2 * i - 1] = numOfSpatialDims + i;
|
||||
temp[2 * i] = i - 1;
|
||||
}
|
||||
for (i = 2 * numOfSpatialDims + 1; i < static_cast<int>(temp.size()); ++i) temp[i] = i;
|
||||
|
||||
outputRearranged0->permutei(temp, false, false);
|
||||
|
||||
// ****** //
|
||||
|
||||
if (input.lengthOf() == output.lengthOf()) {
|
||||
outputRearranged0->assign(&input);
|
||||
} else {
|
||||
//*** construct reshaping std::vector for second reshape of output array ***//
|
||||
temp.resize(rank);
|
||||
|
||||
temp[0] = input.sizeAt(0);
|
||||
|
||||
for (i = 1; i < rank; ++i)
|
||||
temp[i] = (i <= numOfSpatialDims) ? output.sizeAt(i) * blockShape.e<sd::LongType>(i - 1) : output.sizeAt(i);
|
||||
|
||||
NDArray *outputRearranged1 = outputRearranged0->reshape(output.ordering(), temp, false);
|
||||
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), spaceToBatchND_, (input, padding, *outputRearranged1, numOfSpatialDims),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
if (output.buffer() != outputRearranged1->buffer()) outputRearranged0->assign(outputRearranged1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,111 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
//
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <ops/declarable/helpers/s_t_d.h>
|
||||
#if NOT_EXCLUDED(OP_space_to_depth)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static void _spaceTodepth_(NDArray&input, NDArray *output, int block_size, bool isNHWC) {
|
||||
auto input_ptr = reinterpret_cast<T const *>(input.buffer());
|
||||
auto output_ptr = reinterpret_cast<T *>(output->buffer());
|
||||
|
||||
const int batch_size = input.sizeAt(0);
|
||||
const int input_depth = isNHWC ? input.sizeAt(3) : input.sizeAt(1);
|
||||
const int input_height = isNHWC ? input.sizeAt(1) : input.sizeAt(2);
|
||||
const int input_width = isNHWC ? input.sizeAt(2) : input.sizeAt(3);
|
||||
|
||||
const int output_depth = isNHWC ? output->sizeAt(3) : output->sizeAt(1);
|
||||
const int output_height = isNHWC ? output->sizeAt(1) : output->sizeAt(2);
|
||||
const int output_width = isNHWC ? output->sizeAt(2) : output->sizeAt(3);
|
||||
|
||||
const int input_depth_by_output_height = input_depth * output_height;
|
||||
|
||||
const int output_area = output_width * output_height;
|
||||
const int output_depth_by_output_area = output_depth * output_area;
|
||||
|
||||
if (isNHWC) {
|
||||
const int total_count = batch_size * input_height * input_width * input_depth;
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto inp_idx = start; inp_idx < stop; inp_idx++) {
|
||||
// inp_idx = d + input_depth * (w + input_width * (h + input_height * b))
|
||||
const int d = inp_idx % input_depth;
|
||||
const int inp_idx2 = inp_idx / input_depth;
|
||||
const int w = inp_idx2 % input_width;
|
||||
const int inp_idx3 = inp_idx2 / input_width;
|
||||
const int h = inp_idx3 % input_height;
|
||||
const int b = inp_idx3 / input_height;
|
||||
|
||||
const int out_h = h / block_size;
|
||||
const int offset_h = h % block_size;
|
||||
const int out_w = w / block_size;
|
||||
const int offset_w = w % block_size;
|
||||
const int offset_d = (offset_h * block_size + offset_w) * input_depth;
|
||||
const int out_d = d + offset_d;
|
||||
|
||||
const int out_idx = out_d + output_depth * (out_w + output_width * (out_h + output_height * b));
|
||||
*(output_ptr + out_idx) = *(input_ptr + inp_idx);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, total_count);
|
||||
} else {
|
||||
const int total_count = batch_size * output_depth_by_output_area;
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto inp_idx = start; inp_idx < stop; inp_idx++) {
|
||||
const int n_iC_oY_bY_oX = inp_idx / block_size;
|
||||
const int bX = inp_idx - n_iC_oY_bY_oX * block_size;
|
||||
|
||||
const int n_iC_oY_bY = n_iC_oY_bY_oX / output_width;
|
||||
const int oX = n_iC_oY_bY_oX - n_iC_oY_bY * output_width;
|
||||
|
||||
const int n_iC_oY = n_iC_oY_bY / block_size;
|
||||
const int bY = n_iC_oY_bY - n_iC_oY * block_size;
|
||||
|
||||
const int n = n_iC_oY / input_depth_by_output_height;
|
||||
const int iC_oY = n_iC_oY - n * input_depth_by_output_height;
|
||||
|
||||
const int output_idx =
|
||||
oX + (((n * block_size + bY) * block_size + bX) * input_depth_by_output_height + iC_oY) * output_width;
|
||||
|
||||
*(output_ptr + output_idx) = *(input_ptr + inp_idx);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, total_count);
|
||||
}
|
||||
}
|
||||
|
||||
void _spaceTodepth(sd::LaunchContext *context, NDArray&input, NDArray *output, int block_size, bool isNHWC) {
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), _spaceTodepth_, (input, output, block_size, isNHWC), SD_COMMON_TYPES);
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( void _spaceTodepth_,
|
||||
(NDArray&input, NDArray *output, int block_size, bool isNHWC), SD_COMMON_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,208 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/scatter.h>
|
||||
|
||||
#include <numeric>
|
||||
#if NOT_EXCLUDED(OP_scatter)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// x - indices, z - input/output
|
||||
template <typename T>
|
||||
sd::LongType checkIndices_(NDArray& indices, NDArray& output, const int axis) {
|
||||
std::atomic<int64_t> numOfBadIndx{0};
|
||||
|
||||
const auto x = indices.bufferAsT<T>();
|
||||
|
||||
const auto xShapeInfo = indices.shapeInfo();
|
||||
const auto zShapeInfo = output.shapeInfo();
|
||||
|
||||
// Cache shape information
|
||||
const auto xRank = shape::rank(xShapeInfo);
|
||||
const auto* xShape = shape::shapeOf(xShapeInfo);
|
||||
const auto* xStride = shape::stride(xShapeInfo);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
sd::LongType xCoords[SD_MAX_RANK];
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
INDEX2COORDS(i, xRank, xShape, xCoords);
|
||||
|
||||
sd::LongType xOffset;
|
||||
COORDS2INDEX(xRank, xStride, xCoords, xOffset);
|
||||
|
||||
const sd::LongType currentInd = x[xOffset];
|
||||
|
||||
if (currentInd >= shape::sizeAt(zShapeInfo, axis == -1 ? xCoords[xRank - 1] : axis)) {
|
||||
++numOfBadIndx;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, indices.lengthOf());
|
||||
|
||||
return numOfBadIndx;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
sd::LongType checkIndices(sd::LaunchContext* context, NDArray& indices, NDArray& output, const int axis) {
|
||||
BUILD_SINGLE_SELECTOR(indices.dataType(), return checkIndices_, (indices, output, axis), SD_INTEGER_TYPES);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
void scatter(sd::LaunchContext* context, pairwise::Ops op, NDArray& indices, NDArray& updates,
|
||||
NDArray& output, const bool lock) {
|
||||
const int outRank = output.rankOf();
|
||||
const int indRank = indices.rankOf();
|
||||
const int updRank = updates.rankOf();
|
||||
const sd::LongType indLen = indices.lengthOf();
|
||||
|
||||
if (outRank == 1) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType idx = indices.e<sd::LongType>(i);
|
||||
NDArray *out = output({idx, idx + 1});
|
||||
NDArray updateE = updates.e(i);
|
||||
out->applyPairwiseTransform(op, &updateE);
|
||||
delete out;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, indLen, 1, lock ? 1 : sd::Environment::getInstance().maxThreads());
|
||||
} else { // outRank > 1
|
||||
|
||||
int sizeOfDims = indRank;
|
||||
if (outRank == updRank && indices.isVector()) sizeOfDims = 1;
|
||||
|
||||
std::vector<sd::LongType > dimsToExcludeUpd(sizeOfDims);
|
||||
std::iota(dimsToExcludeUpd.begin(), dimsToExcludeUpd.end(), 0);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
NDArray *outSubArr = output(indices.e<sd::LongType>(i), std::vector<sd::LongType >({0}));
|
||||
NDArray *updSubArr = updates(i, dimsToExcludeUpd);
|
||||
outSubArr->applyPairwiseTransform(op, updSubArr);
|
||||
delete outSubArr;
|
||||
delete updSubArr;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, indLen, 1, lock ? 1 : sd::Environment::getInstance().maxThreads());
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
void scatterND(sd::LaunchContext* context, pairwise::Ops op, NDArray& indices, NDArray& updates,
|
||||
NDArray& output, const bool lock) {
|
||||
const sd::LongType indLen = indices.lengthOf();
|
||||
const int outRank = output.rankOf();
|
||||
const int indRank = indices.rankOf();
|
||||
const sd::LongType indLastDim = indices.sizeAt(-1);
|
||||
|
||||
if (outRank == 1) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
sd::LongType idx = indices.e<sd::LongType>(i);
|
||||
NDArray *out = output({idx, idx + 1});
|
||||
NDArray updatesE = updates.e(i);
|
||||
ExtraArguments *extraArgs = nullptr;
|
||||
out->applyPairwiseTransform(op, &updatesE, extraArgs);
|
||||
delete out;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, indLen, 1, lock ? 1 : sd::Environment::getInstance().maxThreads());
|
||||
} else {
|
||||
std::vector<sd::LongType> dims = {indRank - 1};
|
||||
std::vector<sd::LongType > *dimsToExcludeInd = ShapeUtils::evalDimsToExclude(indRank, dims.size(),dims.data());
|
||||
std::vector<sd::LongType > dimsToExcludeUpd(indRank - 1);
|
||||
std::iota(dimsToExcludeUpd.begin(), dimsToExcludeUpd.end(), 0);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
std::vector<sd::LongType> idxRangeOut(2 * outRank, 0);
|
||||
|
||||
for (auto i = start; i < stop; i++) {
|
||||
NDArray *indSubArr = indices(i, *dimsToExcludeInd);
|
||||
for (sd::LongType j = 0; j < indLastDim; ++j) {
|
||||
idxRangeOut[2 * j] = indSubArr->e<sd::LongType>(j);
|
||||
idxRangeOut[2 * j + 1] = idxRangeOut[2 * j] + 1;
|
||||
}
|
||||
|
||||
NDArray *outSubArr = output(idxRangeOut);
|
||||
NDArray *updSubArr = updates(i, dimsToExcludeUpd);
|
||||
|
||||
outSubArr->applyPairwiseTransform(op, updSubArr);
|
||||
delete outSubArr;
|
||||
delete indSubArr;
|
||||
delete updSubArr;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, indLen / indLastDim, 1,
|
||||
lock ? 1 : sd::Environment::getInstance().maxThreads());
|
||||
|
||||
delete dimsToExcludeInd;
|
||||
}
|
||||
}
|
||||
|
||||
void scatterForLoss(sd::LaunchContext* context, NDArray& indices, NDArray& updates, NDArray& output,
|
||||
const bool calcGrad) {
|
||||
const sd::LongType indicesLen = indices.lengthOf();
|
||||
std::vector<sd::LongType> dim = {-1};
|
||||
std::vector<sd::LongType > *dimsToExclude = ShapeUtils::evalDimsToExclude(updates.rankOf(), dim.size(),dim.data());
|
||||
|
||||
if (!calcGrad) {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto subArr = updates(i, *dimsToExclude);
|
||||
auto curr = indices.e<sd::LongType>(i);
|
||||
output.p(i, curr);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, indicesLen);
|
||||
|
||||
delete dimsToExclude;
|
||||
} else {
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto subArr = updates(i, *dimsToExclude);
|
||||
auto ind = indices.e<sd::LongType>(i);
|
||||
auto curr = subArr->e<sd::LongType>(ind) - 1.;
|
||||
subArr->p(ind,curr);
|
||||
delete subArr;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, indicesLen);
|
||||
delete dimsToExclude;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,119 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 20.04.2018
|
||||
//
|
||||
#include <helpers/Loops.h>
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
#if NOT_EXCLUDED(OP_scatter_update)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void scatterUpdate(sd::LaunchContext* context, NDArray& input, NDArray& updates, const std::vector<LongType>* intArgs) {
|
||||
sd::LongType opCode = (*intArgs)[0];
|
||||
sd::LongType dimSize = (*intArgs)[1];
|
||||
sd::LongType e;
|
||||
sd::LongType limg = 2 + dimSize;
|
||||
std::vector<sd::LongType> tadDimensions(dimSize);
|
||||
for (e = 2; e < limg; e++) tadDimensions[e - 2] = (*intArgs)[e];
|
||||
|
||||
std::vector<sd::LongType> *dimsToExclude = ShapeUtils::evalDimsToExclude(input.rankOf(), tadDimensions.size(),tadDimensions.data());
|
||||
|
||||
// increasing counter to skip numIndices
|
||||
e++;
|
||||
std::vector<sd::LongType> indices;
|
||||
for (; e < static_cast<sd::LongType>(intArgs->size()); e++) indices.push_back((*intArgs)[e]);
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto inSubArr = input(indices[i], *dimsToExclude, true);
|
||||
auto updSubArr = updates(i, *dimsToExclude, true);
|
||||
if (inSubArr->lengthOf() != updSubArr->lengthOf()) {
|
||||
delete inSubArr;
|
||||
continue;
|
||||
}
|
||||
|
||||
switch (opCode) {
|
||||
case 0:
|
||||
inSubArr->applyPairwiseTransform(pairwise::Add, updSubArr, inSubArr);
|
||||
break;
|
||||
case 1:
|
||||
inSubArr->applyPairwiseTransform(pairwise::Subtract, updSubArr, inSubArr);
|
||||
break;
|
||||
case 2:
|
||||
inSubArr->applyPairwiseTransform(pairwise::Multiply, updSubArr, inSubArr);
|
||||
break;
|
||||
case 3:
|
||||
inSubArr->applyPairwiseTransform(pairwise::Divide, updSubArr, inSubArr);
|
||||
break;
|
||||
case 4:
|
||||
inSubArr->applyPairwiseTransform(pairwise::ReverseSubtract, updSubArr, inSubArr);
|
||||
break;
|
||||
case 5:
|
||||
inSubArr->applyPairwiseTransform(pairwise::ReverseDivide, updSubArr, inSubArr);
|
||||
break;
|
||||
case 6:
|
||||
inSubArr->applyPairwiseTransform(pairwise::CopyPws, updSubArr, inSubArr);
|
||||
break;
|
||||
default:
|
||||
continue;
|
||||
}
|
||||
|
||||
delete inSubArr;
|
||||
delete updSubArr;
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, indices.size());
|
||||
|
||||
|
||||
delete dimsToExclude;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void scatterSimple(sd::LaunchContext* context, const int opId, NDArray& input, NDArray& updates,
|
||||
NDArray& indices, const std::vector<LongType>& dimensions) {
|
||||
// updates and indices have same length
|
||||
const sd::LongType len = indices.lengthOf();
|
||||
|
||||
switch (opId) {
|
||||
case 6: { // copy
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) {
|
||||
auto inSubArr = input(i, dimensions);
|
||||
auto curr = indices.e(i);
|
||||
inSubArr->p(indices.t<sd::LongType>(i), &curr);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, len);
|
||||
} break;
|
||||
|
||||
default:
|
||||
THROW_EXCEPTION("helpers::scatterSimple: operation is not implemented for given id !");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user