chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,405 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by Yurii Shyrma on 20.11.2017.
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_absolute_difference_loss)
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(absolute_difference_loss, 3, 1, false, 0, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got "
|
||||
"weights = %s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, "
|
||||
"but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(predictions->shapeInfo()));
|
||||
|
||||
NDArray* diff = (*predictions) - (*labels);
|
||||
NDArray* E = diff->transform(transform::Abs);
|
||||
delete diff;
|
||||
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
delete E;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(EWeighted);
|
||||
break;
|
||||
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
auto* sumResult = EWeighted->reduceNumber(reduce::Sum);
|
||||
output->assign(sumResult);
|
||||
delete sumResult;
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * EWeighted->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
(*output) = 0.;
|
||||
} else {
|
||||
NDArray* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*sumE) / (*sum);
|
||||
output->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = EWeighted->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
(*output) = 0.;
|
||||
} else {
|
||||
NDArray* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*sumE) / double(numOfNonZeroWeights);
|
||||
output->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delete EWeighted;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
DECLARE_TYPES(absolute_difference_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
DECLARE_SHAPE_FN(absolute_difference_loss) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s "
|
||||
"and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
LongType * outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) { // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
} else { // in this case output has the same shape as labels and predictions
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(labelsShapeInfo),
|
||||
shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))->primary();
|
||||
}
|
||||
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(absolute_difference_loss_grad, 3, 3, false, 0, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dpredictions
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
// inputs validation
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got "
|
||||
"weights = %s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, "
|
||||
"but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(predictions->shapeInfo()));
|
||||
|
||||
NDArray* E = (*predictions) - (*labels);
|
||||
|
||||
// dE_i/dp_i = sign(p_i - y_i)
|
||||
E->applyTransform(transform::Sign, dLdp); // dE/dp
|
||||
// dE_i/dy_i = -sign(p_i - y_i)
|
||||
|
||||
E->applyTransform(transform::Abs, E);
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
NDArray* dLdpWeighted = (*dLdp) * (*weightsBroad);
|
||||
dLdp->assign(dLdpWeighted);
|
||||
delete dLdpWeighted;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
dLdw->assign(sumE);
|
||||
delete sumE;
|
||||
}
|
||||
else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else
|
||||
dLdw->assign(E);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * E->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* dLdpWeighted = (*dLdp) * (*weightsBroad);
|
||||
NDArray* dLdpResult = (*dLdpWeighted) / (*sum);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
delete dLdpWeighted;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
*dLdw = 0.;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
gradTemp->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete gradTemp;
|
||||
} else {
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
dLdw->assign(gradTemp);
|
||||
delete gradTemp;
|
||||
}
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
auto numOfNonZeroWeightsScalar =
|
||||
NDArrayFactory::create(dLdw->dataType(), numOfNonZeroWeights, block.launchContext());
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / double(numOfNonZeroWeights);
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
NDArray* dLdwResult = (*dLdw) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(dLdwResult);
|
||||
delete dLdwResult;
|
||||
} else {
|
||||
auto* result = (*E) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
}
|
||||
|
||||
NDArray* temp = (*weightsBroad) / (*numOfNonZeroWeightsScalar);
|
||||
NDArray* dLdpResult = (*dLdp) * (*temp);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
delete temp;
|
||||
|
||||
delete numOfNonZeroWeightsScalar;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
NDArray neg = -*dLdp;
|
||||
dLdl->assign(&neg);
|
||||
|
||||
delete E;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
DECLARE_TYPES(absolute_difference_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
DECLARE_SHAPE_FN(absolute_difference_loss_grad) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"ABSOLUTE_DIFFERENCE_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s "
|
||||
"and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ShapeBuilders::copyShapeInfoAndType(predictionsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdlShapeInfo = ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(CONSTANT(dLdpShapeInfo), CONSTANT(dLdwShapeInfo), CONSTANT(dLdlShapeInfo));
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,475 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 22.11.2017
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_cosine_distance_loss)
|
||||
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(cosine_distance_loss, 3, 1, false, 0, 2) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
int dim = INT_ARG(1); // axis along which sum will be made
|
||||
if (dim < 0) dim += labels->rankOf();
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"COSINE_DISTANCE_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// regard 4 possible reduction modes below
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"COSINE_DISTANCE_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but "
|
||||
"got %i instead!",
|
||||
reductionMode);
|
||||
// input dimension can't be larger than labels/predictions/weights rank
|
||||
REQUIRE_TRUE(dim < labels->rankOf(), 0,
|
||||
"COSINE_DISTANCE_LOSS OP: input reduction dimension (got %i) must be < labels rank %i!", dim,
|
||||
labels->rankOf());
|
||||
|
||||
if (!output->isScalar()) {
|
||||
// weights array can be single scalar or has the same shape as output, and must be broadcastable to output shape
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == output->rankOf(), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: weights array should be scalar or have the same rank as output array, "
|
||||
"but got %i and %i correspondingly!",
|
||||
weights->rankOf(), output->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *output), 0,
|
||||
"COSINE_DISTANCE_LOSS OP: shapes of weights and output arrays should be broadcastable, but got "
|
||||
"weights = %s and output = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
}
|
||||
std::vector<LongType> dims;
|
||||
dims.push_back(dim);
|
||||
|
||||
NDArray* predLabels = (*predictions) * (*labels);
|
||||
NDArray* dotProduct = predLabels->reduceAlongDimension(reduce::Sum, &dims, true);
|
||||
delete predLabels;
|
||||
|
||||
NDArray* E = 1. - (*dotProduct);
|
||||
delete dotProduct;
|
||||
|
||||
// perform weights broadcasting/tile to E if it is necessary
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(E))
|
||||
weightsBroad = new NDArray(weights->tileToShape(E->shapeInfo()));
|
||||
|
||||
// multiply E on weights
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(EWeighted);
|
||||
break;
|
||||
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
auto* sumResult = EWeighted->reduceNumber(reduce::Sum);
|
||||
output->assign(sumResult);
|
||||
delete sumResult;
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * EWeighted->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*output = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / (*sum);
|
||||
output->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = EWeighted->lengthOf();
|
||||
} else {
|
||||
auto* countResult = EWeighted->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*output = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / double(numOfNonZeroWeights);
|
||||
output->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
STORE_RESULT(*output);
|
||||
|
||||
delete EWeighted;
|
||||
delete E;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(cosine_distance_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(cosine_distance_loss) {
|
||||
// labels and predictions must have the same shapes
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
int dim = INT_ARG(1);
|
||||
if (dim < 0) dim += labelsShapeInfo[0];
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"COSINE_DISTANCE_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// input dimension can't be larger than labels/predictions/weights rank
|
||||
REQUIRE_TRUE(dim < labelsShapeInfo[0], 0,
|
||||
"COSINE_DISTANCE_LOSS OP: input reduction dimension (got %i) must be < labels rank %i!", dim,
|
||||
labelsShapeInfo[0]);
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
// evaluate output shapeInfo
|
||||
LongType * outShapeInfo = nullptr;
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the same shape as labels reduced by dim axis
|
||||
|
||||
std::vector<LongType> dimensions = {dim};
|
||||
outShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(predictionsShapeInfo), &dimensions, predictionsShapeInfo,
|
||||
outType, true, false, block.getWorkspace());
|
||||
|
||||
// weights array can be single scalar or has the same rank as output, and must be broadcastable to output
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(outShapeInfo), 0,
|
||||
"COSINE_DISTANCE_LOSS OP: weights array should be scalar or have the same rank as output array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(outShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, outShapeInfo), 0,
|
||||
"COSINE_DISTANCE_LOSS OP: shapes of weights and output arrays should be broadcastable, but got weights = %s "
|
||||
"and output = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(outShapeInfo).c_str());
|
||||
}
|
||||
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(cosine_distance_loss_grad, 3, 3, false, 0, 2) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dpredictions
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
int dim = INT_ARG(1); // axis along which sum will be made
|
||||
if (dim < 0) dim += labels->rankOf();
|
||||
|
||||
std::vector<LongType> dimensions = {dim};
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, "
|
||||
"but got %i instead!",
|
||||
reductionMode);
|
||||
auto lossShapeInfo = ShapeUtils::evalReduceShapeInfo(predictions->ordering(), &dimensions, predictions->shapeInfo(),
|
||||
true, false, block.getWorkspace());
|
||||
// weights array can be single scalar or has the same shape as loss, and must be broadcastable to loss shape
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == shape::rank(lossShapeInfo), 0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: weights array should be scalar or have the same rank as loss array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
weights->rankOf(), shape::rank(lossShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(weights->shapeInfo(), lossShapeInfo), 0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: shapes of weights and loss arrays should be broadcastable, but got "
|
||||
"weights = %s and loss = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(lossShapeInfo).c_str());
|
||||
// input dimension can't be larger than labels/predictions/weights rank
|
||||
REQUIRE_TRUE(dim < labels->rankOf(), 0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: input reduction dimension (got %i) must be < labels rank %i!", dim,
|
||||
labels->rankOf());
|
||||
|
||||
std::vector<LongType> dims;
|
||||
dims.push_back(dim);
|
||||
|
||||
NDArray* predLabels = (*predictions) * (*labels);
|
||||
NDArray* dotProduct = predLabels->reduceAlongDimension(reduce::Sum, &dims, true);
|
||||
delete predLabels;
|
||||
|
||||
NDArray* E = 1. - (*dotProduct);
|
||||
delete dotProduct;
|
||||
|
||||
// perform weights broadcasting/tile to E if it is necessary
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(E))
|
||||
weightsBroad = new NDArray(weights->tileToShape(E->shapeInfo()));
|
||||
|
||||
NDArray negLabels = -(*labels);
|
||||
NDArray negPreds = -(*predictions);
|
||||
dLdp->assign(&negLabels);
|
||||
dLdl->assign(&negPreds);
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
NDArray* dLdpWeighted = (*dLdp) * (*weightsBroad);
|
||||
dLdp->assign(dLdpWeighted);
|
||||
delete dLdpWeighted;
|
||||
|
||||
NDArray* dLdlWeighted = (*dLdl) * (*weightsBroad);
|
||||
dLdl->assign(dLdlWeighted);
|
||||
delete dLdlWeighted;
|
||||
|
||||
if (weights->isScalar() || weights->lengthOf() == 1) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
dLdw->assign(sumE);
|
||||
delete sumE;
|
||||
} else {
|
||||
if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else
|
||||
dLdw->assign(E);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * E->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* temp = (*weightsBroad) / (*sum);
|
||||
NDArray* dLdpResult = (*dLdp) * (*temp);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
|
||||
NDArray* dLdlResult = (*dLdl) * (*temp);
|
||||
dLdl->assign(dLdlResult);
|
||||
delete dLdlResult;
|
||||
delete temp;
|
||||
|
||||
if (weights->isScalar() || weights->lengthOf() == 1) {
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
gradTemp->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete gradTemp;
|
||||
} else {
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
dLdw->assign(gradTemp);
|
||||
delete gradTemp;
|
||||
}
|
||||
}
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* temp = (*weightsBroad) / numOfNonZeroWeights;
|
||||
NDArray* dLdpResult = (*dLdp) * (*temp);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
|
||||
NDArray* dLdlResult = (*dLdl) * (*temp);
|
||||
dLdl->assign(dLdlResult);
|
||||
delete dLdlResult;
|
||||
delete temp;
|
||||
|
||||
if (weights->isScalar() || weights->lengthOf() == 1) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / numOfNonZeroWeights;
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
} else {
|
||||
if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
NDArray* dLdwResult = (*dLdw) / numOfNonZeroWeights;
|
||||
dLdw->assign(dLdwResult);
|
||||
delete dLdwResult;
|
||||
} else {
|
||||
NDArray* result = (*E) / numOfNonZeroWeights;
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delete E;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(cosine_distance_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(cosine_distance_loss_grad) {
|
||||
/// labels and predictions must have the same shapes
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
int dim = INT_ARG(1);
|
||||
if (dim < 0) dim += labelsShapeInfo[0];
|
||||
|
||||
std::vector<LongType> dimensions = {dim};
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
auto lossShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(predictionsShapeInfo), &dimensions,
|
||||
predictionsShapeInfo, true, false, block.getWorkspace());
|
||||
// weights array can be single scalar or has the same rank as loss, and must be broadcastable to loss
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(lossShapeInfo), 0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: weights array should be scalar or have the same rank as loss array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(lossShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, lossShapeInfo),
|
||||
0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: shapes of weights and loss arrays should be broadcastable, but got "
|
||||
"weights = %s and loss = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(lossShapeInfo).c_str());
|
||||
// input dimension can't be larger than labels/predictions/weights rank
|
||||
REQUIRE_TRUE(dim < labelsShapeInfo[0], 0,
|
||||
"COSINE_DISTANCE_LOSS_GRAD OP: input reduction dimension (got %i) must be < labels rank %i!", dim,
|
||||
labelsShapeInfo[0]);
|
||||
|
||||
auto outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ShapeBuilders::copyShapeInfoAndType(predictionsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdlShapeInfo = ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(CONSTANT(dLdpShapeInfo), CONSTANT(dLdwShapeInfo), CONSTANT(dLdlShapeInfo));
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,160 @@
|
||||
/*******************************************************************************
|
||||
* Copyright (c) 2021 Deeplearning4j Contributors
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*******************************************************************************/
|
||||
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/helpers/ctc.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_ctc_loss)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(ctc_loss, 4, 1, false, 0, 1) {
|
||||
auto targetLabels = INPUT_VARIABLE(0);
|
||||
auto logitInput = INPUT_VARIABLE(1);
|
||||
auto targetLabelLengths = INPUT_VARIABLE(2);
|
||||
auto logitInputLengths = INPUT_VARIABLE(3);
|
||||
auto outputLosses = OUTPUT_VARIABLE(0);
|
||||
|
||||
int blankIndex = INT_ARG(0);
|
||||
|
||||
REQUIRE_TRUE(
|
||||
targetLabels->rankOf() == 2, 0,
|
||||
"CtcLoss: target labels fails to meet rank requirement (batch_size, max_label_sequence_length): %i == 2 ",
|
||||
targetLabels->rankOf());
|
||||
REQUIRE_TRUE(logitInput->rankOf() == 3, 0,
|
||||
"CtcLoss: logit Input fails to meet rank requirement (batch_size, frames, classes): %i == 3 ",
|
||||
logitInput->rankOf());
|
||||
REQUIRE_TRUE(targetLabelLengths->rankOf() == 1, 0,
|
||||
"CtcLoss: target label length fails to meet rank requirement (batch_size): %i == 1 ",
|
||||
targetLabelLengths->rankOf());
|
||||
REQUIRE_TRUE(logitInputLengths->rankOf() == 1, 0,
|
||||
"CtcLoss: logit Input lengths fails to meet rank requirement (batch_size): %i == 1 ",
|
||||
logitInputLengths->rankOf());
|
||||
|
||||
auto batchSize0 = targetLabels->sizeAt(0);
|
||||
auto batchSize1 = logitInput->sizeAt(0);
|
||||
auto batchSize2 = targetLabelLengths->sizeAt(0);
|
||||
auto batchSize3 = logitInputLengths->sizeAt(0);
|
||||
auto batchSize4 = outputLosses->sizeAt(0);
|
||||
|
||||
bool check_batches = (batchSize0 == batchSize1) && (batchSize2 == batchSize3);
|
||||
check_batches = check_batches && (batchSize0 == batchSize4) && (batchSize0 == batchSize2);
|
||||
|
||||
REQUIRE_TRUE(check_batches, 0, "CtcLoss: All batch sizes should be %i", batchSize0);
|
||||
REQUIRE_TRUE(outputLosses->isSameShape(targetLabelLengths), 0,
|
||||
"CtcLoss: wrong shape of output array, expected is %s but got %s instead !",
|
||||
ShapeUtils::shapeAsString(targetLabelLengths).c_str(), ShapeUtils::shapeAsString(outputLosses).c_str());
|
||||
|
||||
auto emptyGradients = NDArrayFactory::empty<float>();
|
||||
helpers::ctcLoss(block, *logitInput, *targetLabels, *logitInputLengths, *targetLabelLengths, *outputLosses,
|
||||
*emptyGradients, blankIndex);
|
||||
|
||||
delete emptyGradients;
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(ctc_loss) {
|
||||
getOpDescriptor()
|
||||
->setAllowedInputTypes(0,{ALL_INDICES})
|
||||
->setAllowedInputTypes(1, {ALL_FLOATS})
|
||||
->setAllowedInputTypes(2,{ALL_INDICES})
|
||||
->setAllowedInputTypes(3,{ALL_INDICES})
|
||||
->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(ctc_loss) {
|
||||
auto yShapeInfo = inputShape->at(1);
|
||||
auto zShapeInfo = inputShape->at(2);
|
||||
|
||||
auto dtype = ArrayOptions::dataType(yShapeInfo);
|
||||
auto ret = SHAPELIST(ConstantShapeHelper::getInstance().castToDataType(zShapeInfo, dtype));
|
||||
return ret;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(ctc_loss_grad, 4, 1, false, 0, 1) {
|
||||
auto targetLabels = INPUT_VARIABLE(0);
|
||||
auto logitInput = INPUT_VARIABLE(1);
|
||||
auto targetLabelLengths = INPUT_VARIABLE(2);
|
||||
auto logitInputLengths = INPUT_VARIABLE(3);
|
||||
auto outputGradients = OUTPUT_VARIABLE(0);
|
||||
|
||||
int blankIndex = INT_ARG(0);
|
||||
|
||||
REQUIRE_TRUE(
|
||||
targetLabels->rankOf() == 2, 0,
|
||||
"CtcLoss: target labels fails to meet rank requirement (batch_size, max_label_sequence_length): %i == 2 ",
|
||||
targetLabels->rankOf());
|
||||
REQUIRE_TRUE(logitInput->rankOf() == 3, 0,
|
||||
"CtcLoss: logit Input fails to meet rank requirement (batch_size, frames, classes): %i == 3 ",
|
||||
logitInput->rankOf());
|
||||
REQUIRE_TRUE(targetLabelLengths->rankOf() == 1, 0,
|
||||
"CtcLoss: target label length fails to meet rank requirement (batch_size): %i == 1 ",
|
||||
targetLabelLengths->rankOf());
|
||||
REQUIRE_TRUE(logitInputLengths->rankOf() == 1, 0,
|
||||
"CtcLoss: logit Input lengths fails to meet rank requirement (batch_size): %i == 1 ",
|
||||
logitInputLengths->rankOf());
|
||||
|
||||
auto batchSize0 = targetLabels->sizeAt(0);
|
||||
auto batchSize1 = logitInput->sizeAt(0);
|
||||
auto batchSize2 = targetLabelLengths->sizeAt(0);
|
||||
auto batchSize3 = logitInputLengths->sizeAt(0);
|
||||
auto batchSize4 = outputGradients->sizeAt(0);
|
||||
|
||||
bool check_batches = (batchSize0 == batchSize1) && (batchSize2 == batchSize3);
|
||||
check_batches = check_batches && (batchSize0 == batchSize4) && (batchSize0 == batchSize2);
|
||||
|
||||
REQUIRE_TRUE(check_batches, 0, "CtcLoss Gradient: All batch sizes should be %i", batchSize0);
|
||||
REQUIRE_TRUE(outputGradients->isSameShape(logitInput), 0,
|
||||
"CtcLoss Gradient: wrong shape of output array, expected is %s but got %s instead !",
|
||||
ShapeUtils::shapeAsString(logitInput).c_str(), ShapeUtils::shapeAsString(outputGradients).c_str());
|
||||
|
||||
auto emptyLoss = NDArrayFactory::empty<float>();
|
||||
helpers::ctcLoss(block, *logitInput, *targetLabels, *logitInputLengths, *targetLabelLengths, *emptyLoss,
|
||||
*outputGradients, blankIndex);
|
||||
delete emptyLoss;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(ctc_loss_grad) {
|
||||
getOpDescriptor()
|
||||
->setAllowedInputTypes({ALL_INDICES})
|
||||
->setAllowedInputTypes(1, {ALL_FLOATS})
|
||||
->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(ctc_loss_grad) {
|
||||
auto yShapeInfo = inputShape->at(1);
|
||||
auto dtype = ArrayOptions::dataType(yShapeInfo);
|
||||
auto ret = SHAPELIST(ConstantShapeHelper::getInstance().castToDataType(yShapeInfo, dtype));
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,447 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 23.11.2017
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_hinge_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(hinge_loss, 3, 1, false, 0, 1) {
|
||||
auto logits = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(logits), 0,
|
||||
"HINGE_LOSS OP: labels and logits arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(logits).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"HINGE_LOSS OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"HINGE_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got "
|
||||
"weights = %s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(
|
||||
reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"HINGE_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, "
|
||||
"but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to logits if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(logits))
|
||||
weightsBroad = new NDArray(weights->tileToShape(logits->shapeInfo()));
|
||||
|
||||
// We first need to convert binary labels to -1/1 labels (as floats)
|
||||
NDArray* labelsScaled = (*labels) * 2.f;
|
||||
NDArray* labelsTransformed = (*labelsScaled) - 1.f;
|
||||
delete labelsScaled;
|
||||
|
||||
NDArray* logitsScaled = (*labelsTransformed) * (*logits);
|
||||
delete labelsTransformed;
|
||||
|
||||
NDArray* E = 1.f - (*logitsScaled);
|
||||
delete logitsScaled;
|
||||
|
||||
E->applyScalar(scalar::RELU, 0.0f, E);
|
||||
|
||||
// multiply E on weights
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: { // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(EWeighted);
|
||||
break;
|
||||
}
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
auto* sumResult = EWeighted->reduceNumber(reduce::Sum);
|
||||
output->assign(sumResult);
|
||||
delete sumResult;
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * EWeighted->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*output = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / (*sum);
|
||||
output->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = EWeighted->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
(*output) = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / numOfNonZeroWeights;
|
||||
output->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delete EWeighted;
|
||||
delete E;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(hinge_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(hinge_loss) {
|
||||
auto logitsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(
|
||||
shape::shapeEquals(labelsShapeInfo, logitsShapeInfo), 0,
|
||||
"HINGE_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"HINGE_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i and %i "
|
||||
"correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"HINGE_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and labels = "
|
||||
"%s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(logitsShapeInfo));
|
||||
LongType *outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the same shape as labels and predictions
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(labelsShapeInfo),
|
||||
shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))->primary();
|
||||
}
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(hinge_loss_grad, 3, 3, false, 0, 1) {
|
||||
auto logits = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dpredictions
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
// inputs validation
|
||||
REQUIRE_TRUE(
|
||||
labels->isSameShape(logits), 0,
|
||||
"HINGE_LOSS_GRAD OP: labels and logits arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(logits).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"HINGE_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but got %i "
|
||||
"and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"HINGE_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s "
|
||||
"and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(
|
||||
reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"HINGE_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(logits))
|
||||
weightsBroad = new NDArray(weights->tileToShape(logits->shapeInfo()));
|
||||
|
||||
// We first need to convert binary labels to -1/1 labels (as floats)
|
||||
NDArray* labelsScaled = (*labels) * 2.f;
|
||||
NDArray* z = (*labelsScaled) - 1.f;
|
||||
delete labelsScaled;
|
||||
|
||||
NDArray* logitsScaled = (*z) * (*logits);
|
||||
NDArray* E = 1.f - (*logitsScaled);
|
||||
delete logitsScaled;
|
||||
|
||||
E->applyScalar(scalar::RELU, 0.0f, E);
|
||||
// turn E into gradient mask
|
||||
|
||||
NDArray gradientMask(E->shapeInfo(), block.getWorkspace());
|
||||
E->applyTransform(transform::Sign, &gradientMask);
|
||||
|
||||
// For dLdp
|
||||
NDArray negZ = -(*z);
|
||||
NDArray* dLdpTemp = negZ * gradientMask;
|
||||
dLdp->assign(dLdpTemp);
|
||||
delete dLdpTemp;
|
||||
|
||||
// For dLdl
|
||||
NDArray* logitsScaled2 = (*logits) * 2.f;
|
||||
NDArray* dLdlTemp = (*logitsScaled2) * gradientMask;
|
||||
delete logitsScaled2;
|
||||
NDArray dLdlNeg = -(*dLdlTemp);
|
||||
dLdl->assign(&dLdlNeg);
|
||||
delete dLdlTemp;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
NDArray* dLdpWeighted = (*dLdp) * (*weightsBroad);
|
||||
dLdp->assign(dLdpWeighted);
|
||||
delete dLdpWeighted;
|
||||
|
||||
NDArray* dLdlWeighted = (*dLdl) * (*weightsBroad);
|
||||
dLdl->assign(dLdlWeighted);
|
||||
delete dLdlWeighted;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
dLdw->assign(sumE);
|
||||
delete sumE;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else
|
||||
dLdw->assign(E);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * E->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* weightsDivSum = (*weightsBroad) / (*sum);
|
||||
NDArray* dLdpResult = (*dLdp) * (*weightsDivSum);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
|
||||
NDArray* dLdlResult = (*dLdl) * (*weightsDivSum);
|
||||
dLdl->assign(dLdlResult);
|
||||
delete dLdlResult;
|
||||
delete weightsDivSum;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
*dLdw = 0.;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
gradTemp->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete gradTemp;
|
||||
} else {
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
dLdw->assign(gradTemp);
|
||||
delete gradTemp;
|
||||
}
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
auto* numOfNonZeroWeightsScalar =
|
||||
NDArrayFactory::create(dLdw->dataType(), numOfNonZeroWeights, block.launchContext());
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / double(numOfNonZeroWeights);
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
NDArray* dLdwResult = (*dLdw) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(dLdwResult);
|
||||
delete dLdwResult;
|
||||
} else {
|
||||
auto* result = (*E) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
}
|
||||
|
||||
NDArray* temp = (*weightsBroad) / (*numOfNonZeroWeightsScalar);
|
||||
NDArray* dLdpResult = (*dLdp) * (*temp);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
|
||||
NDArray* dLdlResult = (*dLdl) * (*temp);
|
||||
dLdl->assign(dLdlResult);
|
||||
delete dLdlResult;
|
||||
delete temp;
|
||||
|
||||
delete numOfNonZeroWeightsScalar;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delete E;
|
||||
delete z;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
DECLARE_TYPES(hinge_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
DECLARE_SHAPE_FN(hinge_loss_grad) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"HINGE_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"HINGE_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but got %i "
|
||||
"and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"HINGE_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and "
|
||||
"labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
LongType *dLdpShapeInfo =
|
||||
ShapeBuilders::copyShapeInfoAndType(predictionsShapeInfo, outType, false, block.getWorkspace());
|
||||
LongType *dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
LongType *dLdlShapeInfo =
|
||||
ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(dLdpShapeInfo, dLdwShapeInfo, dLdlShapeInfo);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,477 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 23.11.2017
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_huber_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(huber_loss, 3, 1, false, 1, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// FIXME: double?
|
||||
double delta = T_ARG(0);
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(
|
||||
labels->isSameShape(predictions), 0,
|
||||
"HUBER_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"HUBER_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i and %i "
|
||||
"correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"HUBER_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and "
|
||||
"labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(
|
||||
reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"HUBER_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to predictions if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(predictions->shapeInfo()));
|
||||
|
||||
NDArray* error = (*predictions) - (*labels);
|
||||
error->applyTransform(transform::Abs, error);
|
||||
NDArray quadratic(error->shapeInfo(), block.getWorkspace());
|
||||
error->applyScalar(scalar::MinPairwise, delta, &quadratic);
|
||||
|
||||
NDArray* quadraticSquared = quadratic * quadratic;
|
||||
NDArray* scaledQuadratic = (*quadraticSquared) * 0.5f;
|
||||
delete quadraticSquared;
|
||||
|
||||
NDArray* errorMinusQuadratic = (*error) - quadratic;
|
||||
NDArray* linearTerm = (*errorMinusQuadratic) * delta;
|
||||
delete errorMinusQuadratic;
|
||||
|
||||
NDArray* E = (*scaledQuadratic) + (*linearTerm);
|
||||
delete scaledQuadratic;
|
||||
delete linearTerm;
|
||||
|
||||
// multiply E on weights
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
delete E;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: { // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(EWeighted);
|
||||
break;
|
||||
}
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
auto* sumResult = EWeighted->reduceNumber(reduce::Sum);
|
||||
output->assign(sumResult);
|
||||
delete sumResult;
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * EWeighted->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*output = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / (*sum);
|
||||
output->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = EWeighted->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
(*output) = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / double(numOfNonZeroWeights);
|
||||
output->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delete EWeighted;
|
||||
delete error;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(huber_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(huber_loss) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(
|
||||
shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"HUBER_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"HUBER_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i and %i "
|
||||
"correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"HUBER_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and labels = "
|
||||
"%s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
LongType * outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the same shape as labels and predictions
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(labelsShapeInfo),
|
||||
shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))->primary();
|
||||
}
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(huber_loss_grad, 3, 3, false, 1, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dpredictions
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
auto delta = T_ARG(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
// inputs validation
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"HUBER_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"HUBER_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but got %i "
|
||||
"and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"HUBER_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s "
|
||||
"and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(
|
||||
reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"HUBER_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(predictions->shapeInfo()));
|
||||
|
||||
NDArray* diff = (*predictions) - (*labels);
|
||||
NDArray absDiff(*diff);
|
||||
absDiff.applyTransform(transform::Abs, &absDiff);
|
||||
NDArray quadratic(absDiff);
|
||||
absDiff.applyScalar(scalar::MinPairwise, delta, &quadratic);
|
||||
|
||||
NDArray* quadraticSquared = quadratic * quadratic;
|
||||
NDArray* scaledQuadratic = (*quadraticSquared) * 0.5f;
|
||||
delete quadraticSquared;
|
||||
|
||||
NDArray* absDiffMinusQuadratic = absDiff - quadratic;
|
||||
NDArray* linearTerm = (*absDiffMinusQuadratic) * delta;
|
||||
delete absDiffMinusQuadratic;
|
||||
|
||||
NDArray* E = (*scaledQuadratic) + (*linearTerm);
|
||||
delete scaledQuadratic;
|
||||
delete linearTerm;
|
||||
|
||||
NDArray lteMask(diff->shapeInfo(), BOOL, true, block.launchContext());
|
||||
absDiff.applyScalar(scalar::LessThanOrEqual, delta, <eMask);
|
||||
|
||||
NDArray gtMask(diff->shapeInfo(), BOOL, true, block.launchContext());
|
||||
absDiff.applyScalar(scalar::GreaterThan, delta, >Mask);
|
||||
|
||||
NDArray signDiff(*diff);
|
||||
diff->applyTransform(transform::Sign, &signDiff);
|
||||
|
||||
auto gtMaskFloat = gtMask.cast(diff->dataType());
|
||||
auto lteMaskFloat = lteMask.cast(diff->dataType());
|
||||
|
||||
// For dLdp
|
||||
NDArray* lteDiff = (*lteMaskFloat) * (*diff);
|
||||
NDArray* gtSignScaled = (*gtMaskFloat) * delta;
|
||||
NDArray* gtTerm = (*gtSignScaled) * signDiff;
|
||||
delete gtSignScaled;
|
||||
NDArray* dLdpTemp = (*lteDiff) + (*gtTerm);
|
||||
delete lteDiff;
|
||||
delete gtTerm;
|
||||
dLdp->assign(dLdpTemp);
|
||||
delete dLdpTemp;
|
||||
|
||||
// For dLdl
|
||||
NDArray negLteMaskFloat = -(*lteMaskFloat);
|
||||
NDArray* negLteDiff = negLteMaskFloat * (*diff);
|
||||
NDArray* negGtSignScaled = (*gtMaskFloat) * delta;
|
||||
NDArray* negGtTerm = (*negGtSignScaled) * signDiff;
|
||||
delete negGtSignScaled;
|
||||
NDArray negGtTermNeg = -(*negGtTerm);
|
||||
delete negGtTerm;
|
||||
NDArray* dLdlTemp = (*negLteDiff) + negGtTermNeg;
|
||||
delete negLteDiff;
|
||||
dLdl->assign(dLdlTemp);
|
||||
delete dLdlTemp;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
NDArray* dLdpWeighted = (*dLdp) * (*weightsBroad);
|
||||
dLdp->assign(dLdpWeighted);
|
||||
delete dLdpWeighted;
|
||||
|
||||
NDArray* dLdlWeighted = (*dLdl) * (*weightsBroad);
|
||||
dLdl->assign(dLdlWeighted);
|
||||
delete dLdlWeighted;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
dLdw->assign(sumE);
|
||||
delete sumE;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else
|
||||
dLdw->assign(E);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * E->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* weightsDivSum = (*weightsBroad) / (*sum);
|
||||
NDArray* dLdpResult = (*dLdp) * (*weightsDivSum);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
|
||||
NDArray* dLdlResult = (*dLdl) * (*weightsDivSum);
|
||||
dLdl->assign(dLdlResult);
|
||||
delete dLdlResult;
|
||||
delete weightsDivSum;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
*dLdw = 0.;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
gradTemp->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete gradTemp;
|
||||
} else {
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
dLdw->assign(gradTemp);
|
||||
delete gradTemp;
|
||||
}
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
auto numOfNonZeroWeightsScalar =
|
||||
NDArrayFactory::create(dLdw->dataType(), numOfNonZeroWeights, block.launchContext());
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
auto* result = (*sumE) / double(numOfNonZeroWeights);
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
delete sumE;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
NDArray* dLdwResult = (*dLdw) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(dLdwResult);
|
||||
delete dLdwResult;
|
||||
} else {
|
||||
auto* result = (*E) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
}
|
||||
|
||||
NDArray* temp = (*weightsBroad) / (*numOfNonZeroWeightsScalar);
|
||||
NDArray* dLdpResult = (*dLdp) * (*temp);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
|
||||
NDArray* dLdlResult = (*dLdl) * (*temp);
|
||||
dLdl->assign(dLdlResult);
|
||||
delete dLdlResult;
|
||||
delete temp;
|
||||
|
||||
delete numOfNonZeroWeightsScalar;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delete E;
|
||||
delete diff;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
DECLARE_TYPES(huber_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
DECLARE_SHAPE_FN(huber_loss_grad) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"HUBER_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"HUBER_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but got %i "
|
||||
"and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"HUBER_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and "
|
||||
"labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ShapeBuilders::copyShapeInfoAndType(predictionsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdlShapeInfo = ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(dLdpShapeInfo, dLdwShapeInfo, dLdlShapeInfo);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,55 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by GS <sgazeos@gmail.com> 31.01.2018
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_l2_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
CUSTOM_OP_IMPL(l2_loss, 1, 1, false, 0, 0) {
|
||||
auto input = INPUT_VARIABLE(0);
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
REQUIRE_TRUE(output->isScalar(), 0, "Rank output should be scalar");
|
||||
|
||||
// FIXME: output should be used directly here, to avoid sum
|
||||
auto* result = input->reduceNumber(reduce::SquaredNorm);
|
||||
NDArray* divided = (*result) / 2.;
|
||||
output->assign(divided);
|
||||
delete divided;
|
||||
delete result;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
DECLARE_SHAPE_FN(l2_loss) {
|
||||
return SHAPELIST(ConstantShapeHelper::getInstance().scalarShapeInfo(ArrayOptions::dataType(inputShape->at(0))));
|
||||
}
|
||||
|
||||
DECLARE_TYPES(l2_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,472 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 23.11.2017
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_log_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(log_loss, 3, 1, false, 1, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// FIXME: double?
|
||||
double epsilon = T_ARG(0);
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(
|
||||
labels->isSameShape(predictions), 0,
|
||||
"LOG_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"LOG_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i and %i "
|
||||
"correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"LOG_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and "
|
||||
"labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(
|
||||
reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"LOG_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to predictions if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(predictions->shapeInfo()));
|
||||
|
||||
// E = -labels * log(predictions + epsilon) - (1 - labels) * log(1 + epsilon - predictions)
|
||||
// Break this into steps:
|
||||
NDArray* predPlusEps = (*predictions) + epsilon;
|
||||
NDArray* logPredPlusEps = predPlusEps->transform(transform::Log);
|
||||
delete predPlusEps;
|
||||
|
||||
NDArray negLabels = -(*labels); // unary negation returns value
|
||||
NDArray* term1 = negLabels * (*logPredPlusEps);
|
||||
delete logPredPlusEps;
|
||||
|
||||
NDArray* oneMinusLabels = 1. - (*labels);
|
||||
NDArray* onePlusEpsMinusPred = (1. + epsilon) - (*predictions);
|
||||
NDArray* logOnePlusEpsMinusPred = onePlusEpsMinusPred->transform(transform::Log);
|
||||
delete onePlusEpsMinusPred;
|
||||
|
||||
NDArray* term2 = (*oneMinusLabels) * (*logOnePlusEpsMinusPred);
|
||||
delete oneMinusLabels;
|
||||
delete logOnePlusEpsMinusPred;
|
||||
|
||||
NDArray* E_ptr = (*term1) - (*term2);
|
||||
delete term1;
|
||||
delete term2;
|
||||
|
||||
NDArray E = *E_ptr;
|
||||
delete E_ptr;
|
||||
|
||||
// multiply E on weights
|
||||
E *= *weightsBroad;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: { // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(&E);
|
||||
break;
|
||||
}
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
E.reduceNumber(reduce::Sum, output);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
double sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = weights->e<double>(0) * E.lengthOf();
|
||||
} else {
|
||||
NDArray* sumPtr = weightsBroad->reduceNumber(reduce::Sum);
|
||||
sum = sumPtr->e<double>(0);
|
||||
delete sumPtr;
|
||||
}
|
||||
|
||||
if (sum == 0.)
|
||||
*output = 0.;
|
||||
else {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / sum;
|
||||
delete eSum;
|
||||
output->assign(result);
|
||||
delete result;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
NDArray* countNonZero = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countNonZero->e<LongType>(0);
|
||||
delete countNonZero;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0)
|
||||
(*output) = 0.;
|
||||
else {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / double(numOfNonZeroWeights);
|
||||
delete eSum;
|
||||
output->assign(result);
|
||||
delete result;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(log_loss) { getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS}); }
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(log_loss) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(
|
||||
shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"LOG_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"LOG_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i and %i "
|
||||
"correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"LOG_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and labels = %s "
|
||||
"instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
LongType* outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the same shape as labels and predictions
|
||||
outShapeInfo = ConstantShapeHelper::getInstance()
|
||||
.bufferForShapeInfo(outType, shape::order(labelsShapeInfo), shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))
|
||||
->primary();
|
||||
}
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(log_loss_grad, 3, 3, false, 1, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dpredictions
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
// FIXME: double?
|
||||
double epsilon = T_ARG(0);
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(
|
||||
labels->isSameShape(predictions), 0,
|
||||
"LOG_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"LOG_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but got %i and "
|
||||
"%i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"LOG_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s "
|
||||
"and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(
|
||||
reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"LOG_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(predictions->shapeInfo()));
|
||||
|
||||
NDArray* predictPlusEps_ptr = (*predictions) + epsilon;
|
||||
NDArray predictPlusEps = *predictPlusEps_ptr;
|
||||
delete predictPlusEps_ptr;
|
||||
|
||||
NDArray* oneMinusLabels_ptr = 1. - (*labels);
|
||||
NDArray oneMinusLabels = *oneMinusLabels_ptr;
|
||||
delete oneMinusLabels_ptr;
|
||||
|
||||
NDArray* onePlusEpsMinusPredict_ptr = (1. + epsilon) - (*predictions);
|
||||
NDArray onePlusEpsMinusPredict = *onePlusEpsMinusPredict_ptr;
|
||||
delete onePlusEpsMinusPredict_ptr;
|
||||
|
||||
// dE_i/dp_i = (1-y_i)/(1-p_i+eps) - y_i/(p_i+eps)
|
||||
NDArray* oneMinusDiv = oneMinusLabels / onePlusEpsMinusPredict;
|
||||
NDArray* labelsDiv = (*labels) / predictPlusEps;
|
||||
NDArray* dEdp = (*oneMinusDiv) - (*labelsDiv);
|
||||
delete oneMinusDiv;
|
||||
delete labelsDiv;
|
||||
dLdp->assign(dEdp);
|
||||
delete dEdp;
|
||||
|
||||
// dE_i/dy_i = log((1+2eps)/(p_i+eps) - 1)
|
||||
double onePlus2Eps = 1. + 2. * epsilon;
|
||||
NDArray* ratio = onePlus2Eps / predictPlusEps;
|
||||
NDArray* ratioMinus1 = (*ratio) - 1.;
|
||||
delete ratio;
|
||||
ratioMinus1->applyTransform(transform::Log, dLdl);
|
||||
delete ratioMinus1;
|
||||
|
||||
// Compute E for gradient calculations
|
||||
NDArray* logPredPlusEps = predictPlusEps.transform(transform::Log);
|
||||
NDArray* logOnePlusEpsMinusPred = onePlusEpsMinusPredict.transform(transform::Log);
|
||||
|
||||
NDArray negLabels = -(*labels); // unary negation returns value
|
||||
NDArray* term1 = negLabels * (*logPredPlusEps);
|
||||
delete logPredPlusEps;
|
||||
|
||||
NDArray* term2 = oneMinusLabels * (*logOnePlusEpsMinusPred);
|
||||
delete logOnePlusEpsMinusPred;
|
||||
|
||||
NDArray* E_ptr = (*term1) - (*term2);
|
||||
delete term1;
|
||||
delete term2;
|
||||
|
||||
NDArray E = *E_ptr;
|
||||
delete E_ptr;
|
||||
|
||||
// process 3 possible reduction modes below
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
*dLdp *= *weightsBroad;
|
||||
*dLdl *= *weightsBroad;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
dLdw->assign(eSum);
|
||||
delete eSum;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else
|
||||
dLdw->assign(&E);
|
||||
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
|
||||
double sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = weights->e<double>(0) * E.lengthOf();
|
||||
} else {
|
||||
NDArray* sumPtr = weightsBroad->reduceNumber(reduce::Sum);
|
||||
sum = sumPtr->e<double>(0);
|
||||
delete sumPtr;
|
||||
}
|
||||
|
||||
if (sum == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* weightsDivSum = (*weightsBroad) / sum;
|
||||
NDArray temp = *weightsDivSum;
|
||||
delete weightsDivSum;
|
||||
|
||||
*dLdp *= temp;
|
||||
*dLdl *= temp;
|
||||
|
||||
if (weights->isScalar())
|
||||
*dLdw = 0.;
|
||||
else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
|
||||
// Compute (E * sum - (E * weightsBroad).reduceNumber(Sum)) / (sum * sum)
|
||||
NDArray* ETimesSum = E * sum;
|
||||
NDArray* ETimesWeights = E * (*weightsBroad);
|
||||
NDArray* ETimesWeightsSum = ETimesWeights->reduceNumber(reduce::Sum);
|
||||
delete ETimesWeights;
|
||||
|
||||
NDArray* numerator = (*ETimesSum) - (*ETimesWeightsSum);
|
||||
delete ETimesSum;
|
||||
delete ETimesWeightsSum;
|
||||
|
||||
double sumSquared = sum * sum;
|
||||
NDArray* result = (*numerator) / sumSquared;
|
||||
delete numerator;
|
||||
|
||||
result->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete result;
|
||||
} else {
|
||||
// Compute (E * sum - (E * weightsBroad).reduceNumber(Sum)) / (sum * sum)
|
||||
NDArray* ETimesSum = E * sum;
|
||||
NDArray* ETimesWeights = E * (*weightsBroad);
|
||||
NDArray* ETimesWeightsSum = ETimesWeights->reduceNumber(reduce::Sum);
|
||||
delete ETimesWeights;
|
||||
|
||||
NDArray* numerator = (*ETimesSum) - (*ETimesWeightsSum);
|
||||
delete ETimesSum;
|
||||
delete ETimesWeightsSum;
|
||||
|
||||
double sumSquared = sum * sum;
|
||||
NDArray* result = (*numerator) / sumSquared;
|
||||
delete numerator;
|
||||
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
NDArray* countNonZero = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countNonZero->e<LongType>(0);
|
||||
delete countNonZero;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
auto* numOfNonZeroWeightsScalar =
|
||||
NDArrayFactory::create(dLdw->dataType(), numOfNonZeroWeights, block.launchContext());
|
||||
|
||||
if (weights->isScalar()) {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / numOfNonZeroWeights;
|
||||
delete eSum;
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
*dLdw /= *numOfNonZeroWeightsScalar;
|
||||
} else {
|
||||
NDArray* EDivNum = E / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(EDivNum);
|
||||
delete EDivNum;
|
||||
|
||||
NDArray* weightsDivNum = (*weightsBroad) / (*numOfNonZeroWeightsScalar);
|
||||
NDArray temp = *weightsDivNum;
|
||||
delete weightsDivNum;
|
||||
|
||||
*dLdp *= temp;
|
||||
*dLdl *= temp;
|
||||
}
|
||||
|
||||
delete numOfNonZeroWeightsScalar;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(log_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(log_loss_grad) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(
|
||||
shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"LOG_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"LOG_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but got %i and "
|
||||
"%i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"LOG_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and labels "
|
||||
"= %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ShapeBuilders::copyShapeInfoAndType(predictionsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdlShapeInfo = ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(CONSTANT(dLdpShapeInfo), CONSTANT(dLdwShapeInfo), CONSTANT(dLdlShapeInfo));
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,426 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_log_poisson_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
CUSTOM_OP_IMPL(log_poisson_loss, 3, 1, true, 0, 1) {
|
||||
auto log_predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
|
||||
bool computeFullLoss = false;
|
||||
if (block.numI() > 1) computeFullLoss = INT_ARG(1) != 0;
|
||||
|
||||
// inputs validation
|
||||
REQUIRE_TRUE(labels->isSameShape(log_predictions), 0,
|
||||
"LOG_POISSON_LOSS OP: labels and log_predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(log_predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"LOG_POISSON_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i "
|
||||
"and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"LOG_POISSON_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s "
|
||||
"and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"LOG_POISSON_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but got "
|
||||
"%i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(log_predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(log_predictions->shapeInfo()));
|
||||
|
||||
NDArray E(labels->shapeInfo(), block.getWorkspace());
|
||||
if (computeFullLoss)
|
||||
labels->applyPairwiseTransform(pairwise::LogPoissonLossFull, log_predictions, &E);
|
||||
else
|
||||
labels->applyPairwiseTransform(pairwise::LogPoissonLoss, log_predictions, &E);
|
||||
|
||||
// multiply E on weights
|
||||
E *= *weightsBroad;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: { // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(&E);
|
||||
break;
|
||||
}
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
E.reduceNumber(reduce::Sum, output);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum = nullptr;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * E.lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*output = 0.;
|
||||
} else {
|
||||
NDArray* sumResult = E.reduceNumber(reduce::Sum);
|
||||
NDArray* assign = (*sumResult) / (*sum);
|
||||
delete sumResult;
|
||||
output->assign(assign);
|
||||
delete assign;
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
NDArray* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0)
|
||||
(*output) = 0.;
|
||||
else {
|
||||
NDArray* sumResult = E.reduceNumber(reduce::Sum);
|
||||
NDArray* assign = (*sumResult) / double(numOfNonZeroWeights);
|
||||
delete sumResult;
|
||||
output->assign(assign);
|
||||
delete assign;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(log_poisson_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(log_poisson_loss) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"LOG_POISSON_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"LOG_POISSON_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i "
|
||||
"and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"LOG_POISSON_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and "
|
||||
"labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
LongType* outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the same shape as labels and predictions
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(labelsShapeInfo),
|
||||
shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))->primary();
|
||||
}
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(log_poisson_loss_grad, 3, 3, false, 0, 1) {
|
||||
auto log_predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dpredictions
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
bool computeFullLoss = false;
|
||||
if (block.numI() > 1) computeFullLoss = INT_ARG(1) != 0;
|
||||
|
||||
// inputs validation
|
||||
REQUIRE_TRUE(labels->isSameShape(log_predictions), 0,
|
||||
"LOG_POISSON_LOSS_GRAD OP: labels and log_predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(log_predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"LOG_POISSON_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"LOG_POISSON_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights "
|
||||
"= %s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"LOG_POISSON_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but "
|
||||
"got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(log_predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(log_predictions->shapeInfo()));
|
||||
|
||||
NDArray E(labels->shapeInfo(), block.getWorkspace());
|
||||
if (computeFullLoss) {
|
||||
labels->applyPairwiseTransform(pairwise::LogPoissonLossFull, log_predictions, &E);
|
||||
|
||||
NDArray rDiv(labels->shapeInfo(), block.getWorkspace());
|
||||
labels->applyScalar(scalar::ReverseDivide, 0.5f, &rDiv);
|
||||
|
||||
// For dLdl - first case
|
||||
NDArray* logLabels = labels->transform(transform::Log);
|
||||
NDArray negLogPredictions = -(*log_predictions); // unary negation returns value
|
||||
NDArray* temp1 = rDiv + (*logLabels);
|
||||
delete logLabels;
|
||||
NDArray* dLdlTemp1 = (*temp1) + negLogPredictions;
|
||||
delete temp1;
|
||||
dLdl->assign(dLdlTemp1);
|
||||
delete dLdlTemp1;
|
||||
} else {
|
||||
labels->applyPairwiseTransform(pairwise::LogPoissonLoss, log_predictions, &E);
|
||||
|
||||
// For dLdl - second case
|
||||
NDArray dLdlTemp2 = -(*log_predictions); // unary negation returns value
|
||||
dLdl->assign(&dLdlTemp2);
|
||||
}
|
||||
|
||||
// For dLdp
|
||||
NDArray* expResult = log_predictions->transform(transform::Exp);
|
||||
NDArray* dLdpTemp = (*expResult) - (*labels);
|
||||
delete expResult;
|
||||
dLdp->assign(dLdpTemp);
|
||||
delete dLdpTemp;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
*dLdp *= *weightsBroad;
|
||||
*dLdl *= *weightsBroad;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
NDArray* assign = E.reduceNumber(reduce::Sum);
|
||||
dLdw->assign(assign);
|
||||
delete assign;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else
|
||||
dLdw->assign(&E);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
|
||||
NDArray* sum = nullptr;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * E.lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* weightsBroadDivSum = (*weightsBroad) / (*sum);
|
||||
*dLdp *= *weightsBroadDivSum;
|
||||
*dLdl *= *weightsBroadDivSum;
|
||||
delete weightsBroadDivSum;
|
||||
|
||||
if (weights->isScalar())
|
||||
*dLdw = 0.;
|
||||
else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
|
||||
NDArray* eMulWeightsBroad = E * (*weightsBroad);
|
||||
NDArray* sumReduced = eMulWeightsBroad->reduceNumber(reduce::Sum);
|
||||
delete eMulWeightsBroad;
|
||||
|
||||
NDArray* eMulSum = E * (*sum);
|
||||
NDArray* numerator = (*eMulSum) - (*sumReduced);
|
||||
delete eMulSum;
|
||||
delete sumReduced;
|
||||
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* result = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
|
||||
result->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete result;
|
||||
} else {
|
||||
NDArray* eMulWeightsBroad = E * (*weightsBroad);
|
||||
NDArray* sumReduced = eMulWeightsBroad->reduceNumber(reduce::Sum);
|
||||
delete eMulWeightsBroad;
|
||||
|
||||
NDArray* eMulSum = E * (*sum);
|
||||
NDArray* numerator = (*eMulSum) - (*sumReduced);
|
||||
delete eMulSum;
|
||||
delete sumReduced;
|
||||
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* assign = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
|
||||
dLdw->assign(assign);
|
||||
delete assign;
|
||||
}
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
NDArray* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
auto* numOfNonZeroWeightsScalar =
|
||||
NDArrayFactory::create(dLdw->dataType(), numOfNonZeroWeights, block.launchContext());
|
||||
|
||||
if (weights->isScalar()) {
|
||||
NDArray* sumResult = E.reduceNumber(reduce::Sum);
|
||||
NDArray* assign = (*sumResult) / double(numOfNonZeroWeights);
|
||||
delete sumResult;
|
||||
dLdw->assign(assign);
|
||||
delete assign;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
*dLdw /= *numOfNonZeroWeightsScalar;
|
||||
} else {
|
||||
NDArray* assign = E / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(assign);
|
||||
delete assign;
|
||||
}
|
||||
|
||||
NDArray* temp = (*weightsBroad) / (*numOfNonZeroWeightsScalar);
|
||||
*dLdp *= *temp;
|
||||
*dLdl *= *temp;
|
||||
delete temp;
|
||||
delete numOfNonZeroWeightsScalar;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
DECLARE_TYPES(log_poisson_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
DECLARE_SHAPE_FN(log_poisson_loss_grad) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"LOG_POISSON_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"LOG_POISSON_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"LOG_POISSON_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and "
|
||||
"labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ShapeBuilders::copyShapeInfoAndType(predictionsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdlShapeInfo = ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(dLdpShapeInfo, dLdwShapeInfo, dLdlShapeInfo);
|
||||
}
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,550 @@
|
||||
#pragma clang diagnostic push
|
||||
#pragma ide diagnostic ignored "cert-err58-cpp"
|
||||
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 24.11.2017
|
||||
// @author Paul Dubs
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_mean_pairwssqerr_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(mean_pairwssqerr_loss, 3, 1, false, 0, 1) {
|
||||
/*
|
||||
* Implementation of mean pairwise squared error loss
|
||||
*
|
||||
* For context on where this loss function may be useful see:
|
||||
*
|
||||
* Wei, Z., Zhang, J., Shen, X., Lin, Z., Mech, R., Hoai, M. and Samaras, D., 2018.
|
||||
* Good view hunting: learning photo composition from dense view pairs. In Proceedings of the IEEE Conference on
|
||||
* Computer Vision and Pattern Recognition (pp. 5437-5446).
|
||||
*
|
||||
* The paper defines the loss function as:
|
||||
*
|
||||
* L(y,q) = 1/((n*(n-1))/2) * (sum_(i,j=1..n,i!=j)((y_i - y_j) - (q_i - q_j))^2)
|
||||
*
|
||||
* with y: predictions, q: labels, n: length of y and q
|
||||
*
|
||||
* As creating those pairs is computationally expensive, we implement a mathematically equivalent function:
|
||||
*
|
||||
* L(y,q) = 4/(n*(n-1)) * (n * sum (y_i - q_i)^2 - (sum y_i - q_i)^2)
|
||||
*
|
||||
* This equivalency can be derived as:
|
||||
*
|
||||
* sum_(i,j=1..n,i!=j)((y_i - y_j) - (q_i - q_j))^2 = sum_(i,j=1..n,i!=j)((y_i - q_i) - (y_j - q_j))^2
|
||||
*
|
||||
* To simplify the following equations we use
|
||||
*
|
||||
* sum_(i,j=1..n,i!=j)(d_i - d_j)^2 = sum_(i,j=1..n,i!=j)(d_i^2 + d_j^2 - 2*d_i*d_j)
|
||||
*
|
||||
* Due to the pairings each element will appear as both d_i and d_j exactly n-1 times. This allows us to split the
|
||||
* sum:
|
||||
*
|
||||
* sum_(i,j=1..n,i!=j)(d_i^2 + d_j^2 - 2*d_i*d_j) = 2*(n-1)*sum d_i^2 - 2 * sum_(i,j=1..n,i!=j) d_i * d_j
|
||||
* = 2*((n-1) * sum d_i^2 - sum_(i,j=1..n,i!=j) d_i * d_j)
|
||||
*
|
||||
* Now we use the following equivalency:
|
||||
*
|
||||
* (sum d_i)^2 = sum d_i^2 + sum_(i,j=1..n,i!=j) d_i * d_j
|
||||
*
|
||||
* This allows us to now use sum d_i^2 and (sum d_i)^2 as a quick way to calculate the sum:
|
||||
*
|
||||
* (n-1) * sum d_i^2 - sum_(i,j=1..n,i!=j) d_i * d_j = n * sum d_i^2 - (sum d_i)^2
|
||||
*
|
||||
* And by substituting it into the original definition we get:
|
||||
*
|
||||
* 1/((n*(n-1))/2) * 2*(n * sum d_i^2 - (sum d_i)^2)
|
||||
*
|
||||
* Which can be again simplified to
|
||||
*
|
||||
* 4/(n*(n-1)) * (n * sum d_i^2 - (sum d_i)^2)
|
||||
*
|
||||
* After substituting d_i back to (y_i - q_i) this results in the function that we actually implement.
|
||||
*
|
||||
*/
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but "
|
||||
"got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
if (labels->rankOf() == 1) { // If labels and predictions are of rank 1, it means that all data entries are 0-tensor
|
||||
// (scalar) so that the result of becomes always zero.
|
||||
*output = 0.;
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
std::vector<LongType> zero;
|
||||
zero.push_back(0);
|
||||
std::vector<LongType> *reductionIdx = ShapeUtils::evalDimsToExclude(labels->rankOf(),1,zero.data());
|
||||
|
||||
auto n = double(labels->sizeAt(1));
|
||||
|
||||
// Compute diffs = predictions - labels
|
||||
NDArray* diffs_ptr = (*predictions) - (*labels);
|
||||
NDArray diffs = *diffs_ptr;
|
||||
delete diffs_ptr;
|
||||
|
||||
// Compute sumOfSquares = sum(diffs^2)
|
||||
NDArray* diffsSquared = diffs * diffs;
|
||||
NDArray* sumOfSquares_ptr = diffsSquared->reduceAlongDimension(reduce::Sum, reductionIdx, true);
|
||||
delete diffsSquared;
|
||||
NDArray sumOfSquares = *sumOfSquares_ptr;
|
||||
delete sumOfSquares_ptr;
|
||||
|
||||
// Compute squareOfSum = (sum(diffs))^2
|
||||
NDArray* squareOfSum_ptr = diffs.reduceAlongDimension(reduce::Sum, reductionIdx, true);
|
||||
NDArray squareOfSum = *squareOfSum_ptr;
|
||||
delete squareOfSum_ptr;
|
||||
squareOfSum.applyScalar(scalar::Pow, 2, &squareOfSum);
|
||||
|
||||
delete reductionIdx;
|
||||
|
||||
// Compute E = ((sumOfSquares * n) - squareOfSum) * (4 / (n * (n - 1)))
|
||||
NDArray* sumOfSquaresTimesN = sumOfSquares * n;
|
||||
NDArray* numerator = (*sumOfSquaresTimesN) - squareOfSum;
|
||||
delete sumOfSquaresTimesN;
|
||||
|
||||
NDArray* E_ptr = (*numerator) * (4 / (n * (n - 1)));
|
||||
delete numerator;
|
||||
|
||||
NDArray E = *E_ptr;
|
||||
delete E_ptr;
|
||||
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == E.rankOf(), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: weights array should be scalar or have the same rank as results array, "
|
||||
"but got %i and %i correspondingly!",
|
||||
weights->rankOf(), E.rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, E), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got "
|
||||
"weights = %s and results = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(&E).c_str());
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(E)) weightsBroad = new NDArray(weights->tileToShape(E.shapeInfo()));
|
||||
|
||||
E *= *weightsBroad;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(&E);
|
||||
break;
|
||||
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
E.reduceNumber(reduce::Sum, output);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sumPtr = nullptr;
|
||||
if (weights->isScalar()) {
|
||||
NDArray* weightTimesLen = (*weights) * E.lengthOf();
|
||||
sumPtr = weightTimesLen;
|
||||
} else {
|
||||
sumPtr = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sumPtr->e<double>(0) == 0.) {
|
||||
(*output) = 0.;
|
||||
} else {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / (*sumPtr);
|
||||
delete eSum;
|
||||
output->assign(result);
|
||||
delete result;
|
||||
}
|
||||
delete sumPtr;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
NDArray* countNonZero = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countNonZero->e<LongType>(0);
|
||||
delete countNonZero;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0)
|
||||
(*output) = 0.;
|
||||
else {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / double(numOfNonZeroWeights);
|
||||
delete eSum;
|
||||
output->assign(result);
|
||||
delete result;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(mean_pairwssqerr_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(mean_pairwssqerr_loss) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
LongType *outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the shape as labels and logits minus last dimension
|
||||
std::vector<LongType> dimensions = {-1};
|
||||
outShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(predictionsShapeInfo), &dimensions, predictionsShapeInfo,
|
||||
false, true, block.getWorkspace());
|
||||
|
||||
// weights array can be single scalar or has the same rank as output, and must be broadcastable to output
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(outShapeInfo), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS OP: weights array should be scalar or have the same rank as output array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(outShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, outShapeInfo), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS OP: shapes of weights and output arrays should be broadcastable, but got weights = %s "
|
||||
"and output = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(outShapeInfo).c_str());
|
||||
}
|
||||
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(mean_pairwssqerr_loss_grad, 3, 3, false, 0, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dpredictions
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
// inputs validation
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, "
|
||||
"but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
auto n = double(labels->sizeAt(1));
|
||||
|
||||
// Compute diffs = predictions - labels
|
||||
NDArray* diffs_ptr = (*predictions) - (*labels);
|
||||
NDArray diffs = *diffs_ptr;
|
||||
delete diffs_ptr;
|
||||
|
||||
std::vector<LongType> dims2;
|
||||
dims2.push_back(0);
|
||||
std::vector<LongType> *reductionIdx = ShapeUtils::evalDimsToExclude(labels->rankOf(), 1,dims2.data());
|
||||
|
||||
// Compute sumOfSquares
|
||||
NDArray* diffsSquared = diffs * diffs;
|
||||
NDArray* sumOfSquares_ptr = diffsSquared->reduceAlongDimension(reduce::Sum, reductionIdx, true);
|
||||
delete diffsSquared;
|
||||
NDArray sumOfSquares = *sumOfSquares_ptr;
|
||||
delete sumOfSquares_ptr;
|
||||
|
||||
// Compute squareOfSum
|
||||
NDArray* squareOfSum_ptr = diffs.reduceAlongDimension(reduce::Sum, reductionIdx, true);
|
||||
NDArray squareOfSum = *squareOfSum_ptr;
|
||||
delete squareOfSum_ptr;
|
||||
squareOfSum.applyScalar(scalar::Pow, 2, &squareOfSum);
|
||||
|
||||
// Compute E
|
||||
NDArray* sumOfSquaresTimesN = sumOfSquares * n;
|
||||
NDArray* eTerm1 = (*sumOfSquaresTimesN) - squareOfSum;
|
||||
delete sumOfSquaresTimesN;
|
||||
NDArray* E_ptr = (*eTerm1) * (4 / (n * (n - 1)));
|
||||
delete eTerm1;
|
||||
NDArray E = *E_ptr;
|
||||
delete E_ptr;
|
||||
|
||||
// Compute gradients for predictions
|
||||
NDArray* sumPred_ptr = predictions->reduceAlongDimension(reduce::Sum, reductionIdx, true);
|
||||
NDArray sumPred = *sumPred_ptr;
|
||||
delete sumPred_ptr;
|
||||
|
||||
NDArray* sumLabel_ptr = labels->reduceAlongDimension(reduce::Sum, reductionIdx, true);
|
||||
NDArray sumLabel = *sumLabel_ptr;
|
||||
delete sumLabel_ptr;
|
||||
|
||||
delete reductionIdx;
|
||||
|
||||
// Compute dLdp = ((diffs * n) - sumPred + sumLabel) * (8 / (n * (n - 1)))
|
||||
NDArray* diffsTimesN = diffs * n;
|
||||
NDArray* term1 = (*diffsTimesN) - sumPred;
|
||||
delete diffsTimesN;
|
||||
NDArray* term2 = (*term1) + sumLabel;
|
||||
delete term1;
|
||||
NDArray* dLdpTemp = (*term2) * (8 / (n * (n - 1)));
|
||||
delete term2;
|
||||
dLdp->assign(dLdpTemp);
|
||||
delete dLdpTemp;
|
||||
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == E.rankOf(), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: weights array should be scalar or have the same rank as results array, "
|
||||
"but got %i and %i correspondingly!",
|
||||
weights->rankOf(), E.rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, E), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got "
|
||||
"weights = %s and results = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(&E).c_str());
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(E)) weightsBroad = new NDArray(weights->tileToShape(E.shapeInfo()));
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
*dLdp *= *weightsBroad;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
dLdw->assign(eSum);
|
||||
delete eSum;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else
|
||||
dLdw->assign(&E);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
|
||||
NDArray* sum = nullptr;
|
||||
if (weights->isScalar()) {
|
||||
NDArray* weightTimesLen = (*weights) * E.lengthOf();
|
||||
sum = weightTimesLen;
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* weightsDivSum = (*weightsBroad) / (*sum);
|
||||
*dLdp *= *weightsDivSum;
|
||||
delete weightsDivSum;
|
||||
|
||||
if (weights->isScalar())
|
||||
*dLdw = 0.;
|
||||
else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
|
||||
// Compute (E * sum - (E * weightsBroad).reduceNumber(Sum)) / (sum * sum)
|
||||
NDArray* ETimesSum = E * (*sum);
|
||||
NDArray* ETimesWeights = E * (*weightsBroad);
|
||||
NDArray* ETimesWeightsSum = ETimesWeights->reduceNumber(reduce::Sum);
|
||||
delete ETimesWeights;
|
||||
|
||||
NDArray* numerator = (*ETimesSum) - (*ETimesWeightsSum);
|
||||
delete ETimesSum;
|
||||
delete ETimesWeightsSum;
|
||||
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* result = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
|
||||
result->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete result;
|
||||
} else {
|
||||
// Compute (E * sum - (E * weightsBroad).reduceNumber(Sum)) / (sum * sum)
|
||||
NDArray* ETimesSum = E * (*sum);
|
||||
NDArray* ETimesWeights = E * (*weightsBroad);
|
||||
NDArray* ETimesWeightsSum = ETimesWeights->reduceNumber(reduce::Sum);
|
||||
delete ETimesWeights;
|
||||
|
||||
NDArray* numerator = (*ETimesSum) - (*ETimesWeightsSum);
|
||||
delete ETimesSum;
|
||||
delete ETimesWeightsSum;
|
||||
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* result = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
}
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
NDArray* countNonZero = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countNonZero->e<LongType>(0);
|
||||
delete countNonZero;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
auto* numOfNonZeroWeightsScalar =
|
||||
NDArrayFactory::create(dLdw->dataType(), numOfNonZeroWeights, block.launchContext());
|
||||
|
||||
if (weights->isScalar()) {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / double(numOfNonZeroWeights);
|
||||
delete eSum;
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
*dLdw /= *numOfNonZeroWeightsScalar;
|
||||
} else {
|
||||
NDArray* eDivNum = E / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(eDivNum);
|
||||
delete eDivNum;
|
||||
}
|
||||
|
||||
NDArray* weightsDivNum = (*weightsBroad) / (*numOfNonZeroWeightsScalar);
|
||||
NDArray temp = *weightsDivNum;
|
||||
delete weightsDivNum;
|
||||
*dLdp *= temp;
|
||||
|
||||
delete numOfNonZeroWeightsScalar;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
NDArray negDLdp = -(*dLdp); // unary negation returns value
|
||||
dLdl->assign(&negDLdp);
|
||||
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
DECLARE_TYPES(mean_pairwssqerr_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
DECLARE_SHAPE_FN(mean_pairwssqerr_loss_grad) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, "
|
||||
"but got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"MEAN_PAIRWSSQERR_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = "
|
||||
"%s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
LongType *dLdpShapeInfo =
|
||||
ShapeBuilders::copyShapeInfoAndType(predictionsShapeInfo, outType, false, block.getWorkspace());
|
||||
LongType *dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
LongType *dLdlShapeInfo =
|
||||
ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(dLdpShapeInfo, dLdwShapeInfo, dLdlShapeInfo);
|
||||
}
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
#pragma clang diagnostic pop
|
||||
@@ -0,0 +1,409 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 25.11.2017
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_mean_sqerr_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(mean_sqerr_loss, 3, 1, false, 0, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
|
||||
// inputs validation
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"MEAN_SQERR_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"MEAN_SQERR_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i "
|
||||
"and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"MEAN_SQERR_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s "
|
||||
"and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(
|
||||
reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"MEAN_SQERR_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(predictions->shapeInfo()));
|
||||
|
||||
NDArray E(labels->shapeInfo(), false, block.launchContext());
|
||||
predictions->applyPairwiseTransform(pairwise::SquaredSubtract, labels, &E);
|
||||
|
||||
// multiply E on weights
|
||||
NDArray* EWeighted = E * (*weightsBroad);
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(EWeighted);
|
||||
break;
|
||||
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
auto* sumResult = EWeighted->reduceNumber(reduce::Sum);
|
||||
output->assign(sumResult);
|
||||
delete sumResult;
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * EWeighted->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
(*output) = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* outAssign = (*sumE) / (*sum);
|
||||
output->assign(outAssign);
|
||||
delete outAssign;
|
||||
delete sumE;
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = EWeighted->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
(*output) = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* outAssign = (*sumE) / double(numOfNonZeroWeights);
|
||||
output->assign(outAssign);
|
||||
delete outAssign;
|
||||
delete sumE;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
STORE_RESULT(*output);
|
||||
|
||||
delete EWeighted;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
DECLARE_TYPES(mean_sqerr_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
DECLARE_SHAPE_FN(mean_sqerr_loss) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"MEAN_SQERR_LOSS OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"MEAN_SQERR_LOSS OP: weights array should be scalar or have the same rank as labels array, but got %i "
|
||||
"and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"MEAN_SQERR_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and "
|
||||
"labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
LongType * outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the same shape as labels and predictions
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(labelsShapeInfo),
|
||||
shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))->primary();
|
||||
}
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(mean_sqerr_loss_grad, 3, 3, false, 0, 1) {
|
||||
auto predictions = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dpredictions
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
// inputs validation
|
||||
REQUIRE_TRUE(labels->isSameShape(predictions), 0,
|
||||
"MEAN_SQERR_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(predictions).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"MEAN_SQERR_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but got "
|
||||
"%i and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"MEAN_SQERR_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights "
|
||||
"= %s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"MEAN_SQERR_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, but "
|
||||
"got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(predictions))
|
||||
weightsBroad = new NDArray(weights->tileToShape(predictions->shapeInfo()));
|
||||
|
||||
NDArray* diff = (*predictions) - (*labels);
|
||||
|
||||
// dE_i/dp_i = 2 * (p_i - y_i)
|
||||
NDArray* dldpTemp = (*diff) * 2.;
|
||||
dLdp->assign(dldpTemp);
|
||||
delete dldpTemp;
|
||||
|
||||
// dE_i/dy_i = -2 * (p_i - y_i)
|
||||
NDArray* E = (*diff) * (*diff);
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
NDArray* dLdpWeighted = (*dLdp) * (*weightsBroad);
|
||||
dLdp->assign(dLdpWeighted);
|
||||
delete dLdpWeighted;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
dLdw->assign(sumE);
|
||||
delete sumE;
|
||||
}
|
||||
else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
}
|
||||
else {
|
||||
dLdw->assign(E);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * E->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* weightsDivSum = (*weightsBroad) / (*sum);
|
||||
NDArray* dLdpResult = (*dLdp) * (*weightsDivSum);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
delete weightsDivSum;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
*dLdw = 0.;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
gradTemp->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete gradTemp;
|
||||
}
|
||||
else {
|
||||
NDArray* EWeighted = (*E) * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = (*E) * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* dLdwTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
dLdw->assign(dLdwTemp);
|
||||
delete dLdwTemp;
|
||||
}
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
auto numOfNonZeroWeightsScalar =
|
||||
NDArrayFactory::create(dLdw->dataType(), numOfNonZeroWeights, block.launchContext());
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E->reduceNumber(reduce::Sum);
|
||||
auto* dLdwTemp = (*sumE) / double(numOfNonZeroWeights);
|
||||
dLdw->assign(dLdwTemp);
|
||||
delete dLdwTemp;
|
||||
delete sumE;
|
||||
}
|
||||
else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
NDArray* dLdwResult = (*dLdw) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(dLdwResult);
|
||||
delete dLdwResult;
|
||||
}
|
||||
else {
|
||||
auto* dLdwTemp = (*E) / numOfNonZeroWeights;
|
||||
dLdw->assign(dLdwTemp);
|
||||
delete dLdwTemp;
|
||||
}
|
||||
|
||||
NDArray* temp = (*weightsBroad) / (*numOfNonZeroWeightsScalar);
|
||||
NDArray* dLdpResult = (*dLdp) * (*temp);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
delete temp;
|
||||
|
||||
delete numOfNonZeroWeightsScalar;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
NDArray dldlAssign = -*dLdp;
|
||||
dLdl->assign(&dldlAssign);
|
||||
|
||||
delete E;
|
||||
delete diff;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
DECLARE_TYPES(mean_sqerr_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
DECLARE_SHAPE_FN(mean_sqerr_loss_grad) {
|
||||
auto predictionsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and predictions must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, predictionsShapeInfo), 0,
|
||||
"MEAN_SQERR_LOSS_GRAD OP: labels and predictions arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(),
|
||||
ShapeUtils::shapeAsString(predictionsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"MEAN_SQERR_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, but got "
|
||||
"%i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"MEAN_SQERR_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s and "
|
||||
"labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(predictionsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ShapeBuilders::copyShapeInfoAndType(predictionsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdlShapeInfo = ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(CONSTANT(dLdpShapeInfo), CONSTANT(dLdwShapeInfo), CONSTANT(dLdlShapeInfo));
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,441 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 25.11.2017
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_sigm_cross_entropy_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/helpers/legacy_helpers.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(sigm_cross_entropy_loss, 3, 1, false, 1, 1) {
|
||||
auto logits = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
auto labelsSmoothing = T_ARG(0);
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(logits), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS OP: labels and logits arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly!",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(logits).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got "
|
||||
"weights = %s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, "
|
||||
"but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(logits))
|
||||
weightsBroad = new NDArray(weights->tileToShape(logits->shapeInfo()));
|
||||
|
||||
// If labelsSmoothing is nonzero, smooth the labels towards 1/2:
|
||||
auto newLabels = labels;
|
||||
if (labelsSmoothing != 0.) {
|
||||
newLabels = new NDArray(*labels);
|
||||
newLabels->applyScalar(scalar::SXELogitsSmoother, labelsSmoothing, newLabels);
|
||||
}
|
||||
|
||||
NDArray E(labels, false, block.launchContext());
|
||||
|
||||
// logits - labels * logits + log(1 + exp(-logits)) -> take into account numerical stability at large logits
|
||||
helpers::sigmCrossEntropy(block.launchContext(), logits, newLabels, &E);
|
||||
|
||||
// multiply E on weights
|
||||
NDArray* EWeighted = E * (*weightsBroad);
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(EWeighted);
|
||||
break;
|
||||
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
auto* sumResult = EWeighted->reduceNumber(reduce::Sum);
|
||||
output->assign(sumResult);
|
||||
delete sumResult;
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * EWeighted->lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*output = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* outputTemp = (*sumE) / (*sum);
|
||||
output->assign(outputTemp);
|
||||
delete outputTemp;
|
||||
delete sumE;
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = EWeighted->lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
(*output) = 0.;
|
||||
} else {
|
||||
auto* sumE = EWeighted->reduceNumber(reduce::Sum);
|
||||
auto* outputTemp = (*sumE) / double(numOfNonZeroWeights);
|
||||
output->assign(outputTemp);
|
||||
delete outputTemp;
|
||||
delete sumE;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delete EWeighted;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
if (newLabels != labels) delete newLabels;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(sigm_cross_entropy_loss) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(sigm_cross_entropy_loss) {
|
||||
auto logitsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and logits must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, logitsShapeInfo), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS OP: labels and logits arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS OP: weights array should be scalar or have the same rank as labels array, but "
|
||||
"got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS OP: shapes of weights and labels arrays should be broadcastable, but got weights = %s "
|
||||
"and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(logitsShapeInfo));
|
||||
LongType* outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the same shape as labels and logits
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(labelsShapeInfo),
|
||||
shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))->primary();
|
||||
}
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(sigm_cross_entropy_loss_grad, 3, 3, false, 1, 1) {
|
||||
auto logits = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dlogits
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
NDArray *labelsSmoothing = NDArrayFactory::create(logits->dataType(), T_ARG(0), block.launchContext());
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(logits), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS_GRAD OP: labels and logits arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly!",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(logits).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == labels->rankOf(), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, "
|
||||
"but got %i and %i correspondingly!",
|
||||
weights->rankOf(), labels->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *labels), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got "
|
||||
"weights = %s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, 2, "
|
||||
"3, but got %i instead!",
|
||||
reductionMode);
|
||||
|
||||
// perform weights broadcasting/tile to labels if needed
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(logits))
|
||||
weightsBroad = new NDArray(weights->tileToShape(logits->shapeInfo()));
|
||||
|
||||
// If labelsSmoothing is nonzero, smooth the labels towards 1/2:
|
||||
auto newLabels = labels;
|
||||
if (labelsSmoothing->e<float>(0) != 0.f) {
|
||||
newLabels = new NDArray(*labels);
|
||||
newLabels->applyScalar(scalar::SXELogitsSmoother, labelsSmoothing->e<float>(0), newLabels);
|
||||
}
|
||||
|
||||
NDArray E(labels, false, block.launchContext());
|
||||
|
||||
// logits - labels * logits + log(1 + exp(-logits)) -> take into account numerical stability at large logits
|
||||
helpers::sigmCrossEntropy(block.launchContext(), logits, newLabels, &E);
|
||||
|
||||
// dLdp = 1 - labels - 1 / (1 + exp(logits))
|
||||
helpers::sigmCrossEntropyGrad(block.launchContext(), logits, newLabels, dLdp);
|
||||
|
||||
// dLdl = -logits
|
||||
*labelsSmoothing -= 1.f;
|
||||
NDArray* dLdlTemp = (*logits) * (*labelsSmoothing);
|
||||
dLdl->assign(dLdlTemp);
|
||||
delete dLdlTemp;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
NDArray* dLdpWeighted = (*dLdp) * (*weightsBroad);
|
||||
dLdp->assign(dLdpWeighted);
|
||||
delete dLdpWeighted;
|
||||
|
||||
NDArray* dLdlWeighted = (*dLdl) * (*weightsBroad);
|
||||
dLdl->assign(dLdlWeighted);
|
||||
delete dLdlWeighted;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E.reduceNumber(reduce::Sum);
|
||||
dLdw->assign(sumE);
|
||||
delete sumE;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else {
|
||||
dLdw->assign(&E);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum;
|
||||
if (weights->isScalar()) {
|
||||
sum = (*weights) * E.lengthOf();
|
||||
} else {
|
||||
sum = weightsBroad->reduceNumber(reduce::Sum);
|
||||
}
|
||||
|
||||
if (sum->e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* temp = (*weightsBroad) / (*sum);
|
||||
NDArray* dLdpResult = (*dLdp) * (*temp);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
|
||||
NDArray* dLdlResult = (*dLdl) * (*temp);
|
||||
dLdl->assign(dLdlResult);
|
||||
delete dLdlResult;
|
||||
delete temp;
|
||||
|
||||
if (weights->isScalar()) {
|
||||
*dLdw = 0.;
|
||||
} else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
NDArray* EWeighted = E * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = E * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* gradTemp = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
gradTemp->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete gradTemp;
|
||||
} else {
|
||||
NDArray* EWeighted = E * (*weightsBroad);
|
||||
NDArray* EWeightedSum = EWeighted->reduceNumber(reduce::Sum);
|
||||
delete EWeighted;
|
||||
NDArray* ESum = E * (*sum);
|
||||
NDArray* numerator = (*ESum) - (*EWeightedSum);
|
||||
delete ESum;
|
||||
delete EWeightedSum;
|
||||
NDArray* sumSquared = (*sum) * (*sum);
|
||||
NDArray* dLdwTemp1 = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
dLdw->assign(dLdwTemp1);
|
||||
delete dLdwTemp1;
|
||||
}
|
||||
}
|
||||
delete sum;
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
auto* countResult = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countResult->e<LongType>(0);
|
||||
delete countResult;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
auto numOfNonZeroWeightsScalar =
|
||||
NDArrayFactory::create(dLdw->dataType(), numOfNonZeroWeights, block.launchContext());
|
||||
|
||||
if (weights->isScalar()) {
|
||||
auto* sumE = E.reduceNumber(reduce::Sum);
|
||||
auto* dLdwTemp2 = (*sumE) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(dLdwTemp2);
|
||||
delete dLdwTemp2;
|
||||
delete sumE;
|
||||
}
|
||||
else if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
NDArray* dLdwResult = (*dLdw) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(dLdwResult);
|
||||
delete dLdwResult;
|
||||
} else {
|
||||
auto* sumE = E.reduceNumber(reduce::Sum);
|
||||
auto* dLdwTemp2 = (*sumE) / (*numOfNonZeroWeightsScalar);
|
||||
dLdw->assign(dLdwTemp2);
|
||||
delete dLdwTemp2;
|
||||
delete sumE;
|
||||
}
|
||||
|
||||
NDArray* temp = (*weightsBroad) / (*numOfNonZeroWeightsScalar);
|
||||
NDArray* dLdpResult = (*dLdp) * (*temp);
|
||||
dLdp->assign(dLdpResult);
|
||||
delete dLdpResult;
|
||||
|
||||
NDArray* dLdlResult = (*dLdl) * (*temp);
|
||||
dLdl->assign(dLdlResult);
|
||||
delete dLdlResult;
|
||||
delete temp;
|
||||
|
||||
delete numOfNonZeroWeightsScalar;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
delete labelsSmoothing;
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
if (newLabels != labels) delete newLabels;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(sigm_cross_entropy_loss_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(sigm_cross_entropy_loss_grad) {
|
||||
auto logitsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and logits must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(labelsShapeInfo, logitsShapeInfo), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS_GRAD OP: labels and logits arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
// weights array can be single scalar or has the same rank as labels, and must be broadcastable to labels
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(labelsShapeInfo), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS_GRAD OP: weights array should be scalar or have the same rank as labels array, "
|
||||
"but got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(labelsShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, labelsShapeInfo), 0,
|
||||
"SIGM_CROSS_ENTROPY_LOSS_GRAD OP: shapes of weights and labels arrays should be broadcastable, but got weights = "
|
||||
"%s and labels = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(labelsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(logitsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ShapeBuilders::copyShapeInfoAndType(logitsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdwShapeInfo = ShapeBuilders::copyShapeInfoAndType(weightsShapeInfo, outType, false, block.getWorkspace());
|
||||
auto dLdlShapeInfo = ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(CONSTANT(dLdpShapeInfo), CONSTANT(dLdwShapeInfo), CONSTANT(dLdlShapeInfo));
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,571 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 25.11.2017.
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_softmax_cross_entropy_loss)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(softmax_cross_entropy_loss, 3, 1, false, 1, 1) {
|
||||
auto logits = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
double labelsSmoothing = T_ARG(0);
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(logits), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: labels and logits arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(logits).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: reduction mode value is not acceptable, possible values are 0, 1, 2, 3, "
|
||||
"but got %i instead!",
|
||||
reductionMode);
|
||||
// smoothing is possible for rank of logits/labels > 1
|
||||
REQUIRE_TRUE(labels->rankOf() > 1 || (labels->rankOf() == 1 && labelsSmoothing == 0.), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: smoothing is not possible when rank of labels/ logits = 1 !");
|
||||
|
||||
if (!output->isScalar()) {
|
||||
// weights array can be single scalar or has the same shape as output, and must be broadcastable to output shape
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == output->rankOf(), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: weights array should be scalar or have the same rank as output array, "
|
||||
"but got %i and %i correspondingly!",
|
||||
weights->rankOf(), output->rankOf());
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(*weights, *output), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: shapes of weights and output arrays should be broadcastable, but got "
|
||||
"weights = %s and output = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(labels).c_str());
|
||||
}
|
||||
|
||||
// If label_smoothing is nonzero, smooth the labels towards 1/num_classes: new_onehot_labels = onehot_labels * (1 -
|
||||
// label_smoothing) + label_smoothing / num_classes num_classes = labels->sizeAt(1)
|
||||
NDArray* cLabels = labels->cast(weights->dataType()); // cast() already returns NDArray*
|
||||
NDArray* newLabels = cLabels;
|
||||
if (labelsSmoothing != 0.) {
|
||||
newLabels = new NDArray(cLabels);
|
||||
NDArray* term1 = (1.f - labelsSmoothing) * (*cLabels);
|
||||
NDArray* term2 = (*term1) + (labelsSmoothing / cLabels->sizeAt(1));
|
||||
delete term1;
|
||||
newLabels->assign(term2);
|
||||
delete term2;
|
||||
}
|
||||
|
||||
// main formula: result = - sum_i(lables_i * log(softmax_i)) - sum over last dimension
|
||||
// softmax_i = exp(logits_i) / sum_j(exp(logits_j))
|
||||
// so result = sum_i( lables_i * (log(sum_j(exp(logits_j))) - logits_i) )
|
||||
// for numerical stability we use shifted logits (one can approve this using simple math):
|
||||
// softmax_i = exp(logits_i - maxLogit) / sum_j(exp(logits_j - maxLogit))
|
||||
// maxLogit is max among logits_i
|
||||
|
||||
std::vector<LongType> dimensions = {-1};
|
||||
NDArray* maxLogits = logits->reduceAlongDimension(reduce::Max, &dimensions, true);
|
||||
NDArray* shiftedLogits_ptr = (*logits) - (*maxLogits);
|
||||
delete maxLogits;
|
||||
NDArray shiftedLogits = *shiftedLogits_ptr;
|
||||
delete shiftedLogits_ptr;
|
||||
|
||||
NDArray* expShifted = shiftedLogits.transform(transform::Exp);
|
||||
NDArray* sumExp = expShifted->reduceAlongDimension(reduce::Sum, &dimensions, true);
|
||||
delete expShifted;
|
||||
NDArray* logSumExp_ptr = sumExp->transform(transform::Log);
|
||||
delete sumExp;
|
||||
NDArray logSumExp = *logSumExp_ptr;
|
||||
delete logSumExp_ptr;
|
||||
|
||||
// E = (newLabels * (logSumExp - shiftedLogits)).reduceAlongDimension(Sum)
|
||||
NDArray* diff = logSumExp - shiftedLogits;
|
||||
NDArray* product = (*newLabels) * (*diff);
|
||||
delete diff;
|
||||
NDArray* E_ptr = product->reduceAlongDimension(reduce::Sum, &dimensions);
|
||||
delete product;
|
||||
NDArray E = *E_ptr;
|
||||
delete E_ptr;
|
||||
|
||||
// perform weights broadcasting/tile to E if it is necessary
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(&E)) {
|
||||
std::vector<LongType> weightsShape = {weights->lengthOf()};
|
||||
if (E.rankOf() == 1 && weights->isVector() && weights->rankOf() > 1)
|
||||
weightsBroad = weights->reshape(weights->ordering(), weightsShape);
|
||||
else
|
||||
weightsBroad = new NDArray(weights->tileToShape(E.shapeInfo()));
|
||||
}
|
||||
|
||||
// multiply E on weights
|
||||
E *= *weightsBroad;
|
||||
|
||||
switch (reductionMode) {
|
||||
case 0: // 0 - "none", un-reduced weighted losses with the same shape as labels.
|
||||
output->assign(&E);
|
||||
break;
|
||||
|
||||
case 1: { // 1 - "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
E.reduceNumber(reduce::Sum, output);
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
double sum;
|
||||
if (weights->isScalar())
|
||||
sum = weights->e<double>(0) * E.lengthOf();
|
||||
else {
|
||||
NDArray* sumPtr = weightsBroad->reduceNumber(reduce::Sum);
|
||||
sum = sumPtr->e<double>(0);
|
||||
delete sumPtr;
|
||||
}
|
||||
|
||||
if (sum == 0.)
|
||||
*output = 0.;
|
||||
else {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / sum;
|
||||
delete eSum;
|
||||
output->assign(result);
|
||||
delete result;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
NDArray* countNonZero = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countNonZero->e<LongType>(0);
|
||||
delete countNonZero;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0)
|
||||
*output = 0.;
|
||||
else {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / double(numOfNonZeroWeights);
|
||||
delete eSum;
|
||||
output->assign(result);
|
||||
delete result;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
if (newLabels != cLabels) delete newLabels;
|
||||
|
||||
delete cLabels;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(softmax_cross_entropy_loss) {
|
||||
getOpDescriptor()
|
||||
->setAllowedInputTypes(0, {ALL_FLOATS})
|
||||
->setAllowedInputTypes(1, {ALL_FLOATS})
|
||||
->setAllowedInputTypes(2, {ALL_FLOATS, ALL_INTS})
|
||||
->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(softmax_cross_entropy_loss) {
|
||||
auto logitsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
// labels and logits must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(logitsShapeInfo, labelsShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: labels and logits arrays must have the same shapes, but got %s and %s "
|
||||
"correspondingly!",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(logitsShapeInfo));
|
||||
LongType* outShapeInfo = nullptr;
|
||||
|
||||
if (INT_ARG(0) != 0) // in this case output is scalar
|
||||
outShapeInfo = ConstantShapeHelper::getInstance().scalarShapeInfo(outType);
|
||||
else { // in this case output has the shape as labels and logits minus last dimension
|
||||
std::vector<LongType> dimensions = {-1};
|
||||
outShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(logitsShapeInfo), &dimensions, logitsShapeInfo, false,
|
||||
true, block.getWorkspace());
|
||||
|
||||
// weights array can be single scalar or has the same rank as output, and must be broadcastable to output
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(outShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: weights array should be scalar or have the same rank as output array, "
|
||||
"but got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(outShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(
|
||||
shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, outShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS OP: shapes of weights and output arrays should be broadcastable, but got weights = "
|
||||
"%s and output = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(outShapeInfo).c_str());
|
||||
}
|
||||
|
||||
return SHAPELIST(outShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(softmax_cross_entropy_loss_grad, 3, 3, false, 1, 1) {
|
||||
auto logits = INPUT_VARIABLE(0);
|
||||
auto weights = INPUT_VARIABLE(1);
|
||||
auto labels = INPUT_VARIABLE(2);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dlogits
|
||||
auto dLdw = OUTPUT_VARIABLE(1); // dL/dweights
|
||||
auto dLdl = OUTPUT_VARIABLE(2); // dL/dlabels
|
||||
|
||||
auto labelsSmoothing = T_ARG(0);
|
||||
|
||||
int reductionMode =
|
||||
INT_ARG(0); // 0 - "none"; 1 - "weighted_sum"; 2 - "weighted_mean"; 3 - "weighted_sum_by_nonzero_weights"
|
||||
// take into account Alex's proposition to treat "none" the same as "weighted_sum" mode when calculating gradients
|
||||
if (reductionMode == 0) reductionMode = 1;
|
||||
|
||||
std::vector<LongType> *dimensions = new std::vector<LongType>({-1});
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(logits), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_GRAD OP: labels and logits arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(logits).c_str());
|
||||
// only 4 possible reduction modes exist
|
||||
REQUIRE_TRUE(reductionMode == 0 || reductionMode == 1 || reductionMode == 2 || reductionMode == 3, 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_GRAD OP: reduction mode value is not acceptable, possible values are 0, 1, "
|
||||
"2, 3, but got %i instead!",
|
||||
reductionMode);
|
||||
auto lossShapeInfo = ShapeUtils::evalReduceShapeInfo(logits->ordering(), dimensions, logits->shapeInfo(), false,
|
||||
false, block.getWorkspace());
|
||||
// weights array can be single scalar or has the same shape as loss, and must be broadcastable to loss shape
|
||||
REQUIRE_TRUE(weights->isScalar() || weights->rankOf() == shape::rank(lossShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_GRAD OP: weights array should be scalar or have the same rank as loss "
|
||||
"array, but got %i and %i correspondingly!",
|
||||
weights->rankOf(), shape::rank(lossShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(weights->isScalar() || ShapeUtils::areShapesBroadcastable(weights->shapeInfo(), lossShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_GRAD OP: shapes of weights and loss arrays should be broadcastable, but got "
|
||||
"weights = %s and loss = %s instead!",
|
||||
ShapeUtils::shapeAsString(weights).c_str(), ShapeUtils::shapeAsString(lossShapeInfo).c_str());
|
||||
// smoothing is possible for rank of logits/labels > 1
|
||||
REQUIRE_TRUE(labels->rankOf() > 1 || (labels->rankOf() == 1 && labelsSmoothing == 0.), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_GRAD OP: smoothing is not possible when rank of labels/ logits = 1 !");
|
||||
|
||||
// If label_smoothing is nonzero, smooth the labels towards 1/num_classes: new_onehot_labels = onehot_labels * (1 -
|
||||
// label_smoothing) + label_smoothing / num_classes num_classes = labels->sizeAt(1)
|
||||
NDArray* cLabels = labels->cast(weights->dataType()); // cast() already returns NDArray*
|
||||
NDArray* newLabels = cLabels;
|
||||
if (labelsSmoothing != 0.) {
|
||||
newLabels = new NDArray(labels->shapeInfo(), dLdl->dataType(), false, block.launchContext());
|
||||
NDArray* term1 = (1.f - labelsSmoothing) * (*cLabels);
|
||||
NDArray* term2 = (*term1) + (labelsSmoothing / cLabels->sizeAt(1));
|
||||
delete term1;
|
||||
newLabels->assign(term2);
|
||||
delete term2;
|
||||
}
|
||||
|
||||
// Compute softmax
|
||||
NDArray* maxLogits = logits->reduceAlongDimension(reduce::Max, dimensions, true);
|
||||
NDArray* shiftedLogits_ptr = (*logits) - (*maxLogits);
|
||||
delete maxLogits;
|
||||
NDArray* expShifted = shiftedLogits_ptr->transform(transform::Exp);
|
||||
delete shiftedLogits_ptr;
|
||||
NDArray* sumExp = expShifted->reduceAlongDimension(reduce::Sum, dimensions, true);
|
||||
NDArray* softmax_ptr = (*expShifted) / (*sumExp);
|
||||
delete expShifted;
|
||||
delete sumExp;
|
||||
NDArray softmax = *softmax_ptr;
|
||||
delete softmax_ptr;
|
||||
|
||||
// dEdp = softmax * sum_i(lables_i) - labels
|
||||
NDArray* labelSum = newLabels->reduceAlongDimension(reduce::Sum, dimensions, true);
|
||||
NDArray* softmaxTimesLabelSum = softmax * (*labelSum);
|
||||
delete labelSum;
|
||||
NDArray* dLdpTemp_ptr = (*softmaxTimesLabelSum) - (*newLabels);
|
||||
delete softmaxTimesLabelSum;
|
||||
dLdp->assign(dLdpTemp_ptr);
|
||||
delete dLdpTemp_ptr;
|
||||
|
||||
// dEdl = -log(softmax)
|
||||
NDArray* logSoftmax = softmax.transform(transform::Log);
|
||||
NDArray negLogSoftmax = -(*logSoftmax); // unary negation returns value
|
||||
delete logSoftmax;
|
||||
NDArray* dLdlTemp_ptr = negLogSoftmax * (1.f - labelsSmoothing);
|
||||
dLdl->assign(dLdlTemp_ptr);
|
||||
delete dLdlTemp_ptr;
|
||||
|
||||
// Compute E for gradient calculations
|
||||
NDArray* maxLogits2 = logits->reduceAlongDimension(reduce::Max, dimensions, true);
|
||||
NDArray* shiftedLogits2_ptr = (*logits) - (*maxLogits2);
|
||||
delete maxLogits2;
|
||||
NDArray shiftedLogits = *shiftedLogits2_ptr;
|
||||
delete shiftedLogits2_ptr;
|
||||
|
||||
NDArray* expShifted2 = shiftedLogits.transform(transform::Exp);
|
||||
NDArray* sumExp2 = expShifted2->reduceAlongDimension(reduce::Sum, dimensions, true);
|
||||
delete expShifted2;
|
||||
NDArray* logSumExp_ptr = sumExp2->transform(transform::Log);
|
||||
delete sumExp2;
|
||||
NDArray logSumExp = *logSumExp_ptr;
|
||||
delete logSumExp_ptr;
|
||||
|
||||
NDArray* diff = logSumExp - shiftedLogits;
|
||||
NDArray* product = (*newLabels) * (*diff);
|
||||
delete diff;
|
||||
NDArray* E_ptr = product->reduceAlongDimension(reduce::Sum, dimensions);
|
||||
delete product;
|
||||
NDArray E = *E_ptr;
|
||||
delete E_ptr;
|
||||
|
||||
// perform weights broadcasting/tile to E if it is necessary
|
||||
auto weightsBroad = weights;
|
||||
if (!weights->isScalar() && !weights->isSameShape(&E))
|
||||
weightsBroad = new NDArray(weights->tileToShape(E.shapeInfo()));
|
||||
|
||||
auto excludeDims = ShapeUtils::evalDimsToExclude(dLdp->rankOf(), dimensions->size(), dimensions->data());
|
||||
|
||||
switch (reductionMode) {
|
||||
case 1: { // 1 - "none" and "weighted_sum", output is scalar and equal to sum of all elements of E array
|
||||
|
||||
if (weights->isScalar() || weights->lengthOf() == 1) {
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
dLdw->assign(eSum);
|
||||
delete eSum;
|
||||
*dLdp *= *weights;
|
||||
*dLdl *= *weights;
|
||||
} else {
|
||||
dLdp->applyBroadcast(broadcast::Multiply, excludeDims, weightsBroad, dLdp);
|
||||
dLdl->applyBroadcast(broadcast::Multiply, excludeDims, weightsBroad, dLdl);
|
||||
|
||||
if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
} else
|
||||
dLdw->assign(&E);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 2: { // 2 - "weighted_mean", output is scalar and equal to sum of all elements of E array divided by sum of
|
||||
// all elements of weightsBroad array
|
||||
NDArray* sum_ptr = nullptr;
|
||||
if (weights->isScalar())
|
||||
sum_ptr = (*weights) * E.lengthOf();
|
||||
else
|
||||
sum_ptr = weightsBroad->reduceNumber(reduce::Sum);
|
||||
|
||||
NDArray sum = *sum_ptr;
|
||||
delete sum_ptr;
|
||||
|
||||
if (sum.e<double>(0) == 0.) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
if (weights->isScalar() || weights->lengthOf() == 1) {
|
||||
NDArray* temp_ptr = (*weights) / sum;
|
||||
NDArray temp = *temp_ptr;
|
||||
delete temp_ptr;
|
||||
*dLdp *= temp;
|
||||
*dLdl *= temp;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
NDArray* temp_ptr = (*weightsBroad) / sum;
|
||||
NDArray temp = *temp_ptr;
|
||||
delete temp_ptr;
|
||||
dLdp->applyBroadcast(broadcast::Multiply, dimensions, &temp, dLdp);
|
||||
dLdl->applyBroadcast(broadcast::Multiply, dimensions, &temp, dLdl);
|
||||
|
||||
if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
|
||||
// Compute (E * sum - (E * weightsBroad).reduceNumber(Sum)) / (sum * sum)
|
||||
NDArray* ETimesSum = E * sum;
|
||||
NDArray* ETimesWeights = E * (*weightsBroad);
|
||||
NDArray* ETimesWeightsSum = ETimesWeights->reduceNumber(reduce::Sum);
|
||||
delete ETimesWeights;
|
||||
|
||||
NDArray* numerator = (*ETimesSum) - (*ETimesWeightsSum);
|
||||
delete ETimesSum;
|
||||
delete ETimesWeightsSum;
|
||||
|
||||
NDArray* sumSquared = sum * sum;
|
||||
NDArray* result = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
|
||||
result->reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
delete result;
|
||||
} else {
|
||||
// Compute (E * sum - (E * weightsBroad).reduceNumber(Sum)) / (sum * sum)
|
||||
NDArray* ETimesSum = E * sum;
|
||||
NDArray* ETimesWeights = E * (*weightsBroad);
|
||||
NDArray* ETimesWeightsSum = ETimesWeights->reduceNumber(reduce::Sum);
|
||||
delete ETimesWeights;
|
||||
|
||||
NDArray* numerator = (*ETimesSum) - (*ETimesWeightsSum);
|
||||
delete ETimesSum;
|
||||
delete ETimesWeightsSum;
|
||||
|
||||
NDArray* sumSquared = sum * sum;
|
||||
NDArray* result = (*numerator) / (*sumSquared);
|
||||
delete numerator;
|
||||
delete sumSquared;
|
||||
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 3: { // 3 - "weighted_sum_by_nonzero_weights", output is scalar and equal to scalar sum of all elements of E
|
||||
// array divided by number of non-zero weights
|
||||
LongType numOfNonZeroWeights = 0;
|
||||
if (weights->isScalar()) {
|
||||
if (weights->e<double>(0) != 0.) numOfNonZeroWeights = E.lengthOf();
|
||||
} else {
|
||||
NDArray* countNonZero = weightsBroad->reduceNumber(reduce::CountNonZero);
|
||||
numOfNonZeroWeights = countNonZero->e<LongType>(0);
|
||||
delete countNonZero;
|
||||
}
|
||||
|
||||
if (numOfNonZeroWeights == 0) {
|
||||
*dLdp = 0.;
|
||||
*dLdl = 0.;
|
||||
*dLdw = 0.;
|
||||
} else {
|
||||
if (weights->isScalar() || weights->lengthOf() == 1) {
|
||||
NDArray* temp_ptr = (*weights) / numOfNonZeroWeights;
|
||||
NDArray temp = *temp_ptr;
|
||||
delete temp_ptr;
|
||||
*dLdp *= temp;
|
||||
*dLdl *= temp;
|
||||
|
||||
NDArray* eSum = E.reduceNumber(reduce::Sum);
|
||||
NDArray* result = (*eSum) / numOfNonZeroWeights;
|
||||
delete eSum;
|
||||
dLdw->assign(result);
|
||||
delete result;
|
||||
} else {
|
||||
NDArray* temp_ptr = (*weightsBroad) / numOfNonZeroWeights;
|
||||
NDArray temp = *temp_ptr;
|
||||
delete temp_ptr;
|
||||
dLdp->applyBroadcast(broadcast::Multiply, dimensions, &temp, dLdp);
|
||||
dLdl->applyBroadcast(broadcast::Multiply, dimensions, &temp, dLdl);
|
||||
|
||||
if (weights != weightsBroad) {
|
||||
std::vector<LongType> axesToReduceAlong =
|
||||
ShapeUtils::evalBroadcastBackwardAxis(weights->shapeInfo(), weightsBroad->shapeInfo());
|
||||
E.reduceAlongDimension(reduce::Sum, dLdw, &axesToReduceAlong, true);
|
||||
*dLdw /= numOfNonZeroWeights;
|
||||
} else {
|
||||
NDArray* eDivNum = E / numOfNonZeroWeights;
|
||||
dLdw->assign(eDivNum);
|
||||
delete eDivNum;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (weightsBroad != weights) delete weightsBroad;
|
||||
|
||||
if (newLabels != cLabels) delete newLabels;
|
||||
|
||||
delete cLabels;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(softmax_cross_entropy_loss_grad) {
|
||||
getOpDescriptor()
|
||||
->setAllowedInputTypes(0, {ALL_FLOATS})
|
||||
->setAllowedInputTypes(1, {ALL_FLOATS})
|
||||
->setAllowedInputTypes(2, {ALL_FLOATS, ALL_INTS})
|
||||
->setAllowedInputTypes(3, {ALL_FLOATS})
|
||||
->setAllowedInputTypes(4, {ALL_FLOATS})
|
||||
->setAllowedInputTypes(5, {ALL_FLOATS})
|
||||
->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(softmax_cross_entropy_loss_grad) {
|
||||
auto logitsShapeInfo = inputShape->at(0);
|
||||
auto weightsShapeInfo = inputShape->at(1);
|
||||
auto labelsShapeInfo = inputShape->at(2);
|
||||
|
||||
std::vector<LongType> dimensions = {-1};
|
||||
|
||||
// labels and logits must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(logitsShapeInfo, labelsShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_GRAD OP: labels and logits arrays must have the same shapes, but got %s and "
|
||||
"%s correspondingly!",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
auto lossShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(logitsShapeInfo), &dimensions, logitsShapeInfo,
|
||||
false, false, block.getWorkspace());
|
||||
// weights array can be single scalar or has the same rank as loss, and must be broadcastable to loss
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || shape::rank(weightsShapeInfo) == shape::rank(lossShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_GRAD OP: weights array should be scalar or have the same rank as loss "
|
||||
"array, but got %i and %i correspondingly!",
|
||||
shape::rank(weightsShapeInfo), shape::rank(lossShapeInfo));
|
||||
// check whether broadcast operation is possible for weights array
|
||||
REQUIRE_TRUE(shape::isScalar(weightsShapeInfo) || ShapeUtils::areShapesBroadcastable(weightsShapeInfo, lossShapeInfo),
|
||||
0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_GRAD OP: shapes of weights and loss arrays should be broadcastable, but got "
|
||||
"weights = %s and loss = %s instead!",
|
||||
ShapeUtils::shapeAsString(weightsShapeInfo).c_str(), ShapeUtils::shapeAsString(lossShapeInfo).c_str());
|
||||
|
||||
auto outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(logitsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(logitsShapeInfo),
|
||||
shape::rank(logitsShapeInfo),
|
||||
shape::shapeOf(logitsShapeInfo))->primary();
|
||||
|
||||
auto dLdwShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(weightsShapeInfo),
|
||||
shape::rank(weightsShapeInfo),
|
||||
shape::shapeOf(weightsShapeInfo))->primary();
|
||||
|
||||
auto dLdlShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(labelsShapeInfo),
|
||||
shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))->primary();
|
||||
return SHAPELIST(dLdpShapeInfo, dLdwShapeInfo, dLdlShapeInfo);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,202 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 18.06.2018
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_softmax_cross_entropy_loss_with_logits)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(softmax_cross_entropy_loss_with_logits, 2, 1, false, 0, 0) {
|
||||
auto logits = INPUT_VARIABLE(0);
|
||||
auto labels = INPUT_VARIABLE(1);
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
const int classesDim = block.getIArguments()->size() > 0 ? INT_ARG(0) : logits->rankOf() - 1;
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(logits), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: labels and logits arrays must have the same shapes, but got "
|
||||
"%s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(logits).c_str());
|
||||
REQUIRE_TRUE(classesDim < logits->rankOf(), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: class dimension must be smaller than rank of logits, but "
|
||||
"got %i and %i correspondingly !",
|
||||
classesDim, logits->rankOf());
|
||||
|
||||
std::vector<LongType> dimension = {classesDim};
|
||||
|
||||
// Compute softmax log
|
||||
NDArray* maxAlongDim_ptr = logits->reduceAlongDimension(reduce::Max, &dimension, true);
|
||||
NDArray maxAlongDim = *maxAlongDim_ptr;
|
||||
delete maxAlongDim_ptr;
|
||||
|
||||
NDArray* shiftedLogits_ptr = (*logits) - maxAlongDim;
|
||||
NDArray* logExp_ptr = shiftedLogits_ptr->transform(transform::Exp);
|
||||
delete shiftedLogits_ptr;
|
||||
NDArray logExp = *logExp_ptr;
|
||||
delete logExp_ptr;
|
||||
|
||||
NDArray* sumLogExp_ptr = logExp.reduceAlongDimension(reduce::Sum, &dimension, true);
|
||||
NDArray sumLogExp = *sumLogExp_ptr;
|
||||
delete sumLogExp_ptr;
|
||||
|
||||
NDArray* softmaxRatio_ptr = logExp / sumLogExp;
|
||||
NDArray* logSoftMax_ptr = softmaxRatio_ptr->transform(transform::Log);
|
||||
delete softmaxRatio_ptr;
|
||||
NDArray logSoftMax = *logSoftMax_ptr;
|
||||
delete logSoftMax_ptr;
|
||||
|
||||
// Compute cross entropy: -labels * log(softmax)
|
||||
NDArray negLabels = -(*labels); // unary negation returns value
|
||||
NDArray* product_ptr = negLabels * logSoftMax;
|
||||
product_ptr->reduceAlongDimension(reduce::Sum, output, &dimension);
|
||||
delete product_ptr;
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(softmax_cross_entropy_loss_with_logits) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(softmax_cross_entropy_loss_with_logits) {
|
||||
auto logitsShapeInfo = inputShape->at(0);
|
||||
auto labelsShapeInfo = inputShape->at(1);
|
||||
|
||||
const int classesDim = block.getIArguments()->size() > 0 ? INT_ARG(0) : -1;
|
||||
std::vector<LongType> dimensions = {classesDim};
|
||||
|
||||
// labels and logits must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(logitsShapeInfo, labelsShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: labels and logits arrays must have the same shapes, but got "
|
||||
"%s and %s correspondingly!",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
|
||||
auto outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(logitsShapeInfo));
|
||||
auto reducedShapeInfo = ShapeUtils::evalReduceShapeInfo(shape::order(labelsShapeInfo), &dimensions, labelsShapeInfo,
|
||||
outType, false, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(reducedShapeInfo);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(softmax_cross_entropy_loss_with_logits_grad, 2, 2, false, 0, 0) {
|
||||
auto logits = INPUT_VARIABLE(0);
|
||||
auto labels = INPUT_VARIABLE(1);
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dlogits
|
||||
auto dLdl = OUTPUT_VARIABLE(1); // dL/dlabels
|
||||
|
||||
const int classesDim = block.getIArguments()->size() > 0 ? INT_ARG(0) : logits->rankOf()-1;
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labels->isSameShape(logits), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS_GRAD OP: labels and logits arrays must have the same shapes, "
|
||||
"but got %s and %s correspondingly !",
|
||||
ShapeUtils::shapeAsString(labels).c_str(), ShapeUtils::shapeAsString(logits).c_str());
|
||||
REQUIRE_TRUE(classesDim < logits->rankOf(), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS_GRAD OP: class dimension must be smaller than rank of logits, "
|
||||
"but got %i and %i correspondingly !",
|
||||
classesDim, logits->rankOf());
|
||||
|
||||
|
||||
std::vector<LongType> dimension = {classesDim};
|
||||
|
||||
// Compute softmax
|
||||
NDArray* maxAlongDim_ptr = logits->reduceAlongDimension(reduce::Max, &dimension, true);
|
||||
NDArray maxAlongDim = *maxAlongDim_ptr;
|
||||
delete maxAlongDim_ptr;
|
||||
|
||||
NDArray* shiftedLogits_ptr = (*logits) - maxAlongDim;
|
||||
NDArray* softmax_ptr = shiftedLogits_ptr->transform(transform::Exp);
|
||||
delete shiftedLogits_ptr;
|
||||
NDArray softmax = *softmax_ptr;
|
||||
delete softmax_ptr;
|
||||
|
||||
NDArray* sumSoftmax_ptr = softmax.reduceAlongDimension(reduce::Sum, &dimension, true);
|
||||
NDArray sumSoftmax = *sumSoftmax_ptr;
|
||||
delete sumSoftmax_ptr;
|
||||
|
||||
softmax /= sumSoftmax;
|
||||
|
||||
// dEdp = softmax * sum_i(labels_i) - labels
|
||||
// note the eps is to account for exact 0s in the log calculation being nan
|
||||
NDArray* labelsPlusEps_ptr = (*labels) + 1e-6;
|
||||
NDArray labelsPlusEps = *labelsPlusEps_ptr;
|
||||
delete labelsPlusEps_ptr;
|
||||
|
||||
NDArray* labelSum_ptr = labelsPlusEps.reduceAlongDimension(reduce::Sum, &dimension, true);
|
||||
NDArray labelSum = *labelSum_ptr;
|
||||
delete labelSum_ptr;
|
||||
|
||||
NDArray* softmaxTimesLabelSum_ptr = softmax * labelSum;
|
||||
NDArray* dLdpTemp_ptr = (*softmaxTimesLabelSum_ptr) - labelsPlusEps;
|
||||
delete softmaxTimesLabelSum_ptr;
|
||||
dLdp->assign(dLdpTemp_ptr);
|
||||
delete dLdpTemp_ptr;
|
||||
|
||||
// dEdl = -log(softmax)
|
||||
softmax.applyTransform(transform::Log, dLdl);
|
||||
dLdl->applyTransform(transform::Neg, dLdl);
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(softmax_cross_entropy_loss_with_logits_grad) {
|
||||
getOpDescriptor()->setAllowedInputTypes(ANY)->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(softmax_cross_entropy_loss_with_logits_grad) {
|
||||
auto logitsShapeInfo = inputShape->at(0);
|
||||
auto labelsShapeInfo = inputShape->at(1);
|
||||
|
||||
// labels and logits must have the same shapes
|
||||
REQUIRE_TRUE(shape::shapeEquals(logitsShapeInfo, labelsShapeInfo), 0,
|
||||
"SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS_GRAD OP: labels and logits arrays must have the same shapes, "
|
||||
"but got %s and %s correspondingly!",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(logitsShapeInfo));
|
||||
|
||||
auto dLdpShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(logitsShapeInfo),
|
||||
shape::rank(logitsShapeInfo),
|
||||
shape::shapeOf(logitsShapeInfo))->primary();
|
||||
|
||||
auto dLdlShapeInfo = ConstantShapeHelper::getInstance().bufferForShapeInfo(outType, shape::order(labelsShapeInfo),
|
||||
shape::rank(labelsShapeInfo),
|
||||
shape::shapeOf(labelsShapeInfo))->primary();
|
||||
return SHAPELIST(dLdpShapeInfo, dLdlShapeInfo);
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,234 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 29.08.2018
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_sparse_softmax_cross_entropy_loss_with_logits)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/generic/helpers/ScatterHelper.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(sparse_softmax_cross_entropy_loss_with_logits, 2, 1, false, 0, 0) {
|
||||
auto labels = INPUT_VARIABLE(0);
|
||||
auto logits = INPUT_VARIABLE(1);
|
||||
|
||||
auto output = OUTPUT_VARIABLE(0);
|
||||
|
||||
const int labelsRank = labels->rankOf();
|
||||
const int logitsRank = logits->rankOf();
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labelsRank == logitsRank - 1, 0,
|
||||
"SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: input arrays should satisfy relation (labels_rank = "
|
||||
"logits_rank - 1), but got labels_rank = %i and logits_rank = %i instead !",
|
||||
labelsRank, logitsRank);
|
||||
|
||||
auto* labelsShapePtr = labels->getShapeAsVector();
|
||||
std::vector<LongType> labelsShape = *labelsShapePtr;
|
||||
delete labelsShapePtr;
|
||||
auto* logitsShapePtr = logits->getShapeAsVector();
|
||||
std::vector<LongType> logitsShape = *logitsShapePtr;
|
||||
delete logitsShapePtr;
|
||||
logitsShape.pop_back();
|
||||
bool equalSoft = logitsShape == labelsShape;
|
||||
|
||||
REQUIRE_TRUE(
|
||||
equalSoft, 0,
|
||||
"SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: wrong shape of labels array, its shape should be the same as "
|
||||
"logits shape with last dimension excluded, however got labels_shape = %s and logits_shape = %s instead !",
|
||||
ShapeUtils::shapeAsString(labelsShape).c_str(), ShapeUtils::shapeAsString(logitsShape).c_str());
|
||||
|
||||
std::vector<LongType> dimension = {-1};
|
||||
|
||||
// Compute log softmax: -log(exp(logits - max) / sum(exp(logits - max)))
|
||||
NDArray* maxAlongDim_ptr = logits->reduceAlongDimension(reduce::Max, &dimension, true);
|
||||
NDArray maxAlongDim = *maxAlongDim_ptr;
|
||||
delete maxAlongDim_ptr;
|
||||
|
||||
NDArray* shiftedLogits_ptr = (*logits) - maxAlongDim;
|
||||
NDArray* logitsExp_ptr = shiftedLogits_ptr->transform(transform::Exp, nullptr);
|
||||
delete shiftedLogits_ptr;
|
||||
NDArray logitsExp = *logitsExp_ptr;
|
||||
delete logitsExp_ptr;
|
||||
|
||||
NDArray* sumLogitsExp_ptr = logitsExp.reduceAlongDimension(reduce::Sum, &dimension, true);
|
||||
NDArray sumLogitsExp = *sumLogitsExp_ptr;
|
||||
delete sumLogitsExp_ptr;
|
||||
|
||||
NDArray* softmaxRatio_ptr = logitsExp / sumLogitsExp;
|
||||
NDArray* logSoftmax_ptr = softmaxRatio_ptr->transform(transform::Log);
|
||||
delete softmaxRatio_ptr;
|
||||
|
||||
// Apply negation: -log(softmax)
|
||||
NDArray negLogSoftmax = -(*logSoftmax_ptr); // unary negation returns value
|
||||
delete logSoftmax_ptr;
|
||||
|
||||
NDArray logSoftMax = negLogSoftmax;
|
||||
|
||||
helpers::scatterForLoss(block.launchContext(), *labels, logSoftMax, *output, false);
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(sparse_softmax_cross_entropy_loss_with_logits) {
|
||||
getOpDescriptor()
|
||||
->setAllowedInputTypes(0, {ALL_INTS})
|
||||
->setAllowedInputTypes(1, {ALL_FLOATS})
|
||||
->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(sparse_softmax_cross_entropy_loss_with_logits) {
|
||||
auto labelsShapeInfo = inputShape->at(0);
|
||||
auto logitsShapeInfo = inputShape->at(1);
|
||||
|
||||
REQUIRE_TRUE(labelsShapeInfo[0] == logitsShapeInfo[0] - 1, 0,
|
||||
"SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: input arrays should satisfy relation (labels_rank = "
|
||||
"logits_rank - 1), but got labels_rank = %i and logits_rank = %i instead !",
|
||||
labelsShapeInfo[0], logitsShapeInfo[0]);
|
||||
|
||||
bool equalSoft = true;
|
||||
for (int i = 1; i < labelsShapeInfo[0]; ++i)
|
||||
if (labelsShapeInfo[i] != logitsShapeInfo[i]) {
|
||||
equalSoft = false;
|
||||
break;
|
||||
}
|
||||
|
||||
REQUIRE_TRUE(
|
||||
equalSoft, 0,
|
||||
"SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: wrong shape of labels array, its shape should be the same as "
|
||||
"logits shape with last dimension excluded, however got labels_shape = %s and logits_shape = %s instead !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
|
||||
auto outShapeInfo =
|
||||
ShapeBuilders::copyShapeInfoAndType(labelsShapeInfo, logitsShapeInfo, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(CONSTANT(outShapeInfo));
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
CUSTOM_OP_IMPL(sparse_softmax_cross_entropy_loss_with_logits_grad, 2, 1, false, 0, 0) {
|
||||
auto labels = INPUT_VARIABLE(0);
|
||||
auto logits = INPUT_VARIABLE(1);
|
||||
|
||||
auto dLdp = OUTPUT_VARIABLE(0); // dL/dlogits
|
||||
|
||||
const int labelsRank = labels->rankOf();
|
||||
const int logitsRank = logits->rankOf();
|
||||
|
||||
// input validation
|
||||
REQUIRE_TRUE(labelsRank == logitsRank - 1, 0,
|
||||
"SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS_GRAD OP: input arrays should satisfy relation "
|
||||
"(labels_rank = logits_rank - 1), but got labels_rank = %i and logits_rank = %i instead !",
|
||||
labelsRank, logitsRank);
|
||||
|
||||
auto* labelsShapePtr = labels->getShapeAsVector();
|
||||
std::vector<LongType> labelsShape = *labelsShapePtr;
|
||||
delete labelsShapePtr;
|
||||
auto* logitsShapePtr = logits->getShapeAsVector();
|
||||
std::vector<LongType> logitsShape = *logitsShapePtr;
|
||||
delete logitsShapePtr;
|
||||
logitsShape.pop_back();
|
||||
bool equalSoft = logitsShape == labelsShape;
|
||||
|
||||
REQUIRE_TRUE(equalSoft, 0,
|
||||
"SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS_GRAD OP: wrong shape of labels array, its shape should "
|
||||
"be the same as logits shape with last dimension excluded, however got labels_shape = %s and "
|
||||
"logits_shape = %s instead !",
|
||||
ShapeUtils::shapeAsString(labelsShape).c_str(), ShapeUtils::shapeAsString(logitsShape).c_str());
|
||||
|
||||
std::vector<LongType> dimension = {-1};
|
||||
|
||||
// Compute softmax
|
||||
NDArray* maxAlongDim_ptr = logits->reduceAlongDimension(reduce::Max, &dimension, true);
|
||||
NDArray maxAlongDim = *maxAlongDim_ptr;
|
||||
delete maxAlongDim_ptr;
|
||||
|
||||
NDArray* shiftedLogits_ptr = (*logits) - maxAlongDim;
|
||||
NDArray* softmax_ptr = shiftedLogits_ptr->transform(transform::Exp);
|
||||
delete shiftedLogits_ptr;
|
||||
NDArray softmax = *softmax_ptr;
|
||||
delete softmax_ptr;
|
||||
|
||||
NDArray* sumSoftmax_ptr = softmax.reduceAlongDimension(reduce::Sum, &dimension, true);
|
||||
NDArray sumSoftmax = *sumSoftmax_ptr;
|
||||
delete sumSoftmax_ptr;
|
||||
|
||||
softmax /= sumSoftmax;
|
||||
|
||||
// dEdp = softmax - 1 (or 0)
|
||||
dLdp->assign(&softmax);
|
||||
|
||||
// subtract unities at appropriate indexes of dLdp array
|
||||
helpers::scatterForLoss(block.launchContext(), *labels, *dLdp,
|
||||
*labels /*actually third array is unnecessary for gradient calculation*/, true);
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_TYPES(sparse_softmax_cross_entropy_loss_with_logits_grad) {
|
||||
getOpDescriptor()
|
||||
->setAllowedInputTypes(0, {ALL_INTS})
|
||||
->setAllowedInputTypes(1, {ALL_FLOATS})
|
||||
->setAllowedOutputTypes({ALL_FLOATS});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
DECLARE_SHAPE_FN(sparse_softmax_cross_entropy_loss_with_logits_grad) {
|
||||
auto labelsShapeInfo = inputShape->at(0);
|
||||
auto logitsShapeInfo = inputShape->at(1);
|
||||
|
||||
REQUIRE_TRUE(labelsShapeInfo[0] == logitsShapeInfo[0] - 1, 0,
|
||||
"SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS_GRAD OP: input arrays should satisfy relation "
|
||||
"(labels_rank = logits_rank - 1), but got labels_rank = %i and logits_rank = %i instead !",
|
||||
labelsShapeInfo[0], logitsShapeInfo[0]);
|
||||
|
||||
bool equalSoft = true;
|
||||
for (int i = 1; i < labelsShapeInfo[0]; ++i)
|
||||
if (labelsShapeInfo[i] != logitsShapeInfo[i]) {
|
||||
equalSoft = false;
|
||||
break;
|
||||
}
|
||||
|
||||
REQUIRE_TRUE(equalSoft, 0,
|
||||
"SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS_GRAD OP: wrong shape of labels array, its shape should "
|
||||
"be the same as logits shape with last dimension excluded, however got labels_shape = %s and "
|
||||
"logits_shape = %s instead !",
|
||||
ShapeUtils::shapeAsString(labelsShapeInfo).c_str(), ShapeUtils::shapeAsString(logitsShapeInfo).c_str());
|
||||
|
||||
DataType outType = DataTypeUtils::pickFloatingType(ArrayOptions::dataType(logitsShapeInfo));
|
||||
|
||||
LongType *dLdpShapeInfo =
|
||||
ShapeBuilders::copyShapeInfoAndType(logitsShapeInfo, outType, false, block.getWorkspace());
|
||||
|
||||
return SHAPELIST(CONSTANT(dLdpShapeInfo));
|
||||
}
|
||||
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user