/* ****************************************************************************** * * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at * https://www.apache.org/licenses/LICENSE-2.0. * * See the NOTICE file distributed with this work for additional * information regarding copyright ownership. * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. * * SPDX-License-Identifier: Apache-2.0 ******************************************************************************/ // // @author Yurii Shyrma (iuriish@yahoo.com), created on 05.06.2018 // #ifndef LIBND4J_MMULHELPER_CPP #define LIBND4J_MMULHELPER_CPP #include "../MmulHelper.h" #include #include #include #include #include #include #include #include #include #include "ops/declarable/headers/blas.h" namespace sd { ////////////////////////////////////////////////////////////////////////// NDArray* MmulHelper::tensorDot(NDArray* A, NDArray* B, const std::initializer_list& axesA, const std::initializer_list& axesB) { std::vector aA(axesA); std::vector aB(axesB); return tensorDot(A, B, aA, aB); } ////////////////////////////////////////////////////////////////////////// NDArray* MmulHelper::tensorDot(NDArray* A, NDArray* B, const std::vector& axesA, const std::vector& axesB) { std::vector permutAt, permutBt; std::vector shapeAt, shapeBt; auto outShape = ShapeUtils::evalShapeForTensorDot(A, B, axesA, axesB, permutAt, permutBt, shapeAt, shapeBt); // check whether permutation is necessary NDArray* aP = permutAt.empty() ? A : A->permute(permutAt, false, false); NDArray* bP = permutBt.empty() ? B : B->permute(permutBt, false, false); // check whether reshape is necessary NDArray* aPR = aP->isSameShape(shapeAt) ? aP : aP->reshape(aP->ordering(), shapeAt); NDArray* bPR = bP->isSameShape(shapeAt) ? bP : bP->reshape(bP->ordering(), shapeBt); NDArray* c = mmul(aPR, bPR, nullptr, 1.0, 0.0); c->reshapei(outShape); // Delete reshaped arrays first if(aPR != A && aPR != aP) { delete aPR; } if(bPR != B && bPR != bP) { delete bPR; } // Then delete permuted arrays if(aP != A) { delete aP; } if(bP != B) { delete bP; } return c; } void MmulHelper::computeNewShapesAndAxes( NDArray& as_, const std::vector& axes_a, NDArray& bs, const std::vector& axes_b, std::vector& newshape_a, std::vector& newaxes_a, std::vector& newshape_b, std::vector& newaxes_b ) { std::vector *as_shape = as_.getShapeAsVector(); std::vector *bs_shape = bs.getShapeAsVector(); std::vector notin_a; for(size_t k = 0; k < as_shape->size(); ++k) { if(std::find(axes_a.begin(), axes_a.end(), k) == axes_a.end()) notin_a.push_back(k); } newaxes_a.clear(); std::copy(notin_a.begin(), notin_a.end(), std::back_inserter(newaxes_a)); std::copy(axes_a.begin(), axes_a.end(), std::back_inserter(newaxes_a)); LongType N2_a = std::accumulate(axes_a.begin(), axes_a.end(), 1L, [&](LongType product, LongType i){ return product * (*as_shape)[i]; }); newshape_a.clear(); newshape_a.push_back(std::accumulate(notin_a.begin(), notin_a.end(), 1L, [&](LongType product, LongType i){ return product * (*as_shape)[i]; })); newshape_a.push_back(N2_a); std::vector notin_b; for(size_t k = 0; k < bs_shape->size(); ++k) { if(std::find(axes_b.begin(), axes_b.end(), k) == axes_b.end()) notin_b.push_back(k); } newaxes_b.clear(); std::copy(axes_b.begin(), axes_b.end(), std::back_inserter(newaxes_b)); std::copy(notin_b.begin(), notin_b.end(), std::back_inserter(newaxes_b)); LongType N2_b = std::accumulate(axes_b.begin(), axes_b.end(), 1L, [&](LongType product, LongType i){ return product * (*bs_shape)[i]; }); newshape_b.clear(); newshape_b.push_back(N2_b); newshape_b.push_back(std::accumulate(notin_b.begin(), notin_b.end(), 1L, [&](LongType product, LongType i){ return product * (*bs_shape)[i]; })); } ////////////////////////////////////////////////////////////////////////// void MmulHelper::tensorDot2(NDArray* a, NDArray* b, NDArray* c, const std::vector& axes_a, const std::vector& axes_b, std::vector& permutAt, std::vector& permuteBt, std::vector& permuteCt, NDArray* realFinalResult) { // check whether permutation is required NDArray* cP =permuteCt.empty() ? c : c->permute(permuteCt, false, false); std::vector shapeAt, shapeBt; std::vector permutAtDummy, permuteBtDummy; std::vector newshape_a, newaxes_a, newshape_b, newaxes_b; computeNewShapesAndAxes(*a, axes_a, *b, axes_b, newshape_a, newaxes_a, newshape_b, newaxes_b); NDArray* aP = permutAt.empty() ? a : a->permute(permutAt, false, false); NDArray* bP = permuteBt.empty() ? b :b->permute(permuteBt, false, false); NDArray* aPermuted = aP->permute(newaxes_a, false, false); NDArray* aPR = aPermuted->reshape('c', newshape_a, true); NDArray* bPermuted = bP->permute(newaxes_b, false, false); NDArray* bPR = bPermuted->reshape('c', newshape_b, true); std::vector requiredCshape = {aPR->sizeAt(0), bPR->sizeAt(1)}; NDArray *cP2 = cP->reshape('f', requiredCshape, false); NDArray* cPR = cP2; NDArray * ret = mmul(aPR, bPR, cPR, 1.0, 0.0); if (cPR->buffer() != cP->buffer() || cPR->specialBuffer() != cP->specialBuffer()) { // this means both permute and reshape have been performed on c, cP if(c->buffer() == cP->buffer()) { auto copyFromBuff = cP->dataBuffer(); cP->dataBuffer()->copyBufferFrom(*copyFromBuff); } else { auto copyFromBuff = cP->dataBuffer(); c->dataBuffer()->copyBufferFrom(*copyFromBuff); } } if(realFinalResult != c) { realFinalResult->dataBuffer()->copyBufferFrom(*c->dataBuffer()); } if(cP != c) { delete cP; } if(cPR != c) { delete cPR; } if(aP != a && !aP->isView()) { delete aP; } if(bP != b && !bP->isView()) { delete bP; } // Delete in reverse order of creation to avoid use-after-free if(bPR != b && bPR != bP && bPR != bPermuted && !bPR->isView()) { delete bPR; } if(bPermuted != b && bPermuted != bP && !bPermuted->isView()) { delete bPermuted; } if(aPR != a && aPR != aP && aPR != aPermuted && !aPR->isView()) { delete aPR; } if(aPermuted != a && aPermuted != aP && !aPermuted->isView()) { delete aPermuted; } } void MmulHelper::tensorDot(NDArray* a, NDArray* b, NDArray* c, std::vector& axes_a, std::vector& axes_b, std::vector& permutForC) { std::vector permutAt, permutBt; std::vector shapeAt, shapeBt; ShapeUtils::evalShapeForTensorDot(a, b, axes_a, axes_b, permutAt, permutBt, shapeAt, shapeBt); // check whether permutation is required NDArray* cP = permutForC.empty() ? c :c->permute(permutForC, false, false); // check whether permutation is necessary NDArray* aP = permutAt.empty() ? a :a->permute(permutAt, false, false); NDArray* bP = permutBt.empty() ? b : b->permute(permutBt, false, false); // check whether reshape is necessary NDArray* aPR = aP->isSameShape(shapeAt) ? aP : aP->reshape(aP->ordering(), shapeAt); NDArray* bPR = bP->isSameShape(shapeAt) ? bP : bP->reshape(bP->ordering(), shapeBt); std::vector requiredCshape = {aPR->sizeAt(0), bPR->sizeAt(1)}; NDArray* cPR = cP->isSameShape(requiredCshape) ? cP : cP->reshape(cP->ordering(), requiredCshape, false); NDArray *ret = mmul(aPR, bPR, cPR, 1.0, 0.0); if (c != ret) { // this means both permute and reshape have been performed on c, cP // always points on c->buffer() NDArray *assign2 = ret->reshape(c->ordering(),requiredCshape); c->assign(assign2); delete assign2; } if(c != cP && !cP->isView()) { delete cP; } if(aP != a && !aP->isView()) { delete aP; } if(bP != b && !bP->isView()) { delete bP; } if(aPR != a && aPR != aP && !aPR->isView()) { delete aPR; } if(bPR != b && bPR != bP && !bPR->isView()) { delete bPR; } if(cPR != c && cPR != cP && !cPR->isView()) { delete cPR; } } #ifndef __JAVACPP_HACK__ ////////////////////////////////////////////////////////////////////////// void MmulHelper::tensorDot(NDArray* a, NDArray* b, NDArray* c, std::vector>& modifA, std::vector>& modifB, std::vector>& modifC) { NDArray *aPR(const_cast(a)), *bPR(const_cast(b)); std::string whatToDoWithA, whatToDoWithB, whatToDoWithC; // "" - nothing; "p" - permutation; "r" - reshaping; "pr" - permutation+reshaping; "rp" - // reshaping/permutation, and so on; if another string is produced - throw exception for (const auto& arr : modifA) whatToDoWithA = (std::find(arr.begin(), arr.end(), 0) != arr.end()) ? whatToDoWithA + "p" : whatToDoWithA + "r"; // when 0 is present in arr then it is permutation array, otherwise - it is reshaping array for (const auto& arr : modifB) whatToDoWithB = (std::find(arr.begin(), arr.end(), 0) != arr.end()) ? whatToDoWithB + "p" : whatToDoWithB + "r"; for (const auto& arr : modifC) whatToDoWithC = (std::find(arr.begin(), arr.end(), 0) != arr.end()) ? whatToDoWithC + "p" : whatToDoWithC + "r"; // first step for a array if (!whatToDoWithA.empty()) aPR = (whatToDoWithA[0] == 'p') ? a->permute(modifA[0], false, false) :a->reshape(a->ordering(), modifA[0]); // first step for b array if (!whatToDoWithB.empty()) bPR = (whatToDoWithB[0] == 'p') ? b->permute(modifB[0], false, false) : b->reshape(b->ordering(), modifB[0]); // rest steps for a array for (size_t i = 1; i < whatToDoWithA.size(); ++i) if (whatToDoWithA[i] == 'p') aPR->permutei(modifA[i], false, false); else aPR->reshapei(modifA[i]); // rest steps for b array for (size_t i = 1; i < whatToDoWithB.size(); ++i) if (whatToDoWithB[i] == 'p') bPR->permutei(modifB[i], false, false); else bPR->reshapei(modifB[i]); // now work with c array std::vector cArrs = {c}; if (!whatToDoWithC.empty()) { cArrs = std::vector(whatToDoWithC.size() + 1, c); for (size_t i = 0; i < cArrs.size() - 1; ++i) cArrs[i + 1] = (whatToDoWithC[i] == 'p') ? cArrs[i]->permute(modifC[i], false, false) : cArrs[i]->reshape( c->ordering(), modifC[i], false); // since we ignore first element in cArrs (that is cArrs[0]) then it is always equal to c } mmul(aPR, bPR, cArrs[cArrs.size() - 1], 1.0, 0.0); // check whether new buffer allocation was happened for c array if (!whatToDoWithC.empty()) { for (int i = cArrs.size() - 1; i > 0; --i) { if (cArrs[i]->buffer() != cArrs[i - 1]->buffer() || cArrs[i]->specialBuffer() != cArrs[i - 1]->specialBuffer()) cArrs[i - 1]->assign(cArrs[i]); delete cArrs[i]; } } if (aPR != a) delete aPR; if (bPR != b) delete bPR; } ////////////////////////////////////////////////////////////////////////// NDArray* MmulHelper::tensorDot(NDArray* a, NDArray* b, std::vector>& modifA, std::vector>& modifB) { NDArray *aPR(const_cast(a)), *bPR(const_cast(b)); std::string whatToDoWithA, whatToDoWithB; // "" - nothing; "p" - permutation only; "r" - reshaping only; "pr" - permutation+reshaping; "rp" // - reshaping/permutation; another string - throw exception for (const auto& arr : modifA) whatToDoWithA = (std::find(arr.begin(), arr.end(), 0) != arr.end()) ? whatToDoWithA + "p" : whatToDoWithA + "r"; // when 0 is present in arr then it is permutation array, otherwise - it is reshaping array for (const auto& arr : modifB) whatToDoWithB = (std::find(arr.begin(), arr.end(), 0) != arr.end()) ? whatToDoWithB + "p" : whatToDoWithB + "r"; // first step for a array if (!whatToDoWithA.empty()) aPR = (whatToDoWithA[0] == 'p') ?a->permute(modifA[0], false, false) : a->reshape(a->ordering(), modifA[0]); // first step for b array if (!whatToDoWithB.empty()) bPR = (whatToDoWithB[0] == 'p') ? b->permute(modifB[0], false, false) : b->reshape(b->ordering(), modifB[0]); // rest steps for a array for (size_t i = 1; i < whatToDoWithA.size(); ++i) if (whatToDoWithA[i] == 'p') aPR->permutei(modifA[i], false, false); else aPR->reshapei(modifA[i]); // rest steps for b array for (size_t i = 1; i < whatToDoWithB.size(); ++i) if (whatToDoWithB[i] == 'p') bPR->permutei(modifB[i], false, false); else bPR->reshapei(modifB[i]); NDArray* result = mmul(aPR, bPR, nullptr, 1.0, 0.0); return result; } #endif ////////////////////////////////////////////////////////////////////////// NDArray* MmulHelper::mmul(NDArray* A, NDArray* B, NDArray* C, const double alpha, const double beta, const char outOrder) { LongType lenDim; const LongType aRank = A->rankOf(); const LongType bRank = B->rankOf(); const bool isAVector = shape::isCommonVector(A->shapeInfo(), lenDim); const bool isBVector = shape::isCommonVector(B->shapeInfo(), lenDim); // dot product of 2 vectors if (A->lengthOf() == B->lengthOf() && isAVector && isBVector && (aRank != 2 || (aRank == 2 && (A->isSameShape(B) || (bRank == 1 && A->sizeAt(1) == 1))))) { // (1x1x1 * 1x1) or (1x4 * 1*4) or (4x1 * 4x1) or (4x1 * 4) return dot(A, B, C, alpha, beta); } // matrix x matrix if (aRank == 2 && bRank == 2) { return mmulMxM(A, B, C, alpha, beta, outOrder); } // matrix x vector if (aRank == 2 && isBVector) { return mmulMxV(A, B, C, alpha, beta, outOrder); } // vector x matrix, A{M} x B{M,N} = C{N} -> reduce to matrix x matrix A2{1,M} x B{M,N} = C2{1,N}, since there is no // corresponding blas operation sgevm if (isAVector && bRank == 2) { std::vector aShape = {1, A->lengthOf()}; std::vector cShape = {1, C->lengthOf()}; NDArray* A2 = A->reshape(A->ordering(), aShape); // A{M} -> A2{1,M} NDArray* C2 = C ? C->reshape(C->ordering(), cShape, false) : nullptr; // C{N} -> C2{1,N} auto result = mmulMxM(A2, B, C2, alpha, beta, outOrder); // result{1,N} // Cleanup reshaped arrays if (A2 != A) delete A2; if (C2 != nullptr && C2 != C) delete C2; if (!C) { result->reshapei({result->lengthOf()}); // result{1,N} -> result{N} return result; } return C; } // batched matrix multiplication return mmulNxN(A, B, C, alpha, beta, outOrder); } bool MmulHelper::resolveTranspose(sd::NDArray& a, sd::NDArray& b, bool& transA, bool& transB) { int rowsA = a.sizeAt(-2); int colsA = a.sizeAt(-1); int rowsB = b.sizeAt(-2); int colsB = b.sizeAt(-1); transA = false; transB = false; if (colsA == rowsB) { // No transpose needed return true; } else if (rowsA == rowsB) { // Transpose A transA = true; return true; } else if (colsA == colsB) { // Transpose B transB = true; return true; } else { // Dimensions do not match for matrix multiply return false; } } ////////////////////////////////////////////////////////////////////////// void MmulHelper::matmul(NDArray* x, NDArray* y, NDArray* z, const bool transX, const bool transY, double alpha, double beta, NDArray* realFinalResult) { int xRank = x->rankOf(); int yRank = y->rankOf(); auto outShape = ShapeUtils::evalShapeForMatmul(x->shapeInfo(), y->shapeInfo(), transX, transY); if (!z->isSameShape(outShape)) { std::string errorMessage; errorMessage = "NDArrayFactory::matmul static method: input shape of output array is wrong, actual is"; errorMessage += ShapeUtils::shapeAsString(z).c_str(); errorMessage += " and expected is "; errorMessage += ShapeUtils::shapeAsString(outShape).c_str(); errorMessage += " ! \n"; THROW_EXCEPTION(errorMessage.c_str()); } if (z->isEmpty()) return; NDArray *xT = const_cast(x); NDArray *yT = const_cast(y); NDArray *zT = z; // Handle transpose via permute + dup for contiguous data // permute creates a view with swapped strides, dup() makes a contiguous copy if ((transX && xRank > 1) || (transY && yRank > 1)) { const int rank = xRank >= yRank ? xRank : yRank; std::vector permut(rank); for (int i = 0; i < rank - 2; ++i) permut[i] = i; permut[rank - 2] = rank - 1; permut[rank - 1] = rank - 2; if (transX) { NDArray *permutedView = x->permute(permut, false, false); // Create view (non-contiguous) xT = permutedView->dup(); // Make contiguous copy with proper data layout delete permutedView; } if (transY) { NDArray *permutedView = y->permute(permut, false, false); // Create view (non-contiguous) yT = permutedView->dup(); // Make contiguous copy with proper data layout delete permutedView; } } if (xRank <= 2 && yRank <= 2) { // dot (1Dx1D), vector-matrix (1Dx2D), matrix-vector (2Dx1D), matrix-matrix (2Dx2D) product cases NDArray* xReshaped = nullptr; NDArray* zReshaped = nullptr; if (xRank == 1 && yRank == 2) { // reduce vector-matrix to matrix-matrix case std::vector xShape = {1, xT->lengthOf()}; std::vector zShape = {1, z->lengthOf()}; // Remember if we need to delete the permuted versions NDArray* xPermuted = (xT != x) ? xT : nullptr; NDArray* zPermuted = (zT != z) ? zT : nullptr; xReshaped = xT->reshape(xT->ordering(), xShape, false); xT = xReshaped; zReshaped = z->reshape(z->ordering(), zShape, false); zT = zReshaped; // Clean up permuted versions if they exist if(xPermuted != nullptr && !xPermuted->isView()) { delete xPermuted; } if(zPermuted != nullptr && !zPermuted->isView()) { delete zPermuted; } } mmul(xT, yT, zT, alpha, beta); // Copy back result and clean up reshaped output if(zT != z) { z->dataBuffer()->copyBufferFrom(*zT->dataBuffer(), zT->lengthOf() * zT->sizeOfT()); delete zT; zT = z; // Reset to original to prevent double-free at end of function } // Clean up reshaped input if(xReshaped != nullptr && xReshaped != x) { delete xReshaped; xT = x; // Reset to original to prevent double-free at end of function } } else { // Batched matmul: loop over batch dimensions and call 2D gemm for each slice // This is more reliable than mmulNxN which has bugs in batch index calculation // For 3D arrays [batch, M, K] x [batch, K, N] = [batch, M, N] // We iterate over batch dimension and call 2D mmul for each slice const int xRankT = xT->rankOf(); const int yRankT = yT->rankOf(); const int zRankT = zT->rankOf(); if (xRankT == 3 && yRankT == 3 && zRankT == 3) { // Simple case: all 3D with matching batch dimension const LongType batchSize = xT->sizeAt(0); const LongType M = xT->sizeAt(1); const LongType K = xT->sizeAt(2); const LongType N = yT->sizeAt(2); for (LongType b = 0; b < batchSize; ++b) { // Get 2D slices for this batch using subarray auto xSlice = (*xT)(b, {0}); // [M, K] auto ySlice = (*yT)(b, {0}); // [K, N] auto zSlice = (*zT)(b, {0}); // [M, N] // Call 2D matmul - no transpose flags since we already handled them via permute+dup mmul(xSlice, ySlice, zSlice, alpha, beta); } } else { // Fall back to mmulNxN for other cases (4D+, mixed ranks, etc.) mmulNxN(xT, yT, zT, alpha, beta, z->ordering()); } } // Clean up permuted arrays (works for both cases) if (xT != x && xT != nullptr) delete xT; if (yT != y && yT != nullptr) delete yT; if(realFinalResult != nullptr && realFinalResult != z) { realFinalResult->dataBuffer()->copyBufferFrom(*z->dataBuffer()); } } } // namespace sd #endif