/* ****************************************************************************** * * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at * https://www.apache.org/licenses/LICENSE-2.0. * * See the NOTICE file distributed with this work for additional * information regarding copyright ownership. * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. * * SPDX-License-Identifier: Apache-2.0 ******************************************************************************/ // // @author raver119@gmail.com // #include #include #include #include #include #include namespace sd { namespace sparse { template void SparseUtils::printIndex(sd::LongType *indices, int rank, int x) { printf(" ["); for (int e = 0; e < rank; e++) { if (e > 0) printf(", "); printf("%lld", (long long)indices[x * rank + e]); } printf("] "); } template bool SparseUtils::ltIndices(sd::LongType *indices, int rank, sd::LongType x, sd::LongType y) { for (int e = 0; e < rank; e++) { sd::LongType idxX = indices[x * rank + e]; sd::LongType idxY = indices[y * rank + e]; // we're comparing indices one by one, starting from outer dimension if (idxX < idxY) { return true; } else if (idxX == idxY) { // do nothing, continue to next dimension } else return false; } return false; } template bool SparseUtils::gtIndices(sd::LongType *indices, int rank, sd::LongType x, sd::LongType y) { for (int e = 0; e < rank; e++) { // we're comparing indices one by one, starting from outer dimension sd::LongType idxX = indices[x * rank + e]; sd::LongType idxY = indices[y * rank + e]; if (idxX > idxY) { return true; } else if (idxX == idxY) { // do nothing, continue to next dimension } else return false; } return false; } template void SparseUtils::swapEverything(sd::LongType *indices, T *array, int rank, sd::LongType x, sd::LongType y) { // swap indices for (int e = 0; e < rank; e++) { sd::LongType tmp = indices[x * rank + e]; indices[x * rank + e] = indices[y * rank + e]; indices[y * rank + e] = tmp; } // swap values T tmp = array[x]; array[x] = array[y]; array[y] = tmp; } template sd::LongType SparseUtils::coo_quickSort_findPivot(sd::LongType *indices, T *array, sd::LongType left, sd::LongType right, int rank) { sd::LongType mid = (left + right) / 2; // ensure left < mid if (ltIndices(indices, rank, mid, left)) { // ensure lo < mid swapEverything(indices, array, rank, mid, left); } // ensure left < right if (ltIndices(indices, rank, right, left)) { swapEverything(indices, array, rank, right, left); } // ensure mid < right if (ltIndices(indices, rank, right, mid)) { swapEverything(indices, array, rank, right, mid); } // mid is the median of the 3, and is the optimal pivot point return mid; } template void SparseUtils::coo_quickSort_parallel_internal(sd::LongType *indices, T *array, sd::LongType left, sd::LongType right, int cutoff, int rank) { sd::LongType span = right - left; // elements to be partitioned - 1 if (span == 1) { // only 2 elements to partition. swap if needed and return directly without further sorting. if (ltIndices(indices, rank, right, left)) { swapEverything(indices, array, rank, left, right); } return; } // find optimal pivot and sort left < right < right sd::LongType pvt = coo_quickSort_findPivot(indices, array, left, right, rank); if (span == 2) { // only 3 elements to partition. findPivot has already sorted them. no further sorting is needed. return; } // index that is greater than pivot - leftmost element is already partitioned because of findPivot. sd::LongType i = left + 1; // index that is smaller than pivot - rightmost element is already partitioned because of findPivot. sd::LongType j = right - 1; { // flag that indicates that pivot index lies between i and j and *could* be swapped. bool checkPivot = true; /* PARTITION PART */ while (i <= j) { while (ltIndices(indices, rank, i, pvt)) i++; while (gtIndices(indices, rank, j, pvt)) j--; if (i <= j) { if (i != j) { // swap can be fairly expensive. don't swap i -> i swapEverything(indices, array, rank, i, j); } // only check pivot if it hasn't already been swapped. if (checkPivot) { // check if we moved the pivot, if so, change pivot index accordingly if (pvt == j) { pvt = i; checkPivot = false; } else if (pvt == i) { pvt = j; checkPivot = false; } } i++; j--; } } } if ((span < cutoff)) { if (left < j) { coo_quickSort_parallel_internal(indices, array, left, j, cutoff, rank); } if (i < right) { coo_quickSort_parallel_internal(indices, array, i, right, cutoff, rank); } } else { PRAGMA_OMP_TASK { coo_quickSort_parallel_internal(indices, array, left, j, cutoff, rank); } PRAGMA_OMP_TASK { coo_quickSort_parallel_internal(indices, array, i, right, cutoff, rank); } } } template void SparseUtils::coo_quickSort_parallel(sd::LongType *indices, T *array, sd::LongType lenArray, int numThreads, int rank) { int cutoff = 1000; PRAGMA_OMP_PARALLEL_THREADS(numThreads) { PRAGMA_OMP_SINGLE_ARGS(nowait) { coo_quickSort_parallel_internal(indices, array, 0, lenArray - 1, cutoff, rank); } } } template void SparseUtils::sortCooIndicesGeneric(sd::LongType *indices, void *vx, sd::LongType length, int rank) { auto values = reinterpret_cast(vx); #ifdef _OPENMP coo_quickSort_parallel(indices, values, length, omp_get_max_threads(), rank); #else coo_quickSort_parallel(indices, values, length, 1, rank); #endif } BUILD_SINGLE_TEMPLATE( class SparseUtils, , SD_COMMON_TYPES); void IndexUtils::ravelMultiIndex(sd::LongType *indices, sd::LongType *flatIndices, sd::LongType length, sd::LongType *shapeInfo, int mode) { sd::LongType *shape = shape::shapeOf(shapeInfo); sd::LongType *stride = shape::stride(shapeInfo); sd::LongType rank = shape::rank(shapeInfo); int errorCount = 0; PRAGMA_OMP_PARALLEL_FOR for (sd::LongType i = 0; i < length; ++i) { sd::LongType raveledIndex = 0; for (sd::LongType j = 0; j < rank; ++j) { sd::LongType idx = indices[i * rank + j]; if (idx >= shape[j]) { // index does not fit into shape at j dimension. if (mode == ND4J_CLIPMODE_CLIP) { // set idx to largest possible value (clip to shape) idx = shape[j] - 1; } else if (mode == ND4J_CLIPMODE_WRAP) { idx %= shape[j]; } else { // mode is ND4J_CLIPMODE_THROW or is unknown. either way. throw an error later. // cannot throw here because of parallel region sd_printf( "sparse::IndexUtils::ravelMultiIndex Cannot ravel index at element %d, does not fit into specified " "shape.\n", i); ++errorCount; } } raveledIndex += idx * stride[j]; } flatIndices[i] = raveledIndex; } if (errorCount > 0) { // throw error if one ocurred in loop THROW_EXCEPTION("sparse::IndexUtils::ravelMultiIndex Cannot ravel index"); } } void IndexUtils::unravelIndex(sd::LongType *indices, sd::LongType *flatIndices, sd::LongType length, sd::LongType *shapeInfo) { sd::LongType *shape = shape::shapeOf(shapeInfo); sd::LongType *stride = shape::stride(shapeInfo); sd::LongType rank = shape::rank(shapeInfo); int errorCount = 0; // unravelOrder ensures that the dimensions with largest stride are unraveled first. // create vector with elements 0..rank int *unravelOrder = shape::range(0, rank); // sort order according to stride length. std::sort(unravelOrder, unravelOrder + rank, [&](int i1, int i2) { return stride[i1] > stride[i2]; }); // calculate the largest raveled index that will fit into passed shape sd::LongType maxRaveledIndex = shape[unravelOrder[0]] * stride[unravelOrder[0]] - 1; PRAGMA_OMP_PARALLEL_FOR for (sd::LongType i = 0; i < length; ++i) { sd::LongType raveledIndex = flatIndices[i]; if (raveledIndex > maxRaveledIndex) { // cannot throw here because of parallel region sd_printf( "sparse::IndexUtils::unravelIndex Cannot unravel index at element %d. raveled index of %d does not fit into " "specified shape.\n", i, raveledIndex); ++errorCount; } for (int *it = unravelOrder; it != unravelOrder + rank; it++) { int j = *it; // how many strides of this size? indices[i * rank + j] = raveledIndex / stride[j]; // remainder for subsequent smaller strides. raveledIndex %= stride[j]; } } if (errorCount > 0) { // throw error if one occurred in loop sd_printf("Largest raveled index is: %d, ", maxRaveledIndex) std::vector v(shape, shape + rank); sd_printv("Shape: ", v); THROW_EXCEPTION("sparse::IndexUtils::unravelIndex Cannot unravel index"); } delete[] unravelOrder; } } // namespace sparse } // namespace sd