/* ****************************************************************************** * * * This program and the accompanying materials are made available under the * terms of the Apache License, Version 2.0 which is available at * https://www.apache.org/licenses/LICENSE-2.0. * * See the NOTICE file distributed with this work for additional * information regarding copyright ownership. * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. * * SPDX-License-Identifier: Apache-2.0 ******************************************************************************/ // // @author raver119@gmail.com // #ifndef LIBND4J_HELPER_GENERATOR_H #define LIBND4J_HELPER_GENERATOR_H #include #include #ifdef _MSC_VER // include for uint64_t on MSVC #include #elif ANDROID #include #ifndef UINT64_C #if defined(__LP64__) #define UINT64_C(c) c##UL #else #define UINT64_C(c) c##ULL #endif // LP64 #endif // UINT64 #endif // MSVC/ANDROID #ifdef __GNUC__ #include #endif namespace sd { namespace random { #ifdef __CUDACC__ class SD_LIB_EXPORT CudaManaged { private: protected: void *devHolder; public: void *operator new(size_t len) { void *ptr; cudaHostAlloc(&ptr, len, cudaHostAllocDefault); return ptr; } void operator delete(void *ptr) { cudaFreeHost(ptr); } }; class SD_LIB_EXPORT RandomBuffer : public CudaManaged { #else class SD_LIB_EXPORT RandomBuffer { #endif private: sd::LongType size; uint64_t *buffer; uint64_t *devBuffer; sd::LongType offset; sd::LongType seed; sd::LongType generation; sd::LongType currentPosition; sd::LongType amplifier; unsigned int synchronizer; #ifdef __CUDACC__ curandGenerator_t gen; #endif public: /** * This method allocates buffer of size * sizeof(sd::LongType) * * @param size * @return */ #ifdef __CUDACC__ SD_HOST RandomBuffer(sd::LongType seed, sd::LongType size, uint64_t *hostBuffer, uint64_t *devBuffer) { this->buffer = hostBuffer; this->seed = seed; this->size = size; this->generation = 1; this->currentPosition = 0; this->offset = 0; this->amplifier = seed; this->synchronizer = 0; this->devBuffer = devBuffer; cudaMalloc(&devHolder, sizeof(sd::random::RandomBuffer)); } SD_HOST sd::Pointer getDevicePointer() { return reinterpret_cast(devHolder); } SD_HOST ~RandomBuffer() { cudaFree(devHolder); } SD_HOST void propagateToDevice(sd::random::RandomBuffer *buffer, cudaStream_t stream) { cudaMemcpyAsync(devHolder, buffer, sizeof(sd::random::RandomBuffer), cudaMemcpyHostToDevice, stream); } SD_HOST_DEVICE #endif RandomBuffer(sd::LongType seed, sd::LongType size, uint64_t *buffer) { this->buffer = buffer; this->seed = seed; this->size = size; this->generation = 1; this->currentPosition = 0; this->offset = 0; this->amplifier = seed; this->synchronizer = 0; this->devBuffer = buffer; } SD_INLINE SD_HOST_DEVICE uint64_t *getBuffer() { return this->buffer; } SD_INLINE SD_HOST_DEVICE uint64_t *getDeviceBuffer() { return this->devBuffer; } #ifdef __CUDACC__ SD_HOST_DEVICE curandGenerator_t *getGeneratorPointer() { return &gen; } SD_HOST_DEVICE curandGenerator_t getGenerator() { return gen; } SD_HOST void setBuffer(uint64_t *ptr) { this->buffer = ptr; } #endif SD_INLINE SD_HOST_DEVICE sd::LongType getSize() { return this->size; } SD_INLINE SD_HOST_DEVICE sd::LongType getSeed() { return this->seed; } void SD_HOST_DEVICE setSeed(sd::LongType seedIn) { this->seed = seedIn; this->amplifier = seedIn; this->generation = 1; } sd::LongType SD_HOST_DEVICE getAllocatedSize() { return this->size * sizeof(double); } SD_INLINE SD_HOST_DEVICE sd::LongType getOffset() { return this->currentPosition; } void SD_HOST_DEVICE setOffset(sd::LongType offsetIn) { this->currentPosition = offsetIn; } void SD_HOST_DEVICE reSeed(sd::LongType amplifierIn) { this->amplifier = amplifierIn; } SD_INLINE SD_DEVICE uint64_t getElement(sd::LongType positionIn) { sd::LongType actualPosition = this->getOffset() + positionIn; sd::LongType tempGen = generation; if (actualPosition >= this->size) { tempGen += actualPosition / this->size; actualPosition = actualPosition % this->size; } #ifdef __CUDACC__ // __syncthreads(); auto ret = static_cast(devBuffer[actualPosition]); #else auto ret = static_cast(buffer[actualPosition]); #endif if (tempGen != generation) ret = safeShift(ret, tempGen); if (generation > 1) ret = safeShift(ret, generation); if (amplifier != seed) ret = safeShift(ret, amplifier); #ifdef __CUDACC__ // __syncthreads(); #endif if (amplifier != seed || generation > 1 || tempGen != generation) ret = next64(seedConv(static_cast(ret))); return ret; } uint64_t SD_HOST_DEVICE next64(uint64_t shiftedSeed) { const auto s0 = static_cast(shiftedSeed); auto s1 = static_cast(shiftedSeed) % sd::DataTypeUtils::max() + 11; uint64_t r0, r1; s1 ^= s0; r0 = rotl(s0, 55) ^ s1 ^ (s1 << 14); // a, b r1 = rotl(s1, 36); // c return r0 + r1; } static SD_HOST_DEVICE inline uint64_t rotl(const uint64_t x, uint64_t k) { return (x << k) | (x >> (64 - k)); } uint64_t static SD_HOST_DEVICE inline safeShift(uint64_t x, uint64_t y) { if (y != 0 && x > sd::DataTypeUtils::max() / y) { return x / y + 11; } else return (x * y) + 11; } uint64_t SD_HOST_DEVICE seedConv(sd::LongType seedIn) { uint64_t x = static_cast(seedIn); uint64_t z = (x += UINT64_C(0x9E3779B97F4A7C15)); z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9); z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB); return z ^ (z >> 31); } void SD_HOST_DEVICE incrementGeneration() { this->generation++; } sd::LongType SD_HOST_DEVICE getNextIndex() { currentPosition++; if (currentPosition >= size) { currentPosition = 0; generation++; } sd::LongType ret = currentPosition; return ret; } uint64_t SD_HOST_DEVICE getNextElement() { // TODO: proper implementation needed here return generation == 1 ? buffer[getNextIndex()] : buffer[getNextIndex()] * generation; } /** * This method skips X elements from buffer * * @param numberOfElements number of elements to skip */ #ifdef __CUDACC__ SD_DEVICE void rewind(sd::LongType numberOfElements) { if (gridDim.x > 1) { __shared__ bool amLast; if (threadIdx.x == 0) { unsigned int ticket = atomicInc(&synchronizer, gridDim.x); amLast = (ticket == gridDim.x - 1); } __syncthreads(); if (amLast) { if (threadIdx.x == 0) { synchronizer = 0; sd::LongType newPos = this->getOffset() + numberOfElements; if (newPos > this->getSize()) { generation += newPos / this->size; newPos = newPos % this->size; } else if (newPos == this->getSize()) { newPos = 0; generation++; } this->setOffset(newPos); } } } else { if (threadIdx.x == 0) { sd::LongType newPos = this->getOffset() + numberOfElements; if (newPos > this->getSize()) { generation += newPos / this->size; newPos = newPos % this->size; } else if (newPos == this->getSize()) { generation++; newPos = 0; } this->setOffset(newPos); } } } #endif void rewindH(sd::LongType numberOfElements) { sd::LongType newPos = this->getOffset() + numberOfElements; if (newPos > this->getSize()) { generation += newPos / this->size; newPos = newPos % this->size; } else if (newPos == this->getSize()) { generation++; newPos = 0; } this->setOffset(newPos); } /** * This method returns random int in range [0..SD_MAX_INT] * @return */ int SD_DEVICE nextInt() { auto u = nextUInt64(); return static_cast(u) <= sd::DataTypeUtils::max() ? static_cast(u) : static_cast(u % sd::DataTypeUtils::max()); }; uint64_t SD_DEVICE nextUInt64() { return getNextElement(); } /** * This method returns random int in range [0..to] * @param to * @return */ int SD_DEVICE nextInt(int to) { int r = nextInt(); int m = to - 1; if ((to & m) == 0) // i.e., bound is a power of 2 r = ((to * (sd::LongType)r) >> 31); else { for (int u = r; u - (r = u % to) + m < 0; u = nextInt()) ; } return r; }; /** * This method returns random int in range [from..to] * @param from * @param to * @return */ int SD_DEVICE nextInt(int from, int to) { if (from == 0) return nextInt(to); return from + nextInt(to - from); }; /** * This method returns random T in range of [0..1] * @return */ template SD_DEVICE T nextT() { auto u = static_cast(nextUInt64()); auto m = static_cast(sd::DataTypeUtils::max()); return static_cast(u / m); } /** * This method returns random T in range of [0..to] * @param to * @return */ template SD_DEVICE T nextT(T to) { if (to == static_cast(1.0f)) return nextT(); return nextT(static_cast(0.0f), to); } /** * This method returns random T in range [from..to] * @param from * @param to * @return */ template SD_DEVICE T inline nextT(T from, T to) { return from + (nextT() * (to - from)); } SD_INLINE SD_DEVICE uint64_t relativeUInt64(sd::LongType index) { return getElement(index); } /** * relative methods are made as workaround for lock-free concurrent execution */ inline int SD_DEVICE relativeInt(sd::LongType index) { auto u = relativeUInt64(index); return static_cast(u) <= sd::DataTypeUtils::max() ? static_cast(u) : static_cast(u % sd::DataTypeUtils::max()); } /** * This method returns random int within [0..to] * * @param index * @param to * @return */ inline int SD_DEVICE relativeInt(sd::LongType index, int to) { auto rel = relativeInt(index); return rel % to; } /** * This method returns random int within [from..to] * * @param index * @param to * @param from * @return */ SD_INLINE SD_DEVICE int relativeInt(sd::LongType index, int from, int to) { if (from == 0) return relativeInt(index, to); return from + relativeInt(index, to - from); } /** * This method returns random T within [0..1] * * @param index * @return */ template SD_INLINE SD_DEVICE T relativeT(sd::LongType index) { /** * Basically we just get float u/m value, and convert into to * * FIXME: once we add support for additional datatypes this code must be tweaked */ auto u = static_cast(relativeUInt64(index)); auto m = static_cast(sd::DataTypeUtils::max()); return static_cast(u / m); } /** * This method returns random T within [0..to] * * @param index * @param to * @return */ template SD_DEVICE T relativeT(sd::LongType index, T to) { if (to == static_cast(1.0f)) return relativeT(index); return relativeT(index, static_cast(0.0f), to); } /** * This method returns random T within [from..to] * * @param index * @param from * @param to * @return */ template SD_DEVICE T relativeT(sd::LongType index, T from, T to) { return from + (relativeT(index) * (to - from)); } }; class SD_LIB_EXPORT IGenerator { protected: sd::LongType limit; sd::LongType seed; uint64_t *buffer; sd::random::RandomBuffer *realBuffer; public: SD_HOST_DEVICE IGenerator(sd::random::RandomBuffer *buffer) { this->limit = buffer->getSize(); this->buffer = reinterpret_cast(buffer->getBuffer()); this->realBuffer = buffer; this->seed = buffer->getSeed(); } SD_HOST_DEVICE RandomBuffer *getBuffer() { return realBuffer; } SD_HOST_DEVICE void setOffset(sd::LongType offset) { this->realBuffer->setOffset(offset); } SD_HOST_DEVICE sd::LongType getElementAbsolute(sd::LongType position) { return buffer[position]; } SD_HOST_DEVICE sd::LongType getElementRelative(sd::LongType position) { return buffer[realBuffer->getOffset() + position]; } virtual SD_HOST_DEVICE void refreshBuffer() = 0; }; class SD_LIB_EXPORT Xoroshiro128 : public IGenerator { protected: uint64_t state[2]; static SD_INLINE SD_HOST_DEVICE uint64_t rotl(const uint64_t x, int k) { return (x << k) | (x >> (64 - k)); } /** * This method returns 64 random bits * @return */ uint64_t SD_HOST_DEVICE next64() { const uint64_t s0 = state[0]; uint64_t s1 = state[1]; const uint64_t result = s0 + s1; s1 ^= s0; state[0] = rotl(s0, 55) ^ s1 ^ (s1 << 14); // a, b state[1] = rotl(s1, 36); // c return result; } uint64_t SD_HOST_DEVICE seedConv(sd::LongType seed) { uint64_t x = static_cast(seed); uint64_t z = (x += UINT64_C(0x9E3779B97F4A7C15)); z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9); z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB); return z ^ (z >> 31); } void SD_HOST jump(void) { static const uint64_t JUMP[] = {0xbeac0467eba5facb, 0xd86b048b86aa9922}; uint64_t s0 = 0; uint64_t s1 = 0; for (unsigned int i = 0; i < sizeof JUMP / sizeof *JUMP; i++) for (int b = 0; b < 64; b++) { if (JUMP[i] & 1ULL << b) { s0 ^= state[0]; s1 ^= state[1]; } next64(); } state[0] = s0; state[1] = s1; } public: SD_HOST_DEVICE Xoroshiro128(sd::random::RandomBuffer *buffer) : IGenerator(buffer) { // } SD_HOST_DEVICE void refreshBuffer() { state[0] = seedConv(this->seed); state[1] = seedConv(this->seed * 119 + 3); int fd = 3 + 3; for (sd::LongType i = 0; i < limit; i++) { buffer[i] = next64(); } } }; } // namespace random } // namespace sd #endif // LIBND4J_HELPER_GENERATOR_H