/* * ****************************************************************************** * * * * * * This program and the accompanying materials are made available under the * * terms of the Apache License, Version 2.0 which is available at * * https://www.apache.org/licenses/LICENSE-2.0. * * * * See the NOTICE file distributed with this work for additional * * information regarding copyright ownership. * * Unless required by applicable law or agreed to in writing, software * * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * * License for the specific language governing permissions and limitations * * under the License. * * * * SPDX-License-Identifier: Apache-2.0 * ***************************************************************************** */ // // @author Yurii Shyrma, created on 14.02.2018 // // implementation of operation for LSTM cell with peep hole connections: // http://www.bioinf.jku.at/publications/older/2604.pdf // S. Hochreiter and J. Schmidhuber. "Long Short-Term Memory". Neural Computation, 9(8):1735-1780, 1997. // and // https://research.google.com/pubs/archive/43905.pdf // Hasim Sak, Andrew Senior, and Francoise Beaufays. "Long short-term memory recurrent neural network architectures for // large scale acoustic modeling." INTERSPEECH, 2014. #include #include #include #include #include #include #include #include #include # if NOT_EXCLUDED(OP_concat) && NOT_EXCLUDED(OP_lstm_cell) && NOT_EXCLUDED(OP_sigmoid) namespace sd { namespace ops { namespace helpers { ////////////////////////////////////////////////////////////////////////// void lstmCell(sd::LaunchContext* context, NDArray* xt, NDArray* ht_1, NDArray* ct_1, NDArray* Wx, NDArray* Wh, NDArray* Wc, NDArray* Wp, NDArray* b, NDArray* ht, NDArray* ct, const std::vector& params) { // xt input [bS x nIn] // ht_1 previous cell output [bS x numProj], that is at previous time step t-1, in case of projection=false -> // numProj=nOut!!! ct_1 previous cell state [bS x nOut], that is at previous time step t-1 // Wx input-to-hidden weights, [nIn x 4*nOut] // Wh hidden-to-hidden weights, [numProj x 4*nOut] // Wc diagonal weights for peephole connections [3*nOut] // Wp projection weights [nOut x numProj] // b biases, [4*nOut] // ht current cell output [bS x numProj], that is at current time step t // ct current cell state [bS x nOut], that is at current time step t const bool peephole = (bool)params[0]; // if true, provide peephole connections const bool projection = (bool)params[1]; // if true, then projection is performed, if false then numProj==nOut is mandatory!!!! double clippingCellValue = params[2]; // clipping value for ct, if it is not equal to zero, then cell state is clipped double clippingProjValue = params[3]; // clipping value for projected ht, if it is not equal to zero, then projected cell output is clipped const double forgetBias = params[4]; const int bS = xt->sizeAt(0); const int nIn = xt->sizeAt(1); const int numProj = ht_1->sizeAt(1); const int nOut = ct_1->sizeAt(1); NDArray *mmulXt = mmul(*xt, *Wx); NDArray *mmulHt = mmul(*ht_1, *Wh); NDArray *addMmuls = (*mmulXt) + (*mmulHt); NDArray *z = (*addMmuls) + (*b); // [bS x 4*nOut] + [bS x 4*nOut] + [1 x 4*nOut] = [bS x 4*nOut] delete mmulXt; delete mmulHt; delete addMmuls; NDArray *zit = (*z)({0, 0, 0, nOut}); // z for input gate, = mmul(Wxi,xt) + mmul(Whi,ht_1) + bi = [bS x nOut] NDArray *zft = (*z)({0, 0, nOut, 2 * nOut}); // z for forget gate, = mmul(Wxf,xt) + mmul(Whf,ht_1) + bf = [bS x nOut] NDArray *zct = (*z)({0, 0, 2 * nOut, 3 * nOut}); // z for cell state, = mmul(Wxc,xt) + mmul(Whc,ht_1) + bc = [bS x nOut] NDArray *zot = (*z)({0, 0, 3 * nOut, 4 * nOut}); // z for output gate, = mmul(Wxo,xt) + mmul(Who,ht_1) + bo = [bS x nOut] if (peephole) { // add peephole connections: z + ct_1*Wc NDArray *wcFirst = (*Wc)({0, nOut}); NDArray *wcSecond = (*Wc)({nOut, 2 * nOut}); NDArray *peepholeFirst = (*ct_1) * (*wcFirst); NDArray *peepholeSecond = (*ct_1) * (*wcSecond); *zit += (*peepholeFirst); // add peephole connections to input gate *zft += (*peepholeSecond); // add peephole connections to forget gate delete peepholeFirst; delete peepholeSecond; delete wcFirst; delete wcSecond; } // current sell state = ft*ct_1 + it*tanh(mmul(Wxc,xt) + mmul(Whc,ht_1) + bc NDArray *zftPlusBias = (*zft) + forgetBias; NDArray sigmoidZft = sigmoid(*zftPlusBias); NDArray sigmoidZit = sigmoid(*zit); NDArray tanhZct = tanh(*zct); NDArray *sigmoidZftMulCt1 = sigmoidZft * (*ct_1); NDArray *sigmoidZitMulTanhZct = sigmoidZit * tanhZct; NDArray *sigmoidOut = (*sigmoidZftMulCt1) + (*sigmoidZitMulTanhZct); ct->assign(sigmoidOut); delete zftPlusBias; delete sigmoidZftMulCt1; delete sigmoidZitMulTanhZct; delete sigmoidOut; // if clipping value is provided then cell state is clipped by this value prior to the cell output activation if (clippingCellValue > 0.0) ct->applyScalar(scalar::LstmClip, clippingCellValue, ct); if (peephole) { NDArray *wcThird = (*Wc)({{2 * nOut, 3 * nOut}}); NDArray *peepholeThird = (*ct) * (*wcThird); *zot += (*peepholeThird); // add peephole connections to output gate zot + ct*Wc delete peepholeThird; delete wcThird; } // current cell output = ot*tanh(ct) NDArray sigmoidZot = sigmoid(*zot); NDArray tanhCt = tanh(*ct); NDArray *htNoPeepHole = sigmoidZot * tanhCt; // = [bS x nOut] // apply projection if (projection) { NDArray *assign = mmul(*htNoPeepHole, *Wp); ht->assign(assign); // [bS x nOut] * [ nOut x numProj] = [bS x numProj] delete assign; // if clipping projection is provided then projected cell output state is clipped by this value if (clippingProjValue != 0.) ht->applyScalar(scalar::LstmClip, clippingProjValue, ht); } else ht->assign(htNoPeepHole); delete htNoPeepHole; delete z; delete zit; delete zft; delete zct; delete zot; } template static void fusedTanh(NDArray* z, NDArray* i, NDArray* c, NDArray* cLast, NDArray* f, NDArray* h) { // cell state = blockInput .* inputGate + prevCellState .* forgetGate auto uLen = static_cast(z->lengthOf()); auto c_ = c->bufferAsT(); auto z_ = z->bufferAsT(); auto i_ = i->bufferAsT(); auto f_ = f->bufferAsT(); auto cLast_ = cLast->bufferAsT(); auto h_ = h->bufferAsT(); auto func = PRAGMA_THREADS_FOR { for (auto e = start; e < stop; e++) { c_[e] = z_[e] * i_[e] + (f_[e] * cLast_[e]); h_[e] = sd::math::sd_tanh(c_[e]); } }; samediff::Threads::parallel_for(func, 0, uLen); } ////////////////////////////////////////////////////////////////////////// void lstmBlockCell(NDArray* xt, NDArray* cLast, NDArray* yLast, NDArray* W, NDArray* Wci, NDArray* Wcf, NDArray* Wco, NDArray* b, NDArray* i, NDArray* c, NDArray* f, NDArray* o, NDArray* z, NDArray* h, NDArray* y, const std::vector& params) { /* Input arrays: * 0: xt - input [bS, nIn] at time t * 1: cLast (cs_prev) - previous cell state [bS, nOut], time t-1 * 2: yLast (h_prev) - previous output [bS, nOut], time t-1 * 3: W - Weights - concatenated (input-to-hidden, hidden-to-hidden weights) weights, [(nIn+nOut), * 4*nOut] 4: Wci - weights - cell peephole (t-1) connections to input modulation gate, [nOut] 5: Wcf - * weights - cell peephole (t-1) connections to forget gate, [nOut] 6: Wco - weights - cell peephole (t) * connections to output gate, [nOut] 7: b - biases, [4*nOut] * * Input integer arguments: * 0: if not zero, provide peephole connections * * Input float arguments: * 0: the bias added to forget gates in order to reduce the scale of forgetting in the beginning of the training * 1: clipping value for cell state, if it is not equal to zero, then cell state is clipped * * Output arrays: * 0: i - Input modulation gate activations [bS, nOut] * 1: c (cs) - Cell state (pre tanh) [bs, nOut] (cs) * 2: f - Output - forget gate activations [bs, nOut] * 3: o - Output - output gate activations [bs, nOut] * 4: z (ci) - Output - block input [bs, nOut] * 5: h (co) - Cell state, post tanh [bs, nOut] * 6: y (h) - Current cell output [bS, nOut], time t */ const bool peephole = (bool)params[0]; // if true, provide peephole connections const double forgetBias = params[1]; const double clippingCellValue = params[2]; // clipping value for ct, if it is not equal to zero, then cell state is clipped const int bS = xt->sizeAt(0); const int nIn = xt->sizeAt(1); const int nOut = cLast->sizeAt(1); std::vector cOutShape = {xt->sizeAt(0),xt->sizeAt(1), xt->sizeAt(1) + yLast->sizeAt(1)}; // Concat inputs: [xt, yt-1]: concat([bs,nIn],[bs,nOut]) -> [bs, (nIn+nOut)] NDArray concatOut(xt->ordering(), cOutShape, xt->dataType(), xt->getContext()); helpers::concat(xt->getContext(), {const_cast(xt), const_cast(yLast)}, concatOut, 1); NDArray *m = mmul(concatOut, *W); // mmul: [bs, (nIn+nOut)] * [(nIn+nOut), 4*nOut] = [bs, 4*nOut] *m += (*b); // addiRowVector // Note: weights are ordered [inputGate, blockInput, forgetGate, outputGate] to match TF (TF code comments state // [i,f,z/ci,o] but behaviour is [i,z,f,o]) NDArray *zi = (*m)({0, 0, 0, nOut}); // z for input modulation gate, [bS, nOut] NDArray *zz = (*m)({0, 0, nOut, 2 * nOut}); // z for block input, [bS, nOut] NDArray *zf = (*m)({0, 0, 2 * nOut, 3 * nOut}); // z for forget gate, [bS, nOut] NDArray *zo = (*m)({0, 0, 3 * nOut, 4 * nOut}); // z for output gate, [bS, nOut] if (peephole) { // add peephole connections: z + ct_1*Wc NDArray *peepholeI = (*cLast) * (*Wci); NDArray *peepholeF = (*cLast) * (*Wcf); *zi += (*peepholeI); // add peephole connections to input gate *zf += (*peepholeF); // add peephole connections to forget gate delete peepholeI; delete peepholeF; } // current sell state = ft*cLast + it*tanh(mmul(Wxc,xt) + mmul(Whc,ht_1) + bc if (forgetBias != 0.0) *zf += forgetBias; PRAGMA_OMP_PARALLEL PRAGMA_OMP_SINGLE { PRAGMA_OMP_TASK zz->applyTransform(transform::Tanh, z); // z = tanh(zz) PRAGMA_OMP_TASK zi->applyTransform(transform::Sigmoid, i); // i = sigmoid(zi) PRAGMA_OMP_TASK zf->applyTransform(transform::Sigmoid, f); // f = sigmoid(zf); } if (z->ews() == 1 && i->ews() == 1 && c->ews() == 1 && cLast->ews() == 1 && f->ews() == 1 && h->ews() == 1 && z->ordering() == i->ordering() && z->ordering() == c->ordering() && z->ordering() == cLast->ordering() && z->ordering() == f->ordering() && z->ordering() == h->ordering()) { // cell state = blockInput .* inputGate + prevCellState .* forgetGate BUILD_SINGLE_SELECTOR(z->dataType(), fusedTanh, (z, i, c, cLast, f, h), SD_FLOAT_TYPES); } else { // cell state = blockInput .* inputGate + prevCellState .* forgetGate z->applyPairwiseTransform(pairwise::Multiply, i, c); // c = z * i NDArray *temp = (*f) * (*cLast); *c += (*temp); // c = (i * z) + (zf * (*cLast)) delete temp; c->applyTransform(transform::Tanh, h); // h = tanh(c) } // if clipping value is provided then cell state is clipped by this value prior to the cell output activation if (clippingCellValue > 0.0) c->applyScalar(scalar::LstmClip, clippingCellValue, c); // add peephole connections to output gate zot + ct*Wc if (peephole) { NDArray *prod = (*c) * (*Wco); *zo += (*prod); delete prod; } zo->applyTransform(transform::Sigmoid, o); // o = sigmoid(zo) // current cell output = ot*tanh(ct) c->applyTransform(transform::Tanh, h); // h = tanh(c) o->applyPairwiseTransform(pairwise::Multiply, h, y); // y = o * h delete m; delete zi; delete zz; delete zf; delete zo; } } // namespace helpers } // namespace ops } // namespace sd #endif