chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,293 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma, created on 14.02.2018
|
||||
//
|
||||
|
||||
// implementation of operation for LSTM cell with peep hole connections:
|
||||
// http://www.bioinf.jku.at/publications/older/2604.pdf
|
||||
// S. Hochreiter and J. Schmidhuber. "Long Short-Term Memory". Neural Computation, 9(8):1735-1780, 1997.
|
||||
// and
|
||||
// https://research.google.com/pubs/archive/43905.pdf
|
||||
// Hasim Sak, Andrew Senior, and Francoise Beaufays. "Long short-term memory recurrent neural network architectures for
|
||||
// large scale acoustic modeling." INTERSPEECH, 2014.
|
||||
|
||||
#include <array/NDArrayList.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <graph/VariableSpace.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/helpers/legacy_helpers.h>
|
||||
#include <ops/declarable/helpers/lstm.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
|
||||
#include <iterator>
|
||||
# if NOT_EXCLUDED(OP_concat) && NOT_EXCLUDED(OP_lstm_cell) && NOT_EXCLUDED(OP_sigmoid)
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void lstmCell(sd::LaunchContext* context, NDArray* xt, NDArray* ht_1, NDArray* ct_1,
|
||||
NDArray* Wx, NDArray* Wh, NDArray* Wc, NDArray* Wp, NDArray* b, NDArray* ht,
|
||||
NDArray* ct, const std::vector<double>& params) {
|
||||
// xt input [bS x nIn]
|
||||
// ht_1 previous cell output [bS x numProj], that is at previous time step t-1, in case of projection=false ->
|
||||
// numProj=nOut!!! ct_1 previous cell state [bS x nOut], that is at previous time step t-1
|
||||
|
||||
// Wx input-to-hidden weights, [nIn x 4*nOut]
|
||||
// Wh hidden-to-hidden weights, [numProj x 4*nOut]
|
||||
// Wc diagonal weights for peephole connections [3*nOut]
|
||||
// Wp projection weights [nOut x numProj]
|
||||
// b biases, [4*nOut]
|
||||
|
||||
// ht current cell output [bS x numProj], that is at current time step t
|
||||
// ct current cell state [bS x nOut], that is at current time step t
|
||||
|
||||
const bool peephole = (bool)params[0]; // if true, provide peephole connections
|
||||
const bool projection =
|
||||
(bool)params[1]; // if true, then projection is performed, if false then numProj==nOut is mandatory!!!!
|
||||
double clippingCellValue =
|
||||
params[2]; // clipping value for ct, if it is not equal to zero, then cell state is clipped
|
||||
double clippingProjValue =
|
||||
params[3]; // clipping value for projected ht, if it is not equal to zero, then projected cell output is clipped
|
||||
const double forgetBias = params[4];
|
||||
|
||||
const int bS = xt->sizeAt(0);
|
||||
const int nIn = xt->sizeAt(1);
|
||||
const int numProj = ht_1->sizeAt(1);
|
||||
const int nOut = ct_1->sizeAt(1);
|
||||
|
||||
NDArray *mmulXt = mmul(*xt, *Wx);
|
||||
NDArray *mmulHt = mmul(*ht_1, *Wh);
|
||||
NDArray *addMmuls = (*mmulXt) + (*mmulHt);
|
||||
NDArray *z = (*addMmuls) + (*b); // [bS x 4*nOut] + [bS x 4*nOut] + [1 x 4*nOut] = [bS x 4*nOut]
|
||||
delete mmulXt;
|
||||
delete mmulHt;
|
||||
delete addMmuls;
|
||||
|
||||
NDArray *zit = (*z)({0, 0, 0, nOut}); // z for input gate, = mmul(Wxi,xt) + mmul(Whi,ht_1) + bi = [bS x nOut]
|
||||
NDArray *zft = (*z)({0, 0, nOut, 2 * nOut}); // z for forget gate, = mmul(Wxf,xt) + mmul(Whf,ht_1) + bf = [bS x nOut]
|
||||
NDArray *zct = (*z)({0, 0, 2 * nOut, 3 * nOut}); // z for cell state, = mmul(Wxc,xt) + mmul(Whc,ht_1) + bc = [bS x nOut]
|
||||
NDArray *zot = (*z)({0, 0, 3 * nOut, 4 * nOut}); // z for output gate, = mmul(Wxo,xt) + mmul(Who,ht_1) + bo = [bS x nOut]
|
||||
|
||||
if (peephole) { // add peephole connections: z + ct_1*Wc
|
||||
NDArray *wcFirst = (*Wc)({0, nOut});
|
||||
NDArray *wcSecond = (*Wc)({nOut, 2 * nOut});
|
||||
NDArray *peepholeFirst = (*ct_1) * (*wcFirst);
|
||||
NDArray *peepholeSecond = (*ct_1) * (*wcSecond);
|
||||
*zit += (*peepholeFirst); // add peephole connections to input gate
|
||||
*zft += (*peepholeSecond); // add peephole connections to forget gate
|
||||
delete peepholeFirst;
|
||||
delete peepholeSecond;
|
||||
delete wcFirst;
|
||||
delete wcSecond;
|
||||
}
|
||||
|
||||
// current sell state = ft*ct_1 + it*tanh(mmul(Wxc,xt) + mmul(Whc,ht_1) + bc
|
||||
NDArray *zftPlusBias = (*zft) + forgetBias;
|
||||
NDArray sigmoidZft = sigmoid(*zftPlusBias);
|
||||
NDArray sigmoidZit = sigmoid(*zit);
|
||||
NDArray tanhZct = tanh(*zct);
|
||||
NDArray *sigmoidZftMulCt1 = sigmoidZft * (*ct_1);
|
||||
NDArray *sigmoidZitMulTanhZct = sigmoidZit * tanhZct;
|
||||
NDArray *sigmoidOut = (*sigmoidZftMulCt1) + (*sigmoidZitMulTanhZct);
|
||||
ct->assign(sigmoidOut);
|
||||
delete zftPlusBias;
|
||||
delete sigmoidZftMulCt1;
|
||||
delete sigmoidZitMulTanhZct;
|
||||
delete sigmoidOut;
|
||||
|
||||
// if clipping value is provided then cell state is clipped by this value prior to the cell output activation
|
||||
if (clippingCellValue > 0.0) ct->applyScalar(scalar::LstmClip, clippingCellValue, ct);
|
||||
|
||||
if (peephole) {
|
||||
NDArray *wcThird = (*Wc)({{2 * nOut, 3 * nOut}});
|
||||
NDArray *peepholeThird = (*ct) * (*wcThird);
|
||||
*zot += (*peepholeThird); // add peephole connections to output gate zot + ct*Wc
|
||||
delete peepholeThird;
|
||||
delete wcThird;
|
||||
}
|
||||
|
||||
// current cell output = ot*tanh(ct)
|
||||
NDArray sigmoidZot = sigmoid(*zot);
|
||||
NDArray tanhCt = tanh(*ct);
|
||||
NDArray *htNoPeepHole = sigmoidZot * tanhCt; // = [bS x nOut]
|
||||
|
||||
// apply projection
|
||||
if (projection) {
|
||||
NDArray *assign = mmul(*htNoPeepHole, *Wp);
|
||||
ht->assign(assign); // [bS x nOut] * [ nOut x numProj] = [bS x numProj]
|
||||
delete assign;
|
||||
// if clipping projection is provided then projected cell output state is clipped by this value
|
||||
if (clippingProjValue != 0.) ht->applyScalar(scalar::LstmClip, clippingProjValue, ht);
|
||||
} else
|
||||
ht->assign(htNoPeepHole);
|
||||
|
||||
delete htNoPeepHole;
|
||||
delete z;
|
||||
delete zit;
|
||||
delete zft;
|
||||
delete zct;
|
||||
delete zot;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void fusedTanh(NDArray* z, NDArray* i, NDArray* c, NDArray* cLast, NDArray* f, NDArray* h) {
|
||||
// cell state = blockInput .* inputGate + prevCellState .* forgetGate
|
||||
auto uLen = static_cast<sd::LongType>(z->lengthOf());
|
||||
auto c_ = c->bufferAsT<T>();
|
||||
auto z_ = z->bufferAsT<T>();
|
||||
auto i_ = i->bufferAsT<T>();
|
||||
auto f_ = f->bufferAsT<T>();
|
||||
auto cLast_ = cLast->bufferAsT<T>();
|
||||
auto h_ = h->bufferAsT<T>();
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
c_[e] = z_[e] * i_[e] + (f_[e] * cLast_[e]);
|
||||
h_[e] = sd::math::sd_tanh<T, T>(c_[e]);
|
||||
}
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_for(func, 0, uLen);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void lstmBlockCell(NDArray* xt, NDArray* cLast, NDArray* yLast, NDArray* W, NDArray* Wci,
|
||||
NDArray* Wcf, NDArray* Wco, NDArray* b, NDArray* i, NDArray* c, NDArray* f,
|
||||
NDArray* o, NDArray* z, NDArray* h, NDArray* y, const std::vector<double>& params) {
|
||||
/* Input arrays:
|
||||
* 0: xt - input [bS, nIn] at time t
|
||||
* 1: cLast (cs_prev) - previous cell state [bS, nOut], time t-1
|
||||
* 2: yLast (h_prev) - previous output [bS, nOut], time t-1
|
||||
* 3: W - Weights - concatenated (input-to-hidden, hidden-to-hidden weights) weights, [(nIn+nOut),
|
||||
* 4*nOut] 4: Wci - weights - cell peephole (t-1) connections to input modulation gate, [nOut] 5: Wcf -
|
||||
* weights - cell peephole (t-1) connections to forget gate, [nOut] 6: Wco - weights - cell peephole (t)
|
||||
* connections to output gate, [nOut] 7: b - biases, [4*nOut]
|
||||
*
|
||||
* Input integer arguments:
|
||||
* 0: if not zero, provide peephole connections
|
||||
*
|
||||
* Input float arguments:
|
||||
* 0: the bias added to forget gates in order to reduce the scale of forgetting in the beginning of the training
|
||||
* 1: clipping value for cell state, if it is not equal to zero, then cell state is clipped
|
||||
*
|
||||
* Output arrays:
|
||||
* 0: i - Input modulation gate activations [bS, nOut]
|
||||
* 1: c (cs) - Cell state (pre tanh) [bs, nOut] (cs)
|
||||
* 2: f - Output - forget gate activations [bs, nOut]
|
||||
* 3: o - Output - output gate activations [bs, nOut]
|
||||
* 4: z (ci) - Output - block input [bs, nOut]
|
||||
* 5: h (co) - Cell state, post tanh [bs, nOut]
|
||||
* 6: y (h) - Current cell output [bS, nOut], time t
|
||||
*/
|
||||
const bool peephole = (bool)params[0]; // if true, provide peephole connections
|
||||
const double forgetBias = params[1];
|
||||
const double clippingCellValue =
|
||||
params[2]; // clipping value for ct, if it is not equal to zero, then cell state is clipped
|
||||
|
||||
const int bS = xt->sizeAt(0);
|
||||
const int nIn = xt->sizeAt(1);
|
||||
const int nOut = cLast->sizeAt(1);
|
||||
|
||||
std::vector<sd::LongType> cOutShape = {xt->sizeAt(0),xt->sizeAt(1), xt->sizeAt(1) + yLast->sizeAt(1)};
|
||||
// Concat inputs: [xt, yt-1]: concat([bs,nIn],[bs,nOut]) -> [bs, (nIn+nOut)]
|
||||
NDArray concatOut(xt->ordering(), cOutShape, xt->dataType(),
|
||||
xt->getContext());
|
||||
helpers::concat(xt->getContext(), {const_cast<NDArray*>(xt), const_cast<NDArray*>(yLast)}, concatOut, 1);
|
||||
|
||||
NDArray *m = mmul(concatOut, *W); // mmul: [bs, (nIn+nOut)] * [(nIn+nOut), 4*nOut] = [bs, 4*nOut]
|
||||
*m += (*b); // addiRowVector
|
||||
|
||||
// Note: weights are ordered [inputGate, blockInput, forgetGate, outputGate] to match TF (TF code comments state
|
||||
// [i,f,z/ci,o] but behaviour is [i,z,f,o])
|
||||
NDArray *zi = (*m)({0, 0, 0, nOut}); // z for input modulation gate, [bS, nOut]
|
||||
NDArray *zz = (*m)({0, 0, nOut, 2 * nOut}); // z for block input, [bS, nOut]
|
||||
NDArray *zf = (*m)({0, 0, 2 * nOut, 3 * nOut}); // z for forget gate, [bS, nOut]
|
||||
NDArray *zo = (*m)({0, 0, 3 * nOut, 4 * nOut}); // z for output gate, [bS, nOut]
|
||||
|
||||
if (peephole) { // add peephole connections: z + ct_1*Wc
|
||||
NDArray *peepholeI = (*cLast) * (*Wci);
|
||||
NDArray *peepholeF = (*cLast) * (*Wcf);
|
||||
*zi += (*peepholeI); // add peephole connections to input gate
|
||||
*zf += (*peepholeF); // add peephole connections to forget gate
|
||||
delete peepholeI;
|
||||
delete peepholeF;
|
||||
}
|
||||
|
||||
// current sell state = ft*cLast + it*tanh(mmul(Wxc,xt) + mmul(Whc,ht_1) + bc
|
||||
if (forgetBias != 0.0) *zf += forgetBias;
|
||||
|
||||
PRAGMA_OMP_PARALLEL
|
||||
PRAGMA_OMP_SINGLE {
|
||||
PRAGMA_OMP_TASK
|
||||
zz->applyTransform(transform::Tanh, z); // z = tanh(zz)
|
||||
|
||||
PRAGMA_OMP_TASK
|
||||
zi->applyTransform(transform::Sigmoid, i); // i = sigmoid(zi)
|
||||
|
||||
PRAGMA_OMP_TASK
|
||||
zf->applyTransform(transform::Sigmoid, f); // f = sigmoid(zf);
|
||||
}
|
||||
|
||||
if (z->ews() == 1 && i->ews() == 1 && c->ews() == 1 && cLast->ews() == 1 && f->ews() == 1 && h->ews() == 1 &&
|
||||
z->ordering() == i->ordering() && z->ordering() == c->ordering() && z->ordering() == cLast->ordering() &&
|
||||
z->ordering() == f->ordering() && z->ordering() == h->ordering()) {
|
||||
// cell state = blockInput .* inputGate + prevCellState .* forgetGate
|
||||
BUILD_SINGLE_SELECTOR(z->dataType(), fusedTanh, (z, i, c, cLast, f, h), SD_FLOAT_TYPES);
|
||||
} else {
|
||||
// cell state = blockInput .* inputGate + prevCellState .* forgetGate
|
||||
z->applyPairwiseTransform(pairwise::Multiply, i, c); // c = z * i
|
||||
NDArray *temp = (*f) * (*cLast);
|
||||
*c += (*temp); // c = (i * z) + (zf * (*cLast))
|
||||
delete temp;
|
||||
c->applyTransform(transform::Tanh, h); // h = tanh(c)
|
||||
}
|
||||
|
||||
// if clipping value is provided then cell state is clipped by this value prior to the cell output activation
|
||||
if (clippingCellValue > 0.0) c->applyScalar(scalar::LstmClip, clippingCellValue, c);
|
||||
|
||||
// add peephole connections to output gate zot + ct*Wc
|
||||
if (peephole) {
|
||||
NDArray *prod = (*c) * (*Wco);
|
||||
*zo += (*prod);
|
||||
delete prod;
|
||||
}
|
||||
|
||||
zo->applyTransform(transform::Sigmoid, o); // o = sigmoid(zo)
|
||||
|
||||
// current cell output = ot*tanh(ct)
|
||||
c->applyTransform(transform::Tanh, h); // h = tanh(c)
|
||||
o->applyPairwiseTransform(pairwise::Multiply, h, y); // y = o * h
|
||||
|
||||
delete m;
|
||||
delete zi;
|
||||
delete zz;
|
||||
delete zf;
|
||||
delete zo;
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
Reference in New Issue
Block a user