chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1 @@
|
||||
This folder contains generic helpers implementations, for operations that do not require platform-specific code, or have no real sense of having platform-specific code
|
||||
@@ -0,0 +1,154 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_choose)
|
||||
|
||||
#include <array/NDArrayFactory.h>
|
||||
#include <ops/declarable/helpers/choose.h>
|
||||
#include <ops/ops.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static NDArray* processCondition_(int mode, NDArray* arg, NDArray* comp, NDArray& compScalar);
|
||||
|
||||
template <typename T>
|
||||
static T processElementCondition(int mode, T d1, T d2);
|
||||
|
||||
template <typename T>
|
||||
NDArray* processCondition_(int mode, NDArray* arg, NDArray* comp, NDArray* output, NDArray* numResult,
|
||||
NDArray& compScalar) {
|
||||
// Convert to straight ndarray based on input
|
||||
|
||||
int numResults = 0;
|
||||
if (comp != nullptr) {
|
||||
if (comp->isScalar()) {
|
||||
// Other input for compare could be an ndarray or a secondary scalar
|
||||
// for comparison
|
||||
// sd::NDArray arg1 = *arg;
|
||||
// sd::NDArray comp1 = *comp;
|
||||
for (LongType i = 0; i < arg->lengthOf(); i++) {
|
||||
T result2 = processElementCondition(mode, arg->e<T>(i), comp->e<T>(0));
|
||||
if (result2 > static_cast<T>(0)) {
|
||||
if (output != nullptr) output->p(numResults, arg->e<T>(i));
|
||||
numResults++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Other input for compare could be an ndarray or a secondary scalar
|
||||
// for comparison
|
||||
NDArray arg1 = *arg;
|
||||
for (LongType i = 0; i < arg->lengthOf(); i++) {
|
||||
T result2 = processElementCondition(mode, arg->e<T>(i), comp->e<T>(i));
|
||||
if (result2 > static_cast<T>(0)) {
|
||||
if (output != nullptr) output->p(numResults, arg->e<T>(i));
|
||||
numResults++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
// sd::NDArray arg1 = *arg;
|
||||
// Other input for compare could be an ndarray or a secondary scalar
|
||||
// for comparison
|
||||
for (LongType i = 0; i < arg->lengthOf(); i++) {
|
||||
T result2 = processElementCondition(mode, arg->e<T>(i), compScalar.e<T>(0));
|
||||
if (result2 > static_cast<T>(0)) {
|
||||
if (output != nullptr) output->p(numResults, arg->e<T>(i));
|
||||
numResults++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (numResult != nullptr) numResult->p(0, numResults);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
NDArray* processCondition(LaunchContext* context, int mode, NDArray* arg, NDArray* comp, NDArray* output,
|
||||
NDArray* numResult, NDArray& compScalar) {
|
||||
arg->syncToHost();
|
||||
|
||||
if (comp != nullptr) comp->syncToHost();
|
||||
|
||||
if (output != nullptr) output->syncToHost();
|
||||
|
||||
if (numResult != nullptr) numResult->syncToHost();
|
||||
|
||||
compScalar.syncToHost();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(arg->dataType(), return processCondition_, (mode, arg, comp, output, numResult, compScalar),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
arg->syncToDevice();
|
||||
|
||||
if (comp != nullptr) comp->syncToDevice();
|
||||
|
||||
if (output != nullptr) output->syncToDevice();
|
||||
|
||||
if (numResult != nullptr) numResult->syncToDevice();
|
||||
|
||||
compScalar.syncToDevice();
|
||||
return nullptr;
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( NDArray* processCondition_,
|
||||
(int mode, sd::NDArray* arg, sd::NDArray* comp, sd::NDArray* output, sd::NDArray* numResult,
|
||||
sd::NDArray& compScalar),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
template <typename T>
|
||||
T processElementCondition(int mode, T d1, T d2) {
|
||||
T input[3] = {d2, (T)SD_EPSILON, (T)mode};
|
||||
T res = simdOps::MatchCondition<T, T>::op(d1, input);
|
||||
return res;
|
||||
}
|
||||
|
||||
void chooseFunctorArray(LaunchContext* context, NDArray* arg, NDArray* comp, int mode, NDArray* result,
|
||||
NDArray* numResults) {
|
||||
if (arg->isScalar() || comp->isScalar()) {
|
||||
if (arg->isScalar()) {
|
||||
processCondition(context, mode, comp, nullptr, result, numResults, *arg);
|
||||
} else {
|
||||
processCondition(context, mode, arg, nullptr, result, numResults, *comp);
|
||||
}
|
||||
} else {
|
||||
auto zero = NDArrayFactory::create<float>(0);
|
||||
processCondition(context, mode, arg, comp, result, numResults, *zero);
|
||||
delete zero;
|
||||
}
|
||||
}
|
||||
|
||||
void chooseFunctorScalar(LaunchContext* context, NDArray* arg, double scalar, int mode, NDArray* result,
|
||||
NDArray* numResults) {
|
||||
auto scalarA = NDArrayFactory::create(scalar);
|
||||
processCondition(context, mode, arg, nullptr, result, numResults, *scalarA);
|
||||
delete scalarA;
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,658 @@
|
||||
|
||||
/*******************************************************************************
|
||||
* Copyright (c) 2021 Deeplearning4j Contributors
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*******************************************************************************/
|
||||
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_ctcBeam)
|
||||
//
|
||||
// @author AbdelRauf
|
||||
//
|
||||
#include <execution/ThreadPool.h>
|
||||
#include <execution/Threads.h>
|
||||
#include <helpers/LoopsCoordsHelper.h>
|
||||
#include <ops/declarable/helpers/ctc.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
#include <vector>
|
||||
#include <system/selective_rendering.h>
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
struct BeamProb {
|
||||
T total = negative_infinity<T>();
|
||||
T non_blank = negative_infinity<T>();
|
||||
T blank = negative_infinity<T>(); // log(1)
|
||||
};
|
||||
|
||||
template <typename T, typename T2 = void>
|
||||
struct DefaultInvalid {
|
||||
static constexpr T value = T();
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct DefaultInvalid<T, typename std::enable_if<std::is_integral<T>::value>::type> {
|
||||
static constexpr T value = static_cast<T>(-1);
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct SequenceNode {
|
||||
// intrusive double links
|
||||
SequenceNode<T>* prev = nullptr;
|
||||
SequenceNode<T>* next = nullptr;
|
||||
|
||||
// sequence prefix/parent
|
||||
SequenceNode<T>* prefix = nullptr;
|
||||
|
||||
T value = DefaultInvalid<T>::value;
|
||||
|
||||
int state = 0;
|
||||
|
||||
void markAsFullyExtended() { state |= 1; }
|
||||
|
||||
void increaseRef() {
|
||||
// we will have just two copies in bad case. so just or
|
||||
state = state | 2;
|
||||
}
|
||||
|
||||
void decreaseRef() {
|
||||
// we will have just two cases in bad case, so just remove that
|
||||
state = state & (-2);
|
||||
}
|
||||
|
||||
bool safeToRemove() {
|
||||
if (state & 1) return false;
|
||||
|
||||
decreaseRef();
|
||||
// we do not want to remove parent nodes in our case. otherwise just returning state<=1 is ok
|
||||
return state == 0;
|
||||
}
|
||||
|
||||
bool isFullyExtended() const { return state & 1; }
|
||||
};
|
||||
|
||||
/***
|
||||
* Sequence container.
|
||||
*
|
||||
* NOTE: it is not thread-safe
|
||||
*
|
||||
* Extend path - O(1)
|
||||
* Remove path - O(1)
|
||||
* Generating Sequence with backtracking prefix: O(n)
|
||||
*
|
||||
* Note: Sequence container is implemented primitively and only usable within this task.
|
||||
* As it does not behave as a fully capable tree. some cases should be handled manually
|
||||
*
|
||||
* Here is special cases that should be handled manually to exploit tree/graph behaviour:
|
||||
*
|
||||
* Extending new path value:
|
||||
*
|
||||
* To extend the path one need to give path and value and in return get new_path:
|
||||
* new_path = container.extendPath ( path, new_value );
|
||||
*
|
||||
* Also note that:
|
||||
* SequenceContainer has already default empty path as a beginning point for paths.
|
||||
* So as an initial node one should use it.
|
||||
* initial_path = container.getEmptyPath();
|
||||
*
|
||||
* Adding new path that could be already in container:
|
||||
*
|
||||
* Assume we have two paths that can overlap in next step
|
||||
* 1st path: node#0() -> node#1(1) => generated sequence {},{1}
|
||||
* 2nd path: node#0() -> node#1(1) -> node#2(2) => generated sequence {},{1}, {2}
|
||||
*
|
||||
* While extending the first path with value (2). it will be:
|
||||
*
|
||||
* node#0() -> node#0(1) -> node#( either new or old)(2) => generated sequence {},{1}, {2}
|
||||
*
|
||||
* For some tasks its not desired to have additional node that will generate the same sequence.
|
||||
* For example:
|
||||
* Assume you wanted to use it as sequence entry in map with just (entry->prefix, entry->value).
|
||||
* so in that case having different paths is not correct and will not be unique in map.
|
||||
*
|
||||
* there is not direct way to handle that in our container other than searching.
|
||||
* So one should look for the node with prefix node#1(1) and value(2) and return that node instead of adding new
|
||||
one
|
||||
|
||||
* Fortunately, for our beam search case:
|
||||
*
|
||||
* we need only look for such overlapped cases within the candidates list.
|
||||
* which makes it easy to determine them beforehand while finding and marking overlapped cases. instead of looking
|
||||
for it in SequenceContainer
|
||||
*
|
||||
* Removing the same nodes multiple times:
|
||||
* It is fast to remove nodes. As nodes can be stored externally One should follow this rule:
|
||||
*
|
||||
* One should not remove the same node twice as it will lead to double free. as Nodes are pointers the same
|
||||
applies to removing a copy
|
||||
*
|
||||
* There could be cases where you would like to store copy of nodes. in that cases you can use below method to be
|
||||
able to safely remove:
|
||||
* node should have mutable method named safeToRemove().
|
||||
* Basic implementation will be decreasing reference/copy counts and returning true if it is safe to delete
|
||||
*
|
||||
*
|
||||
*/
|
||||
template <typename T>
|
||||
class SequenceContainer {
|
||||
public:
|
||||
SequenceContainer() : count_(1) {
|
||||
empty_path = new SequenceNode<T>();
|
||||
current_ = empty_path;
|
||||
}
|
||||
|
||||
SequenceContainer(const SequenceContainer& s) = delete;
|
||||
|
||||
SequenceContainer(SequenceContainer&& other) noexcept {
|
||||
this->current_ = other.current_;
|
||||
other.current_ = nullptr;
|
||||
}
|
||||
|
||||
SequenceContainer& operator=(const SequenceContainer& other) = delete;
|
||||
|
||||
SequenceContainer& operator=(SequenceContainer&& other) noexcept {
|
||||
if (this != other) {
|
||||
clear();
|
||||
this->current_ = other.current_;
|
||||
this->count_ = other.count_;
|
||||
other.current_ = nullptr;
|
||||
other.count_ = 0;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
SequenceNode<T>* getEmptyPath() { return current_; }
|
||||
|
||||
SequenceNode<T>* extendPath(SequenceNode<T>* prefix, T value) {
|
||||
auto new_node = new SequenceNode<T>();
|
||||
|
||||
new_node->value = value;
|
||||
new_node->prefix = prefix;
|
||||
// add in the holder
|
||||
new_node->next = nullptr;
|
||||
new_node->prev = current_;
|
||||
if (current_) current_->next = new_node;
|
||||
|
||||
current_ = new_node;
|
||||
count_++;
|
||||
return new_node;
|
||||
}
|
||||
|
||||
void remove(SequenceNode<T>* seq) {
|
||||
if (seq == nullptr) return;
|
||||
|
||||
if (!seq->safeToRemove()) return;
|
||||
|
||||
SequenceNode<T>* previous = seq->prev;
|
||||
SequenceNode<T>* next = seq->next;
|
||||
if (previous) previous->next = next;
|
||||
if (next) next->prev = previous;
|
||||
|
||||
if (current_ == seq) {
|
||||
current_ = previous;
|
||||
}
|
||||
|
||||
delete seq;
|
||||
count_--;
|
||||
}
|
||||
|
||||
static std::vector<T> getSequence(SequenceNode<T>* seq, size_t reserve_size = 1024) {
|
||||
std::vector<T> ret;
|
||||
ret.reserve(reserve_size);
|
||||
SequenceNode<T>* backtrack = seq;
|
||||
while (backtrack) {
|
||||
ret.push_back(backtrack->value);
|
||||
backtrack = backtrack->prefix;
|
||||
}
|
||||
if (ret.size() > 1) {
|
||||
// remove last default node
|
||||
ret.pop_back();
|
||||
// reverse
|
||||
std::reverse(std::begin(ret), std::end(ret));
|
||||
return ret;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
void clear() {
|
||||
// destruct all nodes
|
||||
SequenceNode<T>* del = current_;
|
||||
// int i = 0;
|
||||
while (del) {
|
||||
//++i;
|
||||
SequenceNode<T>* temp = del->prev;
|
||||
delete del;
|
||||
del = temp;
|
||||
}
|
||||
current_ = nullptr;
|
||||
}
|
||||
|
||||
~SequenceContainer() { clear(); }
|
||||
|
||||
private:
|
||||
SequenceNode<T>* current_ = nullptr;
|
||||
|
||||
SequenceNode<T>* empty_path = nullptr;
|
||||
|
||||
int count_ = 0;
|
||||
};
|
||||
|
||||
template <typename T, typename U>
|
||||
struct BeamEntry {
|
||||
SequenceNode<U>* sequence{};
|
||||
BeamProb<T> prob;
|
||||
};
|
||||
|
||||
template <typename T, typename U>
|
||||
struct BeamEntryEx {
|
||||
BeamEntry<T, U> entry;
|
||||
// keep indices for lookUp
|
||||
int index_as_child = -1;
|
||||
int index_as_parent = -1;
|
||||
int children_count = 0;
|
||||
};
|
||||
|
||||
template <typename T, typename U>
|
||||
struct LookUpEntry {
|
||||
U last_c; // this is is the same as node->value. just we added for the speed
|
||||
SequenceNode<U>* node = nullptr;
|
||||
int next_beam_index = -1; // index inside next_beam array
|
||||
};
|
||||
|
||||
template <typename T, typename U>
|
||||
static bool compare_beam_prob(const BeamEntry<T, U>& i1, const BeamEntry<T, U>& i2) {
|
||||
return (i1.prob.total > i2.prob.total);
|
||||
}
|
||||
|
||||
template <typename T, typename U>
|
||||
SD_INLINE T pr(const int c, const BeamProb<T>& beam_prob, const SequenceNode<U>* seq, const T prob) {
|
||||
return seq->value == c ? beam_prob.blank + prob : beam_prob.total + prob;
|
||||
}
|
||||
|
||||
template <bool HasElementStride = false, typename Type, typename IndexType>
|
||||
void inner_beam_search(const Type* log_p, const uint64_t inc_p, IndexType* result_sequence, const uint64_t inc_res_seq,
|
||||
const uint64_t max_len_t, Type* result_prob, IndexType* result_seq_length, uint64_t len_t,
|
||||
const uint64_t len_c, const int blank_index, int beam_width, int nbest_len,
|
||||
bool normalize_logits, const uint64_t element_stride = 1L) {
|
||||
using BeamEntryType = BeamEntry<Type, IndexType>;
|
||||
using BeamEntryTypeEx = BeamEntryEx<Type, IndexType>;
|
||||
|
||||
if (beam_width < 1) beam_width = 1;
|
||||
if (nbest_len > beam_width) nbest_len = beam_width;
|
||||
// if len_t is greater than max_len_t truncate it
|
||||
len_t = len_t > max_len_t ? max_len_t : len_t;
|
||||
|
||||
SequenceContainer<IndexType> sequence_container;
|
||||
BeamEntryType empty;
|
||||
empty.prob.blank = 0;
|
||||
empty.prob.total = log_sum_exp(empty.prob.blank, empty.prob.non_blank);
|
||||
empty.sequence = sequence_container.getEmptyPath();
|
||||
|
||||
// vectors: we will use it as array, here
|
||||
std::vector<BeamEntryTypeEx> last_beams;
|
||||
std::vector<BeamEntryType> next_beams;
|
||||
last_beams.resize(beam_width);
|
||||
// as we skip blank indexes the count is beam_width * len_c
|
||||
next_beams.resize(beam_width * len_c);
|
||||
last_beams[0].entry = empty;
|
||||
last_beams[0].index_as_child = -1;
|
||||
last_beams[0].index_as_parent = -1;
|
||||
last_beams[0].children_count = 0;
|
||||
auto last_beam_size = 1;
|
||||
|
||||
// lookupContainer:
|
||||
// it will keep sorted entries. so we will just move and compare the entry
|
||||
// in each step there will be overlapped cases
|
||||
// the size of overlapped cases in last_beam[0:beam_width]:
|
||||
// as we have beam_width size in each step after sort and pruning
|
||||
// there is at least one item who will not have any parent
|
||||
// and for the rest (beam_width-1) it will check has_parent_in_container() ? 1 : 0
|
||||
// so maximum size of overlapped pairs is beam_width-1
|
||||
|
||||
std::vector<LookUpEntry<Type, IndexType>> lookUp;
|
||||
lookUp.resize(beam_width - 1);
|
||||
|
||||
// additional storage to sort overlapped case by classes
|
||||
std::vector<std::pair<IndexType, int>> child_class_sorter_help;
|
||||
child_class_sorter_help.resize(beam_width - 1);
|
||||
Type norm_offset = static_cast<Type>(0);
|
||||
|
||||
for (uint64_t t = 0; t < len_t; t++) {
|
||||
auto next_beam_size = 0;
|
||||
if (normalize_logits) {
|
||||
norm_offset = softmax_normalization_term<HasElementStride, Type, IndexType>(log_p, len_c, element_stride);
|
||||
}
|
||||
for (auto j = 0; j < last_beam_size; j++) {
|
||||
SequenceNode<IndexType>* seq = last_beams[j].entry.sequence;
|
||||
auto& cur_prob = last_beams[j].entry.prob;
|
||||
// if len(seq) > 0 then
|
||||
const auto log_p_blank = element<HasElementStride>(log_p, blank_index, element_stride);
|
||||
Type blank_prob, non_blank_prob;
|
||||
// log_p[seq->value]
|
||||
non_blank_prob = seq->value != -1
|
||||
? (element<HasElementStride>(log_p, seq->value, element_stride) + cur_prob.non_blank)
|
||||
: negative_infinity<Type>();
|
||||
blank_prob = log_p_blank + cur_prob.total;
|
||||
|
||||
if (normalize_logits) {
|
||||
non_blank_prob = non_blank_prob - norm_offset;
|
||||
blank_prob = blank_prob - norm_offset;
|
||||
}
|
||||
|
||||
auto look_up_beam_index = -1;
|
||||
|
||||
if (last_beams[j].index_as_child != -1) {
|
||||
// check entry
|
||||
look_up_beam_index = lookUp[last_beams[j].index_as_child].next_beam_index;
|
||||
}
|
||||
|
||||
if (look_up_beam_index == -1) {
|
||||
BeamEntryType entry;
|
||||
entry.sequence = seq;
|
||||
entry.prob.blank = blank_prob;
|
||||
entry.prob.non_blank = non_blank_prob;
|
||||
entry.prob.total = log_sum_exp(blank_prob, non_blank_prob);
|
||||
next_beams[next_beam_size] = entry;
|
||||
// map if its overlapped one. in this case just being child is enough
|
||||
if (last_beams[j].index_as_child != -1) {
|
||||
lookUp[last_beams[j].index_as_child].next_beam_index = next_beam_size;
|
||||
}
|
||||
++next_beam_size;
|
||||
} else {
|
||||
// note: here we took as ref &
|
||||
auto& entry_prob = next_beams[look_up_beam_index].prob;
|
||||
entry_prob.blank = log_sum_exp(entry_prob.blank, blank_prob);
|
||||
entry_prob.non_blank = log_sum_exp(entry_prob.non_blank, non_blank_prob);
|
||||
entry_prob.total = log_sum_exp(entry_prob.blank, entry_prob.non_blank);
|
||||
}
|
||||
// check to see if it is overlapped parent
|
||||
auto start_index = last_beams[j].index_as_parent;
|
||||
auto end_index = last_beams[j].index_as_parent + last_beams[j].children_count;
|
||||
|
||||
for (int c = 0; c < static_cast<int>(len_c); c++) {
|
||||
if (c == blank_index) continue;
|
||||
|
||||
const auto prob = element<HasElementStride>(log_p, c, element_stride); // log_p[c];
|
||||
|
||||
non_blank_prob = pr(c, cur_prob, seq, prob);
|
||||
if (normalize_logits) non_blank_prob = non_blank_prob - norm_offset;
|
||||
// extend by new character
|
||||
auto look_up_beam_index_ex = -1;
|
||||
int found_index = -1;
|
||||
|
||||
// get index within array if its that class index
|
||||
if (start_index < end_index && lookUp[start_index].last_c == c) {
|
||||
look_up_beam_index_ex = lookUp[start_index].next_beam_index;
|
||||
|
||||
found_index = start_index;
|
||||
++start_index;
|
||||
}
|
||||
|
||||
if (look_up_beam_index_ex == -1) {
|
||||
BeamEntryType entry;
|
||||
SequenceNode<IndexType>* extended_sequence;
|
||||
if (found_index != -1) {
|
||||
extended_sequence = lookUp[found_index].node;
|
||||
// assing next_beam_index for lookup
|
||||
lookUp[found_index].next_beam_index = next_beam_size;
|
||||
extended_sequence->increaseRef();
|
||||
} else {
|
||||
extended_sequence = sequence_container.extendPath(seq, c);
|
||||
}
|
||||
entry.prob.non_blank = non_blank_prob;
|
||||
entry.prob.total = non_blank_prob;
|
||||
entry.sequence = extended_sequence;
|
||||
next_beams[next_beam_size] = entry;
|
||||
|
||||
++next_beam_size;
|
||||
} else {
|
||||
auto& entry_prob = next_beams[look_up_beam_index_ex].prob;
|
||||
entry_prob.non_blank = log_sum_exp(entry_prob.non_blank, non_blank_prob);
|
||||
entry_prob.total = log_sum_exp(entry_prob.total, non_blank_prob);
|
||||
}
|
||||
} // iteration over classes
|
||||
|
||||
// mark it as extended
|
||||
seq->markAsFullyExtended();
|
||||
|
||||
} // iteration over beams
|
||||
|
||||
log_p += inc_p;
|
||||
|
||||
last_beam_size = std::min(next_beam_size, beam_width);
|
||||
#if !defined(NTH_ELEMENT)
|
||||
// sort next beams to get candidates
|
||||
std::partial_sort(std::begin(next_beams), std::begin(next_beams) + last_beam_size,
|
||||
std::begin(next_beams) + next_beam_size, compare_beam_prob<Type, IndexType>);
|
||||
|
||||
#else
|
||||
std::nth_element(std::begin(next_beams), std::begin(next_beams) + last_beam_size,
|
||||
std::begin(next_beams) + next_beam_size, compare_beam_prob<Type, IndexType>);
|
||||
|
||||
#endif
|
||||
|
||||
if (t < len_t) {
|
||||
// copy top beams
|
||||
for (int j = 0; j < last_beam_size; j++) {
|
||||
last_beams[j].entry = next_beams[j];
|
||||
last_beams[j].index_as_child = -1;
|
||||
last_beams[j].index_as_parent = -1;
|
||||
last_beams[j].children_count = 0;
|
||||
}
|
||||
|
||||
// delete sequences from the sequence_holder to decrease memory
|
||||
for (auto j = beam_width; j < next_beam_size; j++) {
|
||||
sequence_container.remove(next_beams[j].sequence);
|
||||
}
|
||||
|
||||
// check overlapping cases and create lookUp with sorted classes as well
|
||||
int look_up_index = 0;
|
||||
for (auto j = 0; j < last_beam_size; j++) {
|
||||
// if it is not parent node then there is not any need to check
|
||||
if (last_beams[j].entry.sequence->isFullyExtended()) {
|
||||
auto parent_seq = last_beams[j].entry.sequence;
|
||||
int children_count = 0;
|
||||
for (int k = 0; k < last_beam_size; k++) {
|
||||
auto current = last_beams[k].entry.sequence;
|
||||
if (current->prefix == parent_seq) {
|
||||
child_class_sorter_help[children_count].second = k;
|
||||
++children_count;
|
||||
}
|
||||
}
|
||||
|
||||
if (children_count > 0) {
|
||||
// sort by class
|
||||
if (children_count < 2) {
|
||||
//
|
||||
if (children_count > 1 && child_class_sorter_help[0].first > child_class_sorter_help[1].first) {
|
||||
std::swap(child_class_sorter_help[0], child_class_sorter_help[1]);
|
||||
}
|
||||
} else {
|
||||
std::sort(std::begin(child_class_sorter_help), std::begin(child_class_sorter_help) + children_count,
|
||||
[](const std::pair<int, int>& left, const std::pair<int, int>& right) {
|
||||
return left.first < right.first;
|
||||
});
|
||||
}
|
||||
last_beams[j].index_as_parent = look_up_index;
|
||||
last_beams[j].children_count = children_count;
|
||||
|
||||
for (int l = 0; l < children_count; l++) {
|
||||
int c = child_class_sorter_help[l].first;
|
||||
int k = child_class_sorter_help[l].second;
|
||||
// std::cout << c <<" , " << k << std::endl;
|
||||
last_beams[k].index_as_child = look_up_index;
|
||||
auto seq = last_beams[k].entry.sequence;
|
||||
lookUp[look_up_index].last_c = c;
|
||||
lookUp[look_up_index].node = seq;
|
||||
lookUp[look_up_index].next_beam_index = -1;
|
||||
// next one
|
||||
++look_up_index;
|
||||
}
|
||||
} // add sorted lookUps
|
||||
}
|
||||
} // overlap_direction identified to speed up lookUp
|
||||
}
|
||||
|
||||
} // iterate over t
|
||||
#if defined(NTH_ELEMENT)
|
||||
// use sort for n elements as only nth_element was used
|
||||
std::sort(std::begin(next_beams), std::begin(next_beams) + last_beam_size, compare_beam_prob<Type, IndexType>);
|
||||
#endif
|
||||
// store nbest results
|
||||
if (nbest_len <= last_beam_size) {
|
||||
for (int j = 0; j < nbest_len; j++) {
|
||||
auto top = next_beams[j];
|
||||
auto result_vector = SequenceContainer<IndexType>::getSequence(top.sequence, len_t);
|
||||
const auto seq_size = result_vector.size();
|
||||
|
||||
result_prob[j] = top.prob.total;
|
||||
result_seq_length[j] = seq_size;
|
||||
// copy sequence
|
||||
for (size_t s = 0; s < seq_size; s++) {
|
||||
result_sequence[s] = result_vector[s];
|
||||
}
|
||||
|
||||
result_sequence += inc_res_seq;
|
||||
}
|
||||
} else {
|
||||
for (int j = 0; j < nbest_len; j++) {
|
||||
result_prob[j] = negative_infinity<Type>();
|
||||
result_seq_length[j] = 0;
|
||||
;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
template <typename Type, typename IndexType = int>
|
||||
void beamSearch_(NDArray& logit, NDArray& sequence_length, NDArray& result_sequences, NDArray& result_probs,
|
||||
NDArray& result_sequences_length, int blank_index, int beam_width, int nbest_len,
|
||||
bool normalize_logits) {
|
||||
const auto shapes = logit.shapeOf();
|
||||
const auto strides = logit.stridesOf();
|
||||
const auto rank = logit.rankOf();
|
||||
|
||||
uint64_t element_stride_t = 1;
|
||||
|
||||
// checks before
|
||||
if (rank < 2) return;
|
||||
auto batch_len = rank > 2 ? shapes[0] : 1;
|
||||
auto max_len_t = shapes[rank - 2];
|
||||
auto len_c = shapes[rank - 1];
|
||||
|
||||
if (len_c < 1 || max_len_t < 1) return;
|
||||
// defaulting blankIndex to the last class if its incorrect or -1
|
||||
if (blank_index > len_c || blank_index < 0) blank_index = static_cast<int>(len_c) - 1;
|
||||
|
||||
// strides
|
||||
auto batch_stride = rank > 2 ? strides[0] : 0;
|
||||
auto inc_p = strides[rank - 2];
|
||||
auto element_stride = logit.stridesOf()[rank - 1];
|
||||
|
||||
#if defined(ASSERT_INNER)
|
||||
// result_probs should be [batch_len, nbest_len]
|
||||
assert(result_probs.ews() == 1 && result_probs.rankOf() == 2 && result_probs.shapeOf()[0] == batch_len &&
|
||||
result_probs.shapeOf()[1] == nbest_len);
|
||||
// result sequence should be [batch_len, nbest_len, max_len_t]
|
||||
assert(result_sequences.ews() == 1 && result_sequences.rankOf() == 3 && result_sequences.shapeOf()[0] == batch_len &&
|
||||
result_sequences.shapeOf()[1] == nbest_len && result_sequences.shapeOf()[2] == max_len_t);
|
||||
#endif
|
||||
// as ctcBeam search runs on Cpu we should make NdArray buffers available on the host side as well
|
||||
NDArray::preparePrimaryUse({&result_sequences, &result_probs, &result_sequences_length}, {&sequence_length, &logit});
|
||||
|
||||
auto logits_ptr = logit.bufferAsT<Type>();
|
||||
auto result_seq_ptr = result_sequences.bufferAsT<IndexType>();
|
||||
auto result_probs_ptr = result_probs.bufferAsT<Type>();
|
||||
auto result_seq_length_ptr = result_sequences_length.bufferAsT<IndexType>();
|
||||
const IndexType* len_t_ptr = nullptr;
|
||||
if (sequence_length.rankOf() == 1 && sequence_length.shapeOf()[0] == batch_len) {
|
||||
len_t_ptr = sequence_length.bufferAsT<IndexType>();
|
||||
element_stride_t = sequence_length.stridesOf()[0];
|
||||
}
|
||||
const auto batch_stride_res = result_sequences.stridesOf()[0];
|
||||
const auto inc_res = result_sequences.stridesOf()[1];
|
||||
const auto batch_stride_res_prob = result_probs.stridesOf()[0];
|
||||
const auto batch_stride_res_seq_length = result_sequences_length.stridesOf()[0];
|
||||
auto func = [max_len_t, len_c, batch_stride, inc_p, element_stride, element_stride_t, logits_ptr, len_t_ptr,
|
||||
blank_index, beam_width, normalize_logits, nbest_len, result_seq_ptr, result_seq_length_ptr,
|
||||
result_probs_ptr, batch_stride_res, inc_res, batch_stride_res_prob, batch_stride_res_seq_length](
|
||||
uint64_t thread_id, int64_t start, int64_t stop, int64_t increment) -> void {
|
||||
auto ptr = logits_ptr + start * batch_stride;
|
||||
|
||||
if (element_stride == 1) {
|
||||
// choose ews one
|
||||
for (auto b = start; b < stop; b += increment) {
|
||||
auto prob_ptr = &(result_probs_ptr[b * batch_stride_res_prob]);
|
||||
auto seq_length_ptr = &(result_seq_length_ptr[b * batch_stride_res_seq_length]);
|
||||
auto seq_ptr = &(result_seq_ptr[b * batch_stride_res]);
|
||||
|
||||
auto len_t = len_t_ptr ? len_t_ptr[b * element_stride_t] : max_len_t;
|
||||
inner_beam_search<false, Type, IndexType>(ptr, inc_p, seq_ptr, inc_res, max_len_t, prob_ptr, seq_length_ptr,
|
||||
len_t, len_c, blank_index, beam_width, nbest_len, normalize_logits);
|
||||
|
||||
ptr += batch_stride;
|
||||
}
|
||||
} else {
|
||||
// element with stride case
|
||||
for (auto b = start; b < stop; b += increment) {
|
||||
auto prob_ptr = &(result_probs_ptr[b * batch_stride_res_prob]);
|
||||
auto seq_length_ptr = &(result_seq_length_ptr[b * batch_stride_res_seq_length]);
|
||||
auto seq_ptr = &(result_seq_ptr[b * batch_stride_res]);
|
||||
|
||||
auto len_t = len_t_ptr ? len_t_ptr[b * element_stride_t] : max_len_t;
|
||||
inner_beam_search<false, Type, IndexType>(ptr, inc_p, seq_ptr, inc_res, max_len_t, prob_ptr, seq_length_ptr,
|
||||
len_t, len_c, blank_index, beam_width, nbest_len, normalize_logits,
|
||||
element_stride);
|
||||
|
||||
ptr += batch_stride;
|
||||
}
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, batch_len, 1);
|
||||
|
||||
NDArray::registerPrimaryUse({&result_sequences, &result_probs, &result_sequences_length}, {&sequence_length, &logit});
|
||||
return;
|
||||
}
|
||||
|
||||
void beamSearch(NDArray& logit, NDArray& sequence_length, NDArray& result_sequences, NDArray& result_probs,
|
||||
NDArray& result_sequences_length, int blank_index, int beam_width, int nbest_len,
|
||||
bool normalize_logits = true) {
|
||||
auto logitDType = logit.dataType();
|
||||
auto resSeqDType = result_sequences.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(logit.dataType(), result_sequences.dataType(), beamSearch_,
|
||||
(logit, sequence_length, result_sequences, result_probs, result_sequences_length, blank_index,
|
||||
beam_width, nbest_len, normalize_logits),
|
||||
SD_FLOAT_TYPES, SD_INDEXING_TYPES);
|
||||
}
|
||||
|
||||
BUILD_DOUBLE_TEMPLATE( void beamSearch_,
|
||||
(NDArray& logit, NDArray& sequence_length, NDArray& result_sequences,
|
||||
NDArray& result_probs, NDArray& result_sequences_length, int blank_index, int beam_width,
|
||||
int nbest_len, bool normalize_logits),
|
||||
SD_FLOAT_TYPES, SD_INDEXING_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,669 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 15.02.2018, Alex Black
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
|
||||
#if NOT_EXCLUDED(OP_gru)
|
||||
|
||||
// implementation of gated Recurrent Unit cell
|
||||
// (cf. https://arxiv.org/abs/1406.1078).
|
||||
// Kyunghyun Cho, Bart van Merrienboer, Caglar Gulcehre, Dzmitry Bahdanau, Fethi Bougares, Holger Schwenk, Yoshua Bengio
|
||||
// "Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation"
|
||||
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/helpers/gru.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void gruCell(sd::LaunchContext* context, NDArray* x, NDArray* hI, NDArray* W, NDArray* Wc,
|
||||
NDArray* b, NDArray* bc, NDArray* r, NDArray* u, NDArray* c, NDArray* h) {
|
||||
// Inputs:
|
||||
// x input [bS, nIn], nIn - input size
|
||||
// hI previous cell output [bS, nOut], that is at previous time step t-1, nOut - number of units
|
||||
// W RU weights - [nIn+nOut, 2*nOut] - reset and update gates
|
||||
// Wc C weights - [nIn+nOut, nOut] - cell gate
|
||||
// b r and u biases, [2*nOut] - reset and update gates
|
||||
// bc c biases, [nOut] - cell gate
|
||||
|
||||
// Outputs:
|
||||
// r Reset gate output [bS, nOut]
|
||||
// u Update gate output [bS, nOut]
|
||||
// c Cell gate output [bS, nOut]
|
||||
// h current cell output [bS, nOut]
|
||||
|
||||
/***************************************************************************************/
|
||||
/************************ THIS IS NOT OPTIMIZED CODE ***********************************/
|
||||
/** however it is more math-friendly and convenient for backprop formulas derivation) **/
|
||||
|
||||
const int bS = x->sizeAt(0);
|
||||
const int nIn = x->sizeAt(1);
|
||||
const int nOut = hI->sizeAt(1);
|
||||
|
||||
NDArray *Wrx = (*W)({0, nIn, 0, nOut}); // [nIn, nOut]
|
||||
NDArray *Wux = (*W)({0, nIn, nOut, 2 * nOut}); // [nIn, nOut]
|
||||
NDArray *Wrh = (*W)({nIn, nIn + nOut, 0, nOut}); // [nOut, nOut]
|
||||
NDArray *Wuh = (*W)({nIn, nIn + nOut, nOut, 2 * nOut}); // [nOut, nOut]
|
||||
|
||||
NDArray *Wcx = (*Wc)({0, nIn, 0, 0}); // reset cell weights [nIn, nOut]
|
||||
NDArray *Wch = (*Wc)({nIn, nIn + nOut, 0, 0}); // updates cell weights [nOut, nOut]
|
||||
|
||||
NDArray *br = (*b)({0, nOut}); // [nOut]
|
||||
NDArray *bu = (*b)({nOut, 2 * nOut}); // [nOut]
|
||||
|
||||
// × means matrix multiplication
|
||||
// * means element-wise product or so called Hadamard product
|
||||
|
||||
// r = sigmoid(x × Wrx + hI × Wrh + br)
|
||||
auto xWrx = mmul(*x, *Wrx);
|
||||
auto hIWrh = mmul(*hI, *Wrh);
|
||||
auto* sum1 = *xWrx + *hIWrh;
|
||||
auto* rAssign = (*sum1) + (*br);
|
||||
delete sum1;
|
||||
delete hIWrh;
|
||||
r->assign(rAssign); // [bS, nIn] × [nIn, nOut] + [bS, nOut] × [nOut, nOut] + [nOut] = [bS, nOut]
|
||||
delete rAssign;
|
||||
r->applyTransform(transform::Sigmoid, r);
|
||||
|
||||
// u = sigmoid(x × Wux + hI × Wuh + bu)
|
||||
auto xWux = mmul(*x, *Wux);
|
||||
auto hIWuh = mmul(*hI, *Wuh);
|
||||
auto* sum2 = *xWux + *hIWuh;
|
||||
auto* uAssign = (*sum2) + (*bu);
|
||||
delete sum2;
|
||||
delete xWux;
|
||||
u->assign(uAssign); // [bS, nIn] × [nIn, nOut] + [bS, nOut] × [nOut, nOut] + [nOut] = [bS, nOut]
|
||||
delete uAssign;
|
||||
u->applyTransform(transform::Sigmoid, u);
|
||||
|
||||
// c = tanh(x × Wcx + (r * hI) × Wch + bc)
|
||||
auto* rTimesHi = (*r) * (*hI);
|
||||
auto xWcx = mmul(*x, *Wcx);
|
||||
auto rTimesHiWch = mmul(*rTimesHi, *Wch);
|
||||
delete rTimesHi;
|
||||
auto* sum3 = *xWcx + *rTimesHiWch;
|
||||
auto* cAssign = (*sum3) + (*bc);
|
||||
delete sum3;
|
||||
delete xWcx;
|
||||
delete rTimesHiWch;
|
||||
c->assign(cAssign); // [bS, nIn] × [nIn, nOut] + [bS, nOut] × [nOut, nOut] + [nOut] = [bS, nOut]
|
||||
delete cAssign;
|
||||
c->applyTransform(transform::Tanh, c);
|
||||
|
||||
// h = (1 - u) * c + u * hI
|
||||
auto* uTimesHi = (*u) * (*hI);
|
||||
auto* oneMinusU = 1.f - (*u);
|
||||
auto* oneMinusUTimesC = (*oneMinusU) * (*c);
|
||||
delete oneMinusU;
|
||||
auto* hAssign = (*uTimesHi) + (*oneMinusUTimesC);
|
||||
delete uTimesHi;
|
||||
delete oneMinusUTimesC;
|
||||
h->assign(hAssign);
|
||||
delete hAssign;
|
||||
|
||||
delete Wrx;
|
||||
delete Wux;
|
||||
delete Wrh;
|
||||
delete Wuh;
|
||||
delete Wcx;
|
||||
delete Wch;
|
||||
delete br;
|
||||
delete bu;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void gruCell(NDArray* x, NDArray* hI, NDArray* Wx, NDArray* Wh, NDArray* b,
|
||||
NDArray* gates, NDArray* h, bool linearBeforeReset) {
|
||||
|
||||
if(linearBeforeReset) {
|
||||
THROW_EXCEPTION("GRU: Linear before reset not implemented. Please set to false.");
|
||||
}
|
||||
|
||||
// Inputs:
|
||||
// x input [bS, nIn]
|
||||
// hI previous cell output [bS, nOut], that is at previous time step t-1
|
||||
// Wx weights for x - [nIn, 3*nOut]
|
||||
// Wh weights for h - [nOut, 3*nOut]
|
||||
// b biases [3*nOut]
|
||||
|
||||
// 3*nOut means following sequence: reset, update, cell
|
||||
|
||||
// Outputs:
|
||||
// gates [bS, 3*nOut] = reset gate [bS, nOut] + update gate [bS, nOut] + cell gate [bS, nOut]
|
||||
// h current cell output [bS, nOut]
|
||||
|
||||
// formulas:
|
||||
// zr = x × Wxr + hI × Whr + br
|
||||
// zu = x × Wxu + hI × Whu + bu
|
||||
// r = sigmoid(zr)
|
||||
// u = sigmoid(zu)
|
||||
// zc = x × Wxc + (r * hI) × Whc + bc
|
||||
// c = tanh(zc)
|
||||
// h = (1-u)*c + u*hI
|
||||
|
||||
const int bS = x->sizeAt(0);
|
||||
const int nIn = x->sizeAt(1);
|
||||
const int nOut = hI->sizeAt(1);
|
||||
|
||||
NDArray *gatesULike = gates->ulike();
|
||||
NDArray temp = *gatesULike;
|
||||
MmulHelper::mmul(x, Wx, &temp); // [bS, nIn] × [nIn, 3*nOut] = [bS, 3*nOut]
|
||||
temp += *b;
|
||||
|
||||
MmulHelper::mmul(hI, Wh, gates); // [bS, nOut] × [nOut, 3*nOut] = [bS, 3*nOut]
|
||||
|
||||
NDArray *ru = (*gates)({0, 0, 0, 2 * nOut}); // [bS, 2*nOut]
|
||||
|
||||
NDArray *r = (*gates)({0, 0, 0, nOut}); // [bS, nOut]
|
||||
NDArray *u = (*gates)({0, 0, nOut, 2 * nOut}); // [bS, nOut]
|
||||
NDArray *c = (*gates)({0, 0, 2 * nOut, 3 * nOut}); // [bS, nOut]
|
||||
|
||||
NDArray *tempView1 = temp({0, 0, 0, 2 * nOut});
|
||||
NDArray *tempView2 = temp({0, 0, 2 * nOut, 3 * nOut});
|
||||
|
||||
// reset and update gates
|
||||
*ru += *tempView1;
|
||||
ru->applyTransform(transform::Sigmoid, ru);
|
||||
|
||||
// cell gate
|
||||
auto* cTimesR = (*c) * (*r);
|
||||
auto* cAssign = (*cTimesR) + (*tempView2);
|
||||
delete cTimesR;
|
||||
c->assign(cAssign);
|
||||
delete cAssign;
|
||||
c->applyTransform(transform::Tanh, c);
|
||||
|
||||
// h = (1-u)*c + u*hI
|
||||
auto* uTimesHi = (*u) * (*hI);
|
||||
auto* oneMinusU = 1.f - (*u);
|
||||
auto* oneMinusUTimesC = (*oneMinusU) * (*c);
|
||||
delete oneMinusU;
|
||||
auto* hAssign = (*uTimesHi) + (*oneMinusUTimesC);
|
||||
delete uTimesHi;
|
||||
delete oneMinusUTimesC;
|
||||
h->assign(hAssign);
|
||||
delete hAssign;
|
||||
|
||||
delete gatesULike;
|
||||
delete ru;
|
||||
delete r;
|
||||
delete u;
|
||||
delete c;
|
||||
delete tempView1;
|
||||
delete tempView2;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void gruTimeLoop(sd::LaunchContext* context, NDArray* x, NDArray* hI, NDArray* Wx, NDArray* Wh,
|
||||
NDArray* b, NDArray* h, bool linearBeforeReset) {
|
||||
// sL means time steps
|
||||
|
||||
// x input [sL, bS, nIn]
|
||||
// hI initial cell output (at time step = 0) [bS, nOut]
|
||||
// Wx input-to-hidden weights, [nIn, 3*nOut]
|
||||
// Wh hidden-to-hidden weights, [nOut, 3*nOut]
|
||||
// b biases, [3*nOut]
|
||||
|
||||
// h cell outputs at each time step [sL, bS, nOut]
|
||||
|
||||
const int sL = x->sizeAt(0);
|
||||
const int bS = x->sizeAt(1);
|
||||
const int nOut = hI->sizeAt(1);
|
||||
|
||||
std::vector<LongType> shape = {bS, 3 * nOut};
|
||||
NDArray gates(h->ordering(), shape, h->dataType(), context);
|
||||
|
||||
auto xSet = x->allTensorsAlongDimension({1, 2}); // sub-arrays with shape [bS, nIn]
|
||||
auto hSet = h->allTensorsAlongDimension({1, 2}); // sub-arrays with shape [bS, nOut]
|
||||
|
||||
// time loop
|
||||
for (int t = 0; t < sL; ++t) {
|
||||
gruCell(xSet.at(t), t == 0 ? hI : hSet.at(t - 1), Wx, Wh, b, &gates, hSet.at(t), linearBeforeReset);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void gruCellBp(sd::LaunchContext* context, NDArray* x, NDArray* hLast, NDArray* W, NDArray* Wc,
|
||||
NDArray* b, NDArray* bc, NDArray* dLdr, NDArray* dLdu, NDArray* dLdc,
|
||||
NDArray* dLdh, NDArray* dLdx, NDArray* dLdhLast, NDArray* dLdW, NDArray* dLdWc, NDArray* dLdb,
|
||||
NDArray* dLdbc) {
|
||||
// Inputs:
|
||||
// x input [bS, iS]
|
||||
// hLast previous cell output [bS, nU], that is at previous time step t-1
|
||||
// W weights - [iS+nU, 2*nU] - reset and update gates
|
||||
// Wc C weights - [iS+nU, nU] - cell gate
|
||||
// b r and u biases, [2*nU] - reset and update gates
|
||||
// bc c biases, [nU] - cell gate
|
||||
// dLdr gradient wrt reset gate, [bS, nU]
|
||||
// dLdu gradient wrt update gate, [bS, nU]
|
||||
// dLdc gradient wrt cell state, [bS, nU]
|
||||
// dLdh gradient wrt current cell output, [bS, nU]
|
||||
|
||||
// Outputs:
|
||||
// dLdx gradient wrt x, [bS, iS],
|
||||
// dLdhLast gradient wrt hLast, [bS, nU]
|
||||
// dLdW gradient wrt W, [iS+nU, 2*nU]
|
||||
// dLdWc gradient wrt Wc, [iS+nU, nU]
|
||||
// dLdb gradient wrt bru [2*nU]
|
||||
// dLdbc gradient wrt bc [nU]
|
||||
|
||||
// * means element-wise product or so called Hadamard product
|
||||
// × means matrix multiplication
|
||||
|
||||
/************************************************************************************************/
|
||||
/******************************* THIS IS NOT OPTIMIZED CODE *************************************/
|
||||
/*** aim is to have math-readable code in order to keep track of backprop formulas derivation ***/
|
||||
|
||||
const int bS = x->sizeAt(0);
|
||||
const int iS = x->sizeAt(1);
|
||||
const int nU = hLast->sizeAt(1);
|
||||
|
||||
NDArray *xT = x->transpose(); // [iS, bS]
|
||||
NDArray *hLastT = hLast->transpose(); // [nU, bS]
|
||||
|
||||
NDArray *Wrx = (*W)({0, iS, 0, nU}); // [iS, nU]
|
||||
NDArray *Wux = (*W)({0, iS, nU, 2 * nU}); // [iS, nU]
|
||||
NDArray *Wrh = (*W)({iS, iS + nU, 0, nU}); // [nU, nU]
|
||||
NDArray *Wuh = (*W)({iS, iS + nU, nU, 2 * nU}); // [nU, nU]
|
||||
|
||||
NDArray *Wcx = (*Wc)({0, iS, 0, 0}); // reset cell weights [iS, nU]
|
||||
NDArray *Wch = (*Wc)({iS, iS + nU, 0, 0}); // updates cell weights [nU, nU]
|
||||
|
||||
NDArray *br = (*b)({0, nU}); // [nU]
|
||||
NDArray *bu = (*b)({nU, 2 * nU}); // [nU]
|
||||
|
||||
NDArray *WrxT = Wrx->transpose(); // [nU, iS]
|
||||
NDArray *WuxT = Wux->transpose(); // [nU, iS]
|
||||
NDArray *WrhT = Wrh->transpose(); // [nU, nU]
|
||||
NDArray *WuhT = Wuh->transpose(); // [nU, nU]
|
||||
|
||||
NDArray *WcxT = Wcx->transpose(); // [nU, iS]
|
||||
NDArray *WchT = Wch->transpose(); // [nU, nU]
|
||||
|
||||
NDArray *dLdWrx = (*dLdW)({0, iS, 0, nU}); // [iS, nU]
|
||||
NDArray *dLdWux = (*dLdW)({0, iS, nU, 2 * nU}); // [iS, nU]
|
||||
NDArray *dLdWrh = (*dLdW)({iS, iS + nU, 0, nU}); // [nU, nU]
|
||||
NDArray *dLdWuh = (*dLdW)({iS, iS + nU, nU, 2 * nU}); // [nU, nU]
|
||||
|
||||
NDArray *dLdWcx = (*dLdWc)({0, iS, 0, 0}); // [iS, nU]
|
||||
NDArray *dLdWch = (*dLdWc)({iS, iS + nU, 0, 0}); // [nU, nU]
|
||||
|
||||
NDArray *dLdbr = (*dLdb)({0, nU}); // [nU]
|
||||
NDArray *dLdbu = (*dLdb)({nU, 2 * nU}); // [nU]
|
||||
|
||||
// ***** feed forward step ***** //
|
||||
|
||||
// r = sigmoid(x × Wrx + hLast × Wrh + br)
|
||||
auto xWrx = mmul(*x, *Wrx);
|
||||
auto hLastWrh = mmul(*hLast, *Wrh);
|
||||
auto* sum1 = *xWrx + *hLastWrh;
|
||||
auto* rTemp = (*sum1) + (*br);
|
||||
delete sum1;
|
||||
NDArray r = *rTemp; // [bS, iS] × [iS, nU] + [bS, nU] × [nU, nU] + [nU] = [bS, nU]
|
||||
delete rTemp;
|
||||
r.applyTransform(transform::Sigmoid, &r);
|
||||
|
||||
// u = sigmoid(x × Wux + hLast × Wuh + bu)
|
||||
auto xWux = mmul(*x, *Wux);
|
||||
auto hLastWuh = mmul(*hLast, *Wuh);
|
||||
auto* sum2 = *xWux + *hLastWuh;
|
||||
auto* uTemp = (*sum2) + (*bu);
|
||||
delete sum2;
|
||||
NDArray u = *uTemp; // [bS, iS] × [iS, nU] + [bS, nU] × [nU, nU] + [nU] = [bS, nU]
|
||||
delete uTemp;
|
||||
delete xWux;
|
||||
u.applyTransform(transform::Sigmoid, &u);
|
||||
|
||||
// c = tanh(x × Wcx + (r * hLast) × Wch + bc)
|
||||
auto* rTimesHLast2 = r * (*hLast);
|
||||
auto xWcx = mmul(*x, *Wcx);
|
||||
auto rTimesHLast2Wch = mmul(*rTimesHLast2, *Wch);
|
||||
delete rTimesHLast2;
|
||||
auto* sum3 = *xWcx + *rTimesHLast2Wch;
|
||||
auto* cTemp = (*sum3) + (*bc);
|
||||
delete sum3;
|
||||
delete xWcx;
|
||||
delete rTimesHLast2Wch;
|
||||
NDArray c = *cTemp; // [bS, iS] × [iS, nU] + [bS, nU] × [nU, nU] + [nU] = [bS, nU]
|
||||
delete cTemp;
|
||||
c.applyTransform(transform::Tanh, &c);
|
||||
|
||||
// h = (1 - u) * c + u * hPrev
|
||||
|
||||
// ***** back prop step ***** //
|
||||
|
||||
auto* hLastMinusC = (*hLast) - c;
|
||||
auto* oneMinusU = 1.f - u;
|
||||
auto* dudZu = u * (*oneMinusU);
|
||||
delete oneMinusU;
|
||||
auto* oneMinusR = 1.f - r;
|
||||
auto* drdZr = r * (*oneMinusR);
|
||||
delete oneMinusR;
|
||||
auto* cSquared = c * c;
|
||||
auto* oneMinusCSquared = 1.f - (*cSquared);
|
||||
delete cSquared;
|
||||
auto dcdZc = *oneMinusCSquared;
|
||||
delete oneMinusCSquared;
|
||||
auto* dLdZc = (*dLdc) * dcdZc;
|
||||
auto* dLdZu = (*dLdu) * (*dudZu);
|
||||
delete dudZu;
|
||||
auto* dLdZr = (*dLdr) * (*drdZr);
|
||||
delete drdZr;
|
||||
|
||||
NDArray *dhdc = 1.f - u; // [bS, nU]
|
||||
NDArray dhdu = *hLastMinusC; // [bS, nU]
|
||||
delete hLastMinusC;
|
||||
|
||||
// dLdx = dLdZu × WuxT + dLdZc × WcxT + dLdZr × WrxT
|
||||
auto dLdZuWuxT = mmul(*dLdZu, *WuxT);
|
||||
auto dLdZcWcxT = mmul(*dLdZc, *WcxT);
|
||||
auto dLdZrWrxT = mmul(*dLdZr, *WrxT);
|
||||
auto* temp1 = *dLdZuWuxT + *dLdZcWcxT;
|
||||
auto* dLdxTemp = (*temp1) + *dLdZrWrxT;
|
||||
delete temp1;
|
||||
|
||||
delete dLdZuWuxT;
|
||||
delete dLdZcWcxT;
|
||||
delete dLdZrWrxT;
|
||||
dLdx->assign(dLdxTemp); // [bS, iS]
|
||||
delete dLdxTemp;
|
||||
|
||||
// dldZTimeR = dLdZc * r
|
||||
auto* dldZTimeR = (*dLdZc) * r;
|
||||
|
||||
// dLdhLast = dLdh * u + dLdZu × WuhT + dldZTimeR × WchT + dLdZr × WrhT
|
||||
auto* dLdhTimesU = (*dLdh) * u;
|
||||
auto dLdZuWuhT = mmul(*dLdZu, *WuhT);
|
||||
auto dldZTimeRWchT = mmul(*dldZTimeR, *WchT);
|
||||
auto dLdZrWrhT = mmul(*dLdZr, *WrhT);
|
||||
auto* temp2 = (*dLdhTimesU) + *dLdZuWuhT;
|
||||
delete dLdhTimesU;
|
||||
delete dLdZuWuhT;
|
||||
auto* temp3 = (*temp2) + *dldZTimeRWchT;
|
||||
delete temp2;
|
||||
delete dldZTimeRWchT;
|
||||
auto* dLdhLastTemp = (*temp3) + *dLdZrWrhT;
|
||||
delete temp3;
|
||||
delete dLdZrWrhT;
|
||||
dLdhLast->assign(dLdhLastTemp); // [bS, nU]
|
||||
delete dLdhLastTemp;
|
||||
|
||||
// dLdWrx = xT × dLdZr
|
||||
auto dLdWrxTemp = mmul(*xT, *dLdZr);
|
||||
dLdWrx->assign(dLdWrxTemp); // [iS, bS] × [bS, nU] = [iS, nU]
|
||||
delete dLdWrxTemp;
|
||||
// dLdWrh = hLastT × dLdZr
|
||||
auto dLdWrhTemp = mmul(*hLastT, *dLdZr);
|
||||
dLdWrh->assign(dLdWrhTemp); // [nU, bS] × [bS, nU] = [nU, nU]
|
||||
delete dLdWrhTemp;
|
||||
// dLdWux = xT × dLdZu
|
||||
auto dLdWuxTemp = mmul(*xT, *dLdZu);
|
||||
dLdWux->assign(dLdWuxTemp); // [iS, bS] × [bS, nU] = [iS, nU]
|
||||
delete dLdWuxTemp;
|
||||
// dLdWuh = hLastT × dLdZu
|
||||
auto dLdWuhTemp = mmul(*hLastT, *dLdZu);
|
||||
dLdWuh->assign(dLdWuhTemp); // [nU, bS] × [bS, nU] = [nU, nU]
|
||||
delete dLdWuhTemp;
|
||||
// dLdWcx = xT × dLdZc
|
||||
auto dLdWcxTemp = mmul(*xT, *dLdZc);
|
||||
dLdWcx->assign(dLdWcxTemp); // [iS, bS] × [bS, nU] = [iS, nU]
|
||||
delete dLdWcxTemp;
|
||||
// dLdWch = (r * hLast)T × dLdZc
|
||||
auto* rTimesHLast = r * (*hLast);
|
||||
NDArray* rTimesHLastT = rTimesHLast->transpose();
|
||||
delete rTimesHLast;
|
||||
auto dLdWchTemp = mmul(*rTimesHLastT, *dLdZc);
|
||||
dLdWch->assign(dLdWchTemp); // [nU, bS] × [bS, nU] = [nU, nU]
|
||||
delete dLdWchTemp;
|
||||
// Calculate reduction for bias gradients
|
||||
std::vector<sd::LongType> zeroVec = {0};
|
||||
auto* dLdbrTemp = dLdZr->reduceAlongDimension(reduce::Sum, &zeroVec);
|
||||
dLdbr->assign(dLdbrTemp); // [nU]
|
||||
delete dLdbrTemp;
|
||||
|
||||
auto* dLdbuTemp = dLdZu->reduceAlongDimension(reduce::Sum, &zeroVec);
|
||||
dLdbu->assign(dLdbuTemp); // [nU]
|
||||
delete dLdbuTemp;
|
||||
|
||||
auto* dLdbcTemp = dLdZc->reduceAlongDimension(reduce::Sum, &zeroVec);
|
||||
dLdbc->assign(dLdbcTemp); // [nU]
|
||||
delete dLdbcTemp;
|
||||
|
||||
delete dhdc;
|
||||
delete dLdZc;
|
||||
delete dLdZu;
|
||||
delete dLdZr;
|
||||
delete dldZTimeR;
|
||||
delete Wrx;
|
||||
delete Wux;
|
||||
delete Wrh;
|
||||
delete Wuh;
|
||||
delete Wcx;
|
||||
delete Wch;
|
||||
delete br;
|
||||
delete bu;
|
||||
delete WrxT;
|
||||
delete WuxT;
|
||||
delete WrhT;
|
||||
delete WuhT;
|
||||
delete WcxT;
|
||||
delete WchT;
|
||||
delete dLdWrx;
|
||||
delete dLdWux;
|
||||
delete dLdWrh;
|
||||
delete dLdWuh;
|
||||
delete dLdWcx;
|
||||
delete dLdWch;
|
||||
delete dLdbr;
|
||||
delete dLdbu;
|
||||
delete xT;
|
||||
delete hLastT;
|
||||
delete rTimesHLastT;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void gruCellBp(sd::LaunchContext* context, NDArray* x, NDArray* hI, NDArray* Wx, NDArray* Wh,
|
||||
NDArray* b, NDArray* dLdh, NDArray* gates, NDArray* dLdx, NDArray* dLdhI,
|
||||
NDArray* dLdWx, NDArray* dLdWh, NDArray* dLdb) {
|
||||
// Inputs:
|
||||
// x input [bS, nIn]
|
||||
// hI previous cell output [bS, nOut], that nIn at previous time step t-1
|
||||
// Wx input-to-hidden weights - [nIn, 3*nOut]
|
||||
// Wh hidden-to-hidden weights - [nOut, 3*nOut]
|
||||
// b biases, [3*nOut] - reset and update gates
|
||||
// dLdh gradient vs. ff output, [bS, nOut]
|
||||
|
||||
// Outputs:
|
||||
// dLdx gradient vs. x, [bS, nIn],
|
||||
// dLdhI gradient vs. hI, [bS, nOut]
|
||||
// dLdWx gradient vs. W, [nIn, 3*nOut]
|
||||
// dLdWh gradient vs. Wc, [nOut, 3*nOut]
|
||||
// dLdb gradient vs. b [3*nOut]
|
||||
|
||||
// 3*nOut means following sequence: reset, update, cell
|
||||
|
||||
// * means element-wise product or so called Hadamard product
|
||||
// × means matrix multiplication
|
||||
|
||||
const int nOut = hI->sizeAt(1);
|
||||
|
||||
NDArray *gatesULike = gates->ulike();
|
||||
NDArray dLdz = *gatesULike; // [bS, 3*nOut]
|
||||
|
||||
NDArray *dLdzru = dLdz({0, 0, 0, 2 * nOut}); // [bS, 2*nOut]
|
||||
|
||||
NDArray *dLdzr = dLdz({0, 0, 0, nOut}); // [bS, nOut]
|
||||
NDArray *dLdzu = dLdz({0, 0, nOut, 2 * nOut}); // [bS, nOut]
|
||||
NDArray *dLdzc = dLdz({0, 0, 2 * nOut, 3 * nOut}); // [bS, nOut]
|
||||
|
||||
NDArray *r = (*gates)({0, 0, 0, nOut}); // [bS, nOut]
|
||||
NDArray *u = (*gates)({0, 0, nOut, 2 * nOut}); // [bS, nOut]
|
||||
NDArray *c = (*gates)({0, 0, 2 * nOut, 3 * nOut}); // [bS, nOut]
|
||||
|
||||
NDArray *WhView = (*Wh)({0, 0, 2 * nOut, 3 * nOut});
|
||||
NDArray *WhcT = WhView->transpose();
|
||||
|
||||
if (dLdh) *dLdhI += *dLdh;
|
||||
|
||||
auto* oneMinusU = 1 - (*u); // [bS, nOut]
|
||||
|
||||
// dLdzc = dLdhI * (1-u) * (1-c²)
|
||||
auto* cSquared = (*c) * (*c);
|
||||
auto* oneMinusCSquared = 1.f - (*cSquared);
|
||||
delete cSquared;
|
||||
auto* temp1 = (*dLdhI) * (*oneMinusU);
|
||||
auto* dLdzcTemp = (*temp1) * (*oneMinusCSquared);
|
||||
delete temp1;
|
||||
delete oneMinusCSquared;
|
||||
dLdzc->assign(dLdzcTemp); // [bS, nOut]
|
||||
delete dLdzcTemp;
|
||||
|
||||
// dLdzu = dLdhI * (hI - c) * u * (1-u)
|
||||
auto* hIMinusC = (*hI) - (*c);
|
||||
auto* uTimesOneMinusU = (*u) * (*oneMinusU);
|
||||
auto* temp2 = (*dLdhI) * (*hIMinusC);
|
||||
auto* dLdzuTemp = (*temp2) * (*uTimesOneMinusU);
|
||||
delete temp2;
|
||||
delete hIMinusC;
|
||||
delete uTimesOneMinusU;
|
||||
dLdzu->assign(dLdzuTemp); // [bS, nOut]
|
||||
delete dLdzuTemp;
|
||||
delete oneMinusU;
|
||||
|
||||
// dLdzr = (dLdzc * hI * r * (1-r)) × WhcT
|
||||
auto* oneMinusR = 1 - (*r);
|
||||
auto* rTimesOneMinusR = (*r) * (*oneMinusR);
|
||||
delete oneMinusR;
|
||||
auto* temp3 = (*dLdzc) * (*hI);
|
||||
auto* temp4 = (*temp3) * (*rTimesOneMinusR);
|
||||
delete temp3;
|
||||
delete rTimesOneMinusR;
|
||||
MmulHelper::mmul(temp4, WhcT, dLdzr); // [bS, nOut] x [nOut, nOut] = [bS, nOut]
|
||||
delete temp4;
|
||||
|
||||
// dLdx = dLdz × WxT
|
||||
NDArray *WxT = Wx->transpose();
|
||||
MmulHelper::mmul(&dLdz, WxT, dLdx); // [bS, 3*nOut] x [3*nOut, nIn] = [bS, nIn]
|
||||
|
||||
// dLdWx += xT × dLdz
|
||||
NDArray *xT = x->transpose();
|
||||
auto dLdWxAdd = mmul(*xT, dLdz);
|
||||
*dLdWx += *dLdWxAdd; // [nIn, bS] x [bS, 3*nOut] = [nIn, 3*nOut]
|
||||
|
||||
delete dLdWxAdd;
|
||||
// dLdb += sum(dLdz, axis=0)
|
||||
std::vector<sd::LongType> zeroVec = {0};
|
||||
auto* dLdbAdd = dLdz.reduceAlongDimension(reduce::Sum, &zeroVec);
|
||||
*dLdb += (*dLdbAdd); // [bS, 3*nOut] -> reduce -> [3*nOut];
|
||||
delete dLdbAdd;
|
||||
|
||||
*dLdzc *= (*r);
|
||||
|
||||
// dLdhI = dLdhI * u + dLdz × WhT
|
||||
NDArray *WhT = Wh->transpose();
|
||||
auto* dLdhIU = (*dLdhI) * (*u);
|
||||
auto mmulResult = mmul(dLdz, *WhT);
|
||||
auto* dLdhIAssign = (*dLdhIU) + *mmulResult;
|
||||
delete dLdhIU;
|
||||
delete mmulResult;
|
||||
dLdhI->assign(dLdhIAssign); // [bS, 3*nOut] x [3*nOut, nOut] = [bS, nOut]
|
||||
delete dLdhIAssign;
|
||||
|
||||
// dLdWh += hIT × dLdz
|
||||
NDArray *hITranspose = hI->transpose();
|
||||
auto dLdWhAdd = mmul(*hITranspose, dLdz);
|
||||
*dLdWh += *dLdWhAdd; // [nOut, bS] x [bS, 3*nOut] = [nOut, 3*nOut]
|
||||
delete dLdWhAdd;
|
||||
|
||||
delete gatesULike;
|
||||
delete dLdzru;
|
||||
delete dLdzr;
|
||||
delete dLdzu;
|
||||
delete dLdzc;
|
||||
delete r;
|
||||
delete u;
|
||||
delete c;
|
||||
delete WhView;
|
||||
delete WhcT;
|
||||
delete hITranspose;
|
||||
delete WhT;
|
||||
delete xT;
|
||||
delete WxT;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void gruTimeLoopBp(sd::LaunchContext* context, NDArray* x, NDArray* hI, NDArray* Wx,
|
||||
NDArray* Wh, NDArray* b, NDArray* dLdh, NDArray* dLdx, NDArray* dLdhI,
|
||||
NDArray* dLdWx, NDArray* dLdWh, NDArray* dLdb) {
|
||||
// sL means time steps
|
||||
|
||||
// x input [sL, bS, nIn]
|
||||
// hI initial cell output (at time step = 0) [bS, nOut]
|
||||
// Wx input-to-hidden weights, [nIn, 3*nOut]
|
||||
// Wh hidden-to-hidden weights, [nOut, 3*nOut]
|
||||
// b biases, [3*nOut]
|
||||
// dLdh gradient vs. ff output, [sL, bS, nOut]
|
||||
|
||||
// dLdx gradient vs. x, [sL, bS, nIn],
|
||||
// dLdhI gradient vs. hI, [bS, nOut]
|
||||
// dLdWx gradient vs. W, [nIn, 3*nOut]
|
||||
// dLdWh gradient vs. Wc, [nOut, 3*nOut]
|
||||
// dLdb gradient vs. b [3*nOut]
|
||||
|
||||
const int sL = x->sizeAt(0);
|
||||
const int bS = x->sizeAt(1);
|
||||
const int nOut = hI->sizeAt(1);
|
||||
|
||||
std::vector<sd::LongType> shape = {bS, 3 * nOut};
|
||||
std::vector<sd::LongType> hShape = {sL + 1, bS, nOut};
|
||||
NDArray gates(x->ordering(), shape, dLdh->dataType(), x->getContext());
|
||||
NDArray h(x->ordering(), hShape, dLdh->dataType(), x->getContext());
|
||||
|
||||
auto xSet = x->allTensorsAlongDimension({1, 2}); // sub-arrays with shape [bS, nIn]
|
||||
auto dLdhSet = dLdh->allTensorsAlongDimension({1, 2}); // sub-arrays with shape [bS, nOut]
|
||||
auto hSet = h.allTensorsAlongDimension({1, 2}); // sub-arrays with shape [bS, nOut]
|
||||
auto gatesSet = gates.allTensorsAlongDimension({1, 2}); // sub-arrays with shape [bS, nOut]
|
||||
auto dLdxSet = dLdx->allTensorsAlongDimension({1, 2}); // sub-arrays with shape [bS, nIn]
|
||||
|
||||
hSet.at(0)->assign(hI);
|
||||
|
||||
// forward time loop
|
||||
for (int t = 0; t < sL; ++t) gruCell(xSet.at(t), hSet.at(t), Wx, Wh, b, gatesSet.at(t), hSet.at(t + 1), false);
|
||||
|
||||
// backward time loop
|
||||
for (int t = sL - 1; t >= 0; --t)
|
||||
gruCellBp(context, xSet.at(t), hSet.at(t), Wx, Wh, b, dLdhSet.at(t), gatesSet.at(t), dLdxSet.at(t), dLdhI, dLdWx,
|
||||
dLdWh, dLdb);
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,70 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_knn_mindistance)
|
||||
|
||||
#include <ops/declarable/helpers/knn.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
void mindistance_(const void *vinput, const void *vlow, const void *vhigh, int32_t length, void *vout) {
|
||||
auto input = reinterpret_cast<const T *>(vinput);
|
||||
auto low = reinterpret_cast<const T *>(vlow);
|
||||
auto high = reinterpret_cast<const T *>(vhigh);
|
||||
auto output = reinterpret_cast<T *>(vout);
|
||||
|
||||
T res = static_cast<T>(0.0f);
|
||||
T po = static_cast<T>(2.f);
|
||||
T o = static_cast<T>(1.f);
|
||||
for (auto e = 0; e < length; e++) {
|
||||
T p = input[e];
|
||||
T l = low[e];
|
||||
T h = high[e];
|
||||
if (!(l <= p || h <= p)) {
|
||||
if (p < l)
|
||||
res += math::sd_pow<T, T, T>((p - o), po);
|
||||
else
|
||||
res += math::sd_pow<T, T, T>((p - h), po);
|
||||
}
|
||||
}
|
||||
|
||||
output[0] = math::sd_pow<T, T, T>(res, static_cast<T>(0.5f));
|
||||
}
|
||||
|
||||
|
||||
void knn_mindistance(NDArray&input, NDArray&lowest, NDArray&highest, NDArray &output) {
|
||||
NDArray::preparePrimaryUse({&output}, {&input, &lowest, &highest});
|
||||
|
||||
BUILD_SINGLE_SELECTOR(input.dataType(), mindistance_,
|
||||
(input.buffer(), lowest.buffer(), highest.buffer(), input.lengthOf(), output.buffer()),
|
||||
SD_FLOAT_TYPES);
|
||||
|
||||
NDArray::registerPrimaryUse({&output}, {&input, &lowest, &highest});
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,134 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_listdiff)
|
||||
#include <ops/declarable/helpers/listdiff.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static sd::LongType listDiffCount_(NDArray* values, NDArray* keep) {
|
||||
sd::LongType saved = 0L;
|
||||
for (sd::LongType e = 0; e < values->lengthOf(); e++) {
|
||||
auto v = values->e<double>(e);
|
||||
ExtraArguments extras({v, 0.0, 10.0});
|
||||
auto idx = keep->indexReduceNumber(indexreduce::FirstIndex, &extras);
|
||||
auto index = idx->e<sd::LongType>(0);
|
||||
if (index < 0) saved++;
|
||||
delete idx;
|
||||
}
|
||||
|
||||
|
||||
return saved;
|
||||
}
|
||||
|
||||
sd::LongType listDiffCount(sd::LaunchContext* context, NDArray* values, NDArray* keep) {
|
||||
auto xType = values->dataType();
|
||||
|
||||
NDArray::preparePrimaryUse({}, {values, keep});
|
||||
|
||||
BUILD_SINGLE_SELECTOR(xType, return listDiffCount_, (values, keep), SD_COMMON_TYPES);
|
||||
|
||||
NDArray::registerPrimaryUse({}, {values, keep});
|
||||
return 0;
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( sd::LongType listDiffCount_, (NDArray * values, NDArray* keep);, SD_COMMON_TYPES);
|
||||
|
||||
template <typename T>
|
||||
static sd::Status listDiffFunctor_(NDArray* values, NDArray* keep, NDArray* output1, NDArray* output2) {
|
||||
std::vector<T> saved;
|
||||
std::vector<sd::LongType> indices;
|
||||
for (sd::LongType e = 0; e < values->lengthOf(); e++) {
|
||||
auto v = values->e<double>(e);
|
||||
ExtraArguments extras({v, 0.0, 10.0});
|
||||
NDArray *idxScalar = keep->indexReduceNumber(indexreduce::FirstIndex, &extras);
|
||||
sd::LongType idx = idxScalar->e<sd::LongType>(0);
|
||||
if (idx < 0) {
|
||||
saved.emplace_back(v);
|
||||
indices.emplace_back(e);
|
||||
}
|
||||
delete idxScalar;
|
||||
}
|
||||
|
||||
if (saved.size() == 0) {
|
||||
sd_printf("ListDiff: search returned no results", "");
|
||||
THROW_EXCEPTION("Op validation failed");
|
||||
} else {
|
||||
auto z0 = output1;
|
||||
auto z1 = output2;
|
||||
|
||||
if (static_cast<size_t>(z0->lengthOf()) != saved.size()) {
|
||||
sd_printf("ListDiff: output/actual size mismatch", "");
|
||||
THROW_EXCEPTION("Op validation failed");
|
||||
}
|
||||
|
||||
if (static_cast<size_t>(z1->lengthOf()) != saved.size()) {
|
||||
sd_printf("ListDiff: output/actual indices size mismatch", "");
|
||||
THROW_EXCEPTION("Op validation failed");
|
||||
}
|
||||
memcpy(z0->buffer(), saved.data(), saved.size() * sizeof(T));
|
||||
for (size_t e = 0; e < indices.size(); e++) {
|
||||
z1->p(e, indices[e]);
|
||||
}
|
||||
|
||||
}
|
||||
return sd::Status::OK;
|
||||
}
|
||||
|
||||
sd::Status listDiffFunctor(sd::LaunchContext* context, NDArray* values, NDArray* keep, NDArray* output1,
|
||||
NDArray* output2) {
|
||||
auto xType = values->dataType();
|
||||
|
||||
NDArray::preparePrimaryUse({output1, output2}, {values, keep});
|
||||
|
||||
sd::Status result = sd::Status::OK;
|
||||
|
||||
if (DataTypeUtils::isR(xType)) {
|
||||
BUILD_SINGLE_SELECTOR(xType, result = listDiffFunctor_, (values, keep, output1, output2), SD_FLOAT_TYPES);
|
||||
} else if (DataTypeUtils::isZ(xType)) {
|
||||
BUILD_SINGLE_SELECTOR(xType, result = listDiffFunctor_, (values, keep, output1, output2), SD_INTEGER_TYPES);
|
||||
} else {
|
||||
return sd::Status::KERNEL_FAILURE;
|
||||
}
|
||||
|
||||
NDArray::registerPrimaryUse({output1, output2}, {values, keep});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status listDiffFunctor_,
|
||||
(NDArray * values, NDArray* keep, NDArray* output1, NDArray* output2);
|
||||
, SD_FLOAT_TYPES);
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status listDiffFunctor_,
|
||||
(NDArray * values, NDArray* keep, NDArray* output1, NDArray* output2);
|
||||
, SD_INTEGER_TYPES);
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma, created on 14.02.2018
|
||||
//
|
||||
|
||||
// implementation of operation for LSTM cell with peep hole connections:
|
||||
// http://www.bioinf.jku.at/publications/older/2604.pdf
|
||||
// S. Hochreiter and J. Schmidhuber. "Long Short-Term Memory". Neural Computation, 9(8):1735-1780, 1997.
|
||||
// and
|
||||
// https://research.google.com/pubs/archive/43905.pdf
|
||||
// Hasim Sak, Andrew Senior, and Francoise Beaufays. "Long short-term memory recurrent neural network architectures for
|
||||
// large scale acoustic modeling." INTERSPEECH, 2014.
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_lstm)
|
||||
|
||||
#include <array/NDArrayList.h>
|
||||
#include <graph/VariableSpace.h>
|
||||
#include <helpers/MmulHelper.h>
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/helpers/legacy_helpers.h>
|
||||
#include <ops/declarable/helpers/lstm.h>
|
||||
#include <ops/declarable/helpers/transforms.h>
|
||||
|
||||
#include <iterator>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
void lstmBlockTimeLoop(NDArray* maxSeqLength, NDArray* xSeq, NDArray* c0, NDArray* y0,
|
||||
NDArray* W, NDArray* Wci, NDArray* Wcf, NDArray* Wco, NDArray* b,
|
||||
NDArray* iSeq, NDArray* cSeq, NDArray* fSeq, NDArray* oSeq,
|
||||
NDArray* zSeq, NDArray* hSeq, NDArray* ySeq, const std::vector<double>& params,
|
||||
const int dataFormat) {
|
||||
int seqLen, bS, nIn, nOut;
|
||||
|
||||
if (dataFormat == 0) {
|
||||
seqLen = xSeq->sizeAt(0);
|
||||
bS = xSeq->sizeAt(1);
|
||||
nIn = xSeq->sizeAt(2);
|
||||
nOut = iSeq->sizeAt(2);
|
||||
} else if (dataFormat == 1) {
|
||||
seqLen = xSeq->sizeAt(2);
|
||||
bS = xSeq->sizeAt(0);
|
||||
nIn = xSeq->sizeAt(1);
|
||||
nOut = iSeq->sizeAt(1);
|
||||
} else if (dataFormat == 2) {
|
||||
seqLen = xSeq->sizeAt(1);
|
||||
bS = xSeq->sizeAt(0);
|
||||
nIn = xSeq->sizeAt(2);
|
||||
nOut = iSeq->sizeAt(2);
|
||||
}
|
||||
|
||||
const std::vector<sd::LongType> inSliceShape({bS, nIn});
|
||||
const std::vector<sd::LongType> outSliceShape({bS, nOut});
|
||||
|
||||
auto c_t1 = const_cast<NDArray*>(c0);
|
||||
auto y_t1 = const_cast<NDArray*>(y0);
|
||||
|
||||
// loop through time steps
|
||||
for (int t = 0; t < seqLen; ++t) {
|
||||
auto xt = timeSubset(xSeq, t, dataFormat);
|
||||
|
||||
auto it = timeSubset(iSeq, t, dataFormat);
|
||||
auto ct = timeSubset(cSeq, t, dataFormat);
|
||||
auto ft = timeSubset(fSeq, t, dataFormat);
|
||||
auto ot = timeSubset(oSeq, t, dataFormat);
|
||||
auto zt = timeSubset(zSeq, t, dataFormat);
|
||||
auto ht = timeSubset(hSeq, t, dataFormat);
|
||||
auto yt = timeSubset(ySeq, t, dataFormat);
|
||||
|
||||
helpers::lstmBlockCell(xt, c_t1, y_t1, W, Wci, Wcf, Wco, b, it, ct, ft, ot, zt, ht, yt, params);
|
||||
|
||||
if (t != 0) {
|
||||
delete c_t1;
|
||||
delete y_t1;
|
||||
}
|
||||
|
||||
if (t < seqLen - 1) {
|
||||
c_t1 = ct;
|
||||
y_t1 = yt;
|
||||
} else {
|
||||
delete ct;
|
||||
delete yt;
|
||||
}
|
||||
|
||||
|
||||
delete it;
|
||||
delete ft;
|
||||
delete ot;
|
||||
delete zt;
|
||||
delete ht;
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void lstmTimeLoop(sd::LaunchContext* context, NDArray* x, NDArray* h0, NDArray* c0, NDArray* Wx,
|
||||
NDArray* Wh, NDArray* Wc, NDArray* Wp, NDArray* b, NDArray* h, NDArray* c,
|
||||
const std::vector<double>& params) {
|
||||
// x input [time x bS x nIn]
|
||||
// h0 initial cell output (at time step = 0) [bS x numProj], in case of projection=false -> numProj == numUnits !!!
|
||||
// c0 initial cell state (at time step = 0) [bS x numUnits],
|
||||
|
||||
// Wx input-to-hidden weights, [nIn x 4*numUnits]
|
||||
// Wh hidden-to-hidden weights, [numProj x 4*numUnits]
|
||||
// Wc diagonal weights for peephole connections [3*numUnits]
|
||||
// Wp projection weights [numUnits x numProj]
|
||||
// b biases, [4*numUnits]
|
||||
|
||||
// h cell outputs [time x bS x numProj], that is per each time step
|
||||
// c cell states [time x bS x numUnits] that is per each time step
|
||||
|
||||
const int time = x->sizeAt(0);
|
||||
|
||||
NDArray currentH(*h0);
|
||||
NDArray currentC(*c0);
|
||||
|
||||
// loop through time steps
|
||||
for (int t = 0; t < time; ++t) {
|
||||
auto xt = (*x)({t, t + 1, 0, 0, 0, 0});
|
||||
auto ht = (*h)({t, t + 1, 0, 0, 0, 0});
|
||||
auto ct = (*c)({t, t + 1, 0, 0, 0, 0});
|
||||
|
||||
helpers::lstmCell(context, xt, ¤tH, ¤tC, Wx, Wh, Wc, Wp, b, ht, ct, params);
|
||||
currentH.assign(ht);
|
||||
currentC.assign(ct);
|
||||
delete ht;
|
||||
delete ct;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,73 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_unique)
|
||||
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
#include <ops/declarable/helpers/multiUnique.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
bool multiUnique(std::vector<NDArray*> const& inputList, sd::memory::Workspace* workspace) {
|
||||
sd::LongType length = 0;
|
||||
std::vector<NDArray*> reshaped(inputList.size());
|
||||
int pos = 0;
|
||||
sd::LongType axis = 0;
|
||||
Context cContext(1);
|
||||
for (auto array : inputList) {
|
||||
if (array->dataType() != sd::DataType::INT32)
|
||||
THROW_EXCEPTION("multiUnique: this op support INT32 data type only.");
|
||||
|
||||
std::vector<sd::LongType> reshape = {-1};
|
||||
reshaped[pos] = array->reshape(array->ordering(), reshape);
|
||||
cContext.setInputArray(pos, reshaped[pos]);
|
||||
|
||||
length += array->lengthOf();
|
||||
pos++;
|
||||
}
|
||||
std::vector<LongType> shape = {length};
|
||||
NDArray arrayFull('c',shape, sd::DataType::INT32, inputList[0]->getContext());
|
||||
cContext.setOutputArray(0, &arrayFull);
|
||||
cContext.setIArguments(&axis, 1);
|
||||
|
||||
sd::ops::concat opConcat;
|
||||
auto cResult = opConcat.execute(&cContext);
|
||||
if (sd::Status::OK != cResult) THROW_EXCEPTION("multiUnique: cannot execute concat op properly.");
|
||||
|
||||
sd::ops::unique opUnique;
|
||||
auto uResult = opUnique.evaluate({&arrayFull});
|
||||
if (sd::Status::OK != uResult.status()) THROW_EXCEPTION("multiUnique: cannot execute unique op properly.");
|
||||
|
||||
auto uniqueVals = uResult.at(0);
|
||||
|
||||
bool res = uniqueVals->lengthOf() == arrayFull.lengthOf();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
#endif
|
||||
@@ -0,0 +1,125 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com), created on 16.04.2018
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_rnn)
|
||||
|
||||
// function nnCell implements an Elman RNN cell: output = activation(Wx*x + bx + Wh*ht + bh)
|
||||
#include <helpers/BlasHelper.h>
|
||||
#include <ops/declarable/helpers/rnn.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void rnnCell(sd::LaunchContext* context, NDArray* xt, NDArray* Wx, NDArray* Wh, NDArray* b,
|
||||
NDArray* hPrev, NDArray* ht) {
|
||||
// xt input [bS x iS]
|
||||
// Wx input-to-hidden weights, [iS x nU]
|
||||
// Wh hidden-to-hidden weights, [nU x nU]
|
||||
// b biases, [2*nU]: {0, nU} are input-to-hidden biases and {nU, 2*nU} are hidden-to-hidden biases
|
||||
// hPrev previous cell output [bS x nU], that is at previous time step t-1, in case of projection=false -> nU=nU!!!
|
||||
|
||||
const int nU = hPrev->sizeAt(1);
|
||||
|
||||
// ht is current cell output [bS x nU], that is at current time step t
|
||||
NDArray *bFirst = (*b)({{0, nU}});
|
||||
NDArray *bSecond = (*b)({{nU, 2 * nU}});
|
||||
|
||||
NDArray *mmulOne = mmul(*xt, *Wx);
|
||||
NDArray *mmulTwo = mmul(*hPrev, *Wh);
|
||||
|
||||
// Chain additions with proper dereferencing since operators return NDArray*
|
||||
NDArray *temp1 = (*mmulOne) + (*bFirst);
|
||||
NDArray *temp2 = (*temp1) + (*mmulTwo);
|
||||
NDArray *temp3 = (*temp2) + (*bSecond);
|
||||
|
||||
ht->assign(temp3); // [bS x nU] + [nU] + [bS x nU] + [nU] = [bS x nU]
|
||||
ht->applyTransform(transform::Tanh, ht);
|
||||
|
||||
// Clean up intermediate results
|
||||
delete temp1;
|
||||
delete temp2;
|
||||
delete temp3;
|
||||
|
||||
delete mmulOne;
|
||||
delete mmulTwo;
|
||||
delete bFirst;
|
||||
delete bSecond;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void rnnTimeLoop(sd::LaunchContext* context, NDArray* x, NDArray* Wx, NDArray* Wh, NDArray* b,
|
||||
NDArray* h0, NDArray* maxTimeStep, NDArray* h, NDArray* hFinal) {
|
||||
// x input [time x bS x iS]
|
||||
// Wx input-to-hidden weights, [iS x nU]
|
||||
// Wh hidden-to-hidden weights, [nU x nU]
|
||||
// b biases for, [2*nU]
|
||||
|
||||
// h0 initial cell output (at time step = 0) [bS x nU]
|
||||
// maxTimeStep vector [bS] containing integer values within [0,time), each element of this vector set max time step
|
||||
// per each input in batch, this means there are no calculations for time >= maxTimeStep
|
||||
|
||||
const int time = x->sizeAt(0);
|
||||
const int bS = x->sizeAt(1);
|
||||
|
||||
// at first time step
|
||||
if (h0)
|
||||
hFinal->assign(h0);
|
||||
else
|
||||
*hFinal = 0.;
|
||||
|
||||
BlasHelper::getInstance(); // to avoid memory leak in pragma parallel loops
|
||||
// loop through batch of inputs
|
||||
for (int e = 0; e < bS; ++e) {
|
||||
// loop through time steps
|
||||
for (int t = 0; t < time; ++t) {
|
||||
int maxStep = maxTimeStep ? maxTimeStep->e<int>(e) : time;
|
||||
|
||||
NDArray *xt = (*x)({t, t + 1, e, e + 1, 0, 0}, true);
|
||||
NDArray *ht = (*h)({t, t + 1, e, e + 1, 0, 0}, true);
|
||||
NDArray *hPrev = (*hFinal)({e, e + 1, 0, 0}, true); // previous state
|
||||
|
||||
if (t >= maxStep) {
|
||||
*ht = 0.;
|
||||
NDArray *hPrevAssign = (*h)({maxStep - 1, maxStep, e, e + 1, 0, 0});
|
||||
if (maxStep != 0) hPrev->assign(hPrevAssign);
|
||||
delete hPrevAssign;
|
||||
} else {
|
||||
helpers::rnnCell(context, xt, Wx, Wh, b, hPrev, ht);
|
||||
hPrev->assign(ht);
|
||||
}
|
||||
|
||||
delete xt;
|
||||
delete ht;
|
||||
delete hPrev;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* ******************************************************************************
|
||||
* *
|
||||
* *
|
||||
* * This program and the accompanying materials are made available under the
|
||||
* * terms of the Apache License, Version 2.0 which is available at
|
||||
* * https://www.apache.org/licenses/LICENSE-2.0.
|
||||
* *
|
||||
* * See the NOTICE file distributed with this work for additional
|
||||
* * information regarding copyright ownership.
|
||||
* * Unless required by applicable law or agreed to in writing, software
|
||||
* * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* * License for the specific language governing permissions and limitations
|
||||
* * under the License.
|
||||
* *
|
||||
* * SPDX-License-Identifier: Apache-2.0
|
||||
* *****************************************************************************
|
||||
*/
|
||||
|
||||
//
|
||||
// @author raver119@gmail.com
|
||||
//
|
||||
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_compat_sparse_to_dense)
|
||||
#include <helpers/ShapeUtils.h>
|
||||
#include <helpers/StringUtils.h>
|
||||
#include <ops/declarable/helpers/sparse_to_dense.h>
|
||||
#include <system/selective_rendering.h>
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename X, typename I>
|
||||
static void fill_(const void *vvalues, const void *vindices, void *voutput, const LongType *zShapeInfo,
|
||||
uint8_t rank, uint64_t length) {
|
||||
auto values = reinterpret_cast<const X *>(vvalues);
|
||||
auto indices = reinterpret_cast<const I *>(vindices);
|
||||
auto output = reinterpret_cast<X *>(voutput);
|
||||
|
||||
LongType coords[SD_MAX_RANK];
|
||||
uint64_t pos = 0;
|
||||
for (uint64_t e = 0L; e < length; e++) {
|
||||
// indices come in blocks
|
||||
for (uint8_t p = 0; p < rank; p++) {
|
||||
coords[p] = indices[pos++];
|
||||
}
|
||||
|
||||
// fill output at given coords with sparse value
|
||||
LongType offset;
|
||||
COORDS2INDEX(rank, shape::stride(zShapeInfo), coords, offset);
|
||||
output[offset] = values[e];
|
||||
}
|
||||
}
|
||||
void compat_sparse_to_dense(NDArray& values, NDArray& indices, NDArray* def, NDArray& output) {
|
||||
// make sure host buffer is updated
|
||||
|
||||
auto rank = output.rankOf();
|
||||
|
||||
if (output.isS()) {
|
||||
NDArray::preparePrimaryUse({&output}, {&values, &indices, def});
|
||||
// string case is not so trivial, since elements might, and probably will, have different sizes
|
||||
auto numValues = values.lengthOf();
|
||||
auto numElements = output.lengthOf();
|
||||
|
||||
// first of all we calculate final buffer sizes and offsets
|
||||
auto defaultLength = def == nullptr ? 0 : StringUtils::byteLength(*def);
|
||||
auto valuesLength = StringUtils::byteLength(values);
|
||||
auto bufferLength = defaultLength * (output.lengthOf() - numValues) + valuesLength;
|
||||
auto headerLength = ShapeUtils::stringBufferHeaderRequirements(numElements);
|
||||
|
||||
// now we make sure our output buffer can hold results
|
||||
output.dataBuffer()->expand(bufferLength + headerLength);
|
||||
|
||||
std::vector<LongType> outputCoords(rank);
|
||||
std::vector<LongType> valueCoords(rank);
|
||||
|
||||
auto offsetsBuffer = output.bufferAsT<LongType>();
|
||||
auto dataBuffer = reinterpret_cast<uint8_t*>(offsetsBuffer + output.lengthOf());
|
||||
|
||||
offsetsBuffer[0] = 0;
|
||||
|
||||
// getting initial value coords
|
||||
for (int e = 0; e < rank; e++) valueCoords[e] = indices.e<LongType>(e);
|
||||
|
||||
// write results individually
|
||||
for (LongType e = 0; e < numElements; e++) {
|
||||
LongType vIndex;
|
||||
COORDS2INDEX(rank, shape::stride(output.shapeInfo()), valueCoords.data(), vIndex);
|
||||
auto cLength = 0L;
|
||||
std::string str;
|
||||
if (vIndex == e) {
|
||||
// we're writing down sparse value here
|
||||
str = values.e<std::string>(e);
|
||||
} else {
|
||||
// we're writing down default value if it exists
|
||||
if (def != nullptr)
|
||||
str = def->e<std::string>(0);
|
||||
else
|
||||
str = "";
|
||||
}
|
||||
|
||||
// TODO: make it unicode compliant
|
||||
memcpy(&dataBuffer[offsetsBuffer[e]], str.c_str(), str.length());
|
||||
|
||||
// writing down offset
|
||||
offsetsBuffer[e + 1] = cLength;
|
||||
}
|
||||
NDArray::registerPrimaryUse({&output}, {&values, &indices, def});
|
||||
} else {
|
||||
// numeric case is trivial, since all elements have equal sizes
|
||||
|
||||
// write out default values, if they are present
|
||||
if (def != nullptr) {
|
||||
output.assign(def);
|
||||
}
|
||||
NDArray::preparePrimaryUse({&output}, {&values, &indices});
|
||||
// write out values
|
||||
auto valuesDType = values.dataType();
|
||||
auto indicesDataType = indices.dataType();
|
||||
BUILD_DOUBLE_SELECTOR(
|
||||
values.dataType(), indices.dataType(), fill_,
|
||||
(values.buffer(), indices.buffer(), output.buffer(), output.shapeInfo(), rank, values.lengthOf()),
|
||||
SD_COMMON_TYPES, SD_INDEXING_TYPES);
|
||||
NDArray::registerPrimaryUse({&output}, {&values, &indices});
|
||||
}
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,60 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author Yurii Shyrma (iuriish@yahoo.com)
|
||||
//
|
||||
#include <system/op_boilerplate.h>
|
||||
#if NOT_EXCLUDED(OP_sqrtm)
|
||||
|
||||
#include <helpers/Sqrtm.h>
|
||||
#include <ops/declarable/CustomOperations.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
static void sqrtm_(NDArray* x, NDArray* z) {
|
||||
if (x->rankOf() == 2) {
|
||||
Sqrtm<T>::calc(*x, *z);
|
||||
} else {
|
||||
auto listX = x->allTensorsAlongDimension({-2, -1});
|
||||
auto listZ = z->allTensorsAlongDimension({-2, -1});
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto i = start; i < stop; i++) Sqrtm<T>::calc(*listX.at(i), *listZ.at(i));
|
||||
};
|
||||
|
||||
samediff::Threads::parallel_tad(func, 0, listX.size());
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
void sqrtm(LaunchContext* context, NDArray* x, NDArray* z) {
|
||||
x->syncToHost();
|
||||
BUILD_SINGLE_SELECTOR(z->dataType(), sqrtm_, (x, z), SD_FLOAT_TYPES);
|
||||
z->syncToDevice();
|
||||
}
|
||||
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,117 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// @author sgazeos@gmail.com
|
||||
//
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_unique)
|
||||
|
||||
#include <execution/Threads.h>
|
||||
#include <graph/Variable.h>
|
||||
#include <ops/declarable/helpers/unique.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
|
||||
template <typename T>
|
||||
static LongType uniqueCount_(NDArray* input) {
|
||||
LongType count = 0;
|
||||
|
||||
std::vector<T> values;
|
||||
|
||||
for (LongType e = 0; e < input->lengthOf(); e++) {
|
||||
T v = input->e<T>(e);
|
||||
if (std::find(values.begin(), values.end(), v) == values.end()) {
|
||||
values.push_back(v);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
LongType uniqueCount(LaunchContext* context, NDArray* input) {
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return uniqueCount_, (input), SD_COMMON_TYPES);
|
||||
return 0;
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( sd::LongType uniqueCount_, (NDArray * input), SD_COMMON_TYPES);
|
||||
|
||||
template <typename T>
|
||||
static Status uniqueFunctor_(NDArray* input, NDArray* values, NDArray* indices, NDArray* counts) {
|
||||
std::vector<T> valuesVector;
|
||||
SD_MAP_IMPL<T, int> indicesMap;
|
||||
SD_MAP_IMPL<T, int> countsMap;
|
||||
|
||||
for (LongType e = 0; e < input->lengthOf(); e++) {
|
||||
T v = input->e<T>(e);
|
||||
if (std::find(valuesVector.begin(), valuesVector.end(), v) == valuesVector.end()) {
|
||||
valuesVector.push_back(v);
|
||||
indicesMap[v] = e;
|
||||
countsMap[v] = 1;
|
||||
} else {
|
||||
countsMap[v]++;
|
||||
}
|
||||
}
|
||||
|
||||
auto func = PRAGMA_THREADS_FOR {
|
||||
for (auto e = start; e < stop; e++) {
|
||||
values->p(e, static_cast<T>(valuesVector[e]));
|
||||
if (counts != nullptr) counts->p(e, countsMap[valuesVector[e]]);
|
||||
}
|
||||
};
|
||||
samediff::Threads::parallel_for(func, 0, values->lengthOf());
|
||||
|
||||
for (LongType e = 0; e < indices->lengthOf(); e++) {
|
||||
auto posI = std::find(valuesVector.begin(), valuesVector.end(), input->e<T>(e));
|
||||
auto dist = std::distance(valuesVector.begin(), posI);
|
||||
indices->p(e, LongType(dist)); // indicesMap[(*input)(e)];
|
||||
}
|
||||
|
||||
return Status::OK;
|
||||
}
|
||||
|
||||
Status uniqueFunctor(LaunchContext* context, NDArray* input, NDArray* values, NDArray* indices,
|
||||
NDArray* counts) {
|
||||
input->syncToHost();
|
||||
values->syncToHost();
|
||||
indices->syncToHost();
|
||||
|
||||
if (counts != nullptr) counts->syncToHost();
|
||||
|
||||
BUILD_SINGLE_SELECTOR(input->dataType(), return uniqueFunctor_, (input, values, indices, counts), SD_COMMON_TYPES);
|
||||
|
||||
input->syncToDevice();
|
||||
values->syncToDevice();
|
||||
indices->syncToDevice();
|
||||
|
||||
if (counts != nullptr) counts->syncToDevice();
|
||||
|
||||
|
||||
return sd::Status::KERNEL_FAILURE;
|
||||
}
|
||||
|
||||
BUILD_SINGLE_TEMPLATE( sd::Status uniqueFunctor_,
|
||||
(NDArray * input, NDArray* values, NDArray* indices, NDArray* counts), SD_COMMON_TYPES);
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,70 @@
|
||||
/* ******************************************************************************
|
||||
*
|
||||
*
|
||||
* This program and the accompanying materials are made available under the
|
||||
* terms of the Apache License, Version 2.0 which is available at
|
||||
* https://www.apache.org/licenses/LICENSE-2.0.
|
||||
*
|
||||
* See the NOTICE file distributed with this work for additional
|
||||
* information regarding copyright ownership.
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
******************************************************************************/
|
||||
|
||||
//
|
||||
// Created by raver119 on 24/09/18.
|
||||
//
|
||||
#include <array/NDArrayList.h>
|
||||
#include <system/op_boilerplate.h>
|
||||
|
||||
#if NOT_EXCLUDED(OP_Where)
|
||||
#include <ops/declarable/helpers/where.h>
|
||||
|
||||
namespace sd {
|
||||
namespace ops {
|
||||
namespace helpers {
|
||||
template <typename T>
|
||||
static void __where(NDArray &condition, NDArray &output, memory::Workspace *workspace) {
|
||||
NDArrayList list(0, true);
|
||||
sd::LongType cnt = 0;
|
||||
|
||||
|
||||
for (sd::LongType e = 0; e < condition.lengthOf(); e++) {
|
||||
sd::LongType coords[SD_MAX_RANK];
|
||||
|
||||
INDEX2COORDS(e, condition.rankOf(), condition.shapeOf(), coords);
|
||||
sd::LongType offset;
|
||||
COORDS2INDEX(condition.rankOf(), shape::stride(condition.shapeInfo()), coords, offset);
|
||||
|
||||
if (condition.e<bool>(offset)) {
|
||||
std::vector<sd::LongType> arrShape = {1, condition.rankOf()};
|
||||
auto array = NDArrayFactory::create_('c', arrShape, output.dataType(), output.getContext());
|
||||
for (sd::LongType f = 0; f < condition.rankOf(); f++) {
|
||||
array->p(f, (T)coords[f]);
|
||||
}
|
||||
list.write(cnt++, array);
|
||||
}
|
||||
}
|
||||
|
||||
auto s = list.stack();
|
||||
output.assign(s);
|
||||
delete s;
|
||||
}
|
||||
BUILD_SINGLE_TEMPLATE( void __where, (NDArray & condition, NDArray &output, memory::Workspace *workspace),
|
||||
SD_COMMON_TYPES);
|
||||
|
||||
void _where(sd::LaunchContext *context, NDArray &condition, NDArray &output, memory::Workspace *workspace) {
|
||||
condition.syncToHost();
|
||||
BUILD_SINGLE_SELECTOR(output.dataType(), __where, (condition, output, workspace), SD_COMMON_TYPES);
|
||||
output.syncToDevice();
|
||||
}
|
||||
} // namespace helpers
|
||||
} // namespace ops
|
||||
} // namespace sd
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user